Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
[muen/linux.git] / drivers / net / ethernet / intel / ixgbevf / ixgbevf_main.c
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 1999 - 2018 Intel Corporation. */
3
4 /******************************************************************************
5  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
6 ******************************************************************************/
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10 #include <linux/types.h>
11 #include <linux/bitops.h>
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/netdevice.h>
15 #include <linux/vmalloc.h>
16 #include <linux/string.h>
17 #include <linux/in.h>
18 #include <linux/ip.h>
19 #include <linux/tcp.h>
20 #include <linux/sctp.h>
21 #include <linux/ipv6.h>
22 #include <linux/slab.h>
23 #include <net/checksum.h>
24 #include <net/ip6_checksum.h>
25 #include <linux/ethtool.h>
26 #include <linux/if.h>
27 #include <linux/if_vlan.h>
28 #include <linux/prefetch.h>
29 #include <net/mpls.h>
30 #include <linux/bpf.h>
31 #include <linux/bpf_trace.h>
32 #include <linux/atomic.h>
33
34 #include "ixgbevf.h"
35
36 const char ixgbevf_driver_name[] = "ixgbevf";
37 static const char ixgbevf_driver_string[] =
38         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
39
40 #define DRV_VERSION "4.1.0-k"
41 const char ixgbevf_driver_version[] = DRV_VERSION;
42 static char ixgbevf_copyright[] =
43         "Copyright (c) 2009 - 2018 Intel Corporation.";
44
45 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
46         [board_82599_vf]        = &ixgbevf_82599_vf_info,
47         [board_82599_vf_hv]     = &ixgbevf_82599_vf_hv_info,
48         [board_X540_vf]         = &ixgbevf_X540_vf_info,
49         [board_X540_vf_hv]      = &ixgbevf_X540_vf_hv_info,
50         [board_X550_vf]         = &ixgbevf_X550_vf_info,
51         [board_X550_vf_hv]      = &ixgbevf_X550_vf_hv_info,
52         [board_X550EM_x_vf]     = &ixgbevf_X550EM_x_vf_info,
53         [board_X550EM_x_vf_hv]  = &ixgbevf_X550EM_x_vf_hv_info,
54         [board_x550em_a_vf]     = &ixgbevf_x550em_a_vf_info,
55 };
56
57 /* ixgbevf_pci_tbl - PCI Device ID Table
58  *
59  * Wildcard entries (PCI_ANY_ID) should come last
60  * Last entry must be all 0s
61  *
62  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
63  *   Class, Class Mask, private data (not used) }
64  */
65 static const struct pci_device_id ixgbevf_pci_tbl[] = {
66         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
67         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv },
68         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
69         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv },
70         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
71         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv },
72         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
73         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv},
74         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf },
75         /* required last entry */
76         {0, }
77 };
78 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
79
80 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
81 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
82 MODULE_LICENSE("GPL v2");
83 MODULE_VERSION(DRV_VERSION);
84
85 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
86 static int debug = -1;
87 module_param(debug, int, 0);
88 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
89
90 static struct workqueue_struct *ixgbevf_wq;
91
92 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter)
93 {
94         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
95             !test_bit(__IXGBEVF_REMOVING, &adapter->state) &&
96             !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state))
97                 queue_work(ixgbevf_wq, &adapter->service_task);
98 }
99
100 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter)
101 {
102         BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state));
103
104         /* flush memory to make sure state is correct before next watchdog */
105         smp_mb__before_atomic();
106         clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state);
107 }
108
109 /* forward decls */
110 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
111 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
112 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
113 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer);
114 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
115                                   struct ixgbevf_rx_buffer *old_buff);
116
117 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
118 {
119         struct ixgbevf_adapter *adapter = hw->back;
120
121         if (!hw->hw_addr)
122                 return;
123         hw->hw_addr = NULL;
124         dev_err(&adapter->pdev->dev, "Adapter removed\n");
125         if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state))
126                 ixgbevf_service_event_schedule(adapter);
127 }
128
129 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
130 {
131         u32 value;
132
133         /* The following check not only optimizes a bit by not
134          * performing a read on the status register when the
135          * register just read was a status register read that
136          * returned IXGBE_FAILED_READ_REG. It also blocks any
137          * potential recursion.
138          */
139         if (reg == IXGBE_VFSTATUS) {
140                 ixgbevf_remove_adapter(hw);
141                 return;
142         }
143         value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
144         if (value == IXGBE_FAILED_READ_REG)
145                 ixgbevf_remove_adapter(hw);
146 }
147
148 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
149 {
150         u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);
151         u32 value;
152
153         if (IXGBE_REMOVED(reg_addr))
154                 return IXGBE_FAILED_READ_REG;
155         value = readl(reg_addr + reg);
156         if (unlikely(value == IXGBE_FAILED_READ_REG))
157                 ixgbevf_check_remove(hw, reg);
158         return value;
159 }
160
161 /**
162  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
163  * @adapter: pointer to adapter struct
164  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
165  * @queue: queue to map the corresponding interrupt to
166  * @msix_vector: the vector to map to the corresponding queue
167  **/
168 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
169                              u8 queue, u8 msix_vector)
170 {
171         u32 ivar, index;
172         struct ixgbe_hw *hw = &adapter->hw;
173
174         if (direction == -1) {
175                 /* other causes */
176                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
177                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
178                 ivar &= ~0xFF;
179                 ivar |= msix_vector;
180                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
181         } else {
182                 /* Tx or Rx causes */
183                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
184                 index = ((16 * (queue & 1)) + (8 * direction));
185                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
186                 ivar &= ~(0xFF << index);
187                 ivar |= (msix_vector << index);
188                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
189         }
190 }
191
192 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
193 {
194         return ring->stats.packets;
195 }
196
197 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring)
198 {
199         struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev);
200         struct ixgbe_hw *hw = &adapter->hw;
201
202         u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx));
203         u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx));
204
205         if (head != tail)
206                 return (head < tail) ?
207                         tail - head : (tail + ring->count - head);
208
209         return 0;
210 }
211
212 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring)
213 {
214         u32 tx_done = ixgbevf_get_tx_completed(tx_ring);
215         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
216         u32 tx_pending = ixgbevf_get_tx_pending(tx_ring);
217
218         clear_check_for_tx_hang(tx_ring);
219
220         /* Check for a hung queue, but be thorough. This verifies
221          * that a transmit has been completed since the previous
222          * check AND there is at least one packet pending. The
223          * ARMED bit is set to indicate a potential hang.
224          */
225         if ((tx_done_old == tx_done) && tx_pending) {
226                 /* make sure it is true for two checks in a row */
227                 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED,
228                                         &tx_ring->state);
229         }
230         /* reset the countdown */
231         clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state);
232
233         /* update completed stats and continue */
234         tx_ring->tx_stats.tx_done_old = tx_done;
235
236         return false;
237 }
238
239 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter)
240 {
241         /* Do the reset outside of interrupt context */
242         if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
243                 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state);
244                 ixgbevf_service_event_schedule(adapter);
245         }
246 }
247
248 /**
249  * ixgbevf_tx_timeout - Respond to a Tx Hang
250  * @netdev: network interface device structure
251  **/
252 static void ixgbevf_tx_timeout(struct net_device *netdev)
253 {
254         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
255
256         ixgbevf_tx_timeout_reset(adapter);
257 }
258
259 /**
260  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
261  * @q_vector: board private structure
262  * @tx_ring: tx ring to clean
263  * @napi_budget: Used to determine if we are in netpoll
264  **/
265 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
266                                  struct ixgbevf_ring *tx_ring, int napi_budget)
267 {
268         struct ixgbevf_adapter *adapter = q_vector->adapter;
269         struct ixgbevf_tx_buffer *tx_buffer;
270         union ixgbe_adv_tx_desc *tx_desc;
271         unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
272         unsigned int budget = tx_ring->count / 2;
273         unsigned int i = tx_ring->next_to_clean;
274
275         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
276                 return true;
277
278         tx_buffer = &tx_ring->tx_buffer_info[i];
279         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
280         i -= tx_ring->count;
281
282         do {
283                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
284
285                 /* if next_to_watch is not set then there is no work pending */
286                 if (!eop_desc)
287                         break;
288
289                 /* prevent any other reads prior to eop_desc */
290                 smp_rmb();
291
292                 /* if DD is not set pending work has not been completed */
293                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
294                         break;
295
296                 /* clear next_to_watch to prevent false hangs */
297                 tx_buffer->next_to_watch = NULL;
298
299                 /* update the statistics for this packet */
300                 total_bytes += tx_buffer->bytecount;
301                 total_packets += tx_buffer->gso_segs;
302                 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
303                         total_ipsec++;
304
305                 /* free the skb */
306                 if (ring_is_xdp(tx_ring))
307                         page_frag_free(tx_buffer->data);
308                 else
309                         napi_consume_skb(tx_buffer->skb, napi_budget);
310
311                 /* unmap skb header data */
312                 dma_unmap_single(tx_ring->dev,
313                                  dma_unmap_addr(tx_buffer, dma),
314                                  dma_unmap_len(tx_buffer, len),
315                                  DMA_TO_DEVICE);
316
317                 /* clear tx_buffer data */
318                 dma_unmap_len_set(tx_buffer, len, 0);
319
320                 /* unmap remaining buffers */
321                 while (tx_desc != eop_desc) {
322                         tx_buffer++;
323                         tx_desc++;
324                         i++;
325                         if (unlikely(!i)) {
326                                 i -= tx_ring->count;
327                                 tx_buffer = tx_ring->tx_buffer_info;
328                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
329                         }
330
331                         /* unmap any remaining paged data */
332                         if (dma_unmap_len(tx_buffer, len)) {
333                                 dma_unmap_page(tx_ring->dev,
334                                                dma_unmap_addr(tx_buffer, dma),
335                                                dma_unmap_len(tx_buffer, len),
336                                                DMA_TO_DEVICE);
337                                 dma_unmap_len_set(tx_buffer, len, 0);
338                         }
339                 }
340
341                 /* move us one more past the eop_desc for start of next pkt */
342                 tx_buffer++;
343                 tx_desc++;
344                 i++;
345                 if (unlikely(!i)) {
346                         i -= tx_ring->count;
347                         tx_buffer = tx_ring->tx_buffer_info;
348                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
349                 }
350
351                 /* issue prefetch for next Tx descriptor */
352                 prefetch(tx_desc);
353
354                 /* update budget accounting */
355                 budget--;
356         } while (likely(budget));
357
358         i += tx_ring->count;
359         tx_ring->next_to_clean = i;
360         u64_stats_update_begin(&tx_ring->syncp);
361         tx_ring->stats.bytes += total_bytes;
362         tx_ring->stats.packets += total_packets;
363         u64_stats_update_end(&tx_ring->syncp);
364         q_vector->tx.total_bytes += total_bytes;
365         q_vector->tx.total_packets += total_packets;
366         adapter->tx_ipsec += total_ipsec;
367
368         if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) {
369                 struct ixgbe_hw *hw = &adapter->hw;
370                 union ixgbe_adv_tx_desc *eop_desc;
371
372                 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch;
373
374                 pr_err("Detected Tx Unit Hang%s\n"
375                        "  Tx Queue             <%d>\n"
376                        "  TDH, TDT             <%x>, <%x>\n"
377                        "  next_to_use          <%x>\n"
378                        "  next_to_clean        <%x>\n"
379                        "tx_buffer_info[next_to_clean]\n"
380                        "  next_to_watch        <%p>\n"
381                        "  eop_desc->wb.status  <%x>\n"
382                        "  time_stamp           <%lx>\n"
383                        "  jiffies              <%lx>\n",
384                        ring_is_xdp(tx_ring) ? " XDP" : "",
385                        tx_ring->queue_index,
386                        IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)),
387                        IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)),
388                        tx_ring->next_to_use, i,
389                        eop_desc, (eop_desc ? eop_desc->wb.status : 0),
390                        tx_ring->tx_buffer_info[i].time_stamp, jiffies);
391
392                 if (!ring_is_xdp(tx_ring))
393                         netif_stop_subqueue(tx_ring->netdev,
394                                             tx_ring->queue_index);
395
396                 /* schedule immediate reset if we believe we hung */
397                 ixgbevf_tx_timeout_reset(adapter);
398
399                 return true;
400         }
401
402         if (ring_is_xdp(tx_ring))
403                 return !!budget;
404
405 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
406         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
407                      (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
408                 /* Make sure that anybody stopping the queue after this
409                  * sees the new next_to_clean.
410                  */
411                 smp_mb();
412
413                 if (__netif_subqueue_stopped(tx_ring->netdev,
414                                              tx_ring->queue_index) &&
415                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
416                         netif_wake_subqueue(tx_ring->netdev,
417                                             tx_ring->queue_index);
418                         ++tx_ring->tx_stats.restart_queue;
419                 }
420         }
421
422         return !!budget;
423 }
424
425 /**
426  * ixgbevf_rx_skb - Helper function to determine proper Rx method
427  * @q_vector: structure containing interrupt and ring information
428  * @skb: packet to send up
429  **/
430 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
431                            struct sk_buff *skb)
432 {
433         napi_gro_receive(&q_vector->napi, skb);
434 }
435
436 #define IXGBE_RSS_L4_TYPES_MASK \
437         ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \
438          (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \
439          (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \
440          (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))
441
442 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring,
443                                    union ixgbe_adv_rx_desc *rx_desc,
444                                    struct sk_buff *skb)
445 {
446         u16 rss_type;
447
448         if (!(ring->netdev->features & NETIF_F_RXHASH))
449                 return;
450
451         rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &
452                    IXGBE_RXDADV_RSSTYPE_MASK;
453
454         if (!rss_type)
455                 return;
456
457         skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
458                      (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?
459                      PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
460 }
461
462 /**
463  * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
464  * @ring: structure containig ring specific data
465  * @rx_desc: current Rx descriptor being processed
466  * @skb: skb currently being received and modified
467  **/
468 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
469                                        union ixgbe_adv_rx_desc *rx_desc,
470                                        struct sk_buff *skb)
471 {
472         skb_checksum_none_assert(skb);
473
474         /* Rx csum disabled */
475         if (!(ring->netdev->features & NETIF_F_RXCSUM))
476                 return;
477
478         /* if IP and error */
479         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
480             ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
481                 ring->rx_stats.csum_err++;
482                 return;
483         }
484
485         if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
486                 return;
487
488         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
489                 ring->rx_stats.csum_err++;
490                 return;
491         }
492
493         /* It must be a TCP or UDP packet with a valid checksum */
494         skb->ip_summed = CHECKSUM_UNNECESSARY;
495 }
496
497 /**
498  * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
499  * @rx_ring: rx descriptor ring packet is being transacted on
500  * @rx_desc: pointer to the EOP Rx descriptor
501  * @skb: pointer to current skb being populated
502  *
503  * This function checks the ring, descriptor, and packet information in
504  * order to populate the checksum, VLAN, protocol, and other fields within
505  * the skb.
506  **/
507 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
508                                        union ixgbe_adv_rx_desc *rx_desc,
509                                        struct sk_buff *skb)
510 {
511         ixgbevf_rx_hash(rx_ring, rx_desc, skb);
512         ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
513
514         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
515                 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
516                 unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
517
518                 if (test_bit(vid & VLAN_VID_MASK, active_vlans))
519                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
520         }
521
522         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
523                 ixgbevf_ipsec_rx(rx_ring, rx_desc, skb);
524
525         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
526 }
527
528 static
529 struct ixgbevf_rx_buffer *ixgbevf_get_rx_buffer(struct ixgbevf_ring *rx_ring,
530                                                 const unsigned int size)
531 {
532         struct ixgbevf_rx_buffer *rx_buffer;
533
534         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
535         prefetchw(rx_buffer->page);
536
537         /* we are reusing so sync this buffer for CPU use */
538         dma_sync_single_range_for_cpu(rx_ring->dev,
539                                       rx_buffer->dma,
540                                       rx_buffer->page_offset,
541                                       size,
542                                       DMA_FROM_DEVICE);
543
544         rx_buffer->pagecnt_bias--;
545
546         return rx_buffer;
547 }
548
549 static void ixgbevf_put_rx_buffer(struct ixgbevf_ring *rx_ring,
550                                   struct ixgbevf_rx_buffer *rx_buffer,
551                                   struct sk_buff *skb)
552 {
553         if (ixgbevf_can_reuse_rx_page(rx_buffer)) {
554                 /* hand second half of page back to the ring */
555                 ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
556         } else {
557                 if (IS_ERR(skb))
558                         /* We are not reusing the buffer so unmap it and free
559                          * any references we are holding to it
560                          */
561                         dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
562                                              ixgbevf_rx_pg_size(rx_ring),
563                                              DMA_FROM_DEVICE,
564                                              IXGBEVF_RX_DMA_ATTR);
565                 __page_frag_cache_drain(rx_buffer->page,
566                                         rx_buffer->pagecnt_bias);
567         }
568
569         /* clear contents of rx_buffer */
570         rx_buffer->page = NULL;
571 }
572
573 /**
574  * ixgbevf_is_non_eop - process handling of non-EOP buffers
575  * @rx_ring: Rx ring being processed
576  * @rx_desc: Rx descriptor for current buffer
577  *
578  * This function updates next to clean.  If the buffer is an EOP buffer
579  * this function exits returning false, otherwise it will place the
580  * sk_buff in the next buffer to be chained and return true indicating
581  * that this is in fact a non-EOP buffer.
582  **/
583 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
584                                union ixgbe_adv_rx_desc *rx_desc)
585 {
586         u32 ntc = rx_ring->next_to_clean + 1;
587
588         /* fetch, update, and store next to clean */
589         ntc = (ntc < rx_ring->count) ? ntc : 0;
590         rx_ring->next_to_clean = ntc;
591
592         prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
593
594         if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
595                 return false;
596
597         return true;
598 }
599
600 static inline unsigned int ixgbevf_rx_offset(struct ixgbevf_ring *rx_ring)
601 {
602         return ring_uses_build_skb(rx_ring) ? IXGBEVF_SKB_PAD : 0;
603 }
604
605 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
606                                       struct ixgbevf_rx_buffer *bi)
607 {
608         struct page *page = bi->page;
609         dma_addr_t dma;
610
611         /* since we are recycling buffers we should seldom need to alloc */
612         if (likely(page))
613                 return true;
614
615         /* alloc new page for storage */
616         page = dev_alloc_pages(ixgbevf_rx_pg_order(rx_ring));
617         if (unlikely(!page)) {
618                 rx_ring->rx_stats.alloc_rx_page_failed++;
619                 return false;
620         }
621
622         /* map page for use */
623         dma = dma_map_page_attrs(rx_ring->dev, page, 0,
624                                  ixgbevf_rx_pg_size(rx_ring),
625                                  DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);
626
627         /* if mapping failed free memory back to system since
628          * there isn't much point in holding memory we can't use
629          */
630         if (dma_mapping_error(rx_ring->dev, dma)) {
631                 __free_pages(page, ixgbevf_rx_pg_order(rx_ring));
632
633                 rx_ring->rx_stats.alloc_rx_page_failed++;
634                 return false;
635         }
636
637         bi->dma = dma;
638         bi->page = page;
639         bi->page_offset = ixgbevf_rx_offset(rx_ring);
640         bi->pagecnt_bias = 1;
641         rx_ring->rx_stats.alloc_rx_page++;
642
643         return true;
644 }
645
646 /**
647  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
648  * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
649  * @cleaned_count: number of buffers to replace
650  **/
651 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
652                                      u16 cleaned_count)
653 {
654         union ixgbe_adv_rx_desc *rx_desc;
655         struct ixgbevf_rx_buffer *bi;
656         unsigned int i = rx_ring->next_to_use;
657
658         /* nothing to do or no valid netdev defined */
659         if (!cleaned_count || !rx_ring->netdev)
660                 return;
661
662         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
663         bi = &rx_ring->rx_buffer_info[i];
664         i -= rx_ring->count;
665
666         do {
667                 if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
668                         break;
669
670                 /* sync the buffer for use by the device */
671                 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
672                                                  bi->page_offset,
673                                                  ixgbevf_rx_bufsz(rx_ring),
674                                                  DMA_FROM_DEVICE);
675
676                 /* Refresh the desc even if pkt_addr didn't change
677                  * because each write-back erases this info.
678                  */
679                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
680
681                 rx_desc++;
682                 bi++;
683                 i++;
684                 if (unlikely(!i)) {
685                         rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
686                         bi = rx_ring->rx_buffer_info;
687                         i -= rx_ring->count;
688                 }
689
690                 /* clear the length for the next_to_use descriptor */
691                 rx_desc->wb.upper.length = 0;
692
693                 cleaned_count--;
694         } while (cleaned_count);
695
696         i += rx_ring->count;
697
698         if (rx_ring->next_to_use != i) {
699                 /* record the next descriptor to use */
700                 rx_ring->next_to_use = i;
701
702                 /* update next to alloc since we have filled the ring */
703                 rx_ring->next_to_alloc = i;
704
705                 /* Force memory writes to complete before letting h/w
706                  * know there are new descriptors to fetch.  (Only
707                  * applicable for weak-ordered memory model archs,
708                  * such as IA-64).
709                  */
710                 wmb();
711                 ixgbevf_write_tail(rx_ring, i);
712         }
713 }
714
715 /**
716  * ixgbevf_cleanup_headers - Correct corrupted or empty headers
717  * @rx_ring: rx descriptor ring packet is being transacted on
718  * @rx_desc: pointer to the EOP Rx descriptor
719  * @skb: pointer to current skb being fixed
720  *
721  * Check for corrupted packet headers caused by senders on the local L2
722  * embedded NIC switch not setting up their Tx Descriptors right.  These
723  * should be very rare.
724  *
725  * Also address the case where we are pulling data in on pages only
726  * and as such no data is present in the skb header.
727  *
728  * In addition if skb is not at least 60 bytes we need to pad it so that
729  * it is large enough to qualify as a valid Ethernet frame.
730  *
731  * Returns true if an error was encountered and skb was freed.
732  **/
733 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
734                                     union ixgbe_adv_rx_desc *rx_desc,
735                                     struct sk_buff *skb)
736 {
737         /* XDP packets use error pointer so abort at this point */
738         if (IS_ERR(skb))
739                 return true;
740
741         /* verify that the packet does not have any known errors */
742         if (unlikely(ixgbevf_test_staterr(rx_desc,
743                                           IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
744                 struct net_device *netdev = rx_ring->netdev;
745
746                 if (!(netdev->features & NETIF_F_RXALL)) {
747                         dev_kfree_skb_any(skb);
748                         return true;
749                 }
750         }
751
752         /* if eth_skb_pad returns an error the skb was freed */
753         if (eth_skb_pad(skb))
754                 return true;
755
756         return false;
757 }
758
759 /**
760  * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
761  * @rx_ring: rx descriptor ring to store buffers on
762  * @old_buff: donor buffer to have page reused
763  *
764  * Synchronizes page for reuse by the adapter
765  **/
766 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
767                                   struct ixgbevf_rx_buffer *old_buff)
768 {
769         struct ixgbevf_rx_buffer *new_buff;
770         u16 nta = rx_ring->next_to_alloc;
771
772         new_buff = &rx_ring->rx_buffer_info[nta];
773
774         /* update, and store next to alloc */
775         nta++;
776         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
777
778         /* transfer page from old buffer to new buffer */
779         new_buff->page = old_buff->page;
780         new_buff->dma = old_buff->dma;
781         new_buff->page_offset = old_buff->page_offset;
782         new_buff->pagecnt_bias = old_buff->pagecnt_bias;
783 }
784
785 static inline bool ixgbevf_page_is_reserved(struct page *page)
786 {
787         return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
788 }
789
790 static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer)
791 {
792         unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
793         struct page *page = rx_buffer->page;
794
795         /* avoid re-using remote pages */
796         if (unlikely(ixgbevf_page_is_reserved(page)))
797                 return false;
798
799 #if (PAGE_SIZE < 8192)
800         /* if we are only owner of page we can reuse it */
801         if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
802                 return false;
803 #else
804 #define IXGBEVF_LAST_OFFSET \
805         (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBEVF_RXBUFFER_2048)
806
807         if (rx_buffer->page_offset > IXGBEVF_LAST_OFFSET)
808                 return false;
809
810 #endif
811
812         /* If we have drained the page fragment pool we need to update
813          * the pagecnt_bias and page count so that we fully restock the
814          * number of references the driver holds.
815          */
816         if (unlikely(!pagecnt_bias)) {
817                 page_ref_add(page, USHRT_MAX);
818                 rx_buffer->pagecnt_bias = USHRT_MAX;
819         }
820
821         return true;
822 }
823
824 /**
825  * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
826  * @rx_ring: rx descriptor ring to transact packets on
827  * @rx_buffer: buffer containing page to add
828  * @skb: sk_buff to place the data into
829  * @size: size of buffer to be added
830  *
831  * This function will add the data contained in rx_buffer->page to the skb.
832  **/
833 static void ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
834                                 struct ixgbevf_rx_buffer *rx_buffer,
835                                 struct sk_buff *skb,
836                                 unsigned int size)
837 {
838 #if (PAGE_SIZE < 8192)
839         unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
840 #else
841         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
842                                 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) :
843                                 SKB_DATA_ALIGN(size);
844 #endif
845         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
846                         rx_buffer->page_offset, size, truesize);
847 #if (PAGE_SIZE < 8192)
848         rx_buffer->page_offset ^= truesize;
849 #else
850         rx_buffer->page_offset += truesize;
851 #endif
852 }
853
854 static
855 struct sk_buff *ixgbevf_construct_skb(struct ixgbevf_ring *rx_ring,
856                                       struct ixgbevf_rx_buffer *rx_buffer,
857                                       struct xdp_buff *xdp,
858                                       union ixgbe_adv_rx_desc *rx_desc)
859 {
860         unsigned int size = xdp->data_end - xdp->data;
861 #if (PAGE_SIZE < 8192)
862         unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
863 #else
864         unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
865                                                xdp->data_hard_start);
866 #endif
867         unsigned int headlen;
868         struct sk_buff *skb;
869
870         /* prefetch first cache line of first page */
871         prefetch(xdp->data);
872 #if L1_CACHE_BYTES < 128
873         prefetch(xdp->data + L1_CACHE_BYTES);
874 #endif
875         /* Note, we get here by enabling legacy-rx via:
876          *
877          *    ethtool --set-priv-flags <dev> legacy-rx on
878          *
879          * In this mode, we currently get 0 extra XDP headroom as
880          * opposed to having legacy-rx off, where we process XDP
881          * packets going to stack via ixgbevf_build_skb().
882          *
883          * For ixgbevf_construct_skb() mode it means that the
884          * xdp->data_meta will always point to xdp->data, since
885          * the helper cannot expand the head. Should this ever
886          * changed in future for legacy-rx mode on, then lets also
887          * add xdp->data_meta handling here.
888          */
889
890         /* allocate a skb to store the frags */
891         skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBEVF_RX_HDR_SIZE);
892         if (unlikely(!skb))
893                 return NULL;
894
895         /* Determine available headroom for copy */
896         headlen = size;
897         if (headlen > IXGBEVF_RX_HDR_SIZE)
898                 headlen = eth_get_headlen(xdp->data, IXGBEVF_RX_HDR_SIZE);
899
900         /* align pull length to size of long to optimize memcpy performance */
901         memcpy(__skb_put(skb, headlen), xdp->data,
902                ALIGN(headlen, sizeof(long)));
903
904         /* update all of the pointers */
905         size -= headlen;
906         if (size) {
907                 skb_add_rx_frag(skb, 0, rx_buffer->page,
908                                 (xdp->data + headlen) -
909                                         page_address(rx_buffer->page),
910                                 size, truesize);
911 #if (PAGE_SIZE < 8192)
912                 rx_buffer->page_offset ^= truesize;
913 #else
914                 rx_buffer->page_offset += truesize;
915 #endif
916         } else {
917                 rx_buffer->pagecnt_bias++;
918         }
919
920         return skb;
921 }
922
923 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
924                                              u32 qmask)
925 {
926         struct ixgbe_hw *hw = &adapter->hw;
927
928         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
929 }
930
931 static struct sk_buff *ixgbevf_build_skb(struct ixgbevf_ring *rx_ring,
932                                          struct ixgbevf_rx_buffer *rx_buffer,
933                                          struct xdp_buff *xdp,
934                                          union ixgbe_adv_rx_desc *rx_desc)
935 {
936         unsigned int metasize = xdp->data - xdp->data_meta;
937 #if (PAGE_SIZE < 8192)
938         unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
939 #else
940         unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
941                                 SKB_DATA_ALIGN(xdp->data_end -
942                                                xdp->data_hard_start);
943 #endif
944         struct sk_buff *skb;
945
946         /* Prefetch first cache line of first page. If xdp->data_meta
947          * is unused, this points to xdp->data, otherwise, we likely
948          * have a consumer accessing first few bytes of meta data,
949          * and then actual data.
950          */
951         prefetch(xdp->data_meta);
952 #if L1_CACHE_BYTES < 128
953         prefetch(xdp->data_meta + L1_CACHE_BYTES);
954 #endif
955
956         /* build an skb around the page buffer */
957         skb = build_skb(xdp->data_hard_start, truesize);
958         if (unlikely(!skb))
959                 return NULL;
960
961         /* update pointers within the skb to store the data */
962         skb_reserve(skb, xdp->data - xdp->data_hard_start);
963         __skb_put(skb, xdp->data_end - xdp->data);
964         if (metasize)
965                 skb_metadata_set(skb, metasize);
966
967         /* update buffer offset */
968 #if (PAGE_SIZE < 8192)
969         rx_buffer->page_offset ^= truesize;
970 #else
971         rx_buffer->page_offset += truesize;
972 #endif
973
974         return skb;
975 }
976
977 #define IXGBEVF_XDP_PASS 0
978 #define IXGBEVF_XDP_CONSUMED 1
979 #define IXGBEVF_XDP_TX 2
980
981 static int ixgbevf_xmit_xdp_ring(struct ixgbevf_ring *ring,
982                                  struct xdp_buff *xdp)
983 {
984         struct ixgbevf_tx_buffer *tx_buffer;
985         union ixgbe_adv_tx_desc *tx_desc;
986         u32 len, cmd_type;
987         dma_addr_t dma;
988         u16 i;
989
990         len = xdp->data_end - xdp->data;
991
992         if (unlikely(!ixgbevf_desc_unused(ring)))
993                 return IXGBEVF_XDP_CONSUMED;
994
995         dma = dma_map_single(ring->dev, xdp->data, len, DMA_TO_DEVICE);
996         if (dma_mapping_error(ring->dev, dma))
997                 return IXGBEVF_XDP_CONSUMED;
998
999         /* record the location of the first descriptor for this packet */
1000         i = ring->next_to_use;
1001         tx_buffer = &ring->tx_buffer_info[i];
1002
1003         dma_unmap_len_set(tx_buffer, len, len);
1004         dma_unmap_addr_set(tx_buffer, dma, dma);
1005         tx_buffer->data = xdp->data;
1006         tx_buffer->bytecount = len;
1007         tx_buffer->gso_segs = 1;
1008         tx_buffer->protocol = 0;
1009
1010         /* Populate minimal context descriptor that will provide for the
1011          * fact that we are expected to process Ethernet frames.
1012          */
1013         if (!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state)) {
1014                 struct ixgbe_adv_tx_context_desc *context_desc;
1015
1016                 set_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state);
1017
1018                 context_desc = IXGBEVF_TX_CTXTDESC(ring, 0);
1019                 context_desc->vlan_macip_lens   =
1020                         cpu_to_le32(ETH_HLEN << IXGBE_ADVTXD_MACLEN_SHIFT);
1021                 context_desc->fceof_saidx       = 0;
1022                 context_desc->type_tucmd_mlhl   =
1023                         cpu_to_le32(IXGBE_TXD_CMD_DEXT |
1024                                     IXGBE_ADVTXD_DTYP_CTXT);
1025                 context_desc->mss_l4len_idx     = 0;
1026
1027                 i = 1;
1028         }
1029
1030         /* put descriptor type bits */
1031         cmd_type = IXGBE_ADVTXD_DTYP_DATA |
1032                    IXGBE_ADVTXD_DCMD_DEXT |
1033                    IXGBE_ADVTXD_DCMD_IFCS;
1034         cmd_type |= len | IXGBE_TXD_CMD;
1035
1036         tx_desc = IXGBEVF_TX_DESC(ring, i);
1037         tx_desc->read.buffer_addr = cpu_to_le64(dma);
1038
1039         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1040         tx_desc->read.olinfo_status =
1041                         cpu_to_le32((len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
1042                                     IXGBE_ADVTXD_CC);
1043
1044         /* Avoid any potential race with cleanup */
1045         smp_wmb();
1046
1047         /* set next_to_watch value indicating a packet is present */
1048         i++;
1049         if (i == ring->count)
1050                 i = 0;
1051
1052         tx_buffer->next_to_watch = tx_desc;
1053         ring->next_to_use = i;
1054
1055         return IXGBEVF_XDP_TX;
1056 }
1057
1058 static struct sk_buff *ixgbevf_run_xdp(struct ixgbevf_adapter *adapter,
1059                                        struct ixgbevf_ring  *rx_ring,
1060                                        struct xdp_buff *xdp)
1061 {
1062         int result = IXGBEVF_XDP_PASS;
1063         struct ixgbevf_ring *xdp_ring;
1064         struct bpf_prog *xdp_prog;
1065         u32 act;
1066
1067         rcu_read_lock();
1068         xdp_prog = READ_ONCE(rx_ring->xdp_prog);
1069
1070         if (!xdp_prog)
1071                 goto xdp_out;
1072
1073         act = bpf_prog_run_xdp(xdp_prog, xdp);
1074         switch (act) {
1075         case XDP_PASS:
1076                 break;
1077         case XDP_TX:
1078                 xdp_ring = adapter->xdp_ring[rx_ring->queue_index];
1079                 result = ixgbevf_xmit_xdp_ring(xdp_ring, xdp);
1080                 break;
1081         default:
1082                 bpf_warn_invalid_xdp_action(act);
1083                 /* fallthrough */
1084         case XDP_ABORTED:
1085                 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
1086                 /* fallthrough -- handle aborts by dropping packet */
1087         case XDP_DROP:
1088                 result = IXGBEVF_XDP_CONSUMED;
1089                 break;
1090         }
1091 xdp_out:
1092         rcu_read_unlock();
1093         return ERR_PTR(-result);
1094 }
1095
1096 static void ixgbevf_rx_buffer_flip(struct ixgbevf_ring *rx_ring,
1097                                    struct ixgbevf_rx_buffer *rx_buffer,
1098                                    unsigned int size)
1099 {
1100 #if (PAGE_SIZE < 8192)
1101         unsigned int truesize = ixgbevf_rx_pg_size(rx_ring) / 2;
1102
1103         rx_buffer->page_offset ^= truesize;
1104 #else
1105         unsigned int truesize = ring_uses_build_skb(rx_ring) ?
1106                                 SKB_DATA_ALIGN(IXGBEVF_SKB_PAD + size) :
1107                                 SKB_DATA_ALIGN(size);
1108
1109         rx_buffer->page_offset += truesize;
1110 #endif
1111 }
1112
1113 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
1114                                 struct ixgbevf_ring *rx_ring,
1115                                 int budget)
1116 {
1117         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1118         struct ixgbevf_adapter *adapter = q_vector->adapter;
1119         u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
1120         struct sk_buff *skb = rx_ring->skb;
1121         bool xdp_xmit = false;
1122         struct xdp_buff xdp;
1123
1124         xdp.rxq = &rx_ring->xdp_rxq;
1125
1126         while (likely(total_rx_packets < budget)) {
1127                 struct ixgbevf_rx_buffer *rx_buffer;
1128                 union ixgbe_adv_rx_desc *rx_desc;
1129                 unsigned int size;
1130
1131                 /* return some buffers to hardware, one at a time is too slow */
1132                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
1133                         ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
1134                         cleaned_count = 0;
1135                 }
1136
1137                 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
1138                 size = le16_to_cpu(rx_desc->wb.upper.length);
1139                 if (!size)
1140                         break;
1141
1142                 /* This memory barrier is needed to keep us from reading
1143                  * any other fields out of the rx_desc until we know the
1144                  * RXD_STAT_DD bit is set
1145                  */
1146                 rmb();
1147
1148                 rx_buffer = ixgbevf_get_rx_buffer(rx_ring, size);
1149
1150                 /* retrieve a buffer from the ring */
1151                 if (!skb) {
1152                         xdp.data = page_address(rx_buffer->page) +
1153                                    rx_buffer->page_offset;
1154                         xdp.data_meta = xdp.data;
1155                         xdp.data_hard_start = xdp.data -
1156                                               ixgbevf_rx_offset(rx_ring);
1157                         xdp.data_end = xdp.data + size;
1158
1159                         skb = ixgbevf_run_xdp(adapter, rx_ring, &xdp);
1160                 }
1161
1162                 if (IS_ERR(skb)) {
1163                         if (PTR_ERR(skb) == -IXGBEVF_XDP_TX) {
1164                                 xdp_xmit = true;
1165                                 ixgbevf_rx_buffer_flip(rx_ring, rx_buffer,
1166                                                        size);
1167                         } else {
1168                                 rx_buffer->pagecnt_bias++;
1169                         }
1170                         total_rx_packets++;
1171                         total_rx_bytes += size;
1172                 } else if (skb) {
1173                         ixgbevf_add_rx_frag(rx_ring, rx_buffer, skb, size);
1174                 } else if (ring_uses_build_skb(rx_ring)) {
1175                         skb = ixgbevf_build_skb(rx_ring, rx_buffer,
1176                                                 &xdp, rx_desc);
1177                 } else {
1178                         skb = ixgbevf_construct_skb(rx_ring, rx_buffer,
1179                                                     &xdp, rx_desc);
1180                 }
1181
1182                 /* exit if we failed to retrieve a buffer */
1183                 if (!skb) {
1184                         rx_ring->rx_stats.alloc_rx_buff_failed++;
1185                         rx_buffer->pagecnt_bias++;
1186                         break;
1187                 }
1188
1189                 ixgbevf_put_rx_buffer(rx_ring, rx_buffer, skb);
1190                 cleaned_count++;
1191
1192                 /* fetch next buffer in frame if non-eop */
1193                 if (ixgbevf_is_non_eop(rx_ring, rx_desc))
1194                         continue;
1195
1196                 /* verify the packet layout is correct */
1197                 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
1198                         skb = NULL;
1199                         continue;
1200                 }
1201
1202                 /* probably a little skewed due to removing CRC */
1203                 total_rx_bytes += skb->len;
1204
1205                 /* Workaround hardware that can't do proper VEPA multicast
1206                  * source pruning.
1207                  */
1208                 if ((skb->pkt_type == PACKET_BROADCAST ||
1209                      skb->pkt_type == PACKET_MULTICAST) &&
1210                     ether_addr_equal(rx_ring->netdev->dev_addr,
1211                                      eth_hdr(skb)->h_source)) {
1212                         dev_kfree_skb_irq(skb);
1213                         continue;
1214                 }
1215
1216                 /* populate checksum, VLAN, and protocol */
1217                 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
1218
1219                 ixgbevf_rx_skb(q_vector, skb);
1220
1221                 /* reset skb pointer */
1222                 skb = NULL;
1223
1224                 /* update budget accounting */
1225                 total_rx_packets++;
1226         }
1227
1228         /* place incomplete frames back on ring for completion */
1229         rx_ring->skb = skb;
1230
1231         if (xdp_xmit) {
1232                 struct ixgbevf_ring *xdp_ring =
1233                         adapter->xdp_ring[rx_ring->queue_index];
1234
1235                 /* Force memory writes to complete before letting h/w
1236                  * know there are new descriptors to fetch.
1237                  */
1238                 wmb();
1239                 ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use);
1240         }
1241
1242         u64_stats_update_begin(&rx_ring->syncp);
1243         rx_ring->stats.packets += total_rx_packets;
1244         rx_ring->stats.bytes += total_rx_bytes;
1245         u64_stats_update_end(&rx_ring->syncp);
1246         q_vector->rx.total_packets += total_rx_packets;
1247         q_vector->rx.total_bytes += total_rx_bytes;
1248
1249         return total_rx_packets;
1250 }
1251
1252 /**
1253  * ixgbevf_poll - NAPI polling calback
1254  * @napi: napi struct with our devices info in it
1255  * @budget: amount of work driver is allowed to do this pass, in packets
1256  *
1257  * This function will clean more than one or more rings associated with a
1258  * q_vector.
1259  **/
1260 static int ixgbevf_poll(struct napi_struct *napi, int budget)
1261 {
1262         struct ixgbevf_q_vector *q_vector =
1263                 container_of(napi, struct ixgbevf_q_vector, napi);
1264         struct ixgbevf_adapter *adapter = q_vector->adapter;
1265         struct ixgbevf_ring *ring;
1266         int per_ring_budget, work_done = 0;
1267         bool clean_complete = true;
1268
1269         ixgbevf_for_each_ring(ring, q_vector->tx) {
1270                 if (!ixgbevf_clean_tx_irq(q_vector, ring, budget))
1271                         clean_complete = false;
1272         }
1273
1274         if (budget <= 0)
1275                 return budget;
1276
1277         /* attempt to distribute budget to each queue fairly, but don't allow
1278          * the budget to go below 1 because we'll exit polling
1279          */
1280         if (q_vector->rx.count > 1)
1281                 per_ring_budget = max(budget/q_vector->rx.count, 1);
1282         else
1283                 per_ring_budget = budget;
1284
1285         ixgbevf_for_each_ring(ring, q_vector->rx) {
1286                 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring,
1287                                                    per_ring_budget);
1288                 work_done += cleaned;
1289                 if (cleaned >= per_ring_budget)
1290                         clean_complete = false;
1291         }
1292
1293         /* If all work not completed, return budget and keep polling */
1294         if (!clean_complete)
1295                 return budget;
1296         /* all work done, exit the polling mode */
1297         napi_complete_done(napi, work_done);
1298         if (adapter->rx_itr_setting == 1)
1299                 ixgbevf_set_itr(q_vector);
1300         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
1301             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
1302                 ixgbevf_irq_enable_queues(adapter,
1303                                           BIT(q_vector->v_idx));
1304
1305         return 0;
1306 }
1307
1308 /**
1309  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
1310  * @q_vector: structure containing interrupt and ring information
1311  **/
1312 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
1313 {
1314         struct ixgbevf_adapter *adapter = q_vector->adapter;
1315         struct ixgbe_hw *hw = &adapter->hw;
1316         int v_idx = q_vector->v_idx;
1317         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
1318
1319         /* set the WDIS bit to not clear the timer bits and cause an
1320          * immediate assertion of the interrupt
1321          */
1322         itr_reg |= IXGBE_EITR_CNT_WDIS;
1323
1324         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
1325 }
1326
1327 /**
1328  * ixgbevf_configure_msix - Configure MSI-X hardware
1329  * @adapter: board private structure
1330  *
1331  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
1332  * interrupts.
1333  **/
1334 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
1335 {
1336         struct ixgbevf_q_vector *q_vector;
1337         int q_vectors, v_idx;
1338
1339         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1340         adapter->eims_enable_mask = 0;
1341
1342         /* Populate the IVAR table and set the ITR values to the
1343          * corresponding register.
1344          */
1345         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1346                 struct ixgbevf_ring *ring;
1347
1348                 q_vector = adapter->q_vector[v_idx];
1349
1350                 ixgbevf_for_each_ring(ring, q_vector->rx)
1351                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1352
1353                 ixgbevf_for_each_ring(ring, q_vector->tx)
1354                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1355
1356                 if (q_vector->tx.ring && !q_vector->rx.ring) {
1357                         /* Tx only vector */
1358                         if (adapter->tx_itr_setting == 1)
1359                                 q_vector->itr = IXGBE_12K_ITR;
1360                         else
1361                                 q_vector->itr = adapter->tx_itr_setting;
1362                 } else {
1363                         /* Rx or Rx/Tx vector */
1364                         if (adapter->rx_itr_setting == 1)
1365                                 q_vector->itr = IXGBE_20K_ITR;
1366                         else
1367                                 q_vector->itr = adapter->rx_itr_setting;
1368                 }
1369
1370                 /* add q_vector eims value to global eims_enable_mask */
1371                 adapter->eims_enable_mask |= BIT(v_idx);
1372
1373                 ixgbevf_write_eitr(q_vector);
1374         }
1375
1376         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
1377         /* setup eims_other and add value to global eims_enable_mask */
1378         adapter->eims_other = BIT(v_idx);
1379         adapter->eims_enable_mask |= adapter->eims_other;
1380 }
1381
1382 enum latency_range {
1383         lowest_latency = 0,
1384         low_latency = 1,
1385         bulk_latency = 2,
1386         latency_invalid = 255
1387 };
1388
1389 /**
1390  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
1391  * @q_vector: structure containing interrupt and ring information
1392  * @ring_container: structure containing ring performance data
1393  *
1394  * Stores a new ITR value based on packets and byte
1395  * counts during the last interrupt.  The advantage of per interrupt
1396  * computation is faster updates and more accurate ITR for the current
1397  * traffic pattern.  Constants in this function were computed
1398  * based on theoretical maximum wire speed and thresholds were set based
1399  * on testing data as well as attempting to minimize response time
1400  * while increasing bulk throughput.
1401  **/
1402 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
1403                                struct ixgbevf_ring_container *ring_container)
1404 {
1405         int bytes = ring_container->total_bytes;
1406         int packets = ring_container->total_packets;
1407         u32 timepassed_us;
1408         u64 bytes_perint;
1409         u8 itr_setting = ring_container->itr;
1410
1411         if (packets == 0)
1412                 return;
1413
1414         /* simple throttle rate management
1415          *    0-20MB/s lowest (100000 ints/s)
1416          *   20-100MB/s low   (20000 ints/s)
1417          *  100-1249MB/s bulk (12000 ints/s)
1418          */
1419         /* what was last interrupt timeslice? */
1420         timepassed_us = q_vector->itr >> 2;
1421         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1422
1423         switch (itr_setting) {
1424         case lowest_latency:
1425                 if (bytes_perint > 10)
1426                         itr_setting = low_latency;
1427                 break;
1428         case low_latency:
1429                 if (bytes_perint > 20)
1430                         itr_setting = bulk_latency;
1431                 else if (bytes_perint <= 10)
1432                         itr_setting = lowest_latency;
1433                 break;
1434         case bulk_latency:
1435                 if (bytes_perint <= 20)
1436                         itr_setting = low_latency;
1437                 break;
1438         }
1439
1440         /* clear work counters since we have the values we need */
1441         ring_container->total_bytes = 0;
1442         ring_container->total_packets = 0;
1443
1444         /* write updated itr to ring container */
1445         ring_container->itr = itr_setting;
1446 }
1447
1448 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1449 {
1450         u32 new_itr = q_vector->itr;
1451         u8 current_itr;
1452
1453         ixgbevf_update_itr(q_vector, &q_vector->tx);
1454         ixgbevf_update_itr(q_vector, &q_vector->rx);
1455
1456         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1457
1458         switch (current_itr) {
1459         /* counts and packets in update_itr are dependent on these numbers */
1460         case lowest_latency:
1461                 new_itr = IXGBE_100K_ITR;
1462                 break;
1463         case low_latency:
1464                 new_itr = IXGBE_20K_ITR;
1465                 break;
1466         case bulk_latency:
1467                 new_itr = IXGBE_12K_ITR;
1468                 break;
1469         default:
1470                 break;
1471         }
1472
1473         if (new_itr != q_vector->itr) {
1474                 /* do an exponential smoothing */
1475                 new_itr = (10 * new_itr * q_vector->itr) /
1476                           ((9 * new_itr) + q_vector->itr);
1477
1478                 /* save the algorithm value here */
1479                 q_vector->itr = new_itr;
1480
1481                 ixgbevf_write_eitr(q_vector);
1482         }
1483 }
1484
1485 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1486 {
1487         struct ixgbevf_adapter *adapter = data;
1488         struct ixgbe_hw *hw = &adapter->hw;
1489
1490         hw->mac.get_link_status = 1;
1491
1492         ixgbevf_service_event_schedule(adapter);
1493
1494         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
1495
1496         return IRQ_HANDLED;
1497 }
1498
1499 /**
1500  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
1501  * @irq: unused
1502  * @data: pointer to our q_vector struct for this interrupt vector
1503  **/
1504 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
1505 {
1506         struct ixgbevf_q_vector *q_vector = data;
1507
1508         /* EIAM disabled interrupts (on this vector) for us */
1509         if (q_vector->rx.ring || q_vector->tx.ring)
1510                 napi_schedule_irqoff(&q_vector->napi);
1511
1512         return IRQ_HANDLED;
1513 }
1514
1515 /**
1516  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1517  * @adapter: board private structure
1518  *
1519  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1520  * interrupts from the kernel.
1521  **/
1522 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1523 {
1524         struct net_device *netdev = adapter->netdev;
1525         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1526         unsigned int ri = 0, ti = 0;
1527         int vector, err;
1528
1529         for (vector = 0; vector < q_vectors; vector++) {
1530                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1531                 struct msix_entry *entry = &adapter->msix_entries[vector];
1532
1533                 if (q_vector->tx.ring && q_vector->rx.ring) {
1534                         snprintf(q_vector->name, sizeof(q_vector->name),
1535                                  "%s-TxRx-%u", netdev->name, ri++);
1536                         ti++;
1537                 } else if (q_vector->rx.ring) {
1538                         snprintf(q_vector->name, sizeof(q_vector->name),
1539                                  "%s-rx-%u", netdev->name, ri++);
1540                 } else if (q_vector->tx.ring) {
1541                         snprintf(q_vector->name, sizeof(q_vector->name),
1542                                  "%s-tx-%u", netdev->name, ti++);
1543                 } else {
1544                         /* skip this unused q_vector */
1545                         continue;
1546                 }
1547                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1548                                   q_vector->name, q_vector);
1549                 if (err) {
1550                         hw_dbg(&adapter->hw,
1551                                "request_irq failed for MSIX interrupt Error: %d\n",
1552                                err);
1553                         goto free_queue_irqs;
1554                 }
1555         }
1556
1557         err = request_irq(adapter->msix_entries[vector].vector,
1558                           &ixgbevf_msix_other, 0, netdev->name, adapter);
1559         if (err) {
1560                 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n",
1561                        err);
1562                 goto free_queue_irqs;
1563         }
1564
1565         return 0;
1566
1567 free_queue_irqs:
1568         while (vector) {
1569                 vector--;
1570                 free_irq(adapter->msix_entries[vector].vector,
1571                          adapter->q_vector[vector]);
1572         }
1573         /* This failure is non-recoverable - it indicates the system is
1574          * out of MSIX vector resources and the VF driver cannot run
1575          * without them.  Set the number of msix vectors to zero
1576          * indicating that not enough can be allocated.  The error
1577          * will be returned to the user indicating device open failed.
1578          * Any further attempts to force the driver to open will also
1579          * fail.  The only way to recover is to unload the driver and
1580          * reload it again.  If the system has recovered some MSIX
1581          * vectors then it may succeed.
1582          */
1583         adapter->num_msix_vectors = 0;
1584         return err;
1585 }
1586
1587 /**
1588  * ixgbevf_request_irq - initialize interrupts
1589  * @adapter: board private structure
1590  *
1591  * Attempts to configure interrupts using the best available
1592  * capabilities of the hardware and kernel.
1593  **/
1594 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1595 {
1596         int err = ixgbevf_request_msix_irqs(adapter);
1597
1598         if (err)
1599                 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err);
1600
1601         return err;
1602 }
1603
1604 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1605 {
1606         int i, q_vectors;
1607
1608         if (!adapter->msix_entries)
1609                 return;
1610
1611         q_vectors = adapter->num_msix_vectors;
1612         i = q_vectors - 1;
1613
1614         free_irq(adapter->msix_entries[i].vector, adapter);
1615         i--;
1616
1617         for (; i >= 0; i--) {
1618                 /* free only the irqs that were actually requested */
1619                 if (!adapter->q_vector[i]->rx.ring &&
1620                     !adapter->q_vector[i]->tx.ring)
1621                         continue;
1622
1623                 free_irq(adapter->msix_entries[i].vector,
1624                          adapter->q_vector[i]);
1625         }
1626 }
1627
1628 /**
1629  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1630  * @adapter: board private structure
1631  **/
1632 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1633 {
1634         struct ixgbe_hw *hw = &adapter->hw;
1635         int i;
1636
1637         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1638         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1639         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1640
1641         IXGBE_WRITE_FLUSH(hw);
1642
1643         for (i = 0; i < adapter->num_msix_vectors; i++)
1644                 synchronize_irq(adapter->msix_entries[i].vector);
1645 }
1646
1647 /**
1648  * ixgbevf_irq_enable - Enable default interrupt generation settings
1649  * @adapter: board private structure
1650  **/
1651 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1652 {
1653         struct ixgbe_hw *hw = &adapter->hw;
1654
1655         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1656         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1657         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1658 }
1659
1660 /**
1661  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1662  * @adapter: board private structure
1663  * @ring: structure containing ring specific data
1664  *
1665  * Configure the Tx descriptor ring after a reset.
1666  **/
1667 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1668                                       struct ixgbevf_ring *ring)
1669 {
1670         struct ixgbe_hw *hw = &adapter->hw;
1671         u64 tdba = ring->dma;
1672         int wait_loop = 10;
1673         u32 txdctl = IXGBE_TXDCTL_ENABLE;
1674         u8 reg_idx = ring->reg_idx;
1675
1676         /* disable queue to avoid issues while updating state */
1677         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1678         IXGBE_WRITE_FLUSH(hw);
1679
1680         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1681         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1682         IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1683                         ring->count * sizeof(union ixgbe_adv_tx_desc));
1684
1685         /* disable head writeback */
1686         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1687         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1688
1689         /* enable relaxed ordering */
1690         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1691                         (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1692                          IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1693
1694         /* reset head and tail pointers */
1695         IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1696         IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1697         ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
1698
1699         /* reset ntu and ntc to place SW in sync with hardwdare */
1700         ring->next_to_clean = 0;
1701         ring->next_to_use = 0;
1702
1703         /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1704          * to or less than the number of on chip descriptors, which is
1705          * currently 40.
1706          */
1707         txdctl |= (8 << 16);    /* WTHRESH = 8 */
1708
1709         /* Setting PTHRESH to 32 both improves performance */
1710         txdctl |= (1u << 8) |    /* HTHRESH = 1 */
1711                    32;           /* PTHRESH = 32 */
1712
1713         /* reinitialize tx_buffer_info */
1714         memset(ring->tx_buffer_info, 0,
1715                sizeof(struct ixgbevf_tx_buffer) * ring->count);
1716
1717         clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state);
1718         clear_bit(__IXGBEVF_TX_XDP_RING_PRIMED, &ring->state);
1719
1720         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1721
1722         /* poll to verify queue is enabled */
1723         do {
1724                 usleep_range(1000, 2000);
1725                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1726         }  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1727         if (!wait_loop)
1728                 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);
1729 }
1730
1731 /**
1732  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1733  * @adapter: board private structure
1734  *
1735  * Configure the Tx unit of the MAC after a reset.
1736  **/
1737 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1738 {
1739         u32 i;
1740
1741         /* Setup the HW Tx Head and Tail descriptor pointers */
1742         for (i = 0; i < adapter->num_tx_queues; i++)
1743                 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
1744         for (i = 0; i < adapter->num_xdp_queues; i++)
1745                 ixgbevf_configure_tx_ring(adapter, adapter->xdp_ring[i]);
1746 }
1747
1748 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1749
1750 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter,
1751                                      struct ixgbevf_ring *ring, int index)
1752 {
1753         struct ixgbe_hw *hw = &adapter->hw;
1754         u32 srrctl;
1755
1756         srrctl = IXGBE_SRRCTL_DROP_EN;
1757
1758         srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
1759         if (ring_uses_large_buffer(ring))
1760                 srrctl |= IXGBEVF_RXBUFFER_3072 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1761         else
1762                 srrctl |= IXGBEVF_RXBUFFER_2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1763         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1764
1765         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1766 }
1767
1768 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1769 {
1770         struct ixgbe_hw *hw = &adapter->hw;
1771
1772         /* PSRTYPE must be initialized in 82599 */
1773         u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1774                       IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1775                       IXGBE_PSRTYPE_L2HDR;
1776
1777         if (adapter->num_rx_queues > 1)
1778                 psrtype |= BIT(29);
1779
1780         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1781 }
1782
1783 #define IXGBEVF_MAX_RX_DESC_POLL 10
1784 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1785                                      struct ixgbevf_ring *ring)
1786 {
1787         struct ixgbe_hw *hw = &adapter->hw;
1788         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1789         u32 rxdctl;
1790         u8 reg_idx = ring->reg_idx;
1791
1792         if (IXGBE_REMOVED(hw->hw_addr))
1793                 return;
1794         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1795         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1796
1797         /* write value back with RXDCTL.ENABLE bit cleared */
1798         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1799
1800         /* the hardware may take up to 100us to really disable the Rx queue */
1801         do {
1802                 udelay(10);
1803                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1804         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1805
1806         if (!wait_loop)
1807                 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1808                        reg_idx);
1809 }
1810
1811 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1812                                          struct ixgbevf_ring *ring)
1813 {
1814         struct ixgbe_hw *hw = &adapter->hw;
1815         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1816         u32 rxdctl;
1817         u8 reg_idx = ring->reg_idx;
1818
1819         if (IXGBE_REMOVED(hw->hw_addr))
1820                 return;
1821         do {
1822                 usleep_range(1000, 2000);
1823                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1824         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1825
1826         if (!wait_loop)
1827                 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1828                        reg_idx);
1829 }
1830
1831 /**
1832  * ixgbevf_init_rss_key - Initialize adapter RSS key
1833  * @adapter: device handle
1834  *
1835  * Allocates and initializes the RSS key if it is not allocated.
1836  **/
1837 static inline int ixgbevf_init_rss_key(struct ixgbevf_adapter *adapter)
1838 {
1839         u32 *rss_key;
1840
1841         if (!adapter->rss_key) {
1842                 rss_key = kzalloc(IXGBEVF_RSS_HASH_KEY_SIZE, GFP_KERNEL);
1843                 if (unlikely(!rss_key))
1844                         return -ENOMEM;
1845
1846                 netdev_rss_key_fill(rss_key, IXGBEVF_RSS_HASH_KEY_SIZE);
1847                 adapter->rss_key = rss_key;
1848         }
1849
1850         return 0;
1851 }
1852
1853 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter)
1854 {
1855         struct ixgbe_hw *hw = &adapter->hw;
1856         u32 vfmrqc = 0, vfreta = 0;
1857         u16 rss_i = adapter->num_rx_queues;
1858         u8 i, j;
1859
1860         /* Fill out hash function seeds */
1861         for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++)
1862                 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), *(adapter->rss_key + i));
1863
1864         for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) {
1865                 if (j == rss_i)
1866                         j = 0;
1867
1868                 adapter->rss_indir_tbl[i] = j;
1869
1870                 vfreta |= j << (i & 0x3) * 8;
1871                 if ((i & 3) == 3) {
1872                         IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta);
1873                         vfreta = 0;
1874                 }
1875         }
1876
1877         /* Perform hash on these packet types */
1878         vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 |
1879                 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP |
1880                 IXGBE_VFMRQC_RSS_FIELD_IPV6 |
1881                 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP;
1882
1883         vfmrqc |= IXGBE_VFMRQC_RSSEN;
1884
1885         IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc);
1886 }
1887
1888 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1889                                       struct ixgbevf_ring *ring)
1890 {
1891         struct ixgbe_hw *hw = &adapter->hw;
1892         union ixgbe_adv_rx_desc *rx_desc;
1893         u64 rdba = ring->dma;
1894         u32 rxdctl;
1895         u8 reg_idx = ring->reg_idx;
1896
1897         /* disable queue to avoid issues while updating state */
1898         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1899         ixgbevf_disable_rx_queue(adapter, ring);
1900
1901         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1902         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1903         IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1904                         ring->count * sizeof(union ixgbe_adv_rx_desc));
1905
1906 #ifndef CONFIG_SPARC
1907         /* enable relaxed ordering */
1908         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1909                         IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1910 #else
1911         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1912                         IXGBE_DCA_RXCTRL_DESC_RRO_EN |
1913                         IXGBE_DCA_RXCTRL_DATA_WRO_EN);
1914 #endif
1915
1916         /* reset head and tail pointers */
1917         IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1918         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1919         ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
1920
1921         /* initialize rx_buffer_info */
1922         memset(ring->rx_buffer_info, 0,
1923                sizeof(struct ixgbevf_rx_buffer) * ring->count);
1924
1925         /* initialize Rx descriptor 0 */
1926         rx_desc = IXGBEVF_RX_DESC(ring, 0);
1927         rx_desc->wb.upper.length = 0;
1928
1929         /* reset ntu and ntc to place SW in sync with hardwdare */
1930         ring->next_to_clean = 0;
1931         ring->next_to_use = 0;
1932         ring->next_to_alloc = 0;
1933
1934         ixgbevf_configure_srrctl(adapter, ring, reg_idx);
1935
1936         /* RXDCTL.RLPML does not work on 82599 */
1937         if (adapter->hw.mac.type != ixgbe_mac_82599_vf) {
1938                 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
1939                             IXGBE_RXDCTL_RLPML_EN);
1940
1941 #if (PAGE_SIZE < 8192)
1942                 /* Limit the maximum frame size so we don't overrun the skb */
1943                 if (ring_uses_build_skb(ring) &&
1944                     !ring_uses_large_buffer(ring))
1945                         rxdctl |= IXGBEVF_MAX_FRAME_BUILD_SKB |
1946                                   IXGBE_RXDCTL_RLPML_EN;
1947 #endif
1948         }
1949
1950         rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1951         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1952
1953         ixgbevf_rx_desc_queue_enable(adapter, ring);
1954         ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
1955 }
1956
1957 static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter,
1958                                       struct ixgbevf_ring *rx_ring)
1959 {
1960         struct net_device *netdev = adapter->netdev;
1961         unsigned int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1962
1963         /* set build_skb and buffer size flags */
1964         clear_ring_build_skb_enabled(rx_ring);
1965         clear_ring_uses_large_buffer(rx_ring);
1966
1967         if (adapter->flags & IXGBEVF_FLAGS_LEGACY_RX)
1968                 return;
1969
1970         set_ring_build_skb_enabled(rx_ring);
1971
1972         if (PAGE_SIZE < 8192) {
1973                 if (max_frame <= IXGBEVF_MAX_FRAME_BUILD_SKB)
1974                         return;
1975
1976                 set_ring_uses_large_buffer(rx_ring);
1977         }
1978 }
1979
1980 /**
1981  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1982  * @adapter: board private structure
1983  *
1984  * Configure the Rx unit of the MAC after a reset.
1985  **/
1986 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1987 {
1988         struct ixgbe_hw *hw = &adapter->hw;
1989         struct net_device *netdev = adapter->netdev;
1990         int i, ret;
1991
1992         ixgbevf_setup_psrtype(adapter);
1993         if (hw->mac.type >= ixgbe_mac_X550_vf)
1994                 ixgbevf_setup_vfmrqc(adapter);
1995
1996         spin_lock_bh(&adapter->mbx_lock);
1997         /* notify the PF of our intent to use this size of frame */
1998         ret = hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
1999         spin_unlock_bh(&adapter->mbx_lock);
2000         if (ret)
2001                 dev_err(&adapter->pdev->dev,
2002                         "Failed to set MTU at %d\n", netdev->mtu);
2003
2004         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2005          * the Base and Length of the Rx Descriptor Ring
2006          */
2007         for (i = 0; i < adapter->num_rx_queues; i++) {
2008                 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
2009
2010                 ixgbevf_set_rx_buffer_len(adapter, rx_ring);
2011                 ixgbevf_configure_rx_ring(adapter, rx_ring);
2012         }
2013 }
2014
2015 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
2016                                    __be16 proto, u16 vid)
2017 {
2018         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2019         struct ixgbe_hw *hw = &adapter->hw;
2020         int err;
2021
2022         spin_lock_bh(&adapter->mbx_lock);
2023
2024         /* add VID to filter table */
2025         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
2026
2027         spin_unlock_bh(&adapter->mbx_lock);
2028
2029         /* translate error return types so error makes sense */
2030         if (err == IXGBE_ERR_MBX)
2031                 return -EIO;
2032
2033         if (err == IXGBE_ERR_INVALID_ARGUMENT)
2034                 return -EACCES;
2035
2036         set_bit(vid, adapter->active_vlans);
2037
2038         return err;
2039 }
2040
2041 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
2042                                     __be16 proto, u16 vid)
2043 {
2044         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2045         struct ixgbe_hw *hw = &adapter->hw;
2046         int err;
2047
2048         spin_lock_bh(&adapter->mbx_lock);
2049
2050         /* remove VID from filter table */
2051         err = hw->mac.ops.set_vfta(hw, vid, 0, false);
2052
2053         spin_unlock_bh(&adapter->mbx_lock);
2054
2055         clear_bit(vid, adapter->active_vlans);
2056
2057         return err;
2058 }
2059
2060 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
2061 {
2062         u16 vid;
2063
2064         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2065                 ixgbevf_vlan_rx_add_vid(adapter->netdev,
2066                                         htons(ETH_P_8021Q), vid);
2067 }
2068
2069 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
2070 {
2071         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2072         struct ixgbe_hw *hw = &adapter->hw;
2073         int count = 0;
2074
2075         if ((netdev_uc_count(netdev)) > 10) {
2076                 pr_err("Too many unicast filters - No Space\n");
2077                 return -ENOSPC;
2078         }
2079
2080         if (!netdev_uc_empty(netdev)) {
2081                 struct netdev_hw_addr *ha;
2082
2083                 netdev_for_each_uc_addr(ha, netdev) {
2084                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
2085                         udelay(200);
2086                 }
2087         } else {
2088                 /* If the list is empty then send message to PF driver to
2089                  * clear all MAC VLANs on this VF.
2090                  */
2091                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
2092         }
2093
2094         return count;
2095 }
2096
2097 /**
2098  * ixgbevf_set_rx_mode - Multicast and unicast set
2099  * @netdev: network interface device structure
2100  *
2101  * The set_rx_method entry point is called whenever the multicast address
2102  * list, unicast address list or the network interface flags are updated.
2103  * This routine is responsible for configuring the hardware for proper
2104  * multicast mode and configuring requested unicast filters.
2105  **/
2106 static void ixgbevf_set_rx_mode(struct net_device *netdev)
2107 {
2108         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2109         struct ixgbe_hw *hw = &adapter->hw;
2110         unsigned int flags = netdev->flags;
2111         int xcast_mode;
2112
2113         /* request the most inclusive mode we need */
2114         if (flags & IFF_PROMISC)
2115                 xcast_mode = IXGBEVF_XCAST_MODE_PROMISC;
2116         else if (flags & IFF_ALLMULTI)
2117                 xcast_mode = IXGBEVF_XCAST_MODE_ALLMULTI;
2118         else if (flags & (IFF_BROADCAST | IFF_MULTICAST))
2119                 xcast_mode = IXGBEVF_XCAST_MODE_MULTI;
2120         else
2121                 xcast_mode = IXGBEVF_XCAST_MODE_NONE;
2122
2123         spin_lock_bh(&adapter->mbx_lock);
2124
2125         hw->mac.ops.update_xcast_mode(hw, xcast_mode);
2126
2127         /* reprogram multicast list */
2128         hw->mac.ops.update_mc_addr_list(hw, netdev);
2129
2130         ixgbevf_write_uc_addr_list(netdev);
2131
2132         spin_unlock_bh(&adapter->mbx_lock);
2133 }
2134
2135 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
2136 {
2137         int q_idx;
2138         struct ixgbevf_q_vector *q_vector;
2139         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2140
2141         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2142                 q_vector = adapter->q_vector[q_idx];
2143                 napi_enable(&q_vector->napi);
2144         }
2145 }
2146
2147 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
2148 {
2149         int q_idx;
2150         struct ixgbevf_q_vector *q_vector;
2151         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2152
2153         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
2154                 q_vector = adapter->q_vector[q_idx];
2155                 napi_disable(&q_vector->napi);
2156         }
2157 }
2158
2159 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
2160 {
2161         struct ixgbe_hw *hw = &adapter->hw;
2162         unsigned int def_q = 0;
2163         unsigned int num_tcs = 0;
2164         unsigned int num_rx_queues = adapter->num_rx_queues;
2165         unsigned int num_tx_queues = adapter->num_tx_queues;
2166         int err;
2167
2168         spin_lock_bh(&adapter->mbx_lock);
2169
2170         /* fetch queue configuration from the PF */
2171         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2172
2173         spin_unlock_bh(&adapter->mbx_lock);
2174
2175         if (err)
2176                 return err;
2177
2178         if (num_tcs > 1) {
2179                 /* we need only one Tx queue */
2180                 num_tx_queues = 1;
2181
2182                 /* update default Tx ring register index */
2183                 adapter->tx_ring[0]->reg_idx = def_q;
2184
2185                 /* we need as many queues as traffic classes */
2186                 num_rx_queues = num_tcs;
2187         }
2188
2189         /* if we have a bad config abort request queue reset */
2190         if ((adapter->num_rx_queues != num_rx_queues) ||
2191             (adapter->num_tx_queues != num_tx_queues)) {
2192                 /* force mailbox timeout to prevent further messages */
2193                 hw->mbx.timeout = 0;
2194
2195                 /* wait for watchdog to come around and bail us out */
2196                 set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state);
2197         }
2198
2199         return 0;
2200 }
2201
2202 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
2203 {
2204         ixgbevf_configure_dcb(adapter);
2205
2206         ixgbevf_set_rx_mode(adapter->netdev);
2207
2208         ixgbevf_restore_vlan(adapter);
2209         ixgbevf_ipsec_restore(adapter);
2210
2211         ixgbevf_configure_tx(adapter);
2212         ixgbevf_configure_rx(adapter);
2213 }
2214
2215 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
2216 {
2217         /* Only save pre-reset stats if there are some */
2218         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
2219                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
2220                         adapter->stats.base_vfgprc;
2221                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
2222                         adapter->stats.base_vfgptc;
2223                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
2224                         adapter->stats.base_vfgorc;
2225                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
2226                         adapter->stats.base_vfgotc;
2227                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
2228                         adapter->stats.base_vfmprc;
2229         }
2230 }
2231
2232 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
2233 {
2234         struct ixgbe_hw *hw = &adapter->hw;
2235
2236         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
2237         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
2238         adapter->stats.last_vfgorc |=
2239                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
2240         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
2241         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
2242         adapter->stats.last_vfgotc |=
2243                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
2244         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
2245
2246         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
2247         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
2248         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
2249         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
2250         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
2251 }
2252
2253 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
2254 {
2255         struct ixgbe_hw *hw = &adapter->hw;
2256         int api[] = { ixgbe_mbox_api_14,
2257                       ixgbe_mbox_api_13,
2258                       ixgbe_mbox_api_12,
2259                       ixgbe_mbox_api_11,
2260                       ixgbe_mbox_api_10,
2261                       ixgbe_mbox_api_unknown };
2262         int err, idx = 0;
2263
2264         spin_lock_bh(&adapter->mbx_lock);
2265
2266         while (api[idx] != ixgbe_mbox_api_unknown) {
2267                 err = hw->mac.ops.negotiate_api_version(hw, api[idx]);
2268                 if (!err)
2269                         break;
2270                 idx++;
2271         }
2272
2273         spin_unlock_bh(&adapter->mbx_lock);
2274 }
2275
2276 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
2277 {
2278         struct net_device *netdev = adapter->netdev;
2279         struct ixgbe_hw *hw = &adapter->hw;
2280
2281         ixgbevf_configure_msix(adapter);
2282
2283         spin_lock_bh(&adapter->mbx_lock);
2284
2285         if (is_valid_ether_addr(hw->mac.addr))
2286                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
2287         else
2288                 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
2289
2290         spin_unlock_bh(&adapter->mbx_lock);
2291
2292         smp_mb__before_atomic();
2293         clear_bit(__IXGBEVF_DOWN, &adapter->state);
2294         ixgbevf_napi_enable_all(adapter);
2295
2296         /* clear any pending interrupts, may auto mask */
2297         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2298         ixgbevf_irq_enable(adapter);
2299
2300         /* enable transmits */
2301         netif_tx_start_all_queues(netdev);
2302
2303         ixgbevf_save_reset_stats(adapter);
2304         ixgbevf_init_last_counter_stats(adapter);
2305
2306         hw->mac.get_link_status = 1;
2307         mod_timer(&adapter->service_timer, jiffies);
2308 }
2309
2310 void ixgbevf_up(struct ixgbevf_adapter *adapter)
2311 {
2312         ixgbevf_configure(adapter);
2313
2314         ixgbevf_up_complete(adapter);
2315 }
2316
2317 /**
2318  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
2319  * @rx_ring: ring to free buffers from
2320  **/
2321 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
2322 {
2323         u16 i = rx_ring->next_to_clean;
2324
2325         /* Free Rx ring sk_buff */
2326         if (rx_ring->skb) {
2327                 dev_kfree_skb(rx_ring->skb);
2328                 rx_ring->skb = NULL;
2329         }
2330
2331         /* Free all the Rx ring pages */
2332         while (i != rx_ring->next_to_alloc) {
2333                 struct ixgbevf_rx_buffer *rx_buffer;
2334
2335                 rx_buffer = &rx_ring->rx_buffer_info[i];
2336
2337                 /* Invalidate cache lines that may have been written to by
2338                  * device so that we avoid corrupting memory.
2339                  */
2340                 dma_sync_single_range_for_cpu(rx_ring->dev,
2341                                               rx_buffer->dma,
2342                                               rx_buffer->page_offset,
2343                                               ixgbevf_rx_bufsz(rx_ring),
2344                                               DMA_FROM_DEVICE);
2345
2346                 /* free resources associated with mapping */
2347                 dma_unmap_page_attrs(rx_ring->dev,
2348                                      rx_buffer->dma,
2349                                      ixgbevf_rx_pg_size(rx_ring),
2350                                      DMA_FROM_DEVICE,
2351                                      IXGBEVF_RX_DMA_ATTR);
2352
2353                 __page_frag_cache_drain(rx_buffer->page,
2354                                         rx_buffer->pagecnt_bias);
2355
2356                 i++;
2357                 if (i == rx_ring->count)
2358                         i = 0;
2359         }
2360
2361         rx_ring->next_to_alloc = 0;
2362         rx_ring->next_to_clean = 0;
2363         rx_ring->next_to_use = 0;
2364 }
2365
2366 /**
2367  * ixgbevf_clean_tx_ring - Free Tx Buffers
2368  * @tx_ring: ring to be cleaned
2369  **/
2370 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
2371 {
2372         u16 i = tx_ring->next_to_clean;
2373         struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
2374
2375         while (i != tx_ring->next_to_use) {
2376                 union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
2377
2378                 /* Free all the Tx ring sk_buffs */
2379                 if (ring_is_xdp(tx_ring))
2380                         page_frag_free(tx_buffer->data);
2381                 else
2382                         dev_kfree_skb_any(tx_buffer->skb);
2383
2384                 /* unmap skb header data */
2385                 dma_unmap_single(tx_ring->dev,
2386                                  dma_unmap_addr(tx_buffer, dma),
2387                                  dma_unmap_len(tx_buffer, len),
2388                                  DMA_TO_DEVICE);
2389
2390                 /* check for eop_desc to determine the end of the packet */
2391                 eop_desc = tx_buffer->next_to_watch;
2392                 tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
2393
2394                 /* unmap remaining buffers */
2395                 while (tx_desc != eop_desc) {
2396                         tx_buffer++;
2397                         tx_desc++;
2398                         i++;
2399                         if (unlikely(i == tx_ring->count)) {
2400                                 i = 0;
2401                                 tx_buffer = tx_ring->tx_buffer_info;
2402                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
2403                         }
2404
2405                         /* unmap any remaining paged data */
2406                         if (dma_unmap_len(tx_buffer, len))
2407                                 dma_unmap_page(tx_ring->dev,
2408                                                dma_unmap_addr(tx_buffer, dma),
2409                                                dma_unmap_len(tx_buffer, len),
2410                                                DMA_TO_DEVICE);
2411                 }
2412
2413                 /* move us one more past the eop_desc for start of next pkt */
2414                 tx_buffer++;
2415                 i++;
2416                 if (unlikely(i == tx_ring->count)) {
2417                         i = 0;
2418                         tx_buffer = tx_ring->tx_buffer_info;
2419                 }
2420         }
2421
2422         /* reset next_to_use and next_to_clean */
2423         tx_ring->next_to_use = 0;
2424         tx_ring->next_to_clean = 0;
2425
2426 }
2427
2428 /**
2429  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
2430  * @adapter: board private structure
2431  **/
2432 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
2433 {
2434         int i;
2435
2436         for (i = 0; i < adapter->num_rx_queues; i++)
2437                 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
2438 }
2439
2440 /**
2441  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
2442  * @adapter: board private structure
2443  **/
2444 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
2445 {
2446         int i;
2447
2448         for (i = 0; i < adapter->num_tx_queues; i++)
2449                 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
2450         for (i = 0; i < adapter->num_xdp_queues; i++)
2451                 ixgbevf_clean_tx_ring(adapter->xdp_ring[i]);
2452 }
2453
2454 void ixgbevf_down(struct ixgbevf_adapter *adapter)
2455 {
2456         struct net_device *netdev = adapter->netdev;
2457         struct ixgbe_hw *hw = &adapter->hw;
2458         int i;
2459
2460         /* signal that we are down to the interrupt handler */
2461         if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
2462                 return; /* do nothing if already down */
2463
2464         /* disable all enabled Rx queues */
2465         for (i = 0; i < adapter->num_rx_queues; i++)
2466                 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
2467
2468         usleep_range(10000, 20000);
2469
2470         netif_tx_stop_all_queues(netdev);
2471
2472         /* call carrier off first to avoid false dev_watchdog timeouts */
2473         netif_carrier_off(netdev);
2474         netif_tx_disable(netdev);
2475
2476         ixgbevf_irq_disable(adapter);
2477
2478         ixgbevf_napi_disable_all(adapter);
2479
2480         del_timer_sync(&adapter->service_timer);
2481
2482         /* disable transmits in the hardware now that interrupts are off */
2483         for (i = 0; i < adapter->num_tx_queues; i++) {
2484                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
2485
2486                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2487                                 IXGBE_TXDCTL_SWFLSH);
2488         }
2489
2490         for (i = 0; i < adapter->num_xdp_queues; i++) {
2491                 u8 reg_idx = adapter->xdp_ring[i]->reg_idx;
2492
2493                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2494                                 IXGBE_TXDCTL_SWFLSH);
2495         }
2496
2497         if (!pci_channel_offline(adapter->pdev))
2498                 ixgbevf_reset(adapter);
2499
2500         ixgbevf_clean_all_tx_rings(adapter);
2501         ixgbevf_clean_all_rx_rings(adapter);
2502 }
2503
2504 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
2505 {
2506         WARN_ON(in_interrupt());
2507
2508         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
2509                 msleep(1);
2510
2511         ixgbevf_down(adapter);
2512         ixgbevf_up(adapter);
2513
2514         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
2515 }
2516
2517 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
2518 {
2519         struct ixgbe_hw *hw = &adapter->hw;
2520         struct net_device *netdev = adapter->netdev;
2521
2522         if (hw->mac.ops.reset_hw(hw)) {
2523                 hw_dbg(hw, "PF still resetting\n");
2524         } else {
2525                 hw->mac.ops.init_hw(hw);
2526                 ixgbevf_negotiate_api(adapter);
2527         }
2528
2529         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
2530                 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr);
2531                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2532         }
2533
2534         adapter->last_reset = jiffies;
2535 }
2536
2537 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
2538                                         int vectors)
2539 {
2540         int vector_threshold;
2541
2542         /* We'll want at least 2 (vector_threshold):
2543          * 1) TxQ[0] + RxQ[0] handler
2544          * 2) Other (Link Status Change, etc.)
2545          */
2546         vector_threshold = MIN_MSIX_COUNT;
2547
2548         /* The more we get, the more we will assign to Tx/Rx Cleanup
2549          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
2550          * Right now, we simply care about how many we'll get; we'll
2551          * set them up later while requesting irq's.
2552          */
2553         vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2554                                         vector_threshold, vectors);
2555
2556         if (vectors < 0) {
2557                 dev_err(&adapter->pdev->dev,
2558                         "Unable to allocate MSI-X interrupts\n");
2559                 kfree(adapter->msix_entries);
2560                 adapter->msix_entries = NULL;
2561                 return vectors;
2562         }
2563
2564         /* Adjust for only the vectors we'll use, which is minimum
2565          * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
2566          * vectors we were allocated.
2567          */
2568         adapter->num_msix_vectors = vectors;
2569
2570         return 0;
2571 }
2572
2573 /**
2574  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
2575  * @adapter: board private structure to initialize
2576  *
2577  * This is the top level queue allocation routine.  The order here is very
2578  * important, starting with the "most" number of features turned on at once,
2579  * and ending with the smallest set of features.  This way large combinations
2580  * can be allocated if they're turned on, and smaller combinations are the
2581  * fallthrough conditions.
2582  *
2583  **/
2584 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
2585 {
2586         struct ixgbe_hw *hw = &adapter->hw;
2587         unsigned int def_q = 0;
2588         unsigned int num_tcs = 0;
2589         int err;
2590
2591         /* Start with base case */
2592         adapter->num_rx_queues = 1;
2593         adapter->num_tx_queues = 1;
2594         adapter->num_xdp_queues = 0;
2595
2596         spin_lock_bh(&adapter->mbx_lock);
2597
2598         /* fetch queue configuration from the PF */
2599         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2600
2601         spin_unlock_bh(&adapter->mbx_lock);
2602
2603         if (err)
2604                 return;
2605
2606         /* we need as many queues as traffic classes */
2607         if (num_tcs > 1) {
2608                 adapter->num_rx_queues = num_tcs;
2609         } else {
2610                 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES);
2611
2612                 switch (hw->api_version) {
2613                 case ixgbe_mbox_api_11:
2614                 case ixgbe_mbox_api_12:
2615                 case ixgbe_mbox_api_13:
2616                 case ixgbe_mbox_api_14:
2617                         if (adapter->xdp_prog &&
2618                             hw->mac.max_tx_queues == rss)
2619                                 rss = rss > 3 ? 2 : 1;
2620
2621                         adapter->num_rx_queues = rss;
2622                         adapter->num_tx_queues = rss;
2623                         adapter->num_xdp_queues = adapter->xdp_prog ? rss : 0;
2624                 default:
2625                         break;
2626                 }
2627         }
2628 }
2629
2630 /**
2631  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2632  * @adapter: board private structure to initialize
2633  *
2634  * Attempt to configure the interrupts using the best available
2635  * capabilities of the hardware and the kernel.
2636  **/
2637 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2638 {
2639         int vector, v_budget;
2640
2641         /* It's easy to be greedy for MSI-X vectors, but it really
2642          * doesn't do us much good if we have a lot more vectors
2643          * than CPU's.  So let's be conservative and only ask for
2644          * (roughly) the same number of vectors as there are CPU's.
2645          * The default is to use pairs of vectors.
2646          */
2647         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2648         v_budget = min_t(int, v_budget, num_online_cpus());
2649         v_budget += NON_Q_VECTORS;
2650
2651         adapter->msix_entries = kcalloc(v_budget,
2652                                         sizeof(struct msix_entry), GFP_KERNEL);
2653         if (!adapter->msix_entries)
2654                 return -ENOMEM;
2655
2656         for (vector = 0; vector < v_budget; vector++)
2657                 adapter->msix_entries[vector].entry = vector;
2658
2659         /* A failure in MSI-X entry allocation isn't fatal, but the VF driver
2660          * does not support any other modes, so we will simply fail here. Note
2661          * that we clean up the msix_entries pointer else-where.
2662          */
2663         return ixgbevf_acquire_msix_vectors(adapter, v_budget);
2664 }
2665
2666 static void ixgbevf_add_ring(struct ixgbevf_ring *ring,
2667                              struct ixgbevf_ring_container *head)
2668 {
2669         ring->next = head->ring;
2670         head->ring = ring;
2671         head->count++;
2672 }
2673
2674 /**
2675  * ixgbevf_alloc_q_vector - Allocate memory for a single interrupt vector
2676  * @adapter: board private structure to initialize
2677  * @v_idx: index of vector in adapter struct
2678  * @txr_count: number of Tx rings for q vector
2679  * @txr_idx: index of first Tx ring to assign
2680  * @xdp_count: total number of XDP rings to allocate
2681  * @xdp_idx: index of first XDP ring to allocate
2682  * @rxr_count: number of Rx rings for q vector
2683  * @rxr_idx: index of first Rx ring to assign
2684  *
2685  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
2686  **/
2687 static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx,
2688                                   int txr_count, int txr_idx,
2689                                   int xdp_count, int xdp_idx,
2690                                   int rxr_count, int rxr_idx)
2691 {
2692         struct ixgbevf_q_vector *q_vector;
2693         int reg_idx = txr_idx + xdp_idx;
2694         struct ixgbevf_ring *ring;
2695         int ring_count, size;
2696
2697         ring_count = txr_count + xdp_count + rxr_count;
2698         size = sizeof(*q_vector) + (sizeof(*ring) * ring_count);
2699
2700         /* allocate q_vector and rings */
2701         q_vector = kzalloc(size, GFP_KERNEL);
2702         if (!q_vector)
2703                 return -ENOMEM;
2704
2705         /* initialize NAPI */
2706         netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64);
2707
2708         /* tie q_vector and adapter together */
2709         adapter->q_vector[v_idx] = q_vector;
2710         q_vector->adapter = adapter;
2711         q_vector->v_idx = v_idx;
2712
2713         /* initialize pointer to rings */
2714         ring = q_vector->ring;
2715
2716         while (txr_count) {
2717                 /* assign generic ring traits */
2718                 ring->dev = &adapter->pdev->dev;
2719                 ring->netdev = adapter->netdev;
2720
2721                 /* configure backlink on ring */
2722                 ring->q_vector = q_vector;
2723
2724                 /* update q_vector Tx values */
2725                 ixgbevf_add_ring(ring, &q_vector->tx);
2726
2727                 /* apply Tx specific ring traits */
2728                 ring->count = adapter->tx_ring_count;
2729                 ring->queue_index = txr_idx;
2730                 ring->reg_idx = reg_idx;
2731
2732                 /* assign ring to adapter */
2733                  adapter->tx_ring[txr_idx] = ring;
2734
2735                 /* update count and index */
2736                 txr_count--;
2737                 txr_idx++;
2738                 reg_idx++;
2739
2740                 /* push pointer to next ring */
2741                 ring++;
2742         }
2743
2744         while (xdp_count) {
2745                 /* assign generic ring traits */
2746                 ring->dev = &adapter->pdev->dev;
2747                 ring->netdev = adapter->netdev;
2748
2749                 /* configure backlink on ring */
2750                 ring->q_vector = q_vector;
2751
2752                 /* update q_vector Tx values */
2753                 ixgbevf_add_ring(ring, &q_vector->tx);
2754
2755                 /* apply Tx specific ring traits */
2756                 ring->count = adapter->tx_ring_count;
2757                 ring->queue_index = xdp_idx;
2758                 ring->reg_idx = reg_idx;
2759                 set_ring_xdp(ring);
2760
2761                 /* assign ring to adapter */
2762                 adapter->xdp_ring[xdp_idx] = ring;
2763
2764                 /* update count and index */
2765                 xdp_count--;
2766                 xdp_idx++;
2767                 reg_idx++;
2768
2769                 /* push pointer to next ring */
2770                 ring++;
2771         }
2772
2773         while (rxr_count) {
2774                 /* assign generic ring traits */
2775                 ring->dev = &adapter->pdev->dev;
2776                 ring->netdev = adapter->netdev;
2777
2778                 /* configure backlink on ring */
2779                 ring->q_vector = q_vector;
2780
2781                 /* update q_vector Rx values */
2782                 ixgbevf_add_ring(ring, &q_vector->rx);
2783
2784                 /* apply Rx specific ring traits */
2785                 ring->count = adapter->rx_ring_count;
2786                 ring->queue_index = rxr_idx;
2787                 ring->reg_idx = rxr_idx;
2788
2789                 /* assign ring to adapter */
2790                 adapter->rx_ring[rxr_idx] = ring;
2791
2792                 /* update count and index */
2793                 rxr_count--;
2794                 rxr_idx++;
2795
2796                 /* push pointer to next ring */
2797                 ring++;
2798         }
2799
2800         return 0;
2801 }
2802
2803 /**
2804  * ixgbevf_free_q_vector - Free memory allocated for specific interrupt vector
2805  * @adapter: board private structure to initialize
2806  * @v_idx: index of vector in adapter struct
2807  *
2808  * This function frees the memory allocated to the q_vector.  In addition if
2809  * NAPI is enabled it will delete any references to the NAPI struct prior
2810  * to freeing the q_vector.
2811  **/
2812 static void ixgbevf_free_q_vector(struct ixgbevf_adapter *adapter, int v_idx)
2813 {
2814         struct ixgbevf_q_vector *q_vector = adapter->q_vector[v_idx];
2815         struct ixgbevf_ring *ring;
2816
2817         ixgbevf_for_each_ring(ring, q_vector->tx) {
2818                 if (ring_is_xdp(ring))
2819                         adapter->xdp_ring[ring->queue_index] = NULL;
2820                 else
2821                         adapter->tx_ring[ring->queue_index] = NULL;
2822         }
2823
2824         ixgbevf_for_each_ring(ring, q_vector->rx)
2825                 adapter->rx_ring[ring->queue_index] = NULL;
2826
2827         adapter->q_vector[v_idx] = NULL;
2828         netif_napi_del(&q_vector->napi);
2829
2830         /* ixgbevf_get_stats() might access the rings on this vector,
2831          * we must wait a grace period before freeing it.
2832          */
2833         kfree_rcu(q_vector, rcu);
2834 }
2835
2836 /**
2837  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2838  * @adapter: board private structure to initialize
2839  *
2840  * We allocate one q_vector per queue interrupt.  If allocation fails we
2841  * return -ENOMEM.
2842  **/
2843 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2844 {
2845         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2846         int rxr_remaining = adapter->num_rx_queues;
2847         int txr_remaining = adapter->num_tx_queues;
2848         int xdp_remaining = adapter->num_xdp_queues;
2849         int rxr_idx = 0, txr_idx = 0, xdp_idx = 0, v_idx = 0;
2850         int err;
2851
2852         if (q_vectors >= (rxr_remaining + txr_remaining + xdp_remaining)) {
2853                 for (; rxr_remaining; v_idx++, q_vectors--) {
2854                         int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2855
2856                         err = ixgbevf_alloc_q_vector(adapter, v_idx,
2857                                                      0, 0, 0, 0, rqpv, rxr_idx);
2858                         if (err)
2859                                 goto err_out;
2860
2861                         /* update counts and index */
2862                         rxr_remaining -= rqpv;
2863                         rxr_idx += rqpv;
2864                 }
2865         }
2866
2867         for (; q_vectors; v_idx++, q_vectors--) {
2868                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors);
2869                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors);
2870                 int xqpv = DIV_ROUND_UP(xdp_remaining, q_vectors);
2871
2872                 err = ixgbevf_alloc_q_vector(adapter, v_idx,
2873                                              tqpv, txr_idx,
2874                                              xqpv, xdp_idx,
2875                                              rqpv, rxr_idx);
2876
2877                 if (err)
2878                         goto err_out;
2879
2880                 /* update counts and index */
2881                 rxr_remaining -= rqpv;
2882                 rxr_idx += rqpv;
2883                 txr_remaining -= tqpv;
2884                 txr_idx += tqpv;
2885                 xdp_remaining -= xqpv;
2886                 xdp_idx += xqpv;
2887         }
2888
2889         return 0;
2890
2891 err_out:
2892         while (v_idx) {
2893                 v_idx--;
2894                 ixgbevf_free_q_vector(adapter, v_idx);
2895         }
2896
2897         return -ENOMEM;
2898 }
2899
2900 /**
2901  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2902  * @adapter: board private structure to initialize
2903  *
2904  * This function frees the memory allocated to the q_vectors.  In addition if
2905  * NAPI is enabled it will delete any references to the NAPI struct prior
2906  * to freeing the q_vector.
2907  **/
2908 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2909 {
2910         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2911
2912         while (q_vectors) {
2913                 q_vectors--;
2914                 ixgbevf_free_q_vector(adapter, q_vectors);
2915         }
2916 }
2917
2918 /**
2919  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2920  * @adapter: board private structure
2921  *
2922  **/
2923 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2924 {
2925         if (!adapter->msix_entries)
2926                 return;
2927
2928         pci_disable_msix(adapter->pdev);
2929         kfree(adapter->msix_entries);
2930         adapter->msix_entries = NULL;
2931 }
2932
2933 /**
2934  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2935  * @adapter: board private structure to initialize
2936  *
2937  **/
2938 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2939 {
2940         int err;
2941
2942         /* Number of supported queues */
2943         ixgbevf_set_num_queues(adapter);
2944
2945         err = ixgbevf_set_interrupt_capability(adapter);
2946         if (err) {
2947                 hw_dbg(&adapter->hw,
2948                        "Unable to setup interrupt capabilities\n");
2949                 goto err_set_interrupt;
2950         }
2951
2952         err = ixgbevf_alloc_q_vectors(adapter);
2953         if (err) {
2954                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n");
2955                 goto err_alloc_q_vectors;
2956         }
2957
2958         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u XDP Queue count %u\n",
2959                (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
2960                adapter->num_rx_queues, adapter->num_tx_queues,
2961                adapter->num_xdp_queues);
2962
2963         set_bit(__IXGBEVF_DOWN, &adapter->state);
2964
2965         return 0;
2966 err_alloc_q_vectors:
2967         ixgbevf_reset_interrupt_capability(adapter);
2968 err_set_interrupt:
2969         return err;
2970 }
2971
2972 /**
2973  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2974  * @adapter: board private structure to clear interrupt scheme on
2975  *
2976  * We go through and clear interrupt specific resources and reset the structure
2977  * to pre-load conditions
2978  **/
2979 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2980 {
2981         adapter->num_tx_queues = 0;
2982         adapter->num_xdp_queues = 0;
2983         adapter->num_rx_queues = 0;
2984
2985         ixgbevf_free_q_vectors(adapter);
2986         ixgbevf_reset_interrupt_capability(adapter);
2987 }
2988
2989 /**
2990  * ixgbevf_sw_init - Initialize general software structures
2991  * @adapter: board private structure to initialize
2992  *
2993  * ixgbevf_sw_init initializes the Adapter private data structure.
2994  * Fields are initialized based on PCI device information and
2995  * OS network device settings (MTU size).
2996  **/
2997 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2998 {
2999         struct ixgbe_hw *hw = &adapter->hw;
3000         struct pci_dev *pdev = adapter->pdev;
3001         struct net_device *netdev = adapter->netdev;
3002         int err;
3003
3004         /* PCI config space info */
3005         hw->vendor_id = pdev->vendor;
3006         hw->device_id = pdev->device;
3007         hw->revision_id = pdev->revision;
3008         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3009         hw->subsystem_device_id = pdev->subsystem_device;
3010
3011         hw->mbx.ops.init_params(hw);
3012
3013         if (hw->mac.type >= ixgbe_mac_X550_vf) {
3014                 err = ixgbevf_init_rss_key(adapter);
3015                 if (err)
3016                         goto out;
3017         }
3018
3019         /* assume legacy case in which PF would only give VF 2 queues */
3020         hw->mac.max_tx_queues = 2;
3021         hw->mac.max_rx_queues = 2;
3022
3023         /* lock to protect mailbox accesses */
3024         spin_lock_init(&adapter->mbx_lock);
3025
3026         err = hw->mac.ops.reset_hw(hw);
3027         if (err) {
3028                 dev_info(&pdev->dev,
3029                          "PF still in reset state.  Is the PF interface up?\n");
3030         } else {
3031                 err = hw->mac.ops.init_hw(hw);
3032                 if (err) {
3033                         pr_err("init_shared_code failed: %d\n", err);
3034                         goto out;
3035                 }
3036                 ixgbevf_negotiate_api(adapter);
3037                 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
3038                 if (err)
3039                         dev_info(&pdev->dev, "Error reading MAC address\n");
3040                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
3041                         dev_info(&pdev->dev,
3042                                  "MAC address not assigned by administrator.\n");
3043                 ether_addr_copy(netdev->dev_addr, hw->mac.addr);
3044         }
3045
3046         if (!is_valid_ether_addr(netdev->dev_addr)) {
3047                 dev_info(&pdev->dev, "Assigning random MAC address\n");
3048                 eth_hw_addr_random(netdev);
3049                 ether_addr_copy(hw->mac.addr, netdev->dev_addr);
3050                 ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr);
3051         }
3052
3053         /* Enable dynamic interrupt throttling rates */
3054         adapter->rx_itr_setting = 1;
3055         adapter->tx_itr_setting = 1;
3056
3057         /* set default ring sizes */
3058         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
3059         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
3060
3061         set_bit(__IXGBEVF_DOWN, &adapter->state);
3062         return 0;
3063
3064 out:
3065         return err;
3066 }
3067
3068 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
3069         {                                                       \
3070                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
3071                 if (current_counter < last_counter)             \
3072                         counter += 0x100000000LL;               \
3073                 last_counter = current_counter;                 \
3074                 counter &= 0xFFFFFFFF00000000LL;                \
3075                 counter |= current_counter;                     \
3076         }
3077
3078 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
3079         {                                                                \
3080                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
3081                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
3082                 u64 current_counter = (current_counter_msb << 32) |      \
3083                         current_counter_lsb;                             \
3084                 if (current_counter < last_counter)                      \
3085                         counter += 0x1000000000LL;                       \
3086                 last_counter = current_counter;                          \
3087                 counter &= 0xFFFFFFF000000000LL;                         \
3088                 counter |= current_counter;                              \
3089         }
3090 /**
3091  * ixgbevf_update_stats - Update the board statistics counters.
3092  * @adapter: board private structure
3093  **/
3094 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
3095 {
3096         struct ixgbe_hw *hw = &adapter->hw;
3097         u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
3098         u64 alloc_rx_page = 0, hw_csum_rx_error = 0;
3099         int i;
3100
3101         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3102             test_bit(__IXGBEVF_RESETTING, &adapter->state))
3103                 return;
3104
3105         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
3106                                 adapter->stats.vfgprc);
3107         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
3108                                 adapter->stats.vfgptc);
3109         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3110                                 adapter->stats.last_vfgorc,
3111                                 adapter->stats.vfgorc);
3112         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3113                                 adapter->stats.last_vfgotc,
3114                                 adapter->stats.vfgotc);
3115         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
3116                                 adapter->stats.vfmprc);
3117
3118         for (i = 0;  i  < adapter->num_rx_queues;  i++) {
3119                 struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
3120
3121                 hw_csum_rx_error += rx_ring->rx_stats.csum_err;
3122                 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
3123                 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
3124                 alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
3125         }
3126
3127         adapter->hw_csum_rx_error = hw_csum_rx_error;
3128         adapter->alloc_rx_page_failed = alloc_rx_page_failed;
3129         adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
3130         adapter->alloc_rx_page = alloc_rx_page;
3131 }
3132
3133 /**
3134  * ixgbevf_service_timer - Timer Call-back
3135  * @t: pointer to timer_list struct
3136  **/
3137 static void ixgbevf_service_timer(struct timer_list *t)
3138 {
3139         struct ixgbevf_adapter *adapter = from_timer(adapter, t,
3140                                                      service_timer);
3141
3142         /* Reset the timer */
3143         mod_timer(&adapter->service_timer, (HZ * 2) + jiffies);
3144
3145         ixgbevf_service_event_schedule(adapter);
3146 }
3147
3148 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter)
3149 {
3150         if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state))
3151                 return;
3152
3153         rtnl_lock();
3154         /* If we're already down or resetting, just bail */