ixgbe: fix MAC anti-spoofing filter after VFLR
[muen/linux.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright (c) 2016-2017 Hisilicon Limited.
3
4 #include <linux/dma-mapping.h>
5 #include <linux/etherdevice.h>
6 #include <linux/interrupt.h>
7 #include <linux/if_vlan.h>
8 #include <linux/ip.h>
9 #include <linux/ipv6.h>
10 #include <linux/module.h>
11 #include <linux/pci.h>
12 #include <linux/aer.h>
13 #include <linux/skbuff.h>
14 #include <linux/sctp.h>
15 #include <linux/vermagic.h>
16 #include <net/gre.h>
17 #include <net/pkt_cls.h>
18 #include <net/vxlan.h>
19
20 #include "hnae3.h"
21 #include "hns3_enet.h"
22
23 static void hns3_clear_all_ring(struct hnae3_handle *h);
24 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
25 static void hns3_remove_hw_addr(struct net_device *netdev);
26
27 static const char hns3_driver_name[] = "hns3";
28 const char hns3_driver_version[] = VERMAGIC_STRING;
29 static const char hns3_driver_string[] =
30                         "Hisilicon Ethernet Network Driver for Hip08 Family";
31 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
32 static struct hnae3_client client;
33
34 /* hns3_pci_tbl - PCI Device ID Table
35  *
36  * Last entry must be all 0s
37  *
38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
39  *   Class, Class Mask, private data (not used) }
40  */
41 static const struct pci_device_id hns3_pci_tbl[] = {
42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
45          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
47          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
49          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
50         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
51          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
52         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
53          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
54         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
55         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
56          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
57         /* required last entry */
58         {0, }
59 };
60 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62 static irqreturn_t hns3_irq_handle(int irq, void *vector)
63 {
64         struct hns3_enet_tqp_vector *tqp_vector = vector;
65
66         napi_schedule(&tqp_vector->napi);
67
68         return IRQ_HANDLED;
69 }
70
71 /* This callback function is used to set affinity changes to the irq affinity
72  * masks when the irq_set_affinity_notifier function is used.
73  */
74 static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
75                                          const cpumask_t *mask)
76 {
77         struct hns3_enet_tqp_vector *tqp_vectors =
78                 container_of(notify, struct hns3_enet_tqp_vector,
79                              affinity_notify);
80
81         tqp_vectors->affinity_mask = *mask;
82 }
83
84 static void hns3_nic_irq_affinity_release(struct kref *ref)
85 {
86 }
87
88 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
89 {
90         struct hns3_enet_tqp_vector *tqp_vectors;
91         unsigned int i;
92
93         for (i = 0; i < priv->vector_num; i++) {
94                 tqp_vectors = &priv->tqp_vector[i];
95
96                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
97                         continue;
98
99                 /* clear the affinity notifier and affinity mask */
100                 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
101                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
102
103                 /* release the irq resource */
104                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
105                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
106         }
107 }
108
109 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
110 {
111         struct hns3_enet_tqp_vector *tqp_vectors;
112         int txrx_int_idx = 0;
113         int rx_int_idx = 0;
114         int tx_int_idx = 0;
115         unsigned int i;
116         int ret;
117
118         for (i = 0; i < priv->vector_num; i++) {
119                 tqp_vectors = &priv->tqp_vector[i];
120
121                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
122                         continue;
123
124                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
125                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
126                                  "%s-%s-%d", priv->netdev->name, "TxRx",
127                                  txrx_int_idx++);
128                         txrx_int_idx++;
129                 } else if (tqp_vectors->rx_group.ring) {
130                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
131                                  "%s-%s-%d", priv->netdev->name, "Rx",
132                                  rx_int_idx++);
133                 } else if (tqp_vectors->tx_group.ring) {
134                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
135                                  "%s-%s-%d", priv->netdev->name, "Tx",
136                                  tx_int_idx++);
137                 } else {
138                         /* Skip this unused q_vector */
139                         continue;
140                 }
141
142                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
143
144                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
145                                   tqp_vectors->name,
146                                        tqp_vectors);
147                 if (ret) {
148                         netdev_err(priv->netdev, "request irq(%d) fail\n",
149                                    tqp_vectors->vector_irq);
150                         return ret;
151                 }
152
153                 tqp_vectors->affinity_notify.notify =
154                                         hns3_nic_irq_affinity_notify;
155                 tqp_vectors->affinity_notify.release =
156                                         hns3_nic_irq_affinity_release;
157                 irq_set_affinity_notifier(tqp_vectors->vector_irq,
158                                           &tqp_vectors->affinity_notify);
159                 irq_set_affinity_hint(tqp_vectors->vector_irq,
160                                       &tqp_vectors->affinity_mask);
161
162                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
163         }
164
165         return 0;
166 }
167
168 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
169                                  u32 mask_en)
170 {
171         writel(mask_en, tqp_vector->mask_addr);
172 }
173
174 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
175 {
176         napi_enable(&tqp_vector->napi);
177
178         /* enable vector */
179         hns3_mask_vector_irq(tqp_vector, 1);
180 }
181
182 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
183 {
184         /* disable vector */
185         hns3_mask_vector_irq(tqp_vector, 0);
186
187         disable_irq(tqp_vector->vector_irq);
188         napi_disable(&tqp_vector->napi);
189 }
190
191 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
192                                  u32 rl_value)
193 {
194         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
195
196         /* this defines the configuration for RL (Interrupt Rate Limiter).
197          * Rl defines rate of interrupts i.e. number of interrupts-per-second
198          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
199          */
200
201         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
202             !tqp_vector->rx_group.coal.gl_adapt_enable)
203                 /* According to the hardware, the range of rl_reg is
204                  * 0-59 and the unit is 4.
205                  */
206                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
207
208         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
209 }
210
211 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
212                                     u32 gl_value)
213 {
214         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
215
216         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
217 }
218
219 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
220                                     u32 gl_value)
221 {
222         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
223
224         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
225 }
226
227 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
228                                    struct hns3_nic_priv *priv)
229 {
230         /* initialize the configuration for interrupt coalescing.
231          * 1. GL (Interrupt Gap Limiter)
232          * 2. RL (Interrupt Rate Limiter)
233          */
234
235         /* Default: enable interrupt coalescing self-adaptive and GL */
236         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
237         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
238
239         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
240         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
241
242         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
243         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
244         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
245 }
246
247 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
248                                       struct hns3_nic_priv *priv)
249 {
250         struct hnae3_handle *h = priv->ae_handle;
251
252         hns3_set_vector_coalesce_tx_gl(tqp_vector,
253                                        tqp_vector->tx_group.coal.int_gl);
254         hns3_set_vector_coalesce_rx_gl(tqp_vector,
255                                        tqp_vector->rx_group.coal.int_gl);
256         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
257 }
258
259 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
260 {
261         struct hnae3_handle *h = hns3_get_handle(netdev);
262         struct hnae3_knic_private_info *kinfo = &h->kinfo;
263         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
264         int i, ret;
265
266         if (kinfo->num_tc <= 1) {
267                 netdev_reset_tc(netdev);
268         } else {
269                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
270                 if (ret) {
271                         netdev_err(netdev,
272                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
273                         return ret;
274                 }
275
276                 for (i = 0; i < HNAE3_MAX_TC; i++) {
277                         if (!kinfo->tc_info[i].enable)
278                                 continue;
279
280                         netdev_set_tc_queue(netdev,
281                                             kinfo->tc_info[i].tc,
282                                             kinfo->tc_info[i].tqp_count,
283                                             kinfo->tc_info[i].tqp_offset);
284                 }
285         }
286
287         ret = netif_set_real_num_tx_queues(netdev, queue_size);
288         if (ret) {
289                 netdev_err(netdev,
290                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
291                            ret);
292                 return ret;
293         }
294
295         ret = netif_set_real_num_rx_queues(netdev, queue_size);
296         if (ret) {
297                 netdev_err(netdev,
298                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
299                 return ret;
300         }
301
302         return 0;
303 }
304
305 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
306 {
307         u16 alloc_tqps, max_rss_size, rss_size;
308
309         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
310         rss_size = alloc_tqps / h->kinfo.num_tc;
311
312         return min_t(u16, rss_size, max_rss_size);
313 }
314
315 static int hns3_nic_net_up(struct net_device *netdev)
316 {
317         struct hns3_nic_priv *priv = netdev_priv(netdev);
318         struct hnae3_handle *h = priv->ae_handle;
319         int i, j;
320         int ret;
321
322         ret = hns3_nic_reset_all_ring(h);
323         if (ret)
324                 return ret;
325
326         /* get irq resource for all vectors */
327         ret = hns3_nic_init_irq(priv);
328         if (ret) {
329                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
330                 return ret;
331         }
332
333         /* enable the vectors */
334         for (i = 0; i < priv->vector_num; i++)
335                 hns3_vector_enable(&priv->tqp_vector[i]);
336
337         /* start the ae_dev */
338         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
339         if (ret)
340                 goto out_start_err;
341
342         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
343
344         return 0;
345
346 out_start_err:
347         for (j = i - 1; j >= 0; j--)
348                 hns3_vector_disable(&priv->tqp_vector[j]);
349
350         hns3_nic_uninit_irq(priv);
351
352         return ret;
353 }
354
355 static int hns3_nic_net_open(struct net_device *netdev)
356 {
357         struct hns3_nic_priv *priv = netdev_priv(netdev);
358         struct hnae3_handle *h = hns3_get_handle(netdev);
359         struct hnae3_knic_private_info *kinfo;
360         int i, ret;
361
362         netif_carrier_off(netdev);
363
364         ret = hns3_nic_set_real_num_queue(netdev);
365         if (ret)
366                 return ret;
367
368         ret = hns3_nic_net_up(netdev);
369         if (ret) {
370                 netdev_err(netdev,
371                            "hns net up fail, ret=%d!\n", ret);
372                 return ret;
373         }
374
375         kinfo = &h->kinfo;
376         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
377                 netdev_set_prio_tc_map(netdev, i,
378                                        kinfo->prio_tc[i]);
379         }
380
381         priv->ae_handle->last_reset_time = jiffies;
382         return 0;
383 }
384
385 static void hns3_nic_net_down(struct net_device *netdev)
386 {
387         struct hns3_nic_priv *priv = netdev_priv(netdev);
388         const struct hnae3_ae_ops *ops;
389         int i;
390
391         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
392                 return;
393
394         /* disable vectors */
395         for (i = 0; i < priv->vector_num; i++)
396                 hns3_vector_disable(&priv->tqp_vector[i]);
397
398         /* stop ae_dev */
399         ops = priv->ae_handle->ae_algo->ops;
400         if (ops->stop)
401                 ops->stop(priv->ae_handle);
402
403         /* free irq resources */
404         hns3_nic_uninit_irq(priv);
405
406         hns3_clear_all_ring(priv->ae_handle);
407 }
408
409 static int hns3_nic_net_stop(struct net_device *netdev)
410 {
411         netif_tx_stop_all_queues(netdev);
412         netif_carrier_off(netdev);
413
414         hns3_nic_net_down(netdev);
415
416         return 0;
417 }
418
419 static int hns3_nic_uc_sync(struct net_device *netdev,
420                             const unsigned char *addr)
421 {
422         struct hnae3_handle *h = hns3_get_handle(netdev);
423
424         if (h->ae_algo->ops->add_uc_addr)
425                 return h->ae_algo->ops->add_uc_addr(h, addr);
426
427         return 0;
428 }
429
430 static int hns3_nic_uc_unsync(struct net_device *netdev,
431                               const unsigned char *addr)
432 {
433         struct hnae3_handle *h = hns3_get_handle(netdev);
434
435         if (h->ae_algo->ops->rm_uc_addr)
436                 return h->ae_algo->ops->rm_uc_addr(h, addr);
437
438         return 0;
439 }
440
441 static int hns3_nic_mc_sync(struct net_device *netdev,
442                             const unsigned char *addr)
443 {
444         struct hnae3_handle *h = hns3_get_handle(netdev);
445
446         if (h->ae_algo->ops->add_mc_addr)
447                 return h->ae_algo->ops->add_mc_addr(h, addr);
448
449         return 0;
450 }
451
452 static int hns3_nic_mc_unsync(struct net_device *netdev,
453                               const unsigned char *addr)
454 {
455         struct hnae3_handle *h = hns3_get_handle(netdev);
456
457         if (h->ae_algo->ops->rm_mc_addr)
458                 return h->ae_algo->ops->rm_mc_addr(h, addr);
459
460         return 0;
461 }
462
463 static u8 hns3_get_netdev_flags(struct net_device *netdev)
464 {
465         u8 flags = 0;
466
467         if (netdev->flags & IFF_PROMISC) {
468                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
469         } else {
470                 flags |= HNAE3_VLAN_FLTR;
471                 if (netdev->flags & IFF_ALLMULTI)
472                         flags |= HNAE3_USER_MPE;
473         }
474
475         return flags;
476 }
477
478 static void hns3_nic_set_rx_mode(struct net_device *netdev)
479 {
480         struct hnae3_handle *h = hns3_get_handle(netdev);
481         u8 new_flags;
482         int ret;
483
484         new_flags = hns3_get_netdev_flags(netdev);
485
486         ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
487         if (ret) {
488                 netdev_err(netdev, "sync uc address fail\n");
489                 if (ret == -ENOSPC)
490                         new_flags |= HNAE3_OVERFLOW_UPE;
491         }
492
493         if (netdev->flags & IFF_MULTICAST) {
494                 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
495                                     hns3_nic_mc_unsync);
496                 if (ret) {
497                         netdev_err(netdev, "sync mc address fail\n");
498                         if (ret == -ENOSPC)
499                                 new_flags |= HNAE3_OVERFLOW_MPE;
500                 }
501         }
502
503         hns3_update_promisc_mode(netdev, new_flags);
504         /* User mode Promisc mode enable and vlan filtering is disabled to
505          * let all packets in. MAC-VLAN Table overflow Promisc enabled and
506          * vlan fitering is enabled
507          */
508         hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
509         h->netdev_flags = new_flags;
510 }
511
512 void hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
513 {
514         struct hns3_nic_priv *priv = netdev_priv(netdev);
515         struct hnae3_handle *h = priv->ae_handle;
516
517         if (h->ae_algo->ops->set_promisc_mode) {
518                 h->ae_algo->ops->set_promisc_mode(h,
519                                                   promisc_flags & HNAE3_UPE,
520                                                   promisc_flags & HNAE3_MPE);
521         }
522 }
523
524 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
525 {
526         struct hns3_nic_priv *priv = netdev_priv(netdev);
527         struct hnae3_handle *h = priv->ae_handle;
528         bool last_state;
529
530         if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
531                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
532                 if (enable != last_state) {
533                         netdev_info(netdev,
534                                     "%s vlan filter\n",
535                                     enable ? "enable" : "disable");
536                         h->ae_algo->ops->enable_vlan_filter(h, enable);
537                 }
538         }
539 }
540
541 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
542                         u16 *mss, u32 *type_cs_vlan_tso)
543 {
544         u32 l4_offset, hdr_len;
545         union l3_hdr_info l3;
546         union l4_hdr_info l4;
547         u32 l4_paylen;
548         int ret;
549
550         if (!skb_is_gso(skb))
551                 return 0;
552
553         ret = skb_cow_head(skb, 0);
554         if (ret)
555                 return ret;
556
557         l3.hdr = skb_network_header(skb);
558         l4.hdr = skb_transport_header(skb);
559
560         /* Software should clear the IPv4's checksum field when tso is
561          * needed.
562          */
563         if (l3.v4->version == 4)
564                 l3.v4->check = 0;
565
566         /* tunnel packet.*/
567         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
568                                          SKB_GSO_GRE_CSUM |
569                                          SKB_GSO_UDP_TUNNEL |
570                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
571                 if ((!(skb_shinfo(skb)->gso_type &
572                     SKB_GSO_PARTIAL)) &&
573                     (skb_shinfo(skb)->gso_type &
574                     SKB_GSO_UDP_TUNNEL_CSUM)) {
575                         /* Software should clear the udp's checksum
576                          * field when tso is needed.
577                          */
578                         l4.udp->check = 0;
579                 }
580                 /* reset l3&l4 pointers from outer to inner headers */
581                 l3.hdr = skb_inner_network_header(skb);
582                 l4.hdr = skb_inner_transport_header(skb);
583
584                 /* Software should clear the IPv4's checksum field when
585                  * tso is needed.
586                  */
587                 if (l3.v4->version == 4)
588                         l3.v4->check = 0;
589         }
590
591         /* normal or tunnel packet*/
592         l4_offset = l4.hdr - skb->data;
593         hdr_len = (l4.tcp->doff * 4) + l4_offset;
594
595         /* remove payload length from inner pseudo checksum when tso*/
596         l4_paylen = skb->len - l4_offset;
597         csum_replace_by_diff(&l4.tcp->check,
598                              (__force __wsum)htonl(l4_paylen));
599
600         /* find the txbd field values */
601         *paylen = skb->len - hdr_len;
602         hnae3_set_bit(*type_cs_vlan_tso,
603                       HNS3_TXD_TSO_B, 1);
604
605         /* get MSS for TSO */
606         *mss = skb_shinfo(skb)->gso_size;
607
608         return 0;
609 }
610
611 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
612                                 u8 *il4_proto)
613 {
614         union {
615                 struct iphdr *v4;
616                 struct ipv6hdr *v6;
617                 unsigned char *hdr;
618         } l3;
619         unsigned char *l4_hdr;
620         unsigned char *exthdr;
621         u8 l4_proto_tmp;
622         __be16 frag_off;
623
624         /* find outer header point */
625         l3.hdr = skb_network_header(skb);
626         l4_hdr = skb_transport_header(skb);
627
628         if (skb->protocol == htons(ETH_P_IPV6)) {
629                 exthdr = l3.hdr + sizeof(*l3.v6);
630                 l4_proto_tmp = l3.v6->nexthdr;
631                 if (l4_hdr != exthdr)
632                         ipv6_skip_exthdr(skb, exthdr - skb->data,
633                                          &l4_proto_tmp, &frag_off);
634         } else if (skb->protocol == htons(ETH_P_IP)) {
635                 l4_proto_tmp = l3.v4->protocol;
636         } else {
637                 return -EINVAL;
638         }
639
640         *ol4_proto = l4_proto_tmp;
641
642         /* tunnel packet */
643         if (!skb->encapsulation) {
644                 *il4_proto = 0;
645                 return 0;
646         }
647
648         /* find inner header point */
649         l3.hdr = skb_inner_network_header(skb);
650         l4_hdr = skb_inner_transport_header(skb);
651
652         if (l3.v6->version == 6) {
653                 exthdr = l3.hdr + sizeof(*l3.v6);
654                 l4_proto_tmp = l3.v6->nexthdr;
655                 if (l4_hdr != exthdr)
656                         ipv6_skip_exthdr(skb, exthdr - skb->data,
657                                          &l4_proto_tmp, &frag_off);
658         } else if (l3.v4->version == 4) {
659                 l4_proto_tmp = l3.v4->protocol;
660         }
661
662         *il4_proto = l4_proto_tmp;
663
664         return 0;
665 }
666
667 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
668                                 u8 il4_proto, u32 *type_cs_vlan_tso,
669                                 u32 *ol_type_vlan_len_msec)
670 {
671         union {
672                 struct iphdr *v4;
673                 struct ipv6hdr *v6;
674                 unsigned char *hdr;
675         } l3;
676         union {
677                 struct tcphdr *tcp;
678                 struct udphdr *udp;
679                 struct gre_base_hdr *gre;
680                 unsigned char *hdr;
681         } l4;
682         unsigned char *l2_hdr;
683         u8 l4_proto = ol4_proto;
684         u32 ol2_len;
685         u32 ol3_len;
686         u32 ol4_len;
687         u32 l2_len;
688         u32 l3_len;
689
690         l3.hdr = skb_network_header(skb);
691         l4.hdr = skb_transport_header(skb);
692
693         /* compute L2 header size for normal packet, defined in 2 Bytes */
694         l2_len = l3.hdr - skb->data;
695         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
696                         HNS3_TXD_L2LEN_S, l2_len >> 1);
697
698         /* tunnel packet*/
699         if (skb->encapsulation) {
700                 /* compute OL2 header size, defined in 2 Bytes */
701                 ol2_len = l2_len;
702                 hnae3_set_field(*ol_type_vlan_len_msec,
703                                 HNS3_TXD_L2LEN_M,
704                                 HNS3_TXD_L2LEN_S, ol2_len >> 1);
705
706                 /* compute OL3 header size, defined in 4 Bytes */
707                 ol3_len = l4.hdr - l3.hdr;
708                 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
709                                 HNS3_TXD_L3LEN_S, ol3_len >> 2);
710
711                 /* MAC in UDP, MAC in GRE (0x6558)*/
712                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
713                         /* switch MAC header ptr from outer to inner header.*/
714                         l2_hdr = skb_inner_mac_header(skb);
715
716                         /* compute OL4 header size, defined in 4 Bytes. */
717                         ol4_len = l2_hdr - l4.hdr;
718                         hnae3_set_field(*ol_type_vlan_len_msec,
719                                         HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
720                                         ol4_len >> 2);
721
722                         /* switch IP header ptr from outer to inner header */
723                         l3.hdr = skb_inner_network_header(skb);
724
725                         /* compute inner l2 header size, defined in 2 Bytes. */
726                         l2_len = l3.hdr - l2_hdr;
727                         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
728                                         HNS3_TXD_L2LEN_S, l2_len >> 1);
729                 } else {
730                         /* skb packet types not supported by hardware,
731                          * txbd len fild doesn't be filled.
732                          */
733                         return;
734                 }
735
736                 /* switch L4 header pointer from outer to inner */
737                 l4.hdr = skb_inner_transport_header(skb);
738
739                 l4_proto = il4_proto;
740         }
741
742         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
743         l3_len = l4.hdr - l3.hdr;
744         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
745                         HNS3_TXD_L3LEN_S, l3_len >> 2);
746
747         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
748         switch (l4_proto) {
749         case IPPROTO_TCP:
750                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
751                                 HNS3_TXD_L4LEN_S, l4.tcp->doff);
752                 break;
753         case IPPROTO_SCTP:
754                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
755                                 HNS3_TXD_L4LEN_S,
756                                 (sizeof(struct sctphdr) >> 2));
757                 break;
758         case IPPROTO_UDP:
759                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
760                                 HNS3_TXD_L4LEN_S,
761                                 (sizeof(struct udphdr) >> 2));
762                 break;
763         default:
764                 /* skb packet types not supported by hardware,
765                  * txbd len fild doesn't be filled.
766                  */
767                 return;
768         }
769 }
770
771 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
772  * and it is udp packet, which has a dest port as the IANA assigned.
773  * the hardware is expected to do the checksum offload, but the
774  * hardware will not do the checksum offload when udp dest port is
775  * 4789.
776  */
777 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
778 {
779 #define IANA_VXLAN_PORT 4789
780         union {
781                 struct tcphdr *tcp;
782                 struct udphdr *udp;
783                 struct gre_base_hdr *gre;
784                 unsigned char *hdr;
785         } l4;
786
787         l4.hdr = skb_transport_header(skb);
788
789         if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
790                 return false;
791
792         skb_checksum_help(skb);
793
794         return true;
795 }
796
797 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
798                                    u8 il4_proto, u32 *type_cs_vlan_tso,
799                                    u32 *ol_type_vlan_len_msec)
800 {
801         union {
802                 struct iphdr *v4;
803                 struct ipv6hdr *v6;
804                 unsigned char *hdr;
805         } l3;
806         u32 l4_proto = ol4_proto;
807
808         l3.hdr = skb_network_header(skb);
809
810         /* define OL3 type and tunnel type(OL4).*/
811         if (skb->encapsulation) {
812                 /* define outer network header type.*/
813                 if (skb->protocol == htons(ETH_P_IP)) {
814                         if (skb_is_gso(skb))
815                                 hnae3_set_field(*ol_type_vlan_len_msec,
816                                                 HNS3_TXD_OL3T_M,
817                                                 HNS3_TXD_OL3T_S,
818                                                 HNS3_OL3T_IPV4_CSUM);
819                         else
820                                 hnae3_set_field(*ol_type_vlan_len_msec,
821                                                 HNS3_TXD_OL3T_M,
822                                                 HNS3_TXD_OL3T_S,
823                                                 HNS3_OL3T_IPV4_NO_CSUM);
824
825                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
826                         hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
827                                         HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
828                 }
829
830                 /* define tunnel type(OL4).*/
831                 switch (l4_proto) {
832                 case IPPROTO_UDP:
833                         hnae3_set_field(*ol_type_vlan_len_msec,
834                                         HNS3_TXD_TUNTYPE_M,
835                                         HNS3_TXD_TUNTYPE_S,
836                                         HNS3_TUN_MAC_IN_UDP);
837                         break;
838                 case IPPROTO_GRE:
839                         hnae3_set_field(*ol_type_vlan_len_msec,
840                                         HNS3_TXD_TUNTYPE_M,
841                                         HNS3_TXD_TUNTYPE_S,
842                                         HNS3_TUN_NVGRE);
843                         break;
844                 default:
845                         /* drop the skb tunnel packet if hardware don't support,
846                          * because hardware can't calculate csum when TSO.
847                          */
848                         if (skb_is_gso(skb))
849                                 return -EDOM;
850
851                         /* the stack computes the IP header already,
852                          * driver calculate l4 checksum when not TSO.
853                          */
854                         skb_checksum_help(skb);
855                         return 0;
856                 }
857
858                 l3.hdr = skb_inner_network_header(skb);
859                 l4_proto = il4_proto;
860         }
861
862         if (l3.v4->version == 4) {
863                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
864                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
865
866                 /* the stack computes the IP header already, the only time we
867                  * need the hardware to recompute it is in the case of TSO.
868                  */
869                 if (skb_is_gso(skb))
870                         hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
871         } else if (l3.v6->version == 6) {
872                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
873                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
874         }
875
876         switch (l4_proto) {
877         case IPPROTO_TCP:
878                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
879                 hnae3_set_field(*type_cs_vlan_tso,
880                                 HNS3_TXD_L4T_M,
881                                 HNS3_TXD_L4T_S,
882                                 HNS3_L4T_TCP);
883                 break;
884         case IPPROTO_UDP:
885                 if (hns3_tunnel_csum_bug(skb))
886                         break;
887
888                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
889                 hnae3_set_field(*type_cs_vlan_tso,
890                                 HNS3_TXD_L4T_M,
891                                 HNS3_TXD_L4T_S,
892                                 HNS3_L4T_UDP);
893                 break;
894         case IPPROTO_SCTP:
895                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
896                 hnae3_set_field(*type_cs_vlan_tso,
897                                 HNS3_TXD_L4T_M,
898                                 HNS3_TXD_L4T_S,
899                                 HNS3_L4T_SCTP);
900                 break;
901         default:
902                 /* drop the skb tunnel packet if hardware don't support,
903                  * because hardware can't calculate csum when TSO.
904                  */
905                 if (skb_is_gso(skb))
906                         return -EDOM;
907
908                 /* the stack computes the IP header already,
909                  * driver calculate l4 checksum when not TSO.
910                  */
911                 skb_checksum_help(skb);
912                 return 0;
913         }
914
915         return 0;
916 }
917
918 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
919 {
920         /* Config bd buffer end */
921         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
922                         HNS3_TXD_BDTYPE_S, 0);
923         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
924         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
925         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
926 }
927
928 static int hns3_fill_desc_vtags(struct sk_buff *skb,
929                                 struct hns3_enet_ring *tx_ring,
930                                 u32 *inner_vlan_flag,
931                                 u32 *out_vlan_flag,
932                                 u16 *inner_vtag,
933                                 u16 *out_vtag)
934 {
935 #define HNS3_TX_VLAN_PRIO_SHIFT 13
936
937         if (skb->protocol == htons(ETH_P_8021Q) &&
938             !(tx_ring->tqp->handle->kinfo.netdev->features &
939             NETIF_F_HW_VLAN_CTAG_TX)) {
940                 /* When HW VLAN acceleration is turned off, and the stack
941                  * sets the protocol to 802.1q, the driver just need to
942                  * set the protocol to the encapsulated ethertype.
943                  */
944                 skb->protocol = vlan_get_protocol(skb);
945                 return 0;
946         }
947
948         if (skb_vlan_tag_present(skb)) {
949                 u16 vlan_tag;
950
951                 vlan_tag = skb_vlan_tag_get(skb);
952                 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
953
954                 /* Based on hw strategy, use out_vtag in two layer tag case,
955                  * and use inner_vtag in one tag case.
956                  */
957                 if (skb->protocol == htons(ETH_P_8021Q)) {
958                         hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
959                         *out_vtag = vlan_tag;
960                 } else {
961                         hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
962                         *inner_vtag = vlan_tag;
963                 }
964         } else if (skb->protocol == htons(ETH_P_8021Q)) {
965                 struct vlan_ethhdr *vhdr;
966                 int rc;
967
968                 rc = skb_cow_head(skb, 0);
969                 if (rc < 0)
970                         return rc;
971                 vhdr = (struct vlan_ethhdr *)skb->data;
972                 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
973                                         << HNS3_TX_VLAN_PRIO_SHIFT);
974         }
975
976         skb->protocol = vlan_get_protocol(skb);
977         return 0;
978 }
979
980 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
981                           int size, int frag_end, enum hns_desc_type type)
982 {
983         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
984         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
985         struct device *dev = ring_to_dev(ring);
986         u32 ol_type_vlan_len_msec = 0;
987         u16 bdtp_fe_sc_vld_ra_ri = 0;
988         struct skb_frag_struct *frag;
989         unsigned int frag_buf_num;
990         u32 type_cs_vlan_tso = 0;
991         struct sk_buff *skb;
992         u16 inner_vtag = 0;
993         u16 out_vtag = 0;
994         unsigned int k;
995         int sizeoflast;
996         u32 paylen = 0;
997         dma_addr_t dma;
998         u16 mss = 0;
999         u8 ol4_proto;
1000         u8 il4_proto;
1001         int ret;
1002
1003         if (type == DESC_TYPE_SKB) {
1004                 skb = (struct sk_buff *)priv;
1005                 paylen = skb->len;
1006
1007                 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1008                                            &ol_type_vlan_len_msec,
1009                                            &inner_vtag, &out_vtag);
1010                 if (unlikely(ret))
1011                         return ret;
1012
1013                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1014                         skb_reset_mac_len(skb);
1015
1016                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1017                         if (ret)
1018                                 return ret;
1019                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1020                                             &type_cs_vlan_tso,
1021                                             &ol_type_vlan_len_msec);
1022                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1023                                                       &type_cs_vlan_tso,
1024                                                       &ol_type_vlan_len_msec);
1025                         if (ret)
1026                                 return ret;
1027
1028                         ret = hns3_set_tso(skb, &paylen, &mss,
1029                                            &type_cs_vlan_tso);
1030                         if (ret)
1031                                 return ret;
1032                 }
1033
1034                 /* Set txbd */
1035                 desc->tx.ol_type_vlan_len_msec =
1036                         cpu_to_le32(ol_type_vlan_len_msec);
1037                 desc->tx.type_cs_vlan_tso_len =
1038                         cpu_to_le32(type_cs_vlan_tso);
1039                 desc->tx.paylen = cpu_to_le32(paylen);
1040                 desc->tx.mss = cpu_to_le16(mss);
1041                 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1042                 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1043
1044                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1045         } else {
1046                 frag = (struct skb_frag_struct *)priv;
1047                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1048         }
1049
1050         if (dma_mapping_error(ring->dev, dma)) {
1051                 ring->stats.sw_err_cnt++;
1052                 return -ENOMEM;
1053         }
1054
1055         desc_cb->length = size;
1056
1057         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1058         sizeoflast = size % HNS3_MAX_BD_SIZE;
1059         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1060
1061         /* When frag size is bigger than hardware limit, split this frag */
1062         for (k = 0; k < frag_buf_num; k++) {
1063                 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1064                 desc_cb->priv = priv;
1065                 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1066                 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1067                                         DESC_TYPE_SKB : DESC_TYPE_PAGE;
1068
1069                 /* now, fill the descriptor */
1070                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1071                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1072                                 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1073                 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1074                                        frag_end && (k == frag_buf_num - 1) ?
1075                                                 1 : 0);
1076                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1077                                 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1078
1079                 /* move ring pointer to next.*/
1080                 ring_ptr_move_fw(ring, next_to_use);
1081
1082                 desc_cb = &ring->desc_cb[ring->next_to_use];
1083                 desc = &ring->desc[ring->next_to_use];
1084         }
1085
1086         return 0;
1087 }
1088
1089 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1090                                    struct hns3_enet_ring *ring)
1091 {
1092         struct sk_buff *skb = *out_skb;
1093         struct skb_frag_struct *frag;
1094         int bdnum_for_frag;
1095         int frag_num;
1096         int buf_num;
1097         int size;
1098         int i;
1099
1100         size = skb_headlen(skb);
1101         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1102
1103         frag_num = skb_shinfo(skb)->nr_frags;
1104         for (i = 0; i < frag_num; i++) {
1105                 frag = &skb_shinfo(skb)->frags[i];
1106                 size = skb_frag_size(frag);
1107                 bdnum_for_frag =
1108                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1109                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1110                         return -ENOMEM;
1111
1112                 buf_num += bdnum_for_frag;
1113         }
1114
1115         if (buf_num > ring_space(ring))
1116                 return -EBUSY;
1117
1118         *bnum = buf_num;
1119         return 0;
1120 }
1121
1122 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1123                                   struct hns3_enet_ring *ring)
1124 {
1125         struct sk_buff *skb = *out_skb;
1126         int buf_num;
1127
1128         /* No. of segments (plus a header) */
1129         buf_num = skb_shinfo(skb)->nr_frags + 1;
1130
1131         if (unlikely(ring_space(ring) < buf_num))
1132                 return -EBUSY;
1133
1134         *bnum = buf_num;
1135
1136         return 0;
1137 }
1138
1139 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1140 {
1141         struct device *dev = ring_to_dev(ring);
1142         unsigned int i;
1143
1144         for (i = 0; i < ring->desc_num; i++) {
1145                 /* check if this is where we started */
1146                 if (ring->next_to_use == next_to_use_orig)
1147                         break;
1148
1149                 /* unmap the descriptor dma address */
1150                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1151                         dma_unmap_single(dev,
1152                                          ring->desc_cb[ring->next_to_use].dma,
1153                                         ring->desc_cb[ring->next_to_use].length,
1154                                         DMA_TO_DEVICE);
1155                 else if (ring->desc_cb[ring->next_to_use].length)
1156                         dma_unmap_page(dev,
1157                                        ring->desc_cb[ring->next_to_use].dma,
1158                                        ring->desc_cb[ring->next_to_use].length,
1159                                        DMA_TO_DEVICE);
1160
1161                 ring->desc_cb[ring->next_to_use].length = 0;
1162
1163                 /* rollback one */
1164                 ring_ptr_move_bw(ring, next_to_use);
1165         }
1166 }
1167
1168 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1169 {
1170         struct hns3_nic_priv *priv = netdev_priv(netdev);
1171         struct hns3_nic_ring_data *ring_data =
1172                 &tx_ring_data(priv, skb->queue_mapping);
1173         struct hns3_enet_ring *ring = ring_data->ring;
1174         struct netdev_queue *dev_queue;
1175         struct skb_frag_struct *frag;
1176         int next_to_use_head;
1177         int next_to_use_frag;
1178         int buf_num;
1179         int seg_num;
1180         int size;
1181         int ret;
1182         int i;
1183
1184         /* Prefetch the data used later */
1185         prefetch(skb->data);
1186
1187         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1188         case -EBUSY:
1189                 u64_stats_update_begin(&ring->syncp);
1190                 ring->stats.tx_busy++;
1191                 u64_stats_update_end(&ring->syncp);
1192
1193                 goto out_net_tx_busy;
1194         case -ENOMEM:
1195                 u64_stats_update_begin(&ring->syncp);
1196                 ring->stats.sw_err_cnt++;
1197                 u64_stats_update_end(&ring->syncp);
1198                 netdev_err(netdev, "no memory to xmit!\n");
1199
1200                 goto out_err_tx_ok;
1201         default:
1202                 break;
1203         }
1204
1205         /* No. of segments (plus a header) */
1206         seg_num = skb_shinfo(skb)->nr_frags + 1;
1207         /* Fill the first part */
1208         size = skb_headlen(skb);
1209
1210         next_to_use_head = ring->next_to_use;
1211
1212         ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1213                                   DESC_TYPE_SKB);
1214         if (ret)
1215                 goto head_fill_err;
1216
1217         next_to_use_frag = ring->next_to_use;
1218         /* Fill the fragments */
1219         for (i = 1; i < seg_num; i++) {
1220                 frag = &skb_shinfo(skb)->frags[i - 1];
1221                 size = skb_frag_size(frag);
1222
1223                 ret = priv->ops.fill_desc(ring, frag, size,
1224                                           seg_num - 1 == i ? 1 : 0,
1225                                           DESC_TYPE_PAGE);
1226
1227                 if (ret)
1228                         goto frag_fill_err;
1229         }
1230
1231         /* Complete translate all packets */
1232         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1233         netdev_tx_sent_queue(dev_queue, skb->len);
1234
1235         wmb(); /* Commit all data before submit */
1236
1237         hnae3_queue_xmit(ring->tqp, buf_num);
1238
1239         return NETDEV_TX_OK;
1240
1241 frag_fill_err:
1242         hns3_clear_desc(ring, next_to_use_frag);
1243
1244 head_fill_err:
1245         hns3_clear_desc(ring, next_to_use_head);
1246
1247 out_err_tx_ok:
1248         dev_kfree_skb_any(skb);
1249         return NETDEV_TX_OK;
1250
1251 out_net_tx_busy:
1252         netif_stop_subqueue(netdev, ring_data->queue_index);
1253         smp_mb(); /* Commit all data before submit */
1254
1255         return NETDEV_TX_BUSY;
1256 }
1257
1258 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1259 {
1260         struct hnae3_handle *h = hns3_get_handle(netdev);
1261         struct sockaddr *mac_addr = p;
1262         int ret;
1263
1264         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1265                 return -EADDRNOTAVAIL;
1266
1267         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1268                 netdev_info(netdev, "already using mac address %pM\n",
1269                             mac_addr->sa_data);
1270                 return 0;
1271         }
1272
1273         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1274         if (ret) {
1275                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1276                 return ret;
1277         }
1278
1279         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1280
1281         return 0;
1282 }
1283
1284 static int hns3_nic_do_ioctl(struct net_device *netdev,
1285                              struct ifreq *ifr, int cmd)
1286 {
1287         struct hnae3_handle *h = hns3_get_handle(netdev);
1288
1289         if (!netif_running(netdev))
1290                 return -EINVAL;
1291
1292         if (!h->ae_algo->ops->do_ioctl)
1293                 return -EOPNOTSUPP;
1294
1295         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1296 }
1297
1298 static int hns3_nic_set_features(struct net_device *netdev,
1299                                  netdev_features_t features)
1300 {
1301         netdev_features_t changed = netdev->features ^ features;
1302         struct hns3_nic_priv *priv = netdev_priv(netdev);
1303         struct hnae3_handle *h = priv->ae_handle;
1304         int ret;
1305
1306         if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1307                 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1308                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1309                 else
1310                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1311         }
1312
1313         if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1314             h->ae_algo->ops->enable_vlan_filter) {
1315                 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1316                         h->ae_algo->ops->enable_vlan_filter(h, true);
1317                 else
1318                         h->ae_algo->ops->enable_vlan_filter(h, false);
1319         }
1320
1321         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1322             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1323                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1324                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1325                 else
1326                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1327
1328                 if (ret)
1329                         return ret;
1330         }
1331
1332         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1333                 if (features & NETIF_F_NTUPLE)
1334                         h->ae_algo->ops->enable_fd(h, true);
1335                 else
1336                         h->ae_algo->ops->enable_fd(h, false);
1337         }
1338
1339         netdev->features = features;
1340         return 0;
1341 }
1342
1343 static void hns3_nic_get_stats64(struct net_device *netdev,
1344                                  struct rtnl_link_stats64 *stats)
1345 {
1346         struct hns3_nic_priv *priv = netdev_priv(netdev);
1347         int queue_num = priv->ae_handle->kinfo.num_tqps;
1348         struct hnae3_handle *handle = priv->ae_handle;
1349         struct hns3_enet_ring *ring;
1350         unsigned int start;
1351         unsigned int idx;
1352         u64 tx_bytes = 0;
1353         u64 rx_bytes = 0;
1354         u64 tx_pkts = 0;
1355         u64 rx_pkts = 0;
1356         u64 tx_drop = 0;
1357         u64 rx_drop = 0;
1358
1359         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1360                 return;
1361
1362         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1363
1364         for (idx = 0; idx < queue_num; idx++) {
1365                 /* fetch the tx stats */
1366                 ring = priv->ring_data[idx].ring;
1367                 do {
1368                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1369                         tx_bytes += ring->stats.tx_bytes;
1370                         tx_pkts += ring->stats.tx_pkts;
1371                         tx_drop += ring->stats.tx_busy;
1372                         tx_drop += ring->stats.sw_err_cnt;
1373                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1374
1375                 /* fetch the rx stats */
1376                 ring = priv->ring_data[idx + queue_num].ring;
1377                 do {
1378                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1379                         rx_bytes += ring->stats.rx_bytes;
1380                         rx_pkts += ring->stats.rx_pkts;
1381                         rx_drop += ring->stats.non_vld_descs;
1382                         rx_drop += ring->stats.err_pkt_len;
1383                         rx_drop += ring->stats.l2_err;
1384                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1385         }
1386
1387         stats->tx_bytes = tx_bytes;
1388         stats->tx_packets = tx_pkts;
1389         stats->rx_bytes = rx_bytes;
1390         stats->rx_packets = rx_pkts;
1391
1392         stats->rx_errors = netdev->stats.rx_errors;
1393         stats->multicast = netdev->stats.multicast;
1394         stats->rx_length_errors = netdev->stats.rx_length_errors;
1395         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1396         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1397
1398         stats->tx_errors = netdev->stats.tx_errors;
1399         stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1400         stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1401         stats->collisions = netdev->stats.collisions;
1402         stats->rx_over_errors = netdev->stats.rx_over_errors;
1403         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1404         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1405         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1406         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1407         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1408         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1409         stats->tx_window_errors = netdev->stats.tx_window_errors;
1410         stats->rx_compressed = netdev->stats.rx_compressed;
1411         stats->tx_compressed = netdev->stats.tx_compressed;
1412 }
1413
1414 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1415 {
1416         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1417         struct hnae3_handle *h = hns3_get_handle(netdev);
1418         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1419         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1420         u8 tc = mqprio_qopt->qopt.num_tc;
1421         u16 mode = mqprio_qopt->mode;
1422         u8 hw = mqprio_qopt->qopt.hw;
1423         bool if_running;
1424         int ret;
1425
1426         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1427                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1428                 return -EOPNOTSUPP;
1429
1430         if (tc > HNAE3_MAX_TC)
1431                 return -EINVAL;
1432
1433         if (!netdev)
1434                 return -EINVAL;
1435
1436         if_running = netif_running(netdev);
1437         if (if_running) {
1438                 hns3_nic_net_stop(netdev);
1439                 msleep(100);
1440         }
1441
1442         ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1443                 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1444         if (ret)
1445                 goto out;
1446
1447         ret = hns3_nic_set_real_num_queue(netdev);
1448
1449 out:
1450         if (if_running)
1451                 hns3_nic_net_open(netdev);
1452
1453         return ret;
1454 }
1455
1456 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1457                              void *type_data)
1458 {
1459         if (type != TC_SETUP_QDISC_MQPRIO)
1460                 return -EOPNOTSUPP;
1461
1462         return hns3_setup_tc(dev, type_data);
1463 }
1464
1465 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1466                                 __be16 proto, u16 vid)
1467 {
1468         struct hnae3_handle *h = hns3_get_handle(netdev);
1469         struct hns3_nic_priv *priv = netdev_priv(netdev);
1470         int ret = -EIO;
1471
1472         if (h->ae_algo->ops->set_vlan_filter)
1473                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1474
1475         if (!ret)
1476                 set_bit(vid, priv->active_vlans);
1477
1478         return ret;
1479 }
1480
1481 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1482                                  __be16 proto, u16 vid)
1483 {
1484         struct hnae3_handle *h = hns3_get_handle(netdev);
1485         struct hns3_nic_priv *priv = netdev_priv(netdev);
1486         int ret = -EIO;
1487
1488         if (h->ae_algo->ops->set_vlan_filter)
1489                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1490
1491         if (!ret)
1492                 clear_bit(vid, priv->active_vlans);
1493
1494         return ret;
1495 }
1496
1497 static void hns3_restore_vlan(struct net_device *netdev)
1498 {
1499         struct hns3_nic_priv *priv = netdev_priv(netdev);
1500         u16 vid;
1501         int ret;
1502
1503         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1504                 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1505                 if (ret)
1506                         netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n",
1507                                     vid, ret);
1508         }
1509 }
1510
1511 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1512                                 u8 qos, __be16 vlan_proto)
1513 {
1514         struct hnae3_handle *h = hns3_get_handle(netdev);
1515         int ret = -EIO;
1516
1517         if (h->ae_algo->ops->set_vf_vlan_filter)
1518                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1519                                                    qos, vlan_proto);
1520
1521         return ret;
1522 }
1523
1524 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1525 {
1526         struct hnae3_handle *h = hns3_get_handle(netdev);
1527         bool if_running = netif_running(netdev);
1528         int ret;
1529
1530         if (!h->ae_algo->ops->set_mtu)
1531                 return -EOPNOTSUPP;
1532
1533         /* if this was called with netdev up then bring netdevice down */
1534         if (if_running) {
1535                 (void)hns3_nic_net_stop(netdev);
1536                 msleep(100);
1537         }
1538
1539         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1540         if (ret)
1541                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1542                            ret);
1543         else
1544                 netdev->mtu = new_mtu;
1545
1546         /* if the netdev was running earlier, bring it up again */
1547         if (if_running && hns3_nic_net_open(netdev))
1548                 ret = -EINVAL;
1549
1550         return ret;
1551 }
1552
1553 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1554 {
1555         struct hns3_nic_priv *priv = netdev_priv(ndev);
1556         struct hns3_enet_ring *tx_ring = NULL;
1557         int timeout_queue = 0;
1558         int hw_head, hw_tail;
1559         int i;
1560
1561         /* Find the stopped queue the same way the stack does */
1562         for (i = 0; i < ndev->real_num_tx_queues; i++) {
1563                 struct netdev_queue *q;
1564                 unsigned long trans_start;
1565
1566                 q = netdev_get_tx_queue(ndev, i);
1567                 trans_start = q->trans_start;
1568                 if (netif_xmit_stopped(q) &&
1569                     time_after(jiffies,
1570                                (trans_start + ndev->watchdog_timeo))) {
1571                         timeout_queue = i;
1572                         break;
1573                 }
1574         }
1575
1576         if (i == ndev->num_tx_queues) {
1577                 netdev_info(ndev,
1578                             "no netdev TX timeout queue found, timeout count: %llu\n",
1579                             priv->tx_timeout_count);
1580                 return false;
1581         }
1582
1583         tx_ring = priv->ring_data[timeout_queue].ring;
1584
1585         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1586                                 HNS3_RING_TX_RING_HEAD_REG);
1587         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1588                                 HNS3_RING_TX_RING_TAIL_REG);
1589         netdev_info(ndev,
1590                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1591                     priv->tx_timeout_count,
1592                     timeout_queue,
1593                     tx_ring->next_to_use,
1594                     tx_ring->next_to_clean,
1595                     hw_head,
1596                     hw_tail,
1597                     readl(tx_ring->tqp_vector->mask_addr));
1598
1599         return true;
1600 }
1601
1602 static void hns3_nic_net_timeout(struct net_device *ndev)
1603 {
1604         struct hns3_nic_priv *priv = netdev_priv(ndev);
1605         struct hnae3_handle *h = priv->ae_handle;
1606
1607         if (!hns3_get_tx_timeo_queue_info(ndev))
1608                 return;
1609
1610         priv->tx_timeout_count++;
1611
1612         if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
1613                 return;
1614
1615         /* request the reset */
1616         if (h->ae_algo->ops->reset_event)
1617                 h->ae_algo->ops->reset_event(h->pdev, h);
1618 }
1619
1620 static const struct net_device_ops hns3_nic_netdev_ops = {
1621         .ndo_open               = hns3_nic_net_open,
1622         .ndo_stop               = hns3_nic_net_stop,
1623         .ndo_start_xmit         = hns3_nic_net_xmit,
1624         .ndo_tx_timeout         = hns3_nic_net_timeout,
1625         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1626         .ndo_do_ioctl           = hns3_nic_do_ioctl,
1627         .ndo_change_mtu         = hns3_nic_change_mtu,
1628         .ndo_set_features       = hns3_nic_set_features,
1629         .ndo_get_stats64        = hns3_nic_get_stats64,
1630         .ndo_setup_tc           = hns3_nic_setup_tc,
1631         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1632         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1633         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1634         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1635 };
1636
1637 static bool hns3_is_phys_func(struct pci_dev *pdev)
1638 {
1639         u32 dev_id = pdev->device;
1640
1641         switch (dev_id) {
1642         case HNAE3_DEV_ID_GE:
1643         case HNAE3_DEV_ID_25GE:
1644         case HNAE3_DEV_ID_25GE_RDMA:
1645         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1646         case HNAE3_DEV_ID_50GE_RDMA:
1647         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1648         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1649                 return true;
1650         case HNAE3_DEV_ID_100G_VF:
1651         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1652                 return false;
1653         default:
1654                 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1655                          dev_id);
1656         }
1657
1658         return false;
1659 }
1660
1661 static void hns3_disable_sriov(struct pci_dev *pdev)
1662 {
1663         /* If our VFs are assigned we cannot shut down SR-IOV
1664          * without causing issues, so just leave the hardware
1665          * available but disabled
1666          */
1667         if (pci_vfs_assigned(pdev)) {
1668                 dev_warn(&pdev->dev,
1669                          "disabling driver while VFs are assigned\n");
1670                 return;
1671         }
1672
1673         pci_disable_sriov(pdev);
1674 }
1675
1676 static void hns3_get_dev_capability(struct pci_dev *pdev,
1677                                     struct hnae3_ae_dev *ae_dev)
1678 {
1679         if (pdev->revision >= 0x21)
1680                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
1681 }
1682
1683 /* hns3_probe - Device initialization routine
1684  * @pdev: PCI device information struct
1685  * @ent: entry in hns3_pci_tbl
1686  *
1687  * hns3_probe initializes a PF identified by a pci_dev structure.
1688  * The OS initialization, configuring of the PF private structure,
1689  * and a hardware reset occur.
1690  *
1691  * Returns 0 on success, negative on failure
1692  */
1693 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1694 {
1695         struct hnae3_ae_dev *ae_dev;
1696         int ret;
1697
1698         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1699                               GFP_KERNEL);
1700         if (!ae_dev) {
1701                 ret = -ENOMEM;
1702                 return ret;
1703         }
1704
1705         ae_dev->pdev = pdev;
1706         ae_dev->flag = ent->driver_data;
1707         ae_dev->dev_type = HNAE3_DEV_KNIC;
1708         ae_dev->reset_type = HNAE3_NONE_RESET;
1709         hns3_get_dev_capability(pdev, ae_dev);
1710         pci_set_drvdata(pdev, ae_dev);
1711
1712         hnae3_register_ae_dev(ae_dev);
1713
1714         return 0;
1715 }
1716
1717 /* hns3_remove - Device removal routine
1718  * @pdev: PCI device information struct
1719  */
1720 static void hns3_remove(struct pci_dev *pdev)
1721 {
1722         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1723
1724         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1725                 hns3_disable_sriov(pdev);
1726
1727         hnae3_unregister_ae_dev(ae_dev);
1728 }
1729
1730 /**
1731  * hns3_pci_sriov_configure
1732  * @pdev: pointer to a pci_dev structure
1733  * @num_vfs: number of VFs to allocate
1734  *
1735  * Enable or change the number of VFs. Called when the user updates the number
1736  * of VFs in sysfs.
1737  **/
1738 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1739 {
1740         int ret;
1741
1742         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1743                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1744                 return -EINVAL;
1745         }
1746
1747         if (num_vfs) {
1748                 ret = pci_enable_sriov(pdev, num_vfs);
1749                 if (ret)
1750                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1751                 else
1752                         return num_vfs;
1753         } else if (!pci_vfs_assigned(pdev)) {
1754                 pci_disable_sriov(pdev);
1755         } else {
1756                 dev_warn(&pdev->dev,
1757                          "Unable to free VFs because some are assigned to VMs.\n");
1758         }
1759
1760         return 0;
1761 }
1762
1763 static void hns3_shutdown(struct pci_dev *pdev)
1764 {
1765         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1766
1767         hnae3_unregister_ae_dev(ae_dev);
1768         devm_kfree(&pdev->dev, ae_dev);
1769         pci_set_drvdata(pdev, NULL);
1770
1771         if (system_state == SYSTEM_POWER_OFF)
1772                 pci_set_power_state(pdev, PCI_D3hot);
1773 }
1774
1775 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1776                                             pci_channel_state_t state)
1777 {
1778         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1779         pci_ers_result_t ret;
1780
1781         dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1782
1783         if (state == pci_channel_io_perm_failure)
1784                 return PCI_ERS_RESULT_DISCONNECT;
1785
1786         if (!ae_dev) {
1787                 dev_err(&pdev->dev,
1788                         "Can't recover - error happened during device init\n");
1789                 return PCI_ERS_RESULT_NONE;
1790         }
1791
1792         if (ae_dev->ops->process_hw_error)
1793                 ret = ae_dev->ops->process_hw_error(ae_dev);
1794         else
1795                 return PCI_ERS_RESULT_NONE;
1796
1797         return ret;
1798 }
1799
1800 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1801 {
1802         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1803         struct device *dev = &pdev->dev;
1804
1805         dev_info(dev, "requesting reset due to PCI error\n");
1806
1807         /* request the reset */
1808         if (ae_dev->ops->reset_event) {
1809                 ae_dev->ops->reset_event(pdev, NULL);
1810                 return PCI_ERS_RESULT_RECOVERED;
1811         }
1812
1813         return PCI_ERS_RESULT_DISCONNECT;
1814 }
1815
1816 static const struct pci_error_handlers hns3_err_handler = {
1817         .error_detected = hns3_error_detected,
1818         .slot_reset     = hns3_slot_reset,
1819 };
1820
1821 static struct pci_driver hns3_driver = {
1822         .name     = hns3_driver_name,
1823         .id_table = hns3_pci_tbl,
1824         .probe    = hns3_probe,
1825         .remove   = hns3_remove,
1826         .shutdown = hns3_shutdown,
1827         .sriov_configure = hns3_pci_sriov_configure,
1828         .err_handler    = &hns3_err_handler,
1829 };
1830
1831 /* set default feature to hns3 */
1832 static void hns3_set_default_feature(struct net_device *netdev)
1833 {
1834         struct hnae3_handle *h = hns3_get_handle(netdev);
1835         struct pci_dev *pdev = h->pdev;
1836
1837         netdev->priv_flags |= IFF_UNICAST_FLT;
1838
1839         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1840                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1841                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1842                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1843                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1844
1845         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1846
1847         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1848
1849         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1850                 NETIF_F_HW_VLAN_CTAG_FILTER |
1851                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1852                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1853                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1854                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1855                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1856
1857         netdev->vlan_features |=
1858                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1859                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1860                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1861                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1862                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1863
1864         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1865                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1866                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1867                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1868                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1869                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1870
1871         if (pdev->revision >= 0x21) {
1872                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1873
1874                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1875                         netdev->hw_features |= NETIF_F_NTUPLE;
1876                         netdev->features |= NETIF_F_NTUPLE;
1877                 }
1878         }
1879 }
1880
1881 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1882                              struct hns3_desc_cb *cb)
1883 {
1884         unsigned int order = hnae3_page_order(ring);
1885         struct page *p;
1886
1887         p = dev_alloc_pages(order);
1888         if (!p)
1889                 return -ENOMEM;
1890
1891         cb->priv = p;
1892         cb->page_offset = 0;
1893         cb->reuse_flag = 0;
1894         cb->buf  = page_address(p);
1895         cb->length = hnae3_page_size(ring);
1896         cb->type = DESC_TYPE_PAGE;
1897
1898         return 0;
1899 }
1900
1901 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1902                              struct hns3_desc_cb *cb)
1903 {
1904         if (cb->type == DESC_TYPE_SKB)
1905                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1906         else if (!HNAE3_IS_TX_RING(ring))
1907                 put_page((struct page *)cb->priv);
1908         memset(cb, 0, sizeof(*cb));
1909 }
1910
1911 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1912 {
1913         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1914                                cb->length, ring_to_dma_dir(ring));
1915
1916         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
1917                 return -EIO;
1918
1919         return 0;
1920 }
1921
1922 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1923                               struct hns3_desc_cb *cb)
1924 {
1925         if (cb->type == DESC_TYPE_SKB)
1926                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1927                                  ring_to_dma_dir(ring));
1928         else if (cb->length)
1929                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1930                                ring_to_dma_dir(ring));
1931 }
1932
1933 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1934 {
1935         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1936         ring->desc[i].addr = 0;
1937 }
1938
1939 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1940 {
1941         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1942
1943         if (!ring->desc_cb[i].dma)
1944                 return;
1945
1946         hns3_buffer_detach(ring, i);
1947         hns3_free_buffer(ring, cb);
1948 }
1949
1950 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1951 {
1952         int i;
1953
1954         for (i = 0; i < ring->desc_num; i++)
1955                 hns3_free_buffer_detach(ring, i);
1956 }
1957
1958 /* free desc along with its attached buffer */
1959 static void hns3_free_desc(struct hns3_enet_ring *ring)
1960 {
1961         int size = ring->desc_num * sizeof(ring->desc[0]);
1962
1963         hns3_free_buffers(ring);
1964
1965         if (ring->desc) {
1966                 dma_free_coherent(ring_to_dev(ring), size,
1967                                   ring->desc, ring->desc_dma_addr);
1968                 ring->desc = NULL;
1969         }
1970 }
1971
1972 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1973 {
1974         int size = ring->desc_num * sizeof(ring->desc[0]);
1975
1976         ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
1977                                          &ring->desc_dma_addr,
1978                                          GFP_KERNEL);
1979         if (!ring->desc)
1980                 return -ENOMEM;
1981
1982         return 0;
1983 }
1984
1985 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1986                                    struct hns3_desc_cb *cb)
1987 {
1988         int ret;
1989
1990         ret = hns3_alloc_buffer(ring, cb);
1991         if (ret)
1992                 goto out;
1993
1994         ret = hns3_map_buffer(ring, cb);
1995         if (ret)
1996                 goto out_with_buf;
1997
1998         return 0;
1999
2000 out_with_buf:
2001         hns3_free_buffer(ring, cb);
2002 out:
2003         return ret;
2004 }
2005
2006 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2007 {
2008         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2009
2010         if (ret)
2011                 return ret;
2012
2013         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2014
2015         return 0;
2016 }
2017
2018 /* Allocate memory for raw pkg, and map with dma */
2019 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2020 {
2021         int i, j, ret;
2022
2023         for (i = 0; i < ring->desc_num; i++) {
2024                 ret = hns3_alloc_buffer_attach(ring, i);
2025                 if (ret)
2026                         goto out_buffer_fail;
2027         }
2028
2029         return 0;
2030
2031 out_buffer_fail:
2032         for (j = i - 1; j >= 0; j--)
2033                 hns3_free_buffer_detach(ring, j);
2034         return ret;
2035 }
2036
2037 /* detach a in-used buffer and replace with a reserved one  */
2038 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2039                                 struct hns3_desc_cb *res_cb)
2040 {
2041         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2042         ring->desc_cb[i] = *res_cb;
2043         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2044         ring->desc[i].rx.bd_base_info = 0;
2045 }
2046
2047 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2048 {
2049         ring->desc_cb[i].reuse_flag = 0;
2050         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2051                 + ring->desc_cb[i].page_offset);
2052         ring->desc[i].rx.bd_base_info = 0;
2053 }
2054
2055 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2056                                       int *pkts)
2057 {
2058         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2059
2060         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2061         (*bytes) += desc_cb->length;
2062         /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
2063         hns3_free_buffer_detach(ring, ring->next_to_clean);
2064
2065         ring_ptr_move_fw(ring, next_to_clean);
2066 }
2067
2068 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2069 {
2070         int u = ring->next_to_use;
2071         int c = ring->next_to_clean;
2072
2073         if (unlikely(h > ring->desc_num))
2074                 return 0;
2075
2076         return u > c ? (h > c && h <= u) : (h > c || h <= u);
2077 }
2078
2079 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2080 {
2081         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2082         struct hns3_nic_priv *priv = netdev_priv(netdev);
2083         struct netdev_queue *dev_queue;
2084         int bytes, pkts;
2085         int head;
2086
2087         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2088         rmb(); /* Make sure head is ready before touch any data */
2089
2090         if (is_ring_empty(ring) || head == ring->next_to_clean)
2091                 return; /* no data to poll */
2092
2093         if (unlikely(!is_valid_clean_head(ring, head))) {
2094                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2095                            ring->next_to_use, ring->next_to_clean);
2096
2097                 u64_stats_update_begin(&ring->syncp);
2098                 ring->stats.io_err_cnt++;
2099                 u64_stats_update_end(&ring->syncp);
2100                 return;
2101         }
2102
2103         bytes = 0;
2104         pkts = 0;
2105         while (head != ring->next_to_clean) {
2106                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2107                 /* Issue prefetch for next Tx descriptor */
2108                 prefetch(&ring->desc_cb[ring->next_to_clean]);
2109         }
2110
2111         ring->tqp_vector->tx_group.total_bytes += bytes;
2112         ring->tqp_vector->tx_group.total_packets += pkts;
2113
2114         u64_stats_update_begin(&ring->syncp);
2115         ring->stats.tx_bytes += bytes;
2116         ring->stats.tx_pkts += pkts;
2117         u64_stats_update_end(&ring->syncp);
2118
2119         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2120         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2121
2122         if (unlikely(pkts && netif_carrier_ok(netdev) &&
2123                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2124                 /* Make sure that anybody stopping the queue after this
2125                  * sees the new next_to_clean.
2126                  */
2127                 smp_mb();
2128                 if (netif_tx_queue_stopped(dev_queue) &&
2129                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2130                         netif_tx_wake_queue(dev_queue);
2131                         ring->stats.restart_queue++;
2132                 }
2133         }
2134 }
2135
2136 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2137 {
2138         int ntc = ring->next_to_clean;
2139         int ntu = ring->next_to_use;
2140
2141         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2142 }
2143
2144 static void
2145 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2146 {
2147         struct hns3_desc_cb *desc_cb;
2148         struct hns3_desc_cb res_cbs;
2149         int i, ret;
2150
2151         for (i = 0; i < cleand_count; i++) {
2152                 desc_cb = &ring->desc_cb[ring->next_to_use];
2153                 if (desc_cb->reuse_flag) {
2154                         u64_stats_update_begin(&ring->syncp);
2155                         ring->stats.reuse_pg_cnt++;
2156                         u64_stats_update_end(&ring->syncp);
2157
2158                         hns3_reuse_buffer(ring, ring->next_to_use);
2159                 } else {
2160                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
2161                         if (ret) {
2162                                 u64_stats_update_begin(&ring->syncp);
2163                                 ring->stats.sw_err_cnt++;
2164                                 u64_stats_update_end(&ring->syncp);
2165
2166                                 netdev_err(ring->tqp->handle->kinfo.netdev,
2167                                            "hnae reserve buffer map failed.\n");
2168                                 break;
2169                         }
2170                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2171                 }
2172
2173                 ring_ptr_move_fw(ring, next_to_use);
2174         }
2175
2176         wmb(); /* Make all data has been write before submit */
2177         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2178 }
2179
2180 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2181                                 struct hns3_enet_ring *ring, int pull_len,
2182                                 struct hns3_desc_cb *desc_cb)
2183 {
2184         struct hns3_desc *desc;
2185         u32 truesize;
2186         int size;
2187         int last_offset;
2188         bool twobufs;
2189
2190         twobufs = ((PAGE_SIZE < 8192) &&
2191                 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2192
2193         desc = &ring->desc[ring->next_to_clean];
2194         size = le16_to_cpu(desc->rx.size);
2195
2196         truesize = hnae3_buf_size(ring);
2197
2198         if (!twobufs)
2199                 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2200
2201         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2202                         size - pull_len, truesize);
2203
2204          /* Avoid re-using remote pages,flag default unreuse */
2205         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2206                 return;
2207
2208         if (twobufs) {
2209                 /* If we are only owner of page we can reuse it */
2210                 if (likely(page_count(desc_cb->priv) == 1)) {
2211                         /* Flip page offset to other buffer */
2212                         desc_cb->page_offset ^= truesize;
2213
2214                         desc_cb->reuse_flag = 1;
2215                         /* bump ref count on page before it is given*/
2216                         get_page(desc_cb->priv);
2217                 }
2218                 return;
2219         }
2220
2221         /* Move offset up to the next cache line */
2222         desc_cb->page_offset += truesize;
2223
2224         if (desc_cb->page_offset <= last_offset) {
2225                 desc_cb->reuse_flag = 1;
2226                 /* Bump ref count on page before it is given*/
2227                 get_page(desc_cb->priv);
2228         }
2229 }
2230
2231 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2232                              struct hns3_desc *desc)
2233 {
2234         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2235         int l3_type, l4_type;
2236         u32 bd_base_info;
2237         int ol4_type;
2238         u32 l234info;
2239
2240         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2241         l234info = le32_to_cpu(desc->rx.l234_info);
2242
2243         skb->ip_summed = CHECKSUM_NONE;
2244
2245         skb_checksum_none_assert(skb);
2246
2247         if (!(netdev->features & NETIF_F_RXCSUM))
2248                 return;
2249
2250         /* check if hardware has done checksum */
2251         if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2252                 return;
2253
2254         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2255                      hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2256                      hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2257                      hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2258                 u64_stats_update_begin(&ring->syncp);
2259                 ring->stats.l3l4_csum_err++;
2260                 u64_stats_update_end(&ring->syncp);
2261
2262                 return;
2263         }
2264
2265         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2266                                   HNS3_RXD_L3ID_S);
2267         l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2268                                   HNS3_RXD_L4ID_S);
2269
2270         ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2271                                    HNS3_RXD_OL4ID_S);
2272         switch (ol4_type) {
2273         case HNS3_OL4_TYPE_MAC_IN_UDP:
2274         case HNS3_OL4_TYPE_NVGRE:
2275                 skb->csum_level = 1;
2276                 /* fall through */
2277         case HNS3_OL4_TYPE_NO_TUN:
2278                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2279                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2280                      l3_type == HNS3_L3_TYPE_IPV6) &&
2281                     (l4_type == HNS3_L4_TYPE_UDP ||
2282                      l4_type == HNS3_L4_TYPE_TCP ||
2283                      l4_type == HNS3_L4_TYPE_SCTP))
2284                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2285                 break;
2286         default:
2287                 break;
2288         }
2289 }
2290
2291 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2292 {
2293         napi_gro_receive(&ring->tqp_vector->napi, skb);
2294 }
2295
2296 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2297                                 struct hns3_desc *desc, u32 l234info,
2298                                 u16 *vlan_tag)
2299 {
2300         struct pci_dev *pdev = ring->tqp->handle->pdev;
2301
2302         if (pdev->revision == 0x20) {
2303                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2304                 if (!(*vlan_tag & VLAN_VID_MASK))
2305                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2306
2307                 return (*vlan_tag != 0);
2308         }
2309
2310 #define HNS3_STRP_OUTER_VLAN    0x1
2311 #define HNS3_STRP_INNER_VLAN    0x2
2312
2313         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2314                                 HNS3_RXD_STRP_TAGP_S)) {
2315         case HNS3_STRP_OUTER_VLAN:
2316                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2317                 return true;
2318         case HNS3_STRP_INNER_VLAN:
2319                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2320                 return true;
2321         default:
2322                 return false;
2323         }
2324 }
2325
2326 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2327                                      struct sk_buff *skb)
2328 {
2329         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2330         struct hnae3_handle *handle = ring->tqp->handle;
2331         enum pkt_hash_types rss_type;
2332
2333         if (le32_to_cpu(desc->rx.rss_hash))
2334                 rss_type = handle->kinfo.rss_type;
2335         else
2336                 rss_type = PKT_HASH_TYPE_NONE;
2337
2338         skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2339 }
2340
2341 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2342                              struct sk_buff **out_skb, int *out_bnum)
2343 {
2344         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2345         struct hns3_desc_cb *desc_cb;
2346         struct hns3_desc *desc;
2347         struct sk_buff *skb;
2348         unsigned char *va;
2349         u32 bd_base_info;
2350         int pull_len;
2351         u32 l234info;
2352         int length;
2353         int bnum;
2354
2355         desc = &ring->desc[ring->next_to_clean];
2356         desc_cb = &ring->desc_cb[ring->next_to_clean];
2357
2358         prefetch(desc);
2359
2360         length = le16_to_cpu(desc->rx.size);
2361         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2362
2363         /* Check valid BD */
2364         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2365                 return -EFAULT;
2366
2367         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2368
2369         /* Prefetch first cache line of first page
2370          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2371          * line size is 64B so need to prefetch twice to make it 128B. But in
2372          * actual we can have greater size of caches with 128B Level 1 cache
2373          * lines. In such a case, single fetch would suffice to cache in the
2374          * relevant part of the header.
2375          */
2376         prefetch(va);
2377 #if L1_CACHE_BYTES < 128
2378         prefetch(va + L1_CACHE_BYTES);
2379 #endif
2380
2381         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2382                                         HNS3_RX_HEAD_SIZE);
2383         if (unlikely(!skb)) {
2384                 netdev_err(netdev, "alloc rx skb fail\n");
2385
2386                 u64_stats_update_begin(&ring->syncp);
2387                 ring->stats.sw_err_cnt++;
2388                 u64_stats_update_end(&ring->syncp);
2389
2390                 return -ENOMEM;
2391         }
2392
2393         prefetchw(skb->data);
2394
2395         bnum = 1;
2396         if (length <= HNS3_RX_HEAD_SIZE) {
2397                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2398
2399                 /* We can reuse buffer as-is, just make sure it is local */
2400                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2401                         desc_cb->reuse_flag = 1;
2402                 else /* This page cannot be reused so discard it */
2403                         put_page(desc_cb->priv);
2404
2405                 ring_ptr_move_fw(ring, next_to_clean);
2406         } else {
2407                 u64_stats_update_begin(&ring->syncp);
2408                 ring->stats.seg_pkt_cnt++;
2409                 u64_stats_update_end(&ring->syncp);
2410
2411                 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2412
2413                 memcpy(__skb_put(skb, pull_len), va,
2414                        ALIGN(pull_len, sizeof(long)));
2415
2416                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2417                 ring_ptr_move_fw(ring, next_to_clean);
2418
2419                 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2420                         desc = &ring->desc[ring->next_to_clean];
2421                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2422                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2423                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2424                         ring_ptr_move_fw(ring, next_to_clean);
2425                         bnum++;
2426                 }
2427         }
2428
2429         *out_bnum = bnum;
2430
2431         l234info = le32_to_cpu(desc->rx.l234_info);
2432
2433         /* Based on hw strategy, the tag offloaded will be stored at
2434          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2435          * in one layer tag case.
2436          */
2437         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2438                 u16 vlan_tag;
2439
2440                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2441                         __vlan_hwaccel_put_tag(skb,
2442                                                htons(ETH_P_8021Q),
2443                                                vlan_tag);
2444         }
2445
2446         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2447                 u64_stats_update_begin(&ring->syncp);
2448                 ring->stats.non_vld_descs++;
2449                 u64_stats_update_end(&ring->syncp);
2450
2451                 dev_kfree_skb_any(skb);
2452                 return -EINVAL;
2453         }
2454
2455         if (unlikely((!desc->rx.pkt_len) ||
2456                      hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2457                 u64_stats_update_begin(&ring->syncp);
2458                 ring->stats.err_pkt_len++;
2459                 u64_stats_update_end(&ring->syncp);
2460
2461                 dev_kfree_skb_any(skb);
2462                 return -EFAULT;
2463         }
2464
2465         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2466                 u64_stats_update_begin(&ring->syncp);
2467                 ring->stats.l2_err++;
2468                 u64_stats_update_end(&ring->syncp);
2469
2470                 dev_kfree_skb_any(skb);
2471                 return -EFAULT;
2472         }
2473
2474         u64_stats_update_begin(&ring->syncp);
2475         ring->stats.rx_pkts++;
2476         ring->stats.rx_bytes += skb->len;
2477         u64_stats_update_end(&ring->syncp);
2478
2479         ring->tqp_vector->rx_group.total_bytes += skb->len;
2480
2481         hns3_rx_checksum(ring, skb, desc);
2482         hns3_set_rx_skb_rss_type(ring, skb);
2483
2484         return 0;
2485 }
2486
2487 int hns3_clean_rx_ring(
2488                 struct hns3_enet_ring *ring, int budget,
2489                 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2490 {
2491 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2492         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2493         int recv_pkts, recv_bds, clean_count, err;
2494         int unused_count = hns3_desc_unused(ring);
2495         struct sk_buff *skb = NULL;
2496         int num, bnum = 0;
2497
2498         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2499         rmb(); /* Make sure num taken effect before the other data is touched */
2500
2501         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2502         num -= unused_count;
2503
2504         while (recv_pkts < budget && recv_bds < num) {
2505                 /* Reuse or realloc buffers */
2506                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2507                         hns3_nic_alloc_rx_buffers(ring,
2508                                                   clean_count + unused_count);
2509                         clean_count = 0;
2510                         unused_count = hns3_desc_unused(ring);
2511                 }
2512
2513                 /* Poll one pkt */
2514                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2515                 if (unlikely(!skb)) /* This fault cannot be repaired */
2516                         goto out;
2517
2518                 recv_bds += bnum;
2519                 clean_count += bnum;
2520                 if (unlikely(err)) {  /* Do jump the err */
2521                         recv_pkts++;
2522                         continue;
2523                 }
2524
2525                 /* Do update ip stack process */
2526                 skb->protocol = eth_type_trans(skb, netdev);
2527                 rx_fn(ring, skb);
2528
2529                 recv_pkts++;
2530         }
2531
2532 out:
2533         /* Make all data has been write before submit */
2534         if (clean_count + unused_count > 0)
2535                 hns3_nic_alloc_rx_buffers(ring,
2536                                           clean_count + unused_count);
2537
2538         return recv_pkts;
2539 }
2540
2541 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2542 {
2543         struct hns3_enet_tqp_vector *tqp_vector =
2544                                         ring_group->ring->tqp_vector;
2545         enum hns3_flow_level_range new_flow_level;
2546         int packets_per_msecs;
2547         int bytes_per_msecs;
2548         u32 time_passed_ms;
2549         u16 new_int_gl;
2550
2551         if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
2552                 return false;
2553
2554         if (ring_group->total_packets == 0) {
2555                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2556                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
2557                 return true;
2558         }
2559
2560         /* Simple throttlerate management
2561          * 0-10MB/s   lower     (50000 ints/s)
2562          * 10-20MB/s   middle    (20000 ints/s)
2563          * 20-1249MB/s high      (18000 ints/s)
2564          * > 40000pps  ultra     (8000 ints/s)
2565          */
2566         new_flow_level = ring_group->coal.flow_level;
2567         new_int_gl = ring_group->coal.int_gl;
2568         time_passed_ms =
2569                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2570
2571         if (!time_passed_ms)
2572                 return false;
2573
2574         do_div(ring_group->total_packets, time_passed_ms);
2575         packets_per_msecs = ring_group->total_packets;
2576
2577         do_div(ring_group->total_bytes, time_passed_ms);
2578         bytes_per_msecs = ring_group->total_bytes;
2579
2580 #define HNS3_RX_LOW_BYTE_RATE 10000
2581 #define HNS3_RX_MID_BYTE_RATE 20000
2582
2583         switch (new_flow_level) {
2584         case HNS3_FLOW_LOW:
2585                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2586                         new_flow_level = HNS3_FLOW_MID;
2587                 break;
2588         case HNS3_FLOW_MID:
2589                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2590                         new_flow_level = HNS3_FLOW_HIGH;
2591                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2592                         new_flow_level = HNS3_FLOW_LOW;
2593                 break;
2594         case HNS3_FLOW_HIGH:
2595         case HNS3_FLOW_ULTRA:
2596         default:
2597                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2598                         new_flow_level = HNS3_FLOW_MID;
2599                 break;
2600         }
2601
2602 #define HNS3_RX_ULTRA_PACKET_RATE 40
2603
2604         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2605             &tqp_vector->rx_group == ring_group)
2606                 new_flow_level = HNS3_FLOW_ULTRA;
2607
2608         switch (new_flow_level) {
2609         case HNS3_FLOW_LOW:
2610                 new_int_gl = HNS3_INT_GL_50K;
2611                 break;
2612         case HNS3_FLOW_MID:
2613                 new_int_gl = HNS3_INT_GL_20K;
2614                 break;
2615         case HNS3_FLOW_HIGH:
2616                 new_int_gl = HNS3_INT_GL_18K;
2617                 break;
2618         case HNS3_FLOW_ULTRA:
2619                 new_int_gl = HNS3_INT_GL_8K;
2620                 break;
2621         default:
2622                 break;
2623         }
2624
2625         ring_group->total_bytes = 0;
2626         ring_group->total_packets = 0;
2627         ring_group->coal.flow_level = new_flow_level;
2628         if (new_int_gl != ring_group->coal.int_gl) {
2629                 ring_group->coal.int_gl = new_int_gl;
2630                 return true;
2631         }
2632         return false;
2633 }
2634
2635 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2636 {
2637         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2638         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2639         bool rx_update, tx_update;
2640
2641         if (tqp_vector->int_adapt_down > 0) {
2642                 tqp_vector->int_adapt_down--;
2643                 return;
2644         }
2645
2646         if (rx_group->coal.gl_adapt_enable) {
2647                 rx_update = hns3_get_new_int_gl(rx_group);
2648                 if (rx_update)
2649                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
2650                                                        rx_group->coal.int_gl);
2651         }
2652
2653         if (tx_group->coal.gl_adapt_enable) {
2654                 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2655                 if (tx_update)
2656                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
2657                                                        tx_group->coal.int_gl);
2658         }
2659
2660         tqp_vector->last_jiffies = jiffies;
2661         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
2662 }
2663
2664 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2665 {
2666         struct hns3_enet_ring *ring;
2667         int rx_pkt_total = 0;
2668
2669         struct hns3_enet_tqp_vector *tqp_vector =
2670                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2671         bool clean_complete = true;
2672         int rx_budget;
2673
2674         /* Since the actual Tx work is minimal, we can give the Tx a larger
2675          * budget and be more aggressive about cleaning up the Tx descriptors.
2676          */
2677         hns3_for_each_ring(ring, tqp_vector->tx_group)
2678                 hns3_clean_tx_ring(ring);
2679
2680         /* make sure rx ring budget not smaller than 1 */
2681         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2682
2683         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2684                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2685                                                     hns3_rx_skb);
2686
2687                 if (rx_cleaned >= rx_budget)
2688                         clean_complete = false;
2689
2690                 rx_pkt_total += rx_cleaned;
2691         }
2692
2693         tqp_vector->rx_group.total_packets += rx_pkt_total;
2694
2695         if (!clean_complete)
2696                 return budget;
2697
2698         napi_complete(napi);
2699         hns3_update_new_int_gl(tqp_vector);
2700         hns3_mask_vector_irq(tqp_vector, 1);
2701
2702         return rx_pkt_total;
2703 }
2704
2705 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2706                                       struct hnae3_ring_chain_node *head)
2707 {
2708         struct pci_dev *pdev = tqp_vector->handle->pdev;
2709         struct hnae3_ring_chain_node *cur_chain = head;
2710         struct hnae3_ring_chain_node *chain;
2711         struct hns3_enet_ring *tx_ring;
2712         struct hns3_enet_ring *rx_ring;
2713
2714         tx_ring = tqp_vector->tx_group.ring;
2715         if (tx_ring) {
2716                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2717                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2718                               HNAE3_RING_TYPE_TX);
2719                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2720                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2721
2722                 cur_chain->next = NULL;
2723
2724                 while (tx_ring->next) {
2725                         tx_ring = tx_ring->next;
2726
2727                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2728                                              GFP_KERNEL);
2729                         if (!chain)
2730                                 return -ENOMEM;
2731
2732                         cur_chain->next = chain;
2733                         chain->tqp_index = tx_ring->tqp->tqp_index;
2734                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2735                                       HNAE3_RING_TYPE_TX);
2736                         hnae3_set_field(chain->int_gl_idx,
2737                                         HNAE3_RING_GL_IDX_M,
2738                                         HNAE3_RING_GL_IDX_S,
2739                                         HNAE3_RING_GL_TX);
2740
2741                         cur_chain = chain;
2742                 }
2743         }
2744
2745         rx_ring = tqp_vector->rx_group.ring;
2746         if (!tx_ring && rx_ring) {
2747                 cur_chain->next = NULL;
2748                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2749                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2750                               HNAE3_RING_TYPE_RX);
2751                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2752                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2753
2754                 rx_ring = rx_ring->next;
2755         }
2756
2757         while (rx_ring) {
2758                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2759                 if (!chain)
2760                         return -ENOMEM;
2761
2762                 cur_chain->next = chain;
2763                 chain->tqp_index = rx_ring->tqp->tqp_index;
2764                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2765                               HNAE3_RING_TYPE_RX);
2766                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2767                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2768
2769                 cur_chain = chain;
2770
2771                 rx_ring = rx_ring->next;
2772         }
2773
2774         return 0;
2775 }
2776
2777 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2778                                         struct hnae3_ring_chain_node *head)
2779 {
2780         struct pci_dev *pdev = tqp_vector->handle->pdev;
2781         struct hnae3_ring_chain_node *chain_tmp, *chain;
2782
2783         chain = head->next;
2784
2785         while (chain) {
2786                 chain_tmp = chain->next;
2787                 devm_kfree(&pdev->dev, chain);
2788                 chain = chain_tmp;
2789         }
2790 }
2791
2792 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2793                                    struct hns3_enet_ring *ring)
2794 {
2795         ring->next = group->ring;
2796         group->ring = ring;
2797
2798         group->count++;
2799 }
2800
2801 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2802 {
2803         struct pci_dev *pdev = priv->ae_handle->pdev;
2804         struct hns3_enet_tqp_vector *tqp_vector;
2805         int num_vectors = priv->vector_num;
2806         int numa_node;
2807         int vector_i;
2808
2809         numa_node = dev_to_node(&pdev->dev);
2810
2811         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2812                 tqp_vector = &priv->tqp_vector[vector_i];
2813                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2814                                 &tqp_vector->affinity_mask);
2815         }
2816 }
2817
2818 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2819 {
2820         struct hnae3_ring_chain_node vector_ring_chain;
2821         struct hnae3_handle *h = priv->ae_handle;
2822         struct hns3_enet_tqp_vector *tqp_vector;
2823         int ret = 0;
2824         u16 i;
2825
2826         hns3_nic_set_cpumask(priv);
2827
2828         for (i = 0; i < priv->vector_num; i++) {
2829                 tqp_vector = &priv->tqp_vector[i];
2830                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2831                 tqp_vector->num_tqps = 0;
2832         }
2833
2834         for (i = 0; i < h->kinfo.num_tqps; i++) {
2835                 u16 vector_i = i % priv->vector_num;
2836                 u16 tqp_num = h->kinfo.num_tqps;
2837
2838                 tqp_vector = &priv->tqp_vector[vector_i];
2839
2840                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2841                                        priv->ring_data[i].ring);
2842
2843                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2844                                        priv->ring_data[i + tqp_num].ring);
2845
2846                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2847                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2848                 tqp_vector->num_tqps++;
2849         }
2850
2851         for (i = 0; i < priv->vector_num; i++) {
2852                 tqp_vector = &priv->tqp_vector[i];
2853
2854                 tqp_vector->rx_group.total_bytes = 0;
2855                 tqp_vector->rx_group.total_packets = 0;
2856                 tqp_vector->tx_group.total_bytes = 0;
2857                 tqp_vector->tx_group.total_packets = 0;
2858                 tqp_vector->handle = h;
2859
2860                 ret = hns3_get_vector_ring_chain(tqp_vector,
2861                                                  &vector_ring_chain);
2862                 if (ret)
2863                         return ret;
2864
2865                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2866                         tqp_vector->vector_irq, &vector_ring_chain);
2867
2868                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2869
2870                 if (ret)
2871                         return ret;
2872
2873                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2874                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2875         }
2876
2877         return 0;
2878 }
2879
2880 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2881 {
2882         struct hnae3_handle *h = priv->ae_handle;
2883         struct hns3_enet_tqp_vector *tqp_vector;
2884         struct hnae3_vector_info *vector;
2885         struct pci_dev *pdev = h->pdev;
2886         u16 tqp_num = h->kinfo.num_tqps;
2887         u16 vector_num;
2888         int ret = 0;
2889         u16 i;
2890
2891         /* RSS size, cpu online and vector_num should be the same */
2892         /* Should consider 2p/4p later */
2893         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2894         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2895                               GFP_KERNEL);
2896         if (!vector)
2897                 return -ENOMEM;
2898
2899         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2900
2901         priv->vector_num = vector_num;
2902         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2903                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2904                              GFP_KERNEL);
2905         if (!priv->tqp_vector) {
2906                 ret = -ENOMEM;
2907                 goto out;
2908         }
2909
2910         for (i = 0; i < priv->vector_num; i++) {
2911                 tqp_vector = &priv->tqp_vector[i];
2912                 tqp_vector->idx = i;
2913                 tqp_vector->mask_addr = vector[i].io_addr;
2914                 tqp_vector->vector_irq = vector[i].vector;
2915                 hns3_vector_gl_rl_init(tqp_vector, priv);
2916         }
2917
2918 out:
2919         devm_kfree(&pdev->dev, vector);
2920         return ret;
2921 }
2922
2923 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2924 {
2925         group->ring = NULL;
2926         group->count = 0;
2927 }
2928
2929 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2930 {
2931         struct hnae3_ring_chain_node vector_ring_chain;
2932         struct hnae3_handle *h = priv->ae_handle;
2933         struct hns3_enet_tqp_vector *tqp_vector;
2934         int i, ret;
2935
2936         for (i = 0; i < priv->vector_num; i++) {
2937                 tqp_vector = &priv->tqp_vector[i];
2938
2939                 ret = hns3_get_vector_ring_chain(tqp_vector,
2940                                                  &vector_ring_chain);
2941                 if (ret)
2942                         return ret;
2943
2944                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2945                         tqp_vector->vector_irq, &vector_ring_chain);
2946                 if (ret)
2947                         return ret;
2948
2949                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2950
2951                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2952                         (void)irq_set_affinity_hint(
2953                                 priv->tqp_vector[i].vector_irq,
2954                                                     NULL);
2955                         free_irq(priv->tqp_vector[i].vector_irq,
2956                                  &priv->tqp_vector[i]);
2957                 }
2958
2959                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2960                 hns3_clear_ring_group(&tqp_vector->rx_group);
2961                 hns3_clear_ring_group(&tqp_vector->tx_group);
2962                 netif_napi_del(&priv->tqp_vector[i].napi);
2963         }
2964
2965         return 0;
2966 }
2967
2968 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2969 {
2970         struct hnae3_handle *h = priv->ae_handle;
2971         struct pci_dev *pdev = h->pdev;
2972         int i, ret;
2973
2974         for (i = 0; i < priv->vector_num; i++) {
2975                 struct hns3_enet_tqp_vector *tqp_vector;
2976
2977                 tqp_vector = &priv->tqp_vector[i];
2978                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2979                 if (ret)
2980                         return ret;
2981         }
2982
2983         devm_kfree(&pdev->dev, priv->tqp_vector);
2984         return 0;
2985 }
2986
2987 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2988                              int ring_type)
2989 {
2990         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2991         int queue_num = priv->ae_handle->kinfo.num_tqps;
2992         struct pci_dev *pdev = priv->ae_handle->pdev;
2993         struct hns3_enet_ring *ring;
2994
2995         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2996         if (!ring)
2997                 return -ENOMEM;
2998
2999         if (ring_type == HNAE3_RING_TYPE_TX) {
3000                 ring_data[q->tqp_index].ring = ring;
3001                 ring_data[q->tqp_index].queue_index = q->tqp_index;
3002                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3003         } else {
3004                 ring_data[q->tqp_index + queue_num].ring = ring;
3005                 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
3006                 ring->io_base = q->io_base;
3007         }
3008
3009         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3010
3011         ring->tqp = q;
3012         ring->desc = NULL;
3013         ring->desc_cb = NULL;
3014         ring->dev = priv->dev;
3015         ring->desc_dma_addr = 0;
3016         ring->buf_size = q->buf_size;
3017         ring->desc_num = q->desc_num;
3018         ring->next_to_use = 0;
3019         ring->next_to_clean = 0;
3020
3021         return 0;
3022 }
3023
3024 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3025                               struct hns3_nic_priv *priv)
3026 {
3027         int ret;
3028
3029         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3030         if (ret)
3031                 return ret;
3032
3033         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3034         if (ret)
3035                 return ret;
3036
3037         return 0;
3038 }
3039
3040 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3041 {
3042         struct hnae3_handle *h = priv->ae_handle;
3043         struct pci_dev *pdev = h->pdev;
3044         int i, ret;
3045
3046         priv->ring_data =  devm_kzalloc(&pdev->dev,
3047                                         array3_size(h->kinfo.num_tqps,
3048                                                     sizeof(*priv->ring_data),
3049                                                     2),
3050                                         GFP_KERNEL);
3051         if (!priv->ring_data)
3052                 return -ENOMEM;
3053
3054         for (i = 0; i < h->kinfo.num_tqps; i++) {
3055                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3056                 if (ret)
3057                         goto err;
3058         }
3059
3060         return 0;
3061 err:
3062         devm_kfree(&pdev->dev, priv->ring_data);
3063         return ret;
3064 }
3065
3066 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3067 {
3068         struct hnae3_handle *h = priv->ae_handle;
3069         int i;
3070
3071         for (i = 0; i < h->kinfo.num_tqps; i++) {
3072                 devm_kfree(priv->dev, priv->ring_data[i].ring);
3073                 devm_kfree(priv->dev,
3074                            priv->ring_data[i + h->kinfo.num_tqps].ring);
3075         }
3076         devm_kfree(priv->dev, priv->ring_data);
3077 }
3078
3079 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3080 {
3081         int ret;
3082
3083         if (ring->desc_num <= 0 || ring->buf_size <= 0)
3084                 return -EINVAL;
3085
3086         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3087                                 GFP_KERNEL);
3088         if (!ring->desc_cb) {
3089                 ret = -ENOMEM;
3090                 goto out;
3091         }
3092
3093         ret = hns3_alloc_desc(ring);
3094         if (ret)
3095                 goto out_with_desc_cb;
3096
3097         if (!HNAE3_IS_TX_RING(ring)) {
3098                 ret = hns3_alloc_ring_buffers(ring);
3099                 if (ret)
3100                         goto out_with_desc;
3101         }
3102
3103         return 0;
3104
3105 out_with_desc:
3106         hns3_free_desc(ring);
3107 out_with_desc_cb:
3108         kfree(ring->desc_cb);
3109         ring->desc_cb = NULL;
3110 out:
3111         return ret;
3112 }
3113
3114 static void hns3_fini_ring(struct hns3_enet_ring *ring)
3115 {
3116         hns3_free_desc(ring);
3117         kfree(ring->desc_cb);
3118         ring->desc_cb = NULL;
3119         ring->next_to_clean = 0;
3120         ring->next_to_use = 0;
3121 }
3122
3123 static int hns3_buf_size2type(u32 buf_size)
3124 {
3125         int bd_size_type;
3126
3127         switch (buf_size) {
3128         case 512:
3129                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3130                 break;
3131         case 1024:
3132                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3133                 break;
3134         case 2048:
3135                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3136                 break;
3137         case 4096:
3138                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3139                 break;
3140         default:
3141                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3142         }
3143
3144         return bd_size_type;
3145 }
3146
3147 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3148 {
3149         dma_addr_t dma = ring->desc_dma_addr;
3150         struct hnae3_queue *q = ring->tqp;
3151