Merge branch '10GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/jkirsher/net...
[muen/linux.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright (c) 2016-2017 Hisilicon Limited.
3
4 #include <linux/dma-mapping.h>
5 #include <linux/etherdevice.h>
6 #include <linux/interrupt.h>
7 #include <linux/if_vlan.h>
8 #include <linux/ip.h>
9 #include <linux/ipv6.h>
10 #include <linux/module.h>
11 #include <linux/pci.h>
12 #include <linux/aer.h>
13 #include <linux/skbuff.h>
14 #include <linux/sctp.h>
15 #include <linux/vermagic.h>
16 #include <net/gre.h>
17 #include <net/pkt_cls.h>
18 #include <net/vxlan.h>
19
20 #include "hnae3.h"
21 #include "hns3_enet.h"
22
23 static void hns3_clear_all_ring(struct hnae3_handle *h);
24 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
25 static void hns3_remove_hw_addr(struct net_device *netdev);
26
27 static const char hns3_driver_name[] = "hns3";
28 const char hns3_driver_version[] = VERMAGIC_STRING;
29 static const char hns3_driver_string[] =
30                         "Hisilicon Ethernet Network Driver for Hip08 Family";
31 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
32 static struct hnae3_client client;
33
34 /* hns3_pci_tbl - PCI Device ID Table
35  *
36  * Last entry must be all 0s
37  *
38  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
39  *   Class, Class Mask, private data (not used) }
40  */
41 static const struct pci_device_id hns3_pci_tbl[] = {
42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
44         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
45          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
46         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
47          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
48         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
49          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
50         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
51          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
52         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
53          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
54         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
55         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
56          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
57         /* required last entry */
58         {0, }
59 };
60 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62 static irqreturn_t hns3_irq_handle(int irq, void *vector)
63 {
64         struct hns3_enet_tqp_vector *tqp_vector = vector;
65
66         napi_schedule(&tqp_vector->napi);
67
68         return IRQ_HANDLED;
69 }
70
71 /* This callback function is used to set affinity changes to the irq affinity
72  * masks when the irq_set_affinity_notifier function is used.
73  */
74 static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
75                                          const cpumask_t *mask)
76 {
77         struct hns3_enet_tqp_vector *tqp_vectors =
78                 container_of(notify, struct hns3_enet_tqp_vector,
79                              affinity_notify);
80
81         tqp_vectors->affinity_mask = *mask;
82 }
83
84 static void hns3_nic_irq_affinity_release(struct kref *ref)
85 {
86 }
87
88 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
89 {
90         struct hns3_enet_tqp_vector *tqp_vectors;
91         unsigned int i;
92
93         for (i = 0; i < priv->vector_num; i++) {
94                 tqp_vectors = &priv->tqp_vector[i];
95
96                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
97                         continue;
98
99                 /* clear the affinity notifier and affinity mask */
100                 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
101                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
102
103                 /* release the irq resource */
104                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
105                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
106         }
107 }
108
109 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
110 {
111         struct hns3_enet_tqp_vector *tqp_vectors;
112         int txrx_int_idx = 0;
113         int rx_int_idx = 0;
114         int tx_int_idx = 0;
115         unsigned int i;
116         int ret;
117
118         for (i = 0; i < priv->vector_num; i++) {
119                 tqp_vectors = &priv->tqp_vector[i];
120
121                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
122                         continue;
123
124                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
125                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
126                                  "%s-%s-%d", priv->netdev->name, "TxRx",
127                                  txrx_int_idx++);
128                         txrx_int_idx++;
129                 } else if (tqp_vectors->rx_group.ring) {
130                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
131                                  "%s-%s-%d", priv->netdev->name, "Rx",
132                                  rx_int_idx++);
133                 } else if (tqp_vectors->tx_group.ring) {
134                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
135                                  "%s-%s-%d", priv->netdev->name, "Tx",
136                                  tx_int_idx++);
137                 } else {
138                         /* Skip this unused q_vector */
139                         continue;
140                 }
141
142                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
143
144                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
145                                   tqp_vectors->name,
146                                        tqp_vectors);
147                 if (ret) {
148                         netdev_err(priv->netdev, "request irq(%d) fail\n",
149                                    tqp_vectors->vector_irq);
150                         return ret;
151                 }
152
153                 tqp_vectors->affinity_notify.notify =
154                                         hns3_nic_irq_affinity_notify;
155                 tqp_vectors->affinity_notify.release =
156                                         hns3_nic_irq_affinity_release;
157                 irq_set_affinity_notifier(tqp_vectors->vector_irq,
158                                           &tqp_vectors->affinity_notify);
159                 irq_set_affinity_hint(tqp_vectors->vector_irq,
160                                       &tqp_vectors->affinity_mask);
161
162                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
163         }
164
165         return 0;
166 }
167
168 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
169                                  u32 mask_en)
170 {
171         writel(mask_en, tqp_vector->mask_addr);
172 }
173
174 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
175 {
176         napi_enable(&tqp_vector->napi);
177
178         /* enable vector */
179         hns3_mask_vector_irq(tqp_vector, 1);
180 }
181
182 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
183 {
184         /* disable vector */
185         hns3_mask_vector_irq(tqp_vector, 0);
186
187         disable_irq(tqp_vector->vector_irq);
188         napi_disable(&tqp_vector->napi);
189 }
190
191 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
192                                  u32 rl_value)
193 {
194         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
195
196         /* this defines the configuration for RL (Interrupt Rate Limiter).
197          * Rl defines rate of interrupts i.e. number of interrupts-per-second
198          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
199          */
200
201         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
202             !tqp_vector->rx_group.coal.gl_adapt_enable)
203                 /* According to the hardware, the range of rl_reg is
204                  * 0-59 and the unit is 4.
205                  */
206                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
207
208         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
209 }
210
211 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
212                                     u32 gl_value)
213 {
214         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
215
216         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
217 }
218
219 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
220                                     u32 gl_value)
221 {
222         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
223
224         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
225 }
226
227 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
228                                    struct hns3_nic_priv *priv)
229 {
230         /* initialize the configuration for interrupt coalescing.
231          * 1. GL (Interrupt Gap Limiter)
232          * 2. RL (Interrupt Rate Limiter)
233          */
234
235         /* Default: enable interrupt coalescing self-adaptive and GL */
236         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
237         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
238
239         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
240         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
241
242         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
243         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
244         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
245 }
246
247 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
248                                       struct hns3_nic_priv *priv)
249 {
250         struct hnae3_handle *h = priv->ae_handle;
251
252         hns3_set_vector_coalesce_tx_gl(tqp_vector,
253                                        tqp_vector->tx_group.coal.int_gl);
254         hns3_set_vector_coalesce_rx_gl(tqp_vector,
255                                        tqp_vector->rx_group.coal.int_gl);
256         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
257 }
258
259 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
260 {
261         struct hnae3_handle *h = hns3_get_handle(netdev);
262         struct hnae3_knic_private_info *kinfo = &h->kinfo;
263         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
264         int i, ret;
265
266         if (kinfo->num_tc <= 1) {
267                 netdev_reset_tc(netdev);
268         } else {
269                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
270                 if (ret) {
271                         netdev_err(netdev,
272                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
273                         return ret;
274                 }
275
276                 for (i = 0; i < HNAE3_MAX_TC; i++) {
277                         if (!kinfo->tc_info[i].enable)
278                                 continue;
279
280                         netdev_set_tc_queue(netdev,
281                                             kinfo->tc_info[i].tc,
282                                             kinfo->tc_info[i].tqp_count,
283                                             kinfo->tc_info[i].tqp_offset);
284                 }
285         }
286
287         ret = netif_set_real_num_tx_queues(netdev, queue_size);
288         if (ret) {
289                 netdev_err(netdev,
290                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
291                            ret);
292                 return ret;
293         }
294
295         ret = netif_set_real_num_rx_queues(netdev, queue_size);
296         if (ret) {
297                 netdev_err(netdev,
298                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
299                 return ret;
300         }
301
302         return 0;
303 }
304
305 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
306 {
307         u16 alloc_tqps, max_rss_size, rss_size;
308
309         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
310         rss_size = alloc_tqps / h->kinfo.num_tc;
311
312         return min_t(u16, rss_size, max_rss_size);
313 }
314
315 static int hns3_nic_net_up(struct net_device *netdev)
316 {
317         struct hns3_nic_priv *priv = netdev_priv(netdev);
318         struct hnae3_handle *h = priv->ae_handle;
319         int i, j;
320         int ret;
321
322         ret = hns3_nic_reset_all_ring(h);
323         if (ret)
324                 return ret;
325
326         /* get irq resource for all vectors */
327         ret = hns3_nic_init_irq(priv);
328         if (ret) {
329                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
330                 return ret;
331         }
332
333         /* enable the vectors */
334         for (i = 0; i < priv->vector_num; i++)
335                 hns3_vector_enable(&priv->tqp_vector[i]);
336
337         /* start the ae_dev */
338         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
339         if (ret)
340                 goto out_start_err;
341
342         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
343
344         return 0;
345
346 out_start_err:
347         for (j = i - 1; j >= 0; j--)
348                 hns3_vector_disable(&priv->tqp_vector[j]);
349
350         hns3_nic_uninit_irq(priv);
351
352         return ret;
353 }
354
355 static int hns3_nic_net_open(struct net_device *netdev)
356 {
357         struct hns3_nic_priv *priv = netdev_priv(netdev);
358         struct hnae3_handle *h = hns3_get_handle(netdev);
359         struct hnae3_knic_private_info *kinfo;
360         int i, ret;
361
362         netif_carrier_off(netdev);
363
364         ret = hns3_nic_set_real_num_queue(netdev);
365         if (ret)
366                 return ret;
367
368         ret = hns3_nic_net_up(netdev);
369         if (ret) {
370                 netdev_err(netdev,
371                            "hns net up fail, ret=%d!\n", ret);
372                 return ret;
373         }
374
375         kinfo = &h->kinfo;
376         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
377                 netdev_set_prio_tc_map(netdev, i,
378                                        kinfo->prio_tc[i]);
379         }
380
381         priv->ae_handle->last_reset_time = jiffies;
382         return 0;
383 }
384
385 static void hns3_nic_net_down(struct net_device *netdev)
386 {
387         struct hns3_nic_priv *priv = netdev_priv(netdev);
388         const struct hnae3_ae_ops *ops;
389         int i;
390
391         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
392                 return;
393
394         /* disable vectors */
395         for (i = 0; i < priv->vector_num; i++)
396                 hns3_vector_disable(&priv->tqp_vector[i]);
397
398         /* stop ae_dev */
399         ops = priv->ae_handle->ae_algo->ops;
400         if (ops->stop)
401                 ops->stop(priv->ae_handle);
402
403         /* free irq resources */
404         hns3_nic_uninit_irq(priv);
405
406         hns3_clear_all_ring(priv->ae_handle);
407 }
408
409 static int hns3_nic_net_stop(struct net_device *netdev)
410 {
411         netif_tx_stop_all_queues(netdev);
412         netif_carrier_off(netdev);
413
414         hns3_nic_net_down(netdev);
415
416         return 0;
417 }
418
419 static int hns3_nic_uc_sync(struct net_device *netdev,
420                             const unsigned char *addr)
421 {
422         struct hnae3_handle *h = hns3_get_handle(netdev);
423
424         if (h->ae_algo->ops->add_uc_addr)
425                 return h->ae_algo->ops->add_uc_addr(h, addr);
426
427         return 0;
428 }
429
430 static int hns3_nic_uc_unsync(struct net_device *netdev,
431                               const unsigned char *addr)
432 {
433         struct hnae3_handle *h = hns3_get_handle(netdev);
434
435         if (h->ae_algo->ops->rm_uc_addr)
436                 return h->ae_algo->ops->rm_uc_addr(h, addr);
437
438         return 0;
439 }
440
441 static int hns3_nic_mc_sync(struct net_device *netdev,
442                             const unsigned char *addr)
443 {
444         struct hnae3_handle *h = hns3_get_handle(netdev);
445
446         if (h->ae_algo->ops->add_mc_addr)
447                 return h->ae_algo->ops->add_mc_addr(h, addr);
448
449         return 0;
450 }
451
452 static int hns3_nic_mc_unsync(struct net_device *netdev,
453                               const unsigned char *addr)
454 {
455         struct hnae3_handle *h = hns3_get_handle(netdev);
456
457         if (h->ae_algo->ops->rm_mc_addr)
458                 return h->ae_algo->ops->rm_mc_addr(h, addr);
459
460         return 0;
461 }
462
463 static u8 hns3_get_netdev_flags(struct net_device *netdev)
464 {
465         u8 flags = 0;
466
467         if (netdev->flags & IFF_PROMISC) {
468                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
469         } else {
470                 flags |= HNAE3_VLAN_FLTR;
471                 if (netdev->flags & IFF_ALLMULTI)
472                         flags |= HNAE3_USER_MPE;
473         }
474
475         return flags;
476 }
477
478 static void hns3_nic_set_rx_mode(struct net_device *netdev)
479 {
480         struct hnae3_handle *h = hns3_get_handle(netdev);
481         u8 new_flags;
482         int ret;
483
484         new_flags = hns3_get_netdev_flags(netdev);
485
486         ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
487         if (ret) {
488                 netdev_err(netdev, "sync uc address fail\n");
489                 if (ret == -ENOSPC)
490                         new_flags |= HNAE3_OVERFLOW_UPE;
491         }
492
493         if (netdev->flags & IFF_MULTICAST) {
494                 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
495                                     hns3_nic_mc_unsync);
496                 if (ret) {
497                         netdev_err(netdev, "sync mc address fail\n");
498                         if (ret == -ENOSPC)
499                                 new_flags |= HNAE3_OVERFLOW_MPE;
500                 }
501         }
502
503         hns3_update_promisc_mode(netdev, new_flags);
504         /* User mode Promisc mode enable and vlan filtering is disabled to
505          * let all packets in. MAC-VLAN Table overflow Promisc enabled and
506          * vlan fitering is enabled
507          */
508         hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
509         h->netdev_flags = new_flags;
510 }
511
512 int hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
513 {
514         struct hns3_nic_priv *priv = netdev_priv(netdev);
515         struct hnae3_handle *h = priv->ae_handle;
516
517         if (h->ae_algo->ops->set_promisc_mode) {
518                 return h->ae_algo->ops->set_promisc_mode(h,
519                                                 promisc_flags & HNAE3_UPE,
520                                                 promisc_flags & HNAE3_MPE);
521         }
522
523         return 0;
524 }
525
526 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
527 {
528         struct hns3_nic_priv *priv = netdev_priv(netdev);
529         struct hnae3_handle *h = priv->ae_handle;
530         bool last_state;
531
532         if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
533                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
534                 if (enable != last_state) {
535                         netdev_info(netdev,
536                                     "%s vlan filter\n",
537                                     enable ? "enable" : "disable");
538                         h->ae_algo->ops->enable_vlan_filter(h, enable);
539                 }
540         }
541 }
542
543 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
544                         u16 *mss, u32 *type_cs_vlan_tso)
545 {
546         u32 l4_offset, hdr_len;
547         union l3_hdr_info l3;
548         union l4_hdr_info l4;
549         u32 l4_paylen;
550         int ret;
551
552         if (!skb_is_gso(skb))
553                 return 0;
554
555         ret = skb_cow_head(skb, 0);
556         if (ret)
557                 return ret;
558
559         l3.hdr = skb_network_header(skb);
560         l4.hdr = skb_transport_header(skb);
561
562         /* Software should clear the IPv4's checksum field when tso is
563          * needed.
564          */
565         if (l3.v4->version == 4)
566                 l3.v4->check = 0;
567
568         /* tunnel packet.*/
569         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
570                                          SKB_GSO_GRE_CSUM |
571                                          SKB_GSO_UDP_TUNNEL |
572                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
573                 if ((!(skb_shinfo(skb)->gso_type &
574                     SKB_GSO_PARTIAL)) &&
575                     (skb_shinfo(skb)->gso_type &
576                     SKB_GSO_UDP_TUNNEL_CSUM)) {
577                         /* Software should clear the udp's checksum
578                          * field when tso is needed.
579                          */
580                         l4.udp->check = 0;
581                 }
582                 /* reset l3&l4 pointers from outer to inner headers */
583                 l3.hdr = skb_inner_network_header(skb);
584                 l4.hdr = skb_inner_transport_header(skb);
585
586                 /* Software should clear the IPv4's checksum field when
587                  * tso is needed.
588                  */
589                 if (l3.v4->version == 4)
590                         l3.v4->check = 0;
591         }
592
593         /* normal or tunnel packet*/
594         l4_offset = l4.hdr - skb->data;
595         hdr_len = (l4.tcp->doff * 4) + l4_offset;
596
597         /* remove payload length from inner pseudo checksum when tso*/
598         l4_paylen = skb->len - l4_offset;
599         csum_replace_by_diff(&l4.tcp->check,
600                              (__force __wsum)htonl(l4_paylen));
601
602         /* find the txbd field values */
603         *paylen = skb->len - hdr_len;
604         hnae3_set_bit(*type_cs_vlan_tso,
605                       HNS3_TXD_TSO_B, 1);
606
607         /* get MSS for TSO */
608         *mss = skb_shinfo(skb)->gso_size;
609
610         return 0;
611 }
612
613 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
614                                 u8 *il4_proto)
615 {
616         union {
617                 struct iphdr *v4;
618                 struct ipv6hdr *v6;
619                 unsigned char *hdr;
620         } l3;
621         unsigned char *l4_hdr;
622         unsigned char *exthdr;
623         u8 l4_proto_tmp;
624         __be16 frag_off;
625
626         /* find outer header point */
627         l3.hdr = skb_network_header(skb);
628         l4_hdr = skb_transport_header(skb);
629
630         if (skb->protocol == htons(ETH_P_IPV6)) {
631                 exthdr = l3.hdr + sizeof(*l3.v6);
632                 l4_proto_tmp = l3.v6->nexthdr;
633                 if (l4_hdr != exthdr)
634                         ipv6_skip_exthdr(skb, exthdr - skb->data,
635                                          &l4_proto_tmp, &frag_off);
636         } else if (skb->protocol == htons(ETH_P_IP)) {
637                 l4_proto_tmp = l3.v4->protocol;
638         } else {
639                 return -EINVAL;
640         }
641
642         *ol4_proto = l4_proto_tmp;
643
644         /* tunnel packet */
645         if (!skb->encapsulation) {
646                 *il4_proto = 0;
647                 return 0;
648         }
649
650         /* find inner header point */
651         l3.hdr = skb_inner_network_header(skb);
652         l4_hdr = skb_inner_transport_header(skb);
653
654         if (l3.v6->version == 6) {
655                 exthdr = l3.hdr + sizeof(*l3.v6);
656                 l4_proto_tmp = l3.v6->nexthdr;
657                 if (l4_hdr != exthdr)
658                         ipv6_skip_exthdr(skb, exthdr - skb->data,
659                                          &l4_proto_tmp, &frag_off);
660         } else if (l3.v4->version == 4) {
661                 l4_proto_tmp = l3.v4->protocol;
662         }
663
664         *il4_proto = l4_proto_tmp;
665
666         return 0;
667 }
668
669 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
670                                 u8 il4_proto, u32 *type_cs_vlan_tso,
671                                 u32 *ol_type_vlan_len_msec)
672 {
673         union {
674                 struct iphdr *v4;
675                 struct ipv6hdr *v6;
676                 unsigned char *hdr;
677         } l3;
678         union {
679                 struct tcphdr *tcp;
680                 struct udphdr *udp;
681                 struct gre_base_hdr *gre;
682                 unsigned char *hdr;
683         } l4;
684         unsigned char *l2_hdr;
685         u8 l4_proto = ol4_proto;
686         u32 ol2_len;
687         u32 ol3_len;
688         u32 ol4_len;
689         u32 l2_len;
690         u32 l3_len;
691
692         l3.hdr = skb_network_header(skb);
693         l4.hdr = skb_transport_header(skb);
694
695         /* compute L2 header size for normal packet, defined in 2 Bytes */
696         l2_len = l3.hdr - skb->data;
697         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
698                         HNS3_TXD_L2LEN_S, l2_len >> 1);
699
700         /* tunnel packet*/
701         if (skb->encapsulation) {
702                 /* compute OL2 header size, defined in 2 Bytes */
703                 ol2_len = l2_len;
704                 hnae3_set_field(*ol_type_vlan_len_msec,
705                                 HNS3_TXD_L2LEN_M,
706                                 HNS3_TXD_L2LEN_S, ol2_len >> 1);
707
708                 /* compute OL3 header size, defined in 4 Bytes */
709                 ol3_len = l4.hdr - l3.hdr;
710                 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
711                                 HNS3_TXD_L3LEN_S, ol3_len >> 2);
712
713                 /* MAC in UDP, MAC in GRE (0x6558)*/
714                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
715                         /* switch MAC header ptr from outer to inner header.*/
716                         l2_hdr = skb_inner_mac_header(skb);
717
718                         /* compute OL4 header size, defined in 4 Bytes. */
719                         ol4_len = l2_hdr - l4.hdr;
720                         hnae3_set_field(*ol_type_vlan_len_msec,
721                                         HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
722                                         ol4_len >> 2);
723
724                         /* switch IP header ptr from outer to inner header */
725                         l3.hdr = skb_inner_network_header(skb);
726
727                         /* compute inner l2 header size, defined in 2 Bytes. */
728                         l2_len = l3.hdr - l2_hdr;
729                         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
730                                         HNS3_TXD_L2LEN_S, l2_len >> 1);
731                 } else {
732                         /* skb packet types not supported by hardware,
733                          * txbd len fild doesn't be filled.
734                          */
735                         return;
736                 }
737
738                 /* switch L4 header pointer from outer to inner */
739                 l4.hdr = skb_inner_transport_header(skb);
740
741                 l4_proto = il4_proto;
742         }
743
744         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
745         l3_len = l4.hdr - l3.hdr;
746         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
747                         HNS3_TXD_L3LEN_S, l3_len >> 2);
748
749         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
750         switch (l4_proto) {
751         case IPPROTO_TCP:
752                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
753                                 HNS3_TXD_L4LEN_S, l4.tcp->doff);
754                 break;
755         case IPPROTO_SCTP:
756                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
757                                 HNS3_TXD_L4LEN_S,
758                                 (sizeof(struct sctphdr) >> 2));
759                 break;
760         case IPPROTO_UDP:
761                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
762                                 HNS3_TXD_L4LEN_S,
763                                 (sizeof(struct udphdr) >> 2));
764                 break;
765         default:
766                 /* skb packet types not supported by hardware,
767                  * txbd len fild doesn't be filled.
768                  */
769                 return;
770         }
771 }
772
773 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
774  * and it is udp packet, which has a dest port as the IANA assigned.
775  * the hardware is expected to do the checksum offload, but the
776  * hardware will not do the checksum offload when udp dest port is
777  * 4789.
778  */
779 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
780 {
781 #define IANA_VXLAN_PORT 4789
782         union {
783                 struct tcphdr *tcp;
784                 struct udphdr *udp;
785                 struct gre_base_hdr *gre;
786                 unsigned char *hdr;
787         } l4;
788
789         l4.hdr = skb_transport_header(skb);
790
791         if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
792                 return false;
793
794         skb_checksum_help(skb);
795
796         return true;
797 }
798
799 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
800                                    u8 il4_proto, u32 *type_cs_vlan_tso,
801                                    u32 *ol_type_vlan_len_msec)
802 {
803         union {
804                 struct iphdr *v4;
805                 struct ipv6hdr *v6;
806                 unsigned char *hdr;
807         } l3;
808         u32 l4_proto = ol4_proto;
809
810         l3.hdr = skb_network_header(skb);
811
812         /* define OL3 type and tunnel type(OL4).*/
813         if (skb->encapsulation) {
814                 /* define outer network header type.*/
815                 if (skb->protocol == htons(ETH_P_IP)) {
816                         if (skb_is_gso(skb))
817                                 hnae3_set_field(*ol_type_vlan_len_msec,
818                                                 HNS3_TXD_OL3T_M,
819                                                 HNS3_TXD_OL3T_S,
820                                                 HNS3_OL3T_IPV4_CSUM);
821                         else
822                                 hnae3_set_field(*ol_type_vlan_len_msec,
823                                                 HNS3_TXD_OL3T_M,
824                                                 HNS3_TXD_OL3T_S,
825                                                 HNS3_OL3T_IPV4_NO_CSUM);
826
827                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
828                         hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
829                                         HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
830                 }
831
832                 /* define tunnel type(OL4).*/
833                 switch (l4_proto) {
834                 case IPPROTO_UDP:
835                         hnae3_set_field(*ol_type_vlan_len_msec,
836                                         HNS3_TXD_TUNTYPE_M,
837                                         HNS3_TXD_TUNTYPE_S,
838                                         HNS3_TUN_MAC_IN_UDP);
839                         break;
840                 case IPPROTO_GRE:
841                         hnae3_set_field(*ol_type_vlan_len_msec,
842                                         HNS3_TXD_TUNTYPE_M,
843                                         HNS3_TXD_TUNTYPE_S,
844                                         HNS3_TUN_NVGRE);
845                         break;
846                 default:
847                         /* drop the skb tunnel packet if hardware don't support,
848                          * because hardware can't calculate csum when TSO.
849                          */
850                         if (skb_is_gso(skb))
851                                 return -EDOM;
852
853                         /* the stack computes the IP header already,
854                          * driver calculate l4 checksum when not TSO.
855                          */
856                         skb_checksum_help(skb);
857                         return 0;
858                 }
859
860                 l3.hdr = skb_inner_network_header(skb);
861                 l4_proto = il4_proto;
862         }
863
864         if (l3.v4->version == 4) {
865                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
866                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
867
868                 /* the stack computes the IP header already, the only time we
869                  * need the hardware to recompute it is in the case of TSO.
870                  */
871                 if (skb_is_gso(skb))
872                         hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
873         } else if (l3.v6->version == 6) {
874                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
875                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
876         }
877
878         switch (l4_proto) {
879         case IPPROTO_TCP:
880                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
881                 hnae3_set_field(*type_cs_vlan_tso,
882                                 HNS3_TXD_L4T_M,
883                                 HNS3_TXD_L4T_S,
884                                 HNS3_L4T_TCP);
885                 break;
886         case IPPROTO_UDP:
887                 if (hns3_tunnel_csum_bug(skb))
888                         break;
889
890                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
891                 hnae3_set_field(*type_cs_vlan_tso,
892                                 HNS3_TXD_L4T_M,
893                                 HNS3_TXD_L4T_S,
894                                 HNS3_L4T_UDP);
895                 break;
896         case IPPROTO_SCTP:
897                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
898                 hnae3_set_field(*type_cs_vlan_tso,
899                                 HNS3_TXD_L4T_M,
900                                 HNS3_TXD_L4T_S,
901                                 HNS3_L4T_SCTP);
902                 break;
903         default:
904                 /* drop the skb tunnel packet if hardware don't support,
905                  * because hardware can't calculate csum when TSO.
906                  */
907                 if (skb_is_gso(skb))
908                         return -EDOM;
909
910                 /* the stack computes the IP header already,
911                  * driver calculate l4 checksum when not TSO.
912                  */
913                 skb_checksum_help(skb);
914                 return 0;
915         }
916
917         return 0;
918 }
919
920 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
921 {
922         /* Config bd buffer end */
923         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
924                         HNS3_TXD_BDTYPE_S, 0);
925         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
926         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
927         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
928 }
929
930 static int hns3_fill_desc_vtags(struct sk_buff *skb,
931                                 struct hns3_enet_ring *tx_ring,
932                                 u32 *inner_vlan_flag,
933                                 u32 *out_vlan_flag,
934                                 u16 *inner_vtag,
935                                 u16 *out_vtag)
936 {
937 #define HNS3_TX_VLAN_PRIO_SHIFT 13
938
939         if (skb->protocol == htons(ETH_P_8021Q) &&
940             !(tx_ring->tqp->handle->kinfo.netdev->features &
941             NETIF_F_HW_VLAN_CTAG_TX)) {
942                 /* When HW VLAN acceleration is turned off, and the stack
943                  * sets the protocol to 802.1q, the driver just need to
944                  * set the protocol to the encapsulated ethertype.
945                  */
946                 skb->protocol = vlan_get_protocol(skb);
947                 return 0;
948         }
949
950         if (skb_vlan_tag_present(skb)) {
951                 u16 vlan_tag;
952
953                 vlan_tag = skb_vlan_tag_get(skb);
954                 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
955
956                 /* Based on hw strategy, use out_vtag in two layer tag case,
957                  * and use inner_vtag in one tag case.
958                  */
959                 if (skb->protocol == htons(ETH_P_8021Q)) {
960                         hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
961                         *out_vtag = vlan_tag;
962                 } else {
963                         hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
964                         *inner_vtag = vlan_tag;
965                 }
966         } else if (skb->protocol == htons(ETH_P_8021Q)) {
967                 struct vlan_ethhdr *vhdr;
968                 int rc;
969
970                 rc = skb_cow_head(skb, 0);
971                 if (rc < 0)
972                         return rc;
973                 vhdr = (struct vlan_ethhdr *)skb->data;
974                 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
975                                         << HNS3_TX_VLAN_PRIO_SHIFT);
976         }
977
978         skb->protocol = vlan_get_protocol(skb);
979         return 0;
980 }
981
982 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
983                           int size, int frag_end, enum hns_desc_type type)
984 {
985         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
986         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
987         struct device *dev = ring_to_dev(ring);
988         u32 ol_type_vlan_len_msec = 0;
989         u16 bdtp_fe_sc_vld_ra_ri = 0;
990         struct skb_frag_struct *frag;
991         unsigned int frag_buf_num;
992         u32 type_cs_vlan_tso = 0;
993         struct sk_buff *skb;
994         u16 inner_vtag = 0;
995         u16 out_vtag = 0;
996         unsigned int k;
997         int sizeoflast;
998         u32 paylen = 0;
999         dma_addr_t dma;
1000         u16 mss = 0;
1001         u8 ol4_proto;
1002         u8 il4_proto;
1003         int ret;
1004
1005         if (type == DESC_TYPE_SKB) {
1006                 skb = (struct sk_buff *)priv;
1007                 paylen = skb->len;
1008
1009                 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1010                                            &ol_type_vlan_len_msec,
1011                                            &inner_vtag, &out_vtag);
1012                 if (unlikely(ret))
1013                         return ret;
1014
1015                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1016                         skb_reset_mac_len(skb);
1017
1018                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1019                         if (ret)
1020                                 return ret;
1021                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1022                                             &type_cs_vlan_tso,
1023                                             &ol_type_vlan_len_msec);
1024                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1025                                                       &type_cs_vlan_tso,
1026                                                       &ol_type_vlan_len_msec);
1027                         if (ret)
1028                                 return ret;
1029
1030                         ret = hns3_set_tso(skb, &paylen, &mss,
1031                                            &type_cs_vlan_tso);
1032                         if (ret)
1033                                 return ret;
1034                 }
1035
1036                 /* Set txbd */
1037                 desc->tx.ol_type_vlan_len_msec =
1038                         cpu_to_le32(ol_type_vlan_len_msec);
1039                 desc->tx.type_cs_vlan_tso_len =
1040                         cpu_to_le32(type_cs_vlan_tso);
1041                 desc->tx.paylen = cpu_to_le32(paylen);
1042                 desc->tx.mss = cpu_to_le16(mss);
1043                 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1044                 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1045
1046                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1047         } else {
1048                 frag = (struct skb_frag_struct *)priv;
1049                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1050         }
1051
1052         if (dma_mapping_error(ring->dev, dma)) {
1053                 ring->stats.sw_err_cnt++;
1054                 return -ENOMEM;
1055         }
1056
1057         desc_cb->length = size;
1058
1059         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1060         sizeoflast = size % HNS3_MAX_BD_SIZE;
1061         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1062
1063         /* When frag size is bigger than hardware limit, split this frag */
1064         for (k = 0; k < frag_buf_num; k++) {
1065                 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1066                 desc_cb->priv = priv;
1067                 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1068                 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1069                                         DESC_TYPE_SKB : DESC_TYPE_PAGE;
1070
1071                 /* now, fill the descriptor */
1072                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1073                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1074                                 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1075                 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1076                                        frag_end && (k == frag_buf_num - 1) ?
1077                                                 1 : 0);
1078                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1079                                 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1080
1081                 /* move ring pointer to next.*/
1082                 ring_ptr_move_fw(ring, next_to_use);
1083
1084                 desc_cb = &ring->desc_cb[ring->next_to_use];
1085                 desc = &ring->desc[ring->next_to_use];
1086         }
1087
1088         return 0;
1089 }
1090
1091 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1092                                    struct hns3_enet_ring *ring)
1093 {
1094         struct sk_buff *skb = *out_skb;
1095         struct skb_frag_struct *frag;
1096         int bdnum_for_frag;
1097         int frag_num;
1098         int buf_num;
1099         int size;
1100         int i;
1101
1102         size = skb_headlen(skb);
1103         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1104
1105         frag_num = skb_shinfo(skb)->nr_frags;
1106         for (i = 0; i < frag_num; i++) {
1107                 frag = &skb_shinfo(skb)->frags[i];
1108                 size = skb_frag_size(frag);
1109                 bdnum_for_frag =
1110                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1111                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1112                         return -ENOMEM;
1113
1114                 buf_num += bdnum_for_frag;
1115         }
1116
1117         if (buf_num > ring_space(ring))
1118                 return -EBUSY;
1119
1120         *bnum = buf_num;
1121         return 0;
1122 }
1123
1124 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1125                                   struct hns3_enet_ring *ring)
1126 {
1127         struct sk_buff *skb = *out_skb;
1128         int buf_num;
1129
1130         /* No. of segments (plus a header) */
1131         buf_num = skb_shinfo(skb)->nr_frags + 1;
1132
1133         if (unlikely(ring_space(ring) < buf_num))
1134                 return -EBUSY;
1135
1136         *bnum = buf_num;
1137
1138         return 0;
1139 }
1140
1141 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1142 {
1143         struct device *dev = ring_to_dev(ring);
1144         unsigned int i;
1145
1146         for (i = 0; i < ring->desc_num; i++) {
1147                 /* check if this is where we started */
1148                 if (ring->next_to_use == next_to_use_orig)
1149                         break;
1150
1151                 /* unmap the descriptor dma address */
1152                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1153                         dma_unmap_single(dev,
1154                                          ring->desc_cb[ring->next_to_use].dma,
1155                                         ring->desc_cb[ring->next_to_use].length,
1156                                         DMA_TO_DEVICE);
1157                 else if (ring->desc_cb[ring->next_to_use].length)
1158                         dma_unmap_page(dev,
1159                                        ring->desc_cb[ring->next_to_use].dma,
1160                                        ring->desc_cb[ring->next_to_use].length,
1161                                        DMA_TO_DEVICE);
1162
1163                 ring->desc_cb[ring->next_to_use].length = 0;
1164
1165                 /* rollback one */
1166                 ring_ptr_move_bw(ring, next_to_use);
1167         }
1168 }
1169
1170 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1171 {
1172         struct hns3_nic_priv *priv = netdev_priv(netdev);
1173         struct hns3_nic_ring_data *ring_data =
1174                 &tx_ring_data(priv, skb->queue_mapping);
1175         struct hns3_enet_ring *ring = ring_data->ring;
1176         struct netdev_queue *dev_queue;
1177         struct skb_frag_struct *frag;
1178         int next_to_use_head;
1179         int next_to_use_frag;
1180         int buf_num;
1181         int seg_num;
1182         int size;
1183         int ret;
1184         int i;
1185
1186         /* Prefetch the data used later */
1187         prefetch(skb->data);
1188
1189         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1190         case -EBUSY:
1191                 u64_stats_update_begin(&ring->syncp);
1192                 ring->stats.tx_busy++;
1193                 u64_stats_update_end(&ring->syncp);
1194
1195                 goto out_net_tx_busy;
1196         case -ENOMEM:
1197                 u64_stats_update_begin(&ring->syncp);
1198                 ring->stats.sw_err_cnt++;
1199                 u64_stats_update_end(&ring->syncp);
1200                 netdev_err(netdev, "no memory to xmit!\n");
1201
1202                 goto out_err_tx_ok;
1203         default:
1204                 break;
1205         }
1206
1207         /* No. of segments (plus a header) */
1208         seg_num = skb_shinfo(skb)->nr_frags + 1;
1209         /* Fill the first part */
1210         size = skb_headlen(skb);
1211
1212         next_to_use_head = ring->next_to_use;
1213
1214         ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1215                                   DESC_TYPE_SKB);
1216         if (ret)
1217                 goto head_fill_err;
1218
1219         next_to_use_frag = ring->next_to_use;
1220         /* Fill the fragments */
1221         for (i = 1; i < seg_num; i++) {
1222                 frag = &skb_shinfo(skb)->frags[i - 1];
1223                 size = skb_frag_size(frag);
1224
1225                 ret = priv->ops.fill_desc(ring, frag, size,
1226                                           seg_num - 1 == i ? 1 : 0,
1227                                           DESC_TYPE_PAGE);
1228
1229                 if (ret)
1230                         goto frag_fill_err;
1231         }
1232
1233         /* Complete translate all packets */
1234         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1235         netdev_tx_sent_queue(dev_queue, skb->len);
1236
1237         wmb(); /* Commit all data before submit */
1238
1239         hnae3_queue_xmit(ring->tqp, buf_num);
1240
1241         return NETDEV_TX_OK;
1242
1243 frag_fill_err:
1244         hns3_clear_desc(ring, next_to_use_frag);
1245
1246 head_fill_err:
1247         hns3_clear_desc(ring, next_to_use_head);
1248
1249 out_err_tx_ok:
1250         dev_kfree_skb_any(skb);
1251         return NETDEV_TX_OK;
1252
1253 out_net_tx_busy:
1254         netif_stop_subqueue(netdev, ring_data->queue_index);
1255         smp_mb(); /* Commit all data before submit */
1256
1257         return NETDEV_TX_BUSY;
1258 }
1259
1260 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1261 {
1262         struct hnae3_handle *h = hns3_get_handle(netdev);
1263         struct sockaddr *mac_addr = p;
1264         int ret;
1265
1266         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1267                 return -EADDRNOTAVAIL;
1268
1269         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1270                 netdev_info(netdev, "already using mac address %pM\n",
1271                             mac_addr->sa_data);
1272                 return 0;
1273         }
1274
1275         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1276         if (ret) {
1277                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1278                 return ret;
1279         }
1280
1281         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1282
1283         return 0;
1284 }
1285
1286 static int hns3_nic_do_ioctl(struct net_device *netdev,
1287                              struct ifreq *ifr, int cmd)
1288 {
1289         struct hnae3_handle *h = hns3_get_handle(netdev);
1290
1291         if (!netif_running(netdev))
1292                 return -EINVAL;
1293
1294         if (!h->ae_algo->ops->do_ioctl)
1295                 return -EOPNOTSUPP;
1296
1297         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1298 }
1299
1300 static int hns3_nic_set_features(struct net_device *netdev,
1301                                  netdev_features_t features)
1302 {
1303         netdev_features_t changed = netdev->features ^ features;
1304         struct hns3_nic_priv *priv = netdev_priv(netdev);
1305         struct hnae3_handle *h = priv->ae_handle;
1306         int ret;
1307
1308         if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1309                 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1310                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1311                 else
1312                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1313         }
1314
1315         if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1316             h->ae_algo->ops->enable_vlan_filter) {
1317                 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1318                         h->ae_algo->ops->enable_vlan_filter(h, true);
1319                 else
1320                         h->ae_algo->ops->enable_vlan_filter(h, false);
1321         }
1322
1323         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1324             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1325                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1326                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1327                 else
1328                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1329
1330                 if (ret)
1331                         return ret;
1332         }
1333
1334         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1335                 if (features & NETIF_F_NTUPLE)
1336                         h->ae_algo->ops->enable_fd(h, true);
1337                 else
1338                         h->ae_algo->ops->enable_fd(h, false);
1339         }
1340
1341         netdev->features = features;
1342         return 0;
1343 }
1344
1345 static void hns3_nic_get_stats64(struct net_device *netdev,
1346                                  struct rtnl_link_stats64 *stats)
1347 {
1348         struct hns3_nic_priv *priv = netdev_priv(netdev);
1349         int queue_num = priv->ae_handle->kinfo.num_tqps;
1350         struct hnae3_handle *handle = priv->ae_handle;
1351         struct hns3_enet_ring *ring;
1352         unsigned int start;
1353         unsigned int idx;
1354         u64 tx_bytes = 0;
1355         u64 rx_bytes = 0;
1356         u64 tx_pkts = 0;
1357         u64 rx_pkts = 0;
1358         u64 tx_drop = 0;
1359         u64 rx_drop = 0;
1360
1361         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1362                 return;
1363
1364         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1365
1366         for (idx = 0; idx < queue_num; idx++) {
1367                 /* fetch the tx stats */
1368                 ring = priv->ring_data[idx].ring;
1369                 do {
1370                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1371                         tx_bytes += ring->stats.tx_bytes;
1372                         tx_pkts += ring->stats.tx_pkts;
1373                         tx_drop += ring->stats.tx_busy;
1374                         tx_drop += ring->stats.sw_err_cnt;
1375                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1376
1377                 /* fetch the rx stats */
1378                 ring = priv->ring_data[idx + queue_num].ring;
1379                 do {
1380                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1381                         rx_bytes += ring->stats.rx_bytes;
1382                         rx_pkts += ring->stats.rx_pkts;
1383                         rx_drop += ring->stats.non_vld_descs;
1384                         rx_drop += ring->stats.err_pkt_len;
1385                         rx_drop += ring->stats.l2_err;
1386                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1387         }
1388
1389         stats->tx_bytes = tx_bytes;
1390         stats->tx_packets = tx_pkts;
1391         stats->rx_bytes = rx_bytes;
1392         stats->rx_packets = rx_pkts;
1393
1394         stats->rx_errors = netdev->stats.rx_errors;
1395         stats->multicast = netdev->stats.multicast;
1396         stats->rx_length_errors = netdev->stats.rx_length_errors;
1397         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1398         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1399
1400         stats->tx_errors = netdev->stats.tx_errors;
1401         stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1402         stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1403         stats->collisions = netdev->stats.collisions;
1404         stats->rx_over_errors = netdev->stats.rx_over_errors;
1405         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1406         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1407         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1408         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1409         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1410         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1411         stats->tx_window_errors = netdev->stats.tx_window_errors;
1412         stats->rx_compressed = netdev->stats.rx_compressed;
1413         stats->tx_compressed = netdev->stats.tx_compressed;
1414 }
1415
1416 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1417 {
1418         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1419         struct hnae3_handle *h = hns3_get_handle(netdev);
1420         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1421         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1422         u8 tc = mqprio_qopt->qopt.num_tc;
1423         u16 mode = mqprio_qopt->mode;
1424         u8 hw = mqprio_qopt->qopt.hw;
1425         bool if_running;
1426         int ret;
1427
1428         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1429                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1430                 return -EOPNOTSUPP;
1431
1432         if (tc > HNAE3_MAX_TC)
1433                 return -EINVAL;
1434
1435         if (!netdev)
1436                 return -EINVAL;
1437
1438         if_running = netif_running(netdev);
1439         if (if_running) {
1440                 hns3_nic_net_stop(netdev);
1441                 msleep(100);
1442         }
1443
1444         ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1445                 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1446         if (ret)
1447                 goto out;
1448
1449         ret = hns3_nic_set_real_num_queue(netdev);
1450
1451 out:
1452         if (if_running)
1453                 hns3_nic_net_open(netdev);
1454
1455         return ret;
1456 }
1457
1458 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1459                              void *type_data)
1460 {
1461         if (type != TC_SETUP_QDISC_MQPRIO)
1462                 return -EOPNOTSUPP;
1463
1464         return hns3_setup_tc(dev, type_data);
1465 }
1466
1467 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1468                                 __be16 proto, u16 vid)
1469 {
1470         struct hnae3_handle *h = hns3_get_handle(netdev);
1471         struct hns3_nic_priv *priv = netdev_priv(netdev);
1472         int ret = -EIO;
1473
1474         if (h->ae_algo->ops->set_vlan_filter)
1475                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1476
1477         if (!ret)
1478                 set_bit(vid, priv->active_vlans);
1479
1480         return ret;
1481 }
1482
1483 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1484                                  __be16 proto, u16 vid)
1485 {
1486         struct hnae3_handle *h = hns3_get_handle(netdev);
1487         struct hns3_nic_priv *priv = netdev_priv(netdev);
1488         int ret = -EIO;
1489
1490         if (h->ae_algo->ops->set_vlan_filter)
1491                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1492
1493         if (!ret)
1494                 clear_bit(vid, priv->active_vlans);
1495
1496         return ret;
1497 }
1498
1499 static int hns3_restore_vlan(struct net_device *netdev)
1500 {
1501         struct hns3_nic_priv *priv = netdev_priv(netdev);
1502         int ret = 0;
1503         u16 vid;
1504
1505         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1506                 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1507                 if (ret) {
1508                         netdev_err(netdev, "Restore vlan: %d filter, ret:%d\n",
1509                                    vid, ret);
1510                         return ret;
1511                 }
1512         }
1513
1514         return ret;
1515 }
1516
1517 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1518                                 u8 qos, __be16 vlan_proto)
1519 {
1520         struct hnae3_handle *h = hns3_get_handle(netdev);
1521         int ret = -EIO;
1522
1523         if (h->ae_algo->ops->set_vf_vlan_filter)
1524                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1525                                                    qos, vlan_proto);
1526
1527         return ret;
1528 }
1529
1530 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1531 {
1532         struct hnae3_handle *h = hns3_get_handle(netdev);
1533         bool if_running = netif_running(netdev);
1534         int ret;
1535
1536         if (!h->ae_algo->ops->set_mtu)
1537                 return -EOPNOTSUPP;
1538
1539         /* if this was called with netdev up then bring netdevice down */
1540         if (if_running) {
1541                 (void)hns3_nic_net_stop(netdev);
1542                 msleep(100);
1543         }
1544
1545         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1546         if (ret)
1547                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1548                            ret);
1549         else
1550                 netdev->mtu = new_mtu;
1551
1552         /* if the netdev was running earlier, bring it up again */
1553         if (if_running && hns3_nic_net_open(netdev))
1554                 ret = -EINVAL;
1555
1556         return ret;
1557 }
1558
1559 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1560 {
1561         struct hns3_nic_priv *priv = netdev_priv(ndev);
1562         struct hns3_enet_ring *tx_ring = NULL;
1563         int timeout_queue = 0;
1564         int hw_head, hw_tail;
1565         int i;
1566
1567         /* Find the stopped queue the same way the stack does */
1568         for (i = 0; i < ndev->real_num_tx_queues; i++) {
1569                 struct netdev_queue *q;
1570                 unsigned long trans_start;
1571
1572                 q = netdev_get_tx_queue(ndev, i);
1573                 trans_start = q->trans_start;
1574                 if (netif_xmit_stopped(q) &&
1575                     time_after(jiffies,
1576                                (trans_start + ndev->watchdog_timeo))) {
1577                         timeout_queue = i;
1578                         break;
1579                 }
1580         }
1581
1582         if (i == ndev->num_tx_queues) {
1583                 netdev_info(ndev,
1584                             "no netdev TX timeout queue found, timeout count: %llu\n",
1585                             priv->tx_timeout_count);
1586                 return false;
1587         }
1588
1589         tx_ring = priv->ring_data[timeout_queue].ring;
1590
1591         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1592                                 HNS3_RING_TX_RING_HEAD_REG);
1593         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1594                                 HNS3_RING_TX_RING_TAIL_REG);
1595         netdev_info(ndev,
1596                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1597                     priv->tx_timeout_count,
1598                     timeout_queue,
1599                     tx_ring->next_to_use,
1600                     tx_ring->next_to_clean,
1601                     hw_head,
1602                     hw_tail,
1603                     readl(tx_ring->tqp_vector->mask_addr));
1604
1605         return true;
1606 }
1607
1608 static void hns3_nic_net_timeout(struct net_device *ndev)
1609 {
1610         struct hns3_nic_priv *priv = netdev_priv(ndev);
1611         struct hnae3_handle *h = priv->ae_handle;
1612
1613         if (!hns3_get_tx_timeo_queue_info(ndev))
1614                 return;
1615
1616         priv->tx_timeout_count++;
1617
1618         if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
1619                 return;
1620
1621         /* request the reset */
1622         if (h->ae_algo->ops->reset_event)
1623                 h->ae_algo->ops->reset_event(h->pdev, h);
1624 }
1625
1626 static const struct net_device_ops hns3_nic_netdev_ops = {
1627         .ndo_open               = hns3_nic_net_open,
1628         .ndo_stop               = hns3_nic_net_stop,
1629         .ndo_start_xmit         = hns3_nic_net_xmit,
1630         .ndo_tx_timeout         = hns3_nic_net_timeout,
1631         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1632         .ndo_do_ioctl           = hns3_nic_do_ioctl,
1633         .ndo_change_mtu         = hns3_nic_change_mtu,
1634         .ndo_set_features       = hns3_nic_set_features,
1635         .ndo_get_stats64        = hns3_nic_get_stats64,
1636         .ndo_setup_tc           = hns3_nic_setup_tc,
1637         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1638         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1639         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1640         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1641 };
1642
1643 static bool hns3_is_phys_func(struct pci_dev *pdev)
1644 {
1645         u32 dev_id = pdev->device;
1646
1647         switch (dev_id) {
1648         case HNAE3_DEV_ID_GE:
1649         case HNAE3_DEV_ID_25GE:
1650         case HNAE3_DEV_ID_25GE_RDMA:
1651         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1652         case HNAE3_DEV_ID_50GE_RDMA:
1653         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1654         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1655                 return true;
1656         case HNAE3_DEV_ID_100G_VF:
1657         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1658                 return false;
1659         default:
1660                 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1661                          dev_id);
1662         }
1663
1664         return false;
1665 }
1666
1667 static void hns3_disable_sriov(struct pci_dev *pdev)
1668 {
1669         /* If our VFs are assigned we cannot shut down SR-IOV
1670          * without causing issues, so just leave the hardware
1671          * available but disabled
1672          */
1673         if (pci_vfs_assigned(pdev)) {
1674                 dev_warn(&pdev->dev,
1675                          "disabling driver while VFs are assigned\n");
1676                 return;
1677         }
1678
1679         pci_disable_sriov(pdev);
1680 }
1681
1682 static void hns3_get_dev_capability(struct pci_dev *pdev,
1683                                     struct hnae3_ae_dev *ae_dev)
1684 {
1685         if (pdev->revision >= 0x21)
1686                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
1687 }
1688
1689 /* hns3_probe - Device initialization routine
1690  * @pdev: PCI device information struct
1691  * @ent: entry in hns3_pci_tbl
1692  *
1693  * hns3_probe initializes a PF identified by a pci_dev structure.
1694  * The OS initialization, configuring of the PF private structure,
1695  * and a hardware reset occur.
1696  *
1697  * Returns 0 on success, negative on failure
1698  */
1699 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1700 {
1701         struct hnae3_ae_dev *ae_dev;
1702         int ret;
1703
1704         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1705                               GFP_KERNEL);
1706         if (!ae_dev) {
1707                 ret = -ENOMEM;
1708                 return ret;
1709         }
1710
1711         ae_dev->pdev = pdev;
1712         ae_dev->flag = ent->driver_data;
1713         ae_dev->dev_type = HNAE3_DEV_KNIC;
1714         ae_dev->reset_type = HNAE3_NONE_RESET;
1715         hns3_get_dev_capability(pdev, ae_dev);
1716         pci_set_drvdata(pdev, ae_dev);
1717
1718         hnae3_register_ae_dev(ae_dev);
1719
1720         return 0;
1721 }
1722
1723 /* hns3_remove - Device removal routine
1724  * @pdev: PCI device information struct
1725  */
1726 static void hns3_remove(struct pci_dev *pdev)
1727 {
1728         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1729
1730         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1731                 hns3_disable_sriov(pdev);
1732
1733         hnae3_unregister_ae_dev(ae_dev);
1734 }
1735
1736 /**
1737  * hns3_pci_sriov_configure
1738  * @pdev: pointer to a pci_dev structure
1739  * @num_vfs: number of VFs to allocate
1740  *
1741  * Enable or change the number of VFs. Called when the user updates the number
1742  * of VFs in sysfs.
1743  **/
1744 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1745 {
1746         int ret;
1747
1748         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1749                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1750                 return -EINVAL;
1751         }
1752
1753         if (num_vfs) {
1754                 ret = pci_enable_sriov(pdev, num_vfs);
1755                 if (ret)
1756                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1757                 else
1758                         return num_vfs;
1759         } else if (!pci_vfs_assigned(pdev)) {
1760                 pci_disable_sriov(pdev);
1761         } else {
1762                 dev_warn(&pdev->dev,
1763                          "Unable to free VFs because some are assigned to VMs.\n");
1764         }
1765
1766         return 0;
1767 }
1768
1769 static void hns3_shutdown(struct pci_dev *pdev)
1770 {
1771         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1772
1773         hnae3_unregister_ae_dev(ae_dev);
1774         devm_kfree(&pdev->dev, ae_dev);
1775         pci_set_drvdata(pdev, NULL);
1776
1777         if (system_state == SYSTEM_POWER_OFF)
1778                 pci_set_power_state(pdev, PCI_D3hot);
1779 }
1780
1781 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1782                                             pci_channel_state_t state)
1783 {
1784         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1785         pci_ers_result_t ret;
1786
1787         dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1788
1789         if (state == pci_channel_io_perm_failure)
1790                 return PCI_ERS_RESULT_DISCONNECT;
1791
1792         if (!ae_dev) {
1793                 dev_err(&pdev->dev,
1794                         "Can't recover - error happened during device init\n");
1795                 return PCI_ERS_RESULT_NONE;
1796         }
1797
1798         if (ae_dev->ops->process_hw_error)
1799                 ret = ae_dev->ops->process_hw_error(ae_dev);
1800         else
1801                 return PCI_ERS_RESULT_NONE;
1802
1803         return ret;
1804 }
1805
1806 static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1807 {
1808         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1809         struct device *dev = &pdev->dev;
1810
1811         dev_info(dev, "requesting reset due to PCI error\n");
1812
1813         /* request the reset */
1814         if (ae_dev->ops->reset_event) {
1815                 ae_dev->ops->reset_event(pdev, NULL);
1816                 return PCI_ERS_RESULT_RECOVERED;
1817         }
1818
1819         return PCI_ERS_RESULT_DISCONNECT;
1820 }
1821
1822 static const struct pci_error_handlers hns3_err_handler = {
1823         .error_detected = hns3_error_detected,
1824         .slot_reset     = hns3_slot_reset,
1825 };
1826
1827 static struct pci_driver hns3_driver = {
1828         .name     = hns3_driver_name,
1829         .id_table = hns3_pci_tbl,
1830         .probe    = hns3_probe,
1831         .remove   = hns3_remove,
1832         .shutdown = hns3_shutdown,
1833         .sriov_configure = hns3_pci_sriov_configure,
1834         .err_handler    = &hns3_err_handler,
1835 };
1836
1837 /* set default feature to hns3 */
1838 static void hns3_set_default_feature(struct net_device *netdev)
1839 {
1840         struct hnae3_handle *h = hns3_get_handle(netdev);
1841         struct pci_dev *pdev = h->pdev;
1842
1843         netdev->priv_flags |= IFF_UNICAST_FLT;
1844
1845         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1846                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1847                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1848                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1849                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1850
1851         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1852
1853         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1854
1855         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1856                 NETIF_F_HW_VLAN_CTAG_FILTER |
1857                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1858                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1859                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1860                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1861                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1862
1863         netdev->vlan_features |=
1864                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1865                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1866                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1867                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1868                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1869
1870         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1871                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1872                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1873                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1874                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1875                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1876
1877         if (pdev->revision >= 0x21) {
1878                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1879
1880                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1881                         netdev->hw_features |= NETIF_F_NTUPLE;
1882                         netdev->features |= NETIF_F_NTUPLE;
1883                 }
1884         }
1885 }
1886
1887 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1888                              struct hns3_desc_cb *cb)
1889 {
1890         unsigned int order = hnae3_page_order(ring);
1891         struct page *p;
1892
1893         p = dev_alloc_pages(order);
1894         if (!p)
1895                 return -ENOMEM;
1896
1897         cb->priv = p;
1898         cb->page_offset = 0;
1899         cb->reuse_flag = 0;
1900         cb->buf  = page_address(p);
1901         cb->length = hnae3_page_size(ring);
1902         cb->type = DESC_TYPE_PAGE;
1903
1904         return 0;
1905 }
1906
1907 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1908                              struct hns3_desc_cb *cb)
1909 {
1910         if (cb->type == DESC_TYPE_SKB)
1911                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1912         else if (!HNAE3_IS_TX_RING(ring))
1913                 put_page((struct page *)cb->priv);
1914         memset(cb, 0, sizeof(*cb));
1915 }
1916
1917 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1918 {
1919         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1920                                cb->length, ring_to_dma_dir(ring));
1921
1922         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
1923                 return -EIO;
1924
1925         return 0;
1926 }
1927
1928 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1929                               struct hns3_desc_cb *cb)
1930 {
1931         if (cb->type == DESC_TYPE_SKB)
1932                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1933                                  ring_to_dma_dir(ring));
1934         else if (cb->length)
1935                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1936                                ring_to_dma_dir(ring));
1937 }
1938
1939 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1940 {
1941         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1942         ring->desc[i].addr = 0;
1943 }
1944
1945 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1946 {
1947         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1948
1949         if (!ring->desc_cb[i].dma)
1950                 return;
1951
1952         hns3_buffer_detach(ring, i);
1953         hns3_free_buffer(ring, cb);
1954 }
1955
1956 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1957 {
1958         int i;
1959
1960         for (i = 0; i < ring->desc_num; i++)
1961                 hns3_free_buffer_detach(ring, i);
1962 }
1963
1964 /* free desc along with its attached buffer */
1965 static void hns3_free_desc(struct hns3_enet_ring *ring)
1966 {
1967         int size = ring->desc_num * sizeof(ring->desc[0]);
1968
1969         hns3_free_buffers(ring);
1970
1971         if (ring->desc) {
1972                 dma_free_coherent(ring_to_dev(ring), size,
1973                                   ring->desc, ring->desc_dma_addr);
1974                 ring->desc = NULL;
1975         }
1976 }
1977
1978 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1979 {
1980         int size = ring->desc_num * sizeof(ring->desc[0]);
1981
1982         ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
1983                                          &ring->desc_dma_addr,
1984                                          GFP_KERNEL);
1985         if (!ring->desc)
1986                 return -ENOMEM;
1987
1988         return 0;
1989 }
1990
1991 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1992                                    struct hns3_desc_cb *cb)
1993 {
1994         int ret;
1995
1996         ret = hns3_alloc_buffer(ring, cb);
1997         if (ret)
1998                 goto out;
1999
2000         ret = hns3_map_buffer(ring, cb);
2001         if (ret)
2002                 goto out_with_buf;
2003
2004         return 0;
2005
2006 out_with_buf:
2007         hns3_free_buffer(ring, cb);
2008 out:
2009         return ret;
2010 }
2011
2012 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2013 {
2014         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2015
2016         if (ret)
2017                 return ret;
2018
2019         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2020
2021         return 0;
2022 }
2023
2024 /* Allocate memory for raw pkg, and map with dma */
2025 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2026 {
2027         int i, j, ret;
2028
2029         for (i = 0; i < ring->desc_num; i++) {
2030                 ret = hns3_alloc_buffer_attach(ring, i);
2031                 if (ret)
2032                         goto out_buffer_fail;
2033         }
2034
2035         return 0;
2036
2037 out_buffer_fail:
2038         for (j = i - 1; j >= 0; j--)
2039                 hns3_free_buffer_detach(ring, j);
2040         return ret;
2041 }
2042
2043 /* detach a in-used buffer and replace with a reserved one  */
2044 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2045                                 struct hns3_desc_cb *res_cb)
2046 {
2047         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2048         ring->desc_cb[i] = *res_cb;
2049         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2050         ring->desc[i].rx.bd_base_info = 0;
2051 }
2052
2053 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2054 {
2055         ring->desc_cb[i].reuse_flag = 0;
2056         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2057                 + ring->desc_cb[i].page_offset);
2058         ring->desc[i].rx.bd_base_info = 0;
2059 }
2060
2061 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2062                                       int *pkts)
2063 {
2064         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2065
2066         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2067         (*bytes) += desc_cb->length;
2068         /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
2069         hns3_free_buffer_detach(ring, ring->next_to_clean);
2070
2071         ring_ptr_move_fw(ring, next_to_clean);
2072 }
2073
2074 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2075 {
2076         int u = ring->next_to_use;
2077         int c = ring->next_to_clean;
2078
2079         if (unlikely(h > ring->desc_num))
2080                 return 0;
2081
2082         return u > c ? (h > c && h <= u) : (h > c || h <= u);
2083 }
2084
2085 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2086 {
2087         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2088         struct hns3_nic_priv *priv = netdev_priv(netdev);
2089         struct netdev_queue *dev_queue;
2090         int bytes, pkts;
2091         int head;
2092
2093         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2094         rmb(); /* Make sure head is ready before touch any data */
2095
2096         if (is_ring_empty(ring) || head == ring->next_to_clean)
2097                 return; /* no data to poll */
2098
2099         if (unlikely(!is_valid_clean_head(ring, head))) {
2100                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2101                            ring->next_to_use, ring->next_to_clean);
2102
2103                 u64_stats_update_begin(&ring->syncp);
2104                 ring->stats.io_err_cnt++;
2105                 u64_stats_update_end(&ring->syncp);
2106                 return;
2107         }
2108
2109         bytes = 0;
2110         pkts = 0;
2111         while (head != ring->next_to_clean) {
2112                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2113                 /* Issue prefetch for next Tx descriptor */
2114                 prefetch(&ring->desc_cb[ring->next_to_clean]);
2115         }
2116
2117         ring->tqp_vector->tx_group.total_bytes += bytes;
2118         ring->tqp_vector->tx_group.total_packets += pkts;
2119
2120         u64_stats_update_begin(&ring->syncp);
2121         ring->stats.tx_bytes += bytes;
2122         ring->stats.tx_pkts += pkts;
2123         u64_stats_update_end(&ring->syncp);
2124
2125         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2126         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2127
2128         if (unlikely(pkts && netif_carrier_ok(netdev) &&
2129                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2130                 /* Make sure that anybody stopping the queue after this
2131                  * sees the new next_to_clean.
2132                  */
2133                 smp_mb();
2134                 if (netif_tx_queue_stopped(dev_queue) &&
2135                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2136                         netif_tx_wake_queue(dev_queue);
2137                         ring->stats.restart_queue++;
2138                 }
2139         }
2140 }
2141
2142 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2143 {
2144         int ntc = ring->next_to_clean;
2145         int ntu = ring->next_to_use;
2146
2147         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2148 }
2149
2150 static void
2151 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2152 {
2153         struct hns3_desc_cb *desc_cb;
2154         struct hns3_desc_cb res_cbs;
2155         int i, ret;
2156
2157         for (i = 0; i < cleand_count; i++) {
2158                 desc_cb = &ring->desc_cb[ring->next_to_use];
2159                 if (desc_cb->reuse_flag) {
2160                         u64_stats_update_begin(&ring->syncp);
2161                         ring->stats.reuse_pg_cnt++;
2162                         u64_stats_update_end(&ring->syncp);
2163
2164                         hns3_reuse_buffer(ring, ring->next_to_use);
2165                 } else {
2166                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
2167                         if (ret) {
2168                                 u64_stats_update_begin(&ring->syncp);
2169                                 ring->stats.sw_err_cnt++;
2170                                 u64_stats_update_end(&ring->syncp);
2171
2172                                 netdev_err(ring->tqp->handle->kinfo.netdev,
2173                                            "hnae reserve buffer map failed.\n");
2174                                 break;
2175                         }
2176                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2177                 }
2178
2179                 ring_ptr_move_fw(ring, next_to_use);
2180         }
2181
2182         wmb(); /* Make all data has been write before submit */
2183         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2184 }
2185
2186 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2187                                 struct hns3_enet_ring *ring, int pull_len,
2188                                 struct hns3_desc_cb *desc_cb)
2189 {
2190         struct hns3_desc *desc;
2191         u32 truesize;
2192         int size;
2193         int last_offset;
2194         bool twobufs;
2195
2196         twobufs = ((PAGE_SIZE < 8192) &&
2197                 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2198
2199         desc = &ring->desc[ring->next_to_clean];
2200         size = le16_to_cpu(desc->rx.size);
2201
2202         truesize = hnae3_buf_size(ring);
2203
2204         if (!twobufs)
2205                 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2206
2207         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2208                         size - pull_len, truesize);
2209
2210          /* Avoid re-using remote pages,flag default unreuse */
2211         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2212                 return;
2213
2214         if (twobufs) {
2215                 /* If we are only owner of page we can reuse it */
2216                 if (likely(page_count(desc_cb->priv) == 1)) {
2217                         /* Flip page offset to other buffer */
2218                         desc_cb->page_offset ^= truesize;
2219
2220                         desc_cb->reuse_flag = 1;
2221                         /* bump ref count on page before it is given*/
2222                         get_page(desc_cb->priv);
2223                 }
2224                 return;
2225         }
2226
2227         /* Move offset up to the next cache line */
2228         desc_cb->page_offset += truesize;
2229
2230         if (desc_cb->page_offset <= last_offset) {
2231                 desc_cb->reuse_flag = 1;
2232                 /* Bump ref count on page before it is given*/
2233                 get_page(desc_cb->priv);
2234         }
2235 }
2236
2237 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2238                              struct hns3_desc *desc)
2239 {
2240         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2241         int l3_type, l4_type;
2242         u32 bd_base_info;
2243         int ol4_type;
2244         u32 l234info;
2245
2246         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2247         l234info = le32_to_cpu(desc->rx.l234_info);
2248
2249         skb->ip_summed = CHECKSUM_NONE;
2250
2251         skb_checksum_none_assert(skb);
2252
2253         if (!(netdev->features & NETIF_F_RXCSUM))
2254                 return;
2255
2256         /* check if hardware has done checksum */
2257         if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2258                 return;
2259
2260         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2261                      hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2262                      hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2263                      hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2264                 u64_stats_update_begin(&ring->syncp);
2265                 ring->stats.l3l4_csum_err++;
2266                 u64_stats_update_end(&ring->syncp);
2267
2268                 return;
2269         }
2270
2271         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2272                                   HNS3_RXD_L3ID_S);
2273         l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2274                                   HNS3_RXD_L4ID_S);
2275
2276         ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2277                                    HNS3_RXD_OL4ID_S);
2278         switch (ol4_type) {
2279         case HNS3_OL4_TYPE_MAC_IN_UDP:
2280         case HNS3_OL4_TYPE_NVGRE:
2281                 skb->csum_level = 1;
2282                 /* fall through */
2283         case HNS3_OL4_TYPE_NO_TUN:
2284                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2285                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2286                      l3_type == HNS3_L3_TYPE_IPV6) &&
2287                     (l4_type == HNS3_L4_TYPE_UDP ||
2288                      l4_type == HNS3_L4_TYPE_TCP ||
2289                      l4_type == HNS3_L4_TYPE_SCTP))
2290                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2291                 break;
2292         default:
2293                 break;
2294         }
2295 }
2296
2297 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2298 {
2299         napi_gro_receive(&ring->tqp_vector->napi, skb);
2300 }
2301
2302 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2303                                 struct hns3_desc *desc, u32 l234info,
2304                                 u16 *vlan_tag)
2305 {
2306         struct pci_dev *pdev = ring->tqp->handle->pdev;
2307
2308         if (pdev->revision == 0x20) {
2309                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2310                 if (!(*vlan_tag & VLAN_VID_MASK))
2311                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2312
2313                 return (*vlan_tag != 0);
2314         }
2315
2316 #define HNS3_STRP_OUTER_VLAN    0x1
2317 #define HNS3_STRP_INNER_VLAN    0x2
2318
2319         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2320                                 HNS3_RXD_STRP_TAGP_S)) {
2321         case HNS3_STRP_OUTER_VLAN:
2322                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2323                 return true;
2324         case HNS3_STRP_INNER_VLAN:
2325                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2326                 return true;
2327         default:
2328                 return false;
2329         }
2330 }
2331
2332 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2333                                      struct sk_buff *skb)
2334 {
2335         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2336         struct hnae3_handle *handle = ring->tqp->handle;
2337         enum pkt_hash_types rss_type;
2338
2339         if (le32_to_cpu(desc->rx.rss_hash))
2340                 rss_type = handle->kinfo.rss_type;
2341         else
2342                 rss_type = PKT_HASH_TYPE_NONE;
2343
2344         skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2345 }
2346
2347 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2348                              struct sk_buff **out_skb, int *out_bnum)
2349 {
2350         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2351         struct hns3_desc_cb *desc_cb;
2352         struct hns3_desc *desc;
2353         struct sk_buff *skb;
2354         unsigned char *va;
2355         u32 bd_base_info;
2356         int pull_len;
2357         u32 l234info;
2358         int length;
2359         int bnum;
2360
2361         desc = &ring->desc[ring->next_to_clean];
2362         desc_cb = &ring->desc_cb[ring->next_to_clean];
2363
2364         prefetch(desc);
2365
2366         length = le16_to_cpu(desc->rx.size);
2367         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2368
2369         /* Check valid BD */
2370         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2371                 return -EFAULT;
2372
2373         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2374
2375         /* Prefetch first cache line of first page
2376          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2377          * line size is 64B so need to prefetch twice to make it 128B. But in
2378          * actual we can have greater size of caches with 128B Level 1 cache
2379          * lines. In such a case, single fetch would suffice to cache in the
2380          * relevant part of the header.
2381          */
2382         prefetch(va);
2383 #if L1_CACHE_BYTES < 128
2384         prefetch(va + L1_CACHE_BYTES);
2385 #endif
2386
2387         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2388                                         HNS3_RX_HEAD_SIZE);
2389         if (unlikely(!skb)) {
2390                 netdev_err(netdev, "alloc rx skb fail\n");
2391
2392                 u64_stats_update_begin(&ring->syncp);
2393                 ring->stats.sw_err_cnt++;
2394                 u64_stats_update_end(&ring->syncp);
2395
2396                 return -ENOMEM;
2397         }
2398
2399         prefetchw(skb->data);
2400
2401         bnum = 1;
2402         if (length <= HNS3_RX_HEAD_SIZE) {
2403                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2404
2405                 /* We can reuse buffer as-is, just make sure it is local */
2406                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2407                         desc_cb->reuse_flag = 1;
2408                 else /* This page cannot be reused so discard it */
2409                         put_page(desc_cb->priv);
2410
2411                 ring_ptr_move_fw(ring, next_to_clean);
2412         } else {
2413                 u64_stats_update_begin(&ring->syncp);
2414                 ring->stats.seg_pkt_cnt++;
2415                 u64_stats_update_end(&ring->syncp);
2416
2417                 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2418
2419                 memcpy(__skb_put(skb, pull_len), va,
2420                        ALIGN(pull_len, sizeof(long)));
2421
2422                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2423                 ring_ptr_move_fw(ring, next_to_clean);
2424
2425                 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2426                         desc = &ring->desc[ring->next_to_clean];
2427                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2428                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2429                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2430                         ring_ptr_move_fw(ring, next_to_clean);
2431                         bnum++;
2432                 }
2433         }
2434
2435         *out_bnum = bnum;
2436
2437         l234info = le32_to_cpu(desc->rx.l234_info);
2438
2439         /* Based on hw strategy, the tag offloaded will be stored at
2440          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2441          * in one layer tag case.
2442          */
2443         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2444                 u16 vlan_tag;
2445
2446                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2447                         __vlan_hwaccel_put_tag(skb,
2448                                                htons(ETH_P_8021Q),
2449                                                vlan_tag);
2450         }
2451
2452         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2453                 u64_stats_update_begin(&ring->syncp);
2454                 ring->stats.non_vld_descs++;
2455                 u64_stats_update_end(&ring->syncp);
2456
2457                 dev_kfree_skb_any(skb);
2458                 return -EINVAL;
2459         }
2460
2461         if (unlikely((!desc->rx.pkt_len) ||
2462                      hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2463                 u64_stats_update_begin(&ring->syncp);
2464                 ring->stats.err_pkt_len++;
2465                 u64_stats_update_end(&ring->syncp);
2466
2467                 dev_kfree_skb_any(skb);
2468                 return -EFAULT;
2469         }
2470
2471         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2472                 u64_stats_update_begin(&ring->syncp);
2473                 ring->stats.l2_err++;
2474                 u64_stats_update_end(&ring->syncp);
2475
2476                 dev_kfree_skb_any(skb);
2477                 return -EFAULT;
2478         }
2479
2480         u64_stats_update_begin(&ring->syncp);
2481         ring->stats.rx_pkts++;
2482         ring->stats.rx_bytes += skb->len;
2483         u64_stats_update_end(&ring->syncp);
2484
2485         ring->tqp_vector->rx_group.total_bytes += skb->len;
2486
2487         hns3_rx_checksum(ring, skb, desc);
2488         hns3_set_rx_skb_rss_type(ring, skb);
2489
2490         return 0;
2491 }
2492
2493 int hns3_clean_rx_ring(
2494                 struct hns3_enet_ring *ring, int budget,
2495                 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2496 {
2497 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2498         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2499         int recv_pkts, recv_bds, clean_count, err;
2500         int unused_count = hns3_desc_unused(ring);
2501         struct sk_buff *skb = NULL;
2502         int num, bnum = 0;
2503
2504         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2505         rmb(); /* Make sure num taken effect before the other data is touched */
2506
2507         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2508         num -= unused_count;
2509
2510         while (recv_pkts < budget && recv_bds < num) {
2511                 /* Reuse or realloc buffers */
2512                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2513                         hns3_nic_alloc_rx_buffers(ring,
2514                                                   clean_count + unused_count);
2515                         clean_count = 0;
2516                         unused_count = hns3_desc_unused(ring);
2517                 }
2518
2519                 /* Poll one pkt */
2520                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2521                 if (unlikely(!skb)) /* This fault cannot be repaired */
2522                         goto out;
2523
2524                 recv_bds += bnum;
2525                 clean_count += bnum;
2526                 if (unlikely(err)) {  /* Do jump the err */
2527                         recv_pkts++;
2528                         continue;
2529                 }
2530
2531                 /* Do update ip stack process */
2532                 skb->protocol = eth_type_trans(skb, netdev);
2533                 rx_fn(ring, skb);
2534
2535                 recv_pkts++;
2536         }
2537
2538 out:
2539         /* Make all data has been write before submit */
2540         if (clean_count + unused_count > 0)
2541                 hns3_nic_alloc_rx_buffers(ring,
2542                                           clean_count + unused_count);
2543
2544         return recv_pkts;
2545 }
2546
2547 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2548 {
2549         struct hns3_enet_tqp_vector *tqp_vector =
2550                                         ring_group->ring->tqp_vector;
2551         enum hns3_flow_level_range new_flow_level;
2552         int packets_per_msecs;
2553         int bytes_per_msecs;
2554         u32 time_passed_ms;
2555         u16 new_int_gl;
2556
2557         if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
2558                 return false;
2559
2560         if (ring_group->total_packets == 0) {
2561                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2562                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
2563                 return true;
2564         }
2565
2566         /* Simple throttlerate management
2567          * 0-10MB/s   lower     (50000 ints/s)
2568          * 10-20MB/s   middle    (20000 ints/s)
2569          * 20-1249MB/s high      (18000 ints/s)
2570          * > 40000pps  ultra     (8000 ints/s)
2571          */
2572         new_flow_level = ring_group->coal.flow_level;
2573         new_int_gl = ring_group->coal.int_gl;
2574         time_passed_ms =
2575                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2576
2577         if (!time_passed_ms)
2578                 return false;
2579
2580         do_div(ring_group->total_packets, time_passed_ms);
2581         packets_per_msecs = ring_group->total_packets;
2582
2583         do_div(ring_group->total_bytes, time_passed_ms);
2584         bytes_per_msecs = ring_group->total_bytes;
2585
2586 #define HNS3_RX_LOW_BYTE_RATE 10000
2587 #define HNS3_RX_MID_BYTE_RATE 20000
2588
2589         switch (new_flow_level) {
2590         case HNS3_FLOW_LOW:
2591                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2592                         new_flow_level = HNS3_FLOW_MID;
2593                 break;
2594         case HNS3_FLOW_MID:
2595                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2596                         new_flow_level = HNS3_FLOW_HIGH;
2597                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2598                         new_flow_level = HNS3_FLOW_LOW;
2599                 break;
2600         case HNS3_FLOW_HIGH:
2601         case HNS3_FLOW_ULTRA:
2602         default:
2603                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2604                         new_flow_level = HNS3_FLOW_MID;
2605                 break;
2606         }
2607
2608 #define HNS3_RX_ULTRA_PACKET_RATE 40
2609
2610         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2611             &tqp_vector->rx_group == ring_group)
2612                 new_flow_level = HNS3_FLOW_ULTRA;
2613
2614         switch (new_flow_level) {
2615         case HNS3_FLOW_LOW:
2616                 new_int_gl = HNS3_INT_GL_50K;
2617                 break;
2618         case HNS3_FLOW_MID:
2619                 new_int_gl = HNS3_INT_GL_20K;
2620                 break;
2621         case HNS3_FLOW_HIGH:
2622                 new_int_gl = HNS3_INT_GL_18K;
2623                 break;
2624         case HNS3_FLOW_ULTRA:
2625                 new_int_gl = HNS3_INT_GL_8K;
2626                 break;
2627         default:
2628                 break;
2629         }
2630
2631         ring_group->total_bytes = 0;
2632         ring_group->total_packets = 0;
2633         ring_group->coal.flow_level = new_flow_level;
2634         if (new_int_gl != ring_group->coal.int_gl) {
2635                 ring_group->coal.int_gl = new_int_gl;
2636                 return true;
2637         }
2638         return false;
2639 }
2640
2641 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2642 {
2643         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2644         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2645         bool rx_update, tx_update;
2646
2647         if (tqp_vector->int_adapt_down > 0) {
2648                 tqp_vector->int_adapt_down--;
2649                 return;
2650         }
2651
2652         if (rx_group->coal.gl_adapt_enable) {
2653                 rx_update = hns3_get_new_int_gl(rx_group);
2654                 if (rx_update)
2655                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
2656                                                        rx_group->coal.int_gl);
2657         }
2658
2659         if (tx_group->coal.gl_adapt_enable) {
2660                 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2661                 if (tx_update)
2662                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
2663                                                        tx_group->coal.int_gl);
2664         }
2665
2666         tqp_vector->last_jiffies = jiffies;
2667         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
2668 }
2669
2670 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2671 {
2672         struct hns3_enet_ring *ring;
2673         int rx_pkt_total = 0;
2674
2675         struct hns3_enet_tqp_vector *tqp_vector =
2676                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2677         bool clean_complete = true;
2678         int rx_budget;
2679
2680         /* Since the actual Tx work is minimal, we can give the Tx a larger
2681          * budget and be more aggressive about cleaning up the Tx descriptors.
2682          */
2683         hns3_for_each_ring(ring, tqp_vector->tx_group)
2684                 hns3_clean_tx_ring(ring);
2685
2686         /* make sure rx ring budget not smaller than 1 */
2687         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2688
2689         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2690                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2691                                                     hns3_rx_skb);
2692
2693                 if (rx_cleaned >= rx_budget)
2694                         clean_complete = false;
2695
2696                 rx_pkt_total += rx_cleaned;
2697         }
2698
2699         tqp_vector->rx_group.total_packets += rx_pkt_total;
2700
2701         if (!clean_complete)
2702                 return budget;
2703
2704         napi_complete(napi);
2705         hns3_update_new_int_gl(tqp_vector);
2706         hns3_mask_vector_irq(tqp_vector, 1);
2707
2708         return rx_pkt_total;
2709 }
2710
2711 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2712                                       struct hnae3_ring_chain_node *head)
2713 {
2714         struct pci_dev *pdev = tqp_vector->handle->pdev;
2715         struct hnae3_ring_chain_node *cur_chain = head;
2716         struct hnae3_ring_chain_node *chain;
2717         struct hns3_enet_ring *tx_ring;
2718         struct hns3_enet_ring *rx_ring;
2719
2720         tx_ring = tqp_vector->tx_group.ring;
2721         if (tx_ring) {
2722                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2723                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2724                               HNAE3_RING_TYPE_TX);
2725                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2726                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2727
2728                 cur_chain->next = NULL;
2729
2730                 while (tx_ring->next) {
2731                         tx_ring = tx_ring->next;
2732
2733                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2734                                              GFP_KERNEL);
2735                         if (!chain)
2736                                 goto err_free_chain;
2737
2738                         cur_chain->next = chain;
2739                         chain->tqp_index = tx_ring->tqp->tqp_index;
2740                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2741                                       HNAE3_RING_TYPE_TX);
2742                         hnae3_set_field(chain->int_gl_idx,
2743                                         HNAE3_RING_GL_IDX_M,
2744                                         HNAE3_RING_GL_IDX_S,
2745                                         HNAE3_RING_GL_TX);
2746
2747                         cur_chain = chain;
2748                 }
2749         }
2750
2751         rx_ring = tqp_vector->rx_group.ring;
2752         if (!tx_ring && rx_ring) {
2753                 cur_chain->next = NULL;
2754                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2755                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2756                               HNAE3_RING_TYPE_RX);
2757                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2758                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2759
2760                 rx_ring = rx_ring->next;
2761         }
2762
2763         while (rx_ring) {
2764                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2765                 if (!chain)
2766                         goto err_free_chain;
2767
2768                 cur_chain->next = chain;
2769                 chain->tqp_index = rx_ring->tqp->tqp_index;
2770                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2771                               HNAE3_RING_TYPE_RX);
2772                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2773                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2774
2775                 cur_chain = chain;
2776
2777                 rx_ring = rx_ring->next;
2778         }
2779
2780         return 0;
2781
2782 err_free_chain:
2783         cur_chain = head->next;
2784         while (cur_chain) {
2785                 chain = cur_chain->next;
2786                 devm_kfree(&pdev->dev, chain);
2787                 cur_chain = chain;
2788         }
2789
2790         return -ENOMEM;
2791 }
2792
2793 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2794                                         struct hnae3_ring_chain_node *head)
2795 {
2796         struct pci_dev *pdev = tqp_vector->handle->pdev;
2797         struct hnae3_ring_chain_node *chain_tmp, *chain;
2798
2799         chain = head->next;
2800
2801         while (chain) {
2802                 chain_tmp = chain->next;
2803                 devm_kfree(&pdev->dev, chain);
2804                 chain = chain_tmp;
2805         }
2806 }
2807
2808 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2809                                    struct hns3_enet_ring *ring)
2810 {
2811         ring->next = group->ring;
2812         group->ring = ring;
2813
2814         group->count++;
2815 }
2816
2817 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2818 {
2819         struct pci_dev *pdev = priv->ae_handle->pdev;
2820         struct hns3_enet_tqp_vector *tqp_vector;
2821         int num_vectors = priv->vector_num;
2822         int numa_node;
2823         int vector_i;
2824
2825         numa_node = dev_to_node(&pdev->dev);
2826
2827         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2828                 tqp_vector = &priv->tqp_vector[vector_i];
2829                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2830                                 &tqp_vector->affinity_mask);
2831         }
2832 }
2833
2834 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2835 {
2836         struct hnae3_ring_chain_node vector_ring_chain;
2837         struct hnae3_handle *h = priv->ae_handle;
2838         struct hns3_enet_tqp_vector *tqp_vector;
2839         int ret = 0;
2840         int i;
2841
2842         hns3_nic_set_cpumask(priv);
2843
2844         for (i = 0; i < priv->vector_num; i++) {
2845                 tqp_vector = &priv->tqp_vector[i];
2846                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2847                 tqp_vector->num_tqps = 0;
2848         }
2849
2850         for (i = 0; i < h->kinfo.num_tqps; i++) {
2851                 u16 vector_i = i % priv->vector_num;
2852                 u16 tqp_num = h->kinfo.num_tqps;
2853
2854                 tqp_vector = &priv->tqp_vector[vector_i];
2855
2856                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2857                                        priv->ring_data[i].ring);
2858
2859                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2860                                        priv->ring_data[i + tqp_num].ring);
2861
2862                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2863                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2864                 tqp_vector->num_tqps++;
2865         }
2866
2867         for (i = 0; i < priv->vector_num; i++) {
2868                 tqp_vector = &priv->tqp_vector[i];
2869
2870                 tqp_vector->rx_group.total_bytes = 0;
2871                 tqp_vector->rx_group.total_packets = 0;
2872                 tqp_vector->tx_group.total_bytes = 0;
2873                 tqp_vector->tx_group.total_packets = 0;
2874                 tqp_vector->handle = h;
2875
2876                 ret = hns3_get_vector_ring_chain(tqp_vector,
2877                                                  &vector_ring_chain);
2878                 if (ret)
2879                         return ret;
2880
2881                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2882                         tqp_vector->vector_irq, &vector_ring_chain);
2883
2884                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2885
2886                 if (ret)
2887                         goto map_ring_fail;
2888
2889                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2890                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2891         }
2892
2893         return 0;
2894
2895 map_ring_fail:
2896         while (i--)
2897                 netif_napi_del(&priv->tqp_vector[i].napi);
2898
2899         return ret;
2900 }
2901
2902 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2903 {
2904         struct hnae3_handle *h = priv->ae_handle;
2905         struct hns3_enet_tqp_vector *tqp_vector;
2906         struct hnae3_vector_info *vector;
2907         struct pci_dev *pdev = h->pdev;
2908         u16 tqp_num = h->kinfo.num_tqps;
2909         u16 vector_num;
2910         int ret = 0;
2911         u16 i;
2912
2913         /* RSS size, cpu online and vector_num should be the same */
2914         /* Should consider 2p/4p later */
2915         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2916         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2917                               GFP_KERNEL);
2918         if (!vector)
2919                 return -ENOMEM;
2920
2921         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2922
2923         priv->vector_num = vector_num;
2924         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2925                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2926                              GFP_KERNEL);
2927         if (!priv->tqp_vector) {
2928                 ret = -ENOMEM;
2929                 goto out;
2930         }
2931
2932         for (i = 0; i < priv->vector_num; i++) {
2933                 tqp_vector = &priv->tqp_vector[i];
2934                 tqp_vector->idx = i;
2935                 tqp_vector->mask_addr = vector[i].io_addr;
2936                 tqp_vector->vector_irq = vector[i].vector;
2937                 hns3_vector_gl_rl_init(tqp_vector, priv);
2938         }
2939
2940 out:
2941         devm_kfree(&pdev->dev, vector);
2942         return ret;
2943 }
2944
2945 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2946 {
2947         group->ring = NULL;
2948         group->count = 0;
2949 }
2950
2951 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2952 {
2953         struct hnae3_ring_chain_node vector_ring_chain;
2954         struct hnae3_handle *h = priv->ae_handle;
2955         struct hns3_enet_tqp_vector *tqp_vector;
2956         int i, ret;
2957
2958         for (i = 0; i < priv->vector_num; i++) {
2959                 tqp_vector = &priv->tqp_vector[i];
2960
2961                 ret = hns3_get_vector_ring_chain(tqp_vector,
2962                                                  &vector_ring_chain);
2963                 if (ret)
2964                         return ret;
2965
2966                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2967                         tqp_vector->vector_irq, &vector_ring_chain);
2968                 if (ret)
2969                         return ret;
2970
2971                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2972
2973                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2974                         (void)irq_set_affinity_hint(
2975                                 priv->tqp_vector[i].vector_irq,
2976                                                     NULL);
2977                         free_irq(priv->tqp_vector[i].vector_irq,
2978                                  &priv->tqp_vector[i]);
2979                 }
2980
2981                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2982                 hns3_clear_ring_group(&tqp_vector->rx_group);
2983                 hns3_clear_ring_group(&tqp_vector->tx_group);
2984                 netif_napi_del(&priv->tqp_vector[i].napi);
2985         }
2986
2987         return 0;
2988 }
2989
2990 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2991 {
2992         struct hnae3_handle *h = priv->ae_handle;
2993         struct pci_dev *pdev = h->pdev;
2994         int i, ret;
2995
2996         for (i = 0; i < priv->vector_num; i++) {
2997                 struct hns3_enet_tqp_vector *tqp_vector;
2998
2999                 tqp_vector = &priv->tqp_vector[i];
3000                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3001                 if (ret)
3002                         return ret;
3003         }
3004
3005         devm_kfree(&pdev->dev, priv->tqp_vector);
3006         return 0;
3007 }
3008
3009 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3010                              int ring_type)
3011 {
3012         struct hns3_nic_ring_data *ring_data = priv->ring_data;
3013         int queue_num = priv->ae_handle->kinfo.num_tqps;
3014         struct pci_dev *pdev = priv->ae_handle->pdev;
3015         struct hns3_enet_ring *ring;
3016
3017         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
3018         if (!ring)
3019                 return -ENOMEM;
3020
3021         if (ring_type == HNAE3_RING_TYPE_TX) {
3022                 ring_data[q->tqp_index].ring = ring;
3023                 ring_data[q->tqp_index].queue_index = q->tqp_index;
3024                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3025         } else {
3026                 ring_data[q->tqp_index + queue_num].ring = ring;
3027                 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
3028                 ring->io_base = q->io_base;
3029         }
3030
3031         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
3032
3033         ring->tqp = q;
3034         ring->desc = NULL;
3035         ring->desc_cb = NULL;
3036         ring->dev = priv->dev;
3037         ring->desc_dma_addr = 0;
3038         ring->buf_size = q->buf_size;
3039         ring->desc_num = q->desc_num;
3040         ring->next_to_use = 0;
3041         ring->next_to_clean = 0;
3042
3043         return 0;
3044 }
3045
3046 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3047                               struct hns3_nic_priv *priv)
3048 {
3049         int ret;
3050
3051         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3052         if (ret)
3053                 return ret;
3054
3055         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3056         if (ret) {
3057                 devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
3058                 return ret;
3059         }
3060
3061         return 0;
3062 }
3063
3064 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3065 {
3066         struct hnae3_handle *h = priv->ae_handle;
3067         struct pci_dev *pdev = h->pdev;
3068         int i, ret;
3069
3070         priv->ring_data =  devm_kzalloc(&pdev->dev,
3071                                         array3_size(h->kinfo.num_tqps,
3072                                                     sizeof(*priv->ring_data),
3073                                                     2),
3074                                         GFP_KERNEL);
3075         if (!priv->ring_data)
3076                 return -ENOMEM;
3077
3078         for (i = 0; i < h->kinfo.num_tqps; i++) {
3079                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3080                 if (ret)
3081                         goto err;
3082         }
3083
3084         return 0;
3085 err:
3086         while (i--) {
3087                 devm_kfree(priv->dev, priv->ring_data[i].ring);
3088                 devm_kfree(priv->dev,
3089                            priv->ring_data[i + h->kinfo.num_tqps].ring);
3090         }
3091
3092         devm_kfree(&pdev->dev, priv->ring_data);
3093         return ret;
3094 }
3095
3096 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3097 {
3098         struct hnae3_handle *h = priv->ae_handle;
3099         int i;
3100
3101         for (i = 0; i < h->kinfo.num_tqps; i++) {
3102                 devm_kfree(priv->dev, priv->ring_data[i].ring);
3103                 devm_kfree(priv->dev,
3104                            priv->ring_data[i + h->kinfo.num_tqps].ring);
3105         }
3106         devm_kfree(priv->dev, priv->ring_data);
3107 }
3108
3109 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3110 {
3111         int ret;
3112
3113         if (ring->desc_num <= 0 || ring->buf_size <= 0)
3114                 return -EINVAL;
3115
3116         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3117                                 GFP_KERNEL);
3118         if (!ring->desc_cb) {
3119                 ret = -ENOMEM;
3120                 goto out;
3121         }
3122
3123         ret = hns3_alloc_desc(ring);
3124         if (ret)
3125                 goto out_with_desc_cb;
3126
3127         if (!HNAE3_IS_TX_RING(ring)) {
3128                 ret = hns3_alloc_ring_buffers(ring);
3129                 if (ret)
3130                         goto out_with_desc;
3131         }
3132
3133         return 0;
3134
3135 out_with_desc:
3136         hns3_free_desc(ring);
3137 out_with_desc_cb:
3138         kfree(ring->desc_cb);
3139         ring->desc_cb = NULL;
3140 out:
3141         return ret;
3142 }
3143
3144 static void hns3_fini_ring(struct hns3_enet_ring *ring)
3145 {
3146         hns3_free_desc(ring);
3147         kfree(ring->desc_cb);
3148         ring->desc_cb = NULL;
3149         ring->next_to_clean = 0;
3150         ring->next_to_use = 0;