drivers/net/ethernet/hisilicon/hns3/hns3_enet.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2016-2017 Hisilicon Limited.
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/etherdevice.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/if_vlan.h>
   8 #include <linux/ip.h>
   9 #include <linux/ipv6.h>
  10 #include <linux/module.h>
  11 #include <linux/pci.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/sctp.h>
  14 #include <linux/vermagic.h>
  15 #include <net/gre.h>
  16 #include <net/pkt_cls.h>
  17 #include <net/vxlan.h>
  18
  19 #include "hnae3.h"
  20 #include "hns3_enet.h"
  21
  22 static void hns3_clear_all_ring(struct hnae3_handle *h);
  23 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
  24 static void hns3_remove_hw_addr(struct net_device *netdev);
  25
  26 static const char hns3_driver_name[] = "hns3";
  27 const char hns3_driver_version[] = VERMAGIC_STRING;
  28 static const char hns3_driver_string[] =
  29                         "Hisilicon Ethernet Network Driver for Hip08 Family";
  30 static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
  31 static struct hnae3_client client;
  32
  33 /* hns3_pci_tbl - PCI Device ID Table
  34  *
  35  * Last entry must be all 0s
  36  *
  37  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  38  *   Class, Class Mask, private data (not used) }
  39  */
  40 static const struct pci_device_id hns3_pci_tbl[] = {
  41         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
  42         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
  43         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
  44          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  45         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
  46          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  47         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
  48          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  49         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
  50          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  51         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
  52          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  53         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
  54         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
  55          HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
  56         /* required last entry */
  57         {0, }
  58 };
  59 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
  60
  61 static irqreturn_t hns3_irq_handle(int irq, void *vector)
  62 {
  63         struct hns3_enet_tqp_vector *tqp_vector = vector;
  64
  65         napi_schedule(&tqp_vector->napi);
  66
  67         return IRQ_HANDLED;
  68 }
  69
  70 /* This callback function is used to set affinity changes to the irq affinity
  71  * masks when the irq_set_affinity_notifier function is used.
  72  */
  73 static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
  74                                          const cpumask_t *mask)
  75 {
  76         struct hns3_enet_tqp_vector *tqp_vectors =
  77                 container_of(notify, struct hns3_enet_tqp_vector,
  78                              affinity_notify);
  79
  80         tqp_vectors->affinity_mask = *mask;
  81 }
  82
  83 static void hns3_nic_irq_affinity_release(struct kref *ref)
  84 {
  85 }
  86
  87 static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
  88 {
  89         struct hns3_enet_tqp_vector *tqp_vectors;
  90         unsigned int i;
  91
  92         for (i = 0; i < priv->vector_num; i++) {
  93                 tqp_vectors = &priv->tqp_vector[i];
  94
  95                 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
  96                         continue;
  97
  98                 /* clear the affinity notifier and affinity mask */
  99                 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
 100                 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
 101
 102                 /* release the irq resource */
 103                 free_irq(tqp_vectors->vector_irq, tqp_vectors);
 104                 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
 105         }
 106 }
 107
 108 static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
 109 {
 110         struct hns3_enet_tqp_vector *tqp_vectors;
 111         int txrx_int_idx = 0;
 112         int rx_int_idx = 0;
 113         int tx_int_idx = 0;
 114         unsigned int i;
 115         int ret;
 116
 117         for (i = 0; i < priv->vector_num; i++) {
 118                 tqp_vectors = &priv->tqp_vector[i];
 119
 120                 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
 121                         continue;
 122
 123                 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
 124                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 125                                  "%s-%s-%d", priv->netdev->name, "TxRx",
 126                                  txrx_int_idx++);
 127                         txrx_int_idx++;
 128                 } else if (tqp_vectors->rx_group.ring) {
 129                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 130                                  "%s-%s-%d", priv->netdev->name, "Rx",
 131                                  rx_int_idx++);
 132                 } else if (tqp_vectors->tx_group.ring) {
 133                         snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
 134                                  "%s-%s-%d", priv->netdev->name, "Tx",
 135                                  tx_int_idx++);
 136                 } else {
 137                         /* Skip this unused q_vector */
 138                         continue;
 139                 }
 140
 141                 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
 142
 143                 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
 144                                   tqp_vectors->name,
 145                                        tqp_vectors);
 146                 if (ret) {
 147                         netdev_err(priv->netdev, "request irq(%d) fail\n",
 148                                    tqp_vectors->vector_irq);
 149                         return ret;
 150                 }
 151
 152                 tqp_vectors->affinity_notify.notify =
 153                                         hns3_nic_irq_affinity_notify;
 154                 tqp_vectors->affinity_notify.release =
 155                                         hns3_nic_irq_affinity_release;
 156                 irq_set_affinity_notifier(tqp_vectors->vector_irq,
 157                                           &tqp_vectors->affinity_notify);
 158                 irq_set_affinity_hint(tqp_vectors->vector_irq,
 159                                       &tqp_vectors->affinity_mask);
 160
 161                 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
 162         }
 163
 164         return 0;
 165 }
 166
 167 static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
 168                                  u32 mask_en)
 169 {
 170         writel(mask_en, tqp_vector->mask_addr);
 171 }
 172
 173 static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
 174 {
 175         napi_enable(&tqp_vector->napi);
 176
 177         /* enable vector */
 178         hns3_mask_vector_irq(tqp_vector, 1);
 179 }
 180
 181 static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 182 {
 183         /* disable vector */
 184         hns3_mask_vector_irq(tqp_vector, 0);
 185
 186         disable_irq(tqp_vector->vector_irq);
 187         napi_disable(&tqp_vector->napi);
 188 }
 189
 190 void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
 191                                  u32 rl_value)
 192 {
 193         u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 194
 195         /* this defines the configuration for RL (Interrupt Rate Limiter).
 196          * Rl defines rate of interrupts i.e. number of interrupts-per-second
 197          * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 198          */
 199
 200         if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
 201             !tqp_vector->rx_group.coal.gl_adapt_enable)
 202                 /* According to the hardware, the range of rl_reg is
 203                  * 0-59 and the unit is 4.
 204                  */
 205                 rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
 206
 207         writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
 208 }
 209
 210 void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 211                                     u32 gl_value)
 212 {
 213         u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 214
 215         writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
 216 }
 217
 218 void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
 219                                     u32 gl_value)
 220 {
 221         u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
 222
 223         writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 224 }
 225
 226 static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
 227                                    struct hns3_nic_priv *priv)
 228 {
 229         /* initialize the configuration for interrupt coalescing.
 230          * 1. GL (Interrupt Gap Limiter)
 231          * 2. RL (Interrupt Rate Limiter)
 232          */
 233
 234         /* Default: enable interrupt coalescing self-adaptive and GL */
 235         tqp_vector->tx_group.coal.gl_adapt_enable = 1;
 236         tqp_vector->rx_group.coal.gl_adapt_enable = 1;
 237
 238         tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
 239         tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
 240
 241         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
 242         tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
 243         tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
 244 }
 245
 246 static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
 247                                       struct hns3_nic_priv *priv)
 248 {
 249         struct hnae3_handle *h = priv->ae_handle;
 250
 251         hns3_set_vector_coalesce_tx_gl(tqp_vector,
 252                                        tqp_vector->tx_group.coal.int_gl);
 253         hns3_set_vector_coalesce_rx_gl(tqp_vector,
 254                                        tqp_vector->rx_group.coal.int_gl);
 255         hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
 256 }
 257
 258 static int hns3_nic_set_real_num_queue(struct net_device *netdev)
 259 {
 260         struct hnae3_handle *h = hns3_get_handle(netdev);
 261         struct hnae3_knic_private_info *kinfo = &h->kinfo;
 262         unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
 263         int i, ret;
 264
 265         if (kinfo->num_tc <= 1) {
 266                 netdev_reset_tc(netdev);
 267         } else {
 268                 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
 269                 if (ret) {
 270                         netdev_err(netdev,
 271                                    "netdev_set_num_tc fail, ret=%d!\n", ret);
 272                         return ret;
 273                 }
 274
 275                 for (i = 0; i < HNAE3_MAX_TC; i++) {
 276                         if (!kinfo->tc_info[i].enable)
 277                                 continue;
 278
 279                         netdev_set_tc_queue(netdev,
 280                                             kinfo->tc_info[i].tc,
 281                                             kinfo->tc_info[i].tqp_count,
 282                                             kinfo->tc_info[i].tqp_offset);
 283                 }
 284         }
 285
 286         ret = netif_set_real_num_tx_queues(netdev, queue_size);
 287         if (ret) {
 288                 netdev_err(netdev,
 289                            "netif_set_real_num_tx_queues fail, ret=%d!\n",
 290                            ret);
 291                 return ret;
 292         }
 293
 294         ret = netif_set_real_num_rx_queues(netdev, queue_size);
 295         if (ret) {
 296                 netdev_err(netdev,
 297                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
 298                 return ret;
 299         }
 300
 301         return 0;
 302 }
 303
 304 static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
 305 {
 306         u16 alloc_tqps, max_rss_size, rss_size;
 307
 308         h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
 309         rss_size = alloc_tqps / h->kinfo.num_tc;
 310
 311         return min_t(u16, rss_size, max_rss_size);
 312 }
 313
 314 static int hns3_nic_net_up(struct net_device *netdev)
 315 {
 316         struct hns3_nic_priv *priv = netdev_priv(netdev);
 317         struct hnae3_handle *h = priv->ae_handle;
 318         int i, j;
 319         int ret;
 320
 321         ret = hns3_nic_reset_all_ring(h);
 322         if (ret)
 323                 return ret;
 324
 325         /* get irq resource for all vectors */
 326         ret = hns3_nic_init_irq(priv);
 327         if (ret) {
 328                 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
 329                 return ret;
 330         }
 331
 332         /* enable the vectors */
 333         for (i = 0; i < priv->vector_num; i++)
 334                 hns3_vector_enable(&priv->tqp_vector[i]);
 335
 336         /* start the ae_dev */
 337         ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
 338         if (ret)
 339                 goto out_start_err;
 340
 341         clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
 342
 343         return 0;
 344
 345 out_start_err:
 346         for (j = i - 1; j >= 0; j--)
 347                 hns3_vector_disable(&priv->tqp_vector[j]);
 348
 349         hns3_nic_uninit_irq(priv);
 350
 351         return ret;
 352 }
 353
 354 static int hns3_nic_net_open(struct net_device *netdev)
 355 {
 356         struct hns3_nic_priv *priv = netdev_priv(netdev);
 357         struct hnae3_handle *h = hns3_get_handle(netdev);
 358         struct hnae3_knic_private_info *kinfo;
 359         int i, ret;
 360
 361         netif_carrier_off(netdev);
 362
 363         ret = hns3_nic_set_real_num_queue(netdev);
 364         if (ret)
 365                 return ret;
 366
 367         ret = hns3_nic_net_up(netdev);
 368         if (ret) {
 369                 netdev_err(netdev,
 370                            "hns net up fail, ret=%d!\n", ret);
 371                 return ret;
 372         }
 373
 374         kinfo = &h->kinfo;
 375         for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
 376                 netdev_set_prio_tc_map(netdev, i,
 377                                        kinfo->prio_tc[i]);
 378         }
 379
 380         priv->ae_handle->last_reset_time = jiffies;
 381         return 0;
 382 }
 383
 384 static void hns3_nic_net_down(struct net_device *netdev)
 385 {
 386         struct hns3_nic_priv *priv = netdev_priv(netdev);
 387         const struct hnae3_ae_ops *ops;
 388         int i;
 389
 390         if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
 391                 return;
 392
 393         /* disable vectors */
 394         for (i = 0; i < priv->vector_num; i++)
 395                 hns3_vector_disable(&priv->tqp_vector[i]);
 396
 397         /* stop ae_dev */
 398         ops = priv->ae_handle->ae_algo->ops;
 399         if (ops->stop)
 400                 ops->stop(priv->ae_handle);
 401
 402         /* free irq resources */
 403         hns3_nic_uninit_irq(priv);
 404
 405         hns3_clear_all_ring(priv->ae_handle);
 406 }
 407
 408 static int hns3_nic_net_stop(struct net_device *netdev)
 409 {
 410         netif_tx_stop_all_queues(netdev);
 411         netif_carrier_off(netdev);
 412
 413         hns3_nic_net_down(netdev);
 414
 415         return 0;
 416 }
 417
 418 static int hns3_nic_uc_sync(struct net_device *netdev,
 419                             const unsigned char *addr)
 420 {
 421         struct hnae3_handle *h = hns3_get_handle(netdev);
 422
 423         if (h->ae_algo->ops->add_uc_addr)
 424                 return h->ae_algo->ops->add_uc_addr(h, addr);
 425
 426         return 0;
 427 }
 428
 429 static int hns3_nic_uc_unsync(struct net_device *netdev,
 430                               const unsigned char *addr)
 431 {
 432         struct hnae3_handle *h = hns3_get_handle(netdev);
 433
 434         if (h->ae_algo->ops->rm_uc_addr)
 435                 return h->ae_algo->ops->rm_uc_addr(h, addr);
 436
 437         return 0;
 438 }
 439
 440 static int hns3_nic_mc_sync(struct net_device *netdev,
 441                             const unsigned char *addr)
 442 {
 443         struct hnae3_handle *h = hns3_get_handle(netdev);
 444
 445         if (h->ae_algo->ops->add_mc_addr)
 446                 return h->ae_algo->ops->add_mc_addr(h, addr);
 447
 448         return 0;
 449 }
 450
 451 static int hns3_nic_mc_unsync(struct net_device *netdev,
 452                               const unsigned char *addr)
 453 {
 454         struct hnae3_handle *h = hns3_get_handle(netdev);
 455
 456         if (h->ae_algo->ops->rm_mc_addr)
 457                 return h->ae_algo->ops->rm_mc_addr(h, addr);
 458
 459         return 0;
 460 }
 461
 462 static u8 hns3_get_netdev_flags(struct net_device *netdev)
 463 {
 464         u8 flags = 0;
 465
 466         if (netdev->flags & IFF_PROMISC) {
 467                 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
 468         } else {
 469                 flags |= HNAE3_VLAN_FLTR;
 470                 if (netdev->flags & IFF_ALLMULTI)
 471                         flags |= HNAE3_USER_MPE;
 472         }
 473
 474         return flags;
 475 }
 476
 477 static void hns3_nic_set_rx_mode(struct net_device *netdev)
 478 {
 479         struct hnae3_handle *h = hns3_get_handle(netdev);
 480         u8 new_flags;
 481         int ret;
 482
 483         new_flags = hns3_get_netdev_flags(netdev);
 484
 485         ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
 486         if (ret) {
 487                 netdev_err(netdev, "sync uc address fail\n");
 488                 if (ret == -ENOSPC)
 489                         new_flags |= HNAE3_OVERFLOW_UPE;
 490         }
 491
 492         if (netdev->flags & IFF_MULTICAST) {
 493                 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
 494                                     hns3_nic_mc_unsync);
 495                 if (ret) {
 496                         netdev_err(netdev, "sync mc address fail\n");
 497                         if (ret == -ENOSPC)
 498                                 new_flags |= HNAE3_OVERFLOW_MPE;
 499                 }
 500         }
 501
 502         hns3_update_promisc_mode(netdev, new_flags);
 503         /* User mode Promisc mode enable and vlan filtering is disabled to
 504          * let all packets in. MAC-VLAN Table overflow Promisc enabled and
 505          * vlan fitering is enabled
 506          */
 507         hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
 508         h->netdev_flags = new_flags;
 509 }
 510
 511 void hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
 512 {
 513         struct hns3_nic_priv *priv = netdev_priv(netdev);
 514         struct hnae3_handle *h = priv->ae_handle;
 515
 516         if (h->ae_algo->ops->set_promisc_mode) {
 517                 h->ae_algo->ops->set_promisc_mode(h,
 518                                                   promisc_flags & HNAE3_UPE,
 519                                                   promisc_flags & HNAE3_MPE);
 520         }
 521 }
 522
 523 void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
 524 {
 525         struct hns3_nic_priv *priv = netdev_priv(netdev);
 526         struct hnae3_handle *h = priv->ae_handle;
 527         bool last_state;
 528
 529         if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
 530                 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
 531                 if (enable != last_state) {
 532                         netdev_info(netdev,
 533                                     "%s vlan filter\n",
 534                                     enable ? "enable" : "disable");
 535                         h->ae_algo->ops->enable_vlan_filter(h, enable);
 536                 }
 537         }
 538 }
 539
 540 static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
 541                         u16 *mss, u32 *type_cs_vlan_tso)
 542 {
 543         u32 l4_offset, hdr_len;
 544         union l3_hdr_info l3;
 545         union l4_hdr_info l4;
 546         u32 l4_paylen;
 547         int ret;
 548
 549         if (!skb_is_gso(skb))
 550                 return 0;
 551
 552         ret = skb_cow_head(skb, 0);
 553         if (ret)
 554                 return ret;
 555
 556         l3.hdr = skb_network_header(skb);
 557         l4.hdr = skb_transport_header(skb);
 558
 559         /* Software should clear the IPv4's checksum field when tso is
 560          * needed.
 561          */
 562         if (l3.v4->version == 4)
 563                 l3.v4->check = 0;
 564
 565         /* tunnel packet.*/
 566         if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
 567                                          SKB_GSO_GRE_CSUM |
 568                                          SKB_GSO_UDP_TUNNEL |
 569                                          SKB_GSO_UDP_TUNNEL_CSUM)) {
 570                 if ((!(skb_shinfo(skb)->gso_type &
 571                     SKB_GSO_PARTIAL)) &&
 572                     (skb_shinfo(skb)->gso_type &
 573                     SKB_GSO_UDP_TUNNEL_CSUM)) {
 574                         /* Software should clear the udp's checksum
 575                          * field when tso is needed.
 576                          */
 577                         l4.udp->check = 0;
 578                 }
 579                 /* reset l3&l4 pointers from outer to inner headers */
 580                 l3.hdr = skb_inner_network_header(skb);
 581                 l4.hdr = skb_inner_transport_header(skb);
 582
 583                 /* Software should clear the IPv4's checksum field when
 584                  * tso is needed.
 585                  */
 586                 if (l3.v4->version == 4)
 587                         l3.v4->check = 0;
 588         }
 589
 590         /* normal or tunnel packet*/
 591         l4_offset = l4.hdr - skb->data;
 592         hdr_len = (l4.tcp->doff * 4) + l4_offset;
 593
 594         /* remove payload length from inner pseudo checksum when tso*/
 595         l4_paylen = skb->len - l4_offset;
 596         csum_replace_by_diff(&l4.tcp->check,
 597                              (__force __wsum)htonl(l4_paylen));
 598
 599         /* find the txbd field values */
 600         *paylen = skb->len - hdr_len;
 601         hnae3_set_bit(*type_cs_vlan_tso,
 602                       HNS3_TXD_TSO_B, 1);
 603
 604         /* get MSS for TSO */
 605         *mss = skb_shinfo(skb)->gso_size;
 606
 607         return 0;
 608 }
 609
 610 static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
 611                                 u8 *il4_proto)
 612 {
 613         union {
 614                 struct iphdr *v4;
 615                 struct ipv6hdr *v6;
 616                 unsigned char *hdr;
 617         } l3;
 618         unsigned char *l4_hdr;
 619         unsigned char *exthdr;
 620         u8 l4_proto_tmp;
 621         __be16 frag_off;
 622
 623         /* find outer header point */
 624         l3.hdr = skb_network_header(skb);
 625         l4_hdr = skb_transport_header(skb);
 626
 627         if (skb->protocol == htons(ETH_P_IPV6)) {
 628                 exthdr = l3.hdr + sizeof(*l3.v6);
 629                 l4_proto_tmp = l3.v6->nexthdr;
 630                 if (l4_hdr != exthdr)
 631                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 632                                          &l4_proto_tmp, &frag_off);
 633         } else if (skb->protocol == htons(ETH_P_IP)) {
 634                 l4_proto_tmp = l3.v4->protocol;
 635         } else {
 636                 return -EINVAL;
 637         }
 638
 639         *ol4_proto = l4_proto_tmp;
 640
 641         /* tunnel packet */
 642         if (!skb->encapsulation) {
 643                 *il4_proto = 0;
 644                 return 0;
 645         }
 646
 647         /* find inner header point */
 648         l3.hdr = skb_inner_network_header(skb);
 649         l4_hdr = skb_inner_transport_header(skb);
 650
 651         if (l3.v6->version == 6) {
 652                 exthdr = l3.hdr + sizeof(*l3.v6);
 653                 l4_proto_tmp = l3.v6->nexthdr;
 654                 if (l4_hdr != exthdr)
 655                         ipv6_skip_exthdr(skb, exthdr - skb->data,
 656                                          &l4_proto_tmp, &frag_off);
 657         } else if (l3.v4->version == 4) {
 658                 l4_proto_tmp = l3.v4->protocol;
 659         }
 660
 661         *il4_proto = l4_proto_tmp;
 662
 663         return 0;
 664 }
 665
 666 static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
 667                                 u8 il4_proto, u32 *type_cs_vlan_tso,
 668                                 u32 *ol_type_vlan_len_msec)
 669 {
 670         union {
 671                 struct iphdr *v4;
 672                 struct ipv6hdr *v6;
 673                 unsigned char *hdr;
 674         } l3;
 675         union {
 676                 struct tcphdr *tcp;
 677                 struct udphdr *udp;
 678                 struct gre_base_hdr *gre;
 679                 unsigned char *hdr;
 680         } l4;
 681         unsigned char *l2_hdr;
 682         u8 l4_proto = ol4_proto;
 683         u32 ol2_len;
 684         u32 ol3_len;
 685         u32 ol4_len;
 686         u32 l2_len;
 687         u32 l3_len;
 688
 689         l3.hdr = skb_network_header(skb);
 690         l4.hdr = skb_transport_header(skb);
 691
 692         /* compute L2 header size for normal packet, defined in 2 Bytes */
 693         l2_len = l3.hdr - skb->data;
 694         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 695                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 696
 697         /* tunnel packet*/
 698         if (skb->encapsulation) {
 699                 /* compute OL2 header size, defined in 2 Bytes */
 700                 ol2_len = l2_len;
 701                 hnae3_set_field(*ol_type_vlan_len_msec,
 702                                 HNS3_TXD_L2LEN_M,
 703                                 HNS3_TXD_L2LEN_S, ol2_len >> 1);
 704
 705                 /* compute OL3 header size, defined in 4 Bytes */
 706                 ol3_len = l4.hdr - l3.hdr;
 707                 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
 708                                 HNS3_TXD_L3LEN_S, ol3_len >> 2);
 709
 710                 /* MAC in UDP, MAC in GRE (0x6558)*/
 711                 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
 712                         /* switch MAC header ptr from outer to inner header.*/
 713                         l2_hdr = skb_inner_mac_header(skb);
 714
 715                         /* compute OL4 header size, defined in 4 Bytes. */
 716                         ol4_len = l2_hdr - l4.hdr;
 717                         hnae3_set_field(*ol_type_vlan_len_msec,
 718                                         HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
 719                                         ol4_len >> 2);
 720
 721                         /* switch IP header ptr from outer to inner header */
 722                         l3.hdr = skb_inner_network_header(skb);
 723
 724                         /* compute inner l2 header size, defined in 2 Bytes. */
 725                         l2_len = l3.hdr - l2_hdr;
 726                         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
 727                                         HNS3_TXD_L2LEN_S, l2_len >> 1);
 728                 } else {
 729                         /* skb packet types not supported by hardware,
 730                          * txbd len fild doesn't be filled.
 731                          */
 732                         return;
 733                 }
 734
 735                 /* switch L4 header pointer from outer to inner */
 736                 l4.hdr = skb_inner_transport_header(skb);
 737
 738                 l4_proto = il4_proto;
 739         }
 740
 741         /* compute inner(/normal) L3 header size, defined in 4 Bytes */
 742         l3_len = l4.hdr - l3.hdr;
 743         hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
 744                         HNS3_TXD_L3LEN_S, l3_len >> 2);
 745
 746         /* compute inner(/normal) L4 header size, defined in 4 Bytes */
 747         switch (l4_proto) {
 748         case IPPROTO_TCP:
 749                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 750                                 HNS3_TXD_L4LEN_S, l4.tcp->doff);
 751                 break;
 752         case IPPROTO_SCTP:
 753                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 754                                 HNS3_TXD_L4LEN_S,
 755                                 (sizeof(struct sctphdr) >> 2));
 756                 break;
 757         case IPPROTO_UDP:
 758                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
 759                                 HNS3_TXD_L4LEN_S,
 760                                 (sizeof(struct udphdr) >> 2));
 761                 break;
 762         default:
 763                 /* skb packet types not supported by hardware,
 764                  * txbd len fild doesn't be filled.
 765                  */
 766                 return;
 767         }
 768 }
 769
 770 /* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
 771  * and it is udp packet, which has a dest port as the IANA assigned.
 772  * the hardware is expected to do the checksum offload, but the
 773  * hardware will not do the checksum offload when udp dest port is
 774  * 4789.
 775  */
 776 static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
 777 {
 778 #define IANA_VXLAN_PORT 4789
 779         union {
 780                 struct tcphdr *tcp;
 781                 struct udphdr *udp;
 782                 struct gre_base_hdr *gre;
 783                 unsigned char *hdr;
 784         } l4;
 785
 786         l4.hdr = skb_transport_header(skb);
 787
 788         if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
 789                 return false;
 790
 791         skb_checksum_help(skb);
 792
 793         return true;
 794 }
 795
 796 static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
 797                                    u8 il4_proto, u32 *type_cs_vlan_tso,
 798                                    u32 *ol_type_vlan_len_msec)
 799 {
 800         union {
 801                 struct iphdr *v4;
 802                 struct ipv6hdr *v6;
 803                 unsigned char *hdr;
 804         } l3;
 805         u32 l4_proto = ol4_proto;
 806
 807         l3.hdr = skb_network_header(skb);
 808
 809         /* define OL3 type and tunnel type(OL4).*/
 810         if (skb->encapsulation) {
 811                 /* define outer network header type.*/
 812                 if (skb->protocol == htons(ETH_P_IP)) {
 813                         if (skb_is_gso(skb))
 814                                 hnae3_set_field(*ol_type_vlan_len_msec,
 815                                                 HNS3_TXD_OL3T_M,
 816                                                 HNS3_TXD_OL3T_S,
 817                                                 HNS3_OL3T_IPV4_CSUM);
 818                         else
 819                                 hnae3_set_field(*ol_type_vlan_len_msec,
 820                                                 HNS3_TXD_OL3T_M,
 821                                                 HNS3_TXD_OL3T_S,
 822                                                 HNS3_OL3T_IPV4_NO_CSUM);
 823
 824                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 825                         hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
 826                                         HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
 827                 }
 828
 829                 /* define tunnel type(OL4).*/
 830                 switch (l4_proto) {
 831                 case IPPROTO_UDP:
 832                         hnae3_set_field(*ol_type_vlan_len_msec,
 833                                         HNS3_TXD_TUNTYPE_M,
 834                                         HNS3_TXD_TUNTYPE_S,
 835                                         HNS3_TUN_MAC_IN_UDP);
 836                         break;
 837                 case IPPROTO_GRE:
 838                         hnae3_set_field(*ol_type_vlan_len_msec,
 839                                         HNS3_TXD_TUNTYPE_M,
 840                                         HNS3_TXD_TUNTYPE_S,
 841                                         HNS3_TUN_NVGRE);
 842                         break;
 843                 default:
 844                         /* drop the skb tunnel packet if hardware don't support,
 845                          * because hardware can't calculate csum when TSO.
 846                          */
 847                         if (skb_is_gso(skb))
 848                                 return -EDOM;
 849
 850                         /* the stack computes the IP header already,
 851                          * driver calculate l4 checksum when not TSO.
 852                          */
 853                         skb_checksum_help(skb);
 854                         return 0;
 855                 }
 856
 857                 l3.hdr = skb_inner_network_header(skb);
 858                 l4_proto = il4_proto;
 859         }
 860
 861         if (l3.v4->version == 4) {
 862                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 863                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
 864
 865                 /* the stack computes the IP header already, the only time we
 866                  * need the hardware to recompute it is in the case of TSO.
 867                  */
 868                 if (skb_is_gso(skb))
 869                         hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
 870         } else if (l3.v6->version == 6) {
 871                 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
 872                                 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
 873         }
 874
 875         switch (l4_proto) {
 876         case IPPROTO_TCP:
 877                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 878                 hnae3_set_field(*type_cs_vlan_tso,
 879                                 HNS3_TXD_L4T_M,
 880                                 HNS3_TXD_L4T_S,
 881                                 HNS3_L4T_TCP);
 882                 break;
 883         case IPPROTO_UDP:
 884                 if (hns3_tunnel_csum_bug(skb))
 885                         break;
 886
 887                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 888                 hnae3_set_field(*type_cs_vlan_tso,
 889                                 HNS3_TXD_L4T_M,
 890                                 HNS3_TXD_L4T_S,
 891                                 HNS3_L4T_UDP);
 892                 break;
 893         case IPPROTO_SCTP:
 894                 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
 895                 hnae3_set_field(*type_cs_vlan_tso,
 896                                 HNS3_TXD_L4T_M,
 897                                 HNS3_TXD_L4T_S,
 898                                 HNS3_L4T_SCTP);
 899                 break;
 900         default:
 901                 /* drop the skb tunnel packet if hardware don't support,
 902                  * because hardware can't calculate csum when TSO.
 903                  */
 904                 if (skb_is_gso(skb))
 905                         return -EDOM;
 906
 907                 /* the stack computes the IP header already,
 908                  * driver calculate l4 checksum when not TSO.
 909                  */
 910                 skb_checksum_help(skb);
 911                 return 0;
 912         }
 913
 914         return 0;
 915 }
 916
 917 static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 918 {
 919         /* Config bd buffer end */
 920         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
 921                         HNS3_TXD_BDTYPE_S, 0);
 922         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
 923         hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
 924         hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
 925 }
 926
 927 static int hns3_fill_desc_vtags(struct sk_buff *skb,
 928                                 struct hns3_enet_ring *tx_ring,
 929                                 u32 *inner_vlan_flag,
 930                                 u32 *out_vlan_flag,
 931                                 u16 *inner_vtag,
 932                                 u16 *out_vtag)
 933 {
 934 #define HNS3_TX_VLAN_PRIO_SHIFT 13
 935
 936         if (skb->protocol == htons(ETH_P_8021Q) &&
 937             !(tx_ring->tqp->handle->kinfo.netdev->features &
 938             NETIF_F_HW_VLAN_CTAG_TX)) {
 939                 /* When HW VLAN acceleration is turned off, and the stack
 940                  * sets the protocol to 802.1q, the driver just need to
 941                  * set the protocol to the encapsulated ethertype.
 942                  */
 943                 skb->protocol = vlan_get_protocol(skb);
 944                 return 0;
 945         }
 946
 947         if (skb_vlan_tag_present(skb)) {
 948                 u16 vlan_tag;
 949
 950                 vlan_tag = skb_vlan_tag_get(skb);
 951                 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
 952
 953                 /* Based on hw strategy, use out_vtag in two layer tag case,
 954                  * and use inner_vtag in one tag case.
 955                  */
 956                 if (skb->protocol == htons(ETH_P_8021Q)) {
 957                         hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
 958                         *out_vtag = vlan_tag;
 959                 } else {
 960                         hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
 961                         *inner_vtag = vlan_tag;
 962                 }
 963         } else if (skb->protocol == htons(ETH_P_8021Q)) {
 964                 struct vlan_ethhdr *vhdr;
 965                 int rc;
 966
 967                 rc = skb_cow_head(skb, 0);
 968                 if (rc < 0)
 969                         return rc;
 970                 vhdr = (struct vlan_ethhdr *)skb->data;
 971                 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
 972                                         << HNS3_TX_VLAN_PRIO_SHIFT);
 973         }
 974
 975         skb->protocol = vlan_get_protocol(skb);
 976         return 0;
 977 }
 978
 979 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 980                           int size, int frag_end, enum hns_desc_type type)
 981 {
 982         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 983         struct hns3_desc *desc = &ring->desc[ring->next_to_use];
 984         struct device *dev = ring_to_dev(ring);
 985         u32 ol_type_vlan_len_msec = 0;
 986         u16 bdtp_fe_sc_vld_ra_ri = 0;
 987         struct skb_frag_struct *frag;
 988         unsigned int frag_buf_num;
 989         u32 type_cs_vlan_tso = 0;
 990         struct sk_buff *skb;
 991         u16 inner_vtag = 0;
 992         u16 out_vtag = 0;
 993         unsigned int k;
 994         int sizeoflast;
 995         u32 paylen = 0;
 996         dma_addr_t dma;
 997         u16 mss = 0;
 998         u8 ol4_proto;
 999         u8 il4_proto;
1000         int ret;
1001
1002         if (type == DESC_TYPE_SKB) {
1003                 skb = (struct sk_buff *)priv;
1004                 paylen = skb->len;
1005
1006                 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1007                                            &ol_type_vlan_len_msec,
1008                                            &inner_vtag, &out_vtag);
1009                 if (unlikely(ret))
1010                         return ret;
1011
1012                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1013                         skb_reset_mac_len(skb);
1014
1015                         ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1016                         if (ret)
1017                                 return ret;
1018                         hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1019                                             &type_cs_vlan_tso,
1020                                             &ol_type_vlan_len_msec);
1021                         ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1022                                                       &type_cs_vlan_tso,
1023                                                       &ol_type_vlan_len_msec);
1024                         if (ret)
1025                                 return ret;
1026
1027                         ret = hns3_set_tso(skb, &paylen, &mss,
1028                                            &type_cs_vlan_tso);
1029                         if (ret)
1030                                 return ret;
1031                 }
1032
1033                 /* Set txbd */
1034                 desc->tx.ol_type_vlan_len_msec =
1035                         cpu_to_le32(ol_type_vlan_len_msec);
1036                 desc->tx.type_cs_vlan_tso_len =
1037                         cpu_to_le32(type_cs_vlan_tso);
1038                 desc->tx.paylen = cpu_to_le32(paylen);
1039                 desc->tx.mss = cpu_to_le16(mss);
1040                 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1041                 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
1042
1043                 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1044         } else {
1045                 frag = (struct skb_frag_struct *)priv;
1046                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1047         }
1048
1049         if (dma_mapping_error(ring->dev, dma)) {
1050                 ring->stats.sw_err_cnt++;
1051                 return -ENOMEM;
1052         }
1053
1054         desc_cb->length = size;
1055
1056         frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1057         sizeoflast = size % HNS3_MAX_BD_SIZE;
1058         sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1059
1060         /* When frag size is bigger than hardware limit, split this frag */
1061         for (k = 0; k < frag_buf_num; k++) {
1062                 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1063                 desc_cb->priv = priv;
1064                 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1065                 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1066                                         DESC_TYPE_SKB : DESC_TYPE_PAGE;
1067
1068                 /* now, fill the descriptor */
1069                 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
1070                 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1071                                 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1072                 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1073                                        frag_end && (k == frag_buf_num - 1) ?
1074                                                 1 : 0);
1075                 desc->tx.bdtp_fe_sc_vld_ra_ri =
1076                                 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1077
1078                 /* move ring pointer to next.*/
1079                 ring_ptr_move_fw(ring, next_to_use);
1080
1081                 desc_cb = &ring->desc_cb[ring->next_to_use];
1082                 desc = &ring->desc[ring->next_to_use];
1083         }
1084
1085         return 0;
1086 }
1087
1088 static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1089                                    struct hns3_enet_ring *ring)
1090 {
1091         struct sk_buff *skb = *out_skb;
1092         struct skb_frag_struct *frag;
1093         int bdnum_for_frag;
1094         int frag_num;
1095         int buf_num;
1096         int size;
1097         int i;
1098
1099         size = skb_headlen(skb);
1100         buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1101
1102         frag_num = skb_shinfo(skb)->nr_frags;
1103         for (i = 0; i < frag_num; i++) {
1104                 frag = &skb_shinfo(skb)->frags[i];
1105                 size = skb_frag_size(frag);
1106                 bdnum_for_frag =
1107                         (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1108                 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1109                         return -ENOMEM;
1110
1111                 buf_num += bdnum_for_frag;
1112         }
1113
1114         if (buf_num > ring_space(ring))
1115                 return -EBUSY;
1116
1117         *bnum = buf_num;
1118         return 0;
1119 }
1120
1121 static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1122                                   struct hns3_enet_ring *ring)
1123 {
1124         struct sk_buff *skb = *out_skb;
1125         int buf_num;
1126
1127         /* No. of segments (plus a header) */
1128         buf_num = skb_shinfo(skb)->nr_frags + 1;
1129
1130         if (unlikely(ring_space(ring) < buf_num))
1131                 return -EBUSY;
1132
1133         *bnum = buf_num;
1134
1135         return 0;
1136 }
1137
1138 static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
1139 {
1140         struct device *dev = ring_to_dev(ring);
1141         unsigned int i;
1142
1143         for (i = 0; i < ring->desc_num; i++) {
1144                 /* check if this is where we started */
1145                 if (ring->next_to_use == next_to_use_orig)
1146                         break;
1147
1148                 /* unmap the descriptor dma address */
1149                 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1150                         dma_unmap_single(dev,
1151                                          ring->desc_cb[ring->next_to_use].dma,
1152                                         ring->desc_cb[ring->next_to_use].length,
1153                                         DMA_TO_DEVICE);
1154                 else if (ring->desc_cb[ring->next_to_use].length)
1155                         dma_unmap_page(dev,
1156                                        ring->desc_cb[ring->next_to_use].dma,
1157                                        ring->desc_cb[ring->next_to_use].length,
1158                                        DMA_TO_DEVICE);
1159
1160                 ring->desc_cb[ring->next_to_use].length = 0;
1161
1162                 /* rollback one */
1163                 ring_ptr_move_bw(ring, next_to_use);
1164         }
1165 }
1166
1167 netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
1168 {
1169         struct hns3_nic_priv *priv = netdev_priv(netdev);
1170         struct hns3_nic_ring_data *ring_data =
1171                 &tx_ring_data(priv, skb->queue_mapping);
1172         struct hns3_enet_ring *ring = ring_data->ring;
1173         struct netdev_queue *dev_queue;
1174         struct skb_frag_struct *frag;
1175         int next_to_use_head;
1176         int next_to_use_frag;
1177         int buf_num;
1178         int seg_num;
1179         int size;
1180         int ret;
1181         int i;
1182
1183         /* Prefetch the data used later */
1184         prefetch(skb->data);
1185
1186         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1187         case -EBUSY:
1188                 u64_stats_update_begin(&ring->syncp);
1189                 ring->stats.tx_busy++;
1190                 u64_stats_update_end(&ring->syncp);
1191
1192                 goto out_net_tx_busy;
1193         case -ENOMEM:
1194                 u64_stats_update_begin(&ring->syncp);
1195                 ring->stats.sw_err_cnt++;
1196                 u64_stats_update_end(&ring->syncp);
1197                 netdev_err(netdev, "no memory to xmit!\n");
1198
1199                 goto out_err_tx_ok;
1200         default:
1201                 break;
1202         }
1203
1204         /* No. of segments (plus a header) */
1205         seg_num = skb_shinfo(skb)->nr_frags + 1;
1206         /* Fill the first part */
1207         size = skb_headlen(skb);
1208
1209         next_to_use_head = ring->next_to_use;
1210
1211         ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1212                                   DESC_TYPE_SKB);
1213         if (ret)
1214                 goto head_fill_err;
1215
1216         next_to_use_frag = ring->next_to_use;
1217         /* Fill the fragments */
1218         for (i = 1; i < seg_num; i++) {
1219                 frag = &skb_shinfo(skb)->frags[i - 1];
1220                 size = skb_frag_size(frag);
1221
1222                 ret = priv->ops.fill_desc(ring, frag, size,
1223                                           seg_num - 1 == i ? 1 : 0,
1224                                           DESC_TYPE_PAGE);
1225
1226                 if (ret)
1227                         goto frag_fill_err;
1228         }
1229
1230         /* Complete translate all packets */
1231         dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1232         netdev_tx_sent_queue(dev_queue, skb->len);
1233
1234         wmb(); /* Commit all data before submit */
1235
1236         hnae3_queue_xmit(ring->tqp, buf_num);
1237
1238         return NETDEV_TX_OK;
1239
1240 frag_fill_err:
1241         hns3_clear_desc(ring, next_to_use_frag);
1242
1243 head_fill_err:
1244         hns3_clear_desc(ring, next_to_use_head);
1245
1246 out_err_tx_ok:
1247         dev_kfree_skb_any(skb);
1248         return NETDEV_TX_OK;
1249
1250 out_net_tx_busy:
1251         netif_stop_subqueue(netdev, ring_data->queue_index);
1252         smp_mb(); /* Commit all data before submit */
1253
1254         return NETDEV_TX_BUSY;
1255 }
1256
1257 static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1258 {
1259         struct hnae3_handle *h = hns3_get_handle(netdev);
1260         struct sockaddr *mac_addr = p;
1261         int ret;
1262
1263         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1264                 return -EADDRNOTAVAIL;
1265
1266         if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1267                 netdev_info(netdev, "already using mac address %pM\n",
1268                             mac_addr->sa_data);
1269                 return 0;
1270         }
1271
1272         ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
1273         if (ret) {
1274                 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1275                 return ret;
1276         }
1277
1278         ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1279
1280         return 0;
1281 }
1282
1283 static int hns3_nic_do_ioctl(struct net_device *netdev,
1284                              struct ifreq *ifr, int cmd)
1285 {
1286         struct hnae3_handle *h = hns3_get_handle(netdev);
1287
1288         if (!netif_running(netdev))
1289                 return -EINVAL;
1290
1291         if (!h->ae_algo->ops->do_ioctl)
1292                 return -EOPNOTSUPP;
1293
1294         return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1295 }
1296
1297 static int hns3_nic_set_features(struct net_device *netdev,
1298                                  netdev_features_t features)
1299 {
1300         netdev_features_t changed = netdev->features ^ features;
1301         struct hns3_nic_priv *priv = netdev_priv(netdev);
1302         struct hnae3_handle *h = priv->ae_handle;
1303         int ret;
1304
1305         if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1306                 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1307                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1308                 else
1309                         priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1310         }
1311
1312         if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1313             h->ae_algo->ops->enable_vlan_filter) {
1314                 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1315                         h->ae_algo->ops->enable_vlan_filter(h, true);
1316                 else
1317                         h->ae_algo->ops->enable_vlan_filter(h, false);
1318         }
1319
1320         if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1321             h->ae_algo->ops->enable_hw_strip_rxvtag) {
1322                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1323                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1324                 else
1325                         ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1326
1327                 if (ret)
1328                         return ret;
1329         }
1330
1331         if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1332                 if (features & NETIF_F_NTUPLE)
1333                         h->ae_algo->ops->enable_fd(h, true);
1334                 else
1335                         h->ae_algo->ops->enable_fd(h, false);
1336         }
1337
1338         netdev->features = features;
1339         return 0;
1340 }
1341
1342 static void hns3_nic_get_stats64(struct net_device *netdev,
1343                                  struct rtnl_link_stats64 *stats)
1344 {
1345         struct hns3_nic_priv *priv = netdev_priv(netdev);
1346         int queue_num = priv->ae_handle->kinfo.num_tqps;
1347         struct hnae3_handle *handle = priv->ae_handle;
1348         struct hns3_enet_ring *ring;
1349         unsigned int start;
1350         unsigned int idx;
1351         u64 tx_bytes = 0;
1352         u64 rx_bytes = 0;
1353         u64 tx_pkts = 0;
1354         u64 rx_pkts = 0;
1355         u64 tx_drop = 0;
1356         u64 rx_drop = 0;
1357
1358         if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1359                 return;
1360
1361         handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1362
1363         for (idx = 0; idx < queue_num; idx++) {
1364                 /* fetch the tx stats */
1365                 ring = priv->ring_data[idx].ring;
1366                 do {
1367                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1368                         tx_bytes += ring->stats.tx_bytes;
1369                         tx_pkts += ring->stats.tx_pkts;
1370                         tx_drop += ring->stats.tx_busy;
1371                         tx_drop += ring->stats.sw_err_cnt;
1372                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1373
1374                 /* fetch the rx stats */
1375                 ring = priv->ring_data[idx + queue_num].ring;
1376                 do {
1377                         start = u64_stats_fetch_begin_irq(&ring->syncp);
1378                         rx_bytes += ring->stats.rx_bytes;
1379                         rx_pkts += ring->stats.rx_pkts;
1380                         rx_drop += ring->stats.non_vld_descs;
1381                         rx_drop += ring->stats.err_pkt_len;
1382                         rx_drop += ring->stats.l2_err;
1383                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1384         }
1385
1386         stats->tx_bytes = tx_bytes;
1387         stats->tx_packets = tx_pkts;
1388         stats->rx_bytes = rx_bytes;
1389         stats->rx_packets = rx_pkts;
1390
1391         stats->rx_errors = netdev->stats.rx_errors;
1392         stats->multicast = netdev->stats.multicast;
1393         stats->rx_length_errors = netdev->stats.rx_length_errors;
1394         stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1395         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1396
1397         stats->tx_errors = netdev->stats.tx_errors;
1398         stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1399         stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
1400         stats->collisions = netdev->stats.collisions;
1401         stats->rx_over_errors = netdev->stats.rx_over_errors;
1402         stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1403         stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1404         stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1405         stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1406         stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1407         stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1408         stats->tx_window_errors = netdev->stats.tx_window_errors;
1409         stats->rx_compressed = netdev->stats.rx_compressed;
1410         stats->tx_compressed = netdev->stats.tx_compressed;
1411 }
1412
1413 static int hns3_setup_tc(struct net_device *netdev, void *type_data)
1414 {
1415         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
1416         struct hnae3_handle *h = hns3_get_handle(netdev);
1417         struct hnae3_knic_private_info *kinfo = &h->kinfo;
1418         u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1419         u8 tc = mqprio_qopt->qopt.num_tc;
1420         u16 mode = mqprio_qopt->mode;
1421         u8 hw = mqprio_qopt->qopt.hw;
1422         bool if_running;
1423         int ret;
1424
1425         if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1426                mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1427                 return -EOPNOTSUPP;
1428
1429         if (tc > HNAE3_MAX_TC)
1430                 return -EINVAL;
1431
1432         if (!netdev)
1433                 return -EINVAL;
1434
1435         if_running = netif_running(netdev);
1436         if (if_running) {
1437                 hns3_nic_net_stop(netdev);
1438                 msleep(100);
1439         }
1440
1441         ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1442                 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
1443         if (ret)
1444                 goto out;
1445
1446         ret = hns3_nic_set_real_num_queue(netdev);
1447
1448 out:
1449         if (if_running)
1450                 hns3_nic_net_open(netdev);
1451
1452         return ret;
1453 }
1454
1455 static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
1456                              void *type_data)
1457 {
1458         if (type != TC_SETUP_QDISC_MQPRIO)
1459                 return -EOPNOTSUPP;
1460
1461         return hns3_setup_tc(dev, type_data);
1462 }
1463
1464 static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1465                                 __be16 proto, u16 vid)
1466 {
1467         struct hnae3_handle *h = hns3_get_handle(netdev);
1468         struct hns3_nic_priv *priv = netdev_priv(netdev);
1469         int ret = -EIO;
1470
1471         if (h->ae_algo->ops->set_vlan_filter)
1472                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1473
1474         if (!ret)
1475                 set_bit(vid, priv->active_vlans);
1476
1477         return ret;
1478 }
1479
1480 static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1481                                  __be16 proto, u16 vid)
1482 {
1483         struct hnae3_handle *h = hns3_get_handle(netdev);
1484         struct hns3_nic_priv *priv = netdev_priv(netdev);
1485         int ret = -EIO;
1486
1487         if (h->ae_algo->ops->set_vlan_filter)
1488                 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1489
1490         if (!ret)
1491                 clear_bit(vid, priv->active_vlans);
1492
1493         return ret;
1494 }
1495
1496 static void hns3_restore_vlan(struct net_device *netdev)
1497 {
1498         struct hns3_nic_priv *priv = netdev_priv(netdev);
1499         u16 vid;
1500         int ret;
1501
1502         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1503                 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
1504                 if (ret)
1505                         netdev_warn(netdev, "Restore vlan: %d filter, ret:%d\n",
1506                                     vid, ret);
1507         }
1508 }
1509
1510 static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1511                                 u8 qos, __be16 vlan_proto)
1512 {
1513         struct hnae3_handle *h = hns3_get_handle(netdev);
1514         int ret = -EIO;
1515
1516         if (h->ae_algo->ops->set_vf_vlan_filter)
1517                 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1518                                                    qos, vlan_proto);
1519
1520         return ret;
1521 }
1522
1523 static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1524 {
1525         struct hnae3_handle *h = hns3_get_handle(netdev);
1526         bool if_running = netif_running(netdev);
1527         int ret;
1528
1529         if (!h->ae_algo->ops->set_mtu)
1530                 return -EOPNOTSUPP;
1531
1532         /* if this was called with netdev up then bring netdevice down */
1533         if (if_running) {
1534                 (void)hns3_nic_net_stop(netdev);
1535                 msleep(100);
1536         }
1537
1538         ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1539         if (ret)
1540                 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1541                            ret);
1542         else
1543                 netdev->mtu = new_mtu;
1544
1545         /* if the netdev was running earlier, bring it up again */
1546         if (if_running && hns3_nic_net_open(netdev))
1547                 ret = -EINVAL;
1548
1549         return ret;
1550 }
1551
1552 static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1553 {
1554         struct hns3_nic_priv *priv = netdev_priv(ndev);
1555         struct hns3_enet_ring *tx_ring = NULL;
1556         int timeout_queue = 0;
1557         int hw_head, hw_tail;
1558         int i;
1559
1560         /* Find the stopped queue the same way the stack does */
1561         for (i = 0; i < ndev->real_num_tx_queues; i++) {
1562                 struct netdev_queue *q;
1563                 unsigned long trans_start;
1564
1565                 q = netdev_get_tx_queue(ndev, i);
1566                 trans_start = q->trans_start;
1567                 if (netif_xmit_stopped(q) &&
1568                     time_after(jiffies,
1569                                (trans_start + ndev->watchdog_timeo))) {
1570                         timeout_queue = i;
1571                         break;
1572                 }
1573         }
1574
1575         if (i == ndev->num_tx_queues) {
1576                 netdev_info(ndev,
1577                             "no netdev TX timeout queue found, timeout count: %llu\n",
1578                             priv->tx_timeout_count);
1579                 return false;
1580         }
1581
1582         tx_ring = priv->ring_data[timeout_queue].ring;
1583
1584         hw_head = readl_relaxed(tx_ring->tqp->io_base +
1585                                 HNS3_RING_TX_RING_HEAD_REG);
1586         hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1587                                 HNS3_RING_TX_RING_TAIL_REG);
1588         netdev_info(ndev,
1589                     "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1590                     priv->tx_timeout_count,
1591                     timeout_queue,
1592                     tx_ring->next_to_use,
1593                     tx_ring->next_to_clean,
1594                     hw_head,
1595                     hw_tail,
1596                     readl(tx_ring->tqp_vector->mask_addr));
1597
1598         return true;
1599 }
1600
1601 static void hns3_nic_net_timeout(struct net_device *ndev)
1602 {
1603         struct hns3_nic_priv *priv = netdev_priv(ndev);
1604         struct hnae3_handle *h = priv->ae_handle;
1605
1606         if (!hns3_get_tx_timeo_queue_info(ndev))
1607                 return;
1608
1609         priv->tx_timeout_count++;
1610
1611         if (time_before(jiffies, (h->last_reset_time + ndev->watchdog_timeo)))
1612                 return;
1613
1614         /* request the reset */
1615         if (h->ae_algo->ops->reset_event)
1616                 h->ae_algo->ops->reset_event(h);
1617 }
1618
1619 static const struct net_device_ops hns3_nic_netdev_ops = {
1620         .ndo_open               = hns3_nic_net_open,
1621         .ndo_stop               = hns3_nic_net_stop,
1622         .ndo_start_xmit         = hns3_nic_net_xmit,
1623         .ndo_tx_timeout         = hns3_nic_net_timeout,
1624         .ndo_set_mac_address    = hns3_nic_net_set_mac_address,
1625         .ndo_do_ioctl           = hns3_nic_do_ioctl,
1626         .ndo_change_mtu         = hns3_nic_change_mtu,
1627         .ndo_set_features       = hns3_nic_set_features,
1628         .ndo_get_stats64        = hns3_nic_get_stats64,
1629         .ndo_setup_tc           = hns3_nic_setup_tc,
1630         .ndo_set_rx_mode        = hns3_nic_set_rx_mode,
1631         .ndo_vlan_rx_add_vid    = hns3_vlan_rx_add_vid,
1632         .ndo_vlan_rx_kill_vid   = hns3_vlan_rx_kill_vid,
1633         .ndo_set_vf_vlan        = hns3_ndo_set_vf_vlan,
1634 };
1635
1636 static bool hns3_is_phys_func(struct pci_dev *pdev)
1637 {
1638         u32 dev_id = pdev->device;
1639
1640         switch (dev_id) {
1641         case HNAE3_DEV_ID_GE:
1642         case HNAE3_DEV_ID_25GE:
1643         case HNAE3_DEV_ID_25GE_RDMA:
1644         case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1645         case HNAE3_DEV_ID_50GE_RDMA:
1646         case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1647         case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1648                 return true;
1649         case HNAE3_DEV_ID_100G_VF:
1650         case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1651                 return false;
1652         default:
1653                 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1654                          dev_id);
1655         }
1656
1657         return false;
1658 }
1659
1660 static void hns3_disable_sriov(struct pci_dev *pdev)
1661 {
1662         /* If our VFs are assigned we cannot shut down SR-IOV
1663          * without causing issues, so just leave the hardware
1664          * available but disabled
1665          */
1666         if (pci_vfs_assigned(pdev)) {
1667                 dev_warn(&pdev->dev,
1668                          "disabling driver while VFs are assigned\n");
1669                 return;
1670         }
1671
1672         pci_disable_sriov(pdev);
1673 }
1674
1675 static void hns3_get_dev_capability(struct pci_dev *pdev,
1676                                     struct hnae3_ae_dev *ae_dev)
1677 {
1678         if (pdev->revision >= 0x21)
1679                 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
1680 }
1681
1682 /* hns3_probe - Device initialization routine
1683  * @pdev: PCI device information struct
1684  * @ent: entry in hns3_pci_tbl
1685  *
1686  * hns3_probe initializes a PF identified by a pci_dev structure.
1687  * The OS initialization, configuring of the PF private structure,
1688  * and a hardware reset occur.
1689  *
1690  * Returns 0 on success, negative on failure
1691  */
1692 static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1693 {
1694         struct hnae3_ae_dev *ae_dev;
1695         int ret;
1696
1697         ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1698                               GFP_KERNEL);
1699         if (!ae_dev) {
1700                 ret = -ENOMEM;
1701                 return ret;
1702         }
1703
1704         ae_dev->pdev = pdev;
1705         ae_dev->flag = ent->driver_data;
1706         ae_dev->dev_type = HNAE3_DEV_KNIC;
1707         ae_dev->reset_type = HNAE3_NONE_RESET;
1708         hns3_get_dev_capability(pdev, ae_dev);
1709         pci_set_drvdata(pdev, ae_dev);
1710
1711         hnae3_register_ae_dev(ae_dev);
1712
1713         return 0;
1714 }
1715
1716 /* hns3_remove - Device removal routine
1717  * @pdev: PCI device information struct
1718  */
1719 static void hns3_remove(struct pci_dev *pdev)
1720 {
1721         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1722
1723         if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1724                 hns3_disable_sriov(pdev);
1725
1726         hnae3_unregister_ae_dev(ae_dev);
1727 }
1728
1729 /**
1730  * hns3_pci_sriov_configure
1731  * @pdev: pointer to a pci_dev structure
1732  * @num_vfs: number of VFs to allocate
1733  *
1734  * Enable or change the number of VFs. Called when the user updates the number
1735  * of VFs in sysfs.
1736  **/
1737 static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
1738 {
1739         int ret;
1740
1741         if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1742                 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1743                 return -EINVAL;
1744         }
1745
1746         if (num_vfs) {
1747                 ret = pci_enable_sriov(pdev, num_vfs);
1748                 if (ret)
1749                         dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
1750                 else
1751                         return num_vfs;
1752         } else if (!pci_vfs_assigned(pdev)) {
1753                 pci_disable_sriov(pdev);
1754         } else {
1755                 dev_warn(&pdev->dev,
1756                          "Unable to free VFs because some are assigned to VMs.\n");
1757         }
1758
1759         return 0;
1760 }
1761
1762 static void hns3_shutdown(struct pci_dev *pdev)
1763 {
1764         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1765
1766         hnae3_unregister_ae_dev(ae_dev);
1767         devm_kfree(&pdev->dev, ae_dev);
1768         pci_set_drvdata(pdev, NULL);
1769
1770         if (system_state == SYSTEM_POWER_OFF)
1771                 pci_set_power_state(pdev, PCI_D3hot);
1772 }
1773
1774 static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1775                                             pci_channel_state_t state)
1776 {
1777         struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1778         pci_ers_result_t ret;
1779
1780         dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1781
1782         if (state == pci_channel_io_perm_failure)
1783                 return PCI_ERS_RESULT_DISCONNECT;
1784
1785         if (!ae_dev) {
1786                 dev_err(&pdev->dev,
1787                         "Can't recover - error happened during device init\n");
1788                 return PCI_ERS_RESULT_NONE;
1789         }
1790
1791         if (ae_dev->ops->process_hw_error)
1792                 ret = ae_dev->ops->process_hw_error(ae_dev);
1793         else
1794                 return PCI_ERS_RESULT_NONE;
1795
1796         return ret;
1797 }
1798
1799 static const struct pci_error_handlers hns3_err_handler = {
1800         .error_detected = hns3_error_detected,
1801 };
1802
1803 static struct pci_driver hns3_driver = {
1804         .name     = hns3_driver_name,
1805         .id_table = hns3_pci_tbl,
1806         .probe    = hns3_probe,
1807         .remove   = hns3_remove,
1808         .shutdown = hns3_shutdown,
1809         .sriov_configure = hns3_pci_sriov_configure,
1810         .err_handler    = &hns3_err_handler,
1811 };
1812
1813 /* set default feature to hns3 */
1814 static void hns3_set_default_feature(struct net_device *netdev)
1815 {
1816         struct hnae3_handle *h = hns3_get_handle(netdev);
1817         struct pci_dev *pdev = h->pdev;
1818
1819         netdev->priv_flags |= IFF_UNICAST_FLT;
1820
1821         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1822                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1823                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1824                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1825                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1826
1827         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1828
1829         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1830
1831         netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1832                 NETIF_F_HW_VLAN_CTAG_FILTER |
1833                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1834                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1835                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1836                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1837                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1838
1839         netdev->vlan_features |=
1840                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1841                 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1842                 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1843                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1844                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1845
1846         netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1847                 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
1848                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1849                 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1850                 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1851                 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
1852
1853         if (pdev->revision >= 0x21) {
1854                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1855
1856                 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1857                         netdev->hw_features |= NETIF_F_NTUPLE;
1858                         netdev->features |= NETIF_F_NTUPLE;
1859                 }
1860         }
1861 }
1862
1863 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1864                              struct hns3_desc_cb *cb)
1865 {
1866         unsigned int order = hnae3_page_order(ring);
1867         struct page *p;
1868
1869         p = dev_alloc_pages(order);
1870         if (!p)
1871                 return -ENOMEM;
1872
1873         cb->priv = p;
1874         cb->page_offset = 0;
1875         cb->reuse_flag = 0;
1876         cb->buf  = page_address(p);
1877         cb->length = hnae3_page_size(ring);
1878         cb->type = DESC_TYPE_PAGE;
1879
1880         return 0;
1881 }
1882
1883 static void hns3_free_buffer(struct hns3_enet_ring *ring,
1884                              struct hns3_desc_cb *cb)
1885 {
1886         if (cb->type == DESC_TYPE_SKB)
1887                 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1888         else if (!HNAE3_IS_TX_RING(ring))
1889                 put_page((struct page *)cb->priv);
1890         memset(cb, 0, sizeof(*cb));
1891 }
1892
1893 static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1894 {
1895         cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1896                                cb->length, ring_to_dma_dir(ring));
1897
1898         if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
1899                 return -EIO;
1900
1901         return 0;
1902 }
1903
1904 static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1905                               struct hns3_desc_cb *cb)
1906 {
1907         if (cb->type == DESC_TYPE_SKB)
1908                 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1909                                  ring_to_dma_dir(ring));
1910         else if (cb->length)
1911                 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1912                                ring_to_dma_dir(ring));
1913 }
1914
1915 static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1916 {
1917         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1918         ring->desc[i].addr = 0;
1919 }
1920
1921 static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1922 {
1923         struct hns3_desc_cb *cb = &ring->desc_cb[i];
1924
1925         if (!ring->desc_cb[i].dma)
1926                 return;
1927
1928         hns3_buffer_detach(ring, i);
1929         hns3_free_buffer(ring, cb);
1930 }
1931
1932 static void hns3_free_buffers(struct hns3_enet_ring *ring)
1933 {
1934         int i;
1935
1936         for (i = 0; i < ring->desc_num; i++)
1937                 hns3_free_buffer_detach(ring, i);
1938 }
1939
1940 /* free desc along with its attached buffer */
1941 static void hns3_free_desc(struct hns3_enet_ring *ring)
1942 {
1943         int size = ring->desc_num * sizeof(ring->desc[0]);
1944
1945         hns3_free_buffers(ring);
1946
1947         if (ring->desc) {
1948                 dma_free_coherent(ring_to_dev(ring), size,
1949                                   ring->desc, ring->desc_dma_addr);
1950                 ring->desc = NULL;
1951         }
1952 }
1953
1954 static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1955 {
1956         int size = ring->desc_num * sizeof(ring->desc[0]);
1957
1958         ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
1959                                          &ring->desc_dma_addr,
1960                                          GFP_KERNEL);
1961         if (!ring->desc)
1962                 return -ENOMEM;
1963
1964         return 0;
1965 }
1966
1967 static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1968                                    struct hns3_desc_cb *cb)
1969 {
1970         int ret;
1971
1972         ret = hns3_alloc_buffer(ring, cb);
1973         if (ret)
1974                 goto out;
1975
1976         ret = hns3_map_buffer(ring, cb);
1977         if (ret)
1978                 goto out_with_buf;
1979
1980         return 0;
1981
1982 out_with_buf:
1983         hns3_free_buffer(ring, cb);
1984 out:
1985         return ret;
1986 }
1987
1988 static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1989 {
1990         int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1991
1992         if (ret)
1993                 return ret;
1994
1995         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1996
1997         return 0;
1998 }
1999
2000 /* Allocate memory for raw pkg, and map with dma */
2001 static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2002 {
2003         int i, j, ret;
2004
2005         for (i = 0; i < ring->desc_num; i++) {
2006                 ret = hns3_alloc_buffer_attach(ring, i);
2007                 if (ret)
2008                         goto out_buffer_fail;
2009         }
2010
2011         return 0;
2012
2013 out_buffer_fail:
2014         for (j = i - 1; j >= 0; j--)
2015                 hns3_free_buffer_detach(ring, j);
2016         return ret;
2017 }
2018
2019 /* detach a in-used buffer and replace with a reserved one  */
2020 static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2021                                 struct hns3_desc_cb *res_cb)
2022 {
2023         hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2024         ring->desc_cb[i] = *res_cb;
2025         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2026         ring->desc[i].rx.bd_base_info = 0;
2027 }
2028
2029 static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2030 {
2031         ring->desc_cb[i].reuse_flag = 0;
2032         ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2033                 + ring->desc_cb[i].page_offset);
2034         ring->desc[i].rx.bd_base_info = 0;
2035 }
2036
2037 static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2038                                       int *pkts)
2039 {
2040         struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2041
2042         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2043         (*bytes) += desc_cb->length;
2044         /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
2045         hns3_free_buffer_detach(ring, ring->next_to_clean);
2046
2047         ring_ptr_move_fw(ring, next_to_clean);
2048 }
2049
2050 static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2051 {
2052         int u = ring->next_to_use;
2053         int c = ring->next_to_clean;
2054
2055         if (unlikely(h > ring->desc_num))
2056                 return 0;
2057
2058         return u > c ? (h > c && h <= u) : (h > c || h <= u);
2059 }
2060
2061 void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
2062 {
2063         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2064         struct hns3_nic_priv *priv = netdev_priv(netdev);
2065         struct netdev_queue *dev_queue;
2066         int bytes, pkts;
2067         int head;
2068
2069         head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2070         rmb(); /* Make sure head is ready before touch any data */
2071
2072         if (is_ring_empty(ring) || head == ring->next_to_clean)
2073                 return; /* no data to poll */
2074
2075         if (unlikely(!is_valid_clean_head(ring, head))) {
2076                 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2077                            ring->next_to_use, ring->next_to_clean);
2078
2079                 u64_stats_update_begin(&ring->syncp);
2080                 ring->stats.io_err_cnt++;
2081                 u64_stats_update_end(&ring->syncp);
2082                 return;
2083         }
2084
2085         bytes = 0;
2086         pkts = 0;
2087         while (head != ring->next_to_clean) {
2088                 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2089                 /* Issue prefetch for next Tx descriptor */
2090                 prefetch(&ring->desc_cb[ring->next_to_clean]);
2091         }
2092
2093         ring->tqp_vector->tx_group.total_bytes += bytes;
2094         ring->tqp_vector->tx_group.total_packets += pkts;
2095
2096         u64_stats_update_begin(&ring->syncp);
2097         ring->stats.tx_bytes += bytes;
2098         ring->stats.tx_pkts += pkts;
2099         u64_stats_update_end(&ring->syncp);
2100
2101         dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2102         netdev_tx_completed_queue(dev_queue, pkts, bytes);
2103
2104         if (unlikely(pkts && netif_carrier_ok(netdev) &&
2105                      (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2106                 /* Make sure that anybody stopping the queue after this
2107                  * sees the new next_to_clean.
2108                  */
2109                 smp_mb();
2110                 if (netif_tx_queue_stopped(dev_queue) &&
2111                     !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
2112                         netif_tx_wake_queue(dev_queue);
2113                         ring->stats.restart_queue++;
2114                 }
2115         }
2116 }
2117
2118 static int hns3_desc_unused(struct hns3_enet_ring *ring)
2119 {
2120         int ntc = ring->next_to_clean;
2121         int ntu = ring->next_to_use;
2122
2123         return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2124 }
2125
2126 static void
2127 hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2128 {
2129         struct hns3_desc_cb *desc_cb;
2130         struct hns3_desc_cb res_cbs;
2131         int i, ret;
2132
2133         for (i = 0; i < cleand_count; i++) {
2134                 desc_cb = &ring->desc_cb[ring->next_to_use];
2135                 if (desc_cb->reuse_flag) {
2136                         u64_stats_update_begin(&ring->syncp);
2137                         ring->stats.reuse_pg_cnt++;
2138                         u64_stats_update_end(&ring->syncp);
2139
2140                         hns3_reuse_buffer(ring, ring->next_to_use);
2141                 } else {
2142                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
2143                         if (ret) {
2144                                 u64_stats_update_begin(&ring->syncp);
2145                                 ring->stats.sw_err_cnt++;
2146                                 u64_stats_update_end(&ring->syncp);
2147
2148                                 netdev_err(ring->tqp->handle->kinfo.netdev,
2149                                            "hnae reserve buffer map failed.\n");
2150                                 break;
2151                         }
2152                         hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2153                 }
2154
2155                 ring_ptr_move_fw(ring, next_to_use);
2156         }
2157
2158         wmb(); /* Make all data has been write before submit */
2159         writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2160 }
2161
2162 static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2163                                 struct hns3_enet_ring *ring, int pull_len,
2164                                 struct hns3_desc_cb *desc_cb)
2165 {
2166         struct hns3_desc *desc;
2167         u32 truesize;
2168         int size;
2169         int last_offset;
2170         bool twobufs;
2171
2172         twobufs = ((PAGE_SIZE < 8192) &&
2173                 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
2174
2175         desc = &ring->desc[ring->next_to_clean];
2176         size = le16_to_cpu(desc->rx.size);
2177
2178         truesize = hnae3_buf_size(ring);
2179
2180         if (!twobufs)
2181                 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
2182
2183         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
2184                         size - pull_len, truesize);
2185
2186          /* Avoid re-using remote pages,flag default unreuse */
2187         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2188                 return;
2189
2190         if (twobufs) {
2191                 /* If we are only owner of page we can reuse it */
2192                 if (likely(page_count(desc_cb->priv) == 1)) {
2193                         /* Flip page offset to other buffer */
2194                         desc_cb->page_offset ^= truesize;
2195
2196                         desc_cb->reuse_flag = 1;
2197                         /* bump ref count on page before it is given*/
2198                         get_page(desc_cb->priv);
2199                 }
2200                 return;
2201         }
2202
2203         /* Move offset up to the next cache line */
2204         desc_cb->page_offset += truesize;
2205
2206         if (desc_cb->page_offset <= last_offset) {
2207                 desc_cb->reuse_flag = 1;
2208                 /* Bump ref count on page before it is given*/
2209                 get_page(desc_cb->priv);
2210         }
2211 }
2212
2213 static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2214                              struct hns3_desc *desc)
2215 {
2216         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2217         int l3_type, l4_type;
2218         u32 bd_base_info;
2219         int ol4_type;
2220         u32 l234info;
2221
2222         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2223         l234info = le32_to_cpu(desc->rx.l234_info);
2224
2225         skb->ip_summed = CHECKSUM_NONE;
2226
2227         skb_checksum_none_assert(skb);
2228
2229         if (!(netdev->features & NETIF_F_RXCSUM))
2230                 return;
2231
2232         /* check if hardware has done checksum */
2233         if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
2234                 return;
2235
2236         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2237                      hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2238                      hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2239                      hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
2240                 u64_stats_update_begin(&ring->syncp);
2241                 ring->stats.l3l4_csum_err++;
2242                 u64_stats_update_end(&ring->syncp);
2243
2244                 return;
2245         }
2246
2247         l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2248                                   HNS3_RXD_L3ID_S);
2249         l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2250                                   HNS3_RXD_L4ID_S);
2251
2252         ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2253                                    HNS3_RXD_OL4ID_S);
2254         switch (ol4_type) {
2255         case HNS3_OL4_TYPE_MAC_IN_UDP:
2256         case HNS3_OL4_TYPE_NVGRE:
2257                 skb->csum_level = 1;
2258                 /* fall through */
2259         case HNS3_OL4_TYPE_NO_TUN:
2260                 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2261                 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2262                      l3_type == HNS3_L3_TYPE_IPV6) &&
2263                     (l4_type == HNS3_L4_TYPE_UDP ||
2264                      l4_type == HNS3_L4_TYPE_TCP ||
2265                      l4_type == HNS3_L4_TYPE_SCTP))
2266                         skb->ip_summed = CHECKSUM_UNNECESSARY;
2267                 break;
2268         default:
2269                 break;
2270         }
2271 }
2272
2273 static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2274 {
2275         napi_gro_receive(&ring->tqp_vector->napi, skb);
2276 }
2277
2278 static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2279                                 struct hns3_desc *desc, u32 l234info,
2280                                 u16 *vlan_tag)
2281 {
2282         struct pci_dev *pdev = ring->tqp->handle->pdev;
2283
2284         if (pdev->revision == 0x20) {
2285                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2286                 if (!(*vlan_tag & VLAN_VID_MASK))
2287                         *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2288
2289                 return (*vlan_tag != 0);
2290         }
2291
2292 #define HNS3_STRP_OUTER_VLAN    0x1
2293 #define HNS3_STRP_INNER_VLAN    0x2
2294
2295         switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2296                                 HNS3_RXD_STRP_TAGP_S)) {
2297         case HNS3_STRP_OUTER_VLAN:
2298                 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2299                 return true;
2300         case HNS3_STRP_INNER_VLAN:
2301                 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2302                 return true;
2303         default:
2304                 return false;
2305         }
2306 }
2307
2308 static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2309                                      struct sk_buff *skb)
2310 {
2311         struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2312         struct hnae3_handle *handle = ring->tqp->handle;
2313         enum pkt_hash_types rss_type;
2314
2315         if (le32_to_cpu(desc->rx.rss_hash))
2316                 rss_type = handle->kinfo.rss_type;
2317         else
2318                 rss_type = PKT_HASH_TYPE_NONE;
2319
2320         skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2321 }
2322
2323 static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2324                              struct sk_buff **out_skb, int *out_bnum)
2325 {
2326         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2327         struct hns3_desc_cb *desc_cb;
2328         struct hns3_desc *desc;
2329         struct sk_buff *skb;
2330         unsigned char *va;
2331         u32 bd_base_info;
2332         int pull_len;
2333         u32 l234info;
2334         int length;
2335         int bnum;
2336
2337         desc = &ring->desc[ring->next_to_clean];
2338         desc_cb = &ring->desc_cb[ring->next_to_clean];
2339
2340         prefetch(desc);
2341
2342         length = le16_to_cpu(desc->rx.size);
2343         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2344
2345         /* Check valid BD */
2346         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
2347                 return -EFAULT;
2348
2349         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2350
2351         /* Prefetch first cache line of first page
2352          * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2353          * line size is 64B so need to prefetch twice to make it 128B. But in
2354          * actual we can have greater size of caches with 128B Level 1 cache
2355          * lines. In such a case, single fetch would suffice to cache in the
2356          * relevant part of the header.
2357          */
2358         prefetch(va);
2359 #if L1_CACHE_BYTES < 128
2360         prefetch(va + L1_CACHE_BYTES);
2361 #endif
2362
2363         skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2364                                         HNS3_RX_HEAD_SIZE);
2365         if (unlikely(!skb)) {
2366                 netdev_err(netdev, "alloc rx skb fail\n");
2367
2368                 u64_stats_update_begin(&ring->syncp);
2369                 ring->stats.sw_err_cnt++;
2370                 u64_stats_update_end(&ring->syncp);
2371
2372                 return -ENOMEM;
2373         }
2374
2375         prefetchw(skb->data);
2376
2377         bnum = 1;
2378         if (length <= HNS3_RX_HEAD_SIZE) {
2379                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2380
2381                 /* We can reuse buffer as-is, just make sure it is local */
2382                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2383                         desc_cb->reuse_flag = 1;
2384                 else /* This page cannot be reused so discard it */
2385                         put_page(desc_cb->priv);
2386
2387                 ring_ptr_move_fw(ring, next_to_clean);
2388         } else {
2389                 u64_stats_update_begin(&ring->syncp);
2390                 ring->stats.seg_pkt_cnt++;
2391                 u64_stats_update_end(&ring->syncp);
2392
2393                 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2394
2395                 memcpy(__skb_put(skb, pull_len), va,
2396                        ALIGN(pull_len, sizeof(long)));
2397
2398                 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2399                 ring_ptr_move_fw(ring, next_to_clean);
2400
2401                 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2402                         desc = &ring->desc[ring->next_to_clean];
2403                         desc_cb = &ring->desc_cb[ring->next_to_clean];
2404                         bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2405                         hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2406                         ring_ptr_move_fw(ring, next_to_clean);
2407                         bnum++;
2408                 }
2409         }
2410
2411         *out_bnum = bnum;
2412
2413         l234info = le32_to_cpu(desc->rx.l234_info);
2414
2415         /* Based on hw strategy, the tag offloaded will be stored at
2416          * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2417          * in one layer tag case.
2418          */
2419         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2420                 u16 vlan_tag;
2421
2422                 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
2423                         __vlan_hwaccel_put_tag(skb,
2424                                                htons(ETH_P_8021Q),
2425                                                vlan_tag);
2426         }
2427
2428         if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2429                 u64_stats_update_begin(&ring->syncp);
2430                 ring->stats.non_vld_descs++;
2431                 u64_stats_update_end(&ring->syncp);
2432
2433                 dev_kfree_skb_any(skb);
2434                 return -EINVAL;
2435         }
2436
2437         if (unlikely((!desc->rx.pkt_len) ||
2438                      hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2439                 u64_stats_update_begin(&ring->syncp);
2440                 ring->stats.err_pkt_len++;
2441                 u64_stats_update_end(&ring->syncp);
2442
2443                 dev_kfree_skb_any(skb);
2444                 return -EFAULT;
2445         }
2446
2447         if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
2448                 u64_stats_update_begin(&ring->syncp);
2449                 ring->stats.l2_err++;
2450                 u64_stats_update_end(&ring->syncp);
2451
2452                 dev_kfree_skb_any(skb);
2453                 return -EFAULT;
2454         }
2455
2456         u64_stats_update_begin(&ring->syncp);
2457         ring->stats.rx_pkts++;
2458         ring->stats.rx_bytes += skb->len;
2459         u64_stats_update_end(&ring->syncp);
2460
2461         ring->tqp_vector->rx_group.total_bytes += skb->len;
2462
2463         hns3_rx_checksum(ring, skb, desc);
2464         hns3_set_rx_skb_rss_type(ring, skb);
2465
2466         return 0;
2467 }
2468
2469 int hns3_clean_rx_ring(
2470                 struct hns3_enet_ring *ring, int budget,
2471                 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
2472 {
2473 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2474         struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2475         int recv_pkts, recv_bds, clean_count, err;
2476         int unused_count = hns3_desc_unused(ring);
2477         struct sk_buff *skb = NULL;
2478         int num, bnum = 0;
2479
2480         num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2481         rmb(); /* Make sure num taken effect before the other data is touched */
2482
2483         recv_pkts = 0, recv_bds = 0, clean_count = 0;
2484         num -= unused_count;
2485
2486         while (recv_pkts < budget && recv_bds < num) {
2487                 /* Reuse or realloc buffers */
2488                 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2489                         hns3_nic_alloc_rx_buffers(ring,
2490                                                   clean_count + unused_count);
2491                         clean_count = 0;
2492                         unused_count = hns3_desc_unused(ring);
2493                 }
2494
2495                 /* Poll one pkt */
2496                 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2497                 if (unlikely(!skb)) /* This fault cannot be repaired */
2498                         goto out;
2499
2500                 recv_bds += bnum;
2501                 clean_count += bnum;
2502                 if (unlikely(err)) {  /* Do jump the err */
2503                         recv_pkts++;
2504                         continue;
2505                 }
2506
2507                 /* Do update ip stack process */
2508                 skb->protocol = eth_type_trans(skb, netdev);
2509                 rx_fn(ring, skb);
2510
2511                 recv_pkts++;
2512         }
2513
2514 out:
2515         /* Make all data has been write before submit */
2516         if (clean_count + unused_count > 0)
2517                 hns3_nic_alloc_rx_buffers(ring,
2518                                           clean_count + unused_count);
2519
2520         return recv_pkts;
2521 }
2522
2523 static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2524 {
2525         struct hns3_enet_tqp_vector *tqp_vector =
2526                                         ring_group->ring->tqp_vector;
2527         enum hns3_flow_level_range new_flow_level;
2528         int packets_per_msecs;
2529         int bytes_per_msecs;
2530         u32 time_passed_ms;
2531         u16 new_int_gl;
2532
2533         if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
2534                 return false;
2535
2536         if (ring_group->total_packets == 0) {
2537                 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2538                 ring_group->coal.flow_level = HNS3_FLOW_LOW;
2539                 return true;
2540         }
2541
2542         /* Simple throttlerate management
2543          * 0-10MB/s   lower     (50000 ints/s)
2544          * 10-20MB/s   middle    (20000 ints/s)
2545          * 20-1249MB/s high      (18000 ints/s)
2546          * > 40000pps  ultra     (8000 ints/s)
2547          */
2548         new_flow_level = ring_group->coal.flow_level;
2549         new_int_gl = ring_group->coal.int_gl;
2550         time_passed_ms =
2551                 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2552
2553         if (!time_passed_ms)
2554                 return false;
2555
2556         do_div(ring_group->total_packets, time_passed_ms);
2557         packets_per_msecs = ring_group->total_packets;
2558
2559         do_div(ring_group->total_bytes, time_passed_ms);
2560         bytes_per_msecs = ring_group->total_bytes;
2561
2562 #define HNS3_RX_LOW_BYTE_RATE 10000
2563 #define HNS3_RX_MID_BYTE_RATE 20000
2564
2565         switch (new_flow_level) {
2566         case HNS3_FLOW_LOW:
2567                 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
2568                         new_flow_level = HNS3_FLOW_MID;
2569                 break;
2570         case HNS3_FLOW_MID:
2571                 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
2572                         new_flow_level = HNS3_FLOW_HIGH;
2573                 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
2574                         new_flow_level = HNS3_FLOW_LOW;
2575                 break;
2576         case HNS3_FLOW_HIGH:
2577         case HNS3_FLOW_ULTRA:
2578         default:
2579                 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
2580                         new_flow_level = HNS3_FLOW_MID;
2581                 break;
2582         }
2583
2584 #define HNS3_RX_ULTRA_PACKET_RATE 40
2585
2586         if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2587             &tqp_vector->rx_group == ring_group)
2588                 new_flow_level = HNS3_FLOW_ULTRA;
2589
2590         switch (new_flow_level) {
2591         case HNS3_FLOW_LOW:
2592                 new_int_gl = HNS3_INT_GL_50K;
2593                 break;
2594         case HNS3_FLOW_MID:
2595                 new_int_gl = HNS3_INT_GL_20K;
2596                 break;
2597         case HNS3_FLOW_HIGH:
2598                 new_int_gl = HNS3_INT_GL_18K;
2599                 break;
2600         case HNS3_FLOW_ULTRA:
2601                 new_int_gl = HNS3_INT_GL_8K;
2602                 break;
2603         default:
2604                 break;
2605         }
2606
2607         ring_group->total_bytes = 0;
2608         ring_group->total_packets = 0;
2609         ring_group->coal.flow_level = new_flow_level;
2610         if (new_int_gl != ring_group->coal.int_gl) {
2611                 ring_group->coal.int_gl = new_int_gl;
2612                 return true;
2613         }
2614         return false;
2615 }
2616
2617 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2618 {
2619         struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2620         struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2621         bool rx_update, tx_update;
2622
2623         if (tqp_vector->int_adapt_down > 0) {
2624                 tqp_vector->int_adapt_down--;
2625                 return;
2626         }
2627
2628         if (rx_group->coal.gl_adapt_enable) {
2629                 rx_update = hns3_get_new_int_gl(rx_group);
2630                 if (rx_update)
2631                         hns3_set_vector_coalesce_rx_gl(tqp_vector,
2632                                                        rx_group->coal.int_gl);
2633         }
2634
2635         if (tx_group->coal.gl_adapt_enable) {
2636                 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2637                 if (tx_update)
2638                         hns3_set_vector_coalesce_tx_gl(tqp_vector,
2639                                                        tx_group->coal.int_gl);
2640         }
2641
2642         tqp_vector->last_jiffies = jiffies;
2643         tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
2644 }
2645
2646 static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2647 {
2648         struct hns3_enet_ring *ring;
2649         int rx_pkt_total = 0;
2650
2651         struct hns3_enet_tqp_vector *tqp_vector =
2652                 container_of(napi, struct hns3_enet_tqp_vector, napi);
2653         bool clean_complete = true;
2654         int rx_budget;
2655
2656         /* Since the actual Tx work is minimal, we can give the Tx a larger
2657          * budget and be more aggressive about cleaning up the Tx descriptors.
2658          */
2659         hns3_for_each_ring(ring, tqp_vector->tx_group)
2660                 hns3_clean_tx_ring(ring);
2661
2662         /* make sure rx ring budget not smaller than 1 */
2663         rx_budget = max(budget / tqp_vector->num_tqps, 1);
2664
2665         hns3_for_each_ring(ring, tqp_vector->rx_group) {
2666                 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2667                                                     hns3_rx_skb);
2668
2669                 if (rx_cleaned >= rx_budget)
2670                         clean_complete = false;
2671
2672                 rx_pkt_total += rx_cleaned;
2673         }
2674
2675         tqp_vector->rx_group.total_packets += rx_pkt_total;
2676
2677         if (!clean_complete)
2678                 return budget;
2679
2680         napi_complete(napi);
2681         hns3_update_new_int_gl(tqp_vector);
2682         hns3_mask_vector_irq(tqp_vector, 1);
2683
2684         return rx_pkt_total;
2685 }
2686
2687 static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2688                                       struct hnae3_ring_chain_node *head)
2689 {
2690         struct pci_dev *pdev = tqp_vector->handle->pdev;
2691         struct hnae3_ring_chain_node *cur_chain = head;
2692         struct hnae3_ring_chain_node *chain;
2693         struct hns3_enet_ring *tx_ring;
2694         struct hns3_enet_ring *rx_ring;
2695
2696         tx_ring = tqp_vector->tx_group.ring;
2697         if (tx_ring) {
2698                 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2699                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2700                               HNAE3_RING_TYPE_TX);
2701                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2702                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
2703
2704                 cur_chain->next = NULL;
2705
2706                 while (tx_ring->next) {
2707                         tx_ring = tx_ring->next;
2708
2709                         chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2710                                              GFP_KERNEL);
2711                         if (!chain)
2712                                 return -ENOMEM;
2713
2714                         cur_chain->next = chain;
2715                         chain->tqp_index = tx_ring->tqp->tqp_index;
2716                         hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2717                                       HNAE3_RING_TYPE_TX);
2718                         hnae3_set_field(chain->int_gl_idx,
2719                                         HNAE3_RING_GL_IDX_M,
2720                                         HNAE3_RING_GL_IDX_S,
2721                                         HNAE3_RING_GL_TX);
2722
2723                         cur_chain = chain;
2724                 }
2725         }
2726
2727         rx_ring = tqp_vector->rx_group.ring;
2728         if (!tx_ring && rx_ring) {
2729                 cur_chain->next = NULL;
2730                 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2731                 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2732                               HNAE3_RING_TYPE_RX);
2733                 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2734                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2735
2736                 rx_ring = rx_ring->next;
2737         }
2738
2739         while (rx_ring) {
2740                 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2741                 if (!chain)
2742                         return -ENOMEM;
2743
2744                 cur_chain->next = chain;
2745                 chain->tqp_index = rx_ring->tqp->tqp_index;
2746                 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2747                               HNAE3_RING_TYPE_RX);
2748                 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2749                                 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
2750
2751                 cur_chain = chain;
2752
2753                 rx_ring = rx_ring->next;
2754         }
2755
2756         return 0;
2757 }
2758
2759 static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2760                                         struct hnae3_ring_chain_node *head)
2761 {
2762         struct pci_dev *pdev = tqp_vector->handle->pdev;
2763         struct hnae3_ring_chain_node *chain_tmp, *chain;
2764
2765         chain = head->next;
2766
2767         while (chain) {
2768                 chain_tmp = chain->next;
2769                 devm_kfree(&pdev->dev, chain);
2770                 chain = chain_tmp;
2771         }
2772 }
2773
2774 static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2775                                    struct hns3_enet_ring *ring)
2776 {
2777         ring->next = group->ring;
2778         group->ring = ring;
2779
2780         group->count++;
2781 }
2782
2783 static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2784 {
2785         struct pci_dev *pdev = priv->ae_handle->pdev;
2786         struct hns3_enet_tqp_vector *tqp_vector;
2787         int num_vectors = priv->vector_num;
2788         int numa_node;
2789         int vector_i;
2790
2791         numa_node = dev_to_node(&pdev->dev);
2792
2793         for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2794                 tqp_vector = &priv->tqp_vector[vector_i];
2795                 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2796                                 &tqp_vector->affinity_mask);
2797         }
2798 }
2799
2800 static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2801 {
2802         struct hnae3_ring_chain_node vector_ring_chain;
2803         struct hnae3_handle *h = priv->ae_handle;
2804         struct hns3_enet_tqp_vector *tqp_vector;
2805         int ret = 0;
2806         u16 i;
2807
2808         hns3_nic_set_cpumask(priv);
2809
2810         for (i = 0; i < priv->vector_num; i++) {
2811                 tqp_vector = &priv->tqp_vector[i];
2812                 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2813                 tqp_vector->num_tqps = 0;
2814         }
2815
2816         for (i = 0; i < h->kinfo.num_tqps; i++) {
2817                 u16 vector_i = i % priv->vector_num;
2818                 u16 tqp_num = h->kinfo.num_tqps;
2819
2820                 tqp_vector = &priv->tqp_vector[vector_i];
2821
2822                 hns3_add_ring_to_group(&tqp_vector->tx_group,
2823                                        priv->ring_data[i].ring);
2824
2825                 hns3_add_ring_to_group(&tqp_vector->rx_group,
2826                                        priv->ring_data[i + tqp_num].ring);
2827
2828                 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2829                 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2830                 tqp_vector->num_tqps++;
2831         }
2832
2833         for (i = 0; i < priv->vector_num; i++) {
2834                 tqp_vector = &priv->tqp_vector[i];
2835
2836                 tqp_vector->rx_group.total_bytes = 0;
2837                 tqp_vector->rx_group.total_packets = 0;
2838                 tqp_vector->tx_group.total_bytes = 0;
2839                 tqp_vector->tx_group.total_packets = 0;
2840                 tqp_vector->handle = h;
2841
2842                 ret = hns3_get_vector_ring_chain(tqp_vector,
2843                                                  &vector_ring_chain);
2844                 if (ret)
2845                         return ret;
2846
2847                 ret = h->ae_algo->ops->map_ring_to_vector(h,
2848                         tqp_vector->vector_irq, &vector_ring_chain);
2849
2850                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2851
2852                 if (ret)
2853                         return ret;
2854
2855                 netif_napi_add(priv->netdev, &tqp_vector->napi,
2856                                hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2857         }
2858
2859         return 0;
2860 }
2861
2862 static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2863 {
2864         struct hnae3_handle *h = priv->ae_handle;
2865         struct hns3_enet_tqp_vector *tqp_vector;
2866         struct hnae3_vector_info *vector;
2867         struct pci_dev *pdev = h->pdev;
2868         u16 tqp_num = h->kinfo.num_tqps;
2869         u16 vector_num;
2870         int ret = 0;
2871         u16 i;
2872
2873         /* RSS size, cpu online and vector_num should be the same */
2874         /* Should consider 2p/4p later */
2875         vector_num = min_t(u16, num_online_cpus(), tqp_num);
2876         vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2877                               GFP_KERNEL);
2878         if (!vector)
2879                 return -ENOMEM;
2880
2881         vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2882
2883         priv->vector_num = vector_num;
2884         priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2885                 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2886                              GFP_KERNEL);
2887         if (!priv->tqp_vector) {
2888                 ret = -ENOMEM;
2889                 goto out;
2890         }
2891
2892         for (i = 0; i < priv->vector_num; i++) {
2893                 tqp_vector = &priv->tqp_vector[i];
2894                 tqp_vector->idx = i;
2895                 tqp_vector->mask_addr = vector[i].io_addr;
2896                 tqp_vector->vector_irq = vector[i].vector;
2897                 hns3_vector_gl_rl_init(tqp_vector, priv);
2898         }
2899
2900 out:
2901         devm_kfree(&pdev->dev, vector);
2902         return ret;
2903 }
2904
2905 static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
2906 {
2907         group->ring = NULL;
2908         group->count = 0;
2909 }
2910
2911 static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2912 {
2913         struct hnae3_ring_chain_node vector_ring_chain;
2914         struct hnae3_handle *h = priv->ae_handle;
2915         struct hns3_enet_tqp_vector *tqp_vector;
2916         int i, ret;
2917
2918         for (i = 0; i < priv->vector_num; i++) {
2919                 tqp_vector = &priv->tqp_vector[i];
2920
2921                 ret = hns3_get_vector_ring_chain(tqp_vector,
2922                                                  &vector_ring_chain);
2923                 if (ret)
2924                         return ret;
2925
2926                 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2927                         tqp_vector->vector_irq, &vector_ring_chain);
2928                 if (ret)
2929                         return ret;
2930
2931                 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2932
2933                 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2934                         (void)irq_set_affinity_hint(
2935                                 priv->tqp_vector[i].vector_irq,
2936                                                     NULL);
2937                         free_irq(priv->tqp_vector[i].vector_irq,
2938                                  &priv->tqp_vector[i]);
2939                 }
2940
2941                 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2942                 hns3_clear_ring_group(&tqp_vector->rx_group);
2943                 hns3_clear_ring_group(&tqp_vector->tx_group);
2944                 netif_napi_del(&priv->tqp_vector[i].napi);
2945         }
2946
2947         return 0;
2948 }
2949
2950 static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
2951 {
2952         struct hnae3_handle *h = priv->ae_handle;
2953         struct pci_dev *pdev = h->pdev;
2954         int i, ret;
2955
2956         for (i = 0; i < priv->vector_num; i++) {
2957                 struct hns3_enet_tqp_vector *tqp_vector;
2958
2959                 tqp_vector = &priv->tqp_vector[i];
2960                 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
2961                 if (ret)
2962                         return ret;
2963         }
2964
2965         devm_kfree(&pdev->dev, priv->tqp_vector);
2966         return 0;
2967 }
2968
2969 static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2970                              int ring_type)
2971 {
2972         struct hns3_nic_ring_data *ring_data = priv->ring_data;
2973         int queue_num = priv->ae_handle->kinfo.num_tqps;
2974         struct pci_dev *pdev = priv->ae_handle->pdev;
2975         struct hns3_enet_ring *ring;
2976
2977         ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2978         if (!ring)
2979                 return -ENOMEM;
2980
2981         if (ring_type == HNAE3_RING_TYPE_TX) {
2982                 ring_data[q->tqp_index].ring = ring;
2983                 ring_data[q->tqp_index].queue_index = q->tqp_index;
2984                 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2985         } else {
2986                 ring_data[q->tqp_index + queue_num].ring = ring;
2987                 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
2988                 ring->io_base = q->io_base;
2989         }
2990
2991         hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2992
2993         ring->tqp = q;
2994         ring->desc = NULL;
2995         ring->desc_cb = NULL;
2996         ring->dev = priv->dev;
2997         ring->desc_dma_addr = 0;
2998         ring->buf_size = q->buf_size;
2999         ring->desc_num = q->desc_num;
3000         ring->next_to_use = 0;
3001         ring->next_to_clean = 0;
3002
3003         return 0;
3004 }
3005
3006 static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3007                               struct hns3_nic_priv *priv)
3008 {
3009         int ret;
3010
3011         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3012         if (ret)
3013                 return ret;
3014
3015         ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
3016         if (ret)
3017                 return ret;
3018
3019         return 0;
3020 }
3021
3022 static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3023 {
3024         struct hnae3_handle *h = priv->ae_handle;
3025         struct pci_dev *pdev = h->pdev;
3026         int i, ret;
3027
3028         priv->ring_data =  devm_kzalloc(&pdev->dev,
3029                                         array3_size(h->kinfo.num_tqps,
3030                                                     sizeof(*priv->ring_data),
3031                                                     2),
3032                                         GFP_KERNEL);
3033         if (!priv->ring_data)
3034                 return -ENOMEM;
3035
3036         for (i = 0; i < h->kinfo.num_tqps; i++) {
3037                 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3038                 if (ret)
3039                         goto err;
3040         }
3041
3042         return 0;
3043 err:
3044         devm_kfree(&pdev->dev, priv->ring_data);
3045         return ret;
3046 }
3047
3048 static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3049 {
3050         struct hnae3_handle *h = priv->ae_handle;
3051         int i;
3052
3053         for (i = 0; i < h->kinfo.num_tqps; i++) {
3054                 devm_kfree(priv->dev, priv->ring_data[i].ring);
3055                 devm_kfree(priv->dev,
3056                            priv->ring_data[i + h->kinfo.num_tqps].ring);
3057         }
3058         devm_kfree(priv->dev, priv->ring_data);
3059 }
3060
3061 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3062 {
3063         int ret;
3064
3065         if (ring->desc_num <= 0 || ring->buf_size <= 0)
3066                 return -EINVAL;
3067
3068         ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3069                                 GFP_KERNEL);
3070         if (!ring->desc_cb) {
3071                 ret = -ENOMEM;
3072                 goto out;
3073         }
3074
3075         ret = hns3_alloc_desc(ring);
3076         if (ret)
3077                 goto out_with_desc_cb;
3078
3079         if (!HNAE3_IS_TX_RING(ring)) {
3080                 ret = hns3_alloc_ring_buffers(ring);
3081                 if (ret)
3082                         goto out_with_desc;
3083         }
3084
3085         return 0;
3086
3087 out_with_desc:
3088         hns3_free_desc(ring);
3089 out_with_desc_cb:
3090         kfree(ring->desc_cb);
3091         ring->desc_cb = NULL;
3092 out:
3093         return ret;
3094 }
3095
3096 static void hns3_fini_ring(struct hns3_enet_ring *ring)
3097 {
3098         hns3_free_desc(ring);
3099         kfree(ring->desc_cb);
3100         ring->desc_cb = NULL;
3101         ring->next_to_clean = 0;
3102         ring->next_to_use = 0;
3103 }
3104
3105 static int hns3_buf_size2type(u32 buf_size)
3106 {
3107         int bd_size_type;
3108
3109         switch (buf_size) {
3110         case 512:
3111                 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3112                 break;
3113         case 1024:
3114                 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3115                 break;
3116         case 2048:
3117                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3118                 break;
3119         case 4096:
3120                 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3121                 break;
3122         default:
3123                 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3124         }
3125
3126         return bd_size_type;
3127 }
3128
3129 static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3130 {
3131         dma_addr_t dma = ring->desc_dma_addr;
3132         struct hnae3_queue *q = ring->tqp;
3133
3134         if (!HNAE3_IS_TX_RING(ring)) {
3135                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3136                                (u32)dma);
3137                 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3138                                (u32)((dma >> 31) >> 1));
3139
3140                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3141                                hns3_buf_size2type(ring->buf_size));
3142                 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3143                                ring->desc_num / 8 - 1);
3144
3145         } else {
3146                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3147                                (u32)dma);
3148                 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3149                                (u32)((dma >> 31) >> 1));
3150
3151                 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3152                                ring->desc_num / 8 - 1);
3153         }
3154 }
3155
3156 static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3157 {
3158         struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3159         int i;
3160
3161         for (i = 0; i < HNAE3_MAX_TC; i++) {
3162                 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3163                 int j;
3164
3165                 if (!tc_info->enable)
3166                         continue;
3167
3168                 for (j = 0; j < tc_info->tqp_count; j++) {
3169                         struct hnae3_queue *q;
3170
3171                         q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3172                         hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3173                                        tc_info->tc);
3174                 }
3175         }
3176 }
3177
3178 int hns3_init_all_ring(struct hns3_nic_priv *priv)
3179 {
3180         struct hnae3_handle *h = priv->ae_handle;
3181         int ring_num = h->kinfo.num_tqps * 2;
3182         int i, j;
3183         int ret;
3184
3185         for (i = 0; i < ring_num; i++) {
3186                 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3187                 if (ret) {
3188                         dev_err(priv->dev,
3189                                 "Alloc ring memory fail! ret=%d\n", ret);
3190                         goto out_when_alloc_ring_memory;
3191                 }
3192
3193                 u64_stats_init(&priv->ring_data[i].ring->syncp);
3194         }
3195
3196         return 0;
3197
3198 out_when_alloc_ring_memory:
3199         for (j = i - 1; j >= 0; j--)
3200                 hns3_fini_ring(priv->ring_data[j].ring);
3201
3202         return -ENOMEM;
3203 }
3204
3205 int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
3206 {
3207         struct hnae3_handle *h = priv->ae_handle;
3208         int i;
3209
3210         for (i = 0; i < h->kinfo.num_tqps; i++) {
3211                 if (h->ae_algo->ops->reset_queue)
3212                         h->ae_algo->ops->reset_queue(h, i);
3213
3214                 hns3_fini_ring(priv->ring_data[i].ring);
3215                 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3216         }
3217         return 0;
3218 }
3219
3220 /* Set mac addr if it is configured. or leave it to the AE driver */
3221 static void hns3_init_mac_addr(struct net_device *netdev, bool init)
3222 {
3223         struct hns3_nic_priv *priv = netdev_priv(netdev);
3224         struct hnae3_handle *h = priv->ae_handle;
3225         u8 mac_addr_temp[ETH_ALEN];
3226
3227         if (h->ae_algo->ops->get_mac_addr && init) {
3228                 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3229                 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3230         }
3231
3232         /* Check if the MAC address is valid, if not get a random one */
3233         if (!is_valid_ether_addr(netdev->dev_addr)) {
3234                 eth_hw_addr_random(netdev);
3235                 dev_warn(priv->dev, "using random MAC address %pM\n",
3236                          netdev->dev_addr);
3237         }
3238
3239         if (h->ae_algo->ops->set_mac_addr)
3240                 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
3241
3242 }
3243
3244 static int hns3_restore_fd_rules(struct net_device *netdev)
3245 {
3246         struct hnae3_handle *h = hns3_get_handle(netdev);
3247         int ret = 0;
3248
3249         if (h->ae_algo->ops->restore_fd_rules)
3250                 ret = h->ae_algo->ops->restore_fd_rules(h);
3251
3252         return ret;
3253 }
3254
3255 static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3256 {
3257         struct hnae3_handle *h = hns3_get_handle(netdev);
3258
3259         if (h->ae_algo->ops->del_all_fd_entries)
3260                 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3261 }
3262
3263 static void hns3_nic_set_priv_ops(struct net_device *netdev)
3264 {
3265         struct hns3_nic_priv *priv = netdev_priv(netdev);
3266
3267         priv->ops.fill_desc = hns3_fill_desc;
3268         if ((netdev->features & NETIF_F_TSO) ||
3269             (netdev->features & NETIF_F_TSO6))
3270                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
3271         else
3272                 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
3273 }
3274
3275 static int hns3_client_init(struct hnae3_handle *handle)
3276 {
3277         struct pci_dev *pdev = handle->pdev;
3278         u16 alloc_tqps, max_rss_size;
3279         struct hns3_nic_priv *priv;
3280         struct net_device *netdev;
3281         int ret;
3282
3283         handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3284                                                     &max_rss_size);
3285         netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
3286         if (!netdev)
3287                 return -ENOMEM;
3288
3289         priv = netdev_priv(netdev);
3290         priv->dev = &pdev->dev;
3291         priv->netdev = netdev;
3292         priv->ae_handle = handle;
3293         priv->ae_handle->last_reset_time = jiffies;
3294         priv->tx_timeout_count = 0;
3295
3296         handle->kinfo.netdev = netdev;
3297         handle->priv = (void *)priv;
3298
3299         hns3_init_mac_addr(netdev, true);
3300
3301         hns3_set_default_feature(netdev);
3302
3303         netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3304         netdev->priv_flags |= IFF_UNICAST_FLT;
3305         netdev->netdev_ops = &hns3_nic_netdev_ops;
3306         SET_NETDEV_DEV(netdev, &pdev->dev);
3307         hns3_ethtool_set_ops(netdev);
3308         hns3_nic_set_priv_ops(netdev);
3309
3310         /* Carrier off reporting is important to ethtool even BEFORE open */
3311         netif_carrier_off(netdev);
3312
3313         if (handle->flags & HNAE3_SUPPORT_VF)
3314                 handle->reset_level = HNAE3_VF_RESET;
3315         else
3316                 handle->reset_level = HNAE3_FUNC_RESET;
3317
3318         ret = hns3_get_ring_config(priv);
3319         if (ret) {
3320                 ret = -ENOMEM;
3321                 goto out_get_ring_cfg;
3322         }
3323
3324         ret = hns3_nic_alloc_vector_data(priv);
3325         if (ret) {
3326                 ret = -ENOMEM;
3327                 goto out_alloc_vector_data;
3328         }
3329
3330         ret = hns3_nic_init_vector_data(priv);
3331         if (ret) {
3332                 ret = -ENOMEM;
3333                 goto out_init_vector_data;
3334         }
3335
3336         ret = hns3_init_all_ring(priv);
3337         if (ret) {
3338                 ret = -ENOMEM;
3339                 goto out_init_ring_data;
3340         }
3341
3342         ret = register_netdev(netdev);
3343         if (ret) {
3344                 dev_err(priv->dev, "probe register netdev fail!\n");
3345                 goto out_reg_netdev_fail;
3346         }
3347
3348         hns3_dcbnl_setup(handle);
3349
3350         /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3351         netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3352
3353         return ret;
3354
3355 out_reg_netdev_fail:
3356 out_init_ring_data:
3357         (void)hns3_nic_uninit_vector_data(priv);
3358 out_init_vector_data:
3359         hns3_nic_dealloc_vector_data(priv);
3360 out_alloc_vector_data:
3361         priv->ring_data = NULL;
3362 out_get_ring_cfg:
3363         priv->ae_handle = NULL;
3364         free_netdev(netdev);
3365         return ret;
3366 }
3367
3368 static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3369 {
3370         struct net_device *netdev = handle->kinfo.netdev;
3371         struct hns3_nic_priv *priv = netdev_priv(netdev);
3372         int ret;
3373
3374         hns3_remove_hw_addr(netdev);
3375
3376         if (netdev->reg_state != NETREG_UNINITIALIZED)
3377                 unregister_netdev(netdev);
3378
3379         hns3_del_all_fd_rules(netdev, true);
3380
3381         hns3_force_clear_all_rx_ring(handle);
3382
3383         ret = hns3_nic_uninit_vector_data(priv);
3384         if (ret)
3385                 netdev_err(netdev, "uninit vector error\n");
3386
3387         ret = hns3_nic_dealloc_vector_data(priv);
3388         if (ret)
3389                 netdev_err(netdev, "dealloc vector error\n");
3390
3391         ret = hns3_uninit_all_ring(priv);
3392         if (ret)
3393                 netdev_err(netdev, "uninit ring error\n");
3394
3395         hns3_put_ring_config(priv);
3396
3397         priv->ring_data = NULL;
3398
3399         free_netdev(netdev);
3400 }
3401
3402 static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3403 {
3404         struct net_device *netdev = handle->kinfo.netdev;
3405
3406         if (!netdev)
3407                 return;
3408
3409         if (linkup) {
3410                 netif_carrier_on(netdev);
3411                 netif_tx_wake_all_queues(netdev);
3412                 netdev_info(netdev, "link up\n");
3413         } else {
3414                 netif_carrier_off(netdev);
3415                 netif_tx_stop_all_queues(netdev);
3416                 netdev_info(netdev, "link down\n");
3417         }
3418 }
3419
3420 static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3421 {
3422         struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3423         struct net_device *ndev = kinfo->netdev;
3424         bool if_running;
3425         int ret;
3426
3427         if (tc > HNAE3_MAX_TC)
3428                 return -EINVAL;
3429
3430         if (!ndev)
3431                 return -ENODEV;
3432
3433         if_running = netif_running(ndev);
3434
3435         if (if_running) {
3436                 (void)hns3_nic_net_stop(ndev);
3437                 msleep(100);
3438         }
3439
3440         ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3441                 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3442         if (ret)
3443                 goto err_out;
3444
3445         ret = hns3_nic_set_real_num_queue(ndev);
3446
3447 err_out:
3448         if (if_running)
3449                 (void)hns3_nic_net_open(ndev);
3450
3451         return ret;
3452 }
3453
3454 static void hns3_recover_hw_addr(struct net_device *ndev)
3455 {
3456         struct netdev_hw_addr_list *list;
3457         struct netdev_hw_addr *ha, *tmp;
3458
3459         /* go through and sync uc_addr entries to the device */
3460         list = &ndev->uc;
3461         list_for_each_entry_safe(ha, tmp, &list->list, list)
3462                 hns3_nic_uc_sync(ndev, ha->addr);
3463
3464         /* go through and sync mc_addr entries to the device */
3465         list = &ndev->mc;
3466         list_for_each_entry_safe(ha, tmp, &list->list, list)
3467                 hns3_nic_mc_sync(ndev, ha->addr);
3468 }
3469
3470 static void hns3_remove_hw_addr(struct net_device *netdev)
3471 {
3472         struct netdev_hw_addr_list *list;
3473         struct netdev_hw_addr *ha, *tmp;
3474
3475         hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3476
3477         /* go through and unsync uc_addr entries to the device */
3478         list = &netdev->uc;
3479         list_for_each_entry_safe(ha, tmp, &list->list, list)
3480                 hns3_nic_uc_unsync(netdev, ha->addr);
3481
3482         /* go through and unsync mc_addr entries to the device */
3483         list = &netdev->mc;
3484         list_for_each_entry_safe(ha, tmp, &list->list, list)
3485                 if (ha->refcount > 1)
3486                         hns3_nic_mc_unsync(netdev, ha->addr);
3487 }
3488
3489 static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
3490 {
3491         while (ring->next_to_clean != ring->next_to_use) {
3492                 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
3493                 hns3_free_buffer_detach(ring, ring->next_to_clean);
3494                 ring_ptr_move_fw(ring, next_to_clean);
3495         }
3496 }
3497
3498 static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3499 {
3500         struct hns3_desc_cb res_cbs;
3501         int ret;
3502
3503         while (ring->next_to_use != ring->next_to_clean) {
3504                 /* When a buffer is not reused, it's memory has been
3505                  * freed in hns3_handle_rx_bd or will be freed by
3506                  * stack, so we need to replace the buffer here.
3507                  */
3508                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3509                         ret = hns3_reserve_buffer_map(ring, &res_cbs);
3510                         if (ret) {
3511                                 u64_stats_update_begin(&ring->syncp);
3512                                 ring->stats.sw_err_cnt++;
3513                                 u64_stats_update_end(&ring->syncp);
3514                                 /* if alloc new buffer fail, exit directly
3515                                  * and reclear in up flow.
3516                                  */
3517                                 netdev_warn(ring->tqp->handle->kinfo.netdev,
3518                                             "reserve buffer map failed, ret = %d\n",
3519                                             ret);
3520                                 return ret;
3521                         }
3522                         hns3_replace_buffer(ring, ring->next_to_use,
3523                                             &res_cbs);
3524                 }
3525                 ring_ptr_move_fw(ring, next_to_use);
3526         }
3527
3528         return 0;
3529 }
3530
3531 static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
3532 {
3533         while (ring->next_to_use != ring->next_to_clean) {
3534                 /* When a buffer is not reused, it's memory has been
3535                  * freed in hns3_handle_rx_bd or will be freed by
3536                  * stack, so only need to unmap the buffer here.
3537                  */
3538                 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3539                         hns3_unmap_buffer(ring,
3540                                           &ring->desc_cb[ring->next_to_use]);
3541                         ring->desc_cb[ring->next_to_use].dma = 0;
3542                 }
3543
3544                 ring_ptr_move_fw(ring, next_to_use);
3545         }
3546 }
3547
3548 static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3549 {
3550         struct net_device *ndev = h->kinfo.netdev;
3551         struct hns3_nic_priv *priv = netdev_priv(ndev);
3552         struct hns3_enet_ring *ring;
3553         u32 i;
3554
3555         for (i = 0; i < h->kinfo.num_tqps; i++) {
3556                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3557                 hns3_force_clear_rx_ring(ring);
3558         }
3559 }
3560
3561 static void hns3_clear_all_ring(struct hnae3_handle *h)
3562 {
3563         struct net_device *ndev = h->kinfo.netdev;
3564         struct hns3_nic_priv *priv = netdev_priv(ndev);
3565         u32 i;
3566
3567         for (i = 0; i < h->kinfo.num_tqps; i++) {
3568                 struct netdev_queue *dev_queue;
3569                 struct hns3_enet_ring *ring;
3570
3571                 ring = priv->ring_data[i].ring;
3572                 hns3_clear_tx_ring(ring);
3573                 dev_queue = netdev_get_tx_queue(ndev,
3574                                                 priv->ring_data[i].queue_index);
3575                 netdev_tx_reset_queue(dev_queue);
3576
3577                 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3578                 /* Continue to clear other rings even if clearing some
3579                  * rings failed.
3580                  */
3581                 hns3_clear_rx_ring(ring);
3582         }
3583 }
3584
3585 int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3586 {
3587         struct net_device *ndev = h->kinfo.netdev;
3588         struct hns3_nic_priv *priv = netdev_priv(ndev);
3589         struct hns3_enet_ring *rx_ring;
3590         int i, j;
3591         int ret;
3592
3593         for (i = 0; i < h->kinfo.num_tqps; i++) {
3594                 h->ae_algo->ops->reset_queue(h, i);
3595                 hns3_init_ring_hw(priv->ring_data[i].ring);
3596
3597                 /* We need to clear tx ring here because self test will
3598                  * use the ring and will not run down before up
3599                  */
3600                 hns3_clear_tx_ring(priv->ring_data[i].ring);
3601                 priv->ring_data[i].ring->next_to_clean = 0;
3602                 priv->ring_data[i].ring->next_to_use = 0;
3603
3604                 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3605                 hns3_init_ring_hw(rx_ring);
3606                 ret = hns3_clear_rx_ring(rx_ring);
3607                 if (ret)
3608                         return ret;
3609
3610                 /* We can not know the hardware head and tail when this
3611                  * function is called in reset flow, so we reuse all desc.
3612                  */
3613                 for (j = 0; j < rx_ring->desc_num; j++)
3614                         hns3_reuse_buffer(rx_ring, j);
3615
3616                 rx_ring->next_to_clean = 0;
3617                 rx_ring->next_to_use = 0;
3618         }
3619
3620         hns3_init_tx_ring_tc(priv);
3621
3622         return 0;
3623 }
3624
3625 static void hns3_store_coal(struct hns3_nic_priv *priv)
3626 {
3627         /* ethtool only support setting and querying one coal
3628          * configuation for now, so save the vector 0' coal
3629          * configuation here in order to restore it.
3630          */
3631         memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3632                sizeof(struct hns3_enet_coalesce));
3633         memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3634                sizeof(struct hns3_enet_coalesce));
3635 }
3636
3637 static void hns3_restore_coal(struct hns3_nic_priv *priv)
3638 {
3639         u16 vector_num = priv->vector_num;
3640         int i;
3641
3642         for (i = 0; i < vector_num; i++) {
3643                 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3644                        sizeof(struct hns3_enet_coalesce));
3645                 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3646                        sizeof(struct hns3_enet_coalesce));
3647         }
3648 }