ec638778661ccd7aadecb857bfdb9b8e59929524
[muen/linux.git] / drivers / infiniband / hw / hns / hns_roce_hw_v2.c
1 /*
2  * Copyright (c) 2016-2017 Hisilicon Limited.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/kernel.h>
37 #include <net/addrconf.h>
38 #include <rdma/ib_umem.h>
39
40 #include "hnae3.h"
41 #include "hns_roce_common.h"
42 #include "hns_roce_device.h"
43 #include "hns_roce_cmd.h"
44 #include "hns_roce_hem.h"
45 #include "hns_roce_hw_v2.h"
46
47 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
48                             struct ib_sge *sg)
49 {
50         dseg->lkey = cpu_to_le32(sg->lkey);
51         dseg->addr = cpu_to_le64(sg->addr);
52         dseg->len  = cpu_to_le32(sg->length);
53 }
54
55 static int set_rwqe_data_seg(struct ib_qp *ibqp, struct ib_send_wr *wr,
56                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
57                              void *wqe, unsigned int *sge_ind,
58                              struct ib_send_wr **bad_wr)
59 {
60         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
61         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
62         struct hns_roce_qp *qp = to_hr_qp(ibqp);
63         int i;
64
65         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
66                 if (le32_to_cpu(rc_sq_wqe->msg_len) >
67                     hr_dev->caps.max_sq_inline) {
68                         *bad_wr = wr;
69                         dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
70                                 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
71                         return -EINVAL;
72                 }
73
74                 for (i = 0; i < wr->num_sge; i++) {
75                         memcpy(wqe, ((void *)wr->sg_list[i].addr),
76                                wr->sg_list[i].length);
77                         wqe += wr->sg_list[i].length;
78                 }
79
80                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
81                              1);
82         } else {
83                 if (wr->num_sge <= 2) {
84                         for (i = 0; i < wr->num_sge; i++) {
85                                 if (likely(wr->sg_list[i].length)) {
86                                         set_data_seg_v2(dseg, wr->sg_list + i);
87                                         dseg++;
88                                 }
89                         }
90                 } else {
91                         roce_set_field(rc_sq_wqe->byte_20,
92                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
93                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
94                                      (*sge_ind) & (qp->sge.sge_cnt - 1));
95
96                         for (i = 0; i < 2; i++) {
97                                 if (likely(wr->sg_list[i].length)) {
98                                         set_data_seg_v2(dseg, wr->sg_list + i);
99                                         dseg++;
100                                 }
101                         }
102
103                         dseg = get_send_extend_sge(qp,
104                                             (*sge_ind) & (qp->sge.sge_cnt - 1));
105
106                         for (i = 0; i < wr->num_sge - 2; i++) {
107                                 if (likely(wr->sg_list[i + 2].length)) {
108                                         set_data_seg_v2(dseg,
109                                                         wr->sg_list + 2 + i);
110                                         dseg++;
111                                         (*sge_ind)++;
112                                 }
113                         }
114                 }
115
116                 roce_set_field(rc_sq_wqe->byte_16,
117                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
118                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
119         }
120
121         return 0;
122 }
123
124 static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
125                                  struct ib_send_wr **bad_wr)
126 {
127         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
128         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
129         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
130         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
131         struct hns_roce_qp *qp = to_hr_qp(ibqp);
132         struct hns_roce_v2_wqe_data_seg *dseg;
133         struct device *dev = hr_dev->dev;
134         struct hns_roce_v2_db sq_db;
135         unsigned int sge_ind = 0;
136         unsigned int owner_bit;
137         unsigned long flags;
138         unsigned int ind;
139         void *wqe = NULL;
140         u32 tmp_len = 0;
141         bool loopback;
142         int ret = 0;
143         u8 *smac;
144         int nreq;
145         int i;
146
147         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
148                      ibqp->qp_type != IB_QPT_GSI &&
149                      ibqp->qp_type != IB_QPT_UD)) {
150                 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
151                 *bad_wr = NULL;
152                 return -EOPNOTSUPP;
153         }
154
155         if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
156                      qp->state == IB_QPS_RTR)) {
157                 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
158                 *bad_wr = wr;
159                 return -EINVAL;
160         }
161
162         spin_lock_irqsave(&qp->sq.lock, flags);
163         ind = qp->sq_next_wqe;
164         sge_ind = qp->next_sge;
165
166         for (nreq = 0; wr; ++nreq, wr = wr->next) {
167                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
168                         ret = -ENOMEM;
169                         *bad_wr = wr;
170                         goto out;
171                 }
172
173                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
174                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
175                                 wr->num_sge, qp->sq.max_gs);
176                         ret = -EINVAL;
177                         *bad_wr = wr;
178                         goto out;
179                 }
180
181                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
182                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
183                                                                       wr->wr_id;
184
185                 owner_bit = ~(qp->sq.head >> ilog2(qp->sq.wqe_cnt)) & 0x1;
186
187                 /* Corresponding to the QP type, wqe process separately */
188                 if (ibqp->qp_type == IB_QPT_GSI) {
189                         ud_sq_wqe = wqe;
190                         memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
191
192                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
193                                        V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
194                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
195                                        V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
196                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
197                                        V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
198                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
199                                        V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
200                         roce_set_field(ud_sq_wqe->byte_48,
201                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
202                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
203                                        ah->av.mac[4]);
204                         roce_set_field(ud_sq_wqe->byte_48,
205                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
206                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
207                                        ah->av.mac[5]);
208
209                         /* MAC loopback */
210                         smac = (u8 *)hr_dev->dev_addr[qp->port];
211                         loopback = ether_addr_equal_unaligned(ah->av.mac,
212                                                               smac) ? 1 : 0;
213
214                         roce_set_bit(ud_sq_wqe->byte_40,
215                                      V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
216
217                         roce_set_field(ud_sq_wqe->byte_4,
218                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
219                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
220                                        HNS_ROCE_V2_WQE_OP_SEND);
221
222                         for (i = 0; i < wr->num_sge; i++)
223                                 tmp_len += wr->sg_list[i].length;
224
225                         ud_sq_wqe->msg_len =
226                          cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
227
228                         switch (wr->opcode) {
229                         case IB_WR_SEND_WITH_IMM:
230                         case IB_WR_RDMA_WRITE_WITH_IMM:
231                                 ud_sq_wqe->immtdata = wr->ex.imm_data;
232                                 break;
233                         default:
234                                 ud_sq_wqe->immtdata = 0;
235                                 break;
236                         }
237
238                         /* Set sig attr */
239                         roce_set_bit(ud_sq_wqe->byte_4,
240                                    V2_UD_SEND_WQE_BYTE_4_CQE_S,
241                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
242
243                         /* Set se attr */
244                         roce_set_bit(ud_sq_wqe->byte_4,
245                                   V2_UD_SEND_WQE_BYTE_4_SE_S,
246                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
247
248                         roce_set_bit(ud_sq_wqe->byte_4,
249                                      V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
250
251                         roce_set_field(ud_sq_wqe->byte_16,
252                                        V2_UD_SEND_WQE_BYTE_16_PD_M,
253                                        V2_UD_SEND_WQE_BYTE_16_PD_S,
254                                        to_hr_pd(ibqp->pd)->pdn);
255
256                         roce_set_field(ud_sq_wqe->byte_16,
257                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
258                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
259                                        wr->num_sge);
260
261                         roce_set_field(ud_sq_wqe->byte_20,
262                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
263                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
264                                      sge_ind & (qp->sge.sge_cnt - 1));
265
266                         roce_set_field(ud_sq_wqe->byte_24,
267                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
268                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
269                         ud_sq_wqe->qkey =
270                              cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
271                              qp->qkey : ud_wr(wr)->remote_qkey);
272                         roce_set_field(ud_sq_wqe->byte_32,
273                                        V2_UD_SEND_WQE_BYTE_32_DQPN_M,
274                                        V2_UD_SEND_WQE_BYTE_32_DQPN_S,
275                                        ud_wr(wr)->remote_qpn);
276
277                         roce_set_field(ud_sq_wqe->byte_36,
278                                        V2_UD_SEND_WQE_BYTE_36_VLAN_M,
279                                        V2_UD_SEND_WQE_BYTE_36_VLAN_S,
280                                        le16_to_cpu(ah->av.vlan));
281                         roce_set_field(ud_sq_wqe->byte_36,
282                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
283                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
284                                        ah->av.hop_limit);
285                         roce_set_field(ud_sq_wqe->byte_36,
286                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
287                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
288                                        0);
289                         roce_set_field(ud_sq_wqe->byte_36,
290                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
291                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
292                                        0);
293                         roce_set_field(ud_sq_wqe->byte_40,
294                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
295                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, 0);
296                         roce_set_field(ud_sq_wqe->byte_40,
297                                        V2_UD_SEND_WQE_BYTE_40_SL_M,
298                                        V2_UD_SEND_WQE_BYTE_40_SL_S,
299                                       le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
300                                       HNS_ROCE_SL_SHIFT);
301                         roce_set_field(ud_sq_wqe->byte_40,
302                                        V2_UD_SEND_WQE_BYTE_40_PORTN_M,
303                                        V2_UD_SEND_WQE_BYTE_40_PORTN_S,
304                                        qp->port);
305
306                         roce_set_field(ud_sq_wqe->byte_48,
307                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
308                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
309                                        hns_get_gid_index(hr_dev, qp->phy_port,
310                                                          ah->av.gid_index));
311
312                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
313                                GID_LEN_V2);
314
315                         dseg = get_send_extend_sge(qp,
316                                             sge_ind & (qp->sge.sge_cnt - 1));
317                         for (i = 0; i < wr->num_sge; i++) {
318                                 set_data_seg_v2(dseg + i, wr->sg_list + i);
319                                 sge_ind++;
320                         }
321
322                         ind++;
323                 } else if (ibqp->qp_type == IB_QPT_RC) {
324                         rc_sq_wqe = wqe;
325                         memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
326                         for (i = 0; i < wr->num_sge; i++)
327                                 tmp_len += wr->sg_list[i].length;
328
329                         rc_sq_wqe->msg_len =
330                          cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
331
332                         switch (wr->opcode) {
333                         case IB_WR_SEND_WITH_IMM:
334                         case IB_WR_RDMA_WRITE_WITH_IMM:
335                                 rc_sq_wqe->immtdata = wr->ex.imm_data;
336                                 break;
337                         case IB_WR_SEND_WITH_INV:
338                                 rc_sq_wqe->inv_key =
339                                         cpu_to_le32(wr->ex.invalidate_rkey);
340                                 break;
341                         default:
342                                 rc_sq_wqe->immtdata = 0;
343                                 break;
344                         }
345
346                         roce_set_bit(rc_sq_wqe->byte_4,
347                                      V2_RC_SEND_WQE_BYTE_4_FENCE_S,
348                                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
349
350                         roce_set_bit(rc_sq_wqe->byte_4,
351                                   V2_RC_SEND_WQE_BYTE_4_SE_S,
352                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
353
354                         roce_set_bit(rc_sq_wqe->byte_4,
355                                    V2_RC_SEND_WQE_BYTE_4_CQE_S,
356                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
357
358                         roce_set_bit(rc_sq_wqe->byte_4,
359                                      V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
360
361                         switch (wr->opcode) {
362                         case IB_WR_RDMA_READ:
363                                 roce_set_field(rc_sq_wqe->byte_4,
364                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
365                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
366                                                HNS_ROCE_V2_WQE_OP_RDMA_READ);
367                                 rc_sq_wqe->rkey =
368                                         cpu_to_le32(rdma_wr(wr)->rkey);
369                                 rc_sq_wqe->va =
370                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
371                                 break;
372                         case IB_WR_RDMA_WRITE:
373                                 roce_set_field(rc_sq_wqe->byte_4,
374                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
375                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
376                                                HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
377                                 rc_sq_wqe->rkey =
378                                         cpu_to_le32(rdma_wr(wr)->rkey);
379                                 rc_sq_wqe->va =
380                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
381                                 break;
382                         case IB_WR_RDMA_WRITE_WITH_IMM:
383                                 roce_set_field(rc_sq_wqe->byte_4,
384                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
385                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
386                                        HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM);
387                                 rc_sq_wqe->rkey =
388                                         cpu_to_le32(rdma_wr(wr)->rkey);
389                                 rc_sq_wqe->va =
390                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
391                                 break;
392                         case IB_WR_SEND:
393                                 roce_set_field(rc_sq_wqe->byte_4,
394                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
395                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
396                                                HNS_ROCE_V2_WQE_OP_SEND);
397                                 break;
398                         case IB_WR_SEND_WITH_INV:
399                                 roce_set_field(rc_sq_wqe->byte_4,
400                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
401                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
402                                        HNS_ROCE_V2_WQE_OP_SEND_WITH_INV);
403                                 break;
404                         case IB_WR_SEND_WITH_IMM:
405                                 roce_set_field(rc_sq_wqe->byte_4,
406                                               V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
407                                               V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
408                                               HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
409                                 break;
410                         case IB_WR_LOCAL_INV:
411                                 roce_set_field(rc_sq_wqe->byte_4,
412                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
413                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
414                                                HNS_ROCE_V2_WQE_OP_LOCAL_INV);
415                                 break;
416                         case IB_WR_ATOMIC_CMP_AND_SWP:
417                                 roce_set_field(rc_sq_wqe->byte_4,
418                                           V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
419                                           V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
420                                           HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
421                                 break;
422                         case IB_WR_ATOMIC_FETCH_AND_ADD:
423                                 roce_set_field(rc_sq_wqe->byte_4,
424                                          V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
425                                          V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
426                                          HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
427                                 break;
428                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
429                                 roce_set_field(rc_sq_wqe->byte_4,
430                                       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
431                                       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
432                                       HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP);
433                                 break;
434                         case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
435                                 roce_set_field(rc_sq_wqe->byte_4,
436                                      V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
437                                      V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
438                                      HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD);
439                                 break;
440                         default:
441                                 roce_set_field(rc_sq_wqe->byte_4,
442                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
443                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
444                                                HNS_ROCE_V2_WQE_OP_MASK);
445                                 break;
446                         }
447
448                         wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
449                         dseg = wqe;
450
451                         ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, wqe,
452                                                 &sge_ind, bad_wr);
453                         if (ret)
454                                 goto out;
455                         ind++;
456                 } else {
457                         dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
458                         spin_unlock_irqrestore(&qp->sq.lock, flags);
459                         return -EOPNOTSUPP;
460                 }
461         }
462
463 out:
464         if (likely(nreq)) {
465                 qp->sq.head += nreq;
466                 /* Memory barrier */
467                 wmb();
468
469                 sq_db.byte_4 = 0;
470                 sq_db.parameter = 0;
471
472                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
473                                V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
474                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
475                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
476                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M,
477                                V2_DB_PARAMETER_CONS_IDX_S,
478                                qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
479                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
480                                V2_DB_PARAMETER_SL_S, qp->sl);
481
482                 hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
483
484                 qp->sq_next_wqe = ind;
485                 qp->next_sge = sge_ind;
486         }
487
488         spin_unlock_irqrestore(&qp->sq.lock, flags);
489
490         return ret;
491 }
492
493 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
494                                  struct ib_recv_wr **bad_wr)
495 {
496         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
497         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
498         struct hns_roce_v2_wqe_data_seg *dseg;
499         struct hns_roce_rinl_sge *sge_list;
500         struct device *dev = hr_dev->dev;
501         struct hns_roce_v2_db rq_db;
502         unsigned long flags;
503         void *wqe = NULL;
504         int ret = 0;
505         int nreq;
506         int ind;
507         int i;
508
509         spin_lock_irqsave(&hr_qp->rq.lock, flags);
510         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
511
512         if (hr_qp->state == IB_QPS_RESET || hr_qp->state == IB_QPS_ERR) {
513                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
514                 *bad_wr = wr;
515                 return -EINVAL;
516         }
517
518         for (nreq = 0; wr; ++nreq, wr = wr->next) {
519                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
520                         hr_qp->ibqp.recv_cq)) {
521                         ret = -ENOMEM;
522                         *bad_wr = wr;
523                         goto out;
524                 }
525
526                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
527                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
528                                 wr->num_sge, hr_qp->rq.max_gs);
529                         ret = -EINVAL;
530                         *bad_wr = wr;
531                         goto out;
532                 }
533
534                 wqe = get_recv_wqe(hr_qp, ind);
535                 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
536                 for (i = 0; i < wr->num_sge; i++) {
537                         if (!wr->sg_list[i].length)
538                                 continue;
539                         set_data_seg_v2(dseg, wr->sg_list + i);
540                         dseg++;
541                 }
542
543                 if (i < hr_qp->rq.max_gs) {
544                         dseg[i].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
545                         dseg[i].addr = 0;
546                 }
547
548                 /* rq support inline data */
549                 sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
550                 hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt = (u32)wr->num_sge;
551                 for (i = 0; i < wr->num_sge; i++) {
552                         sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
553                         sge_list[i].len = wr->sg_list[i].length;
554                 }
555
556                 hr_qp->rq.wrid[ind] = wr->wr_id;
557
558                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
559         }
560
561 out:
562         if (likely(nreq)) {
563                 hr_qp->rq.head += nreq;
564                 /* Memory barrier */
565                 wmb();
566
567                 rq_db.byte_4 = 0;
568                 rq_db.parameter = 0;
569
570                 roce_set_field(rq_db.byte_4, V2_DB_BYTE_4_TAG_M,
571                                V2_DB_BYTE_4_TAG_S, hr_qp->qpn);
572                 roce_set_field(rq_db.byte_4, V2_DB_BYTE_4_CMD_M,
573                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_RQ_DB);
574                 roce_set_field(rq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M,
575                                V2_DB_PARAMETER_CONS_IDX_S, hr_qp->rq.head);
576
577                 hns_roce_write64_k((__le32 *)&rq_db, hr_qp->rq.db_reg_l);
578         }
579         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
580
581         return ret;
582 }
583
584 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
585 {
586         int ntu = ring->next_to_use;
587         int ntc = ring->next_to_clean;
588         int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
589
590         return ring->desc_num - used - 1;
591 }
592
593 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
594                                    struct hns_roce_v2_cmq_ring *ring)
595 {
596         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
597
598         ring->desc = kzalloc(size, GFP_KERNEL);
599         if (!ring->desc)
600                 return -ENOMEM;
601
602         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
603                                              DMA_BIDIRECTIONAL);
604         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
605                 ring->desc_dma_addr = 0;
606                 kfree(ring->desc);
607                 ring->desc = NULL;
608                 return -ENOMEM;
609         }
610
611         return 0;
612 }
613
614 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
615                                    struct hns_roce_v2_cmq_ring *ring)
616 {
617         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
618                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
619                          DMA_BIDIRECTIONAL);
620         kfree(ring->desc);
621 }
622
623 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
624 {
625         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
626         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
627                                             &priv->cmq.csq : &priv->cmq.crq;
628
629         ring->flag = ring_type;
630         ring->next_to_clean = 0;
631         ring->next_to_use = 0;
632
633         return hns_roce_alloc_cmq_desc(hr_dev, ring);
634 }
635
636 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
637 {
638         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
639         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
640                                             &priv->cmq.csq : &priv->cmq.crq;
641         dma_addr_t dma = ring->desc_dma_addr;
642
643         if (ring_type == TYPE_CSQ) {
644                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
645                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
646                            upper_32_bits(dma));
647                 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
648                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
649                            HNS_ROCE_CMQ_ENABLE);
650                 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
651                 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
652         } else {
653                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
654                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
655                            upper_32_bits(dma));
656                 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
657                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
658                            HNS_ROCE_CMQ_ENABLE);
659                 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
660                 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
661         }
662 }
663
664 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
665 {
666         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
667         int ret;
668
669         /* Setup the queue entries for command queue */
670         priv->cmq.csq.desc_num = 1024;
671         priv->cmq.crq.desc_num = 1024;
672
673         /* Setup the lock for command queue */
674         spin_lock_init(&priv->cmq.csq.lock);
675         spin_lock_init(&priv->cmq.crq.lock);
676
677         /* Setup Tx write back timeout */
678         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
679
680         /* Init CSQ */
681         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
682         if (ret) {
683                 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
684                 return ret;
685         }
686
687         /* Init CRQ */
688         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
689         if (ret) {
690                 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
691                 goto err_crq;
692         }
693
694         /* Init CSQ REG */
695         hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
696
697         /* Init CRQ REG */
698         hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
699
700         return 0;
701
702 err_crq:
703         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
704
705         return ret;
706 }
707
708 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
709 {
710         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
711
712         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
713         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
714 }
715
716 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
717                                           enum hns_roce_opcode_type opcode,
718                                           bool is_read)
719 {
720         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
721         desc->opcode = cpu_to_le16(opcode);
722         desc->flag =
723                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
724         if (is_read)
725                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
726         else
727                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
728 }
729
730 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
731 {
732         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
733         u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
734
735         return head == priv->cmq.csq.next_to_use;
736 }
737
738 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
739 {
740         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
741         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
742         struct hns_roce_cmq_desc *desc;
743         u16 ntc = csq->next_to_clean;
744         u32 head;
745         int clean = 0;
746
747         desc = &csq->desc[ntc];
748         head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
749         while (head != ntc) {
750                 memset(desc, 0, sizeof(*desc));
751                 ntc++;
752                 if (ntc == csq->desc_num)
753                         ntc = 0;
754                 desc = &csq->desc[ntc];
755                 clean++;
756         }
757         csq->next_to_clean = ntc;
758
759         return clean;
760 }
761
762 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
763                              struct hns_roce_cmq_desc *desc, int num)
764 {
765         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
766         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
767         struct hns_roce_cmq_desc *desc_to_use;
768         bool complete = false;
769         u32 timeout = 0;
770         int handle = 0;
771         u16 desc_ret;
772         int ret = 0;
773         int ntc;
774
775         spin_lock_bh(&csq->lock);
776
777         if (num > hns_roce_cmq_space(csq)) {
778                 spin_unlock_bh(&csq->lock);
779                 return -EBUSY;
780         }
781
782         /*
783          * Record the location of desc in the cmq for this time
784          * which will be use for hardware to write back
785          */
786         ntc = csq->next_to_use;
787
788         while (handle < num) {
789                 desc_to_use = &csq->desc[csq->next_to_use];
790                 *desc_to_use = desc[handle];
791                 dev_dbg(hr_dev->dev, "set cmq desc:\n");
792                 csq->next_to_use++;
793                 if (csq->next_to_use == csq->desc_num)
794                         csq->next_to_use = 0;
795                 handle++;
796         }
797
798         /* Write to hardware */
799         roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
800
801         /*
802          * If the command is sync, wait for the firmware to write back,
803          * if multi descriptors to be sent, use the first one to check
804          */
805         if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
806                 do {
807                         if (hns_roce_cmq_csq_done(hr_dev))
808                                 break;
809                         udelay(1);
810                         timeout++;
811                 } while (timeout < priv->cmq.tx_timeout);
812         }
813
814         if (hns_roce_cmq_csq_done(hr_dev)) {
815                 complete = true;
816                 handle = 0;
817                 while (handle < num) {
818                         /* get the result of hardware write back */
819                         desc_to_use = &csq->desc[ntc];
820                         desc[handle] = *desc_to_use;
821                         dev_dbg(hr_dev->dev, "Get cmq desc:\n");
822                         desc_ret = desc[handle].retval;
823                         if (desc_ret == CMD_EXEC_SUCCESS)
824                                 ret = 0;
825                         else
826                                 ret = -EIO;
827                         priv->cmq.last_status = desc_ret;
828                         ntc++;
829                         handle++;
830                         if (ntc == csq->desc_num)
831                                 ntc = 0;
832                 }
833         }
834
835         if (!complete)
836                 ret = -EAGAIN;
837
838         /* clean the command send queue */
839         handle = hns_roce_cmq_csq_clean(hr_dev);
840         if (handle != num)
841                 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
842                          handle, num);
843
844         spin_unlock_bh(&csq->lock);
845
846         return ret;
847 }
848
849 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
850 {
851         struct hns_roce_query_version *resp;
852         struct hns_roce_cmq_desc desc;
853         int ret;
854
855         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
856         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
857         if (ret)
858                 return ret;
859
860         resp = (struct hns_roce_query_version *)desc.data;
861         hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
862         hr_dev->vendor_id = le32_to_cpu(resp->rocee_vendor_id);
863
864         return 0;
865 }
866
867 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
868 {
869         struct hns_roce_cfg_global_param *req;
870         struct hns_roce_cmq_desc desc;
871
872         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
873                                       false);
874
875         req = (struct hns_roce_cfg_global_param *)desc.data;
876         memset(req, 0, sizeof(*req));
877         roce_set_field(req->time_cfg_udp_port,
878                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
879                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
880         roce_set_field(req->time_cfg_udp_port,
881                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
882                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
883
884         return hns_roce_cmq_send(hr_dev, &desc, 1);
885 }
886
887 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
888 {
889         struct hns_roce_cmq_desc desc[2];
890         struct hns_roce_pf_res *res;
891         int ret;
892         int i;
893
894         for (i = 0; i < 2; i++) {
895                 hns_roce_cmq_setup_basic_desc(&desc[i],
896                                               HNS_ROCE_OPC_QUERY_PF_RES, true);
897
898                 if (i == 0)
899                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
900                 else
901                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
902         }
903
904         ret = hns_roce_cmq_send(hr_dev, desc, 2);
905         if (ret)
906                 return ret;
907
908         res = (struct hns_roce_pf_res *)desc[0].data;
909
910         hr_dev->caps.qpc_bt_num = roce_get_field(res->qpc_bt_idx_num,
911                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_M,
912                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_S);
913         hr_dev->caps.srqc_bt_num = roce_get_field(res->srqc_bt_idx_num,
914                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
915                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
916         hr_dev->caps.cqc_bt_num = roce_get_field(res->cqc_bt_idx_num,
917                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_M,
918                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_S);
919         hr_dev->caps.mpt_bt_num = roce_get_field(res->mpt_bt_idx_num,
920                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_M,
921                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_S);
922
923         return 0;
924 }
925
926 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
927 {
928         struct hns_roce_cmq_desc desc[2];
929         struct hns_roce_vf_res_a *req_a;
930         struct hns_roce_vf_res_b *req_b;
931         int i;
932
933         req_a = (struct hns_roce_vf_res_a *)desc[0].data;
934         req_b = (struct hns_roce_vf_res_b *)desc[1].data;
935         memset(req_a, 0, sizeof(*req_a));
936         memset(req_b, 0, sizeof(*req_b));
937         for (i = 0; i < 2; i++) {
938                 hns_roce_cmq_setup_basic_desc(&desc[i],
939                                               HNS_ROCE_OPC_ALLOC_VF_RES, false);
940
941                 if (i == 0)
942                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
943                 else
944                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
945
946                 if (i == 0) {
947                         roce_set_field(req_a->vf_qpc_bt_idx_num,
948                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
949                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
950                         roce_set_field(req_a->vf_qpc_bt_idx_num,
951                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
952                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
953                                        HNS_ROCE_VF_QPC_BT_NUM);
954
955                         roce_set_field(req_a->vf_srqc_bt_idx_num,
956                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
957                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
958                         roce_set_field(req_a->vf_srqc_bt_idx_num,
959                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
960                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
961                                        HNS_ROCE_VF_SRQC_BT_NUM);
962
963                         roce_set_field(req_a->vf_cqc_bt_idx_num,
964                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
965                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
966                         roce_set_field(req_a->vf_cqc_bt_idx_num,
967                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
968                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
969                                        HNS_ROCE_VF_CQC_BT_NUM);
970
971                         roce_set_field(req_a->vf_mpt_bt_idx_num,
972                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
973                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
974                         roce_set_field(req_a->vf_mpt_bt_idx_num,
975                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
976                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
977                                        HNS_ROCE_VF_MPT_BT_NUM);
978
979                         roce_set_field(req_a->vf_eqc_bt_idx_num,
980                                        VF_RES_A_DATA_5_VF_EQC_IDX_M,
981                                        VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
982                         roce_set_field(req_a->vf_eqc_bt_idx_num,
983                                        VF_RES_A_DATA_5_VF_EQC_NUM_M,
984                                        VF_RES_A_DATA_5_VF_EQC_NUM_S,
985                                        HNS_ROCE_VF_EQC_NUM);
986                 } else {
987                         roce_set_field(req_b->vf_smac_idx_num,
988                                        VF_RES_B_DATA_1_VF_SMAC_IDX_M,
989                                        VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
990                         roce_set_field(req_b->vf_smac_idx_num,
991                                        VF_RES_B_DATA_1_VF_SMAC_NUM_M,
992                                        VF_RES_B_DATA_1_VF_SMAC_NUM_S,
993                                        HNS_ROCE_VF_SMAC_NUM);
994
995                         roce_set_field(req_b->vf_sgid_idx_num,
996                                        VF_RES_B_DATA_2_VF_SGID_IDX_M,
997                                        VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
998                         roce_set_field(req_b->vf_sgid_idx_num,
999                                        VF_RES_B_DATA_2_VF_SGID_NUM_M,
1000                                        VF_RES_B_DATA_2_VF_SGID_NUM_S,
1001                                        HNS_ROCE_VF_SGID_NUM);
1002
1003                         roce_set_field(req_b->vf_qid_idx_sl_num,
1004                                        VF_RES_B_DATA_3_VF_QID_IDX_M,
1005                                        VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1006                         roce_set_field(req_b->vf_qid_idx_sl_num,
1007                                        VF_RES_B_DATA_3_VF_SL_NUM_M,
1008                                        VF_RES_B_DATA_3_VF_SL_NUM_S,
1009                                        HNS_ROCE_VF_SL_NUM);
1010                 }
1011         }
1012
1013         return hns_roce_cmq_send(hr_dev, desc, 2);
1014 }
1015
1016 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1017 {
1018         u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1019         u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1020         u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1021         u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1022         struct hns_roce_cfg_bt_attr *req;
1023         struct hns_roce_cmq_desc desc;
1024
1025         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1026         req = (struct hns_roce_cfg_bt_attr *)desc.data;
1027         memset(req, 0, sizeof(*req));
1028
1029         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1030                        CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1031                        hr_dev->caps.qpc_ba_pg_sz);
1032         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1033                        CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1034                        hr_dev->caps.qpc_buf_pg_sz);
1035         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1036                        CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1037                        qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1038
1039         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1040                        CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1041                        hr_dev->caps.srqc_ba_pg_sz);
1042         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1043                        CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1044                        hr_dev->caps.srqc_buf_pg_sz);
1045         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1046                        CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1047                        srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1048
1049         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1050                        CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1051                        hr_dev->caps.cqc_ba_pg_sz);
1052         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1053                        CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1054                        hr_dev->caps.cqc_buf_pg_sz);
1055         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1056                        CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1057                        cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1058
1059         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1060                        CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1061                        hr_dev->caps.mpt_ba_pg_sz);
1062         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1063                        CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1064                        hr_dev->caps.mpt_buf_pg_sz);
1065         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1066                        CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1067                        mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1068
1069         return hns_roce_cmq_send(hr_dev, &desc, 1);
1070 }
1071
1072 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1073 {
1074         struct hns_roce_caps *caps = &hr_dev->caps;
1075         int ret;
1076
1077         ret = hns_roce_cmq_query_hw_info(hr_dev);
1078         if (ret) {
1079                 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1080                         ret);
1081                 return ret;
1082         }
1083
1084         ret = hns_roce_config_global_param(hr_dev);
1085         if (ret) {
1086                 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1087                         ret);
1088         }
1089
1090         /* Get pf resource owned by every pf */
1091         ret = hns_roce_query_pf_resource(hr_dev);
1092         if (ret) {
1093                 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1094                         ret);
1095                 return ret;
1096         }
1097
1098         ret = hns_roce_alloc_vf_resource(hr_dev);
1099         if (ret) {
1100                 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1101                         ret);
1102                 return ret;
1103         }
1104
1105         hr_dev->vendor_part_id = 0;
1106         hr_dev->sys_image_guid = 0;
1107
1108         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1109         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1110         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1111         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1112         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1113         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1114         caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
1115         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1116         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1117         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1118         caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
1119         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1120         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1121         caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
1122         caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
1123         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1124         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1125         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1126         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1127         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1128         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1129         caps->qpc_entry_sz      = HNS_ROCE_V2_QPC_ENTRY_SZ;
1130         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1131         caps->trrl_entry_sz     = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1132         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1133         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1134         caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
1135         caps->cq_entry_sz       = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1136         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1137         caps->reserved_lkey     = 0;
1138         caps->reserved_pds      = 0;
1139         caps->reserved_mrws     = 1;
1140         caps->reserved_uars     = 0;
1141         caps->reserved_cqs      = 0;
1142
1143         caps->qpc_ba_pg_sz      = 0;
1144         caps->qpc_buf_pg_sz     = 0;
1145         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1146         caps->srqc_ba_pg_sz     = 0;
1147         caps->srqc_buf_pg_sz    = 0;
1148         caps->srqc_hop_num      = HNS_ROCE_HOP_NUM_0;
1149         caps->cqc_ba_pg_sz      = 0;
1150         caps->cqc_buf_pg_sz     = 0;
1151         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1152         caps->mpt_ba_pg_sz      = 0;
1153         caps->mpt_buf_pg_sz     = 0;
1154         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1155         caps->pbl_ba_pg_sz      = 0;
1156         caps->pbl_buf_pg_sz     = 0;
1157         caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
1158         caps->mtt_ba_pg_sz      = 0;
1159         caps->mtt_buf_pg_sz     = 0;
1160         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1161         caps->cqe_ba_pg_sz      = 0;
1162         caps->cqe_buf_pg_sz     = 0;
1163         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1164         caps->eqe_ba_pg_sz      = 0;
1165         caps->eqe_buf_pg_sz     = 0;
1166         caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
1167         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1168
1169         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1170                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1171                                   HNS_ROCE_CAP_FLAG_RQ_INLINE;
1172         caps->pkey_table_len[0] = 1;
1173         caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1174         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1175         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1176         caps->local_ca_ack_delay = 0;
1177         caps->max_mtu = IB_MTU_4096;
1178
1179         ret = hns_roce_v2_set_bt(hr_dev);
1180         if (ret)
1181                 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1182                         ret);
1183
1184         return ret;
1185 }
1186
1187 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1188 {
1189         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1190
1191         return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1192 }
1193
1194 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1195 {
1196         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1197
1198         return status & HNS_ROCE_HW_MB_STATUS_MASK;
1199 }
1200
1201 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1202                                  u64 out_param, u32 in_modifier, u8 op_modifier,
1203                                  u16 op, u16 token, int event)
1204 {
1205         struct device *dev = hr_dev->dev;
1206         u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base +
1207                                            ROCEE_VF_MB_CFG0_REG);
1208         unsigned long end;
1209         u32 val0 = 0;
1210         u32 val1 = 0;
1211
1212         end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
1213         while (hns_roce_v2_cmd_pending(hr_dev)) {
1214                 if (time_after(jiffies, end)) {
1215                         dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
1216                                 (int)end);
1217                         return -EAGAIN;
1218                 }
1219                 cond_resched();
1220         }
1221
1222         roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK,
1223                        HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier);
1224         roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK,
1225                        HNS_ROCE_VF_MB4_CMD_SHIFT, op);
1226         roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK,
1227                        HNS_ROCE_VF_MB5_EVENT_SHIFT, event);
1228         roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK,
1229                        HNS_ROCE_VF_MB5_TOKEN_SHIFT, token);
1230
1231         __raw_writeq(cpu_to_le64(in_param), hcr + 0);
1232         __raw_writeq(cpu_to_le64(out_param), hcr + 2);
1233
1234         /* Memory barrier */
1235         wmb();
1236
1237         __raw_writel(cpu_to_le32(val0), hcr + 4);
1238         __raw_writel(cpu_to_le32(val1), hcr + 5);
1239
1240         mmiowb();
1241
1242         return 0;
1243 }
1244
1245 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
1246                                 unsigned long timeout)
1247 {
1248         struct device *dev = hr_dev->dev;
1249         unsigned long end = 0;
1250         u32 status;
1251
1252         end = msecs_to_jiffies(timeout) + jiffies;
1253         while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
1254                 cond_resched();
1255
1256         if (hns_roce_v2_cmd_pending(hr_dev)) {
1257                 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1258                 return -ETIMEDOUT;
1259         }
1260
1261         status = hns_roce_v2_cmd_complete(hr_dev);
1262         if (status != 0x1) {
1263                 dev_err(dev, "mailbox status 0x%x!\n", status);
1264                 return -EBUSY;
1265         }
1266
1267         return 0;
1268 }
1269
1270 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
1271                                int gid_index, union ib_gid *gid,
1272                                const struct ib_gid_attr *attr)
1273 {
1274         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
1275         u32 *p;
1276         u32 val;
1277
1278         if (!gid || !attr)
1279                 return -EINVAL;
1280
1281         if (attr->gid_type == IB_GID_TYPE_ROCE)
1282                 sgid_type = GID_TYPE_FLAG_ROCE_V1;
1283
1284         if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
1285                 if (ipv6_addr_v4mapped((void *)gid))
1286                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
1287                 else
1288                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
1289         }
1290
1291         p = (u32 *)&gid->raw[0];
1292         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG0_REG +
1293                        0x20 * gid_index);
1294
1295         p = (u32 *)&gid->raw[4];
1296         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG1_REG +
1297                        0x20 * gid_index);
1298
1299         p = (u32 *)&gid->raw[8];
1300         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG2_REG +
1301                        0x20 * gid_index);
1302
1303         p = (u32 *)&gid->raw[0xc];
1304         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG3_REG +
1305                        0x20 * gid_index);
1306
1307         val = roce_read(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index);
1308         roce_set_field(val, ROCEE_VF_SGID_CFG4_SGID_TYPE_M,
1309                        ROCEE_VF_SGID_CFG4_SGID_TYPE_S, sgid_type);
1310
1311         roce_write(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index, val);
1312
1313         return 0;
1314 }
1315
1316 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
1317                                u8 *addr)
1318 {
1319         u16 reg_smac_h;
1320         u32 reg_smac_l;
1321         u32 val;
1322
1323         reg_smac_l = *(u32 *)(&addr[0]);
1324         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_VF_SMAC_CFG0_REG +
1325                        0x08 * phy_port);
1326         val = roce_read(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port);
1327
1328         reg_smac_h  = *(u16 *)(&addr[4]);
1329         roce_set_field(val, ROCEE_VF_SMAC_CFG1_VF_SMAC_H_M,
1330                        ROCEE_VF_SMAC_CFG1_VF_SMAC_H_S, reg_smac_h);
1331         roce_write(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port, val);
1332
1333         return 0;
1334 }
1335
1336 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1337                                   unsigned long mtpt_idx)
1338 {
1339         struct hns_roce_v2_mpt_entry *mpt_entry;
1340         struct scatterlist *sg;
1341         u64 page_addr;
1342         u64 *pages;
1343         int i, j;
1344         int len;
1345         int entry;
1346
1347         mpt_entry = mb_buf;
1348         memset(mpt_entry, 0, sizeof(*mpt_entry));
1349
1350         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
1351                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
1352         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
1353                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
1354                        HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
1355         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1356                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1357                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, mr->pbl_ba_pg_sz);
1358         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1359                        V2_MPT_BYTE_4_PD_S, mr->pd);
1360         mpt_entry->byte_4_pd_hop_st = cpu_to_le32(mpt_entry->byte_4_pd_hop_st);
1361
1362         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
1363         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1364         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 0);
1365         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
1366                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1367         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 0);
1368         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1369                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1370         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1371                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1372         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1373                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1374         mpt_entry->byte_8_mw_cnt_en = cpu_to_le32(mpt_entry->byte_8_mw_cnt_en);
1375
1376         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
1377                      mr->type == MR_TYPE_MR ? 0 : 1);
1378         mpt_entry->byte_12_mw_pa = cpu_to_le32(mpt_entry->byte_12_mw_pa);
1379
1380         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
1381         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
1382         mpt_entry->lkey = cpu_to_le32(mr->key);
1383         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
1384         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
1385
1386         if (mr->type == MR_TYPE_DMA)
1387                 return 0;
1388
1389         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1390
1391         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1392         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
1393                        V2_MPT_BYTE_48_PBL_BA_H_S,
1394                        upper_32_bits(mr->pbl_ba >> 3));
1395         mpt_entry->byte_48_mode_ba = cpu_to_le32(mpt_entry->byte_48_mode_ba);
1396
1397         pages = (u64 *)__get_free_page(GFP_KERNEL);
1398         if (!pages)
1399                 return -ENOMEM;
1400
1401         i = 0;
1402         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1403                 len = sg_dma_len(sg) >> PAGE_SHIFT;
1404                 for (j = 0; j < len; ++j) {
1405                         page_addr = sg_dma_address(sg) +
1406                                     (j << mr->umem->page_shift);
1407                         pages[i] = page_addr >> 6;
1408
1409                         /* Record the first 2 entry directly to MTPT table */
1410                         if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
1411                                 goto found;
1412                         i++;
1413                 }
1414         }
1415
1416 found:
1417         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
1418         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
1419                        V2_MPT_BYTE_56_PA0_H_S,
1420                        upper_32_bits(pages[0]));
1421         mpt_entry->byte_56_pa0_h = cpu_to_le32(mpt_entry->byte_56_pa0_h);
1422
1423         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
1424         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
1425                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
1426
1427         free_page((unsigned long)pages);
1428
1429         roce_set_field(mpt_entry->byte_64_buf_pa1,
1430                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1431                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, mr->pbl_buf_pg_sz);
1432         mpt_entry->byte_64_buf_pa1 = cpu_to_le32(mpt_entry->byte_64_buf_pa1);
1433
1434         return 0;
1435 }
1436
1437 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
1438                                         struct hns_roce_mr *mr, int flags,
1439                                         u32 pdn, int mr_access_flags, u64 iova,
1440                                         u64 size, void *mb_buf)
1441 {
1442         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
1443
1444         if (flags & IB_MR_REREG_PD) {
1445                 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1446                                V2_MPT_BYTE_4_PD_S, pdn);
1447                 mr->pd = pdn;
1448         }
1449
1450         if (flags & IB_MR_REREG_ACCESS) {
1451                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1452                              V2_MPT_BYTE_8_BIND_EN_S,
1453                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
1454                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1455                            V2_MPT_BYTE_8_ATOMIC_EN_S,
1456                            (mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0));
1457                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1458                              (mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0));
1459                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1460                             (mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1461                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1462                              (mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1463         }
1464
1465         if (flags & IB_MR_REREG_TRANS) {
1466                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
1467                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
1468                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
1469                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
1470
1471                 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1472                 mpt_entry->pbl_ba_l =
1473                                 cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1474                 roce_set_field(mpt_entry->byte_48_mode_ba,
1475                                V2_MPT_BYTE_48_PBL_BA_H_M,
1476                                V2_MPT_BYTE_48_PBL_BA_H_S,
1477                                upper_32_bits(mr->pbl_ba >> 3));
1478                 mpt_entry->byte_48_mode_ba =
1479                                 cpu_to_le32(mpt_entry->byte_48_mode_ba);
1480
1481                 mr->iova = iova;
1482                 mr->size = size;
1483         }
1484
1485         return 0;
1486 }
1487
1488 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1489 {
1490         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1491                                    n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
1492 }
1493
1494 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1495 {
1496         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
1497
1498         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1499         return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
1500                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
1501 }
1502
1503 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
1504 {
1505         return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
1506 }
1507
1508 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1509 {
1510         struct hns_roce_v2_cq_db cq_db;
1511
1512         cq_db.byte_4 = 0;
1513         cq_db.parameter = 0;
1514
1515         roce_set_field(cq_db.byte_4, V2_CQ_DB_BYTE_4_TAG_M,
1516                        V2_CQ_DB_BYTE_4_TAG_S, hr_cq->cqn);
1517         roce_set_field(cq_db.byte_4, V2_CQ_DB_BYTE_4_CMD_M,
1518                        V2_CQ_DB_BYTE_4_CMD_S, HNS_ROCE_V2_CQ_DB_PTR);
1519
1520         roce_set_field(cq_db.parameter, V2_CQ_DB_PARAMETER_CONS_IDX_M,
1521                        V2_CQ_DB_PARAMETER_CONS_IDX_S,
1522                        cons_index & ((hr_cq->cq_depth << 1) - 1));
1523         roce_set_field(cq_db.parameter, V2_CQ_DB_PARAMETER_CMD_SN_M,
1524                        V2_CQ_DB_PARAMETER_CMD_SN_S, 1);
1525
1526         hns_roce_write64_k((__be32 *)&cq_db, hr_cq->cq_db_l);
1527
1528 }
1529
1530 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1531                                    struct hns_roce_srq *srq)
1532 {
1533         struct hns_roce_v2_cqe *cqe, *dest;
1534         u32 prod_index;
1535         int nfreed = 0;
1536         u8 owner_bit;
1537
1538         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
1539              ++prod_index) {
1540                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1541                         break;
1542         }
1543
1544         /*
1545          * Now backwards through the CQ, removing CQ entries
1546          * that match our QP by overwriting them with next entries.
1547          */
1548         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1549                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1550                 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1551                                     V2_CQE_BYTE_16_LCL_QPN_S) &
1552                                     HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
1553                         /* In v1 engine, not support SRQ */
1554                         ++nfreed;
1555                 } else if (nfreed) {
1556                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
1557                                           hr_cq->ib_cq.cqe);
1558                         owner_bit = roce_get_bit(dest->byte_4,
1559                                                  V2_CQE_BYTE_4_OWNER_S);
1560                         memcpy(dest, cqe, sizeof(*cqe));
1561                         roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
1562                                      owner_bit);
1563                 }
1564         }
1565
1566         if (nfreed) {
1567                 hr_cq->cons_index += nfreed;
1568                 /*
1569                  * Make sure update of buffer contents is done before
1570                  * updating consumer index.
1571                  */
1572                 wmb();
1573                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
1574         }
1575 }
1576
1577 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1578                                  struct hns_roce_srq *srq)
1579 {
1580         spin_lock_irq(&hr_cq->lock);
1581         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
1582         spin_unlock_irq(&hr_cq->lock);
1583 }
1584
1585 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
1586                                   struct hns_roce_cq *hr_cq, void *mb_buf,
1587                                   u64 *mtts, dma_addr_t dma_handle, int nent,
1588                                   u32 vector)
1589 {
1590         struct hns_roce_v2_cq_context *cq_context;
1591
1592         cq_context = mb_buf;
1593         memset(cq_context, 0, sizeof(*cq_context));
1594
1595         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
1596                        V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
1597         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
1598                        V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
1599         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
1600                        V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
1601         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
1602                        V2_CQC_BYTE_4_CEQN_S, vector);
1603         cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
1604
1605         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
1606                        V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
1607
1608         cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
1609         cq_context->cqe_cur_blk_addr =
1610                                 cpu_to_le32(cq_context->cqe_cur_blk_addr);
1611
1612         roce_set_field(cq_context->byte_16_hop_addr,
1613                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
1614                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
1615                        cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
1616         roce_set_field(cq_context->byte_16_hop_addr,
1617                        V2_CQC_BYTE_16_CQE_HOP_NUM_M,
1618                        V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
1619                        HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
1620
1621         cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
1622         roce_set_field(cq_context->byte_24_pgsz_addr,
1623                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
1624                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
1625                        cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
1626         roce_set_field(cq_context->byte_24_pgsz_addr,
1627                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
1628                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
1629                        hr_dev->caps.cqe_ba_pg_sz);
1630         roce_set_field(cq_context->byte_24_pgsz_addr,
1631                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
1632                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
1633                        hr_dev->caps.cqe_buf_pg_sz);
1634
1635         cq_context->cqe_ba = (u32)(dma_handle >> 3);
1636
1637         roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
1638                        V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
1639
1640         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1641                        V2_CQC_BYTE_56_CQ_MAX_CNT_M,
1642                        V2_CQC_BYTE_56_CQ_MAX_CNT_S,
1643                        HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
1644         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1645                        V2_CQC_BYTE_56_CQ_PERIOD_M,
1646                        V2_CQC_BYTE_56_CQ_PERIOD_S,
1647                        HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
1648 }
1649
1650 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
1651                                      enum ib_cq_notify_flags flags)
1652 {
1653         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1654         u32 notification_flag;
1655         u32 doorbell[2];
1656
1657         doorbell[0] = 0;
1658         doorbell[1] = 0;
1659
1660         notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
1661                              V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
1662         /*
1663          * flags = 0; Notification Flag = 1, next
1664          * flags = 1; Notification Flag = 0, solocited
1665          */
1666         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
1667                        hr_cq->cqn);
1668         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
1669                        HNS_ROCE_V2_CQ_DB_NTR);
1670         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
1671                        V2_CQ_DB_PARAMETER_CONS_IDX_S,
1672                        hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
1673         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
1674                        V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
1675         roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
1676                      notification_flag);
1677
1678         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1679
1680         return 0;
1681 }
1682
1683 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
1684                                                     struct hns_roce_qp **cur_qp,
1685                                                     struct ib_wc *wc)
1686 {
1687         struct hns_roce_rinl_sge *sge_list;
1688         u32 wr_num, wr_cnt, sge_num;
1689         u32 sge_cnt, data_len, size;
1690         void *wqe_buf;
1691
1692         wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
1693                                 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
1694         wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
1695
1696         sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
1697         sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
1698         wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
1699         data_len = wc->byte_len;
1700
1701         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
1702                 size = min(sge_list[sge_cnt].len, data_len);
1703                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
1704
1705                 data_len -= size;
1706                 wqe_buf += size;
1707         }
1708
1709         if (data_len) {
1710                 wc->status = IB_WC_LOC_LEN_ERR;
1711                 return -EAGAIN;
1712         }
1713
1714         return 0;
1715 }
1716
1717 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
1718                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
1719 {
1720         struct hns_roce_dev *hr_dev;
1721         struct hns_roce_v2_cqe *cqe;
1722         struct hns_roce_qp *hr_qp;
1723         struct hns_roce_wq *wq;
1724         int is_send;
1725         u16 wqe_ctr;
1726         u32 opcode;
1727         u32 status;
1728         int qpn;
1729         int ret;
1730
1731         /* Find cqe according to consumer index */
1732         cqe = next_cqe_sw_v2(hr_cq);
1733         if (!cqe)
1734                 return -EAGAIN;
1735
1736         ++hr_cq->cons_index;
1737         /* Memory barrier */
1738         rmb();
1739
1740         /* 0->SQ, 1->RQ */
1741         is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
1742
1743         qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1744                                 V2_CQE_BYTE_16_LCL_QPN_S);
1745
1746         if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
1747                 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
1748                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
1749                 if (unlikely(!hr_qp)) {
1750                         dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
1751                                 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
1752                         return -EINVAL;
1753                 }
1754                 *cur_qp = hr_qp;
1755         }
1756
1757         wc->qp = &(*cur_qp)->ibqp;
1758         wc->vendor_err = 0;
1759
1760         status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
1761                                 V2_CQE_BYTE_4_STATUS_S);
1762         switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
1763         case HNS_ROCE_CQE_V2_SUCCESS:
1764                 wc->status = IB_WC_SUCCESS;
1765                 break;
1766         case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
1767                 wc->status = IB_WC_LOC_LEN_ERR;
1768                 break;
1769         case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
1770                 wc->status = IB_WC_LOC_QP_OP_ERR;
1771                 break;
1772         case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
1773                 wc->status = IB_WC_LOC_PROT_ERR;
1774                 break;
1775         case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
1776                 wc->status = IB_WC_WR_FLUSH_ERR;
1777                 break;
1778         case HNS_ROCE_CQE_V2_MW_BIND_ERR:
1779                 wc->status = IB_WC_MW_BIND_ERR;
1780                 break;
1781         case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
1782                 wc->status = IB_WC_BAD_RESP_ERR;
1783                 break;
1784         case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
1785                 wc->status = IB_WC_LOC_ACCESS_ERR;
1786                 break;
1787         case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
1788                 wc->status = IB_WC_REM_INV_REQ_ERR;
1789                 break;
1790         case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
1791                 wc->status = IB_WC_REM_ACCESS_ERR;
1792                 break;
1793         case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
1794                 wc->status = IB_WC_REM_OP_ERR;
1795                 break;
1796         case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
1797                 wc->status = IB_WC_RETRY_EXC_ERR;
1798                 break;
1799         case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
1800                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
1801                 break;
1802         case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
1803                 wc->status = IB_WC_REM_ABORT_ERR;
1804                 break;
1805         default:
1806                 wc->status = IB_WC_GENERAL_ERR;
1807                 break;
1808         }
1809
1810         /* CQE status error, directly return */
1811         if (wc->status != IB_WC_SUCCESS)
1812                 return 0;
1813
1814         if (is_send) {
1815                 wc->wc_flags = 0;
1816                 /* SQ corresponding to CQE */
1817                 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
1818                                        V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
1819                 case HNS_ROCE_SQ_OPCODE_SEND:
1820                         wc->opcode = IB_WC_SEND;
1821                         break;
1822                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
1823                         wc->opcode = IB_WC_SEND;
1824                         break;
1825                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
1826                         wc->opcode = IB_WC_SEND;
1827                         wc->wc_flags |= IB_WC_WITH_IMM;
1828                         break;
1829                 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
1830                         wc->opcode = IB_WC_RDMA_READ;
1831                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1832                         break;
1833                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
1834                         wc->opcode = IB_WC_RDMA_WRITE;
1835                         break;
1836                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
1837                         wc->opcode = IB_WC_RDMA_WRITE;
1838                         wc->wc_flags |= IB_WC_WITH_IMM;
1839                         break;
1840                 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
1841                         wc->opcode = IB_WC_LOCAL_INV;
1842                         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
1843                         break;
1844                 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
1845                         wc->opcode = IB_WC_COMP_SWAP;
1846                         wc->byte_len  = 8;
1847                         break;
1848                 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
1849                         wc->opcode = IB_WC_FETCH_ADD;
1850                         wc->byte_len  = 8;
1851                         break;
1852                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
1853                         wc->opcode = IB_WC_MASKED_COMP_SWAP;
1854                         wc->byte_len  = 8;
1855                         break;
1856                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
1857                         wc->opcode = IB_WC_MASKED_FETCH_ADD;
1858                         wc->byte_len  = 8;
1859                         break;
1860                 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
1861                         wc->opcode = IB_WC_REG_MR;
1862                         break;
1863                 case HNS_ROCE_SQ_OPCODE_BIND_MW:
1864                         wc->opcode = IB_WC_REG_MR;
1865                         break;
1866                 default:
1867                         wc->status = IB_WC_GENERAL_ERR;
1868                         break;
1869                 }
1870
1871                 wq = &(*cur_qp)->sq;
1872                 if ((*cur_qp)->sq_signal_bits) {
1873                         /*
1874                          * If sg_signal_bit is 1,
1875                          * firstly tail pointer updated to wqe
1876                          * which current cqe correspond to
1877                          */
1878                         wqe_ctr = (u16)roce_get_field(cqe->byte_4,
1879                                                       V2_CQE_BYTE_4_WQE_INDX_M,
1880                                                       V2_CQE_BYTE_4_WQE_INDX_S);
1881                         wq->tail += (wqe_ctr - (u16)wq->tail) &
1882                                     (wq->wqe_cnt - 1);
1883                 }
1884
1885                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1886                 ++wq->tail;
1887         } else {
1888                 /* RQ correspond to CQE */
1889                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1890
1891                 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
1892                                         V2_CQE_BYTE_4_OPCODE_S);
1893                 switch (opcode & 0x1f) {
1894                 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
1895                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
1896                         wc->wc_flags = IB_WC_WITH_IMM;
1897                         wc->ex.imm_data = cqe->immtdata;
1898                         break;
1899                 case HNS_ROCE_V2_OPCODE_SEND:
1900                         wc->opcode = IB_WC_RECV;
1901                         wc->wc_flags = 0;
1902                         break;
1903                 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
1904                         wc->opcode = IB_WC_RECV;
1905                         wc->wc_flags = IB_WC_WITH_IMM;
1906                         wc->ex.imm_data = cqe->immtdata;
1907                         break;
1908                 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
1909                         wc->opcode = IB_WC_RECV;
1910                         wc->wc_flags = IB_WC_WITH_INVALIDATE;
1911                         wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
1912                         break;
1913                 default:
1914                         wc->status = IB_WC_GENERAL_ERR;
1915                         break;
1916                 }
1917
1918                 if ((wc->qp->qp_type == IB_QPT_RC ||
1919                      wc->qp->qp_type == IB_QPT_UC) &&
1920                     (opcode == HNS_ROCE_V2_OPCODE_SEND ||
1921                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
1922                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
1923                     (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
1924                         ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
1925                         if (ret)
1926                                 return -EAGAIN;
1927                 }
1928
1929                 /* Update tail pointer, record wr_id */
1930                 wq = &(*cur_qp)->rq;
1931                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1932                 ++wq->tail;
1933
1934                 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
1935                                             V2_CQE_BYTE_32_SL_S);
1936                 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
1937                                                 V2_CQE_BYTE_32_RMT_QPN_M,
1938                                                 V2_CQE_BYTE_32_RMT_QPN_S);
1939                 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
1940                                               V2_CQE_BYTE_32_GRH_S) ?
1941                                               IB_WC_GRH : 0);
1942                 wc->port_num = roce_get_field(cqe->byte_32,
1943                                 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
1944                 wc->pkey_index = 0;
1945                 memcpy(wc->smac, cqe->smac, 4);
1946                 wc->smac[4] = roce_get_field(cqe->byte_28,
1947                                              V2_CQE_BYTE_28_SMAC_4_M,
1948                                              V2_CQE_BYTE_28_SMAC_4_S);
1949                 wc->smac[5] = roce_get_field(cqe->byte_28,
1950                                              V2_CQE_BYTE_28_SMAC_5_M,
1951                                              V2_CQE_BYTE_28_SMAC_5_S);
1952                 wc->vlan_id = 0xffff;
1953                 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
1954                 wc->network_hdr_type = roce_get_field(cqe->byte_28,
1955                                                     V2_CQE_BYTE_28_PORT_TYPE_M,
1956                                                     V2_CQE_BYTE_28_PORT_TYPE_S);
1957         }
1958
1959         return 0;
1960 }
1961
1962 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
1963                                struct ib_wc *wc)
1964 {
1965         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1966         struct hns_roce_qp *cur_qp = NULL;
1967         unsigned long flags;
1968         int npolled;
1969
1970         spin_lock_irqsave(&hr_cq->lock, flags);
1971
1972         for (npolled = 0; npolled < num_entries; ++npolled) {
1973                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
1974                         break;
1975         }
1976
1977         if (npolled) {
1978                 /* Memory barrier */
1979                 wmb();
1980                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
1981         }
1982
1983         spin_unlock_irqrestore(&hr_cq->lock, flags);
1984
1985         return npolled;
1986 }
1987
1988 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
1989                                struct hns_roce_hem_table *table, int obj,
1990                                int step_idx)
1991 {
1992         struct device *dev = hr_dev->dev;
1993         struct hns_roce_cmd_mailbox *mailbox;
1994         struct hns_roce_hem_iter iter;
1995         struct hns_roce_hem_mhop mhop;
1996         struct hns_roce_hem *hem;
1997         unsigned long mhop_obj = obj;
1998         int i, j, k;
1999         int ret = 0;
2000         u64 hem_idx = 0;
2001         u64 l1_idx = 0;
2002         u64 bt_ba = 0;
2003         u32 chunk_ba_num;
2004         u32 hop_num;
2005         u16 op = 0xff;
2006
2007         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2008                 return 0;
2009
2010         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
2011         i = mhop.l0_idx;
2012         j = mhop.l1_idx;
2013         k = mhop.l2_idx;
2014         hop_num = mhop.hop_num;
2015         chunk_ba_num = mhop.bt_chunk_size / 8;
2016
2017         if (hop_num == 2) {
2018                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
2019                           k;
2020                 l1_idx = i * chunk_ba_num + j;
2021         } else if (hop_num == 1) {
2022                 hem_idx = i * chunk_ba_num + j;
2023         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
2024                 hem_idx = i;
2025         }
2026
2027         switch (table->type) {
2028         case HEM_TYPE_QPC:
2029                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2030                 break;
2031         case HEM_TYPE_MTPT:
2032                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2033                 break;
2034         case HEM_TYPE_CQC:
2035                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2036                 break;
2037         case HEM_TYPE_SRQC:
2038                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2039                 break;
2040         default:
2041                 dev_warn(dev, "Table %d not to be written by mailbox!\n",
2042                          table->type);
2043                 return 0;
2044         }
2045         op += step_idx;
2046
2047         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2048         if (IS_ERR(mailbox))
2049                 return PTR_ERR(mailbox);
2050
2051         if (check_whether_last_step(hop_num, step_idx)) {
2052                 hem = table->hem[hem_idx];
2053                 for (hns_roce_hem_first(hem, &iter);
2054                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2055                         bt_ba = hns_roce_hem_addr(&iter);
2056
2057                         /* configure the ba, tag, and op */
2058                         ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2059                                                 obj, 0, op,
2060                                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2061                 }
2062         } else {
2063                 if (step_idx == 0)
2064                         bt_ba = table->bt_l0_dma_addr[i];
2065                 else if (step_idx == 1 && hop_num == 2)
2066                         bt_ba = table->bt_l1_dma_addr[l1_idx];
2067
2068                 /* configure the ba, tag, and op */
2069                 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
2070                                         0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
2071         }
2072
2073         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2074         return ret;
2075 }
2076
2077 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
2078                                  struct hns_roce_hem_table *table, int obj,
2079                                  int step_idx)
2080 {
2081         struct device *dev = hr_dev->dev;
2082         struct hns_roce_cmd_mailbox *mailbox;
2083         int ret = 0;
2084         u16 op = 0xff;
2085
2086         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2087                 return 0;
2088
2089         switch (table->type) {
2090         case HEM_TYPE_QPC:
2091                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
2092                 break;
2093         case HEM_TYPE_MTPT:
2094                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
2095                 break;
2096         case HEM_TYPE_CQC:
2097                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
2098                 break;
2099         case HEM_TYPE_SRQC:
2100                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
2101                 break;
2102         default:
2103                 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
2104                          table->type);
2105                 return 0;
2106         }
2107         op += step_idx;
2108
2109         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2110         if (IS_ERR(mailbox))
2111                 return PTR_ERR(mailbox);
2112
2113         /* configure the tag and op */
2114         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
2115                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2116
2117         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2118         return ret;
2119 }
2120
2121 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
2122                                  struct hns_roce_mtt *mtt,
2123                                  enum ib_qp_state cur_state,
2124                                  enum ib_qp_state new_state,
2125                                  struct hns_roce_v2_qp_context *context,
2126                                  struct hns_roce_qp *hr_qp)
2127 {
2128         struct hns_roce_cmd_mailbox *mailbox;
2129         int ret;
2130
2131         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2132         if (IS_ERR(mailbox))
2133                 return PTR_ERR(mailbox);
2134
2135         memcpy(mailbox->buf, context, sizeof(*context) * 2);
2136
2137         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2138                                 HNS_ROCE_CMD_MODIFY_QPC,
2139                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2140
2141         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2142
2143         return ret;
2144 }
2145
2146 static void set_access_flags(struct hns_roce_qp *hr_qp,
2147                              struct hns_roce_v2_qp_context *context,
2148                              struct hns_roce_v2_qp_context *qpc_mask,
2149                              const struct ib_qp_attr *attr, int attr_mask)
2150 {
2151         u8 dest_rd_atomic;
2152         u32 access_flags;
2153
2154         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
2155                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
2156
2157         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
2158                        attr->qp_access_flags : hr_qp->atomic_rd_en;
2159
2160         if (!dest_rd_atomic)
2161                 access_flags &= IB_ACCESS_REMOTE_WRITE;
2162
2163         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2164                      !!(access_flags & IB_ACCESS_REMOTE_READ));
2165         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
2166
2167         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2168                      !!(access_flags & IB_ACCESS_REMOTE_WRITE));
2169         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
2170
2171         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2172                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
2173         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
2174 }
2175
2176 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
2177                                     const struct ib_qp_attr *attr,
2178                                     int attr_mask,
2179                                     struct hns_roce_v2_qp_context *context,
2180                                     struct hns_roce_v2_qp_context *qpc_mask)
2181 {
2182         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2183
2184         /*
2185          * In v2 engine, software pass context and context mask to hardware
2186          * when modifying qp. If software need modify some fields in context,
2187          * we should set all bits of the relevant fields in context mask to
2188          * 0 at the same time, else set them to 0x1.
2189          */
2190         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2191                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2192         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2193                        V2_QPC_BYTE_4_TST_S, 0);
2194
2195         if (ibqp->qp_type == IB_QPT_GSI)
2196                 roce_set_field(context->byte_4_sqpn_tst,
2197                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2198                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2199                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
2200         else
2201                 roce_set_field(context->byte_4_sqpn_tst,
2202                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2203                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2204                                hr_qp->sq.max_gs > 2 ?
2205                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2206
2207         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2208                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2209
2210         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2211                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2212         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2213                        V2_QPC_BYTE_4_SQPN_S, 0);
2214
2215         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2216                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2217         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2218                        V2_QPC_BYTE_16_PD_S, 0);
2219
2220         roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2221                        V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
2222         roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2223                        V2_QPC_BYTE_20_RQWS_S, 0);
2224
2225         roce_set_field(context->byte_20_smac_sgid_idx,
2226                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2227                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2228         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2229                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2230
2231         roce_set_field(context->byte_20_smac_sgid_idx,
2232                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2233                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2234         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2235                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2236
2237         /* No VLAN need to set 0xFFF */
2238         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M,
2239                        V2_QPC_BYTE_24_VLAN_IDX_S, 0xfff);
2240         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M,
2241                        V2_QPC_BYTE_24_VLAN_IDX_S, 0);
2242
2243         /*
2244          * Set some fields in context to zero, Because the default values
2245          * of all fields in context are zero, we need not set them to 0 again.
2246          * but we should set the relevant fields of context mask to 0.
2247          */
2248         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
2249         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
2250         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
2251         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
2252
2253         roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_MAPID_M,
2254                        V2_QPC_BYTE_60_MAPID_S, 0);
2255
2256         roce_set_bit(qpc_mask->byte_60_qpst_mapid,
2257                      V2_QPC_BYTE_60_INNER_MAP_IND_S, 0);
2258         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_MAP_IND_S,
2259                      0);
2260         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_RQ_MAP_IND_S,
2261                      0);
2262         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_EXT_MAP_IND_S,
2263                      0);
2264         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_RLS_IND_S,
2265                      0);
2266         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_EXT_IND_S,
2267                      0);
2268         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
2269         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
2270
2271         if (attr_mask & IB_QP_QKEY) {
2272                 context->qkey_xrcd = attr->qkey;
2273                 qpc_mask->qkey_xrcd = 0;
2274                 hr_qp->qkey = attr->qkey;
2275         }
2276
2277         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 1);
2278         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
2279
2280         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2281                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2282         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2283                        V2_QPC_BYTE_80_RX_CQN_S, 0);
2284         if (ibqp->srq) {
2285                 roce_set_field(context->byte_76_srqn_op_en,
2286                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2287                                to_hr_srq(ibqp->srq)->srqn);
2288                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
2289                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2290                 roce_set_bit(context->byte_76_srqn_op_en,
2291                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
2292                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2293                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
2294         }
2295
2296         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2297                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2298                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2299         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2300                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2301                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2302
2303         roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
2304                        V2_QPC_BYTE_92_SRQ_INFO_S, 0);
2305
2306         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2307                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2308
2309         roce_set_field(qpc_mask->byte_104_rq_sge,
2310                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
2311                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
2312
2313         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2314                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2315         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2316                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2317                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2318         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2319                      V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
2320
2321         qpc_mask->rq_rnr_timer = 0;
2322         qpc_mask->rx_msg_len = 0;
2323         qpc_mask->rx_rkey_pkt_info = 0;
2324         qpc_mask->rx_va = 0;
2325
2326         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2327                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2328         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2329                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2330
2331         roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S, 0);
2332         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
2333                        V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
2334         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
2335                        V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
2336
2337         roce_set_field(qpc_mask->byte_144_raq,
2338                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
2339                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
2340         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S,
2341                      0);
2342         roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
2343                        V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
2344         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
2345
2346         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
2347                        V2_QPC_BYTE_148_RQ_MSN_S, 0);
2348         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
2349                        V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
2350
2351         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2352                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2353         roce_set_field(qpc_mask->byte_152_raq,
2354                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
2355                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
2356
2357         roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
2358                        V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
2359
2360         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2361                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
2362                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
2363         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2364                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
2365                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
2366
2367         roce_set_field(context->byte_168_irrl_idx,
2368                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2369                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2370                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2371         roce_set_field(qpc_mask->byte_168_irrl_idx,
2372                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2373                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2374
2375         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2376                      V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
2377         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2378                      V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
2379         roce_set_field(qpc_mask->byte_168_irrl_idx,
2380                        V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
2381                        V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
2382
2383         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2384                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
2385         roce_set_field(qpc_mask->byte_172_sq_psn,
2386                        V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2387                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
2388
2389         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
2390                      0);
2391
2392         roce_set_field(qpc_mask->byte_176_msg_pktn,
2393                        V2_QPC_BYTE_176_MSG_USE_PKTN_M,
2394                        V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
2395         roce_set_field(qpc_mask->byte_176_msg_pktn,
2396                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
2397                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
2398
2399         roce_set_field(qpc_mask->byte_184_irrl_idx,
2400                        V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
2401                        V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
2402
2403         qpc_mask->cur_sge_offset = 0;
2404
2405         roce_set_field(qpc_mask->byte_192_ext_sge,
2406                        V2_QPC_BYTE_192_CUR_SGE_IDX_M,
2407                        V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
2408         roce_set_field(qpc_mask->byte_192_ext_sge,
2409                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
2410                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
2411
2412         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
2413                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
2414
2415         roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
2416                        V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
2417         roce_set_field(qpc_mask->byte_200_sq_max,
2418                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
2419                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
2420
2421         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
2422         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
2423
2424         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
2425                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
2426
2427         qpc_mask->sq_timer = 0;
2428
2429         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
2430                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
2431                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
2432         roce_set_field(qpc_mask->byte_232_irrl_sge,
2433                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
2434                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
2435
2436         qpc_mask->irrl_cur_sge_offset = 0;
2437
2438         roce_set_field(qpc_mask->byte_240_irrl_tail,
2439                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
2440                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
2441         roce_set_field(qpc_mask->byte_240_irrl_tail,
2442                        V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
2443                        V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
2444         roce_set_field(qpc_mask->byte_240_irrl_tail,
2445                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
2446                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
2447
2448         roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
2449                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
2450         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
2451                      0);
2452         roce_set_field(qpc_mask->byte_248_ack_psn,
2453                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
2454                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
2455         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
2456                      0);
2457         roce_set_bit(qpc_mask->byte_248_ack_psn,
2458                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
2459         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
2460                      0);
2461
2462         hr_qp->access_flags = attr->qp_access_flags;
2463         hr_qp->pkey_index = attr->pkey_index;
2464         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2465                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2466         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2467                        V2_QPC_BYTE_252_TX_CQN_S, 0);
2468
2469         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
2470                        V2_QPC_BYTE_252_ERR_TYPE_S, 0);
2471
2472         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2473                        V2_QPC_BYTE_256_RQ_CQE_IDX_M,
2474                        V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
2475         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2476                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
2477                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
2478 }
2479
2480 static void modify_qp_init_to_init(struct ib_qp *ibqp,
2481                                    const struct ib_qp_attr *attr, int attr_mask,
2482                                    struct hns_roce_v2_qp_context *context,
2483                                    struct hns_roce_v2_qp_context *qpc_mask)
2484 {
2485         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2486
2487         /*
2488          * In v2 engine, software pass context and context mask to hardware
2489          * when modifying qp. If software need modify some fields in context,
2490          * we should set all bits of the relevant fields in context mask to
2491          * 0 at the same time, else set them to 0x1.
2492          */
2493         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2494                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2495         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2496                        V2_QPC_BYTE_4_TST_S, 0);
2497
2498         if (ibqp->qp_type == IB_QPT_GSI)
2499                 roce_set_field(context->byte_4_sqpn_tst,
2500                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2501                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2502                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
2503         else
2504                 roce_set_field(context->byte_4_sqpn_tst,
2505                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2506                                V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
2507                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2508
2509         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2510                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2511
2512         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2513                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2514                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2515                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2516                              0);
2517
2518                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2519                              !!(attr->qp_access_flags &
2520                              IB_ACCESS_REMOTE_WRITE));
2521                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2522                              0);
2523
2524                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2525                              !!(attr->qp_access_flags &
2526                              IB_ACCESS_REMOTE_ATOMIC));
2527                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2528                              0);
2529         } else {
2530                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2531                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
2532                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2533                              0);
2534
2535                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2536                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
2537                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2538                              0);
2539
2540                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2541                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
2542                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2543                              0);
2544         }
2545
2546         roce_set_field(context->byte_20_smac_sgid_idx,
2547                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2548                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2549         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2550                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2551
2552         roce_set_field(context->byte_20_smac_sgid_idx,
2553                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2554                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2555         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2556                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2557
2558         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2559                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2560         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2561                        V2_QPC_BYTE_16_PD_S, 0);
2562
2563         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2564                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2565         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2566                        V2_QPC_BYTE_80_RX_CQN_S, 0);
2567
2568         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2569                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2570         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2571                        V2_QPC_BYTE_252_TX_CQN_S, 0);
2572
2573         if (ibqp->srq) {
2574                 roce_set_bit(context->byte_76_srqn_op_en,
2575                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
2576                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2577                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
2578                 roce_set_field(context->byte_76_srqn_op_en,
2579                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2580                                to_hr_srq(ibqp->srq)->srqn);
2581                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
2582                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2583         }
2584
2585         if (attr_mask & IB_QP_QKEY) {
2586                 context->qkey_xrcd = attr->qkey;
2587                 qpc_mask->qkey_xrcd = 0;
2588         }
2589
2590         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2591                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2592         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2593                        V2_QPC_BYTE_4_SQPN_S, 0);
2594
2595         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2596                        V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
2597         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2598                        V2_QPC_BYTE_56_DQPN_S, 0);
2599         roce_set_field(context->byte_168_irrl_idx,
2600                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2601                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2602                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2603         roce_set_field(qpc_mask->byte_168_irrl_idx,
2604                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2605                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2606 }
2607
2608 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
2609                                  const struct ib_qp_attr *attr, int attr_mask,
2610                                  struct hns_roce_v2_qp_context *context,
2611                                  struct hns_roce_v2_qp_context *qpc_mask)
2612 {
2613         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2614         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2615         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2616         struct device *dev = hr_dev->dev;
2617         dma_addr_t dma_handle_3;
2618         dma_addr_t dma_handle_2;
2619         dma_addr_t dma_handle;
2620         u32 page_size;
2621         u8 port_num;
2622         u64 *mtts_3;
2623         u64 *mtts_2;
2624         u64 *mtts;
2625         u8 *dmac;
2626         u8 *smac;
2627         int port;
2628
2629         /* Search qp buf's mtts */
2630         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2631                                    hr_qp->mtt.first_seg, &dma_handle);
2632         if (!mtts) {
2633                 dev_err(dev, "qp buf pa find failed\n");
2634                 return -EINVAL;
2635         }
2636
2637         /* Search IRRL's mtts */
2638         mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2639                                      hr_qp->qpn, &dma_handle_2);
2640         if (!mtts_2) {
2641                 dev_err(dev, "qp irrl_table find failed\n");
2642                 return -EINVAL;
2643         }
2644
2645         /* Search TRRL's mtts */
2646         mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
2647                                      hr_qp->qpn, &dma_handle_3);
2648         if (!mtts_3) {
2649                 dev_err(dev, "qp trrl_table find failed\n");
2650                 return -EINVAL;
2651         }
2652
2653         if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_ACCESS_FLAGS) ||
2654             (attr_mask & IB_QP_PKEY_INDEX) || (attr_mask & IB_QP_QKEY)) {
2655                 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
2656                 return -EINVAL;
2657         }
2658
2659         dmac = (u8 *)attr->ah_attr.roce.dmac;
2660         context->wqe_sge_ba = (u32)(dma_handle >> 3);
2661         qpc_mask->wqe_sge_ba = 0;
2662
2663         /*
2664          * In v2 engine, software pass context and context mask to hardware
2665          * when modifying qp. If software need modify some fields in context,
2666          * we should set all bits of the relevant fields in context mask to
2667          * 0 at the same time, else set them to 0x1.
2668          */
2669         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
2670                        V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
2671         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
2672                        V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
2673
2674         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
2675                        V2_QPC_BYTE_12_SQ_HOP_NUM_S,
2676                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
2677                        0 : hr_dev->caps.mtt_hop_num);
2678         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
2679                        V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
2680
2681         roce_set_field(context->byte_20_smac_sgid_idx,
2682                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
2683                        V2_QPC_BYTE_20_SGE_HOP_NUM_S,
2684                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
2685                        hr_dev->caps.mtt_hop_num : 0);
2686         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2687                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
2688                        V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
2689
2690         roce_set_field(context->byte_20_smac_sgid_idx,
2691                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
2692                        V2_QPC_BYTE_20_RQ_HOP_NUM_S,
2693                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
2694                        0 : hr_dev->caps.mtt_hop_num);
2695         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2696                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
2697                        V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
2698
2699         roce_set_field(context->byte_16_buf_ba_pg_sz,
2700                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
2701                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
2702                        hr_dev->caps.mtt_ba_pg_sz);
2703         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
2704                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
2705                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
2706
2707         roce_set_field(context->byte_16_buf_ba_pg_sz,
2708                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
2709                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
2710                        hr_dev->caps.mtt_buf_pg_sz);
2711         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
2712                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
2713                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
2714
2715         roce_set_field(context->byte_80_rnr_rx_cqn,
2716                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
2717                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
2718         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
2719                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
2720                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
2721
2722         page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
2723         context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
2724                                     >> PAGE_ADDR_SHIFT);
2725         qpc_mask->rq_cur_blk_addr = 0;
2726
2727         roce_set_field(context->byte_92_srq_info,
2728                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
2729                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
2730                        mtts[hr_qp->rq.offset / page_size]
2731                        >> (32 + PAGE_ADDR_SHIFT));
2732         roce_set_field(qpc_mask->byte_92_srq_info,
2733                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
2734                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
2735
2736         context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
2737                                     >> PAGE_ADDR_SHIFT);
2738         qpc_mask->rq_nxt_blk_addr = 0;
2739
2740         roce_set_field(context->byte_104_rq_sge,
2741                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
2742                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
2743                        mtts[hr_qp->rq.offset / page_size + 1]
2744                        >> (32 + PAGE_ADDR_SHIFT));
2745         roce_set_field(qpc_mask->byte_104_rq_sge,
2746                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
2747                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
2748
2749         roce_set_field(context->byte_108_rx_reqepsn,
2750                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
2751                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
2752         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2753                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
2754                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
2755
2756         roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
2757                        V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
2758         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
2759                        V2_QPC_BYTE_132_TRRL_BA_S, 0);
2760         context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
2761         qpc_mask->trrl_ba = 0;
2762         roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
2763                        V2_QPC_BYTE_140_TRRL_BA_S,
2764                        (u32)(dma_handle_3 >> (32 + 16 + 4)));
2765         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
2766                        V2_QPC_BYTE_140_TRRL_BA_S, 0);
2767
2768         context->irrl_ba = (u32)(dma_handle_2 >> 6);
2769         qpc_mask->irrl_ba = 0;
2770         roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
2771                        V2_QPC_BYTE_208_IRRL_BA_S,
2772                        dma_handle_2 >> (32 + 6));
2773         roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
2774                        V2_QPC_BYTE_208_IRRL_BA_S, 0);
2775
2776         roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
2777         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
2778
2779         roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
2780                      hr_qp->sq_signal_bits);
2781         roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
2782                      0);
2783
2784         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
2785
2786         smac = (u8 *)hr_dev->dev_addr[port];
2787         /* when dmac equals smac or loop_idc is 1, it should loopback */
2788         if (ether_addr_equal_unaligned(dmac, smac) ||
2789             hr_dev->loop_idc == 0x1) {
2790                 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
2791                 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
2792         }
2793
2794         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
2795              attr->max_dest_rd_atomic) {
2796                 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
2797                                V2_QPC_BYTE_140_RR_MAX_S,
2798                                fls(attr->max_dest_rd_atomic - 1));
2799                 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
2800                                V2_QPC_BYTE_140_RR_MAX_S, 0);
2801         }
2802
2803         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2804                        V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
2805         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2806                        V2_QPC_BYTE_56_DQPN_S, 0);
2807
2808         /* Configure GID index */
2809         port_num = rdma_ah_get_port_num(&attr->ah_attr);
2810         roce_set_field(context->byte_20_smac_sgid_idx,
2811                        V2_QPC_BYTE_20_SGID_IDX_M,
2812                        V2_QPC_BYTE_20_SGID_IDX_S,
2813                        hns_get_gid_index(hr_dev, port_num - 1,
2814                                          grh->sgid_index));
2815         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2816                        V2_QPC_BYTE_20_SGID_IDX_M,
2817                        V2_QPC_BYTE_20_SGID_IDX_S, 0);
2818         memcpy(&(context->dmac), dmac, 4);
2819         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
2820                        V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
2821         qpc_mask->dmac = 0;
2822         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
2823                        V2_QPC_BYTE_52_DMAC_S, 0);
2824
2825         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
2826                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
2827         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
2828                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
2829
2830         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
2831                        V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
2832         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
2833                        V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
2834
2835         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
2836                        V2_QPC_BYTE_28_FL_S, grh->flow_label);
2837         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
2838                        V2_QPC_BYTE_28_FL_S, 0);
2839
2840         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
2841                        V2_QPC_BYTE_24_TC_S, grh->traffic_class);
2842         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
2843                        V2_QPC_BYTE_24_TC_S, 0);
2844
2845         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
2846                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2847                                V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
2848         else
2849                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2850                                V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
2851
2852         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2853                        V2_QPC_BYTE_24_MTU_S, 0);
2854
2855         memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
2856         memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
2857
2858         roce_set_field(context->byte_84_rq_ci_pi,
2859                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2860                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
2861         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2862                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2863                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2864
2865         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2866                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2867                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2868         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2869                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2870         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2871                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2872         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2873                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2874                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2875
2876         context->rq_rnr_timer = 0;
2877         qpc_mask->rq_rnr_timer = 0;
2878
2879         roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2880                        V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
2881         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2882                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2883
2884         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2885                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2886         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2887                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2888
2889         roce_set_field(context->byte_168_irrl_idx,
2890                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
2891                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
2892         roce_set_field(qpc_mask->byte_168_irrl_idx,
2893                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
2894                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
2895
2896         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
2897                        V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
2898         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
2899                        V2_QPC_BYTE_28_SL_S, 0);
2900         hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
2901
2902         return 0;
2903 }
2904
2905 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
2906                                 const struct ib_qp_attr *attr, int attr_mask,
2907                                 struct hns_roce_v2_qp_context *context,
2908                                 struct hns_roce_v2_qp_context *qpc_mask)
2909 {
2910         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2911         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2912         struct device *dev = hr_dev->dev;
2913         dma_addr_t dma_handle;
2914         u32 page_size;
2915         u64 *mtts;
2916
2917         /* Search qp buf's mtts */
2918         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2919                                    hr_qp->mtt.first_seg, &dma_handle);
2920         if (!mtts) {
2921                 dev_err(dev, "qp buf pa find failed\n");
2922                 return -EINVAL;
2923         }
2924
2925