Merge tag 'for-linus-unmerged' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma...
[muen/linux.git] / drivers / infiniband / hw / hns / hns_roce_hw_v2.c
1 /*
2  * Copyright (c) 2016-2017 Hisilicon Limited.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32
33 #include <linux/acpi.h>
34 #include <linux/etherdevice.h>
35 #include <linux/interrupt.h>
36 #include <linux/kernel.h>
37 #include <net/addrconf.h>
38 #include <rdma/ib_umem.h>
39
40 #include "hnae3.h"
41 #include "hns_roce_common.h"
42 #include "hns_roce_device.h"
43 #include "hns_roce_cmd.h"
44 #include "hns_roce_hem.h"
45 #include "hns_roce_hw_v2.h"
46
47 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
48                             struct ib_sge *sg)
49 {
50         dseg->lkey = cpu_to_le32(sg->lkey);
51         dseg->addr = cpu_to_le64(sg->addr);
52         dseg->len  = cpu_to_le32(sg->length);
53 }
54
55 static int set_rwqe_data_seg(struct ib_qp *ibqp, struct ib_send_wr *wr,
56                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
57                              void *wqe, unsigned int *sge_ind,
58                              struct ib_send_wr **bad_wr)
59 {
60         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
61         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
62         struct hns_roce_qp *qp = to_hr_qp(ibqp);
63         int i;
64
65         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
66                 if (le32_to_cpu(rc_sq_wqe->msg_len) >
67                     hr_dev->caps.max_sq_inline) {
68                         *bad_wr = wr;
69                         dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
70                                 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
71                         return -EINVAL;
72                 }
73
74                 for (i = 0; i < wr->num_sge; i++) {
75                         memcpy(wqe, ((void *)wr->sg_list[i].addr),
76                                wr->sg_list[i].length);
77                         wqe += wr->sg_list[i].length;
78                 }
79
80                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
81                              1);
82         } else {
83                 if (wr->num_sge <= 2) {
84                         for (i = 0; i < wr->num_sge; i++) {
85                                 if (likely(wr->sg_list[i].length)) {
86                                         set_data_seg_v2(dseg, wr->sg_list + i);
87                                         dseg++;
88                                 }
89                         }
90                 } else {
91                         roce_set_field(rc_sq_wqe->byte_20,
92                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
93                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
94                                      (*sge_ind) & (qp->sge.sge_cnt - 1));
95
96                         for (i = 0; i < 2; i++) {
97                                 if (likely(wr->sg_list[i].length)) {
98                                         set_data_seg_v2(dseg, wr->sg_list + i);
99                                         dseg++;
100                                 }
101                         }
102
103                         dseg = get_send_extend_sge(qp,
104                                             (*sge_ind) & (qp->sge.sge_cnt - 1));
105
106                         for (i = 0; i < wr->num_sge - 2; i++) {
107                                 if (likely(wr->sg_list[i + 2].length)) {
108                                         set_data_seg_v2(dseg,
109                                                         wr->sg_list + 2 + i);
110                                         dseg++;
111                                         (*sge_ind)++;
112                                 }
113                         }
114                 }
115
116                 roce_set_field(rc_sq_wqe->byte_16,
117                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
118                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
119         }
120
121         return 0;
122 }
123
124 static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
125                                  struct ib_send_wr **bad_wr)
126 {
127         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
128         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
129         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
130         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
131         struct hns_roce_qp *qp = to_hr_qp(ibqp);
132         struct hns_roce_v2_wqe_data_seg *dseg;
133         struct device *dev = hr_dev->dev;
134         struct hns_roce_v2_db sq_db;
135         unsigned int sge_ind = 0;
136         unsigned int owner_bit;
137         unsigned long flags;
138         unsigned int ind;
139         void *wqe = NULL;
140         u32 tmp_len = 0;
141         bool loopback;
142         int ret = 0;
143         u8 *smac;
144         int nreq;
145         int i;
146
147         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
148                      ibqp->qp_type != IB_QPT_GSI &&
149                      ibqp->qp_type != IB_QPT_UD)) {
150                 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
151                 *bad_wr = NULL;
152                 return -EOPNOTSUPP;
153         }
154
155         if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
156                      qp->state == IB_QPS_RTR)) {
157                 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
158                 *bad_wr = wr;
159                 return -EINVAL;
160         }
161
162         spin_lock_irqsave(&qp->sq.lock, flags);
163         ind = qp->sq_next_wqe;
164         sge_ind = qp->next_sge;
165
166         for (nreq = 0; wr; ++nreq, wr = wr->next) {
167                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
168                         ret = -ENOMEM;
169                         *bad_wr = wr;
170                         goto out;
171                 }
172
173                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
174                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
175                                 wr->num_sge, qp->sq.max_gs);
176                         ret = -EINVAL;
177                         *bad_wr = wr;
178                         goto out;
179                 }
180
181                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
182                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
183                                                                       wr->wr_id;
184
185                 owner_bit = ~(qp->sq.head >> ilog2(qp->sq.wqe_cnt)) & 0x1;
186
187                 /* Corresponding to the QP type, wqe process separately */
188                 if (ibqp->qp_type == IB_QPT_GSI) {
189                         ud_sq_wqe = wqe;
190                         memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
191
192                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
193                                        V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
194                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
195                                        V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
196                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
197                                        V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
198                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
199                                        V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
200                         roce_set_field(ud_sq_wqe->byte_48,
201                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
202                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
203                                        ah->av.mac[4]);
204                         roce_set_field(ud_sq_wqe->byte_48,
205                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
206                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
207                                        ah->av.mac[5]);
208
209                         /* MAC loopback */
210                         smac = (u8 *)hr_dev->dev_addr[qp->port];
211                         loopback = ether_addr_equal_unaligned(ah->av.mac,
212                                                               smac) ? 1 : 0;
213
214                         roce_set_bit(ud_sq_wqe->byte_40,
215                                      V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
216
217                         roce_set_field(ud_sq_wqe->byte_4,
218                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
219                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
220                                        HNS_ROCE_V2_WQE_OP_SEND);
221
222                         for (i = 0; i < wr->num_sge; i++)
223                                 tmp_len += wr->sg_list[i].length;
224
225                         ud_sq_wqe->msg_len =
226                          cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
227
228                         switch (wr->opcode) {
229                         case IB_WR_SEND_WITH_IMM:
230                         case IB_WR_RDMA_WRITE_WITH_IMM:
231                                 ud_sq_wqe->immtdata = wr->ex.imm_data;
232                                 break;
233                         default:
234                                 ud_sq_wqe->immtdata = 0;
235                                 break;
236                         }
237
238                         /* Set sig attr */
239                         roce_set_bit(ud_sq_wqe->byte_4,
240                                    V2_UD_SEND_WQE_BYTE_4_CQE_S,
241                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
242
243                         /* Set se attr */
244                         roce_set_bit(ud_sq_wqe->byte_4,
245                                   V2_UD_SEND_WQE_BYTE_4_SE_S,
246                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
247
248                         roce_set_bit(ud_sq_wqe->byte_4,
249                                      V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
250
251                         roce_set_field(ud_sq_wqe->byte_16,
252                                        V2_UD_SEND_WQE_BYTE_16_PD_M,
253                                        V2_UD_SEND_WQE_BYTE_16_PD_S,
254                                        to_hr_pd(ibqp->pd)->pdn);
255
256                         roce_set_field(ud_sq_wqe->byte_16,
257                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
258                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
259                                        wr->num_sge);
260
261                         roce_set_field(ud_sq_wqe->byte_20,
262                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
263                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
264                                      sge_ind & (qp->sge.sge_cnt - 1));
265
266                         roce_set_field(ud_sq_wqe->byte_24,
267                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
268                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
269                         ud_sq_wqe->qkey =
270                              cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
271                              qp->qkey : ud_wr(wr)->remote_qkey);
272                         roce_set_field(ud_sq_wqe->byte_32,
273                                        V2_UD_SEND_WQE_BYTE_32_DQPN_M,
274                                        V2_UD_SEND_WQE_BYTE_32_DQPN_S,
275                                        ud_wr(wr)->remote_qpn);
276
277                         roce_set_field(ud_sq_wqe->byte_36,
278                                        V2_UD_SEND_WQE_BYTE_36_VLAN_M,
279                                        V2_UD_SEND_WQE_BYTE_36_VLAN_S,
280                                        le16_to_cpu(ah->av.vlan));
281                         roce_set_field(ud_sq_wqe->byte_36,
282                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
283                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
284                                        ah->av.hop_limit);
285                         roce_set_field(ud_sq_wqe->byte_36,
286                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
287                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
288                                        0);
289                         roce_set_field(ud_sq_wqe->byte_36,
290                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
291                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
292                                        0);
293                         roce_set_field(ud_sq_wqe->byte_40,
294                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
295                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, 0);
296                         roce_set_field(ud_sq_wqe->byte_40,
297                                        V2_UD_SEND_WQE_BYTE_40_SL_M,
298                                        V2_UD_SEND_WQE_BYTE_40_SL_S,
299                                       le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
300                                       HNS_ROCE_SL_SHIFT);
301                         roce_set_field(ud_sq_wqe->byte_40,
302                                        V2_UD_SEND_WQE_BYTE_40_PORTN_M,
303                                        V2_UD_SEND_WQE_BYTE_40_PORTN_S,
304                                        qp->port);
305
306                         roce_set_field(ud_sq_wqe->byte_48,
307                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
308                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
309                                        hns_get_gid_index(hr_dev, qp->phy_port,
310                                                          ah->av.gid_index));
311
312                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
313                                GID_LEN_V2);
314
315                         dseg = get_send_extend_sge(qp,
316                                             sge_ind & (qp->sge.sge_cnt - 1));
317                         for (i = 0; i < wr->num_sge; i++) {
318                                 set_data_seg_v2(dseg + i, wr->sg_list + i);
319                                 sge_ind++;
320                         }
321
322                         ind++;
323                 } else if (ibqp->qp_type == IB_QPT_RC) {
324                         rc_sq_wqe = wqe;
325                         memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
326                         for (i = 0; i < wr->num_sge; i++)
327                                 tmp_len += wr->sg_list[i].length;
328
329                         rc_sq_wqe->msg_len =
330                          cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
331
332                         switch (wr->opcode) {
333                         case IB_WR_SEND_WITH_IMM:
334                         case IB_WR_RDMA_WRITE_WITH_IMM:
335                                 rc_sq_wqe->immtdata = wr->ex.imm_data;
336                                 break;
337                         case IB_WR_SEND_WITH_INV:
338                                 rc_sq_wqe->inv_key =
339                                         cpu_to_le32(wr->ex.invalidate_rkey);
340                                 break;
341                         default:
342                                 rc_sq_wqe->immtdata = 0;
343                                 break;
344                         }
345
346                         roce_set_bit(rc_sq_wqe->byte_4,
347                                      V2_RC_SEND_WQE_BYTE_4_FENCE_S,
348                                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
349
350                         roce_set_bit(rc_sq_wqe->byte_4,
351                                   V2_RC_SEND_WQE_BYTE_4_SE_S,
352                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
353
354                         roce_set_bit(rc_sq_wqe->byte_4,
355                                    V2_RC_SEND_WQE_BYTE_4_CQE_S,
356                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
357
358                         roce_set_bit(rc_sq_wqe->byte_4,
359                                      V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
360
361                         switch (wr->opcode) {
362                         case IB_WR_RDMA_READ:
363                                 roce_set_field(rc_sq_wqe->byte_4,
364                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
365                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
366                                                HNS_ROCE_V2_WQE_OP_RDMA_READ);
367                                 rc_sq_wqe->rkey =
368                                         cpu_to_le32(rdma_wr(wr)->rkey);
369                                 rc_sq_wqe->va =
370                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
371                                 break;
372                         case IB_WR_RDMA_WRITE:
373                                 roce_set_field(rc_sq_wqe->byte_4,
374                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
375                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
376                                                HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
377                                 rc_sq_wqe->rkey =
378                                         cpu_to_le32(rdma_wr(wr)->rkey);
379                                 rc_sq_wqe->va =
380                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
381                                 break;
382                         case IB_WR_RDMA_WRITE_WITH_IMM:
383                                 roce_set_field(rc_sq_wqe->byte_4,
384                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
385                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
386                                        HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM);
387                                 rc_sq_wqe->rkey =
388                                         cpu_to_le32(rdma_wr(wr)->rkey);
389                                 rc_sq_wqe->va =
390                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
391                                 break;
392                         case IB_WR_SEND:
393                                 roce_set_field(rc_sq_wqe->byte_4,
394                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
395                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
396                                                HNS_ROCE_V2_WQE_OP_SEND);
397                                 break;
398                         case IB_WR_SEND_WITH_INV:
399                                 roce_set_field(rc_sq_wqe->byte_4,
400                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
401                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
402                                        HNS_ROCE_V2_WQE_OP_SEND_WITH_INV);
403                                 break;
404                         case IB_WR_SEND_WITH_IMM:
405                                 roce_set_field(rc_sq_wqe->byte_4,
406                                               V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
407                                               V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
408                                               HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
409                                 break;
410                         case IB_WR_LOCAL_INV:
411                                 roce_set_field(rc_sq_wqe->byte_4,
412                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
413                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
414                                                HNS_ROCE_V2_WQE_OP_LOCAL_INV);
415                                 break;
416                         case IB_WR_ATOMIC_CMP_AND_SWP:
417                                 roce_set_field(rc_sq_wqe->byte_4,
418                                           V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
419                                           V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
420                                           HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
421                                 break;
422                         case IB_WR_ATOMIC_FETCH_AND_ADD:
423                                 roce_set_field(rc_sq_wqe->byte_4,
424                                          V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
425                                          V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
426                                          HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
427                                 break;
428                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
429                                 roce_set_field(rc_sq_wqe->byte_4,
430                                       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
431                                       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
432                                       HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP);
433                                 break;
434                         case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
435                                 roce_set_field(rc_sq_wqe->byte_4,
436                                      V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
437                                      V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
438                                      HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD);
439                                 break;
440                         default:
441                                 roce_set_field(rc_sq_wqe->byte_4,
442                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
443                                                V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
444                                                HNS_ROCE_V2_WQE_OP_MASK);
445                                 break;
446                         }
447
448                         wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
449                         dseg = wqe;
450
451                         ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, wqe,
452                                                 &sge_ind, bad_wr);
453                         if (ret)
454                                 goto out;
455                         ind++;
456                 } else {
457                         dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
458                         spin_unlock_irqrestore(&qp->sq.lock, flags);
459                         return -EOPNOTSUPP;
460                 }
461         }
462
463 out:
464         if (likely(nreq)) {
465                 qp->sq.head += nreq;
466                 /* Memory barrier */
467                 wmb();
468
469                 sq_db.byte_4 = 0;
470                 sq_db.parameter = 0;
471
472                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
473                                V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
474                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
475                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
476                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M,
477                                V2_DB_PARAMETER_CONS_IDX_S,
478                                qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
479                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
480                                V2_DB_PARAMETER_SL_S, qp->sl);
481
482                 hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
483
484                 qp->sq_next_wqe = ind;
485                 qp->next_sge = sge_ind;
486         }
487
488         spin_unlock_irqrestore(&qp->sq.lock, flags);
489
490         return ret;
491 }
492
493 static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
494                                  struct ib_recv_wr **bad_wr)
495 {
496         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
497         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
498         struct hns_roce_v2_wqe_data_seg *dseg;
499         struct hns_roce_rinl_sge *sge_list;
500         struct device *dev = hr_dev->dev;
501         unsigned long flags;
502         void *wqe = NULL;
503         int ret = 0;
504         int nreq;
505         int ind;
506         int i;
507
508         spin_lock_irqsave(&hr_qp->rq.lock, flags);
509         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
510
511         if (hr_qp->state == IB_QPS_RESET) {
512                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
513                 *bad_wr = wr;
514                 return -EINVAL;
515         }
516
517         for (nreq = 0; wr; ++nreq, wr = wr->next) {
518                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
519                         hr_qp->ibqp.recv_cq)) {
520                         ret = -ENOMEM;
521                         *bad_wr = wr;
522                         goto out;
523                 }
524
525                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
526                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
527                                 wr->num_sge, hr_qp->rq.max_gs);
528                         ret = -EINVAL;
529                         *bad_wr = wr;
530                         goto out;
531                 }
532
533                 wqe = get_recv_wqe(hr_qp, ind);
534                 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
535                 for (i = 0; i < wr->num_sge; i++) {
536                         if (!wr->sg_list[i].length)
537                                 continue;
538                         set_data_seg_v2(dseg, wr->sg_list + i);
539                         dseg++;
540                 }
541
542                 if (i < hr_qp->rq.max_gs) {
543                         dseg[i].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
544                         dseg[i].addr = 0;
545                 }
546
547                 /* rq support inline data */
548                 sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
549                 hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt = (u32)wr->num_sge;
550                 for (i = 0; i < wr->num_sge; i++) {
551                         sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
552                         sge_list[i].len = wr->sg_list[i].length;
553                 }
554
555                 hr_qp->rq.wrid[ind] = wr->wr_id;
556
557                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
558         }
559
560 out:
561         if (likely(nreq)) {
562                 hr_qp->rq.head += nreq;
563                 /* Memory barrier */
564                 wmb();
565
566                 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
567         }
568         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
569
570         return ret;
571 }
572
573 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
574 {
575         int ntu = ring->next_to_use;
576         int ntc = ring->next_to_clean;
577         int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
578
579         return ring->desc_num - used - 1;
580 }
581
582 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
583                                    struct hns_roce_v2_cmq_ring *ring)
584 {
585         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
586
587         ring->desc = kzalloc(size, GFP_KERNEL);
588         if (!ring->desc)
589                 return -ENOMEM;
590
591         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
592                                              DMA_BIDIRECTIONAL);
593         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
594                 ring->desc_dma_addr = 0;
595                 kfree(ring->desc);
596                 ring->desc = NULL;
597                 return -ENOMEM;
598         }
599
600         return 0;
601 }
602
603 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
604                                    struct hns_roce_v2_cmq_ring *ring)
605 {
606         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
607                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
608                          DMA_BIDIRECTIONAL);
609         kfree(ring->desc);
610 }
611
612 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
613 {
614         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
615         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
616                                             &priv->cmq.csq : &priv->cmq.crq;
617
618         ring->flag = ring_type;
619         ring->next_to_clean = 0;
620         ring->next_to_use = 0;
621
622         return hns_roce_alloc_cmq_desc(hr_dev, ring);
623 }
624
625 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
626 {
627         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
628         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
629                                             &priv->cmq.csq : &priv->cmq.crq;
630         dma_addr_t dma = ring->desc_dma_addr;
631
632         if (ring_type == TYPE_CSQ) {
633                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
634                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
635                            upper_32_bits(dma));
636                 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
637                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
638                            HNS_ROCE_CMQ_ENABLE);
639                 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
640                 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
641         } else {
642                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
643                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
644                            upper_32_bits(dma));
645                 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
646                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
647                            HNS_ROCE_CMQ_ENABLE);
648                 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
649                 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
650         }
651 }
652
653 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
654 {
655         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
656         int ret;
657
658         /* Setup the queue entries for command queue */
659         priv->cmq.csq.desc_num = 1024;
660         priv->cmq.crq.desc_num = 1024;
661
662         /* Setup the lock for command queue */
663         spin_lock_init(&priv->cmq.csq.lock);
664         spin_lock_init(&priv->cmq.crq.lock);
665
666         /* Setup Tx write back timeout */
667         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
668
669         /* Init CSQ */
670         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
671         if (ret) {
672                 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
673                 return ret;
674         }
675
676         /* Init CRQ */
677         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
678         if (ret) {
679                 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
680                 goto err_crq;
681         }
682
683         /* Init CSQ REG */
684         hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
685
686         /* Init CRQ REG */
687         hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
688
689         return 0;
690
691 err_crq:
692         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
693
694         return ret;
695 }
696
697 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
698 {
699         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
700
701         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
702         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
703 }
704
705 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
706                                           enum hns_roce_opcode_type opcode,
707                                           bool is_read)
708 {
709         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
710         desc->opcode = cpu_to_le16(opcode);
711         desc->flag =
712                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
713         if (is_read)
714                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
715         else
716                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
717 }
718
719 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
720 {
721         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
722         u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
723
724         return head == priv->cmq.csq.next_to_use;
725 }
726
727 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
728 {
729         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
730         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
731         struct hns_roce_cmq_desc *desc;
732         u16 ntc = csq->next_to_clean;
733         u32 head;
734         int clean = 0;
735
736         desc = &csq->desc[ntc];
737         head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
738         while (head != ntc) {
739                 memset(desc, 0, sizeof(*desc));
740                 ntc++;
741                 if (ntc == csq->desc_num)
742                         ntc = 0;
743                 desc = &csq->desc[ntc];
744                 clean++;
745         }
746         csq->next_to_clean = ntc;
747
748         return clean;
749 }
750
751 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
752                              struct hns_roce_cmq_desc *desc, int num)
753 {
754         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
755         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
756         struct hns_roce_cmq_desc *desc_to_use;
757         bool complete = false;
758         u32 timeout = 0;
759         int handle = 0;
760         u16 desc_ret;
761         int ret = 0;
762         int ntc;
763
764         spin_lock_bh(&csq->lock);
765
766         if (num > hns_roce_cmq_space(csq)) {
767                 spin_unlock_bh(&csq->lock);
768                 return -EBUSY;
769         }
770
771         /*
772          * Record the location of desc in the cmq for this time
773          * which will be use for hardware to write back
774          */
775         ntc = csq->next_to_use;
776
777         while (handle < num) {
778                 desc_to_use = &csq->desc[csq->next_to_use];
779                 *desc_to_use = desc[handle];
780                 dev_dbg(hr_dev->dev, "set cmq desc:\n");
781                 csq->next_to_use++;
782                 if (csq->next_to_use == csq->desc_num)
783                         csq->next_to_use = 0;
784                 handle++;
785         }
786
787         /* Write to hardware */
788         roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
789
790         /*
791          * If the command is sync, wait for the firmware to write back,
792          * if multi descriptors to be sent, use the first one to check
793          */
794         if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
795                 do {
796                         if (hns_roce_cmq_csq_done(hr_dev))
797                                 break;
798                         udelay(1);
799                         timeout++;
800                 } while (timeout < priv->cmq.tx_timeout);
801         }
802
803         if (hns_roce_cmq_csq_done(hr_dev)) {
804                 complete = true;
805                 handle = 0;
806                 while (handle < num) {
807                         /* get the result of hardware write back */
808                         desc_to_use = &csq->desc[ntc];
809                         desc[handle] = *desc_to_use;
810                         dev_dbg(hr_dev->dev, "Get cmq desc:\n");
811                         desc_ret = desc[handle].retval;
812                         if (desc_ret == CMD_EXEC_SUCCESS)
813                                 ret = 0;
814                         else
815                                 ret = -EIO;
816                         priv->cmq.last_status = desc_ret;
817                         ntc++;
818                         handle++;
819                         if (ntc == csq->desc_num)
820                                 ntc = 0;
821                 }
822         }
823
824         if (!complete)
825                 ret = -EAGAIN;
826
827         /* clean the command send queue */
828         handle = hns_roce_cmq_csq_clean(hr_dev);
829         if (handle != num)
830                 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
831                          handle, num);
832
833         spin_unlock_bh(&csq->lock);
834
835         return ret;
836 }
837
838 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
839 {
840         struct hns_roce_query_version *resp;
841         struct hns_roce_cmq_desc desc;
842         int ret;
843
844         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
845         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
846         if (ret)
847                 return ret;
848
849         resp = (struct hns_roce_query_version *)desc.data;
850         hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
851         hr_dev->vendor_id = le32_to_cpu(resp->rocee_vendor_id);
852
853         return 0;
854 }
855
856 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
857 {
858         struct hns_roce_cfg_global_param *req;
859         struct hns_roce_cmq_desc desc;
860
861         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
862                                       false);
863
864         req = (struct hns_roce_cfg_global_param *)desc.data;
865         memset(req, 0, sizeof(*req));
866         roce_set_field(req->time_cfg_udp_port,
867                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
868                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
869         roce_set_field(req->time_cfg_udp_port,
870                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
871                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
872
873         return hns_roce_cmq_send(hr_dev, &desc, 1);
874 }
875
876 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
877 {
878         struct hns_roce_cmq_desc desc[2];
879         struct hns_roce_pf_res *res;
880         int ret;
881         int i;
882
883         for (i = 0; i < 2; i++) {
884                 hns_roce_cmq_setup_basic_desc(&desc[i],
885                                               HNS_ROCE_OPC_QUERY_PF_RES, true);
886
887                 if (i == 0)
888                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
889                 else
890                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
891         }
892
893         ret = hns_roce_cmq_send(hr_dev, desc, 2);
894         if (ret)
895                 return ret;
896
897         res = (struct hns_roce_pf_res *)desc[0].data;
898
899         hr_dev->caps.qpc_bt_num = roce_get_field(res->qpc_bt_idx_num,
900                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_M,
901                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_S);
902         hr_dev->caps.srqc_bt_num = roce_get_field(res->srqc_bt_idx_num,
903                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
904                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
905         hr_dev->caps.cqc_bt_num = roce_get_field(res->cqc_bt_idx_num,
906                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_M,
907                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_S);
908         hr_dev->caps.mpt_bt_num = roce_get_field(res->mpt_bt_idx_num,
909                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_M,
910                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_S);
911
912         return 0;
913 }
914
915 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
916 {
917         struct hns_roce_cmq_desc desc[2];
918         struct hns_roce_vf_res_a *req_a;
919         struct hns_roce_vf_res_b *req_b;
920         int i;
921
922         req_a = (struct hns_roce_vf_res_a *)desc[0].data;
923         req_b = (struct hns_roce_vf_res_b *)desc[1].data;
924         memset(req_a, 0, sizeof(*req_a));
925         memset(req_b, 0, sizeof(*req_b));
926         for (i = 0; i < 2; i++) {
927                 hns_roce_cmq_setup_basic_desc(&desc[i],
928                                               HNS_ROCE_OPC_ALLOC_VF_RES, false);
929
930                 if (i == 0)
931                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
932                 else
933                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
934
935                 if (i == 0) {
936                         roce_set_field(req_a->vf_qpc_bt_idx_num,
937                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
938                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
939                         roce_set_field(req_a->vf_qpc_bt_idx_num,
940                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
941                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
942                                        HNS_ROCE_VF_QPC_BT_NUM);
943
944                         roce_set_field(req_a->vf_srqc_bt_idx_num,
945                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
946                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
947                         roce_set_field(req_a->vf_srqc_bt_idx_num,
948                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
949                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
950                                        HNS_ROCE_VF_SRQC_BT_NUM);
951
952                         roce_set_field(req_a->vf_cqc_bt_idx_num,
953                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
954                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
955                         roce_set_field(req_a->vf_cqc_bt_idx_num,
956                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
957                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
958                                        HNS_ROCE_VF_CQC_BT_NUM);
959
960                         roce_set_field(req_a->vf_mpt_bt_idx_num,
961                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
962                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
963                         roce_set_field(req_a->vf_mpt_bt_idx_num,
964                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
965                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
966                                        HNS_ROCE_VF_MPT_BT_NUM);
967
968                         roce_set_field(req_a->vf_eqc_bt_idx_num,
969                                        VF_RES_A_DATA_5_VF_EQC_IDX_M,
970                                        VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
971                         roce_set_field(req_a->vf_eqc_bt_idx_num,
972                                        VF_RES_A_DATA_5_VF_EQC_NUM_M,
973                                        VF_RES_A_DATA_5_VF_EQC_NUM_S,
974                                        HNS_ROCE_VF_EQC_NUM);
975                 } else {
976                         roce_set_field(req_b->vf_smac_idx_num,
977                                        VF_RES_B_DATA_1_VF_SMAC_IDX_M,
978                                        VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
979                         roce_set_field(req_b->vf_smac_idx_num,
980                                        VF_RES_B_DATA_1_VF_SMAC_NUM_M,
981                                        VF_RES_B_DATA_1_VF_SMAC_NUM_S,
982                                        HNS_ROCE_VF_SMAC_NUM);
983
984                         roce_set_field(req_b->vf_sgid_idx_num,
985                                        VF_RES_B_DATA_2_VF_SGID_IDX_M,
986                                        VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
987                         roce_set_field(req_b->vf_sgid_idx_num,
988                                        VF_RES_B_DATA_2_VF_SGID_NUM_M,
989                                        VF_RES_B_DATA_2_VF_SGID_NUM_S,
990                                        HNS_ROCE_VF_SGID_NUM);
991
992                         roce_set_field(req_b->vf_qid_idx_sl_num,
993                                        VF_RES_B_DATA_3_VF_QID_IDX_M,
994                                        VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
995                         roce_set_field(req_b->vf_qid_idx_sl_num,
996                                        VF_RES_B_DATA_3_VF_SL_NUM_M,
997                                        VF_RES_B_DATA_3_VF_SL_NUM_S,
998                                        HNS_ROCE_VF_SL_NUM);
999                 }
1000         }
1001
1002         return hns_roce_cmq_send(hr_dev, desc, 2);
1003 }
1004
1005 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1006 {
1007         u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1008         u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1009         u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1010         u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1011         struct hns_roce_cfg_bt_attr *req;
1012         struct hns_roce_cmq_desc desc;
1013
1014         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1015         req = (struct hns_roce_cfg_bt_attr *)desc.data;
1016         memset(req, 0, sizeof(*req));
1017
1018         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1019                        CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1020                        hr_dev->caps.qpc_ba_pg_sz);
1021         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1022                        CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1023                        hr_dev->caps.qpc_buf_pg_sz);
1024         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1025                        CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1026                        qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1027
1028         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1029                        CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1030                        hr_dev->caps.srqc_ba_pg_sz);
1031         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1032                        CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1033                        hr_dev->caps.srqc_buf_pg_sz);
1034         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1035                        CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1036                        srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1037
1038         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1039                        CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1040                        hr_dev->caps.cqc_ba_pg_sz);
1041         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1042                        CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1043                        hr_dev->caps.cqc_buf_pg_sz);
1044         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1045                        CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1046                        cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1047
1048         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1049                        CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1050                        hr_dev->caps.mpt_ba_pg_sz);
1051         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1052                        CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1053                        hr_dev->caps.mpt_buf_pg_sz);
1054         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1055                        CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1056                        mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1057
1058         return hns_roce_cmq_send(hr_dev, &desc, 1);
1059 }
1060
1061 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1062 {
1063         struct hns_roce_caps *caps = &hr_dev->caps;
1064         int ret;
1065
1066         ret = hns_roce_cmq_query_hw_info(hr_dev);
1067         if (ret) {
1068                 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1069                         ret);
1070                 return ret;
1071         }
1072
1073         ret = hns_roce_config_global_param(hr_dev);
1074         if (ret) {
1075                 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1076                         ret);
1077         }
1078
1079         /* Get pf resource owned by every pf */
1080         ret = hns_roce_query_pf_resource(hr_dev);
1081         if (ret) {
1082                 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1083                         ret);
1084                 return ret;
1085         }
1086
1087         ret = hns_roce_alloc_vf_resource(hr_dev);
1088         if (ret) {
1089                 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1090                         ret);
1091                 return ret;
1092         }
1093
1094         hr_dev->vendor_part_id = 0;
1095         hr_dev->sys_image_guid = 0;
1096
1097         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1098         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1099         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1100         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1101         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1102         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1103         caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
1104         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1105         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1106         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1107         caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
1108         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1109         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1110         caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
1111         caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
1112         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1113         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1114         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1115         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1116         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1117         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1118         caps->qpc_entry_sz      = HNS_ROCE_V2_QPC_ENTRY_SZ;
1119         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1120         caps->trrl_entry_sz     = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1121         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1122         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1123         caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
1124         caps->cq_entry_sz       = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1125         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1126         caps->reserved_lkey     = 0;
1127         caps->reserved_pds      = 0;
1128         caps->reserved_mrws     = 1;
1129         caps->reserved_uars     = 0;
1130         caps->reserved_cqs      = 0;
1131
1132         caps->qpc_ba_pg_sz      = 0;
1133         caps->qpc_buf_pg_sz     = 0;
1134         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1135         caps->srqc_ba_pg_sz     = 0;
1136         caps->srqc_buf_pg_sz    = 0;
1137         caps->srqc_hop_num      = HNS_ROCE_HOP_NUM_0;
1138         caps->cqc_ba_pg_sz      = 0;
1139         caps->cqc_buf_pg_sz     = 0;
1140         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1141         caps->mpt_ba_pg_sz      = 0;
1142         caps->mpt_buf_pg_sz     = 0;
1143         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1144         caps->pbl_ba_pg_sz      = 0;
1145         caps->pbl_buf_pg_sz     = 0;
1146         caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
1147         caps->mtt_ba_pg_sz      = 0;
1148         caps->mtt_buf_pg_sz     = 0;
1149         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1150         caps->cqe_ba_pg_sz      = 0;
1151         caps->cqe_buf_pg_sz     = 0;
1152         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1153         caps->eqe_ba_pg_sz      = 0;
1154         caps->eqe_buf_pg_sz     = 0;
1155         caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
1156         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1157
1158         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1159                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1160                                   HNS_ROCE_CAP_FLAG_RQ_INLINE |
1161                                   HNS_ROCE_CAP_FLAG_RECORD_DB;
1162         caps->pkey_table_len[0] = 1;
1163         caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1164         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1165         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1166         caps->local_ca_ack_delay = 0;
1167         caps->max_mtu = IB_MTU_4096;
1168
1169         ret = hns_roce_v2_set_bt(hr_dev);
1170         if (ret)
1171                 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1172                         ret);
1173
1174         return ret;
1175 }
1176
1177 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1178 {
1179         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1180
1181         return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1182 }
1183
1184 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1185 {
1186         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1187
1188         return status & HNS_ROCE_HW_MB_STATUS_MASK;
1189 }
1190
1191 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1192                                  u64 out_param, u32 in_modifier, u8 op_modifier,
1193                                  u16 op, u16 token, int event)
1194 {
1195         struct device *dev = hr_dev->dev;
1196         u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base +
1197                                            ROCEE_VF_MB_CFG0_REG);
1198         unsigned long end;
1199         u32 val0 = 0;
1200         u32 val1 = 0;
1201
1202         end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
1203         while (hns_roce_v2_cmd_pending(hr_dev)) {
1204                 if (time_after(jiffies, end)) {
1205                         dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
1206                                 (int)end);
1207                         return -EAGAIN;
1208                 }
1209                 cond_resched();
1210         }
1211
1212         roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK,
1213                        HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier);
1214         roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK,
1215                        HNS_ROCE_VF_MB4_CMD_SHIFT, op);
1216         roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK,
1217                        HNS_ROCE_VF_MB5_EVENT_SHIFT, event);
1218         roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK,
1219                        HNS_ROCE_VF_MB5_TOKEN_SHIFT, token);
1220
1221         writeq(in_param, hcr + 0);
1222         writeq(out_param, hcr + 2);
1223
1224         /* Memory barrier */
1225         wmb();
1226
1227         writel(val0, hcr + 4);
1228         writel(val1, hcr + 5);
1229
1230         mmiowb();
1231
1232         return 0;
1233 }
1234
1235 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
1236                                 unsigned long timeout)
1237 {
1238         struct device *dev = hr_dev->dev;
1239         unsigned long end = 0;
1240         u32 status;
1241
1242         end = msecs_to_jiffies(timeout) + jiffies;
1243         while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
1244                 cond_resched();
1245
1246         if (hns_roce_v2_cmd_pending(hr_dev)) {
1247                 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1248                 return -ETIMEDOUT;
1249         }
1250
1251         status = hns_roce_v2_cmd_complete(hr_dev);
1252         if (status != 0x1) {
1253                 dev_err(dev, "mailbox status 0x%x!\n", status);
1254                 return -EBUSY;
1255         }
1256
1257         return 0;
1258 }
1259
1260 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
1261                                int gid_index, union ib_gid *gid,
1262                                const struct ib_gid_attr *attr)
1263 {
1264         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
1265         u32 *p;
1266         u32 val;
1267
1268         if (!gid || !attr)
1269                 return -EINVAL;
1270
1271         if (attr->gid_type == IB_GID_TYPE_ROCE)
1272                 sgid_type = GID_TYPE_FLAG_ROCE_V1;
1273
1274         if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
1275                 if (ipv6_addr_v4mapped((void *)gid))
1276                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
1277                 else
1278                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
1279         }
1280
1281         p = (u32 *)&gid->raw[0];
1282         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG0_REG +
1283                        0x20 * gid_index);
1284
1285         p = (u32 *)&gid->raw[4];
1286         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG1_REG +
1287                        0x20 * gid_index);
1288
1289         p = (u32 *)&gid->raw[8];
1290         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG2_REG +
1291                        0x20 * gid_index);
1292
1293         p = (u32 *)&gid->raw[0xc];
1294         roce_raw_write(*p, hr_dev->reg_base + ROCEE_VF_SGID_CFG3_REG +
1295                        0x20 * gid_index);
1296
1297         val = roce_read(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index);
1298         roce_set_field(val, ROCEE_VF_SGID_CFG4_SGID_TYPE_M,
1299                        ROCEE_VF_SGID_CFG4_SGID_TYPE_S, sgid_type);
1300
1301         roce_write(hr_dev, ROCEE_VF_SGID_CFG4_REG + 0x20 * gid_index, val);
1302
1303         return 0;
1304 }
1305
1306 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
1307                                u8 *addr)
1308 {
1309         u16 reg_smac_h;
1310         u32 reg_smac_l;
1311         u32 val;
1312
1313         reg_smac_l = *(u32 *)(&addr[0]);
1314         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_VF_SMAC_CFG0_REG +
1315                        0x08 * phy_port);
1316         val = roce_read(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port);
1317
1318         reg_smac_h  = *(u16 *)(&addr[4]);
1319         roce_set_field(val, ROCEE_VF_SMAC_CFG1_VF_SMAC_H_M,
1320                        ROCEE_VF_SMAC_CFG1_VF_SMAC_H_S, reg_smac_h);
1321         roce_write(hr_dev, ROCEE_VF_SMAC_CFG1_REG + 0x08 * phy_port, val);
1322
1323         return 0;
1324 }
1325
1326 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1327                                   unsigned long mtpt_idx)
1328 {
1329         struct hns_roce_v2_mpt_entry *mpt_entry;
1330         struct scatterlist *sg;
1331         u64 page_addr;
1332         u64 *pages;
1333         int i, j;
1334         int len;
1335         int entry;
1336
1337         mpt_entry = mb_buf;
1338         memset(mpt_entry, 0, sizeof(*mpt_entry));
1339
1340         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
1341                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
1342         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
1343                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
1344                        HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
1345         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1346                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1347                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S, mr->pbl_ba_pg_sz);
1348         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1349                        V2_MPT_BYTE_4_PD_S, mr->pd);
1350         mpt_entry->byte_4_pd_hop_st = cpu_to_le32(mpt_entry->byte_4_pd_hop_st);
1351
1352         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
1353         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1354         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 0);
1355         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
1356                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1357         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S, 0);
1358         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1359                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1360         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1361                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1362         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1363                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1364         mpt_entry->byte_8_mw_cnt_en = cpu_to_le32(mpt_entry->byte_8_mw_cnt_en);
1365
1366         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
1367                      mr->type == MR_TYPE_MR ? 0 : 1);
1368         mpt_entry->byte_12_mw_pa = cpu_to_le32(mpt_entry->byte_12_mw_pa);
1369
1370         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
1371         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
1372         mpt_entry->lkey = cpu_to_le32(mr->key);
1373         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
1374         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
1375
1376         if (mr->type == MR_TYPE_DMA)
1377                 return 0;
1378
1379         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1380
1381         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1382         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
1383                        V2_MPT_BYTE_48_PBL_BA_H_S,
1384                        upper_32_bits(mr->pbl_ba >> 3));
1385         mpt_entry->byte_48_mode_ba = cpu_to_le32(mpt_entry->byte_48_mode_ba);
1386
1387         pages = (u64 *)__get_free_page(GFP_KERNEL);
1388         if (!pages)
1389                 return -ENOMEM;
1390
1391         i = 0;
1392         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1393                 len = sg_dma_len(sg) >> PAGE_SHIFT;
1394                 for (j = 0; j < len; ++j) {
1395                         page_addr = sg_dma_address(sg) +
1396                                     (j << mr->umem->page_shift);
1397                         pages[i] = page_addr >> 6;
1398
1399                         /* Record the first 2 entry directly to MTPT table */
1400                         if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
1401                                 goto found;
1402                         i++;
1403                 }
1404         }
1405
1406 found:
1407         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
1408         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
1409                        V2_MPT_BYTE_56_PA0_H_S,
1410                        upper_32_bits(pages[0]));
1411         mpt_entry->byte_56_pa0_h = cpu_to_le32(mpt_entry->byte_56_pa0_h);
1412
1413         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
1414         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
1415                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
1416
1417         free_page((unsigned long)pages);
1418
1419         roce_set_field(mpt_entry->byte_64_buf_pa1,
1420                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1421                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S, mr->pbl_buf_pg_sz);
1422         mpt_entry->byte_64_buf_pa1 = cpu_to_le32(mpt_entry->byte_64_buf_pa1);
1423
1424         return 0;
1425 }
1426
1427 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
1428                                         struct hns_roce_mr *mr, int flags,
1429                                         u32 pdn, int mr_access_flags, u64 iova,
1430                                         u64 size, void *mb_buf)
1431 {
1432         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
1433
1434         if (flags & IB_MR_REREG_PD) {
1435                 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1436                                V2_MPT_BYTE_4_PD_S, pdn);
1437                 mr->pd = pdn;
1438         }
1439
1440         if (flags & IB_MR_REREG_ACCESS) {
1441                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1442                              V2_MPT_BYTE_8_BIND_EN_S,
1443                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
1444                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1445                            V2_MPT_BYTE_8_ATOMIC_EN_S,
1446                            (mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0));
1447                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1448                              (mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0));
1449                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1450                             (mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1451                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1452                              (mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1453         }
1454
1455         if (flags & IB_MR_REREG_TRANS) {
1456                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
1457                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
1458                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
1459                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
1460
1461                 mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1462                 mpt_entry->pbl_ba_l =
1463                                 cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1464                 roce_set_field(mpt_entry->byte_48_mode_ba,
1465                                V2_MPT_BYTE_48_PBL_BA_H_M,
1466                                V2_MPT_BYTE_48_PBL_BA_H_S,
1467                                upper_32_bits(mr->pbl_ba >> 3));
1468                 mpt_entry->byte_48_mode_ba =
1469                                 cpu_to_le32(mpt_entry->byte_48_mode_ba);
1470
1471                 mr->iova = iova;
1472                 mr->size = size;
1473         }
1474
1475         return 0;
1476 }
1477
1478 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1479 {
1480         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1481                                    n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
1482 }
1483
1484 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1485 {
1486         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
1487
1488         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1489         return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
1490                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
1491 }
1492
1493 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
1494 {
1495         return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
1496 }
1497
1498 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1499 {
1500         *hr_cq->set_ci_db = cons_index & 0xffffff;
1501 }
1502
1503 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1504                                    struct hns_roce_srq *srq)
1505 {
1506         struct hns_roce_v2_cqe *cqe, *dest;
1507         u32 prod_index;
1508         int nfreed = 0;
1509         u8 owner_bit;
1510
1511         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
1512              ++prod_index) {
1513                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1514                         break;
1515         }
1516
1517         /*
1518          * Now backwards through the CQ, removing CQ entries
1519          * that match our QP by overwriting them with next entries.
1520          */
1521         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1522                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1523                 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1524                                     V2_CQE_BYTE_16_LCL_QPN_S) &
1525                                     HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
1526                         /* In v1 engine, not support SRQ */
1527                         ++nfreed;
1528                 } else if (nfreed) {
1529                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
1530                                           hr_cq->ib_cq.cqe);
1531                         owner_bit = roce_get_bit(dest->byte_4,
1532                                                  V2_CQE_BYTE_4_OWNER_S);
1533                         memcpy(dest, cqe, sizeof(*cqe));
1534                         roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
1535                                      owner_bit);
1536                 }
1537         }
1538
1539         if (nfreed) {
1540                 hr_cq->cons_index += nfreed;
1541                 /*
1542                  * Make sure update of buffer contents is done before
1543                  * updating consumer index.
1544                  */
1545                 wmb();
1546                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
1547         }
1548 }
1549
1550 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1551                                  struct hns_roce_srq *srq)
1552 {
1553         spin_lock_irq(&hr_cq->lock);
1554         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
1555         spin_unlock_irq(&hr_cq->lock);
1556 }
1557
1558 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
1559                                   struct hns_roce_cq *hr_cq, void *mb_buf,
1560                                   u64 *mtts, dma_addr_t dma_handle, int nent,
1561                                   u32 vector)
1562 {
1563         struct hns_roce_v2_cq_context *cq_context;
1564
1565         cq_context = mb_buf;
1566         memset(cq_context, 0, sizeof(*cq_context));
1567
1568         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
1569                        V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
1570         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
1571                        V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
1572         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
1573                        V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
1574         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
1575                        V2_CQC_BYTE_4_CEQN_S, vector);
1576         cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
1577
1578         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
1579                        V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
1580
1581         cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
1582         cq_context->cqe_cur_blk_addr =
1583                                 cpu_to_le32(cq_context->cqe_cur_blk_addr);
1584
1585         roce_set_field(cq_context->byte_16_hop_addr,
1586                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
1587                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
1588                        cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
1589         roce_set_field(cq_context->byte_16_hop_addr,
1590                        V2_CQC_BYTE_16_CQE_HOP_NUM_M,
1591                        V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
1592                        HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
1593
1594         cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
1595         roce_set_field(cq_context->byte_24_pgsz_addr,
1596                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
1597                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
1598                        cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
1599         roce_set_field(cq_context->byte_24_pgsz_addr,
1600                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
1601                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
1602                        hr_dev->caps.cqe_ba_pg_sz);
1603         roce_set_field(cq_context->byte_24_pgsz_addr,
1604                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
1605                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
1606                        hr_dev->caps.cqe_buf_pg_sz);
1607
1608         cq_context->cqe_ba = (u32)(dma_handle >> 3);
1609
1610         roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
1611                        V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
1612
1613         if (hr_cq->db_en)
1614                 roce_set_bit(cq_context->byte_44_db_record,
1615                              V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
1616
1617         roce_set_field(cq_context->byte_44_db_record,
1618                        V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
1619                        V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
1620                        ((u32)hr_cq->db.dma) >> 1);
1621         cq_context->db_record_addr = hr_cq->db.dma >> 32;
1622
1623         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1624                        V2_CQC_BYTE_56_CQ_MAX_CNT_M,
1625                        V2_CQC_BYTE_56_CQ_MAX_CNT_S,
1626                        HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
1627         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
1628                        V2_CQC_BYTE_56_CQ_PERIOD_M,
1629                        V2_CQC_BYTE_56_CQ_PERIOD_S,
1630                        HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
1631 }
1632
1633 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
1634                                      enum ib_cq_notify_flags flags)
1635 {
1636         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1637         u32 notification_flag;
1638         u32 doorbell[2];
1639
1640         doorbell[0] = 0;
1641         doorbell[1] = 0;
1642
1643         notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
1644                              V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
1645         /*
1646          * flags = 0; Notification Flag = 1, next
1647          * flags = 1; Notification Flag = 0, solocited
1648          */
1649         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
1650                        hr_cq->cqn);
1651         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
1652                        HNS_ROCE_V2_CQ_DB_NTR);
1653         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
1654                        V2_CQ_DB_PARAMETER_CONS_IDX_S,
1655                        hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
1656         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
1657                        V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
1658         roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
1659                      notification_flag);
1660
1661         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1662
1663         return 0;
1664 }
1665
1666 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
1667                                                     struct hns_roce_qp **cur_qp,
1668                                                     struct ib_wc *wc)
1669 {
1670         struct hns_roce_rinl_sge *sge_list;
1671         u32 wr_num, wr_cnt, sge_num;
1672         u32 sge_cnt, data_len, size;
1673         void *wqe_buf;
1674
1675         wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
1676                                 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
1677         wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
1678
1679         sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
1680         sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
1681         wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
1682         data_len = wc->byte_len;
1683
1684         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
1685                 size = min(sge_list[sge_cnt].len, data_len);
1686                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
1687
1688                 data_len -= size;
1689                 wqe_buf += size;
1690         }
1691
1692         if (data_len) {
1693                 wc->status = IB_WC_LOC_LEN_ERR;
1694                 return -EAGAIN;
1695         }
1696
1697         return 0;
1698 }
1699
1700 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
1701                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
1702 {
1703         struct hns_roce_dev *hr_dev;
1704         struct hns_roce_v2_cqe *cqe;
1705         struct hns_roce_qp *hr_qp;
1706         struct hns_roce_wq *wq;
1707         int is_send;
1708         u16 wqe_ctr;
1709         u32 opcode;
1710         u32 status;
1711         int qpn;
1712         int ret;
1713
1714         /* Find cqe according to consumer index */
1715         cqe = next_cqe_sw_v2(hr_cq);
1716         if (!cqe)
1717                 return -EAGAIN;
1718
1719         ++hr_cq->cons_index;
1720         /* Memory barrier */
1721         rmb();
1722
1723         /* 0->SQ, 1->RQ */
1724         is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
1725
1726         qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
1727                                 V2_CQE_BYTE_16_LCL_QPN_S);
1728
1729         if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
1730                 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
1731                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
1732                 if (unlikely(!hr_qp)) {
1733                         dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
1734                                 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
1735                         return -EINVAL;
1736                 }
1737                 *cur_qp = hr_qp;
1738         }
1739
1740         wc->qp = &(*cur_qp)->ibqp;
1741         wc->vendor_err = 0;
1742
1743         status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
1744                                 V2_CQE_BYTE_4_STATUS_S);
1745         switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
1746         case HNS_ROCE_CQE_V2_SUCCESS:
1747                 wc->status = IB_WC_SUCCESS;
1748                 break;
1749         case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
1750                 wc->status = IB_WC_LOC_LEN_ERR;
1751                 break;
1752         case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
1753                 wc->status = IB_WC_LOC_QP_OP_ERR;
1754                 break;
1755         case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
1756                 wc->status = IB_WC_LOC_PROT_ERR;
1757                 break;
1758         case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
1759                 wc->status = IB_WC_WR_FLUSH_ERR;
1760                 break;
1761         case HNS_ROCE_CQE_V2_MW_BIND_ERR:
1762                 wc->status = IB_WC_MW_BIND_ERR;
1763                 break;
1764         case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
1765                 wc->status = IB_WC_BAD_RESP_ERR;
1766                 break;
1767         case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
1768                 wc->status = IB_WC_LOC_ACCESS_ERR;
1769                 break;
1770         case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
1771                 wc->status = IB_WC_REM_INV_REQ_ERR;
1772                 break;
1773         case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
1774                 wc->status = IB_WC_REM_ACCESS_ERR;
1775                 break;
1776         case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
1777                 wc->status = IB_WC_REM_OP_ERR;
1778                 break;
1779         case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
1780                 wc->status = IB_WC_RETRY_EXC_ERR;
1781                 break;
1782         case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
1783                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
1784                 break;
1785         case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
1786                 wc->status = IB_WC_REM_ABORT_ERR;
1787                 break;
1788         default:
1789                 wc->status = IB_WC_GENERAL_ERR;
1790                 break;
1791         }
1792
1793         /* CQE status error, directly return */
1794         if (wc->status != IB_WC_SUCCESS)
1795                 return 0;
1796
1797         if (is_send) {
1798                 wc->wc_flags = 0;
1799                 /* SQ corresponding to CQE */
1800                 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
1801                                        V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
1802                 case HNS_ROCE_SQ_OPCODE_SEND:
1803                         wc->opcode = IB_WC_SEND;
1804                         break;
1805                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
1806                         wc->opcode = IB_WC_SEND;
1807                         break;
1808                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
1809                         wc->opcode = IB_WC_SEND;
1810                         wc->wc_flags |= IB_WC_WITH_IMM;
1811                         break;
1812                 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
1813                         wc->opcode = IB_WC_RDMA_READ;
1814                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1815                         break;
1816                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
1817                         wc->opcode = IB_WC_RDMA_WRITE;
1818                         break;
1819                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
1820                         wc->opcode = IB_WC_RDMA_WRITE;
1821                         wc->wc_flags |= IB_WC_WITH_IMM;
1822                         break;
1823                 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
1824                         wc->opcode = IB_WC_LOCAL_INV;
1825                         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
1826                         break;
1827                 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
1828                         wc->opcode = IB_WC_COMP_SWAP;
1829                         wc->byte_len  = 8;
1830                         break;
1831                 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
1832                         wc->opcode = IB_WC_FETCH_ADD;
1833                         wc->byte_len  = 8;
1834                         break;
1835                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
1836                         wc->opcode = IB_WC_MASKED_COMP_SWAP;
1837                         wc->byte_len  = 8;
1838                         break;
1839                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
1840                         wc->opcode = IB_WC_MASKED_FETCH_ADD;
1841                         wc->byte_len  = 8;
1842                         break;
1843                 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
1844                         wc->opcode = IB_WC_REG_MR;
1845                         break;
1846                 case HNS_ROCE_SQ_OPCODE_BIND_MW:
1847                         wc->opcode = IB_WC_REG_MR;
1848                         break;
1849                 default:
1850                         wc->status = IB_WC_GENERAL_ERR;
1851                         break;
1852                 }
1853
1854                 wq = &(*cur_qp)->sq;
1855                 if ((*cur_qp)->sq_signal_bits) {
1856                         /*
1857                          * If sg_signal_bit is 1,
1858                          * firstly tail pointer updated to wqe
1859                          * which current cqe correspond to
1860                          */
1861                         wqe_ctr = (u16)roce_get_field(cqe->byte_4,
1862                                                       V2_CQE_BYTE_4_WQE_INDX_M,
1863                                                       V2_CQE_BYTE_4_WQE_INDX_S);
1864                         wq->tail += (wqe_ctr - (u16)wq->tail) &
1865                                     (wq->wqe_cnt - 1);
1866                 }
1867
1868                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1869                 ++wq->tail;
1870         } else {
1871                 /* RQ correspond to CQE */
1872                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1873
1874                 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
1875                                         V2_CQE_BYTE_4_OPCODE_S);
1876                 switch (opcode & 0x1f) {
1877                 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
1878                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
1879                         wc->wc_flags = IB_WC_WITH_IMM;
1880                         wc->ex.imm_data = cqe->immtdata;
1881                         break;
1882                 case HNS_ROCE_V2_OPCODE_SEND:
1883                         wc->opcode = IB_WC_RECV;
1884                         wc->wc_flags = 0;
1885                         break;
1886                 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
1887                         wc->opcode = IB_WC_RECV;
1888                         wc->wc_flags = IB_WC_WITH_IMM;
1889                         wc->ex.imm_data = cqe->immtdata;
1890                         break;
1891                 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
1892                         wc->opcode = IB_WC_RECV;
1893                         wc->wc_flags = IB_WC_WITH_INVALIDATE;
1894                         wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
1895                         break;
1896                 default:
1897                         wc->status = IB_WC_GENERAL_ERR;
1898                         break;
1899                 }
1900
1901                 if ((wc->qp->qp_type == IB_QPT_RC ||
1902                      wc->qp->qp_type == IB_QPT_UC) &&
1903                     (opcode == HNS_ROCE_V2_OPCODE_SEND ||
1904                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
1905                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
1906                     (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
1907                         ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
1908                         if (ret)
1909                                 return -EAGAIN;
1910                 }
1911
1912                 /* Update tail pointer, record wr_id */
1913                 wq = &(*cur_qp)->rq;
1914                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1915                 ++wq->tail;
1916
1917                 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
1918                                             V2_CQE_BYTE_32_SL_S);
1919                 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
1920                                                 V2_CQE_BYTE_32_RMT_QPN_M,
1921                                                 V2_CQE_BYTE_32_RMT_QPN_S);
1922                 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
1923                                               V2_CQE_BYTE_32_GRH_S) ?
1924                                               IB_WC_GRH : 0);
1925                 wc->port_num = roce_get_field(cqe->byte_32,
1926                                 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
1927                 wc->pkey_index = 0;
1928                 memcpy(wc->smac, cqe->smac, 4);
1929                 wc->smac[4] = roce_get_field(cqe->byte_28,
1930                                              V2_CQE_BYTE_28_SMAC_4_M,
1931                                              V2_CQE_BYTE_28_SMAC_4_S);
1932                 wc->smac[5] = roce_get_field(cqe->byte_28,
1933                                              V2_CQE_BYTE_28_SMAC_5_M,
1934                                              V2_CQE_BYTE_28_SMAC_5_S);
1935                 wc->vlan_id = 0xffff;
1936                 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
1937                 wc->network_hdr_type = roce_get_field(cqe->byte_28,
1938                                                     V2_CQE_BYTE_28_PORT_TYPE_M,
1939                                                     V2_CQE_BYTE_28_PORT_TYPE_S);
1940         }
1941
1942         return 0;
1943 }
1944
1945 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
1946                                struct ib_wc *wc)
1947 {
1948         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1949         struct hns_roce_qp *cur_qp = NULL;
1950         unsigned long flags;
1951         int npolled;
1952
1953         spin_lock_irqsave(&hr_cq->lock, flags);
1954
1955         for (npolled = 0; npolled < num_entries; ++npolled) {
1956                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
1957                         break;
1958         }
1959
1960         if (npolled) {
1961                 /* Memory barrier */
1962                 wmb();
1963                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
1964         }
1965
1966         spin_unlock_irqrestore(&hr_cq->lock, flags);
1967
1968         return npolled;
1969 }
1970
1971 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
1972                                struct hns_roce_hem_table *table, int obj,
1973                                int step_idx)
1974 {
1975         struct device *dev = hr_dev->dev;
1976         struct hns_roce_cmd_mailbox *mailbox;
1977         struct hns_roce_hem_iter iter;
1978         struct hns_roce_hem_mhop mhop;
1979         struct hns_roce_hem *hem;
1980         unsigned long mhop_obj = obj;
1981         int i, j, k;
1982         int ret = 0;
1983         u64 hem_idx = 0;
1984         u64 l1_idx = 0;
1985         u64 bt_ba = 0;
1986         u32 chunk_ba_num;
1987         u32 hop_num;
1988         u16 op = 0xff;
1989
1990         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
1991                 return 0;
1992
1993         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
1994         i = mhop.l0_idx;
1995         j = mhop.l1_idx;
1996         k = mhop.l2_idx;
1997         hop_num = mhop.hop_num;
1998         chunk_ba_num = mhop.bt_chunk_size / 8;
1999
2000         if (hop_num == 2) {
2001                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
2002                           k;
2003                 l1_idx = i * chunk_ba_num + j;
2004         } else if (hop_num == 1) {
2005                 hem_idx = i * chunk_ba_num + j;
2006         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
2007                 hem_idx = i;
2008         }
2009
2010         switch (table->type) {
2011         case HEM_TYPE_QPC:
2012                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2013                 break;
2014         case HEM_TYPE_MTPT:
2015                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2016                 break;
2017         case HEM_TYPE_CQC:
2018                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2019                 break;
2020         case HEM_TYPE_SRQC:
2021                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2022                 break;
2023         default:
2024                 dev_warn(dev, "Table %d not to be written by mailbox!\n",
2025                          table->type);
2026                 return 0;
2027         }
2028         op += step_idx;
2029
2030         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2031         if (IS_ERR(mailbox))
2032                 return PTR_ERR(mailbox);
2033
2034         if (check_whether_last_step(hop_num, step_idx)) {
2035                 hem = table->hem[hem_idx];
2036                 for (hns_roce_hem_first(hem, &iter);
2037                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2038                         bt_ba = hns_roce_hem_addr(&iter);
2039
2040                         /* configure the ba, tag, and op */
2041                         ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2042                                                 obj, 0, op,
2043                                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2044                 }
2045         } else {
2046                 if (step_idx == 0)
2047                         bt_ba = table->bt_l0_dma_addr[i];
2048                 else if (step_idx == 1 && hop_num == 2)
2049                         bt_ba = table->bt_l1_dma_addr[l1_idx];
2050
2051                 /* configure the ba, tag, and op */
2052                 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
2053                                         0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
2054         }
2055
2056         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2057         return ret;
2058 }
2059
2060 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
2061                                  struct hns_roce_hem_table *table, int obj,
2062                                  int step_idx)
2063 {
2064         struct device *dev = hr_dev->dev;
2065         struct hns_roce_cmd_mailbox *mailbox;
2066         int ret = 0;
2067         u16 op = 0xff;
2068
2069         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2070                 return 0;
2071
2072         switch (table->type) {
2073         case HEM_TYPE_QPC:
2074                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
2075                 break;
2076         case HEM_TYPE_MTPT:
2077                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
2078                 break;
2079         case HEM_TYPE_CQC:
2080                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
2081                 break;
2082         case HEM_TYPE_SRQC:
2083                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
2084                 break;
2085         default:
2086                 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
2087                          table->type);
2088                 return 0;
2089         }
2090         op += step_idx;
2091
2092         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2093         if (IS_ERR(mailbox))
2094                 return PTR_ERR(mailbox);
2095
2096         /* configure the tag and op */
2097         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
2098                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2099
2100         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2101         return ret;
2102 }
2103
2104 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
2105                                  struct hns_roce_mtt *mtt,
2106                                  enum ib_qp_state cur_state,
2107                                  enum ib_qp_state new_state,
2108                                  struct hns_roce_v2_qp_context *context,
2109                                  struct hns_roce_qp *hr_qp)
2110 {
2111         struct hns_roce_cmd_mailbox *mailbox;
2112         int ret;
2113
2114         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2115         if (IS_ERR(mailbox))
2116                 return PTR_ERR(mailbox);
2117
2118         memcpy(mailbox->buf, context, sizeof(*context) * 2);
2119
2120         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2121                                 HNS_ROCE_CMD_MODIFY_QPC,
2122                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2123
2124         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2125
2126         return ret;
2127 }
2128
2129 static void set_access_flags(struct hns_roce_qp *hr_qp,
2130                              struct hns_roce_v2_qp_context *context,
2131                              struct hns_roce_v2_qp_context *qpc_mask,
2132                              const struct ib_qp_attr *attr, int attr_mask)
2133 {
2134         u8 dest_rd_atomic;
2135         u32 access_flags;
2136
2137         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
2138                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
2139
2140         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
2141                        attr->qp_access_flags : hr_qp->atomic_rd_en;
2142
2143         if (!dest_rd_atomic)
2144                 access_flags &= IB_ACCESS_REMOTE_WRITE;
2145
2146         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2147                      !!(access_flags & IB_ACCESS_REMOTE_READ));
2148         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
2149
2150         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2151                      !!(access_flags & IB_ACCESS_REMOTE_WRITE));
2152         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
2153
2154         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2155                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
2156         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
2157 }
2158
2159 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
2160                                     const struct ib_qp_attr *attr,
2161                                     int attr_mask,
2162                                     struct hns_roce_v2_qp_context *context,
2163                                     struct hns_roce_v2_qp_context *qpc_mask)
2164 {
2165         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2166
2167         /*
2168          * In v2 engine, software pass context and context mask to hardware
2169          * when modifying qp. If software need modify some fields in context,
2170          * we should set all bits of the relevant fields in context mask to
2171          * 0 at the same time, else set them to 0x1.
2172          */
2173         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2174                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2175         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2176                        V2_QPC_BYTE_4_TST_S, 0);
2177
2178         if (ibqp->qp_type == IB_QPT_GSI)
2179                 roce_set_field(context->byte_4_sqpn_tst,
2180                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2181                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2182                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
2183         else
2184                 roce_set_field(context->byte_4_sqpn_tst,
2185                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2186                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2187                                hr_qp->sq.max_gs > 2 ?
2188                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2189
2190         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2191                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2192
2193         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2194                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2195         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2196                        V2_QPC_BYTE_4_SQPN_S, 0);
2197
2198         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2199                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2200         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2201                        V2_QPC_BYTE_16_PD_S, 0);
2202
2203         roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2204                        V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
2205         roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2206                        V2_QPC_BYTE_20_RQWS_S, 0);
2207
2208         roce_set_field(context->byte_20_smac_sgid_idx,
2209                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2210                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2211         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2212                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2213
2214         roce_set_field(context->byte_20_smac_sgid_idx,
2215                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2216                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2217         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2218                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2219
2220         /* No VLAN need to set 0xFFF */
2221         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M,
2222                        V2_QPC_BYTE_24_VLAN_IDX_S, 0xfff);
2223         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_IDX_M,
2224                        V2_QPC_BYTE_24_VLAN_IDX_S, 0);
2225
2226         /*
2227          * Set some fields in context to zero, Because the default values
2228          * of all fields in context are zero, we need not set them to 0 again.
2229          * but we should set the relevant fields of context mask to 0.
2230          */
2231         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
2232         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
2233         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
2234         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
2235
2236         roce_set_field(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_MAPID_M,
2237                        V2_QPC_BYTE_60_MAPID_S, 0);
2238
2239         roce_set_bit(qpc_mask->byte_60_qpst_mapid,
2240                      V2_QPC_BYTE_60_INNER_MAP_IND_S, 0);
2241         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_MAP_IND_S,
2242                      0);
2243         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_RQ_MAP_IND_S,
2244                      0);
2245         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_EXT_MAP_IND_S,
2246                      0);
2247         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_RLS_IND_S,
2248                      0);
2249         roce_set_bit(qpc_mask->byte_60_qpst_mapid, V2_QPC_BYTE_60_SQ_EXT_IND_S,
2250                      0);
2251         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
2252         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
2253
2254         if (attr_mask & IB_QP_QKEY) {
2255                 context->qkey_xrcd = attr->qkey;
2256                 qpc_mask->qkey_xrcd = 0;
2257                 hr_qp->qkey = attr->qkey;
2258         }
2259
2260         if (hr_qp->rdb_en) {
2261                 roce_set_bit(context->byte_68_rq_db,
2262                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
2263                 roce_set_bit(qpc_mask->byte_68_rq_db,
2264                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
2265         }
2266
2267         roce_set_field(context->byte_68_rq_db,
2268                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2269                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
2270                        ((u32)hr_qp->rdb.dma) >> 1);
2271         roce_set_field(qpc_mask->byte_68_rq_db,
2272                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2273                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
2274         context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
2275         qpc_mask->rq_db_record_addr = 0;
2276
2277         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 1);
2278         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
2279
2280         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2281                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2282         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2283                        V2_QPC_BYTE_80_RX_CQN_S, 0);
2284         if (ibqp->srq) {
2285                 roce_set_field(context->byte_76_srqn_op_en,
2286                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2287                                to_hr_srq(ibqp->srq)->srqn);
2288                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
2289                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2290                 roce_set_bit(context->byte_76_srqn_op_en,
2291                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
2292                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2293                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
2294         }
2295
2296         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2297                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2298                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2299         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2300                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2301                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2302
2303         roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
2304                        V2_QPC_BYTE_92_SRQ_INFO_S, 0);
2305
2306         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2307                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2308
2309         roce_set_field(qpc_mask->byte_104_rq_sge,
2310                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
2311                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
2312
2313         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2314                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2315         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2316                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2317                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2318         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2319                      V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
2320
2321         qpc_mask->rq_rnr_timer = 0;
2322         qpc_mask->rx_msg_len = 0;
2323         qpc_mask->rx_rkey_pkt_info = 0;
2324         qpc_mask->rx_va = 0;
2325
2326         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2327                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2328         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2329                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2330
2331         roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RSVD_RAQ_MAP_S, 0);
2332         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
2333                        V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
2334         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
2335                        V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
2336
2337         roce_set_field(qpc_mask->byte_144_raq,
2338                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
2339                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
2340         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_RTY_INI_IND_S,
2341                      0);
2342         roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
2343                        V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
2344         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
2345
2346         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
2347                        V2_QPC_BYTE_148_RQ_MSN_S, 0);
2348         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
2349                        V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
2350
2351         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2352                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2353         roce_set_field(qpc_mask->byte_152_raq,
2354                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
2355                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
2356
2357         roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
2358                        V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
2359
2360         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2361                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
2362                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
2363         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2364                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
2365                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
2366
2367         roce_set_field(context->byte_168_irrl_idx,
2368                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2369                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2370                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2371         roce_set_field(qpc_mask->byte_168_irrl_idx,
2372                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2373                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2374
2375         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2376                      V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
2377         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2378                      V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
2379         roce_set_field(qpc_mask->byte_168_irrl_idx,
2380                        V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
2381                        V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
2382
2383         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2384                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
2385         roce_set_field(qpc_mask->byte_172_sq_psn,
2386                        V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2387                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
2388
2389         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
2390                      0);
2391
2392         roce_set_field(qpc_mask->byte_176_msg_pktn,
2393                        V2_QPC_BYTE_176_MSG_USE_PKTN_M,
2394                        V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
2395         roce_set_field(qpc_mask->byte_176_msg_pktn,
2396                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
2397                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
2398
2399         roce_set_field(qpc_mask->byte_184_irrl_idx,
2400                        V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
2401                        V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
2402
2403         qpc_mask->cur_sge_offset = 0;
2404
2405         roce_set_field(qpc_mask->byte_192_ext_sge,
2406                        V2_QPC_BYTE_192_CUR_SGE_IDX_M,
2407                        V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
2408         roce_set_field(qpc_mask->byte_192_ext_sge,
2409                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
2410                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
2411
2412         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
2413                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
2414
2415         roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
2416                        V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
2417         roce_set_field(qpc_mask->byte_200_sq_max,
2418                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
2419                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
2420
2421         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
2422         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
2423
2424         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
2425                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
2426
2427         qpc_mask->sq_timer = 0;
2428
2429         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
2430                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
2431                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
2432         roce_set_field(qpc_mask->byte_232_irrl_sge,
2433                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
2434                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
2435
2436         qpc_mask->irrl_cur_sge_offset = 0;
2437
2438         roce_set_field(qpc_mask->byte_240_irrl_tail,
2439                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
2440                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
2441         roce_set_field(qpc_mask->byte_240_irrl_tail,
2442                        V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
2443                        V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
2444         roce_set_field(qpc_mask->byte_240_irrl_tail,
2445                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
2446                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
2447
2448         roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
2449                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
2450         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
2451                      0);
2452         roce_set_field(qpc_mask->byte_248_ack_psn,
2453                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
2454                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
2455         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
2456                      0);
2457         roce_set_bit(qpc_mask->byte_248_ack_psn,
2458                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
2459         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
2460                      0);
2461
2462         hr_qp->access_flags = attr->qp_access_flags;
2463         hr_qp->pkey_index = attr->pkey_index;
2464         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2465                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2466         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2467                        V2_QPC_BYTE_252_TX_CQN_S, 0);
2468
2469         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
2470                        V2_QPC_BYTE_252_ERR_TYPE_S, 0);
2471
2472         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2473                        V2_QPC_BYTE_256_RQ_CQE_IDX_M,
2474                        V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
2475         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
2476                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
2477                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
2478 }
2479
2480 static void modify_qp_init_to_init(struct ib_qp *ibqp,
2481                                    const struct ib_qp_attr *attr, int attr_mask,
2482                                    struct hns_roce_v2_qp_context *context,
2483                                    struct hns_roce_v2_qp_context *qpc_mask)
2484 {
2485         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2486
2487         /*
2488          * In v2 engine, software pass context and context mask to hardware
2489          * when modifying qp. If software need modify some fields in context,
2490          * we should set all bits of the relevant fields in context mask to
2491          * 0 at the same time, else set them to 0x1.
2492          */
2493         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2494                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2495         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2496                        V2_QPC_BYTE_4_TST_S, 0);
2497
2498         if (ibqp->qp_type == IB_QPT_GSI)
2499                 roce_set_field(context->byte_4_sqpn_tst,
2500                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2501                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2502                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
2503         else
2504                 roce_set_field(context->byte_4_sqpn_tst,
2505                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2506                                V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
2507                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2508
2509         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2510                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2511
2512         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2513                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2514                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2515                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2516                              0);
2517
2518                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2519                              !!(attr->qp_access_flags &
2520                              IB_ACCESS_REMOTE_WRITE));
2521                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2522                              0);
2523
2524                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2525                              !!(attr->qp_access_flags &
2526                              IB_ACCESS_REMOTE_ATOMIC));
2527                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2528                              0);
2529         } else {
2530                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2531                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
2532                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2533                              0);
2534
2535                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2536                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
2537                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2538                              0);
2539
2540                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2541                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
2542                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2543                              0);
2544         }
2545
2546         roce_set_field(context->byte_20_smac_sgid_idx,
2547                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2548                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2549         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2550                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2551
2552         roce_set_field(context->byte_20_smac_sgid_idx,
2553                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2554                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2555         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2556                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2557
2558         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2559                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2560         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2561                        V2_QPC_BYTE_16_PD_S, 0);
2562
2563         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2564                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2565         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2566                        V2_QPC_BYTE_80_RX_CQN_S, 0);
2567
2568         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2569                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
2570         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
2571                        V2_QPC_BYTE_252_TX_CQN_S, 0);
2572
2573         if (ibqp->srq) {
2574                 roce_set_bit(context->byte_76_srqn_op_en,
2575                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
2576                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2577                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
2578                 roce_set_field(context->byte_76_srqn_op_en,
2579                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2580                                to_hr_srq(ibqp->srq)->srqn);
2581                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
2582                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2583         }
2584
2585         if (attr_mask & IB_QP_QKEY) {
2586                 context->qkey_xrcd = attr->qkey;
2587                 qpc_mask->qkey_xrcd = 0;
2588         }
2589
2590         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2591                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2592         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2593                        V2_QPC_BYTE_4_SQPN_S, 0);
2594
2595         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2596                        V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
2597         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2598                        V2_QPC_BYTE_56_DQPN_S, 0);
2599         roce_set_field(context->byte_168_irrl_idx,
2600                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2601                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S,
2602                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2603         roce_set_field(qpc_mask->byte_168_irrl_idx,
2604                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_M,
2605                        V2_QPC_BYTE_168_SQ_SHIFT_BAK_S, 0);
2606 }
2607
2608 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
2609                                  const struct ib_qp_attr *attr, int attr_mask,
2610                                  struct hns_roce_v2_qp_context *context,
2611                                  struct hns_roce_v2_qp_context *qpc_mask)
2612 {
2613         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2614         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2615         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2616         struct device *dev = hr_dev->dev;
2617         dma_addr_t dma_handle_3;
2618         dma_addr_t dma_handle_2;
2619         dma_addr_t dma_handle;
2620         u32 page_size;
2621         u8 port_num;
2622         u64 *mtts_3;
2623         u64 *mtts_2;
2624         u64 *mtts;
2625         u8 *dmac;
2626         u8 *smac;
2627         int port;
2628
2629         /* Search qp buf's mtts */
2630         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2631                                    hr_qp->mtt.first_seg, &dma_handle);
2632         if (!mtts) {
2633                 dev_err(dev, "qp buf pa find failed\n");
2634                 return -EINVAL;
2635         }
2636
2637         /* Search IRRL's mtts */
2638         mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
2639                                      hr_qp->qpn, &dma_handle_2);
2640         if (!mtts_2) {
2641                 dev_err(dev, "qp irrl_table find failed\n");
2642                 return -EINVAL;
2643         }
2644
2645         /* Search TRRL's mtts */
2646         mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
2647                                      hr_qp->qpn, &dma_handle_3);
2648         if (!mtts_3) {
2649                 dev_err(dev, "qp trrl_table find failed\n");
2650                 return -EINVAL;
2651         }
2652
2653         if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_ACCESS_FLAGS) ||
2654             (attr_mask & IB_QP_PKEY_INDEX) || (attr_mask & IB_QP_QKEY)) {
2655                 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
2656                 return -EINVAL;
2657         }
2658
2659         dmac = (u8 *)attr->ah_attr.roce.dmac;
2660         context->wqe_sge_ba = (u32)(dma_handle >> 3);
2661         qpc_mask->wqe_sge_ba = 0;
2662
2663         /*
2664          * In v2 engine, software pass context and context mask to hardware
2665          * when modifying qp. If software need modify some fields in context,
2666          * we should set all bits of the relevant fields in context mask to
2667          * 0 at the same time, else set them to 0x1.
2668          */
2669         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
2670                        V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
2671         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
2672                        V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
2673
2674         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
2675                        V2_QPC_BYTE_12_SQ_HOP_NUM_S,
2676                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
2677                        0 : hr_dev->caps.mtt_hop_num);
2678         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
2679                        V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
2680
2681         roce_set_field(context->byte_20_smac_sgid_idx,
2682                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
2683                        V2_QPC_BYTE_20_SGE_HOP_NUM_S,
2684                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
2685                        hr_dev->caps.mtt_hop_num : 0);
2686         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2687                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
2688                        V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
2689
2690         roce_set_field(context->byte_20_smac_sgid_idx,
2691                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
2692                        V2_QPC_BYTE_20_RQ_HOP_NUM_S,
2693                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
2694                        0 : hr_dev->caps.mtt_hop_num);
2695         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2696                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
2697                        V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
2698
2699         roce_set_field(context->byte_16_buf_ba_pg_sz,
2700                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
2701                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
2702                        hr_dev->caps.mtt_ba_pg_sz);
2703         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
2704                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
2705                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
2706
2707         roce_set_field(context->byte_16_buf_ba_pg_sz,
2708                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
2709                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
2710                        hr_dev->caps.mtt_buf_pg_sz);
2711         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
2712                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
2713                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
2714
2715         roce_set_field(context->byte_80_rnr_rx_cqn,
2716                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
2717                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
2718         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
2719                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
2720                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
2721
2722         page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
2723         context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
2724                                     >> PAGE_ADDR_SHIFT);
2725         qpc_mask->rq_cur_blk_addr = 0;
2726
2727         roce_set_field(context->byte_92_srq_info,
2728                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
2729                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
2730                        mtts[hr_qp->rq.offset / page_size]
2731                        >> (32 + PAGE_ADDR_SHIFT));
2732         roce_set_field(qpc_mask->byte_92_srq_info,
2733                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
2734                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
2735
2736         context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
2737                                     >> PAGE_ADDR_SHIFT);
2738         qpc_mask->rq_nxt_blk_addr = 0;
2739
2740         roce_set_field(context->byte_104_rq_sge,
2741                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
2742                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
2743                        mtts[hr_qp->rq.offset / page_size + 1]
2744                        >> (32 + PAGE_ADDR_SHIFT));
2745         roce_set_field(qpc_mask->byte_104_rq_sge,
2746                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
2747                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
2748
2749         roce_set_field(context->byte_108_rx_reqepsn,
2750                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
2751                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
2752         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2753                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
2754                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
2755
2756         roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
2757                        V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
2758         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
2759                        V2_QPC_BYTE_132_TRRL_BA_S, 0);
2760         context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
2761         qpc_mask->trrl_ba = 0;
2762         roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
2763                        V2_QPC_BYTE_140_TRRL_BA_S,
2764                        (u32)(dma_handle_3 >> (32 + 16 + 4)));
2765         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
2766                        V2_QPC_BYTE_140_TRRL_BA_S, 0);
2767
2768         context->irrl_ba = (u32)(dma_handle_2 >> 6);
2769         qpc_mask->irrl_ba = 0;
2770         roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
2771                        V2_QPC_BYTE_208_IRRL_BA_S,
2772                        dma_handle_2 >> (32 + 6));
2773         roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
2774                        V2_QPC_BYTE_208_IRRL_BA_S, 0);
2775
2776         roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
2777         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
2778
2779         roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
2780                      hr_qp->sq_signal_bits);
2781         roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
2782                      0);
2783
2784         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
2785
2786         smac = (u8 *)hr_dev->dev_addr[port];
2787         /* when dmac equals smac or loop_idc is 1, it should loopback */
2788         if (ether_addr_equal_unaligned(dmac, smac) ||
2789             hr_dev->loop_idc == 0x1) {
2790                 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
2791                 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
2792         }
2793
2794         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
2795              attr->max_dest_rd_atomic) {
2796                 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
2797                                V2_QPC_BYTE_140_RR_MAX_S,
2798                                fls(attr->max_dest_rd_atomic - 1));
2799                 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
2800                                V2_QPC_BYTE_140_RR_MAX_S, 0);
2801         }
2802
2803         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2804                        V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
2805         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
2806                        V2_QPC_BYTE_56_DQPN_S, 0);
2807
2808         /* Configure GID index */
2809         port_num = rdma_ah_get_port_num(&attr->ah_attr);
2810         roce_set_field(context->byte_20_smac_sgid_idx,
2811                        V2_QPC_BYTE_20_SGID_IDX_M,
2812                        V2_QPC_BYTE_20_SGID_IDX_S,
2813                        hns_get_gid_index(hr_dev, port_num - 1,
2814                                          grh->sgid_index));
2815         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2816                        V2_QPC_BYTE_20_SGID_IDX_M,
2817                        V2_QPC_BYTE_20_SGID_IDX_S, 0);
2818         memcpy(&(context->dmac), dmac, 4);
2819         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
2820                        V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
2821         qpc_mask->dmac = 0;
2822         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
2823                        V2_QPC_BYTE_52_DMAC_S, 0);
2824
2825         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
2826                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
2827         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
2828                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
2829
2830         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
2831                        V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
2832         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_HOP_LIMIT_M,
2833                        V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
2834
2835         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
2836                        V2_QPC_BYTE_28_FL_S, grh->flow_label);
2837         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
2838                        V2_QPC_BYTE_28_FL_S, 0);
2839
2840         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
2841                        V2_QPC_BYTE_24_TC_S, grh->traffic_class);
2842         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
2843                        V2_QPC_BYTE_24_TC_S, 0);
2844
2845         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
2846                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2847                                V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
2848         else
2849                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2850                                V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
2851
2852         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
2853                        V2_QPC_BYTE_24_MTU_S, 0);
2854
2855         memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
2856         memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
2857
2858         roce_set_field(context->byte_84_rq_ci_pi,
2859                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2860                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
2861         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2862                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2863                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2864
2865         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2866                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2867                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2868         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2869                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2870         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2871                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2872         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2873                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2874                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2875
2876         context->rq_rnr_timer = 0;
2877         qpc_mask->rq_rnr_timer = 0;
2878
2879         roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2880                        V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
2881         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2882                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2883
2884         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2885                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2886         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2887                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2888
2889         roce_set_field(context->byte_168_irrl_idx,
2890                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
2891                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
2892         roce_set_field(qpc_mask->byte_168_irrl_idx,
2893                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
2894                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
2895
2896         roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
2897                        V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
2898         roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
2899                        V2_QPC_BYTE_28_SL_S, 0);
2900         hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
2901
2902         return 0;
2903 }
2904
2905 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
2906                                 const struct ib_qp_attr *attr, int attr_mask,
2907                                 struct hns_roce_v2_qp_context *context,
2908                                 struct hns_roce_v2_qp_context *qpc_mask)
2909 {
2910         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2911         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2912         struct device *dev = hr_dev->dev;
2913         dma_addr_t dma_handle;
2914         u32 page_size;
2915         u64 *mtts;
2916
2917         /* Search qp buf's mtts */
2918         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
2919                                    hr_qp->mtt.first_seg, &dma_handle);
2920         if (!mtts) {