Merge tag 'mtd/fixes-for-4.16' of git://git.infradead.org/linux-mtd
[muen/linux.git] / drivers / infiniband / hw / qedr / verbs.c
1 /* QLogic qedr NIC Driver
2  * Copyright (c) 2015-2016  QLogic Corporation
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and /or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
49 #include "qedr.h"
50 #include "verbs.h"
51 #include <rdma/qedr-abi.h>
52 #include "qedr_roce_cm.h"
53
54 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
55
56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
57                                         size_t len)
58 {
59         size_t min_len = min_t(size_t, len, udata->outlen);
60
61         return ib_copy_to_udata(udata, src, min_len);
62 }
63
64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
65 {
66         if (index > QEDR_ROCE_PKEY_TABLE_LEN)
67                 return -EINVAL;
68
69         *pkey = QEDR_ROCE_PKEY_DEFAULT;
70         return 0;
71 }
72
73 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
74                       int index, union ib_gid *sgid)
75 {
76         struct qedr_dev *dev = get_qedr_dev(ibdev);
77
78         memset(sgid->raw, 0, sizeof(sgid->raw));
79         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
80
81         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
82                  sgid->global.interface_id, sgid->global.subnet_prefix);
83
84         return 0;
85 }
86
87 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
88                    union ib_gid *sgid)
89 {
90         struct qedr_dev *dev = get_qedr_dev(ibdev);
91         int rc = 0;
92
93         if (!rdma_cap_roce_gid_table(ibdev, port))
94                 return -ENODEV;
95
96         rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
97         if (rc == -EAGAIN) {
98                 memcpy(sgid, &zgid, sizeof(*sgid));
99                 return 0;
100         }
101
102         DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
103                  sgid->global.interface_id, sgid->global.subnet_prefix);
104
105         return rc;
106 }
107
108 int qedr_add_gid(struct ib_device *device, u8 port_num,
109                  unsigned int index, const union ib_gid *gid,
110                  const struct ib_gid_attr *attr, void **context)
111 {
112         if (!rdma_cap_roce_gid_table(device, port_num))
113                 return -EINVAL;
114
115         if (port_num > QEDR_MAX_PORT)
116                 return -EINVAL;
117
118         if (!context)
119                 return -EINVAL;
120
121         return 0;
122 }
123
124 int qedr_del_gid(struct ib_device *device, u8 port_num,
125                  unsigned int index, void **context)
126 {
127         if (!rdma_cap_roce_gid_table(device, port_num))
128                 return -EINVAL;
129
130         if (port_num > QEDR_MAX_PORT)
131                 return -EINVAL;
132
133         if (!context)
134                 return -EINVAL;
135
136         return 0;
137 }
138
139 int qedr_query_device(struct ib_device *ibdev,
140                       struct ib_device_attr *attr, struct ib_udata *udata)
141 {
142         struct qedr_dev *dev = get_qedr_dev(ibdev);
143         struct qedr_device_attr *qattr = &dev->attr;
144
145         if (!dev->rdma_ctx) {
146                 DP_ERR(dev,
147                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
148                        dev->rdma_ctx);
149                 return -EINVAL;
150         }
151
152         memset(attr, 0, sizeof(*attr));
153
154         attr->fw_ver = qattr->fw_ver;
155         attr->sys_image_guid = qattr->sys_image_guid;
156         attr->max_mr_size = qattr->max_mr_size;
157         attr->page_size_cap = qattr->page_size_caps;
158         attr->vendor_id = qattr->vendor_id;
159         attr->vendor_part_id = qattr->vendor_part_id;
160         attr->hw_ver = qattr->hw_ver;
161         attr->max_qp = qattr->max_qp;
162         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
163         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
164             IB_DEVICE_RC_RNR_NAK_GEN |
165             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
166
167         attr->max_sge = qattr->max_sge;
168         attr->max_sge_rd = qattr->max_sge;
169         attr->max_cq = qattr->max_cq;
170         attr->max_cqe = qattr->max_cqe;
171         attr->max_mr = qattr->max_mr;
172         attr->max_mw = qattr->max_mw;
173         attr->max_pd = qattr->max_pd;
174         attr->atomic_cap = dev->atomic_cap;
175         attr->max_fmr = qattr->max_fmr;
176         attr->max_map_per_fmr = 16;
177         attr->max_qp_init_rd_atom =
178             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
179         attr->max_qp_rd_atom =
180             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
181                 attr->max_qp_init_rd_atom);
182
183         attr->max_srq = qattr->max_srq;
184         attr->max_srq_sge = qattr->max_srq_sge;
185         attr->max_srq_wr = qattr->max_srq_wr;
186
187         attr->local_ca_ack_delay = qattr->dev_ack_delay;
188         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
189         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
190         attr->max_ah = qattr->max_ah;
191
192         return 0;
193 }
194
195 #define QEDR_SPEED_SDR          (1)
196 #define QEDR_SPEED_DDR          (2)
197 #define QEDR_SPEED_QDR          (4)
198 #define QEDR_SPEED_FDR10        (8)
199 #define QEDR_SPEED_FDR          (16)
200 #define QEDR_SPEED_EDR          (32)
201
202 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
203                                             u8 *ib_width)
204 {
205         switch (speed) {
206         case 1000:
207                 *ib_speed = QEDR_SPEED_SDR;
208                 *ib_width = IB_WIDTH_1X;
209                 break;
210         case 10000:
211                 *ib_speed = QEDR_SPEED_QDR;
212                 *ib_width = IB_WIDTH_1X;
213                 break;
214
215         case 20000:
216                 *ib_speed = QEDR_SPEED_DDR;
217                 *ib_width = IB_WIDTH_4X;
218                 break;
219
220         case 25000:
221                 *ib_speed = QEDR_SPEED_EDR;
222                 *ib_width = IB_WIDTH_1X;
223                 break;
224
225         case 40000:
226                 *ib_speed = QEDR_SPEED_QDR;
227                 *ib_width = IB_WIDTH_4X;
228                 break;
229
230         case 50000:
231                 *ib_speed = QEDR_SPEED_QDR;
232                 *ib_width = IB_WIDTH_4X;
233                 break;
234
235         case 100000:
236                 *ib_speed = QEDR_SPEED_EDR;
237                 *ib_width = IB_WIDTH_4X;
238                 break;
239
240         default:
241                 /* Unsupported */
242                 *ib_speed = QEDR_SPEED_SDR;
243                 *ib_width = IB_WIDTH_1X;
244         }
245 }
246
247 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
248 {
249         struct qedr_dev *dev;
250         struct qed_rdma_port *rdma_port;
251
252         dev = get_qedr_dev(ibdev);
253         if (port > 1) {
254                 DP_ERR(dev, "invalid_port=0x%x\n", port);
255                 return -EINVAL;
256         }
257
258         if (!dev->rdma_ctx) {
259                 DP_ERR(dev, "rdma_ctx is NULL\n");
260                 return -EINVAL;
261         }
262
263         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
264
265         /* *attr being zeroed by the caller, avoid zeroing it here */
266         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
267                 attr->state = IB_PORT_ACTIVE;
268                 attr->phys_state = 5;
269         } else {
270                 attr->state = IB_PORT_DOWN;
271                 attr->phys_state = 3;
272         }
273         attr->max_mtu = IB_MTU_4096;
274         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
275         attr->lid = 0;
276         attr->lmc = 0;
277         attr->sm_lid = 0;
278         attr->sm_sl = 0;
279         attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
280         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
281                 attr->gid_tbl_len = 1;
282                 attr->pkey_tbl_len = 1;
283         } else {
284                 attr->gid_tbl_len = QEDR_MAX_SGID;
285                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
286         }
287         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
288         attr->qkey_viol_cntr = 0;
289         get_link_speed_and_width(rdma_port->link_speed,
290                                  &attr->active_speed, &attr->active_width);
291         attr->max_msg_sz = rdma_port->max_msg_size;
292         attr->max_vl_num = 4;
293
294         return 0;
295 }
296
297 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
298                      struct ib_port_modify *props)
299 {
300         struct qedr_dev *dev;
301
302         dev = get_qedr_dev(ibdev);
303         if (port > 1) {
304                 DP_ERR(dev, "invalid_port=0x%x\n", port);
305                 return -EINVAL;
306         }
307
308         return 0;
309 }
310
311 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
312                          unsigned long len)
313 {
314         struct qedr_mm *mm;
315
316         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
317         if (!mm)
318                 return -ENOMEM;
319
320         mm->key.phy_addr = phy_addr;
321         /* This function might be called with a length which is not a multiple
322          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
323          * forces this granularity by increasing the requested size if needed.
324          * When qedr_mmap is called, it will search the list with the updated
325          * length as a key. To prevent search failures, the length is rounded up
326          * in advance to PAGE_SIZE.
327          */
328         mm->key.len = roundup(len, PAGE_SIZE);
329         INIT_LIST_HEAD(&mm->entry);
330
331         mutex_lock(&uctx->mm_list_lock);
332         list_add(&mm->entry, &uctx->mm_head);
333         mutex_unlock(&uctx->mm_list_lock);
334
335         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
336                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
337                  (unsigned long long)mm->key.phy_addr,
338                  (unsigned long)mm->key.len, uctx);
339
340         return 0;
341 }
342
343 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
344                              unsigned long len)
345 {
346         bool found = false;
347         struct qedr_mm *mm;
348
349         mutex_lock(&uctx->mm_list_lock);
350         list_for_each_entry(mm, &uctx->mm_head, entry) {
351                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
352                         continue;
353
354                 found = true;
355                 break;
356         }
357         mutex_unlock(&uctx->mm_list_lock);
358         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
359                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
360                  mm->key.phy_addr, mm->key.len, uctx, found);
361
362         return found;
363 }
364
365 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
366                                         struct ib_udata *udata)
367 {
368         int rc;
369         struct qedr_ucontext *ctx;
370         struct qedr_alloc_ucontext_resp uresp;
371         struct qedr_dev *dev = get_qedr_dev(ibdev);
372         struct qed_rdma_add_user_out_params oparams;
373
374         if (!udata)
375                 return ERR_PTR(-EFAULT);
376
377         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
378         if (!ctx)
379                 return ERR_PTR(-ENOMEM);
380
381         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
382         if (rc) {
383                 DP_ERR(dev,
384                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
385                        rc);
386                 goto err;
387         }
388
389         ctx->dpi = oparams.dpi;
390         ctx->dpi_addr = oparams.dpi_addr;
391         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
392         ctx->dpi_size = oparams.dpi_size;
393         INIT_LIST_HEAD(&ctx->mm_head);
394         mutex_init(&ctx->mm_list_lock);
395
396         memset(&uresp, 0, sizeof(uresp));
397
398         uresp.dpm_enabled = dev->user_dpm_enabled;
399         uresp.wids_enabled = 1;
400         uresp.wid_count = oparams.wid_count;
401         uresp.db_pa = ctx->dpi_phys_addr;
402         uresp.db_size = ctx->dpi_size;
403         uresp.max_send_wr = dev->attr.max_sqe;
404         uresp.max_recv_wr = dev->attr.max_rqe;
405         uresp.max_srq_wr = dev->attr.max_srq_wr;
406         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
407         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
408         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
409         uresp.max_cqes = QEDR_MAX_CQES;
410
411         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
412         if (rc)
413                 goto err;
414
415         ctx->dev = dev;
416
417         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
418         if (rc)
419                 goto err;
420
421         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
422                  &ctx->ibucontext);
423         return &ctx->ibucontext;
424
425 err:
426         kfree(ctx);
427         return ERR_PTR(rc);
428 }
429
430 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
431 {
432         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
433         struct qedr_mm *mm, *tmp;
434         int status = 0;
435
436         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
437                  uctx);
438         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
439
440         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
441                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
442                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
443                          mm->key.phy_addr, mm->key.len, uctx);
444                 list_del(&mm->entry);
445                 kfree(mm);
446         }
447
448         kfree(uctx);
449         return status;
450 }
451
452 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
453 {
454         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
455         struct qedr_dev *dev = get_qedr_dev(context->device);
456         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
457         u64 unmapped_db = dev->db_phys_addr;
458         unsigned long len = (vma->vm_end - vma->vm_start);
459         int rc = 0;
460         bool found;
461
462         DP_DEBUG(dev, QEDR_MSG_INIT,
463                  "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
464                  vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
465         if (vma->vm_start & (PAGE_SIZE - 1)) {
466                 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
467                        vma->vm_start);
468                 return -EINVAL;
469         }
470
471         found = qedr_search_mmap(ucontext, vm_page, len);
472         if (!found) {
473                 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
474                        vma->vm_pgoff);
475                 return -EINVAL;
476         }
477
478         DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
479
480         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
481                                                      dev->db_size))) {
482                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
483                 if (vma->vm_flags & VM_READ) {
484                         DP_ERR(dev, "Trying to map doorbell bar for read\n");
485                         return -EPERM;
486                 }
487
488                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
489
490                 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
491                                         PAGE_SIZE, vma->vm_page_prot);
492         } else {
493                 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
494                 rc = remap_pfn_range(vma, vma->vm_start,
495                                      vma->vm_pgoff, len, vma->vm_page_prot);
496         }
497         DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
498         return rc;
499 }
500
501 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
502                             struct ib_ucontext *context, struct ib_udata *udata)
503 {
504         struct qedr_dev *dev = get_qedr_dev(ibdev);
505         struct qedr_pd *pd;
506         u16 pd_id;
507         int rc;
508
509         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
510                  (udata && context) ? "User Lib" : "Kernel");
511
512         if (!dev->rdma_ctx) {
513                 DP_ERR(dev, "invalid RDMA context\n");
514                 return ERR_PTR(-EINVAL);
515         }
516
517         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
518         if (!pd)
519                 return ERR_PTR(-ENOMEM);
520
521         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
522         if (rc)
523                 goto err;
524
525         pd->pd_id = pd_id;
526
527         if (udata && context) {
528                 struct qedr_alloc_pd_uresp uresp;
529
530                 uresp.pd_id = pd_id;
531
532                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
533                 if (rc) {
534                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
535                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
536                         goto err;
537                 }
538
539                 pd->uctx = get_qedr_ucontext(context);
540                 pd->uctx->pd = pd;
541         }
542
543         return &pd->ibpd;
544
545 err:
546         kfree(pd);
547         return ERR_PTR(rc);
548 }
549
550 int qedr_dealloc_pd(struct ib_pd *ibpd)
551 {
552         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
553         struct qedr_pd *pd = get_qedr_pd(ibpd);
554
555         if (!pd) {
556                 pr_err("Invalid PD received in dealloc_pd\n");
557                 return -EINVAL;
558         }
559
560         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
561         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
562
563         kfree(pd);
564
565         return 0;
566 }
567
568 static void qedr_free_pbl(struct qedr_dev *dev,
569                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
570 {
571         struct pci_dev *pdev = dev->pdev;
572         int i;
573
574         for (i = 0; i < pbl_info->num_pbls; i++) {
575                 if (!pbl[i].va)
576                         continue;
577                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
578                                   pbl[i].va, pbl[i].pa);
579         }
580
581         kfree(pbl);
582 }
583
584 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
585 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
586
587 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
588 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
589 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
590
591 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
592                                            struct qedr_pbl_info *pbl_info,
593                                            gfp_t flags)
594 {
595         struct pci_dev *pdev = dev->pdev;
596         struct qedr_pbl *pbl_table;
597         dma_addr_t *pbl_main_tbl;
598         dma_addr_t pa;
599         void *va;
600         int i;
601
602         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
603         if (!pbl_table)
604                 return ERR_PTR(-ENOMEM);
605
606         for (i = 0; i < pbl_info->num_pbls; i++) {
607                 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
608                                          &pa, flags);
609                 if (!va)
610                         goto err;
611
612                 pbl_table[i].va = va;
613                 pbl_table[i].pa = pa;
614         }
615
616         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
617          * the first one with physical pointers to all of the rest
618          */
619         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
620         for (i = 0; i < pbl_info->num_pbls - 1; i++)
621                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
622
623         return pbl_table;
624
625 err:
626         for (i--; i >= 0; i--)
627                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
628                                   pbl_table[i].va, pbl_table[i].pa);
629
630         qedr_free_pbl(dev, pbl_info, pbl_table);
631
632         return ERR_PTR(-ENOMEM);
633 }
634
635 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
636                                 struct qedr_pbl_info *pbl_info,
637                                 u32 num_pbes, int two_layer_capable)
638 {
639         u32 pbl_capacity;
640         u32 pbl_size;
641         u32 num_pbls;
642
643         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
644                 if (num_pbes > MAX_PBES_TWO_LAYER) {
645                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
646                                num_pbes);
647                         return -EINVAL;
648                 }
649
650                 /* calculate required pbl page size */
651                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
652                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
653                                NUM_PBES_ON_PAGE(pbl_size);
654
655                 while (pbl_capacity < num_pbes) {
656                         pbl_size *= 2;
657                         pbl_capacity = pbl_size / sizeof(u64);
658                         pbl_capacity = pbl_capacity * pbl_capacity;
659                 }
660
661                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
662                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
663                 pbl_info->two_layered = true;
664         } else {
665                 /* One layered PBL */
666                 num_pbls = 1;
667                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
668                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
669                 pbl_info->two_layered = false;
670         }
671
672         pbl_info->num_pbls = num_pbls;
673         pbl_info->pbl_size = pbl_size;
674         pbl_info->num_pbes = num_pbes;
675
676         DP_DEBUG(dev, QEDR_MSG_MR,
677                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
678                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
679
680         return 0;
681 }
682
683 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
684                                struct qedr_pbl *pbl,
685                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
686 {
687         int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
688         u32 fw_pg_cnt, fw_pg_per_umem_pg;
689         struct qedr_pbl *pbl_tbl;
690         struct scatterlist *sg;
691         struct regpair *pbe;
692         u64 pg_addr;
693         int entry;
694
695         if (!pbl_info->num_pbes)
696                 return;
697
698         /* If we have a two layered pbl, the first pbl points to the rest
699          * of the pbls and the first entry lays on the second pbl in the table
700          */
701         if (pbl_info->two_layered)
702                 pbl_tbl = &pbl[1];
703         else
704                 pbl_tbl = pbl;
705
706         pbe = (struct regpair *)pbl_tbl->va;
707         if (!pbe) {
708                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
709                 return;
710         }
711
712         pbe_cnt = 0;
713
714         shift = umem->page_shift;
715
716         fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
717
718         for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
719                 pages = sg_dma_len(sg) >> shift;
720                 pg_addr = sg_dma_address(sg);
721                 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
722                         for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
723                                 pbe->lo = cpu_to_le32(pg_addr);
724                                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
725
726                                 pg_addr += BIT(pg_shift);
727                                 pbe_cnt++;
728                                 total_num_pbes++;
729                                 pbe++;
730
731                                 if (total_num_pbes == pbl_info->num_pbes)
732                                         return;
733
734                                 /* If the given pbl is full storing the pbes,
735                                  * move to next pbl.
736                                  */
737                                 if (pbe_cnt ==
738                                     (pbl_info->pbl_size / sizeof(u64))) {
739                                         pbl_tbl++;
740                                         pbe = (struct regpair *)pbl_tbl->va;
741                                         pbe_cnt = 0;
742                                 }
743
744                                 fw_pg_cnt++;
745                         }
746                 }
747         }
748 }
749
750 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
751                               struct qedr_cq *cq, struct ib_udata *udata)
752 {
753         struct qedr_create_cq_uresp uresp;
754         int rc;
755
756         memset(&uresp, 0, sizeof(uresp));
757
758         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
759         uresp.icid = cq->icid;
760
761         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
762         if (rc)
763                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
764
765         return rc;
766 }
767
768 static void consume_cqe(struct qedr_cq *cq)
769 {
770         if (cq->latest_cqe == cq->toggle_cqe)
771                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
772
773         cq->latest_cqe = qed_chain_consume(&cq->pbl);
774 }
775
776 static inline int qedr_align_cq_entries(int entries)
777 {
778         u64 size, aligned_size;
779
780         /* We allocate an extra entry that we don't report to the FW. */
781         size = (entries + 1) * QEDR_CQE_SIZE;
782         aligned_size = ALIGN(size, PAGE_SIZE);
783
784         return aligned_size / QEDR_CQE_SIZE;
785 }
786
787 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
788                                        struct qedr_dev *dev,
789                                        struct qedr_userq *q,
790                                        u64 buf_addr, size_t buf_len,
791                                        int access, int dmasync,
792                                        int alloc_and_init)
793 {
794         u32 fw_pages;
795         int rc;
796
797         q->buf_addr = buf_addr;
798         q->buf_len = buf_len;
799         q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
800         if (IS_ERR(q->umem)) {
801                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
802                        PTR_ERR(q->umem));
803                 return PTR_ERR(q->umem);
804         }
805
806         fw_pages = ib_umem_page_count(q->umem) <<
807             (q->umem->page_shift - FW_PAGE_SHIFT);
808
809         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
810         if (rc)
811                 goto err0;
812
813         if (alloc_and_init) {
814                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
815                 if (IS_ERR(q->pbl_tbl)) {
816                         rc = PTR_ERR(q->pbl_tbl);
817                         goto err0;
818                 }
819                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
820                                    FW_PAGE_SHIFT);
821         } else {
822                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
823                 if (!q->pbl_tbl) {
824                         rc = -ENOMEM;
825                         goto err0;
826                 }
827         }
828
829         return 0;
830
831 err0:
832         ib_umem_release(q->umem);
833         q->umem = NULL;
834
835         return rc;
836 }
837
838 static inline void qedr_init_cq_params(struct qedr_cq *cq,
839                                        struct qedr_ucontext *ctx,
840                                        struct qedr_dev *dev, int vector,
841                                        int chain_entries, int page_cnt,
842                                        u64 pbl_ptr,
843                                        struct qed_rdma_create_cq_in_params
844                                        *params)
845 {
846         memset(params, 0, sizeof(*params));
847         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
848         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
849         params->cnq_id = vector;
850         params->cq_size = chain_entries - 1;
851         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
852         params->pbl_num_pages = page_cnt;
853         params->pbl_ptr = pbl_ptr;
854         params->pbl_two_level = 0;
855 }
856
857 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
858 {
859         /* Flush data before signalling doorbell */
860         wmb();
861         cq->db.data.agg_flags = flags;
862         cq->db.data.value = cpu_to_le32(cons);
863         writeq(cq->db.raw, cq->db_addr);
864
865         /* Make sure write would stick */
866         mmiowb();
867 }
868
869 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
870 {
871         struct qedr_cq *cq = get_qedr_cq(ibcq);
872         unsigned long sflags;
873         struct qedr_dev *dev;
874
875         dev = get_qedr_dev(ibcq->device);
876
877         if (cq->destroyed) {
878                 DP_ERR(dev,
879                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
880                        cq, cq->icid);
881                 return -EINVAL;
882         }
883
884
885         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
886                 return 0;
887
888         spin_lock_irqsave(&cq->cq_lock, sflags);
889
890         cq->arm_flags = 0;
891
892         if (flags & IB_CQ_SOLICITED)
893                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
894
895         if (flags & IB_CQ_NEXT_COMP)
896                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
897
898         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
899
900         spin_unlock_irqrestore(&cq->cq_lock, sflags);
901
902         return 0;
903 }
904
905 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
906                              const struct ib_cq_init_attr *attr,
907                              struct ib_ucontext *ib_ctx, struct ib_udata *udata)
908 {
909         struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
910         struct qed_rdma_destroy_cq_out_params destroy_oparams;
911         struct qed_rdma_destroy_cq_in_params destroy_iparams;
912         struct qedr_dev *dev = get_qedr_dev(ibdev);
913         struct qed_rdma_create_cq_in_params params;
914         struct qedr_create_cq_ureq ureq;
915         int vector = attr->comp_vector;
916         int entries = attr->cqe;
917         struct qedr_cq *cq;
918         int chain_entries;
919         int page_cnt;
920         u64 pbl_ptr;
921         u16 icid;
922         int rc;
923
924         DP_DEBUG(dev, QEDR_MSG_INIT,
925                  "create_cq: called from %s. entries=%d, vector=%d\n",
926                  udata ? "User Lib" : "Kernel", entries, vector);
927
928         if (entries > QEDR_MAX_CQES) {
929                 DP_ERR(dev,
930                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
931                        entries, QEDR_MAX_CQES);
932                 return ERR_PTR(-EINVAL);
933         }
934
935         chain_entries = qedr_align_cq_entries(entries);
936         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
937
938         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
939         if (!cq)
940                 return ERR_PTR(-ENOMEM);
941
942         if (udata) {
943                 memset(&ureq, 0, sizeof(ureq));
944                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
945                         DP_ERR(dev,
946                                "create cq: problem copying data from user space\n");
947                         goto err0;
948                 }
949
950                 if (!ureq.len) {
951                         DP_ERR(dev,
952                                "create cq: cannot create a cq with 0 entries\n");
953                         goto err0;
954                 }
955
956                 cq->cq_type = QEDR_CQ_TYPE_USER;
957
958                 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
959                                           ureq.len, IB_ACCESS_LOCAL_WRITE,
960                                           1, 1);
961                 if (rc)
962                         goto err0;
963
964                 pbl_ptr = cq->q.pbl_tbl->pa;
965                 page_cnt = cq->q.pbl_info.num_pbes;
966
967                 cq->ibcq.cqe = chain_entries;
968         } else {
969                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
970
971                 rc = dev->ops->common->chain_alloc(dev->cdev,
972                                                    QED_CHAIN_USE_TO_CONSUME,
973                                                    QED_CHAIN_MODE_PBL,
974                                                    QED_CHAIN_CNT_TYPE_U32,
975                                                    chain_entries,
976                                                    sizeof(union rdma_cqe),
977                                                    &cq->pbl, NULL);
978                 if (rc)
979                         goto err1;
980
981                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
982                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
983                 cq->ibcq.cqe = cq->pbl.capacity;
984         }
985
986         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
987                             pbl_ptr, &params);
988
989         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
990         if (rc)
991                 goto err2;
992
993         cq->icid = icid;
994         cq->sig = QEDR_CQ_MAGIC_NUMBER;
995         spin_lock_init(&cq->cq_lock);
996
997         if (ib_ctx) {
998                 rc = qedr_copy_cq_uresp(dev, cq, udata);
999                 if (rc)
1000                         goto err3;
1001         } else {
1002                 /* Generate doorbell address. */
1003                 cq->db_addr = dev->db_addr +
1004                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1005                 cq->db.data.icid = cq->icid;
1006                 cq->db.data.params = DB_AGG_CMD_SET <<
1007                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1008
1009                 /* point to the very last element, passing it we will toggle */
1010                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1011                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1012                 cq->latest_cqe = NULL;
1013                 consume_cqe(cq);
1014                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1015         }
1016
1017         DP_DEBUG(dev, QEDR_MSG_CQ,
1018                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1019                  cq->icid, cq, params.cq_size);
1020
1021         return &cq->ibcq;
1022
1023 err3:
1024         destroy_iparams.icid = cq->icid;
1025         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1026                                   &destroy_oparams);
1027 err2:
1028         if (udata)
1029                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1030         else
1031                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1032 err1:
1033         if (udata)
1034                 ib_umem_release(cq->q.umem);
1035 err0:
1036         kfree(cq);
1037         return ERR_PTR(-EINVAL);
1038 }
1039
1040 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1041 {
1042         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1043         struct qedr_cq *cq = get_qedr_cq(ibcq);
1044
1045         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1046
1047         return 0;
1048 }
1049
1050 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1051 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1052
1053 int qedr_destroy_cq(struct ib_cq *ibcq)
1054 {
1055         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1056         struct qed_rdma_destroy_cq_out_params oparams;
1057         struct qed_rdma_destroy_cq_in_params iparams;
1058         struct qedr_cq *cq = get_qedr_cq(ibcq);
1059         int iter;
1060         int rc;
1061
1062         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1063
1064         cq->destroyed = 1;
1065
1066         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1067         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1068                 goto done;
1069
1070         iparams.icid = cq->icid;
1071         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1072         if (rc)
1073                 return rc;
1074
1075         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1076
1077         if (ibcq->uobject && ibcq->uobject->context) {
1078                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1079                 ib_umem_release(cq->q.umem);
1080         }
1081
1082         /* We don't want the IRQ handler to handle a non-existing CQ so we
1083          * wait until all CNQ interrupts, if any, are received. This will always
1084          * happen and will always happen very fast. If not, then a serious error
1085          * has occured. That is why we can use a long delay.
1086          * We spin for a short time so we don’t lose time on context switching
1087          * in case all the completions are handled in that span. Otherwise
1088          * we sleep for a while and check again. Since the CNQ may be
1089          * associated with (only) the current CPU we use msleep to allow the
1090          * current CPU to be freed.
1091          * The CNQ notification is increased in qedr_irq_handler().
1092          */
1093         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1094         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1095                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1096                 iter--;
1097         }
1098
1099         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1100         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1101                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1102                 iter--;
1103         }
1104
1105         if (oparams.num_cq_notif != cq->cnq_notif)
1106                 goto err;
1107
1108         /* Note that we don't need to have explicit code to wait for the
1109          * completion of the event handler because it is invoked from the EQ.
1110          * Since the destroy CQ ramrod has also been received on the EQ we can
1111          * be certain that there's no event handler in process.
1112          */
1113 done:
1114         cq->sig = ~cq->sig;
1115
1116         kfree(cq);
1117
1118         return 0;
1119
1120 err:
1121         DP_ERR(dev,
1122                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1123                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1124
1125         return -EINVAL;
1126 }
1127
1128 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1129                                           struct ib_qp_attr *attr,
1130                                           int attr_mask,
1131                                           struct qed_rdma_modify_qp_in_params
1132                                           *qp_params)
1133 {
1134         enum rdma_network_type nw_type;
1135         struct ib_gid_attr gid_attr;
1136         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1137         union ib_gid gid;
1138         u32 ipv4_addr;
1139         int rc = 0;
1140         int i;
1141
1142         rc = ib_get_cached_gid(ibqp->device,
1143                                rdma_ah_get_port_num(&attr->ah_attr),
1144                                grh->sgid_index, &gid, &gid_attr);
1145         if (rc)
1146                 return rc;
1147
1148         if (!memcmp(&gid, &zgid, sizeof(gid)))
1149                 return -ENOENT;
1150
1151         if (gid_attr.ndev) {
1152                 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1153
1154                 dev_put(gid_attr.ndev);
1155                 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1156                 switch (nw_type) {
1157                 case RDMA_NETWORK_IPV6:
1158                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1159                                sizeof(qp_params->sgid));
1160                         memcpy(&qp_params->dgid.bytes[0],
1161                                &grh->dgid,
1162                                sizeof(qp_params->dgid));
1163                         qp_params->roce_mode = ROCE_V2_IPV6;
1164                         SET_FIELD(qp_params->modify_flags,
1165                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1166                         break;
1167                 case RDMA_NETWORK_IB:
1168                         memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1169                                sizeof(qp_params->sgid));
1170                         memcpy(&qp_params->dgid.bytes[0],
1171                                &grh->dgid,
1172                                sizeof(qp_params->dgid));
1173                         qp_params->roce_mode = ROCE_V1;
1174                         break;
1175                 case RDMA_NETWORK_IPV4:
1176                         memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1177                         memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1178                         ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1179                         qp_params->sgid.ipv4_addr = ipv4_addr;
1180                         ipv4_addr =
1181                             qedr_get_ipv4_from_gid(grh->dgid.raw);
1182                         qp_params->dgid.ipv4_addr = ipv4_addr;
1183                         SET_FIELD(qp_params->modify_flags,
1184                                   QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1185                         qp_params->roce_mode = ROCE_V2_IPV4;
1186                         break;
1187                 }
1188         }
1189
1190         for (i = 0; i < 4; i++) {
1191                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1192                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1193         }
1194
1195         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1196                 qp_params->vlan_id = 0;
1197
1198         return 0;
1199 }
1200
1201 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1202                                struct ib_qp_init_attr *attrs)
1203 {
1204         struct qedr_device_attr *qattr = &dev->attr;
1205
1206         /* QP0... attrs->qp_type == IB_QPT_GSI */
1207         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1208                 DP_DEBUG(dev, QEDR_MSG_QP,
1209                          "create qp: unsupported qp type=0x%x requested\n",
1210                          attrs->qp_type);
1211                 return -EINVAL;
1212         }
1213
1214         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1215                 DP_ERR(dev,
1216                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1217                        attrs->cap.max_send_wr, qattr->max_sqe);
1218                 return -EINVAL;
1219         }
1220
1221         if (attrs->cap.max_inline_data > qattr->max_inline) {
1222                 DP_ERR(dev,
1223                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1224                        attrs->cap.max_inline_data, qattr->max_inline);
1225                 return -EINVAL;
1226         }
1227
1228         if (attrs->cap.max_send_sge > qattr->max_sge) {
1229                 DP_ERR(dev,
1230                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1231                        attrs->cap.max_send_sge, qattr->max_sge);
1232                 return -EINVAL;
1233         }
1234
1235         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1236                 DP_ERR(dev,
1237                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1238                        attrs->cap.max_recv_sge, qattr->max_sge);
1239                 return -EINVAL;
1240         }
1241
1242         /* Unprivileged user space cannot create special QP */
1243         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1244                 DP_ERR(dev,
1245                        "create qp: userspace can't create special QPs of type=0x%x\n",
1246                        attrs->qp_type);
1247                 return -EINVAL;
1248         }
1249
1250         return 0;
1251 }
1252
1253 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1254                                struct qedr_create_qp_uresp *uresp,
1255                                struct qedr_qp *qp)
1256 {
1257         /* iWARP requires two doorbells per RQ. */
1258         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1259                 uresp->rq_db_offset =
1260                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1261                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1262         } else {
1263                 uresp->rq_db_offset =
1264                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1265         }
1266
1267         uresp->rq_icid = qp->icid;
1268 }
1269
1270 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1271                                struct qedr_create_qp_uresp *uresp,
1272                                struct qedr_qp *qp)
1273 {
1274         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1275
1276         /* iWARP uses the same cid for rq and sq */
1277         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1278                 uresp->sq_icid = qp->icid;
1279         else
1280                 uresp->sq_icid = qp->icid + 1;
1281 }
1282
1283 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1284                               struct qedr_qp *qp, struct ib_udata *udata)
1285 {
1286         struct qedr_create_qp_uresp uresp;
1287         int rc;
1288
1289         memset(&uresp, 0, sizeof(uresp));
1290         qedr_copy_sq_uresp(dev, &uresp, qp);
1291         qedr_copy_rq_uresp(dev, &uresp, qp);
1292
1293         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1294         uresp.qp_id = qp->qp_id;
1295
1296         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1297         if (rc)
1298                 DP_ERR(dev,
1299                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1300                        qp->icid);
1301
1302         return rc;
1303 }
1304
1305 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1306                                       struct qedr_qp *qp,
1307                                       struct qedr_pd *pd,
1308                                       struct ib_qp_init_attr *attrs)
1309 {
1310         spin_lock_init(&qp->q_lock);
1311         atomic_set(&qp->refcnt, 1);
1312         qp->pd = pd;
1313         qp->qp_type = attrs->qp_type;
1314         qp->max_inline_data = attrs->cap.max_inline_data;
1315         qp->sq.max_sges = attrs->cap.max_send_sge;
1316         qp->state = QED_ROCE_QP_STATE_RESET;
1317         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1318         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1319         qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1320         qp->dev = dev;
1321         qp->rq.max_sges = attrs->cap.max_recv_sge;
1322
1323         DP_DEBUG(dev, QEDR_MSG_QP,
1324                  "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1325                  qp->rq.max_sges, qp->rq_cq->icid);
1326         DP_DEBUG(dev, QEDR_MSG_QP,
1327                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1328                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1329                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1330         DP_DEBUG(dev, QEDR_MSG_QP,
1331                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1332                  qp->sq.max_sges, qp->sq_cq->icid);
1333 }
1334
1335 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1336 {
1337         qp->sq.db = dev->db_addr +
1338                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1339         qp->sq.db_data.data.icid = qp->icid + 1;
1340         qp->rq.db = dev->db_addr +
1341                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1342         qp->rq.db_data.data.icid = qp->icid;
1343 }
1344
1345 static inline void
1346 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1347                               struct qedr_pd *pd,
1348                               struct qedr_qp *qp,
1349                               struct ib_qp_init_attr *attrs,
1350                               bool fmr_and_reserved_lkey,
1351                               struct qed_rdma_create_qp_in_params *params)
1352 {
1353         /* QP handle to be written in an async event */
1354         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1355         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1356
1357         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1358         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1359         params->pd = pd->pd_id;
1360         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1361         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1362         params->stats_queue = 0;
1363         params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1364         params->srq_id = 0;
1365         params->use_srq = false;
1366 }
1367
1368 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1369 {
1370         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1371                  "qp=%p. "
1372                  "sq_addr=0x%llx, "
1373                  "sq_len=%zd, "
1374                  "rq_addr=0x%llx, "
1375                  "rq_len=%zd"
1376                  "\n",
1377                  qp,
1378                  qp->usq.buf_addr,
1379                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1380 }
1381
1382 static int qedr_idr_add(struct qedr_dev *dev, void *ptr, u32 id)
1383 {
1384         int rc;
1385
1386         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1387                 return 0;
1388
1389         idr_preload(GFP_KERNEL);
1390         spin_lock_irq(&dev->idr_lock);
1391
1392         rc = idr_alloc(&dev->qpidr, ptr, id, id + 1, GFP_ATOMIC);
1393
1394         spin_unlock_irq(&dev->idr_lock);
1395         idr_preload_end();
1396
1397         return rc < 0 ? rc : 0;
1398 }
1399
1400 static void qedr_idr_remove(struct qedr_dev *dev, u32 id)
1401 {
1402         if (!rdma_protocol_iwarp(&dev->ibdev, 1))
1403                 return;
1404
1405         spin_lock_irq(&dev->idr_lock);
1406         idr_remove(&dev->qpidr, id);
1407         spin_unlock_irq(&dev->idr_lock);
1408 }
1409
1410 static inline void
1411 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1412                             struct qedr_qp *qp,
1413                             struct qed_rdma_create_qp_out_params *out_params)
1414 {
1415         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1416         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1417
1418         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1419                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1420
1421         qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1422         qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1423
1424         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1425                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1426 }
1427
1428 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1429 {
1430         if (qp->usq.umem)
1431                 ib_umem_release(qp->usq.umem);
1432         qp->usq.umem = NULL;
1433
1434         if (qp->urq.umem)
1435                 ib_umem_release(qp->urq.umem);
1436         qp->urq.umem = NULL;
1437 }
1438
1439 static int qedr_create_user_qp(struct qedr_dev *dev,
1440                                struct qedr_qp *qp,
1441                                struct ib_pd *ibpd,
1442                                struct ib_udata *udata,
1443                                struct ib_qp_init_attr *attrs)
1444 {
1445         struct qed_rdma_create_qp_in_params in_params;
1446         struct qed_rdma_create_qp_out_params out_params;
1447         struct qedr_pd *pd = get_qedr_pd(ibpd);
1448         struct ib_ucontext *ib_ctx = NULL;
1449         struct qedr_create_qp_ureq ureq;
1450         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1451         int rc = -EINVAL;
1452
1453         ib_ctx = ibpd->uobject->context;
1454
1455         memset(&ureq, 0, sizeof(ureq));
1456         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1457         if (rc) {
1458                 DP_ERR(dev, "Problem copying data from user space\n");
1459                 return rc;
1460         }
1461
1462         /* SQ - read access only (0), dma sync not required (0) */
1463         rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1464                                   ureq.sq_len, 0, 0, alloc_and_init);
1465         if (rc)
1466                 return rc;
1467
1468         /* RQ - read access only (0), dma sync not required (0) */
1469         rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1470                                   ureq.rq_len, 0, 0, alloc_and_init);
1471         if (rc)
1472                 return rc;
1473
1474         memset(&in_params, 0, sizeof(in_params));
1475         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1476         in_params.qp_handle_lo = ureq.qp_handle_lo;
1477         in_params.qp_handle_hi = ureq.qp_handle_hi;
1478         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1479         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1480         in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1481         in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1482
1483         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1484                                               &in_params, &out_params);
1485
1486         if (!qp->qed_qp) {
1487                 rc = -ENOMEM;
1488                 goto err1;
1489         }
1490
1491         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1492                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1493
1494         qp->qp_id = out_params.qp_id;
1495         qp->icid = out_params.icid;
1496
1497         rc = qedr_copy_qp_uresp(dev, qp, udata);
1498         if (rc)
1499                 goto err;
1500
1501         qedr_qp_user_print(dev, qp);
1502
1503         return 0;
1504 err:
1505         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1506         if (rc)
1507                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1508
1509 err1:
1510         qedr_cleanup_user(dev, qp);
1511         return rc;
1512 }
1513
1514 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1515 {
1516         qp->sq.db = dev->db_addr +
1517             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1518         qp->sq.db_data.data.icid = qp->icid;
1519
1520         qp->rq.db = dev->db_addr +
1521                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1522         qp->rq.db_data.data.icid = qp->icid;
1523         qp->rq.iwarp_db2 = dev->db_addr +
1524                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1525         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1526         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1527 }
1528
1529 static int
1530 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1531                            struct qedr_qp *qp,
1532                            struct qed_rdma_create_qp_in_params *in_params,
1533                            u32 n_sq_elems, u32 n_rq_elems)
1534 {
1535         struct qed_rdma_create_qp_out_params out_params;
1536         int rc;
1537
1538         rc = dev->ops->common->chain_alloc(dev->cdev,
1539                                            QED_CHAIN_USE_TO_PRODUCE,
1540                                            QED_CHAIN_MODE_PBL,
1541                                            QED_CHAIN_CNT_TYPE_U32,
1542                                            n_sq_elems,
1543                                            QEDR_SQE_ELEMENT_SIZE,
1544                                            &qp->sq.pbl, NULL);
1545
1546         if (rc)
1547                 return rc;
1548
1549         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1550         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1551
1552         rc = dev->ops->common->chain_alloc(dev->cdev,
1553                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1554                                            QED_CHAIN_MODE_PBL,
1555                                            QED_CHAIN_CNT_TYPE_U32,
1556                                            n_rq_elems,
1557                                            QEDR_RQE_ELEMENT_SIZE,
1558                                            &qp->rq.pbl, NULL);
1559         if (rc)
1560                 return rc;
1561
1562         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1563         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1564
1565         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1566                                               in_params, &out_params);
1567
1568         if (!qp->qed_qp)
1569                 return -EINVAL;
1570
1571         qp->qp_id = out_params.qp_id;
1572         qp->icid = out_params.icid;
1573
1574         qedr_set_roce_db_info(dev, qp);
1575         return rc;
1576 }
1577
1578 static int
1579 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1580                             struct qedr_qp *qp,
1581                             struct qed_rdma_create_qp_in_params *in_params,
1582                             u32 n_sq_elems, u32 n_rq_elems)
1583 {
1584         struct qed_rdma_create_qp_out_params out_params;
1585         struct qed_chain_ext_pbl ext_pbl;
1586         int rc;
1587
1588         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1589                                                      QEDR_SQE_ELEMENT_SIZE,
1590                                                      QED_CHAIN_MODE_PBL);
1591         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1592                                                      QEDR_RQE_ELEMENT_SIZE,
1593                                                      QED_CHAIN_MODE_PBL);
1594
1595         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1596                                               in_params, &out_params);
1597
1598         if (!qp->qed_qp)
1599                 return -EINVAL;
1600
1601         /* Now we allocate the chain */
1602         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1603         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1604
1605         rc = dev->ops->common->chain_alloc(dev->cdev,
1606                                            QED_CHAIN_USE_TO_PRODUCE,
1607                                            QED_CHAIN_MODE_PBL,
1608                                            QED_CHAIN_CNT_TYPE_U32,
1609                                            n_sq_elems,
1610                                            QEDR_SQE_ELEMENT_SIZE,
1611                                            &qp->sq.pbl, &ext_pbl);
1612
1613         if (rc)
1614                 goto err;
1615
1616         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1617         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1618
1619         rc = dev->ops->common->chain_alloc(dev->cdev,
1620                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1621                                            QED_CHAIN_MODE_PBL,
1622                                            QED_CHAIN_CNT_TYPE_U32,
1623                                            n_rq_elems,
1624                                            QEDR_RQE_ELEMENT_SIZE,
1625                                            &qp->rq.pbl, &ext_pbl);
1626
1627         if (rc)
1628                 goto err;
1629
1630         qp->qp_id = out_params.qp_id;
1631         qp->icid = out_params.icid;
1632
1633         qedr_set_iwarp_db_info(dev, qp);
1634         return rc;
1635
1636 err:
1637         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1638
1639         return rc;
1640 }
1641
1642 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1643 {
1644         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1645         kfree(qp->wqe_wr_id);
1646
1647         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1648         kfree(qp->rqe_wr_id);
1649 }
1650
1651 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1652                                  struct qedr_qp *qp,
1653                                  struct ib_pd *ibpd,
1654                                  struct ib_qp_init_attr *attrs)
1655 {
1656         struct qed_rdma_create_qp_in_params in_params;
1657         struct qedr_pd *pd = get_qedr_pd(ibpd);
1658         int rc = -EINVAL;
1659         u32 n_rq_elems;
1660         u32 n_sq_elems;
1661         u32 n_sq_entries;
1662
1663         memset(&in_params, 0, sizeof(in_params));
1664
1665         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1666          * the ring. The ring should allow at least a single WR, even if the
1667          * user requested none, due to allocation issues.
1668          * We should add an extra WR since the prod and cons indices of
1669          * wqe_wr_id are managed in such a way that the WQ is considered full
1670          * when (prod+1)%max_wr==cons. We currently don't do that because we
1671          * double the number of entries due an iSER issue that pushes far more
1672          * WRs than indicated. If we decline its ib_post_send() then we get
1673          * error prints in the dmesg we'd like to avoid.
1674          */
1675         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1676                               dev->attr.max_sqe);
1677
1678         qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1679                                 GFP_KERNEL);
1680         if (!qp->wqe_wr_id) {
1681                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1682                 return -ENOMEM;
1683         }
1684
1685         /* QP handle to be written in CQE */
1686         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1687         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1688
1689         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1690          * the ring. There ring should allow at least a single WR, even if the
1691          * user requested none, due to allocation issues.
1692          */
1693         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1694
1695         /* Allocate driver internal RQ array */
1696         qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1697                                 GFP_KERNEL);
1698         if (!qp->rqe_wr_id) {
1699                 DP_ERR(dev,
1700                        "create qp: failed RQ shadow memory allocation\n");
1701                 kfree(qp->wqe_wr_id);
1702                 return -ENOMEM;
1703         }
1704
1705         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1706
1707         n_sq_entries = attrs->cap.max_send_wr;
1708         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1709         n_sq_entries = max_t(u32, n_sq_entries, 1);
1710         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1711
1712         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1713
1714         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1715                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1716                                                  n_sq_elems, n_rq_elems);
1717         else
1718                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1719                                                 n_sq_elems, n_rq_elems);
1720         if (rc)
1721                 qedr_cleanup_kernel(dev, qp);
1722
1723         return rc;
1724 }
1725
1726 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1727                              struct ib_qp_init_attr *attrs,
1728                              struct ib_udata *udata)
1729 {
1730         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1731         struct qedr_pd *pd = get_qedr_pd(ibpd);
1732         struct qedr_qp *qp;
1733         struct ib_qp *ibqp;
1734         int rc = 0;
1735
1736         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1737                  udata ? "user library" : "kernel", pd);
1738
1739         rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1740         if (rc)
1741                 return ERR_PTR(rc);
1742
1743         if (attrs->srq)
1744                 return ERR_PTR(-EINVAL);
1745
1746         DP_DEBUG(dev, QEDR_MSG_QP,
1747                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1748                  udata ? "user library" : "kernel", attrs->event_handler, pd,
1749                  get_qedr_cq(attrs->send_cq),
1750                  get_qedr_cq(attrs->send_cq)->icid,
1751                  get_qedr_cq(attrs->recv_cq),
1752                  get_qedr_cq(attrs->recv_cq)->icid);
1753
1754         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1755         if (!qp) {
1756                 DP_ERR(dev, "create qp: failed allocating memory\n");
1757                 return ERR_PTR(-ENOMEM);
1758         }
1759
1760         qedr_set_common_qp_params(dev, qp, pd, attrs);
1761
1762         if (attrs->qp_type == IB_QPT_GSI) {
1763                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1764                 if (IS_ERR(ibqp))
1765                         kfree(qp);
1766                 return ibqp;
1767         }
1768
1769         if (udata)
1770                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1771         else
1772                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1773
1774         if (rc)
1775                 goto err;
1776
1777         qp->ibqp.qp_num = qp->qp_id;
1778
1779         rc = qedr_idr_add(dev, qp, qp->qp_id);
1780         if (rc)
1781                 goto err;
1782
1783         return &qp->ibqp;
1784
1785 err:
1786         kfree(qp);
1787
1788         return ERR_PTR(-EFAULT);
1789 }
1790
1791 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1792 {
1793         switch (qp_state) {
1794         case QED_ROCE_QP_STATE_RESET:
1795                 return IB_QPS_RESET;
1796         case QED_ROCE_QP_STATE_INIT:
1797                 return IB_QPS_INIT;
1798         case QED_ROCE_QP_STATE_RTR:
1799                 return IB_QPS_RTR;
1800         case QED_ROCE_QP_STATE_RTS:
1801                 return IB_QPS_RTS;
1802         case QED_ROCE_QP_STATE_SQD:
1803                 return IB_QPS_SQD;
1804         case QED_ROCE_QP_STATE_ERR:
1805                 return IB_QPS_ERR;
1806         case QED_ROCE_QP_STATE_SQE:
1807                 return IB_QPS_SQE;
1808         }
1809         return IB_QPS_ERR;
1810 }
1811
1812 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1813                                         enum ib_qp_state qp_state)
1814 {
1815         switch (qp_state) {
1816         case IB_QPS_RESET:
1817                 return QED_ROCE_QP_STATE_RESET;
1818         case IB_QPS_INIT:
1819                 return QED_ROCE_QP_STATE_INIT;
1820         case IB_QPS_RTR:
1821                 return QED_ROCE_QP_STATE_RTR;
1822         case IB_QPS_RTS:
1823                 return QED_ROCE_QP_STATE_RTS;
1824         case IB_QPS_SQD:
1825                 return QED_ROCE_QP_STATE_SQD;
1826         case IB_QPS_ERR:
1827                 return QED_ROCE_QP_STATE_ERR;
1828         default:
1829                 return QED_ROCE_QP_STATE_ERR;
1830         }
1831 }
1832
1833 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1834 {
1835         qed_chain_reset(&qph->pbl);
1836         qph->prod = 0;
1837         qph->cons = 0;
1838         qph->wqe_cons = 0;
1839         qph->db_data.data.value = cpu_to_le16(0);
1840 }
1841
1842 static int qedr_update_qp_state(struct qedr_dev *dev,
1843                                 struct qedr_qp *qp,
1844                                 enum qed_roce_qp_state new_state)
1845 {
1846         int status = 0;
1847
1848         if (new_state == qp->state)
1849                 return 0;
1850
1851         switch (qp->state) {
1852         case QED_ROCE_QP_STATE_RESET:
1853                 switch (new_state) {
1854                 case QED_ROCE_QP_STATE_INIT:
1855                         qp->prev_wqe_size = 0;
1856                         qedr_reset_qp_hwq_info(&qp->sq);
1857                         qedr_reset_qp_hwq_info(&qp->rq);
1858                         break;
1859                 default:
1860                         status = -EINVAL;
1861                         break;
1862                 };
1863                 break;
1864         case QED_ROCE_QP_STATE_INIT:
1865                 switch (new_state) {
1866                 case QED_ROCE_QP_STATE_RTR:
1867                         /* Update doorbell (in case post_recv was
1868                          * done before move to RTR)
1869                          */
1870
1871                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1872                                 wmb();
1873                                 writel(qp->rq.db_data.raw, qp->rq.db);
1874                                 /* Make sure write takes effect */
1875                                 mmiowb();
1876                         }
1877                         break;
1878                 case QED_ROCE_QP_STATE_ERR:
1879                         break;
1880                 default:
1881                         /* Invalid state change. */
1882                         status = -EINVAL;
1883                         break;
1884                 };
1885                 break;
1886         case QED_ROCE_QP_STATE_RTR:
1887                 /* RTR->XXX */
1888                 switch (new_state) {
1889                 case QED_ROCE_QP_STATE_RTS:
1890                         break;
1891                 case QED_ROCE_QP_STATE_ERR:
1892                         break;
1893                 default:
1894                         /* Invalid state change. */
1895                         status = -EINVAL;
1896                         break;
1897                 };
1898                 break;
1899         case QED_ROCE_QP_STATE_RTS:
1900                 /* RTS->XXX */
1901                 switch (new_state) {
1902                 case QED_ROCE_QP_STATE_SQD:
1903                         break;
1904                 case QED_ROCE_QP_STATE_ERR:
1905                         break;
1906                 default:
1907                         /* Invalid state change. */
1908                         status = -EINVAL;
1909                         break;
1910                 };
1911                 break;
1912         case QED_ROCE_QP_STATE_SQD:
1913                 /* SQD->XXX */
1914                 switch (new_state) {
1915                 case QED_ROCE_QP_STATE_RTS:
1916                 case QED_ROCE_QP_STATE_ERR:
1917                         break;
1918                 default:
1919                         /* Invalid state change. */
1920                         status = -EINVAL;
1921                         break;
1922                 };
1923                 break;
1924         case QED_ROCE_QP_STATE_ERR:
1925                 /* ERR->XXX */
1926                 switch (new_state) {
1927                 case QED_ROCE_QP_STATE_RESET:
1928                         if ((qp->rq.prod != qp->rq.cons) ||
1929                             (qp->sq.prod != qp->sq.cons)) {
1930                                 DP_NOTICE(dev,
1931                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1932                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
1933                                           qp->sq.cons);
1934                                 status = -EINVAL;
1935                         }
1936                         break;
1937                 default:
1938                         status = -EINVAL;
1939                         break;
1940                 };
1941                 break;
1942         default:
1943                 status = -EINVAL;
1944                 break;
1945         };
1946
1947         return status;
1948 }
1949
1950 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1951                    int attr_mask, struct ib_udata *udata)
1952 {
1953         struct qedr_qp *qp = get_qedr_qp(ibqp);
1954         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1955         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1956         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1957         enum ib_qp_state old_qp_state, new_qp_state;
1958         int rc = 0;
1959
1960         DP_DEBUG(dev, QEDR_MSG_QP,
1961                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1962                  attr->qp_state);
1963
1964         old_qp_state = qedr_get_ibqp_state(qp->state);
1965         if (attr_mask & IB_QP_STATE)
1966                 new_qp_state = attr->qp_state;
1967         else
1968                 new_qp_state = old_qp_state;
1969
1970         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1971                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
1972                                         ibqp->qp_type, attr_mask,
1973                                         IB_LINK_LAYER_ETHERNET)) {
1974                         DP_ERR(dev,
1975                                "modify qp: invalid attribute mask=0x%x specified for\n"
1976                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1977                                attr_mask, qp->qp_id, ibqp->qp_type,
1978                                old_qp_state, new_qp_state);
1979                         rc = -EINVAL;
1980                         goto err;
1981                 }
1982         }
1983
1984         /* Translate the masks... */
1985         if (attr_mask & IB_QP_STATE) {
1986                 SET_FIELD(qp_params.modify_flags,
1987                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1988                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1989         }
1990
1991         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1992                 qp_params.sqd_async = true;
1993
1994         if (attr_mask & IB_QP_PKEY_INDEX) {
1995                 SET_FIELD(qp_params.modify_flags,
1996                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1997                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1998                         rc = -EINVAL;
1999                         goto err;
2000                 }
2001
2002                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2003         }
2004
2005         if (attr_mask & IB_QP_QKEY)
2006                 qp->qkey = attr->qkey;
2007
2008         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2009                 SET_FIELD(qp_params.modify_flags,
2010                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2011                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2012                                                   IB_ACCESS_REMOTE_READ;
2013                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2014                                                    IB_ACCESS_REMOTE_WRITE;
2015                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2016                                                IB_ACCESS_REMOTE_ATOMIC;
2017         }
2018
2019         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2020                 if (attr_mask & IB_QP_PATH_MTU) {
2021                         if (attr->path_mtu < IB_MTU_256 ||
2022                             attr->path_mtu > IB_MTU_4096) {
2023                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2024                                 rc = -EINVAL;
2025                                 goto err;
2026                         }
2027                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2028                                       ib_mtu_enum_to_int(iboe_get_mtu
2029                                                          (dev->ndev->mtu)));
2030                 }
2031
2032                 if (!qp->mtu) {
2033                         qp->mtu =
2034                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2035                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2036                 }
2037
2038                 SET_FIELD(qp_params.modify_flags,
2039                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2040
2041                 qp_params.traffic_class_tos = grh->traffic_class;
2042                 qp_params.flow_label = grh->flow_label;
2043                 qp_params.hop_limit_ttl = grh->hop_limit;
2044
2045                 qp->sgid_idx = grh->sgid_index;
2046
2047                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2048                 if (rc) {
2049                         DP_ERR(dev,
2050                                "modify qp: problems with GID index %d (rc=%d)\n",
2051                                grh->sgid_index, rc);
2052                         return rc;
2053                 }
2054
2055                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2056                                    qp_params.remote_mac_addr);
2057                 if (rc)
2058                         return rc;
2059
2060                 qp_params.use_local_mac = true;
2061                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2062
2063                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2064                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2065                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2066                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2067                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2068                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2069                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2070                          qp_params.remote_mac_addr);
2071
2072                 qp_params.mtu = qp->mtu;
2073                 qp_params.lb_indication = false;
2074         }
2075
2076         if (!qp_params.mtu) {
2077                 /* Stay with current MTU */
2078                 if (qp->mtu)
2079                         qp_params.mtu = qp->mtu;
2080                 else
2081                         qp_params.mtu =
2082                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2083         }
2084
2085         if (attr_mask & IB_QP_TIMEOUT) {
2086                 SET_FIELD(qp_params.modify_flags,
2087                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2088
2089                 qp_params.ack_timeout = attr->timeout;
2090                 if (attr->timeout) {
2091                         u32 temp;
2092
2093                         temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
2094                         /* FW requires [msec] */
2095                         qp_params.ack_timeout = temp;
2096                 } else {
2097                         /* Infinite */
2098                         qp_params.ack_timeout = 0;
2099                 }
2100         }
2101         if (attr_mask & IB_QP_RETRY_CNT) {
2102                 SET_FIELD(qp_params.modify_flags,
2103                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2104                 qp_params.retry_cnt = attr->retry_cnt;
2105         }
2106
2107         if (attr_mask & IB_QP_RNR_RETRY) {
2108                 SET_FIELD(qp_params.modify_flags,
2109                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2110                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2111         }
2112
2113         if (attr_mask & IB_QP_RQ_PSN) {
2114                 SET_FIELD(qp_params.modify_flags,
2115                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2116                 qp_params.rq_psn = attr->rq_psn;
2117                 qp->rq_psn = attr->rq_psn;
2118         }
2119
2120         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2121                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2122                         rc = -EINVAL;
2123                         DP_ERR(dev,
2124                                "unsupported max_rd_atomic=%d, supported=%d\n",
2125                                attr->max_rd_atomic,
2126                                dev->attr.max_qp_req_rd_atomic_resc);
2127                         goto err;
2128                 }
2129
2130                 SET_FIELD(qp_params.modify_flags,
2131                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2132                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2133         }
2134
2135         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2136                 SET_FIELD(qp_params.modify_flags,
2137                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2138                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2139         }
2140
2141         if (attr_mask & IB_QP_SQ_PSN) {
2142                 SET_FIELD(qp_params.modify_flags,
2143                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2144                 qp_params.sq_psn = attr->sq_psn;
2145                 qp->sq_psn = attr->sq_psn;
2146         }
2147
2148         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2149                 if (attr->max_dest_rd_atomic >
2150                     dev->attr.max_qp_resp_rd_atomic_resc) {
2151                         DP_ERR(dev,
2152                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2153                                attr->max_dest_rd_atomic,
2154                                dev->attr.max_qp_resp_rd_atomic_resc);
2155
2156                         rc = -EINVAL;
2157                         goto err;
2158                 }
2159
2160                 SET_FIELD(qp_params.modify_flags,
2161                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2162                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2163         }
2164
2165         if (attr_mask & IB_QP_DEST_QPN) {
2166                 SET_FIELD(qp_params.modify_flags,
2167                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2168
2169                 qp_params.dest_qp = attr->dest_qp_num;
2170                 qp->dest_qp_num = attr->dest_qp_num;
2171         }
2172
2173         if (qp->qp_type != IB_QPT_GSI)
2174                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2175                                               qp->qed_qp, &qp_params);
2176
2177         if (attr_mask & IB_QP_STATE) {
2178                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2179                         rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
2180                 qp->state = qp_params.new_state;
2181         }
2182
2183 err:
2184         return rc;
2185 }
2186
2187 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2188 {
2189         int ib_qp_acc_flags = 0;
2190
2191         if (params->incoming_rdma_write_en)
2192                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2193         if (params->incoming_rdma_read_en)
2194                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2195         if (params->incoming_atomic_en)
2196                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2197         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2198         return ib_qp_acc_flags;
2199 }
2200
2201 int qedr_query_qp(struct ib_qp *ibqp,
2202                   struct ib_qp_attr *qp_attr,
2203                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2204 {
2205         struct qed_rdma_query_qp_out_params params;
2206         struct qedr_qp *qp = get_qedr_qp(ibqp);
2207         struct qedr_dev *dev = qp->dev;
2208         int rc = 0;
2209
2210         memset(&params, 0, sizeof(params));
2211
2212         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2213         if (rc)
2214                 goto err;
2215
2216         memset(qp_attr, 0, sizeof(*qp_attr));
2217         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2218
2219         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2220         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2221         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2222         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2223         qp_attr->rq_psn = params.rq_psn;
2224         qp_attr->sq_psn = params.sq_psn;
2225         qp_attr->dest_qp_num = params.dest_qp;
2226
2227         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2228
2229         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2230         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2231         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2232         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2233         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2234         qp_init_attr->cap = qp_attr->cap;
2235
2236         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2237         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2238                         params.flow_label, qp->sgid_idx,
2239                         params.hop_limit_ttl, params.traffic_class_tos);
2240         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2241         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2242         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2243         qp_attr->timeout = params.timeout;
2244         qp_attr->rnr_retry = params.rnr_retry;
2245         qp_attr->retry_cnt = params.retry_cnt;
2246         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2247         qp_attr->pkey_index = params.pkey_index;
2248         qp_attr->port_num = 1;
2249         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2250         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2251         qp_attr->alt_pkey_index = 0;
2252         qp_attr->alt_port_num = 0;
2253         qp_attr->alt_timeout = 0;
2254         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2255
2256         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2257         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2258         qp_attr->max_rd_atomic = params.max_rd_atomic;
2259         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2260
2261         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2262                  qp_attr->cap.max_inline_data);
2263
2264 err:
2265         return rc;
2266 }
2267
2268 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2269 {
2270         int rc = 0;
2271
2272         if (qp->qp_type != IB_QPT_GSI) {
2273                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2274                 if (rc)
2275                         return rc;
2276         }
2277
2278         if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2279                 qedr_cleanup_user(dev, qp);
2280         else
2281                 qedr_cleanup_kernel(dev, qp);
2282
2283         return 0;
2284 }
2285
2286 int qedr_destroy_qp(struct ib_qp *ibqp)
2287 {
2288         struct qedr_qp *qp = get_qedr_qp(ibqp);
2289         struct qedr_dev *dev = qp->dev;
2290         struct ib_qp_attr attr;
2291         int attr_mask = 0;
2292         int rc = 0;
2293
2294         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2295                  qp, qp->qp_type);
2296
2297         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2298                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2299                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2300                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2301
2302                         attr.qp_state = IB_QPS_ERR;
2303                         attr_mask |= IB_QP_STATE;
2304
2305                         /* Change the QP state to ERROR */
2306                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2307                 }
2308         } else {
2309                 /* Wait for the connect/accept to complete */
2310                 if (qp->ep) {
2311                         int wait_count = 1;
2312
2313                         while (qp->ep->during_connect) {
2314                                 DP_DEBUG(dev, QEDR_MSG_QP,
2315                                          "Still in during connect/accept\n");
2316
2317                                 msleep(100);
2318                                 if (wait_count++ > 200) {
2319                                         DP_NOTICE(dev,
2320                                                   "during connect timeout\n");
2321                                         break;
2322                                 }
2323                         }
2324                 }
2325         }
2326
2327         if (qp->qp_type == IB_QPT_GSI)
2328                 qedr_destroy_gsi_qp(dev);
2329
2330         qedr_free_qp_resources(dev, qp);
2331
2332         if (atomic_dec_and_test(&qp->refcnt)) {
2333                 qedr_idr_remove(dev, qp->qp_id);
2334                 kfree(qp);
2335         }
2336         return rc;
2337 }
2338
2339 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2340                              struct ib_udata *udata)
2341 {
2342         struct qedr_ah *ah;
2343
2344         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2345         if (!ah)
2346                 return ERR_PTR(-ENOMEM);
2347
2348         ah->attr = *attr;
2349
2350         return &ah->ibah;
2351 }
2352
2353 int qedr_destroy_ah(struct ib_ah *ibah)
2354 {
2355         struct qedr_ah *ah = get_qedr_ah(ibah);
2356
2357         kfree(ah);
2358         return 0;
2359 }
2360
2361 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2362 {
2363         struct qedr_pbl *pbl, *tmp;
2364
2365         if (info->pbl_table)
2366                 list_add_tail(&info->pbl_table->list_entry,
2367                               &info->free_pbl_list);
2368
2369         if (!list_empty(&info->inuse_pbl_list))
2370                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2371
2372         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2373                 list_del(&pbl->list_entry);
2374                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2375         }
2376 }
2377
2378 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2379                         size_t page_list_len, bool two_layered)
2380 {
2381         struct qedr_pbl *tmp;
2382         int rc;
2383
2384         INIT_LIST_HEAD(&info->free_pbl_list);
2385         INIT_LIST_HEAD(&info->inuse_pbl_list);
2386
2387         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2388                                   page_list_len, two_layered);
2389         if (rc)
2390                 goto done;
2391
2392         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2393         if (IS_ERR(info->pbl_table)) {
2394                 rc = PTR_ERR(info->pbl_table);
2395                 goto done;
2396         }
2397
2398         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2399                  &info->pbl_table->pa);
2400
2401         /* in usual case we use 2 PBLs, so we add one to free
2402          * list and allocating another one
2403          */
2404         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2405         if (IS_ERR(tmp)) {
2406                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2407                 goto done;
2408         }
2409
2410         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2411
2412         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2413
2414 done:
2415         if (rc)
2416                 free_mr_info(dev, info);
2417
2418         return rc;
2419 }
2420
2421 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2422                                u64 usr_addr, int acc, struct ib_udata *udata)
2423 {
2424         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2425         struct qedr_mr *mr;
2426         struct qedr_pd *pd;
2427         int rc = -ENOMEM;
2428
2429         pd = get_qedr_pd(ibpd);
2430         DP_DEBUG(dev, QEDR_MSG_MR,
2431                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2432                  pd->pd_id, start, len, usr_addr, acc);
2433
2434         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2435                 return ERR_PTR(-EINVAL);
2436
2437         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2438         if (!mr)
2439                 return ERR_PTR(rc);
2440
2441         mr->type = QEDR_MR_USER;
2442
2443         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2444         if (IS_ERR(mr->umem)) {
2445                 rc = -EFAULT;
2446                 goto err0;
2447         }
2448
2449         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2450         if (rc)
2451                 goto err1;
2452
2453         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2454                            &mr->info.pbl_info, mr->umem->page_shift);
2455
2456         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2457         if (rc) {
2458                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2459                 goto err1;
2460         }
2461
2462         /* Index only, 18 bit long, lkey = itid << 8 | key */
2463         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2464         mr->hw_mr.key = 0;
2465         mr->hw_mr.pd = pd->pd_id;
2466         mr->hw_mr.local_read = 1;
2467         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2468         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2469         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2470         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2471         mr->hw_mr.mw_bind = false;
2472         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2473         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2474         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2475         mr->hw_mr.page_size_log = mr->umem->page_shift;
2476         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2477         mr->hw_mr.length = len;
2478         mr->hw_mr.vaddr = usr_addr;
2479         mr->hw_mr.zbva = false;
2480         mr->hw_mr.phy_mr = false;
2481         mr->hw_mr.dma_mr = false;
2482
2483         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2484         if (rc) {
2485                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2486                 goto err2;
2487         }
2488
2489         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2490         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2491             mr->hw_mr.remote_atomic)
2492                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2493
2494         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2495                  mr->ibmr.lkey);
2496         return &mr->ibmr;
2497
2498 err2:
2499         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2500 err1:
2501         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2502 err0:
2503         kfree(mr);
2504         return ERR_PTR(rc);
2505 }
2506
2507 int qedr_dereg_mr(struct ib_mr *ib_mr)
2508 {
2509         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2510         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2511         int rc = 0;
2512
2513         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2514         if (rc)
2515                 return rc;
2516
2517         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2518
2519         if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2520                 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2521
2522         /* it could be user registered memory. */
2523         if (mr->umem)
2524                 ib_umem_release(mr->umem);
2525
2526         kfree(mr);
2527
2528         return rc;
2529 }
2530
2531 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2532                                        int max_page_list_len)
2533 {
2534         struct qedr_pd *pd = get_qedr_pd(ibpd);
2535         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2536         struct qedr_mr *mr;
2537         int rc = -ENOMEM;
2538
2539         DP_DEBUG(dev, QEDR_MSG_MR,
2540                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2541                  max_page_list_len);
2542
2543         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2544         if (!mr)
2545                 return ERR_PTR(rc);
2546
2547         mr->dev = dev;
2548         mr->type = QEDR_MR_FRMR;
2549
2550         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2551         if (rc)
2552                 goto err0;
2553
2554         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2555         if (rc) {
2556                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2557                 goto err0;
2558         }
2559
2560         /* Index only, 18 bit long, lkey = itid << 8 | key */
2561         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2562         mr->hw_mr.key = 0;
2563         mr->hw_mr.pd = pd->pd_id;
2564         mr->hw_mr.local_read = 1;
2565         mr->hw_mr.local_write = 0;
2566         mr->hw_mr.remote_read = 0;
2567         mr->hw_mr.remote_write = 0;
2568         mr->hw_mr.remote_atomic = 0;
2569         mr->hw_mr.mw_bind = false;
2570         mr->hw_mr.pbl_ptr = 0;
2571         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2572         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2573         mr->hw_mr.fbo = 0;
2574         mr->hw_mr.length = 0;
2575         mr->hw_mr.vaddr = 0;
2576         mr->hw_mr.zbva = false;
2577         mr->hw_mr.phy_mr = true;
2578         mr->hw_mr.dma_mr = false;
2579
2580         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2581         if (rc) {
2582                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2583                 goto err1;
2584         }
2585
2586         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2587         mr->ibmr.rkey = mr->ibmr.lkey;
2588
2589         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2590         return mr;
2591
2592 err1:
2593         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2594 err0:
2595         kfree(mr);
2596         return ERR_PTR(rc);
2597 }
2598
2599 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2600                             enum ib_mr_type mr_type, u32 max_num_sg)
2601 {
2602         struct qedr_mr *mr;
2603
2604         if (mr_type != IB_MR_TYPE_MEM_REG)
2605                 return ERR_PTR(-EINVAL);
2606
2607         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2608
2609         if (IS_ERR(mr))
2610                 return ERR_PTR(-EINVAL);
2611
2612         return &mr->ibmr;
2613 }
2614
2615 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2616 {
2617         struct qedr_mr *mr = get_qedr_mr(ibmr);
2618         struct qedr_pbl *pbl_table;
2619         struct regpair *pbe;
2620         u32 pbes_in_page;
2621
2622         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2623                 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2624                 return -ENOMEM;
2625         }
2626
2627         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2628                  mr->npages, addr);
2629
2630         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2631         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2632         pbe = (struct regpair *)pbl_table->va;
2633         pbe +=  mr->npages % pbes_in_page;
2634         pbe->lo = cpu_to_le32((u32)addr);
2635         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2636
2637         mr->npages++;
2638
2639         return 0;
2640 }
2641
2642 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2643 {
2644         int work = info->completed - info->completed_handled - 1;
2645
2646         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2647         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2648                 struct qedr_pbl *pbl;
2649
2650                 /* Free all the page list that are possible to be freed
2651                  * (all the ones that were invalidated), under the assumption
2652                  * that if an FMR was completed successfully that means that
2653                  * if there was an invalidate operation before it also ended
2654                  */
2655                 pbl = list_first_entry(&info->inuse_pbl_list,
2656                                        struct qedr_pbl, list_entry);
2657                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2658                 info->completed_handled++;
2659         }
2660 }
2661
2662 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2663                    int sg_nents, unsigned int *sg_offset)
2664 {
2665         struct qedr_mr *mr = get_qedr_mr(ibmr);
2666
2667         mr->npages = 0;
2668
2669         handle_completed_mrs(mr->dev, &mr->info);
2670         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2671 }
2672
2673 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2674 {
2675         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2676         struct qedr_pd *pd = get_qedr_pd(ibpd);
2677         struct qedr_mr *mr;
2678         int rc;
2679
2680         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2681         if (!mr)
2682                 return ERR_PTR(-ENOMEM);
2683
2684         mr->type = QEDR_MR_DMA;
2685
2686         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2687         if (rc) {
2688                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2689                 goto err1;
2690         }
2691
2692         /* index only, 18 bit long, lkey = itid << 8 | key */
2693         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2694         mr->hw_mr.pd = pd->pd_id;
2695         mr->hw_mr.local_read = 1;
2696         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2697         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2698         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2699         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2700         mr->hw_mr.dma_mr = true;
2701
2702         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2703         if (rc) {
2704                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2705                 goto err2;
2706         }
2707
2708         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2709         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2710             mr->hw_mr.remote_atomic)
2711                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2712
2713         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2714         return &mr->ibmr;
2715
2716 err2:
2717         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2718 err1:
2719         kfree(mr);
2720         return ERR_PTR(rc);
2721 }
2722
2723 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2724 {
2725         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2726 }
2727
2728 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2729 {
2730         int i, len = 0;
2731
2732         for (i = 0; i < num_sge; i++)
2733                 len += sg_list[i].length;
2734
2735         return len;
2736 }
2737
2738 static void swap_wqe_data64(u64 *p)
2739 {
2740         int i;
2741
2742         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2743                 *p = cpu_to_be64(cpu_to_le64(*p));
2744 }
2745
2746 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2747                                        struct qedr_qp *qp, u8 *wqe_size,
2748                                        struct ib_send_wr *wr,
2749                                        struct ib_send_wr **bad_wr, u8 *bits,
2750                                        u8 bit)
2751 {
2752         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2753         char *seg_prt, *wqe;
2754         int i, seg_siz;
2755
2756         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2757                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2758                 *bad_wr = wr;
2759                 return 0;
2760         }
2761
2762         if (!data_size)
2763                 return data_size;
2764
2765         *bits |= bit;
2766
2767         seg_prt = NULL;
2768         wqe = NULL;
2769         seg_siz = 0;
2770
2771         /* Copy data inline */
2772         for (i = 0; i < wr->num_sge; i++) {
2773                 u32 len = wr->sg_list[i].length;
2774                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2775
2776                 while (len > 0) {
2777                         u32 cur;
2778
2779                         /* New segment required */
2780                         if (!seg_siz) {
2781                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2782                                 seg_prt = wqe;
2783                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
2784                                 (*wqe_size)++;
2785                         }
2786
2787                         /* Calculate currently allowed length */
2788                         cur = min_t(u32, len, seg_siz);
2789                         memcpy(seg_prt, src, cur);
2790
2791                         /* Update segment variables */
2792                         seg_prt += cur;
2793                         seg_siz -= cur;
2794
2795                         /* Update sge variables */
2796                         src += cur;
2797                         len -= cur;
2798
2799                         /* Swap fully-completed segments */
2800                         if (!seg_siz)
2801                                 swap_wqe_data64((u64 *)wqe);
2802                 }
2803         }
2804
2805         /* swap last not completed segment */
2806         if (seg_siz)
2807                 swap_wqe_data64((u64 *)wqe);
2808
2809         return data_size;
2810 }
2811
2812 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
2813         do {                                                    \
2814                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2815                 (sge)->length = cpu_to_le32(vlength);           \
2816                 (sge)->flags = cpu_to_le32(vflags);             \
2817         } while (0)
2818
2819 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
2820         do {                                                    \
2821                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
2822                 (hdr)->num_sges = num_sge;                      \
2823         } while (0)
2824
2825 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
2826         do {                                                    \
2827                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
2828                 (sge)->length = cpu_to_le32(vlength);           \
2829                 (sge)->l_key = cpu_to_le32(vlkey);              \
2830         } while (0)
2831
2832 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2833                                 struct ib_send_wr *wr)
2834 {
2835         u32 data_size = 0;
2836         int i;
2837
2838         for (i = 0; i < wr->num_sge; i++) {
2839                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2840
2841                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2842                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2843                 sge->length = cpu_to_le32(wr->sg_list[i].length);
2844                 data_size += wr->sg_list[i].length;
2845         }
2846
2847         if (wqe_size)
2848                 *wqe_size += wr->num_sge;
2849
2850         return data_size;
2851 }
2852
2853 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2854                                      struct qedr_qp *qp,
2855                                      struct rdma_sq_rdma_wqe_1st *rwqe,
2856                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
2857                                      struct ib_send_wr *wr,
2858                                      struct ib_send_wr **bad_wr)
2859 {
2860         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2861         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2862
2863         if (wr->send_flags & IB_SEND_INLINE &&
2864             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2865              wr->opcode == IB_WR_RDMA_WRITE)) {
2866                 u8 flags = 0;
2867
2868                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2869                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2870                                                    bad_wr, &rwqe->flags, flags);
2871         }
2872
2873         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2874 }
2875
2876 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2877                                      struct qedr_qp *qp,
2878                                      struct rdma_sq_send_wqe_1st *swqe,
2879                                      struct rdma_sq_send_wqe_2st *swqe2,
2880                                      struct ib_send_wr *wr,
2881                                      struct ib_send_wr **bad_wr)
2882 {
2883         memset(swqe2, 0, sizeof(*swqe2));
2884         if (wr->send_flags & IB_SEND_INLINE) {
2885                 u8 flags = 0;
2886
2887                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2888                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2889                                                    bad_wr, &swqe->flags, flags);
2890         }
2891
2892         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2893 }
2894
2895 static int qedr_prepare_reg(struct qedr_qp *qp,
2896                             struct rdma_sq_fmr_wqe_1st *fwqe1,
2897                             struct ib_reg_wr *wr)
2898 {
2899         struct qedr_mr *mr = get_qedr_mr(wr->mr);
2900         struct rdma_sq_fmr_wqe_2nd *fwqe2;
2901
2902         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2903         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2904         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2905         fwqe1->l_key = wr->key;
2906
2907         fwqe2->access_ctrl = 0;
2908
2909         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2910                    !!(wr->access & IB_ACCESS_REMOTE_READ));
2911         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2912                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2913         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2914                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2915         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2916         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2917                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2918         fwqe2->fmr_ctrl = 0;
2919
2920         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2921                    ilog2(mr->ibmr.page_size) - 12);
2922
2923         fwqe2->length_hi = 0;
2924         fwqe2->length_lo = mr->ibmr.length;
2925         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2926         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2927
2928         qp->wqe_wr_id[qp->sq.prod].mr = mr;
2929
2930         return 0;
2931 }
2932
2933 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2934 {
2935         switch (opcode) {
2936         case IB_WR_RDMA_WRITE:
2937         case IB_WR_RDMA_WRITE_WITH_IMM:
2938                 return IB_WC_RDMA_WRITE;
2939         case IB_WR_SEND_WITH_IMM:
2940         case IB_WR_SEND:
2941         case IB_WR_SEND_WITH_INV:
2942                 return IB_WC_SEND;
2943         case IB_WR_RDMA_READ:
2944         case IB_WR_RDMA_READ_WITH_INV:
2945                 return IB_WC_RDMA_READ;
2946         case IB_WR_ATOMIC_CMP_AND_SWP:
2947                 return IB_WC_COMP_SWAP;
2948         case IB_WR_ATOMIC_FETCH_AND_ADD:
2949                 return IB_WC_FETCH_ADD;
2950         case IB_WR_REG_MR:
2951                 return IB_WC_REG_MR;
2952         case IB_WR_LOCAL_INV:
2953                 return IB_WC_LOCAL_INV;
2954         default:
2955                 return IB_WC_SEND;
2956         }
2957 }
2958
2959 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2960 {
2961         int wq_is_full, err_wr, pbl_is_full;
2962         struct qedr_dev *dev = qp->dev;
2963
2964         /* prevent SQ overflow and/or processing of a bad WR */
2965         err_wr = wr->num_sge > qp->sq.max_sges;
2966         wq_is_full = qedr_wq_is_full(&qp->sq);
2967         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2968                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2969         if (wq_is_full || err_wr || pbl_is_full) {
2970                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2971                         DP_ERR(dev,
2972                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2973                                qp);
2974                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2975                 }
2976
2977                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2978                         DP_ERR(dev,
2979                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2980                                qp);
2981                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2982                 }
2983
2984                 if (pbl_is_full &&
2985                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2986                         DP_ERR(dev,
2987                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2988                                qp);
2989                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2990                 }
2991                 return false;
2992         }
2993         return true;
2994 }
2995
2996 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2997                      struct ib_send_wr **bad_wr)
2998 {
2999         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3000         struct qedr_qp *qp = get_qedr_qp(ibqp);
3001         struct rdma_sq_atomic_wqe_1st *awqe1;
3002         struct rdma_sq_atomic_wqe_2nd *awqe2;
3003         struct rdma_sq_atomic_wqe_3rd *awqe3;
3004         struct rdma_sq_send_wqe_2st *swqe2;
3005         struct rdma_sq_local_inv_wqe *iwqe;
3006         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3007         struct rdma_sq_send_wqe_1st *swqe;
3008         struct rdma_sq_rdma_wqe_1st *rwqe;
3009         struct rdma_sq_fmr_wqe_1st *fwqe1;
3010         struct rdma_sq_common_wqe *wqe;
3011         u32 length;
3012         int rc = 0;
3013         bool comp;
3014
3015         if (!qedr_can_post_send(qp, wr)) {
3016                 *bad_wr = wr;
3017                 return -ENOMEM;
3018         }
3019
3020         wqe = qed_chain_produce(&qp->sq.pbl);
3021         qp->wqe_wr_id[qp->sq.prod].signaled =
3022                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3023
3024         wqe->flags = 0;
3025         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3026                    !!(wr->send_flags & IB_SEND_SOLICITED));
3027         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3028         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3029         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3030                    !!(wr->send_flags & IB_SEND_FENCE));
3031         wqe->prev_wqe_size = qp->prev_wqe_size;
3032
3033         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3034
3035         switch (wr->opcode) {
3036         case IB_WR_SEND_WITH_IMM:
3037                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3038                         rc = -EINVAL;
3039                         *bad_wr = wr;
3040                         break;
3041                 }
3042                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3043                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3044                 swqe->wqe_size = 2;
3045                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3046
3047                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3048                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3049                                                    wr, bad_wr);
3050                 swqe->length = cpu_to_le32(length);
3051                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3052                 qp->prev_wqe_size = swqe->wqe_size;
3053                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3054                 break;
3055         case IB_WR_SEND:
3056                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3057                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3058
3059                 swqe->wqe_size = 2;
3060                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3061                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3062                                                    wr, bad_wr);
3063                 swqe->length = cpu_to_le32(length);
3064                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3065                 qp->prev_wqe_size = swqe->wqe_size;
3066                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3067                 break;
3068         case IB_WR_SEND_WITH_INV:
3069                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3070                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3071                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3072                 swqe->wqe_size = 2;
3073                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3074                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3075                                                    wr, bad_wr);
3076                 swqe->length = cpu_to_le32(length);
3077                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3078                 qp->prev_wqe_size = swqe->wqe_size;
3079                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3080                 break;
3081
3082         case IB_WR_RDMA_WRITE_WITH_IMM:
3083                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3084                         rc = -EINVAL;
3085                         *bad_wr = wr;
3086                         break;
3087                 }
3088                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3089                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3090
3091                 rwqe->wqe_size = 2;
3092                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3093                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3094                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3095                                                    wr, bad_wr);
3096                 rwqe->length = cpu_to_le32(length);
3097                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3098                 qp->prev_wqe_size = rwqe->wqe_size;
3099                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3100                 break;
3101         case IB_WR_RDMA_WRITE:
3102                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3103                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3104
3105                 rwqe->wqe_size = 2;
3106                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3107                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3108                                                    wr, bad_wr);
3109                 rwqe->length = cpu_to_le32(length);
3110                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3111                 qp->prev_wqe_size = rwqe->wqe_size;
3112                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3113                 break;
3114         case IB_WR_RDMA_READ_WITH_INV:
3115                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3116                 /* fallthrough -- same is identical to RDMA READ */
3117
3118         case IB_WR_RDMA_READ:
3119                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3120                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3121
3122                 rwqe->wqe_size = 2;
3123                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3124                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3125                                                    wr, bad_wr);
3126                 rwqe->length = cpu_to_le32(length);
3127                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3128                 qp->prev_wqe_size = rwqe->wqe_size;
3129                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3130                 break;
3131
3132         case IB_WR_ATOMIC_CMP_AND_SWP:
3133         case IB_WR_ATOMIC_FETCH_AND_ADD:
3134                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3135                 awqe1->wqe_size = 4;
3136
3137                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3138                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3139                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3140
3141                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3142
3143                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3144                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3145                         DMA_REGPAIR_LE(awqe3->swap_data,
3146                                        atomic_wr(wr)->compare_add);
3147                 } else {
3148                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3149                         DMA_REGPAIR_LE(awqe3->swap_data,
3150                                        atomic_wr(wr)->swap);
3151                         DMA_REGPAIR_LE(awqe3->cmp_data,
3152                                        atomic_wr(wr)->compare_add);
3153                 }
3154
3155                 qedr_prepare_sq_sges(qp, NULL, wr);
3156
3157                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3158                 qp->prev_wqe_size = awqe1->wqe_size;
3159                 break;
3160
3161         case IB_WR_LOCAL_INV:
3162                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3163                 iwqe->wqe_size = 1;
3164
3165                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3166                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3167                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3168                 qp->prev_wqe_size = iwqe->wqe_size;
3169                 break;
3170         case IB_WR_REG_MR:
3171                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3172                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3173                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3174                 fwqe1->wqe_size = 2;
3175
3176                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3177                 if (rc) {
3178                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3179                         *bad_wr = wr;
3180                         break;
3181                 }
3182
3183                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3184                 qp->prev_wqe_size = fwqe1->wqe_size;
3185                 break;
3186         default:
3187                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3188                 rc = -EINVAL;
3189                 *bad_wr = wr;
3190                 break;
3191         }
3192
3193         if (*bad_wr) {
3194                 u16 value;
3195
3196                 /* Restore prod to its position before
3197                  * this WR was processed
3198                  */
3199                 value = le16_to_cpu(qp->sq.db_data.data.value);
3200                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3201
3202                 /* Restore prev_wqe_size */
3203                 qp->prev_wqe_size = wqe->prev_wqe_size;
3204                 rc = -EINVAL;
3205                 DP_ERR(dev, "POST SEND FAILED\n");
3206         }
3207
3208         return rc;