Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
[muen/linux.git] / drivers / infiniband / sw / rxe / rxe_req.c
1 /*
2  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #include <linux/skbuff.h>
35 #include <crypto/hash.h>
36
37 #include "rxe.h"
38 #include "rxe_loc.h"
39 #include "rxe_queue.h"
40
41 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
42                        u32 opcode);
43
44 static inline void retry_first_write_send(struct rxe_qp *qp,
45                                           struct rxe_send_wqe *wqe,
46                                           unsigned int mask, int npsn)
47 {
48         int i;
49
50         for (i = 0; i < npsn; i++) {
51                 int to_send = (wqe->dma.resid > qp->mtu) ?
52                                 qp->mtu : wqe->dma.resid;
53
54                 qp->req.opcode = next_opcode(qp, wqe,
55                                              wqe->wr.opcode);
56
57                 if (wqe->wr.send_flags & IB_SEND_INLINE) {
58                         wqe->dma.resid -= to_send;
59                         wqe->dma.sge_offset += to_send;
60                 } else {
61                         advance_dma_data(&wqe->dma, to_send);
62                 }
63                 if (mask & WR_WRITE_MASK)
64                         wqe->iova += qp->mtu;
65         }
66 }
67
68 static void req_retry(struct rxe_qp *qp)
69 {
70         struct rxe_send_wqe *wqe;
71         unsigned int wqe_index;
72         unsigned int mask;
73         int npsn;
74         int first = 1;
75
76         wqe = queue_head(qp->sq.queue);
77         npsn = (qp->comp.psn - wqe->first_psn) & BTH_PSN_MASK;
78
79         qp->req.wqe_index       = consumer_index(qp->sq.queue);
80         qp->req.psn             = qp->comp.psn;
81         qp->req.opcode          = -1;
82
83         for (wqe_index = consumer_index(qp->sq.queue);
84                 wqe_index != producer_index(qp->sq.queue);
85                 wqe_index = next_index(qp->sq.queue, wqe_index)) {
86                 wqe = addr_from_index(qp->sq.queue, wqe_index);
87                 mask = wr_opcode_mask(wqe->wr.opcode, qp);
88
89                 if (wqe->state == wqe_state_posted)
90                         break;
91
92                 if (wqe->state == wqe_state_done)
93                         continue;
94
95                 wqe->iova = (mask & WR_ATOMIC_MASK) ?
96                              wqe->wr.wr.atomic.remote_addr :
97                              (mask & WR_READ_OR_WRITE_MASK) ?
98                              wqe->wr.wr.rdma.remote_addr :
99                              0;
100
101                 if (!first || (mask & WR_READ_MASK) == 0) {
102                         wqe->dma.resid = wqe->dma.length;
103                         wqe->dma.cur_sge = 0;
104                         wqe->dma.sge_offset = 0;
105                 }
106
107                 if (first) {
108                         first = 0;
109
110                         if (mask & WR_WRITE_OR_SEND_MASK)
111                                 retry_first_write_send(qp, wqe, mask, npsn);
112
113                         if (mask & WR_READ_MASK)
114                                 wqe->iova += npsn * qp->mtu;
115                 }
116
117                 wqe->state = wqe_state_posted;
118         }
119 }
120
121 void rnr_nak_timer(struct timer_list *t)
122 {
123         struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
124
125         pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
126         rxe_run_task(&qp->req.task, 1);
127 }
128
129 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
130 {
131         struct rxe_send_wqe *wqe = queue_head(qp->sq.queue);
132         unsigned long flags;
133
134         if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
135                 /* check to see if we are drained;
136                  * state_lock used by requester and completer
137                  */
138                 spin_lock_irqsave(&qp->state_lock, flags);
139                 do {
140                         if (qp->req.state != QP_STATE_DRAIN) {
141                                 /* comp just finished */
142                                 spin_unlock_irqrestore(&qp->state_lock,
143                                                        flags);
144                                 break;
145                         }
146
147                         if (wqe && ((qp->req.wqe_index !=
148                                 consumer_index(qp->sq.queue)) ||
149                                 (wqe->state != wqe_state_posted))) {
150                                 /* comp not done yet */
151                                 spin_unlock_irqrestore(&qp->state_lock,
152                                                        flags);
153                                 break;
154                         }
155
156                         qp->req.state = QP_STATE_DRAINED;
157                         spin_unlock_irqrestore(&qp->state_lock, flags);
158
159                         if (qp->ibqp.event_handler) {
160                                 struct ib_event ev;
161
162                                 ev.device = qp->ibqp.device;
163                                 ev.element.qp = &qp->ibqp;
164                                 ev.event = IB_EVENT_SQ_DRAINED;
165                                 qp->ibqp.event_handler(&ev,
166                                         qp->ibqp.qp_context);
167                         }
168                 } while (0);
169         }
170
171         if (qp->req.wqe_index == producer_index(qp->sq.queue))
172                 return NULL;
173
174         wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index);
175
176         if (unlikely((qp->req.state == QP_STATE_DRAIN ||
177                       qp->req.state == QP_STATE_DRAINED) &&
178                      (wqe->state != wqe_state_processing)))
179                 return NULL;
180
181         if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) &&
182                      (qp->req.wqe_index != consumer_index(qp->sq.queue)))) {
183                 qp->req.wait_fence = 1;
184                 return NULL;
185         }
186
187         wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
188         return wqe;
189 }
190
191 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)
192 {
193         switch (opcode) {
194         case IB_WR_RDMA_WRITE:
195                 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
196                     qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
197                         return fits ?
198                                 IB_OPCODE_RC_RDMA_WRITE_LAST :
199                                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
200                 else
201                         return fits ?
202                                 IB_OPCODE_RC_RDMA_WRITE_ONLY :
203                                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
204
205         case IB_WR_RDMA_WRITE_WITH_IMM:
206                 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
207                     qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
208                         return fits ?
209                                 IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
210                                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
211                 else
212                         return fits ?
213                                 IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
214                                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
215
216         case IB_WR_SEND:
217                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
218                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
219                         return fits ?
220                                 IB_OPCODE_RC_SEND_LAST :
221                                 IB_OPCODE_RC_SEND_MIDDLE;
222                 else
223                         return fits ?
224                                 IB_OPCODE_RC_SEND_ONLY :
225                                 IB_OPCODE_RC_SEND_FIRST;
226
227         case IB_WR_SEND_WITH_IMM:
228                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
229                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
230                         return fits ?
231                                 IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE :
232                                 IB_OPCODE_RC_SEND_MIDDLE;
233                 else
234                         return fits ?
235                                 IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
236                                 IB_OPCODE_RC_SEND_FIRST;
237
238         case IB_WR_RDMA_READ:
239                 return IB_OPCODE_RC_RDMA_READ_REQUEST;
240
241         case IB_WR_ATOMIC_CMP_AND_SWP:
242                 return IB_OPCODE_RC_COMPARE_SWAP;
243
244         case IB_WR_ATOMIC_FETCH_AND_ADD:
245                 return IB_OPCODE_RC_FETCH_ADD;
246
247         case IB_WR_SEND_WITH_INV:
248                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
249                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
250                         return fits ? IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE :
251                                 IB_OPCODE_RC_SEND_MIDDLE;
252                 else
253                         return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
254                                 IB_OPCODE_RC_SEND_FIRST;
255         case IB_WR_REG_MR:
256         case IB_WR_LOCAL_INV:
257                 return opcode;
258         }
259
260         return -EINVAL;
261 }
262
263 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)
264 {
265         switch (opcode) {
266         case IB_WR_RDMA_WRITE:
267                 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
268                     qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
269                         return fits ?
270                                 IB_OPCODE_UC_RDMA_WRITE_LAST :
271                                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
272                 else
273                         return fits ?
274                                 IB_OPCODE_UC_RDMA_WRITE_ONLY :
275                                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
276
277         case IB_WR_RDMA_WRITE_WITH_IMM:
278                 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
279                     qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
280                         return fits ?
281                                 IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
282                                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
283                 else
284                         return fits ?
285                                 IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
286                                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
287
288         case IB_WR_SEND:
289                 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
290                     qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
291                         return fits ?
292                                 IB_OPCODE_UC_SEND_LAST :
293                                 IB_OPCODE_UC_SEND_MIDDLE;
294                 else
295                         return fits ?
296                                 IB_OPCODE_UC_SEND_ONLY :
297                                 IB_OPCODE_UC_SEND_FIRST;
298
299         case IB_WR_SEND_WITH_IMM:
300                 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
301                     qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
302                         return fits ?
303                                 IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE :
304                                 IB_OPCODE_UC_SEND_MIDDLE;
305                 else
306                         return fits ?
307                                 IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE :
308                                 IB_OPCODE_UC_SEND_FIRST;
309         }
310
311         return -EINVAL;
312 }
313
314 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
315                        u32 opcode)
316 {
317         int fits = (wqe->dma.resid <= qp->mtu);
318
319         switch (qp_type(qp)) {
320         case IB_QPT_RC:
321                 return next_opcode_rc(qp, opcode, fits);
322
323         case IB_QPT_UC:
324                 return next_opcode_uc(qp, opcode, fits);
325
326         case IB_QPT_SMI:
327         case IB_QPT_UD:
328         case IB_QPT_GSI:
329                 switch (opcode) {
330                 case IB_WR_SEND:
331                         return IB_OPCODE_UD_SEND_ONLY;
332
333                 case IB_WR_SEND_WITH_IMM:
334                         return IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
335                 }
336                 break;
337
338         default:
339                 break;
340         }
341
342         return -EINVAL;
343 }
344
345 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
346 {
347         int depth;
348
349         if (wqe->has_rd_atomic)
350                 return 0;
351
352         qp->req.need_rd_atomic = 1;
353         depth = atomic_dec_return(&qp->req.rd_atomic);
354
355         if (depth >= 0) {
356                 qp->req.need_rd_atomic = 0;
357                 wqe->has_rd_atomic = 1;
358                 return 0;
359         }
360
361         atomic_inc(&qp->req.rd_atomic);
362         return -EAGAIN;
363 }
364
365 static inline int get_mtu(struct rxe_qp *qp)
366 {
367         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
368
369         if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
370                 return qp->mtu;
371
372         return rxe->port.mtu_cap;
373 }
374
375 static struct sk_buff *init_req_packet(struct rxe_qp *qp,
376                                        struct rxe_send_wqe *wqe,
377                                        int opcode, int payload,
378                                        struct rxe_pkt_info *pkt)
379 {
380         struct rxe_dev          *rxe = to_rdev(qp->ibqp.device);
381         struct rxe_port         *port = &rxe->port;
382         struct sk_buff          *skb;
383         struct rxe_send_wr      *ibwr = &wqe->wr;
384         struct rxe_av           *av;
385         int                     pad = (-payload) & 0x3;
386         int                     paylen;
387         int                     solicited;
388         u16                     pkey;
389         u32                     qp_num;
390         int                     ack_req;
391
392         /* length from start of bth to end of icrc */
393         paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
394
395         /* pkt->hdr, rxe, port_num and mask are initialized in ifc
396          * layer
397          */
398         pkt->opcode     = opcode;
399         pkt->qp         = qp;
400         pkt->psn        = qp->req.psn;
401         pkt->mask       = rxe_opcode[opcode].mask;
402         pkt->paylen     = paylen;
403         pkt->offset     = 0;
404         pkt->wqe        = wqe;
405
406         /* init skb */
407         av = rxe_get_av(pkt);
408         skb = rxe_init_packet(rxe, av, paylen, pkt);
409         if (unlikely(!skb))
410                 return NULL;
411
412         /* init bth */
413         solicited = (ibwr->send_flags & IB_SEND_SOLICITED) &&
414                         (pkt->mask & RXE_END_MASK) &&
415                         ((pkt->mask & (RXE_SEND_MASK)) ||
416                         (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
417                         (RXE_WRITE_MASK | RXE_IMMDT_MASK));
418
419         pkey = (qp_type(qp) == IB_QPT_GSI) ?
420                  port->pkey_tbl[ibwr->wr.ud.pkey_index] :
421                  port->pkey_tbl[qp->attr.pkey_index];
422
423         qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
424                                          qp->attr.dest_qp_num;
425
426         ack_req = ((pkt->mask & RXE_END_MASK) ||
427                 (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK));
428         if (ack_req)
429                 qp->req.noack_pkts = 0;
430
431         bth_init(pkt, pkt->opcode, solicited, 0, pad, pkey, qp_num,
432                  ack_req, pkt->psn);
433
434         /* init optional headers */
435         if (pkt->mask & RXE_RETH_MASK) {
436                 reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
437                 reth_set_va(pkt, wqe->iova);
438                 reth_set_len(pkt, wqe->dma.length);
439         }
440
441         if (pkt->mask & RXE_IMMDT_MASK)
442                 immdt_set_imm(pkt, ibwr->ex.imm_data);
443
444         if (pkt->mask & RXE_IETH_MASK)
445                 ieth_set_rkey(pkt, ibwr->ex.invalidate_rkey);
446
447         if (pkt->mask & RXE_ATMETH_MASK) {
448                 atmeth_set_va(pkt, wqe->iova);
449                 if (opcode == IB_OPCODE_RC_COMPARE_SWAP ||
450                     opcode == IB_OPCODE_RD_COMPARE_SWAP) {
451                         atmeth_set_swap_add(pkt, ibwr->wr.atomic.swap);
452                         atmeth_set_comp(pkt, ibwr->wr.atomic.compare_add);
453                 } else {
454                         atmeth_set_swap_add(pkt, ibwr->wr.atomic.compare_add);
455                 }
456                 atmeth_set_rkey(pkt, ibwr->wr.atomic.rkey);
457         }
458
459         if (pkt->mask & RXE_DETH_MASK) {
460                 if (qp->ibqp.qp_num == 1)
461                         deth_set_qkey(pkt, GSI_QKEY);
462                 else
463                         deth_set_qkey(pkt, ibwr->wr.ud.remote_qkey);
464                 deth_set_sqp(pkt, qp->ibqp.qp_num);
465         }
466
467         return skb;
468 }
469
470 static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
471                        struct rxe_pkt_info *pkt, struct sk_buff *skb,
472                        int paylen)
473 {
474         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
475         u32 crc = 0;
476         u32 *p;
477         int err;
478
479         err = rxe_prepare(rxe, pkt, skb, &crc);
480         if (err)
481                 return err;
482
483         if (pkt->mask & RXE_WRITE_OR_SEND) {
484                 if (wqe->wr.send_flags & IB_SEND_INLINE) {
485                         u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
486
487                         crc = rxe_crc32(rxe, crc, tmp, paylen);
488                         memcpy(payload_addr(pkt), tmp, paylen);
489
490                         wqe->dma.resid -= paylen;
491                         wqe->dma.sge_offset += paylen;
492                 } else {
493                         err = copy_data(qp->pd, 0, &wqe->dma,
494                                         payload_addr(pkt), paylen,
495                                         from_mem_obj,
496                                         &crc);
497                         if (err)
498                                 return err;
499                 }
500         }
501         p = payload_addr(pkt) + paylen + bth_pad(pkt);
502
503         *p = ~crc;
504
505         return 0;
506 }
507
508 static void update_wqe_state(struct rxe_qp *qp,
509                 struct rxe_send_wqe *wqe,
510                 struct rxe_pkt_info *pkt)
511 {
512         if (pkt->mask & RXE_END_MASK) {
513                 if (qp_type(qp) == IB_QPT_RC)
514                         wqe->state = wqe_state_pending;
515         } else {
516                 wqe->state = wqe_state_processing;
517         }
518 }
519
520 static void update_wqe_psn(struct rxe_qp *qp,
521                            struct rxe_send_wqe *wqe,
522                            struct rxe_pkt_info *pkt,
523                            int payload)
524 {
525         /* number of packets left to send including current one */
526         int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
527
528         /* handle zero length packet case */
529         if (num_pkt == 0)
530                 num_pkt = 1;
531
532         if (pkt->mask & RXE_START_MASK) {
533                 wqe->first_psn = qp->req.psn;
534                 wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK;
535         }
536
537         if (pkt->mask & RXE_READ_MASK)
538                 qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;
539         else
540                 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
541 }
542
543 static void save_state(struct rxe_send_wqe *wqe,
544                        struct rxe_qp *qp,
545                        struct rxe_send_wqe *rollback_wqe,
546                        u32 *rollback_psn)
547 {
548         rollback_wqe->state     = wqe->state;
549         rollback_wqe->first_psn = wqe->first_psn;
550         rollback_wqe->last_psn  = wqe->last_psn;
551         *rollback_psn           = qp->req.psn;
552 }
553
554 static void rollback_state(struct rxe_send_wqe *wqe,
555                            struct rxe_qp *qp,
556                            struct rxe_send_wqe *rollback_wqe,
557                            u32 rollback_psn)
558 {
559         wqe->state     = rollback_wqe->state;
560         wqe->first_psn = rollback_wqe->first_psn;
561         wqe->last_psn  = rollback_wqe->last_psn;
562         qp->req.psn    = rollback_psn;
563 }
564
565 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
566                          struct rxe_pkt_info *pkt, int payload)
567 {
568         qp->req.opcode = pkt->opcode;
569
570         if (pkt->mask & RXE_END_MASK)
571                 qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index);
572
573         qp->need_req_skb = 0;
574
575         if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer))
576                 mod_timer(&qp->retrans_timer,
577                           jiffies + qp->qp_timeout_jiffies);
578 }
579
580 int rxe_requester(void *arg)
581 {
582         struct rxe_qp *qp = (struct rxe_qp *)arg;
583         struct rxe_pkt_info pkt;
584         struct sk_buff *skb;
585         struct rxe_send_wqe *wqe;
586         enum rxe_hdr_mask mask;
587         int payload;
588         int mtu;
589         int opcode;
590         int ret;
591         struct rxe_send_wqe rollback_wqe;
592         u32 rollback_psn;
593
594         rxe_add_ref(qp);
595
596 next_wqe:
597         if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
598                 goto exit;
599
600         if (unlikely(qp->req.state == QP_STATE_RESET)) {
601                 qp->req.wqe_index = consumer_index(qp->sq.queue);
602                 qp->req.opcode = -1;
603                 qp->req.need_rd_atomic = 0;
604                 qp->req.wait_psn = 0;
605                 qp->req.need_retry = 0;
606                 goto exit;
607         }
608
609         if (unlikely(qp->req.need_retry)) {
610                 req_retry(qp);
611                 qp->req.need_retry = 0;
612         }
613
614         wqe = req_next_wqe(qp);
615         if (unlikely(!wqe))
616                 goto exit;
617
618         if (wqe->mask & WR_REG_MASK) {
619                 if (wqe->wr.opcode == IB_WR_LOCAL_INV) {
620                         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
621                         struct rxe_mem *rmr;
622
623                         rmr = rxe_pool_get_index(&rxe->mr_pool,
624                                                  wqe->wr.ex.invalidate_rkey >> 8);
625                         if (!rmr) {
626                                 pr_err("No mr for key %#x\n",
627                                        wqe->wr.ex.invalidate_rkey);
628                                 wqe->state = wqe_state_error;
629                                 wqe->status = IB_WC_MW_BIND_ERR;
630                                 goto exit;
631                         }
632                         rmr->state = RXE_MEM_STATE_FREE;
633                         rxe_drop_ref(rmr);
634                         wqe->state = wqe_state_done;
635                         wqe->status = IB_WC_SUCCESS;
636                 } else if (wqe->wr.opcode == IB_WR_REG_MR) {
637                         struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr);
638
639                         rmr->state = RXE_MEM_STATE_VALID;
640                         rmr->access = wqe->wr.wr.reg.access;
641                         rmr->lkey = wqe->wr.wr.reg.key;
642                         rmr->rkey = wqe->wr.wr.reg.key;
643                         wqe->state = wqe_state_done;
644                         wqe->status = IB_WC_SUCCESS;
645                 } else {
646                         goto exit;
647                 }
648                 if ((wqe->wr.send_flags & IB_SEND_SIGNALED) ||
649                     qp->sq_sig_type == IB_SIGNAL_ALL_WR)
650                         rxe_run_task(&qp->comp.task, 1);
651                 qp->req.wqe_index = next_index(qp->sq.queue,
652                                                 qp->req.wqe_index);
653                 goto next_wqe;
654         }
655
656         if (unlikely(qp_type(qp) == IB_QPT_RC &&
657                      qp->req.psn > (qp->comp.psn + RXE_MAX_UNACKED_PSNS))) {
658                 qp->req.wait_psn = 1;
659                 goto exit;
660         }
661
662         /* Limit the number of inflight SKBs per QP */
663         if (unlikely(atomic_read(&qp->skb_out) >
664                      RXE_INFLIGHT_SKBS_PER_QP_HIGH)) {
665                 qp->need_req_skb = 1;
666                 goto exit;
667         }
668
669         opcode = next_opcode(qp, wqe, wqe->wr.opcode);
670         if (unlikely(opcode < 0)) {
671                 wqe->status = IB_WC_LOC_QP_OP_ERR;
672                 goto exit;
673         }
674
675         mask = rxe_opcode[opcode].mask;
676         if (unlikely(mask & RXE_READ_OR_ATOMIC)) {
677                 if (check_init_depth(qp, wqe))
678                         goto exit;
679         }
680
681         mtu = get_mtu(qp);
682         payload = (mask & RXE_WRITE_OR_SEND) ? wqe->dma.resid : 0;
683         if (payload > mtu) {
684                 if (qp_type(qp) == IB_QPT_UD) {
685                         /* C10-93.1.1: If the total sum of all the buffer lengths specified for a
686                          * UD message exceeds the MTU of the port as returned by QueryHCA, the CI
687                          * shall not emit any packets for this message. Further, the CI shall not
688                          * generate an error due to this condition.
689                          */
690
691                         /* fake a successful UD send */
692                         wqe->first_psn = qp->req.psn;
693                         wqe->last_psn = qp->req.psn;
694                         qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
695                         qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;
696                         qp->req.wqe_index = next_index(qp->sq.queue,
697                                                        qp->req.wqe_index);
698                         wqe->state = wqe_state_done;
699                         wqe->status = IB_WC_SUCCESS;
700                         __rxe_do_task(&qp->comp.task);
701                         rxe_drop_ref(qp);
702                         return 0;
703                 }
704                 payload = mtu;
705         }
706
707         skb = init_req_packet(qp, wqe, opcode, payload, &pkt);
708         if (unlikely(!skb)) {
709                 pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
710                 goto err;
711         }
712
713         if (fill_packet(qp, wqe, &pkt, skb, payload)) {
714                 pr_debug("qp#%d Error during fill packet\n", qp_num(qp));
715                 kfree_skb(skb);
716                 goto err;
717         }
718
719         /*
720          * To prevent a race on wqe access between requester and completer,
721          * wqe members state and psn need to be set before calling
722          * rxe_xmit_packet().
723          * Otherwise, completer might initiate an unjustified retry flow.
724          */
725         save_state(wqe, qp, &rollback_wqe, &rollback_psn);
726         update_wqe_state(qp, wqe, &pkt);
727         update_wqe_psn(qp, wqe, &pkt, payload);
728         ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb);
729         if (ret) {
730                 qp->need_req_skb = 1;
731
732                 rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
733
734                 if (ret == -EAGAIN) {
735                         rxe_run_task(&qp->req.task, 1);
736                         goto exit;
737                 }
738
739                 goto err;
740         }
741
742         update_state(qp, wqe, &pkt, payload);
743
744         goto next_wqe;
745
746 err:
747         wqe->status = IB_WC_LOC_PROT_ERR;
748         wqe->state = wqe_state_error;
749         __rxe_do_task(&qp->comp.task);
750
751 exit:
752         rxe_drop_ref(qp);
753         return -EAGAIN;
754 }