e7e08889865e732d503a6ac2af5d38cac4dd9672
[muen/linux.git] / drivers / nvme / host / tcp.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVMe over Fabrics TCP host.
4  * Copyright (c) 2018 Lightbits Labs. All rights reserved.
5  */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/slab.h>
10 #include <linux/err.h>
11 #include <linux/nvme-tcp.h>
12 #include <net/sock.h>
13 #include <net/tcp.h>
14 #include <linux/blk-mq.h>
15 #include <crypto/hash.h>
16
17 #include "nvme.h"
18 #include "fabrics.h"
19
20 struct nvme_tcp_queue;
21
22 enum nvme_tcp_send_state {
23         NVME_TCP_SEND_CMD_PDU = 0,
24         NVME_TCP_SEND_H2C_PDU,
25         NVME_TCP_SEND_DATA,
26         NVME_TCP_SEND_DDGST,
27 };
28
29 struct nvme_tcp_request {
30         struct nvme_request     req;
31         void                    *pdu;
32         struct nvme_tcp_queue   *queue;
33         u32                     data_len;
34         u32                     pdu_len;
35         u32                     pdu_sent;
36         u16                     ttag;
37         struct list_head        entry;
38         __le32                  ddgst;
39
40         struct bio              *curr_bio;
41         struct iov_iter         iter;
42
43         /* send state */
44         size_t                  offset;
45         size_t                  data_sent;
46         enum nvme_tcp_send_state state;
47 };
48
49 enum nvme_tcp_queue_flags {
50         NVME_TCP_Q_ALLOCATED    = 0,
51         NVME_TCP_Q_LIVE         = 1,
52 };
53
54 enum nvme_tcp_recv_state {
55         NVME_TCP_RECV_PDU = 0,
56         NVME_TCP_RECV_DATA,
57         NVME_TCP_RECV_DDGST,
58 };
59
60 struct nvme_tcp_ctrl;
61 struct nvme_tcp_queue {
62         struct socket           *sock;
63         struct work_struct      io_work;
64         int                     io_cpu;
65
66         spinlock_t              lock;
67         struct list_head        send_list;
68
69         /* recv state */
70         void                    *pdu;
71         int                     pdu_remaining;
72         int                     pdu_offset;
73         size_t                  data_remaining;
74         size_t                  ddgst_remaining;
75
76         /* send state */
77         struct nvme_tcp_request *request;
78
79         int                     queue_size;
80         size_t                  cmnd_capsule_len;
81         struct nvme_tcp_ctrl    *ctrl;
82         unsigned long           flags;
83         bool                    rd_enabled;
84
85         bool                    hdr_digest;
86         bool                    data_digest;
87         struct ahash_request    *rcv_hash;
88         struct ahash_request    *snd_hash;
89         __le32                  exp_ddgst;
90         __le32                  recv_ddgst;
91
92         struct page_frag_cache  pf_cache;
93
94         void (*state_change)(struct sock *);
95         void (*data_ready)(struct sock *);
96         void (*write_space)(struct sock *);
97 };
98
99 struct nvme_tcp_ctrl {
100         /* read only in the hot path */
101         struct nvme_tcp_queue   *queues;
102         struct blk_mq_tag_set   tag_set;
103
104         /* other member variables */
105         struct list_head        list;
106         struct blk_mq_tag_set   admin_tag_set;
107         struct sockaddr_storage addr;
108         struct sockaddr_storage src_addr;
109         struct nvme_ctrl        ctrl;
110
111         struct work_struct      err_work;
112         struct delayed_work     connect_work;
113         struct nvme_tcp_request async_req;
114 };
115
116 static LIST_HEAD(nvme_tcp_ctrl_list);
117 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
118 static struct workqueue_struct *nvme_tcp_wq;
119 static struct blk_mq_ops nvme_tcp_mq_ops;
120 static struct blk_mq_ops nvme_tcp_admin_mq_ops;
121
122 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
123 {
124         return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
125 }
126
127 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
128 {
129         return queue - queue->ctrl->queues;
130 }
131
132 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
133 {
134         u32 queue_idx = nvme_tcp_queue_id(queue);
135
136         if (queue_idx == 0)
137                 return queue->ctrl->admin_tag_set.tags[queue_idx];
138         return queue->ctrl->tag_set.tags[queue_idx - 1];
139 }
140
141 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
142 {
143         return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
144 }
145
146 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
147 {
148         return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
149 }
150
151 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
152 {
153         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
154 }
155
156 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
157 {
158         return req == &req->queue->ctrl->async_req;
159 }
160
161 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
162 {
163         struct request *rq;
164         unsigned int bytes;
165
166         if (unlikely(nvme_tcp_async_req(req)))
167                 return false; /* async events don't have a request */
168
169         rq = blk_mq_rq_from_pdu(req);
170         bytes = blk_rq_payload_bytes(rq);
171
172         return rq_data_dir(rq) == WRITE && bytes &&
173                 bytes <= nvme_tcp_inline_data_size(req->queue);
174 }
175
176 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
177 {
178         return req->iter.bvec->bv_page;
179 }
180
181 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
182 {
183         return req->iter.bvec->bv_offset + req->iter.iov_offset;
184 }
185
186 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
187 {
188         return min_t(size_t, req->iter.bvec->bv_len - req->iter.iov_offset,
189                         req->pdu_len - req->pdu_sent);
190 }
191
192 static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req)
193 {
194         return req->iter.iov_offset;
195 }
196
197 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
198 {
199         return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
200                         req->pdu_len - req->pdu_sent : 0;
201 }
202
203 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
204                 int len)
205 {
206         return nvme_tcp_pdu_data_left(req) <= len;
207 }
208
209 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
210                 unsigned int dir)
211 {
212         struct request *rq = blk_mq_rq_from_pdu(req);
213         struct bio_vec *vec;
214         unsigned int size;
215         int nsegs;
216         size_t offset;
217
218         if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
219                 vec = &rq->special_vec;
220                 nsegs = 1;
221                 size = blk_rq_payload_bytes(rq);
222                 offset = 0;
223         } else {
224                 struct bio *bio = req->curr_bio;
225
226                 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
227                 nsegs = bio_segments(bio);
228                 size = bio->bi_iter.bi_size;
229                 offset = bio->bi_iter.bi_bvec_done;
230         }
231
232         iov_iter_bvec(&req->iter, dir, vec, nsegs, size);
233         req->iter.iov_offset = offset;
234 }
235
236 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
237                 int len)
238 {
239         req->data_sent += len;
240         req->pdu_sent += len;
241         iov_iter_advance(&req->iter, len);
242         if (!iov_iter_count(&req->iter) &&
243             req->data_sent < req->data_len) {
244                 req->curr_bio = req->curr_bio->bi_next;
245                 nvme_tcp_init_iter(req, WRITE);
246         }
247 }
248
249 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req)
250 {
251         struct nvme_tcp_queue *queue = req->queue;
252
253         spin_lock(&queue->lock);
254         list_add_tail(&req->entry, &queue->send_list);
255         spin_unlock(&queue->lock);
256
257         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
258 }
259
260 static inline struct nvme_tcp_request *
261 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
262 {
263         struct nvme_tcp_request *req;
264
265         spin_lock(&queue->lock);
266         req = list_first_entry_or_null(&queue->send_list,
267                         struct nvme_tcp_request, entry);
268         if (req)
269                 list_del(&req->entry);
270         spin_unlock(&queue->lock);
271
272         return req;
273 }
274
275 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
276                 __le32 *dgst)
277 {
278         ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
279         crypto_ahash_final(hash);
280 }
281
282 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
283                 struct page *page, off_t off, size_t len)
284 {
285         struct scatterlist sg;
286
287         sg_init_marker(&sg, 1);
288         sg_set_page(&sg, page, len, off);
289         ahash_request_set_crypt(hash, &sg, NULL, len);
290         crypto_ahash_update(hash);
291 }
292
293 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
294                 void *pdu, size_t len)
295 {
296         struct scatterlist sg;
297
298         sg_init_one(&sg, pdu, len);
299         ahash_request_set_crypt(hash, &sg, pdu + len, len);
300         crypto_ahash_digest(hash);
301 }
302
303 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
304                 void *pdu, size_t pdu_len)
305 {
306         struct nvme_tcp_hdr *hdr = pdu;
307         __le32 recv_digest;
308         __le32 exp_digest;
309
310         if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
311                 dev_err(queue->ctrl->ctrl.device,
312                         "queue %d: header digest flag is cleared\n",
313                         nvme_tcp_queue_id(queue));
314                 return -EPROTO;
315         }
316
317         recv_digest = *(__le32 *)(pdu + hdr->hlen);
318         nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
319         exp_digest = *(__le32 *)(pdu + hdr->hlen);
320         if (recv_digest != exp_digest) {
321                 dev_err(queue->ctrl->ctrl.device,
322                         "header digest error: recv %#x expected %#x\n",
323                         le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
324                 return -EIO;
325         }
326
327         return 0;
328 }
329
330 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
331 {
332         struct nvme_tcp_hdr *hdr = pdu;
333         u8 digest_len = nvme_tcp_hdgst_len(queue);
334         u32 len;
335
336         len = le32_to_cpu(hdr->plen) - hdr->hlen -
337                 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
338
339         if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
340                 dev_err(queue->ctrl->ctrl.device,
341                         "queue %d: data digest flag is cleared\n",
342                 nvme_tcp_queue_id(queue));
343                 return -EPROTO;
344         }
345         crypto_ahash_init(queue->rcv_hash);
346
347         return 0;
348 }
349
350 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
351                 struct request *rq, unsigned int hctx_idx)
352 {
353         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
354
355         page_frag_free(req->pdu);
356 }
357
358 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
359                 struct request *rq, unsigned int hctx_idx,
360                 unsigned int numa_node)
361 {
362         struct nvme_tcp_ctrl *ctrl = set->driver_data;
363         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
364         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
365         struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
366         u8 hdgst = nvme_tcp_hdgst_len(queue);
367
368         req->pdu = page_frag_alloc(&queue->pf_cache,
369                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
370                 GFP_KERNEL | __GFP_ZERO);
371         if (!req->pdu)
372                 return -ENOMEM;
373
374         req->queue = queue;
375         nvme_req(rq)->ctrl = &ctrl->ctrl;
376
377         return 0;
378 }
379
380 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
381                 unsigned int hctx_idx)
382 {
383         struct nvme_tcp_ctrl *ctrl = data;
384         struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
385
386         hctx->driver_data = queue;
387         return 0;
388 }
389
390 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
391                 unsigned int hctx_idx)
392 {
393         struct nvme_tcp_ctrl *ctrl = data;
394         struct nvme_tcp_queue *queue = &ctrl->queues[0];
395
396         hctx->driver_data = queue;
397         return 0;
398 }
399
400 static enum nvme_tcp_recv_state
401 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
402 {
403         return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
404                 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
405                 NVME_TCP_RECV_DATA;
406 }
407
408 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
409 {
410         queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
411                                 nvme_tcp_hdgst_len(queue);
412         queue->pdu_offset = 0;
413         queue->data_remaining = -1;
414         queue->ddgst_remaining = 0;
415 }
416
417 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
418 {
419         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
420                 return;
421
422         queue_work(nvme_wq, &to_tcp_ctrl(ctrl)->err_work);
423 }
424
425 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
426                 struct nvme_completion *cqe)
427 {
428         struct request *rq;
429
430         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), cqe->command_id);
431         if (!rq) {
432                 dev_err(queue->ctrl->ctrl.device,
433                         "queue %d tag 0x%x not found\n",
434                         nvme_tcp_queue_id(queue), cqe->command_id);
435                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
436                 return -EINVAL;
437         }
438
439         nvme_end_request(rq, cqe->status, cqe->result);
440
441         return 0;
442 }
443
444 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
445                 struct nvme_tcp_data_pdu *pdu)
446 {
447         struct request *rq;
448
449         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
450         if (!rq) {
451                 dev_err(queue->ctrl->ctrl.device,
452                         "queue %d tag %#x not found\n",
453                         nvme_tcp_queue_id(queue), pdu->command_id);
454                 return -ENOENT;
455         }
456
457         if (!blk_rq_payload_bytes(rq)) {
458                 dev_err(queue->ctrl->ctrl.device,
459                         "queue %d tag %#x unexpected data\n",
460                         nvme_tcp_queue_id(queue), rq->tag);
461                 return -EIO;
462         }
463
464         queue->data_remaining = le32_to_cpu(pdu->data_length);
465
466         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
467             unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
468                 dev_err(queue->ctrl->ctrl.device,
469                         "queue %d tag %#x SUCCESS set but not last PDU\n",
470                         nvme_tcp_queue_id(queue), rq->tag);
471                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
472                 return -EPROTO;
473         }
474
475         return 0;
476
477 }
478
479 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
480                 struct nvme_tcp_rsp_pdu *pdu)
481 {
482         struct nvme_completion *cqe = &pdu->cqe;
483         int ret = 0;
484
485         /*
486          * AEN requests are special as they don't time out and can
487          * survive any kind of queue freeze and often don't respond to
488          * aborts.  We don't even bother to allocate a struct request
489          * for them but rather special case them here.
490          */
491         if (unlikely(nvme_tcp_queue_id(queue) == 0 &&
492             cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH))
493                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
494                                 &cqe->result);
495         else
496                 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
497
498         return ret;
499 }
500
501 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req,
502                 struct nvme_tcp_r2t_pdu *pdu)
503 {
504         struct nvme_tcp_data_pdu *data = req->pdu;
505         struct nvme_tcp_queue *queue = req->queue;
506         struct request *rq = blk_mq_rq_from_pdu(req);
507         u8 hdgst = nvme_tcp_hdgst_len(queue);
508         u8 ddgst = nvme_tcp_ddgst_len(queue);
509
510         req->pdu_len = le32_to_cpu(pdu->r2t_length);
511         req->pdu_sent = 0;
512
513         if (unlikely(req->data_sent + req->pdu_len > req->data_len)) {
514                 dev_err(queue->ctrl->ctrl.device,
515                         "req %d r2t len %u exceeded data len %u (%zu sent)\n",
516                         rq->tag, req->pdu_len, req->data_len,
517                         req->data_sent);
518                 return -EPROTO;
519         }
520
521         if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {
522                 dev_err(queue->ctrl->ctrl.device,
523                         "req %d unexpected r2t offset %u (expected %zu)\n",
524                         rq->tag, le32_to_cpu(pdu->r2t_offset),
525                         req->data_sent);
526                 return -EPROTO;
527         }
528
529         memset(data, 0, sizeof(*data));
530         data->hdr.type = nvme_tcp_h2c_data;
531         data->hdr.flags = NVME_TCP_F_DATA_LAST;
532         if (queue->hdr_digest)
533                 data->hdr.flags |= NVME_TCP_F_HDGST;
534         if (queue->data_digest)
535                 data->hdr.flags |= NVME_TCP_F_DDGST;
536         data->hdr.hlen = sizeof(*data);
537         data->hdr.pdo = data->hdr.hlen + hdgst;
538         data->hdr.plen =
539                 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
540         data->ttag = pdu->ttag;
541         data->command_id = rq->tag;
542         data->data_offset = cpu_to_le32(req->data_sent);
543         data->data_length = cpu_to_le32(req->pdu_len);
544         return 0;
545 }
546
547 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
548                 struct nvme_tcp_r2t_pdu *pdu)
549 {
550         struct nvme_tcp_request *req;
551         struct request *rq;
552         int ret;
553
554         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
555         if (!rq) {
556                 dev_err(queue->ctrl->ctrl.device,
557                         "queue %d tag %#x not found\n",
558                         nvme_tcp_queue_id(queue), pdu->command_id);
559                 return -ENOENT;
560         }
561         req = blk_mq_rq_to_pdu(rq);
562
563         ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
564         if (unlikely(ret))
565                 return ret;
566
567         req->state = NVME_TCP_SEND_H2C_PDU;
568         req->offset = 0;
569
570         nvme_tcp_queue_request(req);
571
572         return 0;
573 }
574
575 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
576                 unsigned int *offset, size_t *len)
577 {
578         struct nvme_tcp_hdr *hdr;
579         char *pdu = queue->pdu;
580         size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
581         int ret;
582
583         ret = skb_copy_bits(skb, *offset,
584                 &pdu[queue->pdu_offset], rcv_len);
585         if (unlikely(ret))
586                 return ret;
587
588         queue->pdu_remaining -= rcv_len;
589         queue->pdu_offset += rcv_len;
590         *offset += rcv_len;
591         *len -= rcv_len;
592         if (queue->pdu_remaining)
593                 return 0;
594
595         hdr = queue->pdu;
596         if (queue->hdr_digest) {
597                 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
598                 if (unlikely(ret))
599                         return ret;
600         }
601
602
603         if (queue->data_digest) {
604                 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
605                 if (unlikely(ret))
606                         return ret;
607         }
608
609         switch (hdr->type) {
610         case nvme_tcp_c2h_data:
611                 ret = nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
612                 break;
613         case nvme_tcp_rsp:
614                 nvme_tcp_init_recv_ctx(queue);
615                 ret = nvme_tcp_handle_comp(queue, (void *)queue->pdu);
616                 break;
617         case nvme_tcp_r2t:
618                 nvme_tcp_init_recv_ctx(queue);
619                 ret = nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
620                 break;
621         default:
622                 dev_err(queue->ctrl->ctrl.device,
623                         "unsupported pdu type (%d)\n", hdr->type);
624                 return -EINVAL;
625         }
626
627         return ret;
628 }
629
630 static inline void nvme_tcp_end_request(struct request *rq, __le16 status)
631 {
632         union nvme_result res = {};
633
634         nvme_end_request(rq, cpu_to_le16(status << 1), res);
635 }
636
637
638 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
639                               unsigned int *offset, size_t *len)
640 {
641         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
642         struct nvme_tcp_request *req;
643         struct request *rq;
644
645         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
646         if (!rq) {
647                 dev_err(queue->ctrl->ctrl.device,
648                         "queue %d tag %#x not found\n",
649                         nvme_tcp_queue_id(queue), pdu->command_id);
650                 return -ENOENT;
651         }
652         req = blk_mq_rq_to_pdu(rq);
653
654         while (true) {
655                 int recv_len, ret;
656
657                 recv_len = min_t(size_t, *len, queue->data_remaining);
658                 if (!recv_len)
659                         break;
660
661                 if (!iov_iter_count(&req->iter)) {
662                         req->curr_bio = req->curr_bio->bi_next;
663
664                         /*
665                          * If we don`t have any bios it means that controller
666                          * sent more data than we requested, hence error
667                          */
668                         if (!req->curr_bio) {
669                                 dev_err(queue->ctrl->ctrl.device,
670                                         "queue %d no space in request %#x",
671                                         nvme_tcp_queue_id(queue), rq->tag);
672                                 nvme_tcp_init_recv_ctx(queue);
673                                 return -EIO;
674                         }
675                         nvme_tcp_init_iter(req, READ);
676                 }
677
678                 /* we can read only from what is left in this bio */
679                 recv_len = min_t(size_t, recv_len,
680                                 iov_iter_count(&req->iter));
681
682                 if (queue->data_digest)
683                         ret = skb_copy_and_hash_datagram_iter(skb, *offset,
684                                 &req->iter, recv_len, queue->rcv_hash);
685                 else
686                         ret = skb_copy_datagram_iter(skb, *offset,
687                                         &req->iter, recv_len);
688                 if (ret) {
689                         dev_err(queue->ctrl->ctrl.device,
690                                 "queue %d failed to copy request %#x data",
691                                 nvme_tcp_queue_id(queue), rq->tag);
692                         return ret;
693                 }
694
695                 *len -= recv_len;
696                 *offset += recv_len;
697                 queue->data_remaining -= recv_len;
698         }
699
700         if (!queue->data_remaining) {
701                 if (queue->data_digest) {
702                         nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
703                         queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
704                 } else {
705                         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS)
706                                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
707                         nvme_tcp_init_recv_ctx(queue);
708                 }
709         }
710
711         return 0;
712 }
713
714 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
715                 struct sk_buff *skb, unsigned int *offset, size_t *len)
716 {
717         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
718         char *ddgst = (char *)&queue->recv_ddgst;
719         size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
720         off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
721         int ret;
722
723         ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
724         if (unlikely(ret))
725                 return ret;
726
727         queue->ddgst_remaining -= recv_len;
728         *offset += recv_len;
729         *len -= recv_len;
730         if (queue->ddgst_remaining)
731                 return 0;
732
733         if (queue->recv_ddgst != queue->exp_ddgst) {
734                 dev_err(queue->ctrl->ctrl.device,
735                         "data digest error: recv %#x expected %#x\n",
736                         le32_to_cpu(queue->recv_ddgst),
737                         le32_to_cpu(queue->exp_ddgst));
738                 return -EIO;
739         }
740
741         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
742                 struct request *rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue),
743                                                 pdu->command_id);
744
745                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
746         }
747
748         nvme_tcp_init_recv_ctx(queue);
749         return 0;
750 }
751
752 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
753                              unsigned int offset, size_t len)
754 {
755         struct nvme_tcp_queue *queue = desc->arg.data;
756         size_t consumed = len;
757         int result;
758
759         while (len) {
760                 switch (nvme_tcp_recv_state(queue)) {
761                 case NVME_TCP_RECV_PDU:
762                         result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
763                         break;
764                 case NVME_TCP_RECV_DATA:
765                         result = nvme_tcp_recv_data(queue, skb, &offset, &len);
766                         break;
767                 case NVME_TCP_RECV_DDGST:
768                         result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
769                         break;
770                 default:
771                         result = -EFAULT;
772                 }
773                 if (result) {
774                         dev_err(queue->ctrl->ctrl.device,
775                                 "receive failed:  %d\n", result);
776                         queue->rd_enabled = false;
777                         nvme_tcp_error_recovery(&queue->ctrl->ctrl);
778                         return result;
779                 }
780         }
781
782         return consumed;
783 }
784
785 static void nvme_tcp_data_ready(struct sock *sk)
786 {
787         struct nvme_tcp_queue *queue;
788
789         read_lock(&sk->sk_callback_lock);
790         queue = sk->sk_user_data;
791         if (likely(queue && queue->rd_enabled))
792                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
793         read_unlock(&sk->sk_callback_lock);
794 }
795
796 static void nvme_tcp_write_space(struct sock *sk)
797 {
798         struct nvme_tcp_queue *queue;
799
800         read_lock_bh(&sk->sk_callback_lock);
801         queue = sk->sk_user_data;
802         if (likely(queue && sk_stream_is_writeable(sk))) {
803                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
804                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
805         }
806         read_unlock_bh(&sk->sk_callback_lock);
807 }
808
809 static void nvme_tcp_state_change(struct sock *sk)
810 {
811         struct nvme_tcp_queue *queue;
812
813         read_lock(&sk->sk_callback_lock);
814         queue = sk->sk_user_data;
815         if (!queue)
816                 goto done;
817
818         switch (sk->sk_state) {
819         case TCP_CLOSE:
820         case TCP_CLOSE_WAIT:
821         case TCP_LAST_ACK:
822         case TCP_FIN_WAIT1:
823         case TCP_FIN_WAIT2:
824                 /* fallthrough */
825                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
826                 break;
827         default:
828                 dev_info(queue->ctrl->ctrl.device,
829                         "queue %d socket state %d\n",
830                         nvme_tcp_queue_id(queue), sk->sk_state);
831         }
832
833         queue->state_change(sk);
834 done:
835         read_unlock(&sk->sk_callback_lock);
836 }
837
838 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
839 {
840         queue->request = NULL;
841 }
842
843 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
844 {
845         nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_DATA_XFER_ERROR);
846 }
847
848 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
849 {
850         struct nvme_tcp_queue *queue = req->queue;
851
852         while (true) {
853                 struct page *page = nvme_tcp_req_cur_page(req);
854                 size_t offset = nvme_tcp_req_cur_offset(req);
855                 size_t len = nvme_tcp_req_cur_length(req);
856                 bool last = nvme_tcp_pdu_last_send(req, len);
857                 int ret, flags = MSG_DONTWAIT;
858
859                 if (last && !queue->data_digest)
860                         flags |= MSG_EOR;
861                 else
862                         flags |= MSG_MORE;
863
864                 ret = kernel_sendpage(queue->sock, page, offset, len, flags);
865                 if (ret <= 0)
866                         return ret;
867
868                 nvme_tcp_advance_req(req, ret);
869                 if (queue->data_digest)
870                         nvme_tcp_ddgst_update(queue->snd_hash, page,
871                                         offset, ret);
872
873                 /* fully successful last write*/
874                 if (last && ret == len) {
875                         if (queue->data_digest) {
876                                 nvme_tcp_ddgst_final(queue->snd_hash,
877                                         &req->ddgst);
878                                 req->state = NVME_TCP_SEND_DDGST;
879                                 req->offset = 0;
880                         } else {
881                                 nvme_tcp_done_send_req(queue);
882                         }
883                         return 1;
884                 }
885         }
886         return -EAGAIN;
887 }
888
889 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
890 {
891         struct nvme_tcp_queue *queue = req->queue;
892         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
893         bool inline_data = nvme_tcp_has_inline_data(req);
894         int flags = MSG_DONTWAIT | (inline_data ? MSG_MORE : MSG_EOR);
895         u8 hdgst = nvme_tcp_hdgst_len(queue);
896         int len = sizeof(*pdu) + hdgst - req->offset;
897         int ret;
898
899         if (queue->hdr_digest && !req->offset)
900                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
901
902         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
903                         offset_in_page(pdu) + req->offset, len,  flags);
904         if (unlikely(ret <= 0))
905                 return ret;
906
907         len -= ret;
908         if (!len) {
909                 if (inline_data) {
910                         req->state = NVME_TCP_SEND_DATA;
911                         if (queue->data_digest)
912                                 crypto_ahash_init(queue->snd_hash);
913                         nvme_tcp_init_iter(req, WRITE);
914                 } else {
915                         nvme_tcp_done_send_req(queue);
916                 }
917                 return 1;
918         }
919         req->offset += ret;
920
921         return -EAGAIN;
922 }
923
924 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
925 {
926         struct nvme_tcp_queue *queue = req->queue;
927         struct nvme_tcp_data_pdu *pdu = req->pdu;
928         u8 hdgst = nvme_tcp_hdgst_len(queue);
929         int len = sizeof(*pdu) - req->offset + hdgst;
930         int ret;
931
932         if (queue->hdr_digest && !req->offset)
933                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
934
935         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
936                         offset_in_page(pdu) + req->offset, len,
937                         MSG_DONTWAIT | MSG_MORE);
938         if (unlikely(ret <= 0))
939                 return ret;
940
941         len -= ret;
942         if (!len) {
943                 req->state = NVME_TCP_SEND_DATA;
944                 if (queue->data_digest)
945                         crypto_ahash_init(queue->snd_hash);
946                 if (!req->data_sent)
947                         nvme_tcp_init_iter(req, WRITE);
948                 return 1;
949         }
950         req->offset += ret;
951
952         return -EAGAIN;
953 }
954
955 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
956 {
957         struct nvme_tcp_queue *queue = req->queue;
958         int ret;
959         struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
960         struct kvec iov = {
961                 .iov_base = &req->ddgst + req->offset,
962                 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
963         };
964
965         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
966         if (unlikely(ret <= 0))
967                 return ret;
968
969         if (req->offset + ret == NVME_TCP_DIGEST_LENGTH) {
970                 nvme_tcp_done_send_req(queue);
971                 return 1;
972         }
973
974         req->offset += ret;
975         return -EAGAIN;
976 }
977
978 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
979 {
980         struct nvme_tcp_request *req;
981         int ret = 1;
982
983         if (!queue->request) {
984                 queue->request = nvme_tcp_fetch_request(queue);
985                 if (!queue->request)
986                         return 0;
987         }
988         req = queue->request;
989
990         if (req->state == NVME_TCP_SEND_CMD_PDU) {
991                 ret = nvme_tcp_try_send_cmd_pdu(req);
992                 if (ret <= 0)
993                         goto done;
994                 if (!nvme_tcp_has_inline_data(req))
995                         return ret;
996         }
997
998         if (req->state == NVME_TCP_SEND_H2C_PDU) {
999                 ret = nvme_tcp_try_send_data_pdu(req);
1000                 if (ret <= 0)
1001                         goto done;
1002         }
1003
1004         if (req->state == NVME_TCP_SEND_DATA) {
1005                 ret = nvme_tcp_try_send_data(req);
1006                 if (ret <= 0)
1007                         goto done;
1008         }
1009
1010         if (req->state == NVME_TCP_SEND_DDGST)
1011                 ret = nvme_tcp_try_send_ddgst(req);
1012 done:
1013         if (ret == -EAGAIN)
1014                 ret = 0;
1015         return ret;
1016 }
1017
1018 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1019 {
1020         struct sock *sk = queue->sock->sk;
1021         read_descriptor_t rd_desc;
1022         int consumed;
1023
1024         rd_desc.arg.data = queue;
1025         rd_desc.count = 1;
1026         lock_sock(sk);
1027         consumed = tcp_read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1028         release_sock(sk);
1029         return consumed;
1030 }
1031
1032 static void nvme_tcp_io_work(struct work_struct *w)
1033 {
1034         struct nvme_tcp_queue *queue =
1035                 container_of(w, struct nvme_tcp_queue, io_work);
1036         unsigned long start = jiffies + msecs_to_jiffies(1);
1037
1038         do {
1039                 bool pending = false;
1040                 int result;
1041
1042                 result = nvme_tcp_try_send(queue);
1043                 if (result > 0) {
1044                         pending = true;
1045                 } else if (unlikely(result < 0)) {
1046                         dev_err(queue->ctrl->ctrl.device,
1047                                 "failed to send request %d\n", result);
1048                         if (result != -EPIPE)
1049                                 nvme_tcp_fail_request(queue->request);
1050                         nvme_tcp_done_send_req(queue);
1051                         return;
1052                 }
1053
1054                 result = nvme_tcp_try_recv(queue);
1055                 if (result > 0)
1056                         pending = true;
1057
1058                 if (!pending)
1059                         return;
1060
1061         } while (time_after(jiffies, start)); /* quota is exhausted */
1062
1063         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1064 }
1065
1066 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1067 {
1068         struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1069
1070         ahash_request_free(queue->rcv_hash);
1071         ahash_request_free(queue->snd_hash);
1072         crypto_free_ahash(tfm);
1073 }
1074
1075 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1076 {
1077         struct crypto_ahash *tfm;
1078
1079         tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1080         if (IS_ERR(tfm))
1081                 return PTR_ERR(tfm);
1082
1083         queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1084         if (!queue->snd_hash)
1085                 goto free_tfm;
1086         ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1087
1088         queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1089         if (!queue->rcv_hash)
1090                 goto free_snd_hash;
1091         ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1092
1093         return 0;
1094 free_snd_hash:
1095         ahash_request_free(queue->snd_hash);
1096 free_tfm:
1097         crypto_free_ahash(tfm);
1098         return -ENOMEM;
1099 }
1100
1101 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1102 {
1103         struct nvme_tcp_request *async = &ctrl->async_req;
1104
1105         page_frag_free(async->pdu);
1106 }
1107
1108 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1109 {
1110         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1111         struct nvme_tcp_request *async = &ctrl->async_req;
1112         u8 hdgst = nvme_tcp_hdgst_len(queue);
1113
1114         async->pdu = page_frag_alloc(&queue->pf_cache,
1115                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1116                 GFP_KERNEL | __GFP_ZERO);
1117         if (!async->pdu)
1118                 return -ENOMEM;
1119
1120         async->queue = &ctrl->queues[0];
1121         return 0;
1122 }
1123
1124 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1125 {
1126         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1127         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1128
1129         if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1130                 return;
1131
1132         if (queue->hdr_digest || queue->data_digest)
1133                 nvme_tcp_free_crypto(queue);
1134
1135         sock_release(queue->sock);
1136         kfree(queue->pdu);
1137 }
1138
1139 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1140 {
1141         struct nvme_tcp_icreq_pdu *icreq;
1142         struct nvme_tcp_icresp_pdu *icresp;
1143         struct msghdr msg = {};
1144         struct kvec iov;
1145         bool ctrl_hdgst, ctrl_ddgst;
1146         int ret;
1147
1148         icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1149         if (!icreq)
1150                 return -ENOMEM;
1151
1152         icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1153         if (!icresp) {
1154                 ret = -ENOMEM;
1155                 goto free_icreq;
1156         }
1157
1158         icreq->hdr.type = nvme_tcp_icreq;
1159         icreq->hdr.hlen = sizeof(*icreq);
1160         icreq->hdr.pdo = 0;
1161         icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1162         icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1163         icreq->maxr2t = 0; /* single inflight r2t supported */
1164         icreq->hpda = 0; /* no alignment constraint */
1165         if (queue->hdr_digest)
1166                 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1167         if (queue->data_digest)
1168                 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1169
1170         iov.iov_base = icreq;
1171         iov.iov_len = sizeof(*icreq);
1172         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1173         if (ret < 0)
1174                 goto free_icresp;
1175
1176         memset(&msg, 0, sizeof(msg));
1177         iov.iov_base = icresp;
1178         iov.iov_len = sizeof(*icresp);
1179         ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1180                         iov.iov_len, msg.msg_flags);
1181         if (ret < 0)
1182                 goto free_icresp;
1183
1184         ret = -EINVAL;
1185         if (icresp->hdr.type != nvme_tcp_icresp) {
1186                 pr_err("queue %d: bad type returned %d\n",
1187                         nvme_tcp_queue_id(queue), icresp->hdr.type);
1188                 goto free_icresp;
1189         }
1190
1191         if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1192                 pr_err("queue %d: bad pdu length returned %d\n",
1193                         nvme_tcp_queue_id(queue), icresp->hdr.plen);
1194                 goto free_icresp;
1195         }
1196
1197         if (icresp->pfv != NVME_TCP_PFV_1_0) {
1198                 pr_err("queue %d: bad pfv returned %d\n",
1199                         nvme_tcp_queue_id(queue), icresp->pfv);
1200                 goto free_icresp;
1201         }
1202
1203         ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1204         if ((queue->data_digest && !ctrl_ddgst) ||
1205             (!queue->data_digest && ctrl_ddgst)) {
1206                 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1207                         nvme_tcp_queue_id(queue),
1208                         queue->data_digest ? "enabled" : "disabled",
1209                         ctrl_ddgst ? "enabled" : "disabled");
1210                 goto free_icresp;
1211         }
1212
1213         ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1214         if ((queue->hdr_digest && !ctrl_hdgst) ||
1215             (!queue->hdr_digest && ctrl_hdgst)) {
1216                 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1217                         nvme_tcp_queue_id(queue),
1218                         queue->hdr_digest ? "enabled" : "disabled",
1219                         ctrl_hdgst ? "enabled" : "disabled");
1220                 goto free_icresp;
1221         }
1222
1223         if (icresp->cpda != 0) {
1224                 pr_err("queue %d: unsupported cpda returned %d\n",
1225                         nvme_tcp_queue_id(queue), icresp->cpda);
1226                 goto free_icresp;
1227         }
1228
1229         ret = 0;
1230 free_icresp:
1231         kfree(icresp);
1232 free_icreq:
1233         kfree(icreq);
1234         return ret;
1235 }
1236
1237 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1238                 int qid, size_t queue_size)
1239 {
1240         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1241         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1242         struct linger sol = { .l_onoff = 1, .l_linger = 0 };
1243         int ret, opt, rcv_pdu_size, n;
1244
1245         queue->ctrl = ctrl;
1246         INIT_LIST_HEAD(&queue->send_list);
1247         spin_lock_init(&queue->lock);
1248         INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1249         queue->queue_size = queue_size;
1250
1251         if (qid > 0)
1252                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
1253         else
1254                 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1255                                                 NVME_TCP_ADMIN_CCSZ;
1256
1257         ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1258                         IPPROTO_TCP, &queue->sock);
1259         if (ret) {
1260                 dev_err(ctrl->ctrl.device,
1261                         "failed to create socket: %d\n", ret);
1262                 return ret;
1263         }
1264
1265         /* Single syn retry */
1266         opt = 1;
1267         ret = kernel_setsockopt(queue->sock, IPPROTO_TCP, TCP_SYNCNT,
1268                         (char *)&opt, sizeof(opt));
1269         if (ret) {
1270                 dev_err(ctrl->ctrl.device,
1271                         "failed to set TCP_SYNCNT sock opt %d\n", ret);
1272                 goto err_sock;
1273         }
1274
1275         /* Set TCP no delay */
1276         opt = 1;
1277         ret = kernel_setsockopt(queue->sock, IPPROTO_TCP,
1278                         TCP_NODELAY, (char *)&opt, sizeof(opt));
1279         if (ret) {
1280                 dev_err(ctrl->ctrl.device,
1281                         "failed to set TCP_NODELAY sock opt %d\n", ret);
1282                 goto err_sock;
1283         }
1284
1285         /*
1286          * Cleanup whatever is sitting in the TCP transmit queue on socket
1287          * close. This is done to prevent stale data from being sent should
1288          * the network connection be restored before TCP times out.
1289          */
1290         ret = kernel_setsockopt(queue->sock, SOL_SOCKET, SO_LINGER,
1291                         (char *)&sol, sizeof(sol));
1292         if (ret) {
1293                 dev_err(ctrl->ctrl.device,
1294                         "failed to set SO_LINGER sock opt %d\n", ret);
1295                 goto err_sock;
1296         }
1297
1298         queue->sock->sk->sk_allocation = GFP_ATOMIC;
1299         if (!qid)
1300                 n = 0;
1301         else
1302                 n = (qid - 1) % num_online_cpus();
1303         queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1304         queue->request = NULL;
1305         queue->data_remaining = 0;
1306         queue->ddgst_remaining = 0;
1307         queue->pdu_remaining = 0;
1308         queue->pdu_offset = 0;
1309         sk_set_memalloc(queue->sock->sk);
1310
1311         if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR) {
1312                 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1313                         sizeof(ctrl->src_addr));
1314                 if (ret) {
1315                         dev_err(ctrl->ctrl.device,
1316                                 "failed to bind queue %d socket %d\n",
1317                                 qid, ret);
1318                         goto err_sock;
1319                 }
1320         }
1321
1322         queue->hdr_digest = nctrl->opts->hdr_digest;
1323         queue->data_digest = nctrl->opts->data_digest;
1324         if (queue->hdr_digest || queue->data_digest) {
1325                 ret = nvme_tcp_alloc_crypto(queue);
1326                 if (ret) {
1327                         dev_err(ctrl->ctrl.device,
1328                                 "failed to allocate queue %d crypto\n", qid);
1329                         goto err_sock;
1330                 }
1331         }
1332
1333         rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1334                         nvme_tcp_hdgst_len(queue);
1335         queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1336         if (!queue->pdu) {
1337                 ret = -ENOMEM;
1338                 goto err_crypto;
1339         }
1340
1341         dev_dbg(ctrl->ctrl.device, "connecting queue %d\n",
1342                         nvme_tcp_queue_id(queue));
1343
1344         ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1345                 sizeof(ctrl->addr), 0);
1346         if (ret) {
1347                 dev_err(ctrl->ctrl.device,
1348                         "failed to connect socket: %d\n", ret);
1349                 goto err_rcv_pdu;
1350         }
1351
1352         ret = nvme_tcp_init_connection(queue);
1353         if (ret)
1354                 goto err_init_connect;
1355
1356         queue->rd_enabled = true;
1357         set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1358         nvme_tcp_init_recv_ctx(queue);
1359
1360         write_lock_bh(&queue->sock->sk->sk_callback_lock);
1361         queue->sock->sk->sk_user_data = queue;
1362         queue->state_change = queue->sock->sk->sk_state_change;
1363         queue->data_ready = queue->sock->sk->sk_data_ready;
1364         queue->write_space = queue->sock->sk->sk_write_space;
1365         queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1366         queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1367         queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1368         write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1369
1370         return 0;
1371
1372 err_init_connect:
1373         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1374 err_rcv_pdu:
1375         kfree(queue->pdu);
1376 err_crypto:
1377         if (queue->hdr_digest || queue->data_digest)
1378                 nvme_tcp_free_crypto(queue);
1379 err_sock:
1380         sock_release(queue->sock);
1381         queue->sock = NULL;
1382         return ret;
1383 }
1384
1385 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1386 {
1387         struct socket *sock = queue->sock;
1388
1389         write_lock_bh(&sock->sk->sk_callback_lock);
1390         sock->sk->sk_user_data  = NULL;
1391         sock->sk->sk_data_ready = queue->data_ready;
1392         sock->sk->sk_state_change = queue->state_change;
1393         sock->sk->sk_write_space  = queue->write_space;
1394         write_unlock_bh(&sock->sk->sk_callback_lock);
1395 }
1396
1397 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1398 {
1399         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1400         nvme_tcp_restore_sock_calls(queue);
1401         cancel_work_sync(&queue->io_work);
1402 }
1403
1404 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1405 {
1406         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1407         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1408
1409         if (!test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1410                 return;
1411
1412         __nvme_tcp_stop_queue(queue);
1413 }
1414
1415 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1416 {
1417         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1418         int ret;
1419
1420         if (idx)
1421                 ret = nvmf_connect_io_queue(nctrl, idx, false);
1422         else
1423                 ret = nvmf_connect_admin_queue(nctrl);
1424
1425         if (!ret) {
1426                 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1427         } else {
1428                 __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1429                 dev_err(nctrl->device,
1430                         "failed to connect queue: %d ret=%d\n", idx, ret);
1431         }
1432         return ret;
1433 }
1434
1435 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1436                 bool admin)
1437 {
1438         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1439         struct blk_mq_tag_set *set;
1440         int ret;
1441
1442         if (admin) {
1443                 set = &ctrl->admin_tag_set;
1444                 memset(set, 0, sizeof(*set));
1445                 set->ops = &nvme_tcp_admin_mq_ops;
1446                 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1447                 set->reserved_tags = 2; /* connect + keep-alive */
1448                 set->numa_node = NUMA_NO_NODE;
1449                 set->cmd_size = sizeof(struct nvme_tcp_request);
1450                 set->driver_data = ctrl;
1451                 set->nr_hw_queues = 1;
1452                 set->timeout = ADMIN_TIMEOUT;
1453         } else {
1454                 set = &ctrl->tag_set;
1455                 memset(set, 0, sizeof(*set));
1456                 set->ops = &nvme_tcp_mq_ops;
1457                 set->queue_depth = nctrl->sqsize + 1;
1458                 set->reserved_tags = 1; /* fabric connect */
1459                 set->numa_node = NUMA_NO_NODE;
1460                 set->flags = BLK_MQ_F_SHOULD_MERGE;
1461                 set->cmd_size = sizeof(struct nvme_tcp_request);
1462                 set->driver_data = ctrl;
1463                 set->nr_hw_queues = nctrl->queue_count - 1;
1464                 set->timeout = NVME_IO_TIMEOUT;
1465                 set->nr_maps = 2 /* default + read */;
1466         }
1467
1468         ret = blk_mq_alloc_tag_set(set);
1469         if (ret)
1470                 return ERR_PTR(ret);
1471
1472         return set;
1473 }
1474
1475 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1476 {
1477         if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1478                 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1479                 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1480         }
1481
1482         nvme_tcp_free_queue(ctrl, 0);
1483 }
1484
1485 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1486 {
1487         int i;
1488
1489         for (i = 1; i < ctrl->queue_count; i++)
1490                 nvme_tcp_free_queue(ctrl, i);
1491 }
1492
1493 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1494 {
1495         int i;
1496
1497         for (i = 1; i < ctrl->queue_count; i++)
1498                 nvme_tcp_stop_queue(ctrl, i);
1499 }
1500
1501 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1502 {
1503         int i, ret = 0;
1504
1505         for (i = 1; i < ctrl->queue_count; i++) {
1506                 ret = nvme_tcp_start_queue(ctrl, i);
1507                 if (ret)
1508                         goto out_stop_queues;
1509         }
1510
1511         return 0;
1512
1513 out_stop_queues:
1514         for (i--; i >= 1; i--)
1515                 nvme_tcp_stop_queue(ctrl, i);
1516         return ret;
1517 }
1518
1519 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1520 {
1521         int ret;
1522
1523         ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1524         if (ret)
1525                 return ret;
1526
1527         ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1528         if (ret)
1529                 goto out_free_queue;
1530
1531         return 0;
1532
1533 out_free_queue:
1534         nvme_tcp_free_queue(ctrl, 0);
1535         return ret;
1536 }
1537
1538 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1539 {
1540         int i, ret;
1541
1542         for (i = 1; i < ctrl->queue_count; i++) {
1543                 ret = nvme_tcp_alloc_queue(ctrl, i,
1544                                 ctrl->sqsize + 1);
1545                 if (ret)
1546                         goto out_free_queues;
1547         }
1548
1549         return 0;
1550
1551 out_free_queues:
1552         for (i--; i >= 1; i--)
1553                 nvme_tcp_free_queue(ctrl, i);
1554
1555         return ret;
1556 }
1557
1558 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1559 {
1560         unsigned int nr_io_queues;
1561
1562         nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1563         nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1564
1565         return nr_io_queues;
1566 }
1567
1568 static int nvme_alloc_io_queues(struct nvme_ctrl *ctrl)
1569 {
1570         unsigned int nr_io_queues;
1571         int ret;
1572
1573         nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1574         ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1575         if (ret)
1576                 return ret;
1577
1578         ctrl->queue_count = nr_io_queues + 1;
1579         if (ctrl->queue_count < 2)
1580                 return 0;
1581
1582         dev_info(ctrl->device,
1583                 "creating %d I/O queues.\n", nr_io_queues);
1584
1585         return nvme_tcp_alloc_io_queues(ctrl);
1586 }
1587
1588 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1589 {
1590         nvme_tcp_stop_io_queues(ctrl);
1591         if (remove) {
1592                 blk_cleanup_queue(ctrl->connect_q);
1593                 blk_mq_free_tag_set(ctrl->tagset);
1594         }
1595         nvme_tcp_free_io_queues(ctrl);
1596 }
1597
1598 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1599 {
1600         int ret;
1601
1602         ret = nvme_alloc_io_queues(ctrl);
1603         if (ret)
1604                 return ret;
1605
1606         if (new) {
1607                 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1608                 if (IS_ERR(ctrl->tagset)) {
1609                         ret = PTR_ERR(ctrl->tagset);
1610                         goto out_free_io_queues;
1611                 }
1612
1613                 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
1614                 if (IS_ERR(ctrl->connect_q)) {
1615                         ret = PTR_ERR(ctrl->connect_q);
1616                         goto out_free_tag_set;
1617                 }
1618         } else {
1619                 blk_mq_update_nr_hw_queues(ctrl->tagset,
1620                         ctrl->queue_count - 1);
1621         }
1622
1623         ret = nvme_tcp_start_io_queues(ctrl);
1624         if (ret)
1625                 goto out_cleanup_connect_q;
1626
1627         return 0;
1628
1629 out_cleanup_connect_q:
1630         if (new)
1631                 blk_cleanup_queue(ctrl->connect_q);
1632 out_free_tag_set:
1633         if (new)
1634                 blk_mq_free_tag_set(ctrl->tagset);
1635 out_free_io_queues:
1636         nvme_tcp_free_io_queues(ctrl);
1637         return ret;
1638 }
1639
1640 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1641 {
1642         nvme_tcp_stop_queue(ctrl, 0);
1643         if (remove) {
1644                 blk_cleanup_queue(ctrl->admin_q);
1645                 blk_mq_free_tag_set(ctrl->admin_tagset);
1646         }
1647         nvme_tcp_free_admin_queue(ctrl);
1648 }
1649
1650 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1651 {
1652         int error;
1653
1654         error = nvme_tcp_alloc_admin_queue(ctrl);
1655         if (error)
1656                 return error;
1657
1658         if (new) {
1659                 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1660                 if (IS_ERR(ctrl->admin_tagset)) {
1661                         error = PTR_ERR(ctrl->admin_tagset);
1662                         goto out_free_queue;
1663                 }
1664
1665                 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1666                 if (IS_ERR(ctrl->admin_q)) {
1667                         error = PTR_ERR(ctrl->admin_q);
1668                         goto out_free_tagset;
1669                 }
1670         }
1671
1672         error = nvme_tcp_start_queue(ctrl, 0);
1673         if (error)
1674                 goto out_cleanup_queue;
1675
1676         error = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
1677         if (error) {
1678                 dev_err(ctrl->device,
1679                         "prop_get NVME_REG_CAP failed\n");
1680                 goto out_stop_queue;
1681         }
1682
1683         ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
1684
1685         error = nvme_enable_ctrl(ctrl, ctrl->cap);
1686         if (error)
1687                 goto out_stop_queue;
1688
1689         error = nvme_init_identify(ctrl);
1690         if (error)
1691                 goto out_stop_queue;
1692
1693         return 0;
1694
1695 out_stop_queue:
1696         nvme_tcp_stop_queue(ctrl, 0);
1697 out_cleanup_queue:
1698         if (new)
1699                 blk_cleanup_queue(ctrl->admin_q);
1700 out_free_tagset:
1701         if (new)
1702                 blk_mq_free_tag_set(ctrl->admin_tagset);
1703 out_free_queue:
1704         nvme_tcp_free_admin_queue(ctrl);
1705         return error;
1706 }
1707
1708 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
1709                 bool remove)
1710 {
1711         blk_mq_quiesce_queue(ctrl->admin_q);
1712         nvme_tcp_stop_queue(ctrl, 0);
1713         blk_mq_tagset_busy_iter(ctrl->admin_tagset, nvme_cancel_request, ctrl);
1714         blk_mq_unquiesce_queue(ctrl->admin_q);
1715         nvme_tcp_destroy_admin_queue(ctrl, remove);
1716 }
1717
1718 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
1719                 bool remove)
1720 {
1721         if (ctrl->queue_count <= 1)
1722                 return;
1723         nvme_stop_queues(ctrl);
1724         nvme_tcp_stop_io_queues(ctrl);
1725         blk_mq_tagset_busy_iter(ctrl->tagset, nvme_cancel_request, ctrl);
1726         if (remove)
1727                 nvme_start_queues(ctrl);
1728         nvme_tcp_destroy_io_queues(ctrl, remove);
1729 }
1730
1731 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
1732 {
1733         /* If we are resetting/deleting then do nothing */
1734         if (ctrl->state != NVME_CTRL_CONNECTING) {
1735                 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
1736                         ctrl->state == NVME_CTRL_LIVE);
1737                 return;
1738         }
1739
1740         if (nvmf_should_reconnect(ctrl)) {
1741                 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
1742                         ctrl->opts->reconnect_delay);
1743                 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
1744                                 ctrl->opts->reconnect_delay * HZ);
1745         } else {
1746                 dev_info(ctrl->device, "Removing controller...\n");
1747                 nvme_delete_ctrl(ctrl);
1748         }
1749 }
1750
1751 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
1752 {
1753         struct nvmf_ctrl_options *opts = ctrl->opts;
1754         int ret = -EINVAL;
1755
1756         ret = nvme_tcp_configure_admin_queue(ctrl, new);
1757         if (ret)
1758                 return ret;
1759
1760         if (ctrl->icdoff) {
1761                 dev_err(ctrl->device, "icdoff is not supported!\n");
1762                 goto destroy_admin;
1763         }
1764
1765         if (opts->queue_size > ctrl->sqsize + 1)
1766                 dev_warn(ctrl->device,
1767                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
1768                         opts->queue_size, ctrl->sqsize + 1);
1769
1770         if (ctrl->sqsize + 1 > ctrl->maxcmd) {
1771                 dev_warn(ctrl->device,
1772                         "sqsize %u > ctrl maxcmd %u, clamping down\n",
1773                         ctrl->sqsize + 1, ctrl->maxcmd);
1774                 ctrl->sqsize = ctrl->maxcmd - 1;
1775         }
1776
1777         if (ctrl->queue_count > 1) {
1778                 ret = nvme_tcp_configure_io_queues(ctrl, new);
1779                 if (ret)
1780                         goto destroy_admin;
1781         }
1782
1783         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
1784                 /* state change failure is ok if we're in DELETING state */
1785                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING);
1786                 ret = -EINVAL;
1787                 goto destroy_io;
1788         }
1789
1790         nvme_start_ctrl(ctrl);
1791         return 0;
1792
1793 destroy_io:
1794         if (ctrl->queue_count > 1)
1795                 nvme_tcp_destroy_io_queues(ctrl, new);
1796 destroy_admin:
1797         nvme_tcp_stop_queue(ctrl, 0);
1798         nvme_tcp_destroy_admin_queue(ctrl, new);
1799         return ret;
1800 }
1801
1802 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
1803 {
1804         struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
1805                         struct nvme_tcp_ctrl, connect_work);
1806         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
1807
1808         ++ctrl->nr_reconnects;
1809
1810         if (nvme_tcp_setup_ctrl(ctrl, false))
1811                 goto requeue;
1812
1813         dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
1814                         ctrl->nr_reconnects);
1815
1816         ctrl->nr_reconnects = 0;
1817
1818         return;
1819
1820 requeue:
1821         dev_info(ctrl->device, "Failed reconnect attempt %d\n",
1822                         ctrl->nr_reconnects);
1823         nvme_tcp_reconnect_or_remove(ctrl);
1824 }
1825
1826 static void nvme_tcp_error_recovery_work(struct work_struct *work)
1827 {
1828         struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
1829                                 struct nvme_tcp_ctrl, err_work);
1830         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
1831
1832         nvme_stop_keep_alive(ctrl);
1833         nvme_tcp_teardown_io_queues(ctrl, false);
1834         /* unquiesce to fail fast pending requests */
1835         nvme_start_queues(ctrl);
1836         nvme_tcp_teardown_admin_queue(ctrl, false);
1837
1838         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
1839                 /* state change failure is ok if we're in DELETING state */
1840                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING);
1841                 return;
1842         }
1843
1844         nvme_tcp_reconnect_or_remove(ctrl);
1845 }
1846
1847 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
1848 {
1849         cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
1850         cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
1851
1852         nvme_tcp_teardown_io_queues(ctrl, shutdown);
1853         if (shutdown)
1854                 nvme_shutdown_ctrl(ctrl);
1855         else
1856                 nvme_disable_ctrl(ctrl, ctrl->cap);
1857         nvme_tcp_teardown_admin_queue(ctrl, shutdown);
1858 }
1859
1860 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
1861 {
1862         nvme_tcp_teardown_ctrl(ctrl, true);
1863 }
1864
1865 static void nvme_reset_ctrl_work(struct work_struct *work)
1866 {
1867         struct nvme_ctrl *ctrl =
1868                 container_of(work, struct nvme_ctrl, reset_work);
1869
1870         nvme_stop_ctrl(ctrl);
1871         nvme_tcp_teardown_ctrl(ctrl, false);
1872
1873         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
1874                 /* state change failure is ok if we're in DELETING state */
1875                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING);
1876                 return;
1877         }
1878
1879         if (nvme_tcp_setup_ctrl(ctrl, false))
1880                 goto out_fail;
1881
1882         return;
1883
1884 out_fail:
1885         ++ctrl->nr_reconnects;
1886         nvme_tcp_reconnect_or_remove(ctrl);
1887 }
1888
1889 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
1890 {
1891         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1892
1893         if (list_empty(&ctrl->list))
1894                 goto free_ctrl;
1895
1896         mutex_lock(&nvme_tcp_ctrl_mutex);
1897         list_del(&ctrl->list);
1898         mutex_unlock(&nvme_tcp_ctrl_mutex);
1899
1900         nvmf_free_options(nctrl->opts);
1901 free_ctrl:
1902         kfree(ctrl->queues);
1903         kfree(ctrl);
1904 }
1905
1906 static void nvme_tcp_set_sg_null(struct nvme_command *c)
1907 {
1908         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1909
1910         sg->addr = 0;
1911         sg->length = 0;
1912         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
1913                         NVME_SGL_FMT_TRANSPORT_A;
1914 }
1915
1916 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
1917                 struct nvme_command *c, u32 data_len)
1918 {
1919         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1920
1921         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
1922         sg->length = cpu_to_le32(data_len);
1923         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
1924 }
1925
1926 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
1927                 u32 data_len)
1928 {
1929         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
1930
1931         sg->addr = 0;
1932         sg->length = cpu_to_le32(data_len);
1933         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
1934                         NVME_SGL_FMT_TRANSPORT_A;
1935 }
1936
1937 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
1938 {
1939         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
1940         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1941         struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
1942         struct nvme_command *cmd = &pdu->cmd;
1943         u8 hdgst = nvme_tcp_hdgst_len(queue);
1944
1945         memset(pdu, 0, sizeof(*pdu));
1946         pdu->hdr.type = nvme_tcp_cmd;
1947         if (queue->hdr_digest)
1948                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
1949         pdu->hdr.hlen = sizeof(*pdu);
1950         pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
1951
1952         cmd->common.opcode = nvme_admin_async_event;
1953         cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1954         cmd->common.flags |= NVME_CMD_SGL_METABUF;
1955         nvme_tcp_set_sg_null(cmd);
1956
1957         ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
1958         ctrl->async_req.offset = 0;
1959         ctrl->async_req.curr_bio = NULL;
1960         ctrl->async_req.data_len = 0;
1961
1962         nvme_tcp_queue_request(&ctrl->async_req);
1963 }
1964
1965 static enum blk_eh_timer_return
1966 nvme_tcp_timeout(struct request *rq, bool reserved)
1967 {
1968         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
1969         struct nvme_tcp_ctrl *ctrl = req->queue->ctrl;
1970         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1971
1972         dev_warn(ctrl->ctrl.device,
1973                 "queue %d: timeout request %#x type %d\n",
1974                 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
1975
1976         if (ctrl->ctrl.state != NVME_CTRL_LIVE) {
1977                 /*
1978                  * Teardown immediately if controller times out while starting
1979                  * or we are already started error recovery. all outstanding
1980                  * requests are completed on shutdown, so we return BLK_EH_DONE.
1981                  */
1982                 flush_work(&ctrl->err_work);
1983                 nvme_tcp_teardown_io_queues(&ctrl->ctrl, false);
1984                 nvme_tcp_teardown_admin_queue(&ctrl->ctrl, false);
1985                 return BLK_EH_DONE;
1986         }
1987
1988         dev_warn(ctrl->ctrl.device, "starting error recovery\n");
1989         nvme_tcp_error_recovery(&ctrl->ctrl);
1990
1991         return BLK_EH_RESET_TIMER;
1992 }
1993
1994 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
1995                         struct request *rq)
1996 {
1997         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
1998         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1999         struct nvme_command *c = &pdu->cmd;
2000
2001         c->common.flags |= NVME_CMD_SGL_METABUF;
2002
2003         if (rq_data_dir(rq) == WRITE && req->data_len &&
2004             req->data_len <= nvme_tcp_inline_data_size(queue))
2005                 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2006         else
2007                 nvme_tcp_set_sg_host_data(c, req->data_len);
2008
2009         return 0;
2010 }
2011
2012 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2013                 struct request *rq)
2014 {
2015         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2016         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2017         struct nvme_tcp_queue *queue = req->queue;
2018         u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2019         blk_status_t ret;
2020
2021         ret = nvme_setup_cmd(ns, rq, &pdu->cmd);
2022         if (ret)
2023                 return ret;
2024
2025         req->state = NVME_TCP_SEND_CMD_PDU;
2026         req->offset = 0;
2027         req->data_sent = 0;
2028         req->pdu_len = 0;
2029         req->pdu_sent = 0;
2030         req->data_len = blk_rq_payload_bytes(rq);
2031         req->curr_bio = rq->bio;
2032
2033         if (rq_data_dir(rq) == WRITE &&
2034             req->data_len <= nvme_tcp_inline_data_size(queue))
2035                 req->pdu_len = req->data_len;
2036         else if (req->curr_bio)
2037                 nvme_tcp_init_iter(req, READ);
2038
2039         pdu->hdr.type = nvme_tcp_cmd;
2040         pdu->hdr.flags = 0;
2041         if (queue->hdr_digest)
2042                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2043         if (queue->data_digest && req->pdu_len) {
2044                 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2045                 ddgst = nvme_tcp_ddgst_len(queue);
2046         }
2047         pdu->hdr.hlen = sizeof(*pdu);
2048         pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2049         pdu->hdr.plen =
2050                 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2051
2052         ret = nvme_tcp_map_data(queue, rq);
2053         if (unlikely(ret)) {
2054                 dev_err(queue->ctrl->ctrl.device,
2055                         "Failed to map data (%d)\n", ret);
2056                 return ret;
2057         }
2058
2059         return 0;
2060 }
2061
2062 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2063                 const struct blk_mq_queue_data *bd)
2064 {
2065         struct nvme_ns *ns = hctx->queue->queuedata;
2066         struct nvme_tcp_queue *queue = hctx->driver_data;
2067         struct request *rq = bd->rq;
2068         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2069         bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2070         blk_status_t ret;
2071
2072         if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2073                 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
2074
2075         ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2076         if (unlikely(ret))
2077                 return ret;
2078
2079         blk_mq_start_request(rq);
2080
2081         nvme_tcp_queue_request(req);
2082
2083         return BLK_STS_OK;
2084 }
2085
2086 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2087 {
2088         struct nvme_tcp_ctrl *ctrl = set->driver_data;
2089
2090         set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2091         set->map[HCTX_TYPE_READ].nr_queues = ctrl->ctrl.opts->nr_io_queues;
2092         if (ctrl->ctrl.opts->nr_write_queues) {
2093                 /* separate read/write queues */
2094                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2095                                 ctrl->ctrl.opts->nr_write_queues;
2096                 set->map[HCTX_TYPE_READ].queue_offset =
2097                                 ctrl->ctrl.opts->nr_write_queues;
2098         } else {
2099                 /* mixed read/write queues */
2100                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2101                                 ctrl->ctrl.opts->nr_io_queues;
2102                 set->map[HCTX_TYPE_READ].queue_offset = 0;
2103         }
2104         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2105         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2106         return 0;
2107 }
2108
2109 static struct blk_mq_ops nvme_tcp_mq_ops = {
2110         .queue_rq       = nvme_tcp_queue_rq,
2111         .complete       = nvme_complete_rq,
2112         .init_request   = nvme_tcp_init_request,
2113         .exit_request   = nvme_tcp_exit_request,
2114         .init_hctx      = nvme_tcp_init_hctx,
2115         .timeout        = nvme_tcp_timeout,
2116         .map_queues     = nvme_tcp_map_queues,
2117 };
2118
2119 static struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2120         .queue_rq       = nvme_tcp_queue_rq,
2121         .complete       = nvme_complete_rq,
2122         .init_request   = nvme_tcp_init_request,
2123         .exit_request   = nvme_tcp_exit_request,
2124         .init_hctx      = nvme_tcp_init_admin_hctx,
2125         .timeout        = nvme_tcp_timeout,
2126 };
2127
2128 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2129         .name                   = "tcp",
2130         .module                 = THIS_MODULE,
2131         .flags                  = NVME_F_FABRICS,
2132         .reg_read32             = nvmf_reg_read32,
2133         .reg_read64             = nvmf_reg_read64,
2134         .reg_write32            = nvmf_reg_write32,
2135         .free_ctrl              = nvme_tcp_free_ctrl,
2136         .submit_async_event     = nvme_tcp_submit_async_event,
2137         .delete_ctrl            = nvme_tcp_delete_ctrl,
2138         .get_address            = nvmf_get_address,
2139 };
2140
2141 static bool
2142 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2143 {
2144         struct nvme_tcp_ctrl *ctrl;
2145         bool found = false;
2146
2147         mutex_lock(&nvme_tcp_ctrl_mutex);
2148         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2149                 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2150                 if (found)
2151                         break;
2152         }
2153         mutex_unlock(&nvme_tcp_ctrl_mutex);
2154
2155         return found;
2156 }
2157
2158 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2159                 struct nvmf_ctrl_options *opts)
2160 {
2161         struct nvme_tcp_ctrl *ctrl;
2162         int ret;
2163
2164         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2165         if (!ctrl)
2166                 return ERR_PTR(-ENOMEM);
2167
2168         INIT_LIST_HEAD(&ctrl->list);
2169         ctrl->ctrl.opts = opts;
2170         ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 1;
2171         ctrl->ctrl.sqsize = opts->queue_size - 1;
2172         ctrl->ctrl.kato = opts->kato;
2173
2174         INIT_DELAYED_WORK(&ctrl->connect_work,
2175                         nvme_tcp_reconnect_ctrl_work);
2176         INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2177         INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2178
2179         if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2180                 opts->trsvcid =
2181                         kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2182                 if (!opts->trsvcid) {
2183                         ret = -ENOMEM;
2184                         goto out_free_ctrl;
2185                 }
2186                 opts->mask |= NVMF_OPT_TRSVCID;
2187         }
2188
2189         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2190                         opts->traddr, opts->trsvcid, &ctrl->addr);
2191         if (ret) {
2192                 pr_err("malformed address passed: %s:%s\n",
2193                         opts->traddr, opts->trsvcid);
2194                 goto out_free_ctrl;
2195         }
2196
2197         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2198                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2199                         opts->host_traddr, NULL, &ctrl->src_addr);
2200                 if (ret) {
2201                         pr_err("malformed src address passed: %s\n",
2202                                opts->host_traddr);
2203                         goto out_free_ctrl;
2204                 }
2205         }
2206
2207         if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2208                 ret = -EALREADY;
2209                 goto out_free_ctrl;
2210         }
2211
2212         ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2213                                 GFP_KERNEL);
2214         if (!ctrl->queues) {
2215                 ret = -ENOMEM;
2216                 goto out_free_ctrl;
2217         }
2218
2219         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2220         if (ret)
2221                 goto out_kfree_queues;
2222
2223         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2224                 WARN_ON_ONCE(1);
2225                 ret = -EINTR;
2226                 goto out_uninit_ctrl;
2227         }
2228
2229         ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2230         if (ret)
2231                 goto out_uninit_ctrl;
2232
2233         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2234                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2235
2236         nvme_get_ctrl(&ctrl->ctrl);
2237
2238         mutex_lock(&nvme_tcp_ctrl_mutex);
2239         list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2240         mutex_unlock(&nvme_tcp_ctrl_mutex);
2241
2242         return &ctrl->ctrl;
2243
2244 out_uninit_ctrl:
2245         nvme_uninit_ctrl(&ctrl->ctrl);
2246         nvme_put_ctrl(&ctrl->ctrl);
2247         if (ret > 0)
2248                 ret = -EIO;
2249         return ERR_PTR(ret);
2250 out_kfree_queues:
2251         kfree(ctrl->queues);
2252 out_free_ctrl:
2253         kfree(ctrl);
2254         return ERR_PTR(ret);
2255 }
2256
2257 static struct nvmf_transport_ops nvme_tcp_transport = {
2258         .name           = "tcp",
2259         .module         = THIS_MODULE,
2260         .required_opts  = NVMF_OPT_TRADDR,
2261         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2262                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2263                           NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2264                           NVMF_OPT_NR_WRITE_QUEUES,
2265         .create_ctrl    = nvme_tcp_create_ctrl,
2266 };
2267
2268 static int __init nvme_tcp_init_module(void)
2269 {
2270         nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2271                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2272         if (!nvme_tcp_wq)
2273                 return -ENOMEM;
2274
2275         nvmf_register_transport(&nvme_tcp_transport);
2276         return 0;
2277 }
2278
2279 static void __exit nvme_tcp_cleanup_module(void)
2280 {
2281         struct nvme_tcp_ctrl *ctrl;
2282
2283         nvmf_unregister_transport(&nvme_tcp_transport);
2284
2285         mutex_lock(&nvme_tcp_ctrl_mutex);
2286         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2287                 nvme_delete_ctrl(&ctrl->ctrl);
2288         mutex_unlock(&nvme_tcp_ctrl_mutex);
2289         flush_workqueue(nvme_delete_wq);
2290
2291         destroy_workqueue(nvme_tcp_wq);
2292 }
2293
2294 module_init(nvme_tcp_init_module);
2295 module_exit(nvme_tcp_cleanup_module);
2296
2297 MODULE_LICENSE("GPL v2");