88a0166f54776836b9207bc2f8e6ee3a730e1617
[muen/linux.git] / drivers / crypto / cavium / cpt / cptvf_main.c
1 /*
2  * Copyright (C) 2016 Cavium, Inc.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of version 2 of the GNU General Public License
6  * as published by the Free Software Foundation.
7  */
8
9 #include <linux/interrupt.h>
10 #include <linux/module.h>
11
12 #include "cptvf.h"
13
14 #define DRV_NAME        "thunder-cptvf"
15 #define DRV_VERSION     "1.0"
16
17 struct cptvf_wqe {
18         struct tasklet_struct twork;
19         void *cptvf;
20         u32 qno;
21 };
22
23 struct cptvf_wqe_info {
24         struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
25 };
26
27 static void vq_work_handler(unsigned long data)
28 {
29         struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
30         struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
31
32         vq_post_process(cwqe->cptvf, cwqe->qno);
33 }
34
35 static int init_worker_threads(struct cpt_vf *cptvf)
36 {
37         struct pci_dev *pdev = cptvf->pdev;
38         struct cptvf_wqe_info *cwqe_info;
39         int i;
40
41         cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
42         if (!cwqe_info)
43                 return -ENOMEM;
44
45         if (cptvf->nr_queues) {
46                 dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
47                          cptvf->nr_queues);
48         }
49
50         for (i = 0; i < cptvf->nr_queues; i++) {
51                 tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
52                              (u64)cwqe_info);
53                 cwqe_info->vq_wqe[i].qno = i;
54                 cwqe_info->vq_wqe[i].cptvf = cptvf;
55         }
56
57         cptvf->wqe_info = cwqe_info;
58
59         return 0;
60 }
61
62 static void cleanup_worker_threads(struct cpt_vf *cptvf)
63 {
64         struct cptvf_wqe_info *cwqe_info;
65         struct pci_dev *pdev = cptvf->pdev;
66         int i;
67
68         cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
69         if (!cwqe_info)
70                 return;
71
72         if (cptvf->nr_queues) {
73                 dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
74                          cptvf->nr_queues);
75         }
76
77         for (i = 0; i < cptvf->nr_queues; i++)
78                 tasklet_kill(&cwqe_info->vq_wqe[i].twork);
79
80         kzfree(cwqe_info);
81         cptvf->wqe_info = NULL;
82 }
83
84 static void free_pending_queues(struct pending_qinfo *pqinfo)
85 {
86         int i;
87         struct pending_queue *queue;
88
89         for_each_pending_queue(pqinfo, queue, i) {
90                 if (!queue->head)
91                         continue;
92
93                 /* free single queue */
94                 kzfree((queue->head));
95
96                 queue->front = 0;
97                 queue->rear = 0;
98
99                 return;
100         }
101
102         pqinfo->qlen = 0;
103         pqinfo->nr_queues = 0;
104 }
105
106 static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
107                                 u32 nr_queues)
108 {
109         u32 i;
110         size_t size;
111         int ret;
112         struct pending_queue *queue = NULL;
113
114         pqinfo->nr_queues = nr_queues;
115         pqinfo->qlen = qlen;
116
117         size = (qlen * sizeof(struct pending_entry));
118
119         for_each_pending_queue(pqinfo, queue, i) {
120                 queue->head = kzalloc((size), GFP_KERNEL);
121                 if (!queue->head) {
122                         ret = -ENOMEM;
123                         goto pending_qfail;
124                 }
125
126                 queue->front = 0;
127                 queue->rear = 0;
128                 atomic64_set((&queue->pending_count), (0));
129
130                 /* init queue spin lock */
131                 spin_lock_init(&queue->lock);
132         }
133
134         return 0;
135
136 pending_qfail:
137         free_pending_queues(pqinfo);
138
139         return ret;
140 }
141
142 static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
143 {
144         struct pci_dev *pdev = cptvf->pdev;
145         int ret;
146
147         if (!nr_queues)
148                 return 0;
149
150         ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
151         if (ret) {
152                 dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
153                         nr_queues);
154                 return ret;
155         }
156
157         return 0;
158 }
159
160 static void cleanup_pending_queues(struct cpt_vf *cptvf)
161 {
162         struct pci_dev *pdev = cptvf->pdev;
163
164         if (!cptvf->nr_queues)
165                 return;
166
167         dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
168                  cptvf->nr_queues);
169         free_pending_queues(&cptvf->pqinfo);
170 }
171
172 static void free_command_queues(struct cpt_vf *cptvf,
173                                 struct command_qinfo *cqinfo)
174 {
175         int i;
176         struct command_queue *queue = NULL;
177         struct command_chunk *chunk = NULL;
178         struct pci_dev *pdev = cptvf->pdev;
179         struct hlist_node *node;
180
181         /* clean up for each queue */
182         for (i = 0; i < cptvf->nr_queues; i++) {
183                 queue = &cqinfo->queue[i];
184                 if (hlist_empty(&cqinfo->queue[i].chead))
185                         continue;
186
187                 hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead,
188                                           nextchunk) {
189                         dma_free_coherent(&pdev->dev, chunk->size,
190                                           chunk->head,
191                                           chunk->dma_addr);
192                         chunk->head = NULL;
193                         chunk->dma_addr = 0;
194                         hlist_del(&chunk->nextchunk);
195                         kzfree(chunk);
196                 }
197
198                 queue->nchunks = 0;
199                 queue->idx = 0;
200         }
201
202         /* common cleanup */
203         cqinfo->cmd_size = 0;
204 }
205
206 static int alloc_command_queues(struct cpt_vf *cptvf,
207                                 struct command_qinfo *cqinfo, size_t cmd_size,
208                                 u32 qlen)
209 {
210         int i;
211         size_t q_size;
212         struct command_queue *queue = NULL;
213         struct pci_dev *pdev = cptvf->pdev;
214
215         /* common init */
216         cqinfo->cmd_size = cmd_size;
217         /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
218         cptvf->qsize = min(qlen, cqinfo->qchunksize) *
219                         CPT_NEXT_CHUNK_PTR_SIZE + 1;
220         /* Qsize in bytes to create space for alignment */
221         q_size = qlen * cqinfo->cmd_size;
222
223         /* per queue initialization */
224         for (i = 0; i < cptvf->nr_queues; i++) {
225                 size_t c_size = 0;
226                 size_t rem_q_size = q_size;
227                 struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
228                 u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
229
230                 queue = &cqinfo->queue[i];
231                 INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
232                 do {
233                         curr = kzalloc(sizeof(*curr), GFP_KERNEL);
234                         if (!curr)
235                                 goto cmd_qfail;
236
237                         c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
238                                         rem_q_size;
239                         curr->head = (u8 *)dma_alloc_coherent(&pdev->dev,
240                                                               c_size + CPT_NEXT_CHUNK_PTR_SIZE,
241                                                               &curr->dma_addr,
242                                                               GFP_KERNEL);
243                         if (!curr->head) {
244                                 dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
245                                         i, queue->nchunks);
246                                 kfree(curr);
247                                 goto cmd_qfail;
248                         }
249
250                         curr->size = c_size;
251                         if (queue->nchunks == 0) {
252                                 hlist_add_head(&curr->nextchunk,
253                                                &cqinfo->queue[i].chead);
254                                 first = curr;
255                         } else {
256                                 hlist_add_behind(&curr->nextchunk,
257                                                  &last->nextchunk);
258                         }
259
260                         queue->nchunks++;
261                         rem_q_size -= c_size;
262                         if (last)
263                                 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
264
265                         last = curr;
266                 } while (rem_q_size);
267
268                 /* Make the queue circular */
269                 /* Tie back last chunk entry to head */
270                 curr = first;
271                 *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
272                 queue->qhead = curr;
273                 spin_lock_init(&queue->lock);
274         }
275         return 0;
276
277 cmd_qfail:
278         free_command_queues(cptvf, cqinfo);
279         return -ENOMEM;
280 }
281
282 static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
283 {
284         struct pci_dev *pdev = cptvf->pdev;
285         int ret;
286
287         /* setup AE command queues */
288         ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
289                                    qlen);
290         if (ret) {
291                 dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
292                         cptvf->nr_queues);
293                 return ret;
294         }
295
296         return ret;
297 }
298
299 static void cleanup_command_queues(struct cpt_vf *cptvf)
300 {
301         struct pci_dev *pdev = cptvf->pdev;
302
303         if (!cptvf->nr_queues)
304                 return;
305
306         dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
307                  cptvf->nr_queues);
308         free_command_queues(cptvf, &cptvf->cqinfo);
309 }
310
311 static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
312 {
313         cleanup_worker_threads(cptvf);
314         cleanup_pending_queues(cptvf);
315         cleanup_command_queues(cptvf);
316 }
317
318 static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
319 {
320         struct pci_dev *pdev = cptvf->pdev;
321         int ret = 0;
322         u32 max_dev_queues = 0;
323
324         max_dev_queues = CPT_NUM_QS_PER_VF;
325         /* possible cpus */
326         nr_queues = min_t(u32, nr_queues, max_dev_queues);
327         cptvf->nr_queues = nr_queues;
328
329         ret = init_command_queues(cptvf, qlen);
330         if (ret) {
331                 dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
332                         nr_queues);
333                 return ret;
334         }
335
336         ret = init_pending_queues(cptvf, qlen, nr_queues);
337         if (ret) {
338                 dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
339                         nr_queues);
340                 goto setup_pqfail;
341         }
342
343         /* Create worker threads for BH processing */
344         ret = init_worker_threads(cptvf);
345         if (ret) {
346                 dev_err(&pdev->dev, "Failed to setup worker threads\n");
347                 goto init_work_fail;
348         }
349
350         return 0;
351
352 init_work_fail:
353         cleanup_worker_threads(cptvf);
354         cleanup_pending_queues(cptvf);
355
356 setup_pqfail:
357         cleanup_command_queues(cptvf);
358
359         return ret;
360 }
361
362 static void cptvf_free_irq_affinity(struct cpt_vf *cptvf, int vec)
363 {
364         irq_set_affinity_hint(pci_irq_vector(cptvf->pdev, vec), NULL);
365         free_cpumask_var(cptvf->affinity_mask[vec]);
366 }
367
368 static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
369 {
370         union cptx_vqx_ctl vqx_ctl;
371
372         vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
373         vqx_ctl.s.ena = val;
374         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
375 }
376
377 void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
378 {
379         union cptx_vqx_doorbell vqx_dbell;
380
381         vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
382                                      CPTX_VQX_DOORBELL(0, 0));
383         vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
384         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
385                         vqx_dbell.u);
386 }
387
388 static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
389 {
390         union cptx_vqx_inprog vqx_inprg;
391
392         vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
393         vqx_inprg.s.inflight = val;
394         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
395 }
396
397 static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
398 {
399         union cptx_vqx_done_wait vqx_dwait;
400
401         vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
402                                      CPTX_VQX_DONE_WAIT(0, 0));
403         vqx_dwait.s.num_wait = val;
404         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
405                         vqx_dwait.u);
406 }
407
408 static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
409 {
410         union cptx_vqx_done_wait vqx_dwait;
411
412         vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
413                                      CPTX_VQX_DONE_WAIT(0, 0));
414         vqx_dwait.s.time_wait = time;
415         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
416                         vqx_dwait.u);
417 }
418
419 static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
420 {
421         union cptx_vqx_misc_ena_w1s vqx_misc_ena;
422
423         vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
424                                         CPTX_VQX_MISC_ENA_W1S(0, 0));
425         /* Set mbox(0) interupts for the requested vf */
426         vqx_misc_ena.s.swerr = 1;
427         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
428                         vqx_misc_ena.u);
429 }
430
431 static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
432 {
433         union cptx_vqx_misc_ena_w1s vqx_misc_ena;
434
435         vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
436                                         CPTX_VQX_MISC_ENA_W1S(0, 0));
437         /* Set mbox(0) interupts for the requested vf */
438         vqx_misc_ena.s.mbox = 1;
439         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
440                         vqx_misc_ena.u);
441 }
442
443 static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
444 {
445         union cptx_vqx_done_ena_w1s vqx_done_ena;
446
447         vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
448                                         CPTX_VQX_DONE_ENA_W1S(0, 0));
449         /* Set DONE interrupt for the requested vf */
450         vqx_done_ena.s.done = 1;
451         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
452                         vqx_done_ena.u);
453 }
454
455 static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
456 {
457         union cptx_vqx_misc_int vqx_misc_int;
458
459         vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
460                                         CPTX_VQX_MISC_INT(0, 0));
461         /* W1C for the VF */
462         vqx_misc_int.s.dovf = 1;
463         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
464                         vqx_misc_int.u);
465 }
466
467 static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
468 {
469         union cptx_vqx_misc_int vqx_misc_int;
470
471         vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
472                                         CPTX_VQX_MISC_INT(0, 0));
473         /* W1C for the VF */
474         vqx_misc_int.s.irde = 1;
475         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
476                         vqx_misc_int.u);
477 }
478
479 static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
480 {
481         union cptx_vqx_misc_int vqx_misc_int;
482
483         vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
484                                         CPTX_VQX_MISC_INT(0, 0));
485         /* W1C for the VF */
486         vqx_misc_int.s.nwrp = 1;
487         cpt_write_csr64(cptvf->reg_base,
488                         CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
489 }
490
491 static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
492 {
493         union cptx_vqx_misc_int vqx_misc_int;
494
495         vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
496                                         CPTX_VQX_MISC_INT(0, 0));
497         /* W1C for the VF */
498         vqx_misc_int.s.mbox = 1;
499         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
500                         vqx_misc_int.u);
501 }
502
503 static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
504 {
505         union cptx_vqx_misc_int vqx_misc_int;
506
507         vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
508                                         CPTX_VQX_MISC_INT(0, 0));
509         /* W1C for the VF */
510         vqx_misc_int.s.swerr = 1;
511         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
512                         vqx_misc_int.u);
513 }
514
515 static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
516 {
517         return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
518 }
519
520 static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
521 {
522         struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
523         struct pci_dev *pdev = cptvf->pdev;
524         u64 intr;
525
526         intr = cptvf_read_vf_misc_intr_status(cptvf);
527         /*Check for MISC interrupt types*/
528         if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
529                 dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
530                         intr, cptvf->vfid);
531                 cptvf_handle_mbox_intr(cptvf);
532                 cptvf_clear_mbox_intr(cptvf);
533         } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
534                 cptvf_clear_dovf_intr(cptvf);
535                 /*Clear doorbell count*/
536                 cptvf_write_vq_doorbell(cptvf, 0);
537                 dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
538                         intr, cptvf->vfid);
539         } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
540                 cptvf_clear_irde_intr(cptvf);
541                 dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
542                         intr, cptvf->vfid);
543         } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
544                 cptvf_clear_nwrp_intr(cptvf);
545                 dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
546                         intr, cptvf->vfid);
547         } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
548                 cptvf_clear_swerr_intr(cptvf);
549                 dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
550                         intr, cptvf->vfid);
551         } else {
552                 dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
553                         cptvf->vfid);
554         }
555
556         return IRQ_HANDLED;
557 }
558
559 static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
560                                                  int qno)
561 {
562         struct cptvf_wqe_info *nwqe_info;
563
564         if (unlikely(qno >= cptvf->nr_queues))
565                 return NULL;
566         nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
567
568         return &nwqe_info->vq_wqe[qno];
569 }
570
571 static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
572 {
573         union cptx_vqx_done vqx_done;
574
575         vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
576         return vqx_done.s.done;
577 }
578
579 static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
580                                            u32 ackcnt)
581 {
582         union cptx_vqx_done_ack vqx_dack_cnt;
583
584         vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
585                                         CPTX_VQX_DONE_ACK(0, 0));
586         vqx_dack_cnt.s.done_ack = ackcnt;
587         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
588                         vqx_dack_cnt.u);
589 }
590
591 static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
592 {
593         struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
594         struct pci_dev *pdev = cptvf->pdev;
595         /* Read the number of completions */
596         u32 intr = cptvf_read_vq_done_count(cptvf);
597
598         if (intr) {
599                 struct cptvf_wqe *wqe;
600
601                 /* Acknowledge the number of
602                  * scheduled completions for processing
603                  */
604                 cptvf_write_vq_done_ack(cptvf, intr);
605                 wqe = get_cptvf_vq_wqe(cptvf, 0);
606                 if (unlikely(!wqe)) {
607                         dev_err(&pdev->dev, "No work to schedule for VF (%d)",
608                                 cptvf->vfid);
609                         return IRQ_NONE;
610                 }
611                 tasklet_hi_schedule(&wqe->twork);
612         }
613
614         return IRQ_HANDLED;
615 }
616
617 static void cptvf_set_irq_affinity(struct cpt_vf *cptvf, int vec)
618 {
619         struct pci_dev *pdev = cptvf->pdev;
620         int cpu;
621
622         if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
623                                 GFP_KERNEL)) {
624                 dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
625                         cptvf->vfid);
626                 return;
627         }
628
629         cpu = cptvf->vfid % num_online_cpus();
630         cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
631                         cptvf->affinity_mask[vec]);
632         irq_set_affinity_hint(pci_irq_vector(pdev, vec),
633                         cptvf->affinity_mask[vec]);
634 }
635
636 static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
637 {
638         union cptx_vqx_saddr vqx_saddr;
639
640         vqx_saddr.u = val;
641         cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
642 }
643
644 static void cptvf_device_init(struct cpt_vf *cptvf)
645 {
646         u64 base_addr = 0;
647
648         /* Disable the VQ */
649         cptvf_write_vq_ctl(cptvf, 0);
650         /* Reset the doorbell */
651         cptvf_write_vq_doorbell(cptvf, 0);
652         /* Clear inflight */
653         cptvf_write_vq_inprog(cptvf, 0);
654         /* Write VQ SADDR */
655         /* TODO: for now only one queue, so hard coded */
656         base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
657         cptvf_write_vq_saddr(cptvf, base_addr);
658         /* Configure timerhold / coalescence */
659         cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
660         cptvf_write_vq_done_numwait(cptvf, 1);
661         /* Enable the VQ */
662         cptvf_write_vq_ctl(cptvf, 1);
663         /* Flag the VF ready */
664         cptvf->flags |= CPT_FLAG_DEVICE_READY;
665 }
666
667 static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
668 {
669         struct device *dev = &pdev->dev;
670         struct cpt_vf *cptvf;
671         int    err;
672
673         cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
674         if (!cptvf)
675                 return -ENOMEM;
676
677         pci_set_drvdata(pdev, cptvf);
678         cptvf->pdev = pdev;
679         err = pci_enable_device(pdev);
680         if (err) {
681                 dev_err(dev, "Failed to enable PCI device\n");
682                 pci_set_drvdata(pdev, NULL);
683                 return err;
684         }
685
686         err = pci_request_regions(pdev, DRV_NAME);
687         if (err) {
688                 dev_err(dev, "PCI request regions failed 0x%x\n", err);
689                 goto cptvf_err_disable_device;
690         }
691         /* Mark as VF driver */
692         cptvf->flags |= CPT_FLAG_VF_DRIVER;
693         err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
694         if (err) {
695                 dev_err(dev, "Unable to get usable DMA configuration\n");
696                 goto cptvf_err_release_regions;
697         }
698
699         err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
700         if (err) {
701                 dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
702                 goto cptvf_err_release_regions;
703         }
704
705         /* MAP PF's configuration registers */
706         cptvf->reg_base = pcim_iomap(pdev, 0, 0);
707         if (!cptvf->reg_base) {
708                 dev_err(dev, "Cannot map config register space, aborting\n");
709                 err = -ENOMEM;
710                 goto cptvf_err_release_regions;
711         }
712
713         cptvf->node = dev_to_node(&pdev->dev);
714         err = pci_alloc_irq_vectors(pdev, CPT_VF_MSIX_VECTORS,
715                         CPT_VF_MSIX_VECTORS, PCI_IRQ_MSIX);
716         if (err < 0) {
717                 dev_err(dev, "Request for #%d msix vectors failed\n",
718                         CPT_VF_MSIX_VECTORS);
719                 goto cptvf_err_release_regions;
720         }
721
722         err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC),
723                           cptvf_misc_intr_handler, 0, "CPT VF misc intr",
724                           cptvf);
725         if (err) {
726                 dev_err(dev, "Request misc irq failed");
727                 goto cptvf_free_vectors;
728         }
729
730         /* Enable mailbox interrupt */
731         cptvf_enable_mbox_interrupts(cptvf);
732         cptvf_enable_swerr_interrupts(cptvf);
733
734         /* Check ready with PF */
735         /* Gets chip ID / device Id from PF if ready */
736         err = cptvf_check_pf_ready(cptvf);
737         if (err) {
738                 dev_err(dev, "PF not responding to READY msg");
739                 goto cptvf_free_misc_irq;
740         }
741
742         /* CPT VF software resources initialization */
743         cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
744         err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
745         if (err) {
746                 dev_err(dev, "cptvf_sw_init() failed");
747                 goto cptvf_free_misc_irq;
748         }
749         /* Convey VQ LEN to PF */
750         err = cptvf_send_vq_size_msg(cptvf);
751         if (err) {
752                 dev_err(dev, "PF not responding to QLEN msg");
753                 goto cptvf_free_misc_irq;
754         }
755
756         /* CPT VF device initialization */
757         cptvf_device_init(cptvf);
758         /* Send msg to PF to assign currnet Q to required group */
759         cptvf->vfgrp = 1;
760         err = cptvf_send_vf_to_grp_msg(cptvf);
761         if (err) {
762                 dev_err(dev, "PF not responding to VF_GRP msg");
763                 goto cptvf_free_misc_irq;
764         }
765
766         cptvf->priority = 1;
767         err = cptvf_send_vf_priority_msg(cptvf);
768         if (err) {
769                 dev_err(dev, "PF not responding to VF_PRIO msg");
770                 goto cptvf_free_misc_irq;
771         }
772
773         err = request_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE),
774                           cptvf_done_intr_handler, 0, "CPT VF done intr",
775                           cptvf);
776         if (err) {
777                 dev_err(dev, "Request done irq failed\n");
778                 goto cptvf_free_misc_irq;
779         }
780
781         /* Enable mailbox interrupt */
782         cptvf_enable_done_interrupts(cptvf);
783
784         /* Set irq affinity masks */
785         cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
786         cptvf_set_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
787
788         err = cptvf_send_vf_up(cptvf);
789         if (err) {
790                 dev_err(dev, "PF not responding to UP msg");
791                 goto cptvf_free_irq_affinity;
792         }
793         err = cvm_crypto_init(cptvf);
794         if (err) {
795                 dev_err(dev, "Algorithm register failed\n");
796                 goto cptvf_free_irq_affinity;
797         }
798         return 0;
799
800 cptvf_free_irq_affinity:
801         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
802         cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
803 cptvf_free_misc_irq:
804         free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
805 cptvf_free_vectors:
806         pci_free_irq_vectors(cptvf->pdev);
807 cptvf_err_release_regions:
808         pci_release_regions(pdev);
809 cptvf_err_disable_device:
810         pci_disable_device(pdev);
811         pci_set_drvdata(pdev, NULL);
812
813         return err;
814 }
815
816 static void cptvf_remove(struct pci_dev *pdev)
817 {
818         struct cpt_vf *cptvf = pci_get_drvdata(pdev);
819
820         if (!cptvf) {
821                 dev_err(&pdev->dev, "Invalid CPT-VF device\n");
822                 return;
823         }
824
825         /* Convey DOWN to PF */
826         if (cptvf_send_vf_down(cptvf)) {
827                 dev_err(&pdev->dev, "PF not responding to DOWN msg");
828         } else {
829                 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_DONE);
830                 cptvf_free_irq_affinity(cptvf, CPT_VF_INT_VEC_E_MISC);
831                 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_DONE), cptvf);
832                 free_irq(pci_irq_vector(pdev, CPT_VF_INT_VEC_E_MISC), cptvf);
833                 pci_free_irq_vectors(cptvf->pdev);
834                 cptvf_sw_cleanup(cptvf);
835                 pci_set_drvdata(pdev, NULL);
836                 pci_release_regions(pdev);
837                 pci_disable_device(pdev);
838                 cvm_crypto_exit();
839         }
840 }
841
842 static void cptvf_shutdown(struct pci_dev *pdev)
843 {
844         cptvf_remove(pdev);
845 }
846
847 /* Supported devices */
848 static const struct pci_device_id cptvf_id_table[] = {
849         {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
850         { 0, }  /* end of table */
851 };
852
853 static struct pci_driver cptvf_pci_driver = {
854         .name = DRV_NAME,
855         .id_table = cptvf_id_table,
856         .probe = cptvf_probe,
857         .remove = cptvf_remove,
858         .shutdown = cptvf_shutdown,
859 };
860
861 module_pci_driver(cptvf_pci_driver);
862
863 MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
864 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
865 MODULE_LICENSE("GPL v2");
866 MODULE_VERSION(DRV_VERSION);
867 MODULE_DEVICE_TABLE(pci, cptvf_id_table);