88750a34e85987f352ef0e32ded6b22fb8f5115f
[muen/linux.git] / drivers / dma / pl330.c
1 /*
2  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
3  *              http://www.samsung.com
4  *
5  * Copyright (C) 2010 Samsung Electronics Co. Ltd.
6  *      Jaswinder Singh <jassi.brar@samsung.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  */
13
14 #include <linux/kernel.h>
15 #include <linux/io.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <linux/string.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/dmaengine.h>
24 #include <linux/amba/bus.h>
25 #include <linux/scatterlist.h>
26 #include <linux/of.h>
27 #include <linux/of_dma.h>
28 #include <linux/err.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/bug.h>
31
32 #include "dmaengine.h"
33 #define PL330_MAX_CHAN          8
34 #define PL330_MAX_IRQS          32
35 #define PL330_MAX_PERI          32
36 #define PL330_MAX_BURST         16
37
38 #define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
39
40 enum pl330_cachectrl {
41         CCTRL0,         /* Noncacheable and nonbufferable */
42         CCTRL1,         /* Bufferable only */
43         CCTRL2,         /* Cacheable, but do not allocate */
44         CCTRL3,         /* Cacheable and bufferable, but do not allocate */
45         INVALID1,       /* AWCACHE = 0x1000 */
46         INVALID2,
47         CCTRL6,         /* Cacheable write-through, allocate on writes only */
48         CCTRL7,         /* Cacheable write-back, allocate on writes only */
49 };
50
51 enum pl330_byteswap {
52         SWAP_NO,
53         SWAP_2,
54         SWAP_4,
55         SWAP_8,
56         SWAP_16,
57 };
58
59 /* Register and Bit field Definitions */
60 #define DS                      0x0
61 #define DS_ST_STOP              0x0
62 #define DS_ST_EXEC              0x1
63 #define DS_ST_CMISS             0x2
64 #define DS_ST_UPDTPC            0x3
65 #define DS_ST_WFE               0x4
66 #define DS_ST_ATBRR             0x5
67 #define DS_ST_QBUSY             0x6
68 #define DS_ST_WFP               0x7
69 #define DS_ST_KILL              0x8
70 #define DS_ST_CMPLT             0x9
71 #define DS_ST_FLTCMP            0xe
72 #define DS_ST_FAULT             0xf
73
74 #define DPC                     0x4
75 #define INTEN                   0x20
76 #define ES                      0x24
77 #define INTSTATUS               0x28
78 #define INTCLR                  0x2c
79 #define FSM                     0x30
80 #define FSC                     0x34
81 #define FTM                     0x38
82
83 #define _FTC                    0x40
84 #define FTC(n)                  (_FTC + (n)*0x4)
85
86 #define _CS                     0x100
87 #define CS(n)                   (_CS + (n)*0x8)
88 #define CS_CNS                  (1 << 21)
89
90 #define _CPC                    0x104
91 #define CPC(n)                  (_CPC + (n)*0x8)
92
93 #define _SA                     0x400
94 #define SA(n)                   (_SA + (n)*0x20)
95
96 #define _DA                     0x404
97 #define DA(n)                   (_DA + (n)*0x20)
98
99 #define _CC                     0x408
100 #define CC(n)                   (_CC + (n)*0x20)
101
102 #define CC_SRCINC               (1 << 0)
103 #define CC_DSTINC               (1 << 14)
104 #define CC_SRCPRI               (1 << 8)
105 #define CC_DSTPRI               (1 << 22)
106 #define CC_SRCNS                (1 << 9)
107 #define CC_DSTNS                (1 << 23)
108 #define CC_SRCIA                (1 << 10)
109 #define CC_DSTIA                (1 << 24)
110 #define CC_SRCBRSTLEN_SHFT      4
111 #define CC_DSTBRSTLEN_SHFT      18
112 #define CC_SRCBRSTSIZE_SHFT     1
113 #define CC_DSTBRSTSIZE_SHFT     15
114 #define CC_SRCCCTRL_SHFT        11
115 #define CC_SRCCCTRL_MASK        0x7
116 #define CC_DSTCCTRL_SHFT        25
117 #define CC_DRCCCTRL_MASK        0x7
118 #define CC_SWAP_SHFT            28
119
120 #define _LC0                    0x40c
121 #define LC0(n)                  (_LC0 + (n)*0x20)
122
123 #define _LC1                    0x410
124 #define LC1(n)                  (_LC1 + (n)*0x20)
125
126 #define DBGSTATUS               0xd00
127 #define DBG_BUSY                (1 << 0)
128
129 #define DBGCMD                  0xd04
130 #define DBGINST0                0xd08
131 #define DBGINST1                0xd0c
132
133 #define CR0                     0xe00
134 #define CR1                     0xe04
135 #define CR2                     0xe08
136 #define CR3                     0xe0c
137 #define CR4                     0xe10
138 #define CRD                     0xe14
139
140 #define PERIPH_ID               0xfe0
141 #define PERIPH_REV_SHIFT        20
142 #define PERIPH_REV_MASK         0xf
143 #define PERIPH_REV_R0P0         0
144 #define PERIPH_REV_R1P0         1
145 #define PERIPH_REV_R1P1         2
146
147 #define CR0_PERIPH_REQ_SET      (1 << 0)
148 #define CR0_BOOT_EN_SET         (1 << 1)
149 #define CR0_BOOT_MAN_NS         (1 << 2)
150 #define CR0_NUM_CHANS_SHIFT     4
151 #define CR0_NUM_CHANS_MASK      0x7
152 #define CR0_NUM_PERIPH_SHIFT    12
153 #define CR0_NUM_PERIPH_MASK     0x1f
154 #define CR0_NUM_EVENTS_SHIFT    17
155 #define CR0_NUM_EVENTS_MASK     0x1f
156
157 #define CR1_ICACHE_LEN_SHIFT    0
158 #define CR1_ICACHE_LEN_MASK     0x7
159 #define CR1_NUM_ICACHELINES_SHIFT       4
160 #define CR1_NUM_ICACHELINES_MASK        0xf
161
162 #define CRD_DATA_WIDTH_SHIFT    0
163 #define CRD_DATA_WIDTH_MASK     0x7
164 #define CRD_WR_CAP_SHIFT        4
165 #define CRD_WR_CAP_MASK         0x7
166 #define CRD_WR_Q_DEP_SHIFT      8
167 #define CRD_WR_Q_DEP_MASK       0xf
168 #define CRD_RD_CAP_SHIFT        12
169 #define CRD_RD_CAP_MASK         0x7
170 #define CRD_RD_Q_DEP_SHIFT      16
171 #define CRD_RD_Q_DEP_MASK       0xf
172 #define CRD_DATA_BUFF_SHIFT     20
173 #define CRD_DATA_BUFF_MASK      0x3ff
174
175 #define PART                    0x330
176 #define DESIGNER                0x41
177 #define REVISION                0x0
178 #define INTEG_CFG               0x0
179 #define PERIPH_ID_VAL           ((PART << 0) | (DESIGNER << 12))
180
181 #define PL330_STATE_STOPPED             (1 << 0)
182 #define PL330_STATE_EXECUTING           (1 << 1)
183 #define PL330_STATE_WFE                 (1 << 2)
184 #define PL330_STATE_FAULTING            (1 << 3)
185 #define PL330_STATE_COMPLETING          (1 << 4)
186 #define PL330_STATE_WFP                 (1 << 5)
187 #define PL330_STATE_KILLING             (1 << 6)
188 #define PL330_STATE_FAULT_COMPLETING    (1 << 7)
189 #define PL330_STATE_CACHEMISS           (1 << 8)
190 #define PL330_STATE_UPDTPC              (1 << 9)
191 #define PL330_STATE_ATBARRIER           (1 << 10)
192 #define PL330_STATE_QUEUEBUSY           (1 << 11)
193 #define PL330_STATE_INVALID             (1 << 15)
194
195 #define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
196                                 | PL330_STATE_WFE | PL330_STATE_FAULTING)
197
198 #define CMD_DMAADDH             0x54
199 #define CMD_DMAEND              0x00
200 #define CMD_DMAFLUSHP           0x35
201 #define CMD_DMAGO               0xa0
202 #define CMD_DMALD               0x04
203 #define CMD_DMALDP              0x25
204 #define CMD_DMALP               0x20
205 #define CMD_DMALPEND            0x28
206 #define CMD_DMAKILL             0x01
207 #define CMD_DMAMOV              0xbc
208 #define CMD_DMANOP              0x18
209 #define CMD_DMARMB              0x12
210 #define CMD_DMASEV              0x34
211 #define CMD_DMAST               0x08
212 #define CMD_DMASTP              0x29
213 #define CMD_DMASTZ              0x0c
214 #define CMD_DMAWFE              0x36
215 #define CMD_DMAWFP              0x30
216 #define CMD_DMAWMB              0x13
217
218 #define SZ_DMAADDH              3
219 #define SZ_DMAEND               1
220 #define SZ_DMAFLUSHP            2
221 #define SZ_DMALD                1
222 #define SZ_DMALDP               2
223 #define SZ_DMALP                2
224 #define SZ_DMALPEND             2
225 #define SZ_DMAKILL              1
226 #define SZ_DMAMOV               6
227 #define SZ_DMANOP               1
228 #define SZ_DMARMB               1
229 #define SZ_DMASEV               2
230 #define SZ_DMAST                1
231 #define SZ_DMASTP               2
232 #define SZ_DMASTZ               1
233 #define SZ_DMAWFE               2
234 #define SZ_DMAWFP               2
235 #define SZ_DMAWMB               1
236 #define SZ_DMAGO                6
237
238 #define BRST_LEN(ccr)           ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
239 #define BRST_SIZE(ccr)          (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
240
241 #define BYTE_TO_BURST(b, ccr)   ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
242 #define BURST_TO_BYTE(c, ccr)   ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
243
244 /*
245  * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
246  * at 1byte/burst for P<->M and M<->M respectively.
247  * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
248  * should be enough for P<->M and M<->M respectively.
249  */
250 #define MCODE_BUFF_PER_REQ      256
251
252 /* Use this _only_ to wait on transient states */
253 #define UNTIL(t, s)     while (!(_state(t) & (s))) cpu_relax();
254
255 #ifdef PL330_DEBUG_MCGEN
256 static unsigned cmd_line;
257 #define PL330_DBGCMD_DUMP(off, x...)    do { \
258                                                 printk("%x:", cmd_line); \
259                                                 printk(x); \
260                                                 cmd_line += off; \
261                                         } while (0)
262 #define PL330_DBGMC_START(addr)         (cmd_line = addr)
263 #else
264 #define PL330_DBGCMD_DUMP(off, x...)    do {} while (0)
265 #define PL330_DBGMC_START(addr)         do {} while (0)
266 #endif
267
268 /* The number of default descriptors */
269
270 #define NR_DEFAULT_DESC 16
271
272 /* Delay for runtime PM autosuspend, ms */
273 #define PL330_AUTOSUSPEND_DELAY 20
274
275 /* Populated by the PL330 core driver for DMA API driver's info */
276 struct pl330_config {
277         u32     periph_id;
278 #define DMAC_MODE_NS    (1 << 0)
279         unsigned int    mode;
280         unsigned int    data_bus_width:10; /* In number of bits */
281         unsigned int    data_buf_dep:11;
282         unsigned int    num_chan:4;
283         unsigned int    num_peri:6;
284         u32             peri_ns;
285         unsigned int    num_events:6;
286         u32             irq_ns;
287 };
288
289 /**
290  * Request Configuration.
291  * The PL330 core does not modify this and uses the last
292  * working configuration if the request doesn't provide any.
293  *
294  * The Client may want to provide this info only for the
295  * first request and a request with new settings.
296  */
297 struct pl330_reqcfg {
298         /* Address Incrementing */
299         unsigned dst_inc:1;
300         unsigned src_inc:1;
301
302         /*
303          * For now, the SRC & DST protection levels
304          * and burst size/length are assumed same.
305          */
306         bool nonsecure;
307         bool privileged;
308         bool insnaccess;
309         unsigned brst_len:5;
310         unsigned brst_size:3; /* in power of 2 */
311
312         enum pl330_cachectrl dcctl;
313         enum pl330_cachectrl scctl;
314         enum pl330_byteswap swap;
315         struct pl330_config *pcfg;
316 };
317
318 /*
319  * One cycle of DMAC operation.
320  * There may be more than one xfer in a request.
321  */
322 struct pl330_xfer {
323         u32 src_addr;
324         u32 dst_addr;
325         /* Size to xfer */
326         u32 bytes;
327 };
328
329 /* The xfer callbacks are made with one of these arguments. */
330 enum pl330_op_err {
331         /* The all xfers in the request were success. */
332         PL330_ERR_NONE,
333         /* If req aborted due to global error. */
334         PL330_ERR_ABORT,
335         /* If req failed due to problem with Channel. */
336         PL330_ERR_FAIL,
337 };
338
339 enum dmamov_dst {
340         SAR = 0,
341         CCR,
342         DAR,
343 };
344
345 enum pl330_dst {
346         SRC = 0,
347         DST,
348 };
349
350 enum pl330_cond {
351         SINGLE,
352         BURST,
353         ALWAYS,
354 };
355
356 struct dma_pl330_desc;
357
358 struct _pl330_req {
359         u32 mc_bus;
360         void *mc_cpu;
361         struct dma_pl330_desc *desc;
362 };
363
364 /* ToBeDone for tasklet */
365 struct _pl330_tbd {
366         bool reset_dmac;
367         bool reset_mngr;
368         u8 reset_chan;
369 };
370
371 /* A DMAC Thread */
372 struct pl330_thread {
373         u8 id;
374         int ev;
375         /* If the channel is not yet acquired by any client */
376         bool free;
377         /* Parent DMAC */
378         struct pl330_dmac *dmac;
379         /* Only two at a time */
380         struct _pl330_req req[2];
381         /* Index of the last enqueued request */
382         unsigned lstenq;
383         /* Index of the last submitted request or -1 if the DMA is stopped */
384         int req_running;
385 };
386
387 enum pl330_dmac_state {
388         UNINIT,
389         INIT,
390         DYING,
391 };
392
393 enum desc_status {
394         /* In the DMAC pool */
395         FREE,
396         /*
397          * Allocated to some channel during prep_xxx
398          * Also may be sitting on the work_list.
399          */
400         PREP,
401         /*
402          * Sitting on the work_list and already submitted
403          * to the PL330 core. Not more than two descriptors
404          * of a channel can be BUSY at any time.
405          */
406         BUSY,
407         /*
408          * Sitting on the channel work_list but xfer done
409          * by PL330 core
410          */
411         DONE,
412 };
413
414 struct dma_pl330_chan {
415         /* Schedule desc completion */
416         struct tasklet_struct task;
417
418         /* DMA-Engine Channel */
419         struct dma_chan chan;
420
421         /* List of submitted descriptors */
422         struct list_head submitted_list;
423         /* List of issued descriptors */
424         struct list_head work_list;
425         /* List of completed descriptors */
426         struct list_head completed_list;
427
428         /* Pointer to the DMAC that manages this channel,
429          * NULL if the channel is available to be acquired.
430          * As the parent, this DMAC also provides descriptors
431          * to the channel.
432          */
433         struct pl330_dmac *dmac;
434
435         /* To protect channel manipulation */
436         spinlock_t lock;
437
438         /*
439          * Hardware channel thread of PL330 DMAC. NULL if the channel is
440          * available.
441          */
442         struct pl330_thread *thread;
443
444         /* For D-to-M and M-to-D channels */
445         int burst_sz; /* the peripheral fifo width */
446         int burst_len; /* the number of burst */
447         phys_addr_t fifo_addr;
448         /* DMA-mapped view of the FIFO; may differ if an IOMMU is present */
449         dma_addr_t fifo_dma;
450         enum dma_data_direction dir;
451
452         /* for cyclic capability */
453         bool cyclic;
454
455         /* for runtime pm tracking */
456         bool active;
457 };
458
459 struct pl330_dmac {
460         /* DMA-Engine Device */
461         struct dma_device ddma;
462
463         /* Holds info about sg limitations */
464         struct device_dma_parameters dma_parms;
465
466         /* Pool of descriptors available for the DMAC's channels */
467         struct list_head desc_pool;
468         /* To protect desc_pool manipulation */
469         spinlock_t pool_lock;
470
471         /* Size of MicroCode buffers for each channel. */
472         unsigned mcbufsz;
473         /* ioremap'ed address of PL330 registers. */
474         void __iomem    *base;
475         /* Populated by the PL330 core driver during pl330_add */
476         struct pl330_config     pcfg;
477
478         spinlock_t              lock;
479         /* Maximum possible events/irqs */
480         int                     events[32];
481         /* BUS address of MicroCode buffer */
482         dma_addr_t              mcode_bus;
483         /* CPU address of MicroCode buffer */
484         void                    *mcode_cpu;
485         /* List of all Channel threads */
486         struct pl330_thread     *channels;
487         /* Pointer to the MANAGER thread */
488         struct pl330_thread     *manager;
489         /* To handle bad news in interrupt */
490         struct tasklet_struct   tasks;
491         struct _pl330_tbd       dmac_tbd;
492         /* State of DMAC operation */
493         enum pl330_dmac_state   state;
494         /* Holds list of reqs with due callbacks */
495         struct list_head        req_done;
496
497         /* Peripheral channels connected to this DMAC */
498         unsigned int num_peripherals;
499         struct dma_pl330_chan *peripherals; /* keep at end */
500         int quirks;
501 };
502
503 static struct pl330_of_quirks {
504         char *quirk;
505         int id;
506 } of_quirks[] = {
507         {
508                 .quirk = "arm,pl330-broken-no-flushp",
509                 .id = PL330_QUIRK_BROKEN_NO_FLUSHP,
510         }
511 };
512
513 struct dma_pl330_desc {
514         /* To attach to a queue as child */
515         struct list_head node;
516
517         /* Descriptor for the DMA Engine API */
518         struct dma_async_tx_descriptor txd;
519
520         /* Xfer for PL330 core */
521         struct pl330_xfer px;
522
523         struct pl330_reqcfg rqcfg;
524
525         enum desc_status status;
526
527         int bytes_requested;
528         bool last;
529
530         /* The channel which currently holds this desc */
531         struct dma_pl330_chan *pchan;
532
533         enum dma_transfer_direction rqtype;
534         /* Index of peripheral for the xfer. */
535         unsigned peri:5;
536         /* Hook to attach to DMAC's list of reqs with due callback */
537         struct list_head rqd;
538 };
539
540 struct _xfer_spec {
541         u32 ccr;
542         struct dma_pl330_desc *desc;
543 };
544
545 static inline bool _queue_full(struct pl330_thread *thrd)
546 {
547         return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;
548 }
549
550 static inline bool is_manager(struct pl330_thread *thrd)
551 {
552         return thrd->dmac->manager == thrd;
553 }
554
555 /* If manager of the thread is in Non-Secure mode */
556 static inline bool _manager_ns(struct pl330_thread *thrd)
557 {
558         return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;
559 }
560
561 static inline u32 get_revision(u32 periph_id)
562 {
563         return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
564 }
565
566 static inline u32 _emit_END(unsigned dry_run, u8 buf[])
567 {
568         if (dry_run)
569                 return SZ_DMAEND;
570
571         buf[0] = CMD_DMAEND;
572
573         PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
574
575         return SZ_DMAEND;
576 }
577
578 static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
579 {
580         if (dry_run)
581                 return SZ_DMAFLUSHP;
582
583         buf[0] = CMD_DMAFLUSHP;
584
585         peri &= 0x1f;
586         peri <<= 3;
587         buf[1] = peri;
588
589         PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
590
591         return SZ_DMAFLUSHP;
592 }
593
594 static inline u32 _emit_LD(unsigned dry_run, u8 buf[],  enum pl330_cond cond)
595 {
596         if (dry_run)
597                 return SZ_DMALD;
598
599         buf[0] = CMD_DMALD;
600
601         if (cond == SINGLE)
602                 buf[0] |= (0 << 1) | (1 << 0);
603         else if (cond == BURST)
604                 buf[0] |= (1 << 1) | (1 << 0);
605
606         PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
607                 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
608
609         return SZ_DMALD;
610 }
611
612 static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
613                 enum pl330_cond cond, u8 peri)
614 {
615         if (dry_run)
616                 return SZ_DMALDP;
617
618         buf[0] = CMD_DMALDP;
619
620         if (cond == BURST)
621                 buf[0] |= (1 << 1);
622
623         peri &= 0x1f;
624         peri <<= 3;
625         buf[1] = peri;
626
627         PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
628                 cond == SINGLE ? 'S' : 'B', peri >> 3);
629
630         return SZ_DMALDP;
631 }
632
633 static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
634                 unsigned loop, u8 cnt)
635 {
636         if (dry_run)
637                 return SZ_DMALP;
638
639         buf[0] = CMD_DMALP;
640
641         if (loop)
642                 buf[0] |= (1 << 1);
643
644         cnt--; /* DMAC increments by 1 internally */
645         buf[1] = cnt;
646
647         PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
648
649         return SZ_DMALP;
650 }
651
652 struct _arg_LPEND {
653         enum pl330_cond cond;
654         bool forever;
655         unsigned loop;
656         u8 bjump;
657 };
658
659 static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
660                 const struct _arg_LPEND *arg)
661 {
662         enum pl330_cond cond = arg->cond;
663         bool forever = arg->forever;
664         unsigned loop = arg->loop;
665         u8 bjump = arg->bjump;
666
667         if (dry_run)
668                 return SZ_DMALPEND;
669
670         buf[0] = CMD_DMALPEND;
671
672         if (loop)
673                 buf[0] |= (1 << 2);
674
675         if (!forever)
676                 buf[0] |= (1 << 4);
677
678         if (cond == SINGLE)
679                 buf[0] |= (0 << 1) | (1 << 0);
680         else if (cond == BURST)
681                 buf[0] |= (1 << 1) | (1 << 0);
682
683         buf[1] = bjump;
684
685         PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
686                         forever ? "FE" : "END",
687                         cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
688                         loop ? '1' : '0',
689                         bjump);
690
691         return SZ_DMALPEND;
692 }
693
694 static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
695 {
696         if (dry_run)
697                 return SZ_DMAKILL;
698
699         buf[0] = CMD_DMAKILL;
700
701         return SZ_DMAKILL;
702 }
703
704 static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
705                 enum dmamov_dst dst, u32 val)
706 {
707         if (dry_run)
708                 return SZ_DMAMOV;
709
710         buf[0] = CMD_DMAMOV;
711         buf[1] = dst;
712         buf[2] = val;
713         buf[3] = val >> 8;
714         buf[4] = val >> 16;
715         buf[5] = val >> 24;
716
717         PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
718                 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
719
720         return SZ_DMAMOV;
721 }
722
723 static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
724 {
725         if (dry_run)
726                 return SZ_DMARMB;
727
728         buf[0] = CMD_DMARMB;
729
730         PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
731
732         return SZ_DMARMB;
733 }
734
735 static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
736 {
737         if (dry_run)
738                 return SZ_DMASEV;
739
740         buf[0] = CMD_DMASEV;
741
742         ev &= 0x1f;
743         ev <<= 3;
744         buf[1] = ev;
745
746         PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
747
748         return SZ_DMASEV;
749 }
750
751 static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
752 {
753         if (dry_run)
754                 return SZ_DMAST;
755
756         buf[0] = CMD_DMAST;
757
758         if (cond == SINGLE)
759                 buf[0] |= (0 << 1) | (1 << 0);
760         else if (cond == BURST)
761                 buf[0] |= (1 << 1) | (1 << 0);
762
763         PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
764                 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
765
766         return SZ_DMAST;
767 }
768
769 static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
770                 enum pl330_cond cond, u8 peri)
771 {
772         if (dry_run)
773                 return SZ_DMASTP;
774
775         buf[0] = CMD_DMASTP;
776
777         if (cond == BURST)
778                 buf[0] |= (1 << 1);
779
780         peri &= 0x1f;
781         peri <<= 3;
782         buf[1] = peri;
783
784         PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
785                 cond == SINGLE ? 'S' : 'B', peri >> 3);
786
787         return SZ_DMASTP;
788 }
789
790 static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
791                 enum pl330_cond cond, u8 peri)
792 {
793         if (dry_run)
794                 return SZ_DMAWFP;
795
796         buf[0] = CMD_DMAWFP;
797
798         if (cond == SINGLE)
799                 buf[0] |= (0 << 1) | (0 << 0);
800         else if (cond == BURST)
801                 buf[0] |= (1 << 1) | (0 << 0);
802         else
803                 buf[0] |= (0 << 1) | (1 << 0);
804
805         peri &= 0x1f;
806         peri <<= 3;
807         buf[1] = peri;
808
809         PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
810                 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
811
812         return SZ_DMAWFP;
813 }
814
815 static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
816 {
817         if (dry_run)
818                 return SZ_DMAWMB;
819
820         buf[0] = CMD_DMAWMB;
821
822         PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
823
824         return SZ_DMAWMB;
825 }
826
827 struct _arg_GO {
828         u8 chan;
829         u32 addr;
830         unsigned ns;
831 };
832
833 static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
834                 const struct _arg_GO *arg)
835 {
836         u8 chan = arg->chan;
837         u32 addr = arg->addr;
838         unsigned ns = arg->ns;
839
840         if (dry_run)
841                 return SZ_DMAGO;
842
843         buf[0] = CMD_DMAGO;
844         buf[0] |= (ns << 1);
845         buf[1] = chan & 0x7;
846         buf[2] = addr;
847         buf[3] = addr >> 8;
848         buf[4] = addr >> 16;
849         buf[5] = addr >> 24;
850
851         return SZ_DMAGO;
852 }
853
854 #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
855
856 /* Returns Time-Out */
857 static bool _until_dmac_idle(struct pl330_thread *thrd)
858 {
859         void __iomem *regs = thrd->dmac->base;
860         unsigned long loops = msecs_to_loops(5);
861
862         do {
863                 /* Until Manager is Idle */
864                 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
865                         break;
866
867                 cpu_relax();
868         } while (--loops);
869
870         if (!loops)
871                 return true;
872
873         return false;
874 }
875
876 static inline void _execute_DBGINSN(struct pl330_thread *thrd,
877                 u8 insn[], bool as_manager)
878 {
879         void __iomem *regs = thrd->dmac->base;
880         u32 val;
881
882         val = (insn[0] << 16) | (insn[1] << 24);
883         if (!as_manager) {
884                 val |= (1 << 0);
885                 val |= (thrd->id << 8); /* Channel Number */
886         }
887         writel(val, regs + DBGINST0);
888
889         val = le32_to_cpu(*((__le32 *)&insn[2]));
890         writel(val, regs + DBGINST1);
891
892         /* If timed out due to halted state-machine */
893         if (_until_dmac_idle(thrd)) {
894                 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
895                 return;
896         }
897
898         /* Get going */
899         writel(0, regs + DBGCMD);
900 }
901
902 static inline u32 _state(struct pl330_thread *thrd)
903 {
904         void __iomem *regs = thrd->dmac->base;
905         u32 val;
906
907         if (is_manager(thrd))
908                 val = readl(regs + DS) & 0xf;
909         else
910                 val = readl(regs + CS(thrd->id)) & 0xf;
911
912         switch (val) {
913         case DS_ST_STOP:
914                 return PL330_STATE_STOPPED;
915         case DS_ST_EXEC:
916                 return PL330_STATE_EXECUTING;
917         case DS_ST_CMISS:
918                 return PL330_STATE_CACHEMISS;
919         case DS_ST_UPDTPC:
920                 return PL330_STATE_UPDTPC;
921         case DS_ST_WFE:
922                 return PL330_STATE_WFE;
923         case DS_ST_FAULT:
924                 return PL330_STATE_FAULTING;
925         case DS_ST_ATBRR:
926                 if (is_manager(thrd))
927                         return PL330_STATE_INVALID;
928                 else
929                         return PL330_STATE_ATBARRIER;
930         case DS_ST_QBUSY:
931                 if (is_manager(thrd))
932                         return PL330_STATE_INVALID;
933                 else
934                         return PL330_STATE_QUEUEBUSY;
935         case DS_ST_WFP:
936                 if (is_manager(thrd))
937                         return PL330_STATE_INVALID;
938                 else
939                         return PL330_STATE_WFP;
940         case DS_ST_KILL:
941                 if (is_manager(thrd))
942                         return PL330_STATE_INVALID;
943                 else
944                         return PL330_STATE_KILLING;
945         case DS_ST_CMPLT:
946                 if (is_manager(thrd))
947                         return PL330_STATE_INVALID;
948                 else
949                         return PL330_STATE_COMPLETING;
950         case DS_ST_FLTCMP:
951                 if (is_manager(thrd))
952                         return PL330_STATE_INVALID;
953                 else
954                         return PL330_STATE_FAULT_COMPLETING;
955         default:
956                 return PL330_STATE_INVALID;
957         }
958 }
959
960 static void _stop(struct pl330_thread *thrd)
961 {
962         void __iomem *regs = thrd->dmac->base;
963         u8 insn[6] = {0, 0, 0, 0, 0, 0};
964
965         if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
966                 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
967
968         /* Return if nothing needs to be done */
969         if (_state(thrd) == PL330_STATE_COMPLETING
970                   || _state(thrd) == PL330_STATE_KILLING
971                   || _state(thrd) == PL330_STATE_STOPPED)
972                 return;
973
974         _emit_KILL(0, insn);
975
976         /* Stop generating interrupts for SEV */
977         writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
978
979         _execute_DBGINSN(thrd, insn, is_manager(thrd));
980 }
981
982 /* Start doing req 'idx' of thread 'thrd' */
983 static bool _trigger(struct pl330_thread *thrd)
984 {
985         void __iomem *regs = thrd->dmac->base;
986         struct _pl330_req *req;
987         struct dma_pl330_desc *desc;
988         struct _arg_GO go;
989         unsigned ns;
990         u8 insn[6] = {0, 0, 0, 0, 0, 0};
991         int idx;
992
993         /* Return if already ACTIVE */
994         if (_state(thrd) != PL330_STATE_STOPPED)
995                 return true;
996
997         idx = 1 - thrd->lstenq;
998         if (thrd->req[idx].desc != NULL) {
999                 req = &thrd->req[idx];
1000         } else {
1001                 idx = thrd->lstenq;
1002                 if (thrd->req[idx].desc != NULL)
1003                         req = &thrd->req[idx];
1004                 else
1005                         req = NULL;
1006         }
1007
1008         /* Return if no request */
1009         if (!req)
1010                 return true;
1011
1012         /* Return if req is running */
1013         if (idx == thrd->req_running)
1014                 return true;
1015
1016         desc = req->desc;
1017
1018         ns = desc->rqcfg.nonsecure ? 1 : 0;
1019
1020         /* See 'Abort Sources' point-4 at Page 2-25 */
1021         if (_manager_ns(thrd) && !ns)
1022                 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",
1023                         __func__, __LINE__);
1024
1025         go.chan = thrd->id;
1026         go.addr = req->mc_bus;
1027         go.ns = ns;
1028         _emit_GO(0, insn, &go);
1029
1030         /* Set to generate interrupts for SEV */
1031         writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1032
1033         /* Only manager can execute GO */
1034         _execute_DBGINSN(thrd, insn, true);
1035
1036         thrd->req_running = idx;
1037
1038         return true;
1039 }
1040
1041 static bool _start(struct pl330_thread *thrd)
1042 {
1043         switch (_state(thrd)) {
1044         case PL330_STATE_FAULT_COMPLETING:
1045                 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1046
1047                 if (_state(thrd) == PL330_STATE_KILLING)
1048                         UNTIL(thrd, PL330_STATE_STOPPED)
1049                 /* fall through */
1050
1051         case PL330_STATE_FAULTING:
1052                 _stop(thrd);
1053                 /* fall through */
1054
1055         case PL330_STATE_KILLING:
1056         case PL330_STATE_COMPLETING:
1057                 UNTIL(thrd, PL330_STATE_STOPPED)
1058                 /* fall through */
1059
1060         case PL330_STATE_STOPPED:
1061                 return _trigger(thrd);
1062
1063         case PL330_STATE_WFP:
1064         case PL330_STATE_QUEUEBUSY:
1065         case PL330_STATE_ATBARRIER:
1066         case PL330_STATE_UPDTPC:
1067         case PL330_STATE_CACHEMISS:
1068         case PL330_STATE_EXECUTING:
1069                 return true;
1070
1071         case PL330_STATE_WFE: /* For RESUME, nothing yet */
1072         default:
1073                 return false;
1074         }
1075 }
1076
1077 static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1078                 const struct _xfer_spec *pxs, int cyc)
1079 {
1080         int off = 0;
1081         struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;
1082
1083         /* check lock-up free version */
1084         if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1085                 while (cyc--) {
1086                         off += _emit_LD(dry_run, &buf[off], ALWAYS);
1087                         off += _emit_ST(dry_run, &buf[off], ALWAYS);
1088                 }
1089         } else {
1090                 while (cyc--) {
1091                         off += _emit_LD(dry_run, &buf[off], ALWAYS);
1092                         off += _emit_RMB(dry_run, &buf[off]);
1093                         off += _emit_ST(dry_run, &buf[off], ALWAYS);
1094                         off += _emit_WMB(dry_run, &buf[off]);
1095                 }
1096         }
1097
1098         return off;
1099 }
1100
1101 static u32 _emit_load(unsigned int dry_run, u8 buf[],
1102         enum pl330_cond cond, enum dma_transfer_direction direction,
1103         u8 peri)
1104 {
1105         int off = 0;
1106
1107         switch (direction) {
1108         case DMA_MEM_TO_MEM:
1109                 /* fall through */
1110         case DMA_MEM_TO_DEV:
1111                 off += _emit_LD(dry_run, &buf[off], cond);
1112                 break;
1113
1114         case DMA_DEV_TO_MEM:
1115                 if (cond == ALWAYS) {
1116                         off += _emit_LDP(dry_run, &buf[off], SINGLE,
1117                                 peri);
1118                         off += _emit_LDP(dry_run, &buf[off], BURST,
1119                                 peri);
1120                 } else {
1121                         off += _emit_LDP(dry_run, &buf[off], cond,
1122                                 peri);
1123                 }
1124                 break;
1125
1126         default:
1127                 /* this code should be unreachable */
1128                 WARN_ON(1);
1129                 break;
1130         }
1131
1132         return off;
1133 }
1134
1135 static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
1136         enum pl330_cond cond, enum dma_transfer_direction direction,
1137         u8 peri)
1138 {
1139         int off = 0;
1140
1141         switch (direction) {
1142         case DMA_MEM_TO_MEM:
1143                 /* fall through */
1144         case DMA_DEV_TO_MEM:
1145                 off += _emit_ST(dry_run, &buf[off], cond);
1146                 break;
1147
1148         case DMA_MEM_TO_DEV:
1149                 if (cond == ALWAYS) {
1150                         off += _emit_STP(dry_run, &buf[off], SINGLE,
1151                                 peri);
1152                         off += _emit_STP(dry_run, &buf[off], BURST,
1153                                 peri);
1154                 } else {
1155                         off += _emit_STP(dry_run, &buf[off], cond,
1156                                 peri);
1157                 }
1158                 break;
1159
1160         default:
1161                 /* this code should be unreachable */
1162                 WARN_ON(1);
1163                 break;
1164         }
1165
1166         return off;
1167 }
1168
1169 static inline int _ldst_peripheral(struct pl330_dmac *pl330,
1170                                  unsigned dry_run, u8 buf[],
1171                                  const struct _xfer_spec *pxs, int cyc,
1172                                  enum pl330_cond cond)
1173 {
1174         int off = 0;
1175
1176         if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
1177                 cond = BURST;
1178
1179         /*
1180          * do FLUSHP at beginning to clear any stale dma requests before the
1181          * first WFP.
1182          */
1183         if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
1184                 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
1185         while (cyc--) {
1186                 off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
1187                 off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
1188                         pxs->desc->peri);
1189                 off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
1190                         pxs->desc->peri);
1191         }
1192
1193         return off;
1194 }
1195
1196 static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
1197                 const struct _xfer_spec *pxs, int cyc)
1198 {
1199         int off = 0;
1200         enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;
1201
1202         switch (pxs->desc->rqtype) {
1203         case DMA_MEM_TO_DEV:
1204                 /* fall through */
1205         case DMA_DEV_TO_MEM:
1206                 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
1207                         cond);
1208                 break;
1209
1210         case DMA_MEM_TO_MEM:
1211                 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1212                 break;
1213
1214         default:
1215                 /* this code should be unreachable */
1216                 WARN_ON(1);
1217                 break;
1218         }
1219
1220         return off;
1221 }
1222
1223 /*
1224  * transfer dregs with single transfers to peripheral, or a reduced size burst
1225  * for mem-to-mem.
1226  */
1227 static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],
1228                 const struct _xfer_spec *pxs, int transfer_length)
1229 {
1230         int off = 0;
1231         int dregs_ccr;
1232
1233         if (transfer_length == 0)
1234                 return off;
1235
1236         switch (pxs->desc->rqtype) {
1237         case DMA_MEM_TO_DEV:
1238                 /* fall through */
1239         case DMA_DEV_TO_MEM:
1240                 off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs,
1241                         transfer_length, SINGLE);
1242                 break;
1243
1244         case DMA_MEM_TO_MEM:
1245                 dregs_ccr = pxs->ccr;
1246                 dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |
1247                         (0xf << CC_DSTBRSTLEN_SHFT));
1248                 dregs_ccr |= (((transfer_length - 1) & 0xf) <<
1249                         CC_SRCBRSTLEN_SHFT);
1250                 dregs_ccr |= (((transfer_length - 1) & 0xf) <<
1251                         CC_DSTBRSTLEN_SHFT);
1252                 off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
1253                 off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);
1254                 break;
1255
1256         default:
1257                 /* this code should be unreachable */
1258                 WARN_ON(1);
1259                 break;
1260         }
1261
1262         return off;
1263 }
1264
1265 /* Returns bytes consumed and updates bursts */
1266 static inline int _loop(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
1267                 unsigned long *bursts, const struct _xfer_spec *pxs)
1268 {
1269         int cyc, cycmax, szlp, szlpend, szbrst, off;
1270         unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1271         struct _arg_LPEND lpend;
1272
1273         if (*bursts == 1)
1274                 return _bursts(pl330, dry_run, buf, pxs, 1);
1275
1276         /* Max iterations possible in DMALP is 256 */
1277         if (*bursts >= 256*256) {
1278                 lcnt1 = 256;
1279                 lcnt0 = 256;
1280                 cyc = *bursts / lcnt1 / lcnt0;
1281         } else if (*bursts > 256) {
1282                 lcnt1 = 256;
1283                 lcnt0 = *bursts / lcnt1;
1284                 cyc = 1;
1285         } else {
1286                 lcnt1 = *bursts;
1287                 lcnt0 = 0;
1288                 cyc = 1;
1289         }
1290
1291         szlp = _emit_LP(1, buf, 0, 0);
1292         szbrst = _bursts(pl330, 1, buf, pxs, 1);
1293
1294         lpend.cond = ALWAYS;
1295         lpend.forever = false;
1296         lpend.loop = 0;
1297         lpend.bjump = 0;
1298         szlpend = _emit_LPEND(1, buf, &lpend);
1299
1300         if (lcnt0) {
1301                 szlp *= 2;
1302                 szlpend *= 2;
1303         }
1304
1305         /*
1306          * Max bursts that we can unroll due to limit on the
1307          * size of backward jump that can be encoded in DMALPEND
1308          * which is 8-bits and hence 255
1309          */
1310         cycmax = (255 - (szlp + szlpend)) / szbrst;
1311
1312         cyc = (cycmax < cyc) ? cycmax : cyc;
1313
1314         off = 0;
1315
1316         if (lcnt0) {
1317                 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1318                 ljmp0 = off;
1319         }
1320
1321         off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1322         ljmp1 = off;
1323
1324         off += _bursts(pl330, dry_run, &buf[off], pxs, cyc);
1325
1326         lpend.cond = ALWAYS;
1327         lpend.forever = false;
1328         lpend.loop = 1;
1329         lpend.bjump = off - ljmp1;
1330         off += _emit_LPEND(dry_run, &buf[off], &lpend);
1331
1332         if (lcnt0) {
1333                 lpend.cond = ALWAYS;
1334                 lpend.forever = false;
1335                 lpend.loop = 0;
1336                 lpend.bjump = off - ljmp0;
1337                 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1338         }
1339
1340         *bursts = lcnt1 * cyc;
1341         if (lcnt0)
1342                 *bursts *= lcnt0;
1343
1344         return off;
1345 }
1346
1347 static inline int _setup_loops(struct pl330_dmac *pl330,
1348                                unsigned dry_run, u8 buf[],
1349                                const struct _xfer_spec *pxs)
1350 {
1351         struct pl330_xfer *x = &pxs->desc->px;
1352         u32 ccr = pxs->ccr;
1353         unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1354         int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
1355                 BRST_SIZE(ccr);
1356         int off = 0;
1357
1358         while (bursts) {
1359                 c = bursts;
1360                 off += _loop(pl330, dry_run, &buf[off], &c, pxs);
1361                 bursts -= c;
1362         }
1363         off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
1364
1365         return off;
1366 }
1367
1368 static inline int _setup_xfer(struct pl330_dmac *pl330,
1369                               unsigned dry_run, u8 buf[],
1370                               const struct _xfer_spec *pxs)
1371 {
1372         struct pl330_xfer *x = &pxs->desc->px;
1373         int off = 0;
1374
1375         /* DMAMOV SAR, x->src_addr */
1376         off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1377         /* DMAMOV DAR, x->dst_addr */
1378         off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1379
1380         /* Setup Loop(s) */
1381         off += _setup_loops(pl330, dry_run, &buf[off], pxs);
1382
1383         return off;
1384 }
1385
1386 /*
1387  * A req is a sequence of one or more xfer units.
1388  * Returns the number of bytes taken to setup the MC for the req.
1389  */
1390 static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
1391                       struct pl330_thread *thrd, unsigned index,
1392                       struct _xfer_spec *pxs)
1393 {
1394         struct _pl330_req *req = &thrd->req[index];
1395         u8 *buf = req->mc_cpu;
1396         int off = 0;
1397
1398         PL330_DBGMC_START(req->mc_bus);
1399
1400         /* DMAMOV CCR, ccr */
1401         off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1402
1403         off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
1404
1405         /* DMASEV peripheral/event */
1406         off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1407         /* DMAEND */
1408         off += _emit_END(dry_run, &buf[off]);
1409
1410         return off;
1411 }
1412
1413 static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1414 {
1415         u32 ccr = 0;
1416
1417         if (rqc->src_inc)
1418                 ccr |= CC_SRCINC;
1419
1420         if (rqc->dst_inc)
1421                 ccr |= CC_DSTINC;
1422
1423         /* We set same protection levels for Src and DST for now */
1424         if (rqc->privileged)
1425                 ccr |= CC_SRCPRI | CC_DSTPRI;
1426         if (rqc->nonsecure)
1427                 ccr |= CC_SRCNS | CC_DSTNS;
1428         if (rqc->insnaccess)
1429                 ccr |= CC_SRCIA | CC_DSTIA;
1430
1431         ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1432         ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1433
1434         ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1435         ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1436
1437         ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1438         ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1439
1440         ccr |= (rqc->swap << CC_SWAP_SHFT);
1441
1442         return ccr;
1443 }
1444
1445 /*
1446  * Submit a list of xfers after which the client wants notification.
1447  * Client is not notified after each xfer unit, just once after all
1448  * xfer units are done or some error occurs.
1449  */
1450 static int pl330_submit_req(struct pl330_thread *thrd,
1451         struct dma_pl330_desc *desc)
1452 {
1453         struct pl330_dmac *pl330 = thrd->dmac;
1454         struct _xfer_spec xs;
1455         unsigned long flags;
1456         unsigned idx;
1457         u32 ccr;
1458         int ret = 0;
1459
1460         switch (desc->rqtype) {
1461         case DMA_MEM_TO_DEV:
1462                 break;
1463
1464         case DMA_DEV_TO_MEM:
1465                 break;
1466
1467         case DMA_MEM_TO_MEM:
1468                 break;
1469
1470         default:
1471                 return -ENOTSUPP;
1472         }
1473
1474         if (pl330->state == DYING
1475                 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1476                 dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
1477                         __func__, __LINE__);
1478                 return -EAGAIN;
1479         }
1480
1481         /* If request for non-existing peripheral */
1482         if (desc->rqtype != DMA_MEM_TO_MEM &&
1483             desc->peri >= pl330->pcfg.num_peri) {
1484                 dev_info(thrd->dmac->ddma.dev,
1485                                 "%s:%d Invalid peripheral(%u)!\n",
1486                                 __func__, __LINE__, desc->peri);
1487                 return -EINVAL;
1488         }
1489
1490         spin_lock_irqsave(&pl330->lock, flags);
1491
1492         if (_queue_full(thrd)) {
1493                 ret = -EAGAIN;
1494                 goto xfer_exit;
1495         }
1496
1497         /* Prefer Secure Channel */
1498         if (!_manager_ns(thrd))
1499                 desc->rqcfg.nonsecure = 0;
1500         else
1501                 desc->rqcfg.nonsecure = 1;
1502
1503         ccr = _prepare_ccr(&desc->rqcfg);
1504
1505         idx = thrd->req[0].desc == NULL ? 0 : 1;
1506
1507         xs.ccr = ccr;
1508         xs.desc = desc;
1509
1510         /* First dry run to check if req is acceptable */
1511         ret = _setup_req(pl330, 1, thrd, idx, &xs);
1512         if (ret < 0)
1513                 goto xfer_exit;
1514
1515         if (ret > pl330->mcbufsz / 2) {
1516                 dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",
1517                                 __func__, __LINE__, ret, pl330->mcbufsz / 2);
1518                 ret = -ENOMEM;
1519                 goto xfer_exit;
1520         }
1521
1522         /* Hook the request */
1523         thrd->lstenq = idx;
1524         thrd->req[idx].desc = desc;
1525         _setup_req(pl330, 0, thrd, idx, &xs);
1526
1527         ret = 0;
1528
1529 xfer_exit:
1530         spin_unlock_irqrestore(&pl330->lock, flags);
1531
1532         return ret;
1533 }
1534
1535 static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)
1536 {
1537         struct dma_pl330_chan *pch;
1538         unsigned long flags;
1539
1540         if (!desc)
1541                 return;
1542
1543         pch = desc->pchan;
1544
1545         /* If desc aborted */
1546         if (!pch)
1547                 return;
1548
1549         spin_lock_irqsave(&pch->lock, flags);
1550
1551         desc->status = DONE;
1552
1553         spin_unlock_irqrestore(&pch->lock, flags);
1554
1555         tasklet_schedule(&pch->task);
1556 }
1557
1558 static void pl330_dotask(unsigned long data)
1559 {
1560         struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1561         unsigned long flags;
1562         int i;
1563
1564         spin_lock_irqsave(&pl330->lock, flags);
1565
1566         /* The DMAC itself gone nuts */
1567         if (pl330->dmac_tbd.reset_dmac) {
1568                 pl330->state = DYING;
1569                 /* Reset the manager too */
1570                 pl330->dmac_tbd.reset_mngr = true;
1571                 /* Clear the reset flag */
1572                 pl330->dmac_tbd.reset_dmac = false;
1573         }
1574
1575         if (pl330->dmac_tbd.reset_mngr) {
1576                 _stop(pl330->manager);
1577                 /* Reset all channels */
1578                 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1;
1579                 /* Clear the reset flag */
1580                 pl330->dmac_tbd.reset_mngr = false;
1581         }
1582
1583         for (i = 0; i < pl330->pcfg.num_chan; i++) {
1584
1585                 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1586                         struct pl330_thread *thrd = &pl330->channels[i];
1587                         void __iomem *regs = pl330->base;
1588                         enum pl330_op_err err;
1589
1590                         _stop(thrd);
1591
1592                         if (readl(regs + FSC) & (1 << thrd->id))
1593                                 err = PL330_ERR_FAIL;
1594                         else
1595                                 err = PL330_ERR_ABORT;
1596
1597                         spin_unlock_irqrestore(&pl330->lock, flags);
1598                         dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);
1599                         dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);
1600                         spin_lock_irqsave(&pl330->lock, flags);
1601
1602                         thrd->req[0].desc = NULL;
1603                         thrd->req[1].desc = NULL;
1604                         thrd->req_running = -1;
1605
1606                         /* Clear the reset flag */
1607                         pl330->dmac_tbd.reset_chan &= ~(1 << i);
1608                 }
1609         }
1610
1611         spin_unlock_irqrestore(&pl330->lock, flags);
1612
1613         return;
1614 }
1615
1616 /* Returns 1 if state was updated, 0 otherwise */
1617 static int pl330_update(struct pl330_dmac *pl330)
1618 {
1619         struct dma_pl330_desc *descdone;
1620         unsigned long flags;
1621         void __iomem *regs;
1622         u32 val;
1623         int id, ev, ret = 0;
1624
1625         regs = pl330->base;
1626
1627         spin_lock_irqsave(&pl330->lock, flags);
1628
1629         val = readl(regs + FSM) & 0x1;
1630         if (val)
1631                 pl330->dmac_tbd.reset_mngr = true;
1632         else
1633                 pl330->dmac_tbd.reset_mngr = false;
1634
1635         val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1);
1636         pl330->dmac_tbd.reset_chan |= val;
1637         if (val) {
1638                 int i = 0;
1639                 while (i < pl330->pcfg.num_chan) {
1640                         if (val & (1 << i)) {
1641                                 dev_info(pl330->ddma.dev,
1642                                         "Reset Channel-%d\t CS-%x FTC-%x\n",
1643                                                 i, readl(regs + CS(i)),
1644                                                 readl(regs + FTC(i)));
1645                                 _stop(&pl330->channels[i]);
1646                         }
1647                         i++;
1648                 }
1649         }
1650
1651         /* Check which event happened i.e, thread notified */
1652         val = readl(regs + ES);
1653         if (pl330->pcfg.num_events < 32
1654                         && val & ~((1 << pl330->pcfg.num_events) - 1)) {
1655                 pl330->dmac_tbd.reset_dmac = true;
1656                 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,
1657                         __LINE__);
1658                 ret = 1;
1659                 goto updt_exit;
1660         }
1661
1662         for (ev = 0; ev < pl330->pcfg.num_events; ev++) {
1663                 if (val & (1 << ev)) { /* Event occurred */
1664                         struct pl330_thread *thrd;
1665                         u32 inten = readl(regs + INTEN);
1666                         int active;
1667
1668                         /* Clear the event */
1669                         if (inten & (1 << ev))
1670                                 writel(1 << ev, regs + INTCLR);
1671
1672                         ret = 1;
1673
1674                         id = pl330->events[ev];
1675
1676                         thrd = &pl330->channels[id];
1677
1678                         active = thrd->req_running;
1679                         if (active == -1) /* Aborted */
1680                                 continue;
1681
1682                         /* Detach the req */
1683                         descdone = thrd->req[active].desc;
1684                         thrd->req[active].desc = NULL;
1685
1686                         thrd->req_running = -1;
1687
1688                         /* Get going again ASAP */
1689                         _start(thrd);
1690
1691                         /* For now, just make a list of callbacks to be done */
1692                         list_add_tail(&descdone->rqd, &pl330->req_done);
1693                 }
1694         }
1695
1696         /* Now that we are in no hurry, do the callbacks */
1697         while (!list_empty(&pl330->req_done)) {
1698                 descdone = list_first_entry(&pl330->req_done,
1699                                             struct dma_pl330_desc, rqd);
1700                 list_del(&descdone->rqd);
1701                 spin_unlock_irqrestore(&pl330->lock, flags);
1702                 dma_pl330_rqcb(descdone, PL330_ERR_NONE);
1703                 spin_lock_irqsave(&pl330->lock, flags);
1704         }
1705
1706 updt_exit:
1707         spin_unlock_irqrestore(&pl330->lock, flags);
1708
1709         if (pl330->dmac_tbd.reset_dmac
1710                         || pl330->dmac_tbd.reset_mngr
1711                         || pl330->dmac_tbd.reset_chan) {
1712                 ret = 1;
1713                 tasklet_schedule(&pl330->tasks);
1714         }
1715
1716         return ret;
1717 }
1718
1719 /* Reserve an event */
1720 static inline int _alloc_event(struct pl330_thread *thrd)
1721 {
1722         struct pl330_dmac *pl330 = thrd->dmac;
1723         int ev;
1724
1725         for (ev = 0; ev < pl330->pcfg.num_events; ev++)
1726                 if (pl330->events[ev] == -1) {
1727                         pl330->events[ev] = thrd->id;
1728                         return ev;
1729                 }
1730
1731         return -1;
1732 }
1733
1734 static bool _chan_ns(const struct pl330_dmac *pl330, int i)
1735 {
1736         return pl330->pcfg.irq_ns & (1 << i);
1737 }
1738
1739 /* Upon success, returns IdentityToken for the
1740  * allocated channel, NULL otherwise.
1741  */
1742 static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
1743 {
1744         struct pl330_thread *thrd = NULL;
1745         int chans, i;
1746
1747         if (pl330->state == DYING)
1748                 return NULL;
1749
1750         chans = pl330->pcfg.num_chan;
1751
1752         for (i = 0; i < chans; i++) {
1753                 thrd = &pl330->channels[i];
1754                 if ((thrd->free) && (!_manager_ns(thrd) ||
1755                                         _chan_ns(pl330, i))) {
1756                         thrd->ev = _alloc_event(thrd);
1757                         if (thrd->ev >= 0) {
1758                                 thrd->free = false;
1759                                 thrd->lstenq = 1;
1760                                 thrd->req[0].desc = NULL;
1761                                 thrd->req[1].desc = NULL;
1762                                 thrd->req_running = -1;
1763                                 break;
1764                         }
1765                 }
1766                 thrd = NULL;
1767         }
1768
1769         return thrd;
1770 }
1771
1772 /* Release an event */
1773 static inline void _free_event(struct pl330_thread *thrd, int ev)
1774 {
1775         struct pl330_dmac *pl330 = thrd->dmac;
1776
1777         /* If the event is valid and was held by the thread */
1778         if (ev >= 0 && ev < pl330->pcfg.num_events
1779                         && pl330->events[ev] == thrd->id)
1780                 pl330->events[ev] = -1;
1781 }
1782
1783 static void pl330_release_channel(struct pl330_thread *thrd)
1784 {
1785         if (!thrd || thrd->free)
1786                 return;
1787
1788         _stop(thrd);
1789
1790         dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
1791         dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
1792
1793         _free_event(thrd, thrd->ev);
1794         thrd->free = true;
1795 }
1796
1797 /* Initialize the structure for PL330 configuration, that can be used
1798  * by the client driver the make best use of the DMAC
1799  */
1800 static void read_dmac_config(struct pl330_dmac *pl330)
1801 {
1802         void __iomem *regs = pl330->base;
1803         u32 val;
1804
1805         val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1806         val &= CRD_DATA_WIDTH_MASK;
1807         pl330->pcfg.data_bus_width = 8 * (1 << val);
1808
1809         val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1810         val &= CRD_DATA_BUFF_MASK;
1811         pl330->pcfg.data_buf_dep = val + 1;
1812
1813         val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1814         val &= CR0_NUM_CHANS_MASK;
1815         val += 1;
1816         pl330->pcfg.num_chan = val;
1817
1818         val = readl(regs + CR0);
1819         if (val & CR0_PERIPH_REQ_SET) {
1820                 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1821                 val += 1;
1822                 pl330->pcfg.num_peri = val;
1823                 pl330->pcfg.peri_ns = readl(regs + CR4);
1824         } else {
1825                 pl330->pcfg.num_peri = 0;
1826         }
1827
1828         val = readl(regs + CR0);
1829         if (val & CR0_BOOT_MAN_NS)
1830                 pl330->pcfg.mode |= DMAC_MODE_NS;
1831         else
1832                 pl330->pcfg.mode &= ~DMAC_MODE_NS;
1833
1834         val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1835         val &= CR0_NUM_EVENTS_MASK;
1836         val += 1;
1837         pl330->pcfg.num_events = val;
1838
1839         pl330->pcfg.irq_ns = readl(regs + CR3);
1840 }
1841
1842 static inline void _reset_thread(struct pl330_thread *thrd)
1843 {
1844         struct pl330_dmac *pl330 = thrd->dmac;
1845
1846         thrd->req[0].mc_cpu = pl330->mcode_cpu
1847                                 + (thrd->id * pl330->mcbufsz);
1848         thrd->req[0].mc_bus = pl330->mcode_bus
1849                                 + (thrd->id * pl330->mcbufsz);
1850         thrd->req[0].desc = NULL;
1851
1852         thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
1853                                 + pl330->mcbufsz / 2;
1854         thrd->req[1].mc_bus = thrd->req[0].mc_bus
1855                                 + pl330->mcbufsz / 2;
1856         thrd->req[1].desc = NULL;
1857
1858         thrd->req_running = -1;
1859 }
1860
1861 static int dmac_alloc_threads(struct pl330_dmac *pl330)
1862 {
1863         int chans = pl330->pcfg.num_chan;
1864         struct pl330_thread *thrd;
1865         int i;
1866
1867         /* Allocate 1 Manager and 'chans' Channel threads */
1868         pl330->channels = kcalloc(1 + chans, sizeof(*thrd),
1869                                         GFP_KERNEL);
1870         if (!pl330->channels)
1871                 return -ENOMEM;
1872
1873         /* Init Channel threads */
1874         for (i = 0; i < chans; i++) {
1875                 thrd = &pl330->channels[i];
1876                 thrd->id = i;
1877                 thrd->dmac = pl330;
1878                 _reset_thread(thrd);
1879                 thrd->free = true;
1880         }
1881
1882         /* MANAGER is indexed at the end */
1883         thrd = &pl330->channels[chans];
1884         thrd->id = chans;
1885         thrd->dmac = pl330;
1886         thrd->free = false;
1887         pl330->manager = thrd;
1888
1889         return 0;
1890 }
1891
1892 static int dmac_alloc_resources(struct pl330_dmac *pl330)
1893 {
1894         int chans = pl330->pcfg.num_chan;
1895         int ret;
1896
1897         /*
1898          * Alloc MicroCode buffer for 'chans' Channel threads.
1899          * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
1900          */
1901         pl330->mcode_cpu = dma_alloc_attrs(pl330->ddma.dev,
1902                                 chans * pl330->mcbufsz,
1903                                 &pl330->mcode_bus, GFP_KERNEL,
1904                                 DMA_ATTR_PRIVILEGED);
1905         if (!pl330->mcode_cpu) {
1906                 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
1907                         __func__, __LINE__);
1908                 return -ENOMEM;
1909         }
1910
1911         ret = dmac_alloc_threads(pl330);
1912         if (ret) {
1913                 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
1914                         __func__, __LINE__);
1915                 dma_free_coherent(pl330->ddma.dev,
1916                                 chans * pl330->mcbufsz,
1917                                 pl330->mcode_cpu, pl330->mcode_bus);
1918                 return ret;
1919         }
1920
1921         return 0;
1922 }
1923
1924 static int pl330_add(struct pl330_dmac *pl330)
1925 {
1926         int i, ret;
1927
1928         /* Check if we can handle this DMAC */
1929         if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {
1930                 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
1931                         pl330->pcfg.periph_id);
1932                 return -EINVAL;
1933         }
1934
1935         /* Read the configuration of the DMAC */
1936         read_dmac_config(pl330);
1937
1938         if (pl330->pcfg.num_events == 0) {
1939                 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",
1940                         __func__, __LINE__);
1941                 return -EINVAL;
1942         }
1943
1944         spin_lock_init(&pl330->lock);
1945
1946         INIT_LIST_HEAD(&pl330->req_done);
1947
1948         /* Use default MC buffer size if not provided */
1949         if (!pl330->mcbufsz)
1950                 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2;
1951
1952         /* Mark all events as free */
1953         for (i = 0; i < pl330->pcfg.num_events; i++)
1954                 pl330->events[i] = -1;
1955
1956         /* Allocate resources needed by the DMAC */
1957         ret = dmac_alloc_resources(pl330);
1958         if (ret) {
1959                 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");
1960                 return ret;
1961         }
1962
1963         tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
1964
1965         pl330->state = INIT;
1966
1967         return 0;
1968 }
1969
1970 static int dmac_free_threads(struct pl330_dmac *pl330)
1971 {
1972         struct pl330_thread *thrd;
1973         int i;
1974
1975         /* Release Channel threads */
1976         for (i = 0; i < pl330->pcfg.num_chan; i++) {
1977                 thrd = &pl330->channels[i];
1978                 pl330_release_channel(thrd);
1979         }
1980
1981         /* Free memory */
1982         kfree(pl330->channels);
1983
1984         return 0;
1985 }
1986
1987 static void pl330_del(struct pl330_dmac *pl330)
1988 {
1989         pl330->state = UNINIT;
1990
1991         tasklet_kill(&pl330->tasks);
1992
1993         /* Free DMAC resources */
1994         dmac_free_threads(pl330);
1995
1996         dma_free_coherent(pl330->ddma.dev,
1997                 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
1998                 pl330->mcode_bus);
1999 }
2000
2001 /* forward declaration */
2002 static struct amba_driver pl330_driver;
2003
2004 static inline struct dma_pl330_chan *
2005 to_pchan(struct dma_chan *ch)
2006 {
2007         if (!ch)
2008                 return NULL;
2009
2010         return container_of(ch, struct dma_pl330_chan, chan);
2011 }
2012
2013 static inline struct dma_pl330_desc *
2014 to_desc(struct dma_async_tx_descriptor *tx)
2015 {
2016         return container_of(tx, struct dma_pl330_desc, txd);
2017 }
2018
2019 static inline void fill_queue(struct dma_pl330_chan *pch)
2020 {
2021         struct dma_pl330_desc *desc;
2022         int ret;
2023
2024         list_for_each_entry(desc, &pch->work_list, node) {
2025
2026                 /* If already submitted */
2027                 if (desc->status == BUSY)
2028                         continue;
2029
2030                 ret = pl330_submit_req(pch->thread, desc);
2031                 if (!ret) {
2032                         desc->status = BUSY;
2033                 } else if (ret == -EAGAIN) {
2034                         /* QFull or DMAC Dying */
2035                         break;
2036                 } else {
2037                         /* Unacceptable request */
2038                         desc->status = DONE;
2039                         dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",
2040                                         __func__, __LINE__, desc->txd.cookie);
2041                         tasklet_schedule(&pch->task);
2042                 }
2043         }
2044 }
2045
2046 static void pl330_tasklet(unsigned long data)
2047 {
2048         struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
2049         struct dma_pl330_desc *desc, *_dt;
2050         unsigned long flags;
2051         bool power_down = false;
2052
2053         spin_lock_irqsave(&pch->lock, flags);
2054
2055         /* Pick up ripe tomatoes */
2056         list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2057                 if (desc->status == DONE) {
2058                         if (!pch->cyclic)
2059                                 dma_cookie_complete(&desc->txd);
2060                         list_move_tail(&desc->node, &pch->completed_list);
2061                 }
2062
2063         /* Try to submit a req imm. next to the last completed cookie */
2064         fill_queue(pch);
2065
2066         if (list_empty(&pch->work_list)) {
2067                 spin_lock(&pch->thread->dmac->lock);
2068                 _stop(pch->thread);
2069                 spin_unlock(&pch->thread->dmac->lock);
2070                 power_down = true;
2071                 pch->active = false;
2072         } else {
2073                 /* Make sure the PL330 Channel thread is active */
2074                 spin_lock(&pch->thread->dmac->lock);
2075                 _start(pch->thread);
2076                 spin_unlock(&pch->thread->dmac->lock);
2077         }
2078
2079         while (!list_empty(&pch->completed_list)) {
2080                 struct dmaengine_desc_callback cb;
2081
2082                 desc = list_first_entry(&pch->completed_list,
2083                                         struct dma_pl330_desc, node);
2084
2085                 dmaengine_desc_get_callback(&desc->txd, &cb);
2086
2087                 if (pch->cyclic) {
2088                         desc->status = PREP;
2089                         list_move_tail(&desc->node, &pch->work_list);
2090                         if (power_down) {
2091                                 pch->active = true;
2092                                 spin_lock(&pch->thread->dmac->lock);
2093                                 _start(pch->thread);
2094                                 spin_unlock(&pch->thread->dmac->lock);
2095                                 power_down = false;
2096                         }
2097                 } else {
2098                         desc->status = FREE;
2099                         list_move_tail(&desc->node, &pch->dmac->desc_pool);
2100                 }
2101
2102                 dma_descriptor_unmap(&desc->txd);
2103
2104                 if (dmaengine_desc_callback_valid(&cb)) {
2105                         spin_unlock_irqrestore(&pch->lock, flags);
2106                         dmaengine_desc_callback_invoke(&cb, NULL);
2107                         spin_lock_irqsave(&pch->lock, flags);
2108                 }
2109         }
2110         spin_unlock_irqrestore(&pch->lock, flags);
2111
2112         /* If work list empty, power down */
2113         if (power_down) {
2114                 pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2115                 pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
2116         }
2117 }
2118
2119 static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2120                                                 struct of_dma *ofdma)
2121 {
2122         int count = dma_spec->args_count;
2123         struct pl330_dmac *pl330 = ofdma->of_dma_data;
2124         unsigned int chan_id;
2125
2126         if (!pl330)
2127                 return NULL;
2128
2129         if (count != 1)
2130                 return NULL;
2131
2132         chan_id = dma_spec->args[0];
2133         if (chan_id >= pl330->num_peripherals)
2134                 return NULL;
2135
2136         return dma_get_slave_channel(&pl330->peripherals[chan_id].chan);
2137 }
2138
2139 static int pl330_alloc_chan_resources(struct dma_chan *chan)
2140 {
2141         struct dma_pl330_chan *pch = to_pchan(chan);
2142         struct pl330_dmac *pl330 = pch->dmac;
2143         unsigned long flags;
2144
2145         spin_lock_irqsave(&pl330->lock, flags);
2146
2147         dma_cookie_init(chan);
2148         pch->cyclic = false;
2149
2150         pch->thread = pl330_request_channel(pl330);
2151         if (!pch->thread) {
2152                 spin_unlock_irqrestore(&pl330->lock, flags);
2153                 return -ENOMEM;
2154         }
2155
2156         tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2157
2158         spin_unlock_irqrestore(&pl330->lock, flags);
2159
2160         return 1;
2161 }
2162
2163 /*
2164  * We need the data direction between the DMAC (the dma-mapping "device") and
2165  * the FIFO (the dmaengine "dev"), from the FIFO's point of view. Confusing!
2166  */
2167 static enum dma_data_direction
2168 pl330_dma_slave_map_dir(enum dma_transfer_direction dir)
2169 {
2170         switch (dir) {
2171         case DMA_MEM_TO_DEV:
2172                 return DMA_FROM_DEVICE;
2173         case DMA_DEV_TO_MEM:
2174                 return DMA_TO_DEVICE;
2175         case DMA_DEV_TO_DEV:
2176                 return DMA_BIDIRECTIONAL;
2177         default:
2178                 return DMA_NONE;
2179         }
2180 }
2181
2182 static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)
2183 {
2184         if (pch->dir != DMA_NONE)
2185                 dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma,
2186                                    1 << pch->burst_sz, pch->dir, 0);
2187         pch->dir = DMA_NONE;
2188 }
2189
2190
2191 static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
2192                                   enum dma_transfer_direction dir)
2193 {
2194         struct device *dev = pch->chan.device->dev;
2195         enum dma_data_direction dma_dir = pl330_dma_slave_map_dir(dir);
2196
2197         /* Already mapped for this config? */
2198         if (pch->dir == dma_dir)
2199                 return true;
2200
2201         pl330_unprep_slave_fifo(pch);
2202         pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr,
2203                                          1 << pch->burst_sz, dma_dir, 0);
2204         if (dma_mapping_error(dev, pch->fifo_dma))
2205                 return false;
2206
2207         pch->dir = dma_dir;
2208         return true;
2209 }
2210
2211 static int fixup_burst_len(int max_burst_len, int quirks)
2212 {
2213         if (quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
2214                 return 1;
2215         else if (max_burst_len > PL330_MAX_BURST)
2216                 return PL330_MAX_BURST;
2217         else if (max_burst_len < 1)
2218                 return 1;
2219         else
2220                 return max_burst_len;
2221 }
2222
2223 static int pl330_config(struct dma_chan *chan,
2224                         struct dma_slave_config *slave_config)
2225 {
2226         struct dma_pl330_chan *pch = to_pchan(chan);
2227
2228         pl330_unprep_slave_fifo(pch);
2229         if (slave_config->direction == DMA_MEM_TO_DEV) {
2230                 if (slave_config->dst_addr)
2231                         pch->fifo_addr = slave_config->dst_addr;
2232                 if (slave_config->dst_addr_width)
2233                         pch->burst_sz = __ffs(slave_config->dst_addr_width);
2234                 pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
2235                         pch->dmac->quirks);
2236         } else if (slave_config->direction == DMA_DEV_TO_MEM) {
2237                 if (slave_config->src_addr)
2238                         pch->fifo_addr = slave_config->src_addr;
2239                 if (slave_config->src_addr_width)
2240                         pch->burst_sz = __ffs(slave_config->src_addr_width);
2241                 pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
2242                         pch->dmac->quirks);
2243         }
2244
2245         return 0;
2246 }
2247
2248 static int pl330_terminate_all(struct dma_chan *chan)
2249 {
2250         struct dma_pl330_chan *pch = to_pchan(chan);
2251         struct dma_pl330_desc *desc;
2252         unsigned long flags;
2253         struct pl330_dmac *pl330 = pch->dmac;
2254         LIST_HEAD(list);
2255         bool power_down = false;
2256
2257         pm_runtime_get_sync(pl330->ddma.dev);
2258         spin_lock_irqsave(&pch->lock, flags);
2259
2260         spin_lock(&pl330->lock);
2261         _stop(pch->thread);
2262         pch->thread->req[0].desc = NULL;
2263         pch->thread->req[1].desc = NULL;
2264         pch->thread->req_running = -1;
2265         spin_unlock(&pl330->lock);
2266
2267         power_down = pch->active;
2268         pch->active = false;
2269
2270         /* Mark all desc done */
2271         list_for_each_entry(desc, &pch->submitted_list, node) {
2272                 desc->status = FREE;
2273                 dma_cookie_complete(&desc->txd);
2274         }
2275
2276         list_for_each_entry(desc, &pch->work_list , node) {
2277                 desc->status = FREE;
2278                 dma_cookie_complete(&desc->txd);
2279         }
2280
2281         list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);
2282         list_splice_tail_init(&pch->work_list, &pl330->desc_pool);
2283         list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
2284         spin_unlock_irqrestore(&pch->lock, flags);
2285         pm_runtime_mark_last_busy(pl330->ddma.dev);
2286         if (power_down)
2287                 pm_runtime_put_autosuspend(pl330->ddma.dev);
2288         pm_runtime_put_autosuspend(pl330->ddma.dev);
2289
2290         return 0;
2291 }
2292
2293 /*
2294  * We don't support DMA_RESUME command because of hardware
2295  * limitations, so after pausing the channel we cannot restore
2296  * it to active state. We have to terminate channel and setup
2297  * DMA transfer again. This pause feature was implemented to
2298  * allow safely read residue before channel termination.
2299  */
2300 static int pl330_pause(struct dma_chan *chan)
2301 {
2302         struct dma_pl330_chan *pch = to_pchan(chan);
2303         struct pl330_dmac *pl330 = pch->dmac;
2304         unsigned long flags;
2305
2306         pm_runtime_get_sync(pl330->ddma.dev);
2307         spin_lock_irqsave(&pch->lock, flags);
2308
2309         spin_lock(&pl330->lock);
2310         _stop(pch->thread);
2311         spin_unlock(&pl330->lock);
2312
2313         spin_unlock_irqrestore(&pch->lock, flags);
2314         pm_runtime_mark_last_busy(pl330->ddma.dev);
2315         pm_runtime_put_autosuspend(pl330->ddma.dev);
2316
2317         return 0;
2318 }
2319
2320 static void pl330_free_chan_resources(struct dma_chan *chan)
2321 {
2322         struct dma_pl330_chan *pch = to_pchan(chan);
2323         struct pl330_dmac *pl330 = pch->dmac;
2324         unsigned long flags;
2325
2326         tasklet_kill(&pch->task);
2327
2328         pm_runtime_get_sync(pch->dmac->ddma.dev);
2329         spin_lock_irqsave(&pl330->lock, flags);
2330
2331         pl330_release_channel(pch->thread);
2332         pch->thread = NULL;
2333
2334         if (pch->cyclic)
2335                 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2336
2337         spin_unlock_irqrestore(&pl330->lock, flags);
2338         pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2339         pm_runtime_put_autosuspend(pch->dmac->ddma.dev);
2340         pl330_unprep_slave_fifo(pch);
2341 }
2342
2343 static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,
2344                                            struct dma_pl330_desc *desc)
2345 {
2346         struct pl330_thread *thrd = pch->thread;
2347         struct pl330_dmac *pl330 = pch->dmac;
2348         void __iomem *regs = thrd->dmac->base;
2349         u32 val, addr;
2350
2351         pm_runtime_get_sync(pl330->ddma.dev);
2352         val = addr = 0;
2353         if (desc->rqcfg.src_inc) {
2354                 val = readl(regs + SA(thrd->id));
2355                 addr = desc->px.src_addr;
2356         } else {
2357                 val = readl(regs + DA(thrd->id));
2358                 addr = desc->px.dst_addr;
2359         }
2360         pm_runtime_mark_last_busy(pch->dmac->ddma.dev);
2361         pm_runtime_put_autosuspend(pl330->ddma.dev);
2362
2363         /* If DMAMOV hasn't finished yet, SAR/DAR can be zero */
2364         if (!val)
2365                 return 0;
2366
2367         return val - addr;
2368 }
2369
2370 static enum dma_status
2371 pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2372                  struct dma_tx_state *txstate)
2373 {
2374         enum dma_status ret;
2375         unsigned long flags;
2376         struct dma_pl330_desc *desc, *running = NULL, *last_enq = NULL;
2377         struct dma_pl330_chan *pch = to_pchan(chan);
2378         unsigned int transferred, residual = 0;
2379
2380         ret = dma_cookie_status(chan, cookie, txstate);
2381
2382         if (!txstate)
2383                 return ret;
2384
2385         if (ret == DMA_COMPLETE)
2386                 goto out;
2387
2388         spin_lock_irqsave(&pch->lock, flags);
2389         spin_lock(&pch->thread->dmac->lock);
2390
2391         if (pch->thread->req_running != -1)
2392                 running = pch->thread->req[pch->thread->req_running].desc;
2393
2394         last_enq = pch->thread->req[pch->thread->lstenq].desc;
2395
2396         /* Check in pending list */
2397         list_for_each_entry(desc, &pch->work_list, node) {
2398                 if (desc->status == DONE)
2399                         transferred = desc->bytes_requested;
2400                 else if (running && desc == running)
2401                         transferred =
2402                                 pl330_get_current_xferred_count(pch, desc);
2403                 else if (desc->status == BUSY)
2404                         /*
2405                          * Busy but not running means either just enqueued,
2406                          * or finished and not yet marked done
2407                          */
2408                         if (desc == last_enq)
2409                                 transferred = 0;
2410                         else
2411                                 transferred = desc->bytes_requested;
2412                 else
2413                         transferred = 0;
2414                 residual += desc->bytes_requested - transferred;
2415                 if (desc->txd.cookie == cookie) {
2416                         switch (desc->status) {
2417                         case DONE:
2418                                 ret = DMA_COMPLETE;
2419                                 break;
2420                         case PREP:
2421                         case BUSY:
2422                                 ret = DMA_IN_PROGRESS;
2423                                 break;
2424                         default:
2425                                 WARN_ON(1);
2426                         }
2427                         break;
2428                 }
2429                 if (desc->last)
2430                         residual = 0;
2431         }
2432         spin_unlock(&pch->thread->dmac->lock);
2433         spin_unlock_irqrestore(&pch->lock, flags);
2434
2435 out:
2436         dma_set_residue(txstate, residual);
2437
2438         return ret;
2439 }
2440
2441 static void pl330_issue_pending(struct dma_chan *chan)
2442 {
2443         struct dma_pl330_chan *pch = to_pchan(chan);
2444         unsigned long flags;
2445
2446         spin_lock_irqsave(&pch->lock, flags);
2447         if (list_empty(&pch->work_list)) {
2448                 /*
2449                  * Warn on nothing pending. Empty submitted_list may
2450                  * break our pm_runtime usage counter as it is
2451                  * updated on work_list emptiness status.
2452                  */
2453                 WARN_ON(list_empty(&pch->submitted_list));
2454                 pch->active = true;
2455                 pm_runtime_get_sync(pch->dmac->ddma.dev);
2456         }
2457         list_splice_tail_init(&pch->submitted_list, &pch->work_list);
2458         spin_unlock_irqrestore(&pch->lock, flags);
2459
2460         pl330_tasklet((unsigned long)pch);
2461 }
2462
2463 /*
2464  * We returned the last one of the circular list of descriptor(s)
2465  * from prep_xxx, so the argument to submit corresponds to the last
2466  * descriptor of the list.
2467  */
2468 static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2469 {
2470         struct dma_pl330_desc *desc, *last = to_desc(tx);
2471         struct dma_pl330_chan *pch = to_pchan(tx->chan);
2472         dma_cookie_t cookie;
2473         unsigned long flags;
2474
2475         spin_lock_irqsave(&pch->lock, flags);
2476
2477         /* Assign cookies to all nodes */
2478         while (!list_empty(&last->node)) {
2479                 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2480                 if (pch->cyclic) {
2481                         desc->txd.callback = last->txd.callback;
2482                         desc->txd.callback_param = last->txd.callback_param;
2483                 }
2484                 desc->last = false;
2485
2486                 dma_cookie_assign(&desc->txd);
2487
2488                 list_move_tail(&desc->node, &pch->submitted_list);
2489         }
2490
2491         last->last = true;
2492         cookie = dma_cookie_assign(&last->txd);
2493         list_add_tail(&last->node, &pch->submitted_list);
2494         spin_unlock_irqrestore(&pch->lock, flags);
2495
2496         return cookie;
2497 }
2498
2499 static inline void _init_desc(struct dma_pl330_desc *desc)
2500 {
2501         desc->rqcfg.swap = SWAP_NO;
2502         desc->rqcfg.scctl = CCTRL0;
2503         desc->rqcfg.dcctl = CCTRL0;
2504         desc->txd.tx_submit = pl330_tx_submit;
2505
2506         INIT_LIST_HEAD(&desc->node);
2507 }
2508
2509 /* Returns the number of descriptors added to the DMAC pool */
2510 static int add_desc(struct list_head *pool, spinlock_t *lock,
2511                     gfp_t flg, int count)
2512 {
2513         struct dma_pl330_desc *desc;
2514         unsigned long flags;
2515         int i;
2516
2517         desc = kcalloc(count, sizeof(*desc), flg);
2518         if (!desc)
2519                 return 0;
2520
2521         spin_lock_irqsave(lock, flags);
2522
2523         for (i = 0; i < count; i++) {
2524                 _init_desc(&desc[i]);
2525                 list_add_tail(&desc[i].node, pool);
2526         }
2527
2528         spin_unlock_irqrestore(lock, flags);
2529
2530         return count;
2531 }
2532
2533 static struct dma_pl330_desc *pluck_desc(struct list_head *pool,
2534                                          spinlock_t *lock)
2535 {
2536         struct dma_pl330_desc *desc = NULL;
2537         unsigned long flags;
2538
2539         spin_lock_irqsave(lock, flags);
2540
2541         if (!list_empty(pool)) {
2542                 desc = list_entry(pool->next,
2543                                 struct dma_pl330_desc, node);
2544
2545                 list_del_init(&desc->node);
2546
2547                 desc->status = PREP;
2548                 desc->txd.callback = NULL;
2549         }
2550
2551         spin_unlock_irqrestore(lock, flags);
2552
2553         return desc;
2554 }
2555
2556 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2557 {
2558         struct pl330_dmac *pl330 = pch->dmac;
2559         u8 *peri_id = pch->chan.private;
2560         struct dma_pl330_desc *desc;
2561
2562         /* Pluck one desc from the pool of DMAC */
2563         desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock);
2564
2565         /* If the DMAC pool is empty, alloc new */
2566         if (!desc) {
2567                 DEFINE_SPINLOCK(lock);
2568                 LIST_HEAD(pool);
2569
2570                 if (!add_desc(&pool, &lock, GFP_ATOMIC, 1))
2571                         return NULL;
2572
2573                 desc = pluck_desc(&pool, &lock);
2574                 WARN_ON(!desc || !list_empty(&pool));
2575         }
2576
2577         /* Initialize the descriptor */
2578         desc->pchan = pch;
2579         desc->txd.cookie = 0;
2580         async_tx_ack(&desc->txd);
2581
2582         desc->peri = peri_id ? pch->chan.chan_id : 0;
2583         desc->rqcfg.pcfg = &pch->dmac->pcfg;
2584
2585         dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2586
2587         return desc;
2588 }
2589
2590 static inline void fill_px(struct pl330_xfer *px,
2591                 dma_addr_t dst, dma_addr_t src, size_t len)
2592 {
2593         px->bytes = len;
2594         px->dst_addr = dst;
2595         px->src_addr = src;
2596 }
2597
2598 static struct dma_pl330_desc *
2599 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2600                 dma_addr_t src, size_t len)
2601 {
2602         struct dma_pl330_desc *desc = pl330_get_desc(pch);
2603
2604         if (!desc) {
2605                 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2606                         __func__, __LINE__);
2607                 return NULL;
2608         }
2609
2610         /*
2611          * Ideally we should lookout for reqs bigger than
2612          * those that can be programmed with 256 bytes of
2613          * MC buffer, but considering a req size is seldom
2614          * going to be word-unaligned and more than 200MB,
2615          * we take it easy.
2616          * Also, should the limit is reached we'd rather
2617          * have the platform increase MC buffer size than
2618          * complicating this API driver.
2619          */
2620         fill_px(&desc->px, dst, src, len);
2621
2622         return desc;
2623 }
2624
2625 /* Call after fixing burst size */
2626 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2627 {
2628         struct dma_pl330_chan *pch = desc->pchan;
2629         struct pl330_dmac *pl330 = pch->dmac;
2630         int burst_len;
2631
2632         burst_len = pl330->pcfg.data_bus_width / 8;
2633         burst_len *= pl330->pcfg.data_buf_dep / pl330->pcfg.num_chan;
2634         burst_len >>= desc->rqcfg.brst_size;
2635
2636         /* src/dst_burst_len can't be more than 16 */
2637         if (burst_len > PL330_MAX_BURST)
2638                 burst_len = PL330_MAX_BURST;
2639
2640         return burst_len;
2641 }
2642
2643 static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2644                 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2645                 size_t period_len, enum dma_transfer_direction direction,
2646                 unsigned long flags)
2647 {
2648         struct dma_pl330_desc *desc = NULL, *first = NULL;
2649         struct dma_pl330_chan *pch = to_pchan(chan);
2650         struct pl330_dmac *pl330 = pch->dmac;
2651         unsigned int i;
2652         dma_addr_t dst;
2653         dma_addr_t src;
2654
2655         if (len % period_len != 0)
2656                 return NULL;
2657
2658         if (!is_slave_direction(direction)) {
2659                 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",
2660                 __func__, __LINE__);
2661                 return NULL;
2662         }
2663
2664         if (!pl330_prep_slave_fifo(pch, direction))
2665                 return NULL;
2666
2667         for (i = 0; i < len / period_len; i++) {
2668                 desc = pl330_get_desc(pch);
2669                 if (!desc) {
2670                         dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
2671                                 __func__, __LINE__);
2672
2673                         if (!first)
2674                                 return NULL;
2675
2676                         spin_lock_irqsave(&pl330->pool_lock, flags);
2677
2678                         while (!list_empty(&first->node)) {
2679                                 desc = list_entry(first->node.next,
2680                                                 struct dma_pl330_desc, node);
2681                                 list_move_tail(&desc->node, &pl330->desc_pool);
2682                         }
2683
2684                         list_move_tail(&first->node, &pl330->desc_pool);
2685
2686                         spin_unlock_irqrestore(&pl330->pool_lock, flags);
2687
2688                         return NULL;
2689                 }
2690
2691                 switch (direction) {
2692                 case DMA_MEM_TO_DEV:
2693                         desc->rqcfg.src_inc = 1;
2694                         desc->rqcfg.dst_inc = 0;
2695                         src = dma_addr;
2696                         dst = pch->fifo_dma;
2697                         break;
2698                 case DMA_DEV_TO_MEM:
2699                         desc->rqcfg.src_inc = 0;
2700                         desc->rqcfg.dst_inc = 1;
2701                         src = pch->fifo_dma;
2702                         dst = dma_addr;
2703                         break;
2704                 default:
2705                         break;
2706                 }
2707
2708                 desc->rqtype = direction;
2709                 desc->rqcfg.brst_size = pch->burst_sz;
2710                 desc->rqcfg.brst_len = pch->burst_len;
2711                 desc->bytes_requested = period_len;
2712                 fill_px(&desc->px, dst, src, period_len);
2713
2714                 if (!first)
2715                         first = desc;
2716                 else
2717                         list_add_tail(&desc->node, &first->node);
2718
2719                 dma_addr += period_len;
2720         }
2721
2722         if (!desc)
2723                 return NULL;
2724
2725         pch->cyclic = true;
2726         desc->txd.flags = flags;
2727
2728         return &desc->txd;
2729 }
2730
2731 static struct dma_async_tx_descriptor *
2732 pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2733                 dma_addr_t src, size_t len, unsigned long flags)
2734 {
2735         struct dma_pl330_desc *desc;
2736         struct dma_pl330_chan *pch = to_pchan(chan);
2737         struct pl330_dmac *pl330;
2738         int burst;
2739
2740         if (unlikely(!pch || !len))
2741                 return NULL;
2742
2743         pl330 = pch->dmac;
2744
2745         desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2746         if (!desc)
2747                 return NULL;
2748
2749         desc->rqcfg.src_inc = 1;
2750         desc->rqcfg.dst_inc = 1;
2751         desc->rqtype = DMA_MEM_TO_MEM;
2752
2753         /* Select max possible burst size */
2754         burst = pl330->pcfg.data_bus_width / 8;
2755
2756         /*
2757          * Make sure we use a burst size that aligns with all the memcpy
2758          * parameters because our DMA programming algorithm doesn't cope with
2759          * transfers which straddle an entry in the DMA device's MFIFO.
2760          */
2761         while ((src | dst | len) & (burst - 1))
2762                 burst /= 2;
2763
2764         desc->rqcfg.brst_size = 0;
2765         while (burst != (1 << desc->rqcfg.brst_size))
2766                 desc->rqcfg.brst_size++;
2767
2768         /*
2769          * If burst size is smaller than bus width then make sure we only
2770          * transfer one at a time to avoid a burst stradling an MFIFO entry.
2771          */
2772         if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width)
2773                 desc->rqcfg.brst_len = 1;
2774
2775         desc->rqcfg.brst_len = get_burst_len(desc, len);
2776         desc->bytes_requested = len;
2777
2778         desc->txd.flags = flags;
2779
2780         return &desc->txd;
2781 }
2782
2783 static void __pl330_giveback_desc(struct pl330_dmac *pl330,
2784                                   struct dma_pl330_desc *first)
2785 {
2786         unsigned long flags;
2787         struct dma_pl330_desc *desc;
2788
2789         if (!first)
2790                 return;
2791
2792         spin_lock_irqsave(&pl330->pool_lock, flags);
2793
2794         while (!list_empty(&first->node)) {
2795                 desc = list_entry(first->node.next,
2796                                 struct dma_pl330_desc, node);
2797                 list_move_tail(&desc->node, &pl330->desc_pool);
2798         }
2799
2800         list_move_tail(&first->node, &pl330->desc_pool);
2801
2802         spin_unlock_irqrestore(&pl330->pool_lock, flags);
2803 }
2804
2805 static struct dma_async_tx_descriptor *
2806 pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2807                 unsigned int sg_len, enum dma_transfer_direction direction,
2808                 unsigned long flg, void *context)
2809 {
2810         struct dma_pl330_desc *first, *desc = NULL;
2811         struct dma_pl330_chan *pch = to_pchan(chan);
2812         struct scatterlist *sg;
2813         int i;
2814
2815         if (unlikely(!pch || !sgl || !sg_len))
2816                 return NULL;
2817
2818         if (!pl330_prep_slave_fifo(pch, direction))
2819                 return NULL;
2820
2821         first = NULL;
2822
2823         for_each_sg(sgl, sg, sg_len, i) {
2824
2825                 desc = pl330_get_desc(pch);
2826                 if (!desc) {
2827                         struct pl330_dmac *pl330 = pch->dmac;
2828
2829                         dev_err(pch->dmac->ddma.dev,
2830                                 "%s:%d Unable to fetch desc\n",
2831                                 __func__, __LINE__);
2832                         __pl330_giveback_desc(pl330, first);
2833
2834                         return NULL;
2835                 }
2836
2837                 if (!first)
2838                         first = desc;
2839                 else
2840                         list_add_tail(&desc->node, &first->node);
2841
2842                 if (direction == DMA_MEM_TO_DEV) {
2843                         desc->rqcfg.src_inc = 1;
2844                         desc->rqcfg.dst_inc = 0;
2845                         fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg),
2846                                 sg_dma_len(sg));
2847                 } else {
2848                         desc->rqcfg.src_inc = 0;
2849                         desc->rqcfg.dst_inc = 1;
2850                         fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma,
2851                                 sg_dma_len(sg));
2852                 }
2853
2854                 desc->rqcfg.brst_size = pch->burst_sz;
2855                 desc->rqcfg.brst_len = pch->burst_len;
2856                 desc->rqtype = direction;
2857                 desc->bytes_requested = sg_dma_len(sg);
2858         }
2859
2860         /* Return the last desc in the chain */
2861         desc->txd.flags = flg;
2862         return &desc->txd;
2863 }
2864
2865 static irqreturn_t pl330_irq_handler(int irq, void *data)
2866 {
2867         if (pl330_update(data))
2868                 return IRQ_HANDLED;
2869         else
2870                 return IRQ_NONE;
2871 }
2872
2873 #define PL330_DMA_BUSWIDTHS \
2874         BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
2875         BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
2876         BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
2877         BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
2878         BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
2879
2880 /*
2881  * Runtime PM callbacks are provided by amba/bus.c driver.
2882  *
2883  * It is assumed here that IRQ safe runtime PM is chosen in probe and amba
2884  * bus driver will only disable/enable the clock in runtime PM callbacks.
2885  */
2886 static int __maybe_unused pl330_suspend(struct device *dev)
2887 {
2888         struct amba_device *pcdev = to_amba_device(dev);
2889
2890         pm_runtime_disable(dev);
2891
2892         if (!pm_runtime_status_suspended(dev)) {
2893                 /* amba did not disable the clock */
2894                 amba_pclk_disable(pcdev);
2895         }
2896         amba_pclk_unprepare(pcdev);
2897
2898         return 0;
2899 }
2900
2901 static int __maybe_unused pl330_resume(struct device *dev)
2902 {
2903         struct amba_device *pcdev = to_amba_device(dev);
2904         int ret;
2905
2906         ret = amba_pclk_prepare(pcdev);
2907         if (ret)
2908                 return ret;
2909
2910         if (!pm_runtime_status_suspended(dev))
2911                 ret = amba_pclk_enable(pcdev);
2912
2913         pm_runtime_enable(dev);
2914
2915         return ret;
2916 }
2917
2918 static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume);
2919
2920 static int
2921 pl330_probe(struct amba_device *adev, const struct amba_id *id)
2922 {
2923         struct pl330_config *pcfg;
2924         struct pl330_dmac *pl330;
2925         struct dma_pl330_chan *pch, *_p;
2926         struct dma_device *pd;
2927         struct resource *res;
2928         int i, ret, irq;
2929         int num_chan;
2930         struct device_node *np = adev->dev.of_node;
2931
2932         ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2933         if (ret)
2934                 return ret;
2935
2936         /* Allocate a new DMAC and its Channels */
2937         pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL);
2938         if (!pl330)
2939                 return -ENOMEM;
2940
2941         pd = &pl330->ddma;
2942         pd->dev = &adev->dev;
2943
2944         pl330->mcbufsz = 0;
2945
2946         /* get quirk */
2947         for (i = 0; i < ARRAY_SIZE(of_quirks); i++)
2948                 if (of_property_read_bool(np, of_quirks[i].quirk))
2949                         pl330->quirks |= of_quirks[i].id;
2950
2951         res = &adev->res;
2952         pl330->base = devm_ioremap_resource(&adev->dev, res);
2953         if (IS_ERR(pl330->base))
2954                 return PTR_ERR(pl330->base);
2955
2956         amba_set_drvdata(adev, pl330);
2957
2958         for (i = 0; i < AMBA_NR_IRQS; i++) {
2959                 irq = adev->irq[i];
2960                 if (irq) {
2961                         ret = devm_request_irq(&adev->dev, irq,
2962                                                pl330_irq_handler, 0,
2963                                                dev_name(&adev->dev), pl330);
2964                         if (ret)
2965                                 return ret;
2966                 } else {
2967                         break;
2968                 }
2969         }
2970
2971         pcfg = &pl330->pcfg;
2972
2973         pcfg->periph_id = adev->periphid;
2974         ret = pl330_add(pl330);
2975         if (ret)
2976                 return ret;
2977
2978         INIT_LIST_HEAD(&pl330->desc_pool);
2979         spin_lock_init(&pl330->pool_lock);
2980
2981         /* Create a descriptor pool of default size */
2982         if (!add_desc(&pl330->desc_pool, &pl330->pool_lock,
2983                       GFP_KERNEL, NR_DEFAULT_DESC))
2984                 dev_warn(&adev->dev, "unable to allocate desc\n");
2985
2986         INIT_LIST_HEAD(&pd->channels);
2987
2988         /* Initialize channel parameters */
2989         num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);
2990
2991         pl330->num_peripherals = num_chan;
2992
2993         pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL);
2994         if (!pl330->peripherals) {
2995                 ret = -ENOMEM;
2996                 goto probe_err2;
2997         }
2998
2999         for (i = 0; i < num_chan; i++) {
3000                 pch = &pl330->peripherals[i];
3001
3002                 pch->chan.private = adev->dev.of_node;
3003                 INIT_LIST_HEAD(&pch->submitted_list);
3004                 INIT_LIST_HEAD(&pch->work_list);
3005                 INIT_LIST_HEAD(&pch->completed_list);
3006                 spin_lock_init(&pch->lock);
3007                 pch->thread = NULL;
3008                 pch->chan.device = pd;
3009                 pch->dmac = pl330;
3010                 pch->dir = DMA_NONE;
3011
3012                 /* Add the channel to the DMAC list */
3013                 list_add_tail(&pch->chan.device_node, &pd->channels);
3014         }
3015
3016         dma_cap_set(DMA_MEMCPY, pd->cap_mask);
3017         if (pcfg->num_peri) {
3018                 dma_cap_set(DMA_SLAVE, pd->cap_mask);
3019                 dma_cap_set(DMA_CYCLIC, pd->cap_mask);
3020                 dma_cap_set(DMA_PRIVATE, pd->cap_mask);
3021         }
3022
3023         pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
3024         pd->device_free_chan_resources = pl330_free_chan_resources;
3025         pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
3026         pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
3027         pd->device_tx_status = pl330_tx_status;
3028         pd->device_prep_slave_sg = pl330_prep_slave_sg;
3029         pd->device_config = pl330_config;
3030         pd->device_pause = pl330_pause;
3031         pd->device_terminate_all = pl330_terminate_all;
3032         pd->device_issue_pending = pl330_issue_pending;
3033         pd->src_addr_widths = PL330_DMA_BUSWIDTHS;
3034         pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;
3035         pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
3036         pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
3037         pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ?
3038                          1 : PL330_MAX_BURST);
3039
3040         ret = dma_async_device_register(pd);
3041         if (ret) {
3042                 dev_err(&adev->dev, "unable to register DMAC\n");
3043                 goto probe_err3;
3044         }
3045
3046         if (adev->dev.of_node) {
3047                 ret = of_dma_controller_register(adev->dev.of_node,
3048                                          of_dma_pl330_xlate, pl330);
3049                 if (ret) {
3050                         dev_err(&adev->dev,
3051                         "unable to register DMA to the generic DT DMA helpers\n");
3052                 }
3053         }
3054
3055         adev->dev.dma_parms = &pl330->dma_parms;
3056
3057         /*
3058          * This is the limit for transfers with a buswidth of 1, larger
3059          * buswidths will have larger limits.
3060          */
3061         ret = dma_set_max_seg_size(&adev->dev, 1900800);
3062         if (ret)
3063                 dev_err(&adev->dev, "unable to set the seg size\n");
3064
3065
3066         dev_info(&adev->dev,
3067                 "Loaded driver for PL330 DMAC-%x\n", adev->periphid);
3068         dev_info(&adev->dev,
3069                 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
3070                 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan,
3071                 pcfg->num_peri, pcfg->num_events);
3072
3073         pm_runtime_irq_safe(&adev->dev);
3074         pm_runtime_use_autosuspend(&adev->dev);
3075         pm_runtime_set_autosuspend_delay(&adev->dev, PL330_AUTOSUSPEND_DELAY);
3076         pm_runtime_mark_last_busy(&adev->dev);
3077         pm_runtime_put_autosuspend(&adev->dev);
3078
3079         return 0;
3080 probe_err3:
3081         /* Idle the DMAC */
3082         list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3083                         chan.device_node) {
3084
3085                 /* Remove the channel */
3086                 list_del(&pch->chan.device_node);
3087
3088                 /* Flush the channel */
3089                 if (pch->thread) {
3090                         pl330_terminate_all(&pch->chan);
3091                         pl330_free_chan_resources(&pch->chan);
3092                 }
3093         }
3094 probe_err2:
3095         pl330_del(pl330);
3096
3097         return ret;
3098 }
3099
3100 static int pl330_remove(struct amba_device *adev)
3101 {
3102         struct pl330_dmac *pl330 = amba_get_drvdata(adev);
3103         struct dma_pl330_chan *pch, *_p;
3104         int i, irq;
3105
3106         pm_runtime_get_noresume(pl330->ddma.dev);
3107
3108         if (adev->dev.of_node)
3109                 of_dma_controller_free(adev->dev.of_node);
3110
3111         for (i = 0; i < AMBA_NR_IRQS; i++) {
3112                 irq = adev->irq[i];
3113                 if (irq)
3114                         devm_free_irq(&adev->dev, irq, pl330);
3115         }
3116
3117         dma_async_device_unregister(&pl330->ddma);
3118
3119         /* Idle the DMAC */
3120         list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
3121                         chan.device_node) {
3122
3123                 /* Remove the channel */
3124                 list_del(&pch->chan.device_node);
3125
3126                 /* Flush the channel */
3127                 if (pch->thread) {
3128                         pl330_terminate_all(&pch->chan);
3129                         pl330_free_chan_resources(&pch->chan);
3130                 }
3131         }
3132
3133         pl330_del(pl330);
3134
3135         return 0;
3136 }
3137
3138 static const struct amba_id pl330_ids[] = {
3139         {
3140                 .id     = 0x00041330,
3141                 .mask   = 0x000fffff,
3142         },
3143         { 0, 0 },
3144 };
3145
3146 MODULE_DEVICE_TABLE(amba, pl330_ids);
3147
3148 static struct amba_driver pl330_driver = {
3149         .drv = {
3150                 .owner = THIS_MODULE,
3151                 .name = "dma-pl330",
3152                 .pm = &pl330_pm,
3153         },
3154         .id_table = pl330_ids,
3155         .probe = pl330_probe,
3156         .remove = pl330_remove,
3157 };
3158
3159 module_amba_driver(pl330_driver);
3160
3161 MODULE_AUTHOR("Jaswinder Singh <jassisinghbrar@gmail.com>");
3162 MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3163 MODULE_LICENSE("GPL");