b1628d5db6d7536107b669220514e9709a5188c5
[muen/linux.git] / drivers / dma / qcom / bam_dma.c
1 /*
2  * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 and
6  * only version 2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  */
14 /*
15  * QCOM BAM DMA engine driver
16  *
17  * QCOM BAM DMA blocks are distributed amongst a number of the on-chip
18  * peripherals on the MSM 8x74.  The configuration of the channels are dependent
19  * on the way they are hard wired to that specific peripheral.  The peripheral
20  * device tree entries specify the configuration of each channel.
21  *
22  * The DMA controller requires the use of external memory for storage of the
23  * hardware descriptors for each channel.  The descriptor FIFO is accessed as a
24  * circular buffer and operations are managed according to the offset within the
25  * FIFO.  After pipe/channel reset, all of the pipe registers and internal state
26  * are back to defaults.
27  *
28  * During DMA operations, we write descriptors to the FIFO, being careful to
29  * handle wrapping and then write the last FIFO offset to that channel's
30  * P_EVNT_REG register to kick off the transaction.  The P_SW_OFSTS register
31  * indicates the current FIFO offset that is being processed, so there is some
32  * indication of where the hardware is currently working.
33  */
34
35 #include <linux/kernel.h>
36 #include <linux/io.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/module.h>
40 #include <linux/interrupt.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/scatterlist.h>
43 #include <linux/device.h>
44 #include <linux/platform_device.h>
45 #include <linux/of.h>
46 #include <linux/of_address.h>
47 #include <linux/of_irq.h>
48 #include <linux/of_dma.h>
49 #include <linux/circ_buf.h>
50 #include <linux/clk.h>
51 #include <linux/dmaengine.h>
52 #include <linux/pm_runtime.h>
53
54 #include "../dmaengine.h"
55 #include "../virt-dma.h"
56
57 struct bam_desc_hw {
58         __le32 addr;            /* Buffer physical address */
59         __le16 size;            /* Buffer size in bytes */
60         __le16 flags;
61 };
62
63 #define BAM_DMA_AUTOSUSPEND_DELAY 100
64
65 #define DESC_FLAG_INT BIT(15)
66 #define DESC_FLAG_EOT BIT(14)
67 #define DESC_FLAG_EOB BIT(13)
68 #define DESC_FLAG_NWD BIT(12)
69 #define DESC_FLAG_CMD BIT(11)
70
71 struct bam_async_desc {
72         struct virt_dma_desc vd;
73
74         u32 num_desc;
75         u32 xfer_len;
76
77         /* transaction flags, EOT|EOB|NWD */
78         u16 flags;
79
80         struct bam_desc_hw *curr_desc;
81
82         /* list node for the desc in the bam_chan list of descriptors */
83         struct list_head desc_node;
84         enum dma_transfer_direction dir;
85         size_t length;
86         struct bam_desc_hw desc[0];
87 };
88
89 enum bam_reg {
90         BAM_CTRL,
91         BAM_REVISION,
92         BAM_NUM_PIPES,
93         BAM_DESC_CNT_TRSHLD,
94         BAM_IRQ_SRCS,
95         BAM_IRQ_SRCS_MSK,
96         BAM_IRQ_SRCS_UNMASKED,
97         BAM_IRQ_STTS,
98         BAM_IRQ_CLR,
99         BAM_IRQ_EN,
100         BAM_CNFG_BITS,
101         BAM_IRQ_SRCS_EE,
102         BAM_IRQ_SRCS_MSK_EE,
103         BAM_P_CTRL,
104         BAM_P_RST,
105         BAM_P_HALT,
106         BAM_P_IRQ_STTS,
107         BAM_P_IRQ_CLR,
108         BAM_P_IRQ_EN,
109         BAM_P_EVNT_DEST_ADDR,
110         BAM_P_EVNT_REG,
111         BAM_P_SW_OFSTS,
112         BAM_P_DATA_FIFO_ADDR,
113         BAM_P_DESC_FIFO_ADDR,
114         BAM_P_EVNT_GEN_TRSHLD,
115         BAM_P_FIFO_SIZES,
116 };
117
118 struct reg_offset_data {
119         u32 base_offset;
120         unsigned int pipe_mult, evnt_mult, ee_mult;
121 };
122
123 static const struct reg_offset_data bam_v1_3_reg_info[] = {
124         [BAM_CTRL]              = { 0x0F80, 0x00, 0x00, 0x00 },
125         [BAM_REVISION]          = { 0x0F84, 0x00, 0x00, 0x00 },
126         [BAM_NUM_PIPES]         = { 0x0FBC, 0x00, 0x00, 0x00 },
127         [BAM_DESC_CNT_TRSHLD]   = { 0x0F88, 0x00, 0x00, 0x00 },
128         [BAM_IRQ_SRCS]          = { 0x0F8C, 0x00, 0x00, 0x00 },
129         [BAM_IRQ_SRCS_MSK]      = { 0x0F90, 0x00, 0x00, 0x00 },
130         [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 },
131         [BAM_IRQ_STTS]          = { 0x0F94, 0x00, 0x00, 0x00 },
132         [BAM_IRQ_CLR]           = { 0x0F98, 0x00, 0x00, 0x00 },
133         [BAM_IRQ_EN]            = { 0x0F9C, 0x00, 0x00, 0x00 },
134         [BAM_CNFG_BITS]         = { 0x0FFC, 0x00, 0x00, 0x00 },
135         [BAM_IRQ_SRCS_EE]       = { 0x1800, 0x00, 0x00, 0x80 },
136         [BAM_IRQ_SRCS_MSK_EE]   = { 0x1804, 0x00, 0x00, 0x80 },
137         [BAM_P_CTRL]            = { 0x0000, 0x80, 0x00, 0x00 },
138         [BAM_P_RST]             = { 0x0004, 0x80, 0x00, 0x00 },
139         [BAM_P_HALT]            = { 0x0008, 0x80, 0x00, 0x00 },
140         [BAM_P_IRQ_STTS]        = { 0x0010, 0x80, 0x00, 0x00 },
141         [BAM_P_IRQ_CLR]         = { 0x0014, 0x80, 0x00, 0x00 },
142         [BAM_P_IRQ_EN]          = { 0x0018, 0x80, 0x00, 0x00 },
143         [BAM_P_EVNT_DEST_ADDR]  = { 0x102C, 0x00, 0x40, 0x00 },
144         [BAM_P_EVNT_REG]        = { 0x1018, 0x00, 0x40, 0x00 },
145         [BAM_P_SW_OFSTS]        = { 0x1000, 0x00, 0x40, 0x00 },
146         [BAM_P_DATA_FIFO_ADDR]  = { 0x1024, 0x00, 0x40, 0x00 },
147         [BAM_P_DESC_FIFO_ADDR]  = { 0x101C, 0x00, 0x40, 0x00 },
148         [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 },
149         [BAM_P_FIFO_SIZES]      = { 0x1020, 0x00, 0x40, 0x00 },
150 };
151
152 static const struct reg_offset_data bam_v1_4_reg_info[] = {
153         [BAM_CTRL]              = { 0x0000, 0x00, 0x00, 0x00 },
154         [BAM_REVISION]          = { 0x0004, 0x00, 0x00, 0x00 },
155         [BAM_NUM_PIPES]         = { 0x003C, 0x00, 0x00, 0x00 },
156         [BAM_DESC_CNT_TRSHLD]   = { 0x0008, 0x00, 0x00, 0x00 },
157         [BAM_IRQ_SRCS]          = { 0x000C, 0x00, 0x00, 0x00 },
158         [BAM_IRQ_SRCS_MSK]      = { 0x0010, 0x00, 0x00, 0x00 },
159         [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 },
160         [BAM_IRQ_STTS]          = { 0x0014, 0x00, 0x00, 0x00 },
161         [BAM_IRQ_CLR]           = { 0x0018, 0x00, 0x00, 0x00 },
162         [BAM_IRQ_EN]            = { 0x001C, 0x00, 0x00, 0x00 },
163         [BAM_CNFG_BITS]         = { 0x007C, 0x00, 0x00, 0x00 },
164         [BAM_IRQ_SRCS_EE]       = { 0x0800, 0x00, 0x00, 0x80 },
165         [BAM_IRQ_SRCS_MSK_EE]   = { 0x0804, 0x00, 0x00, 0x80 },
166         [BAM_P_CTRL]            = { 0x1000, 0x1000, 0x00, 0x00 },
167         [BAM_P_RST]             = { 0x1004, 0x1000, 0x00, 0x00 },
168         [BAM_P_HALT]            = { 0x1008, 0x1000, 0x00, 0x00 },
169         [BAM_P_IRQ_STTS]        = { 0x1010, 0x1000, 0x00, 0x00 },
170         [BAM_P_IRQ_CLR]         = { 0x1014, 0x1000, 0x00, 0x00 },
171         [BAM_P_IRQ_EN]          = { 0x1018, 0x1000, 0x00, 0x00 },
172         [BAM_P_EVNT_DEST_ADDR]  = { 0x182C, 0x00, 0x1000, 0x00 },
173         [BAM_P_EVNT_REG]        = { 0x1818, 0x00, 0x1000, 0x00 },
174         [BAM_P_SW_OFSTS]        = { 0x1800, 0x00, 0x1000, 0x00 },
175         [BAM_P_DATA_FIFO_ADDR]  = { 0x1824, 0x00, 0x1000, 0x00 },
176         [BAM_P_DESC_FIFO_ADDR]  = { 0x181C, 0x00, 0x1000, 0x00 },
177         [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 },
178         [BAM_P_FIFO_SIZES]      = { 0x1820, 0x00, 0x1000, 0x00 },
179 };
180
181 static const struct reg_offset_data bam_v1_7_reg_info[] = {
182         [BAM_CTRL]              = { 0x00000, 0x00, 0x00, 0x00 },
183         [BAM_REVISION]          = { 0x01000, 0x00, 0x00, 0x00 },
184         [BAM_NUM_PIPES]         = { 0x01008, 0x00, 0x00, 0x00 },
185         [BAM_DESC_CNT_TRSHLD]   = { 0x00008, 0x00, 0x00, 0x00 },
186         [BAM_IRQ_SRCS]          = { 0x03010, 0x00, 0x00, 0x00 },
187         [BAM_IRQ_SRCS_MSK]      = { 0x03014, 0x00, 0x00, 0x00 },
188         [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 },
189         [BAM_IRQ_STTS]          = { 0x00014, 0x00, 0x00, 0x00 },
190         [BAM_IRQ_CLR]           = { 0x00018, 0x00, 0x00, 0x00 },
191         [BAM_IRQ_EN]            = { 0x0001C, 0x00, 0x00, 0x00 },
192         [BAM_CNFG_BITS]         = { 0x0007C, 0x00, 0x00, 0x00 },
193         [BAM_IRQ_SRCS_EE]       = { 0x03000, 0x00, 0x00, 0x1000 },
194         [BAM_IRQ_SRCS_MSK_EE]   = { 0x03004, 0x00, 0x00, 0x1000 },
195         [BAM_P_CTRL]            = { 0x13000, 0x1000, 0x00, 0x00 },
196         [BAM_P_RST]             = { 0x13004, 0x1000, 0x00, 0x00 },
197         [BAM_P_HALT]            = { 0x13008, 0x1000, 0x00, 0x00 },
198         [BAM_P_IRQ_STTS]        = { 0x13010, 0x1000, 0x00, 0x00 },
199         [BAM_P_IRQ_CLR]         = { 0x13014, 0x1000, 0x00, 0x00 },
200         [BAM_P_IRQ_EN]          = { 0x13018, 0x1000, 0x00, 0x00 },
201         [BAM_P_EVNT_DEST_ADDR]  = { 0x1382C, 0x00, 0x1000, 0x00 },
202         [BAM_P_EVNT_REG]        = { 0x13818, 0x00, 0x1000, 0x00 },
203         [BAM_P_SW_OFSTS]        = { 0x13800, 0x00, 0x1000, 0x00 },
204         [BAM_P_DATA_FIFO_ADDR]  = { 0x13824, 0x00, 0x1000, 0x00 },
205         [BAM_P_DESC_FIFO_ADDR]  = { 0x1381C, 0x00, 0x1000, 0x00 },
206         [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 },
207         [BAM_P_FIFO_SIZES]      = { 0x13820, 0x00, 0x1000, 0x00 },
208 };
209
210 /* BAM CTRL */
211 #define BAM_SW_RST                      BIT(0)
212 #define BAM_EN                          BIT(1)
213 #define BAM_EN_ACCUM                    BIT(4)
214 #define BAM_TESTBUS_SEL_SHIFT           5
215 #define BAM_TESTBUS_SEL_MASK            0x3F
216 #define BAM_DESC_CACHE_SEL_SHIFT        13
217 #define BAM_DESC_CACHE_SEL_MASK         0x3
218 #define BAM_CACHED_DESC_STORE           BIT(15)
219 #define IBC_DISABLE                     BIT(16)
220
221 /* BAM REVISION */
222 #define REVISION_SHIFT          0
223 #define REVISION_MASK           0xFF
224 #define NUM_EES_SHIFT           8
225 #define NUM_EES_MASK            0xF
226 #define CE_BUFFER_SIZE          BIT(13)
227 #define AXI_ACTIVE              BIT(14)
228 #define USE_VMIDMT              BIT(15)
229 #define SECURED                 BIT(16)
230 #define BAM_HAS_NO_BYPASS       BIT(17)
231 #define HIGH_FREQUENCY_BAM      BIT(18)
232 #define INACTIV_TMRS_EXST       BIT(19)
233 #define NUM_INACTIV_TMRS        BIT(20)
234 #define DESC_CACHE_DEPTH_SHIFT  21
235 #define DESC_CACHE_DEPTH_1      (0 << DESC_CACHE_DEPTH_SHIFT)
236 #define DESC_CACHE_DEPTH_2      (1 << DESC_CACHE_DEPTH_SHIFT)
237 #define DESC_CACHE_DEPTH_3      (2 << DESC_CACHE_DEPTH_SHIFT)
238 #define DESC_CACHE_DEPTH_4      (3 << DESC_CACHE_DEPTH_SHIFT)
239 #define CMD_DESC_EN             BIT(23)
240 #define INACTIV_TMR_BASE_SHIFT  24
241 #define INACTIV_TMR_BASE_MASK   0xFF
242
243 /* BAM NUM PIPES */
244 #define BAM_NUM_PIPES_SHIFT             0
245 #define BAM_NUM_PIPES_MASK              0xFF
246 #define PERIPH_NON_PIPE_GRP_SHIFT       16
247 #define PERIPH_NON_PIP_GRP_MASK         0xFF
248 #define BAM_NON_PIPE_GRP_SHIFT          24
249 #define BAM_NON_PIPE_GRP_MASK           0xFF
250
251 /* BAM CNFG BITS */
252 #define BAM_PIPE_CNFG           BIT(2)
253 #define BAM_FULL_PIPE           BIT(11)
254 #define BAM_NO_EXT_P_RST        BIT(12)
255 #define BAM_IBC_DISABLE         BIT(13)
256 #define BAM_SB_CLK_REQ          BIT(14)
257 #define BAM_PSM_CSW_REQ         BIT(15)
258 #define BAM_PSM_P_RES           BIT(16)
259 #define BAM_AU_P_RES            BIT(17)
260 #define BAM_SI_P_RES            BIT(18)
261 #define BAM_WB_P_RES            BIT(19)
262 #define BAM_WB_BLK_CSW          BIT(20)
263 #define BAM_WB_CSW_ACK_IDL      BIT(21)
264 #define BAM_WB_RETR_SVPNT       BIT(22)
265 #define BAM_WB_DSC_AVL_P_RST    BIT(23)
266 #define BAM_REG_P_EN            BIT(24)
267 #define BAM_PSM_P_HD_DATA       BIT(25)
268 #define BAM_AU_ACCUMED          BIT(26)
269 #define BAM_CMD_ENABLE          BIT(27)
270
271 #define BAM_CNFG_BITS_DEFAULT   (BAM_PIPE_CNFG |        \
272                                  BAM_NO_EXT_P_RST |     \
273                                  BAM_IBC_DISABLE |      \
274                                  BAM_SB_CLK_REQ |       \
275                                  BAM_PSM_CSW_REQ |      \
276                                  BAM_PSM_P_RES |        \
277                                  BAM_AU_P_RES |         \
278                                  BAM_SI_P_RES |         \
279                                  BAM_WB_P_RES |         \
280                                  BAM_WB_BLK_CSW |       \
281                                  BAM_WB_CSW_ACK_IDL |   \
282                                  BAM_WB_RETR_SVPNT |    \
283                                  BAM_WB_DSC_AVL_P_RST | \
284                                  BAM_REG_P_EN |         \
285                                  BAM_PSM_P_HD_DATA |    \
286                                  BAM_AU_ACCUMED |       \
287                                  BAM_CMD_ENABLE)
288
289 /* PIPE CTRL */
290 #define P_EN                    BIT(1)
291 #define P_DIRECTION             BIT(3)
292 #define P_SYS_STRM              BIT(4)
293 #define P_SYS_MODE              BIT(5)
294 #define P_AUTO_EOB              BIT(6)
295 #define P_AUTO_EOB_SEL_SHIFT    7
296 #define P_AUTO_EOB_SEL_512      (0 << P_AUTO_EOB_SEL_SHIFT)
297 #define P_AUTO_EOB_SEL_256      (1 << P_AUTO_EOB_SEL_SHIFT)
298 #define P_AUTO_EOB_SEL_128      (2 << P_AUTO_EOB_SEL_SHIFT)
299 #define P_AUTO_EOB_SEL_64       (3 << P_AUTO_EOB_SEL_SHIFT)
300 #define P_PREFETCH_LIMIT_SHIFT  9
301 #define P_PREFETCH_LIMIT_32     (0 << P_PREFETCH_LIMIT_SHIFT)
302 #define P_PREFETCH_LIMIT_16     (1 << P_PREFETCH_LIMIT_SHIFT)
303 #define P_PREFETCH_LIMIT_4      (2 << P_PREFETCH_LIMIT_SHIFT)
304 #define P_WRITE_NWD             BIT(11)
305 #define P_LOCK_GROUP_SHIFT      16
306 #define P_LOCK_GROUP_MASK       0x1F
307
308 /* BAM_DESC_CNT_TRSHLD */
309 #define CNT_TRSHLD              0xffff
310 #define DEFAULT_CNT_THRSHLD     0x4
311
312 /* BAM_IRQ_SRCS */
313 #define BAM_IRQ                 BIT(31)
314 #define P_IRQ                   0x7fffffff
315
316 /* BAM_IRQ_SRCS_MSK */
317 #define BAM_IRQ_MSK             BAM_IRQ
318 #define P_IRQ_MSK               P_IRQ
319
320 /* BAM_IRQ_STTS */
321 #define BAM_TIMER_IRQ           BIT(4)
322 #define BAM_EMPTY_IRQ           BIT(3)
323 #define BAM_ERROR_IRQ           BIT(2)
324 #define BAM_HRESP_ERR_IRQ       BIT(1)
325
326 /* BAM_IRQ_CLR */
327 #define BAM_TIMER_CLR           BIT(4)
328 #define BAM_EMPTY_CLR           BIT(3)
329 #define BAM_ERROR_CLR           BIT(2)
330 #define BAM_HRESP_ERR_CLR       BIT(1)
331
332 /* BAM_IRQ_EN */
333 #define BAM_TIMER_EN            BIT(4)
334 #define BAM_EMPTY_EN            BIT(3)
335 #define BAM_ERROR_EN            BIT(2)
336 #define BAM_HRESP_ERR_EN        BIT(1)
337
338 /* BAM_P_IRQ_EN */
339 #define P_PRCSD_DESC_EN         BIT(0)
340 #define P_TIMER_EN              BIT(1)
341 #define P_WAKE_EN               BIT(2)
342 #define P_OUT_OF_DESC_EN        BIT(3)
343 #define P_ERR_EN                BIT(4)
344 #define P_TRNSFR_END_EN         BIT(5)
345 #define P_DEFAULT_IRQS_EN       (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
346
347 /* BAM_P_SW_OFSTS */
348 #define P_SW_OFSTS_MASK         0xffff
349
350 #define BAM_DESC_FIFO_SIZE      SZ_32K
351 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
352 #define BAM_FIFO_SIZE   (SZ_32K - 8)
353 #define IS_BUSY(chan)   (CIRC_SPACE(bchan->tail, bchan->head,\
354                          MAX_DESCRIPTORS + 1) == 0)
355
356 struct bam_chan {
357         struct virt_dma_chan vc;
358
359         struct bam_device *bdev;
360
361         /* configuration from device tree */
362         u32 id;
363
364         /* runtime configuration */
365         struct dma_slave_config slave;
366
367         /* fifo storage */
368         struct bam_desc_hw *fifo_virt;
369         dma_addr_t fifo_phys;
370
371         /* fifo markers */
372         unsigned short head;            /* start of active descriptor entries */
373         unsigned short tail;            /* end of active descriptor entries */
374
375         unsigned int initialized;       /* is the channel hw initialized? */
376         unsigned int paused;            /* is the channel paused? */
377         unsigned int reconfigure;       /* new slave config? */
378         /* list of descriptors currently processed */
379         struct list_head desc_list;
380
381         struct list_head node;
382 };
383
384 static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
385 {
386         return container_of(common, struct bam_chan, vc.chan);
387 }
388
389 struct bam_device {
390         void __iomem *regs;
391         struct device *dev;
392         struct dma_device common;
393         struct device_dma_parameters dma_parms;
394         struct bam_chan *channels;
395         u32 num_channels;
396         u32 num_ees;
397
398         /* execution environment ID, from DT */
399         u32 ee;
400         bool controlled_remotely;
401
402         const struct reg_offset_data *layout;
403
404         struct clk *bamclk;
405         int irq;
406
407         /* dma start transaction tasklet */
408         struct tasklet_struct task;
409 };
410
411 /**
412  * bam_addr - returns BAM register address
413  * @bdev: bam device
414  * @pipe: pipe instance (ignored when register doesn't have multiple instances)
415  * @reg:  register enum
416  */
417 static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
418                 enum bam_reg reg)
419 {
420         const struct reg_offset_data r = bdev->layout[reg];
421
422         return bdev->regs + r.base_offset +
423                 r.pipe_mult * pipe +
424                 r.evnt_mult * pipe +
425                 r.ee_mult * bdev->ee;
426 }
427
428 /**
429  * bam_reset_channel - Reset individual BAM DMA channel
430  * @bchan: bam channel
431  *
432  * This function resets a specific BAM channel
433  */
434 static void bam_reset_channel(struct bam_chan *bchan)
435 {
436         struct bam_device *bdev = bchan->bdev;
437
438         lockdep_assert_held(&bchan->vc.lock);
439
440         /* reset channel */
441         writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));
442         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));
443
444         /* don't allow cpu to reorder BAM register accesses done after this */
445         wmb();
446
447         /* make sure hw is initialized when channel is used the first time  */
448         bchan->initialized = 0;
449 }
450
451 /**
452  * bam_chan_init_hw - Initialize channel hardware
453  * @bchan: bam channel
454  * @dir: DMA transfer direction
455  *
456  * This function resets and initializes the BAM channel
457  */
458 static void bam_chan_init_hw(struct bam_chan *bchan,
459         enum dma_transfer_direction dir)
460 {
461         struct bam_device *bdev = bchan->bdev;
462         u32 val;
463
464         /* Reset the channel to clear internal state of the FIFO */
465         bam_reset_channel(bchan);
466
467         /*
468          * write out 8 byte aligned address.  We have enough space for this
469          * because we allocated 1 more descriptor (8 bytes) than we can use
470          */
471         writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),
472                         bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));
473         writel_relaxed(BAM_FIFO_SIZE,
474                         bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));
475
476         /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */
477         writel_relaxed(P_DEFAULT_IRQS_EN,
478                         bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
479
480         /* unmask the specific pipe and EE combo */
481         val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
482         val |= BIT(bchan->id);
483         writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
484
485         /* don't allow cpu to reorder the channel enable done below */
486         wmb();
487
488         /* set fixed direction and mode, then enable channel */
489         val = P_EN | P_SYS_MODE;
490         if (dir == DMA_DEV_TO_MEM)
491                 val |= P_DIRECTION;
492
493         writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));
494
495         bchan->initialized = 1;
496
497         /* init FIFO pointers */
498         bchan->head = 0;
499         bchan->tail = 0;
500 }
501
502 /**
503  * bam_alloc_chan - Allocate channel resources for DMA channel.
504  * @chan: specified channel
505  *
506  * This function allocates the FIFO descriptor memory
507  */
508 static int bam_alloc_chan(struct dma_chan *chan)
509 {
510         struct bam_chan *bchan = to_bam_chan(chan);
511         struct bam_device *bdev = bchan->bdev;
512
513         if (bchan->fifo_virt)
514                 return 0;
515
516         /* allocate FIFO descriptor space, but only if necessary */
517         bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
518                                         &bchan->fifo_phys, GFP_KERNEL);
519
520         if (!bchan->fifo_virt) {
521                 dev_err(bdev->dev, "Failed to allocate desc fifo\n");
522                 return -ENOMEM;
523         }
524
525         return 0;
526 }
527
528 /**
529  * bam_free_chan - Frees dma resources associated with specific channel
530  * @chan: specified channel
531  *
532  * Free the allocated fifo descriptor memory and channel resources
533  *
534  */
535 static void bam_free_chan(struct dma_chan *chan)
536 {
537         struct bam_chan *bchan = to_bam_chan(chan);
538         struct bam_device *bdev = bchan->bdev;
539         u32 val;
540         unsigned long flags;
541         int ret;
542
543         ret = pm_runtime_get_sync(bdev->dev);
544         if (ret < 0)
545                 return;
546
547         vchan_free_chan_resources(to_virt_chan(chan));
548
549         if (!list_empty(&bchan->desc_list)) {
550                 dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
551                 goto err;
552         }
553
554         spin_lock_irqsave(&bchan->vc.lock, flags);
555         bam_reset_channel(bchan);
556         spin_unlock_irqrestore(&bchan->vc.lock, flags);
557
558         dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
559                     bchan->fifo_phys);
560         bchan->fifo_virt = NULL;
561
562         /* mask irq for pipe/channel */
563         val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
564         val &= ~BIT(bchan->id);
565         writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
566
567         /* disable irq */
568         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
569
570 err:
571         pm_runtime_mark_last_busy(bdev->dev);
572         pm_runtime_put_autosuspend(bdev->dev);
573 }
574
575 /**
576  * bam_slave_config - set slave configuration for channel
577  * @chan: dma channel
578  * @cfg: slave configuration
579  *
580  * Sets slave configuration for channel
581  *
582  */
583 static int bam_slave_config(struct dma_chan *chan,
584                             struct dma_slave_config *cfg)
585 {
586         struct bam_chan *bchan = to_bam_chan(chan);
587         unsigned long flag;
588
589         spin_lock_irqsave(&bchan->vc.lock, flag);
590         memcpy(&bchan->slave, cfg, sizeof(*cfg));
591         bchan->reconfigure = 1;
592         spin_unlock_irqrestore(&bchan->vc.lock, flag);
593
594         return 0;
595 }
596
597 /**
598  * bam_prep_slave_sg - Prep slave sg transaction
599  *
600  * @chan: dma channel
601  * @sgl: scatter gather list
602  * @sg_len: length of sg
603  * @direction: DMA transfer direction
604  * @flags: DMA flags
605  * @context: transfer context (unused)
606  */
607 static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
608         struct scatterlist *sgl, unsigned int sg_len,
609         enum dma_transfer_direction direction, unsigned long flags,
610         void *context)
611 {
612         struct bam_chan *bchan = to_bam_chan(chan);
613         struct bam_device *bdev = bchan->bdev;
614         struct bam_async_desc *async_desc;
615         struct scatterlist *sg;
616         u32 i;
617         struct bam_desc_hw *desc;
618         unsigned int num_alloc = 0;
619
620
621         if (!is_slave_direction(direction)) {
622                 dev_err(bdev->dev, "invalid dma direction\n");
623                 return NULL;
624         }
625
626         /* calculate number of required entries */
627         for_each_sg(sgl, sg, sg_len, i)
628                 num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE);
629
630         /* allocate enough room to accomodate the number of entries */
631         async_desc = kzalloc(sizeof(*async_desc) +
632                         (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT);
633
634         if (!async_desc)
635                 goto err_out;
636
637         if (flags & DMA_PREP_FENCE)
638                 async_desc->flags |= DESC_FLAG_NWD;
639
640         if (flags & DMA_PREP_INTERRUPT)
641                 async_desc->flags |= DESC_FLAG_EOT;
642
643         async_desc->num_desc = num_alloc;
644         async_desc->curr_desc = async_desc->desc;
645         async_desc->dir = direction;
646
647         /* fill in temporary descriptors */
648         desc = async_desc->desc;
649         for_each_sg(sgl, sg, sg_len, i) {
650                 unsigned int remainder = sg_dma_len(sg);
651                 unsigned int curr_offset = 0;
652
653                 do {
654                         if (flags & DMA_PREP_CMD)
655                                 desc->flags |= cpu_to_le16(DESC_FLAG_CMD);
656
657                         desc->addr = cpu_to_le32(sg_dma_address(sg) +
658                                                  curr_offset);
659
660                         if (remainder > BAM_FIFO_SIZE) {
661                                 desc->size = cpu_to_le16(BAM_FIFO_SIZE);
662                                 remainder -= BAM_FIFO_SIZE;
663                                 curr_offset += BAM_FIFO_SIZE;
664                         } else {
665                                 desc->size = cpu_to_le16(remainder);
666                                 remainder = 0;
667                         }
668
669                         async_desc->length += le16_to_cpu(desc->size);
670                         desc++;
671                 } while (remainder > 0);
672         }
673
674         return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
675
676 err_out:
677         kfree(async_desc);
678         return NULL;
679 }
680
681 /**
682  * bam_dma_terminate_all - terminate all transactions on a channel
683  * @chan: bam dma channel
684  *
685  * Dequeues and frees all transactions
686  * No callbacks are done
687  *
688  */
689 static int bam_dma_terminate_all(struct dma_chan *chan)
690 {
691         struct bam_chan *bchan = to_bam_chan(chan);
692         struct bam_async_desc *async_desc, *tmp;
693         unsigned long flag;
694         LIST_HEAD(head);
695
696         /* remove all transactions, including active transaction */
697         spin_lock_irqsave(&bchan->vc.lock, flag);
698         list_for_each_entry_safe(async_desc, tmp,
699                                  &bchan->desc_list, desc_node) {
700                 list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
701                 list_del(&async_desc->desc_node);
702         }
703
704         vchan_get_all_descriptors(&bchan->vc, &head);
705         spin_unlock_irqrestore(&bchan->vc.lock, flag);
706
707         vchan_dma_desc_free_list(&bchan->vc, &head);
708
709         return 0;
710 }
711
712 /**
713  * bam_pause - Pause DMA channel
714  * @chan: dma channel
715  *
716  */
717 static int bam_pause(struct dma_chan *chan)
718 {
719         struct bam_chan *bchan = to_bam_chan(chan);
720         struct bam_device *bdev = bchan->bdev;
721         unsigned long flag;
722         int ret;
723
724         ret = pm_runtime_get_sync(bdev->dev);
725         if (ret < 0)
726                 return ret;
727
728         spin_lock_irqsave(&bchan->vc.lock, flag);
729         writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
730         bchan->paused = 1;
731         spin_unlock_irqrestore(&bchan->vc.lock, flag);
732         pm_runtime_mark_last_busy(bdev->dev);
733         pm_runtime_put_autosuspend(bdev->dev);
734
735         return 0;
736 }
737
738 /**
739  * bam_resume - Resume DMA channel operations
740  * @chan: dma channel
741  *
742  */
743 static int bam_resume(struct dma_chan *chan)
744 {
745         struct bam_chan *bchan = to_bam_chan(chan);
746         struct bam_device *bdev = bchan->bdev;
747         unsigned long flag;
748         int ret;
749
750         ret = pm_runtime_get_sync(bdev->dev);
751         if (ret < 0)
752                 return ret;
753
754         spin_lock_irqsave(&bchan->vc.lock, flag);
755         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
756         bchan->paused = 0;
757         spin_unlock_irqrestore(&bchan->vc.lock, flag);
758         pm_runtime_mark_last_busy(bdev->dev);
759         pm_runtime_put_autosuspend(bdev->dev);
760
761         return 0;
762 }
763
764 /**
765  * process_channel_irqs - processes the channel interrupts
766  * @bdev: bam controller
767  *
768  * This function processes the channel interrupts
769  *
770  */
771 static u32 process_channel_irqs(struct bam_device *bdev)
772 {
773         u32 i, srcs, pipe_stts, offset, avail;
774         unsigned long flags;
775         struct bam_async_desc *async_desc, *tmp;
776
777         srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
778
779         /* return early if no pipe/channel interrupts are present */
780         if (!(srcs & P_IRQ))
781                 return srcs;
782
783         for (i = 0; i < bdev->num_channels; i++) {
784                 struct bam_chan *bchan = &bdev->channels[i];
785
786                 if (!(srcs & BIT(i)))
787                         continue;
788
789                 /* clear pipe irq */
790                 pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));
791
792                 writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
793
794                 spin_lock_irqsave(&bchan->vc.lock, flags);
795
796                 offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
797                                        P_SW_OFSTS_MASK;
798                 offset /= sizeof(struct bam_desc_hw);
799
800                 /* Number of bytes available to read */
801                 avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);
802
803                 list_for_each_entry_safe(async_desc, tmp,
804                                          &bchan->desc_list, desc_node) {
805                         /* Not enough data to read */
806                         if (avail < async_desc->xfer_len)
807                                 break;
808
809                         /* manage FIFO */
810                         bchan->head += async_desc->xfer_len;
811                         bchan->head %= MAX_DESCRIPTORS;
812
813                         async_desc->num_desc -= async_desc->xfer_len;
814                         async_desc->curr_desc += async_desc->xfer_len;
815                         avail -= async_desc->xfer_len;
816
817                         /*
818                          * if complete, process cookie. Otherwise
819                          * push back to front of desc_issued so that
820                          * it gets restarted by the tasklet
821                          */
822                         if (!async_desc->num_desc) {
823                                 vchan_cookie_complete(&async_desc->vd);
824                         } else {
825                                 list_add(&async_desc->vd.node,
826                                          &bchan->vc.desc_issued);
827                         }
828                         list_del(&async_desc->desc_node);
829                 }
830
831                 spin_unlock_irqrestore(&bchan->vc.lock, flags);
832         }
833
834         return srcs;
835 }
836
837 /**
838  * bam_dma_irq - irq handler for bam controller
839  * @irq: IRQ of interrupt
840  * @data: callback data
841  *
842  * IRQ handler for the bam controller
843  */
844 static irqreturn_t bam_dma_irq(int irq, void *data)
845 {
846         struct bam_device *bdev = data;
847         u32 clr_mask = 0, srcs = 0;
848         int ret;
849
850         srcs |= process_channel_irqs(bdev);
851
852         /* kick off tasklet to start next dma transfer */
853         if (srcs & P_IRQ)
854                 tasklet_schedule(&bdev->task);
855
856         ret = pm_runtime_get_sync(bdev->dev);
857         if (ret < 0)
858                 return ret;
859
860         if (srcs & BAM_IRQ) {
861                 clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
862
863                 /*
864                  * don't allow reorder of the various accesses to the BAM
865                  * registers
866                  */
867                 mb();
868
869                 writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));
870         }
871
872         pm_runtime_mark_last_busy(bdev->dev);
873         pm_runtime_put_autosuspend(bdev->dev);
874
875         return IRQ_HANDLED;
876 }
877
878 /**
879  * bam_tx_status - returns status of transaction
880  * @chan: dma channel
881  * @cookie: transaction cookie
882  * @txstate: DMA transaction state
883  *
884  * Return status of dma transaction
885  */
886 static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
887                 struct dma_tx_state *txstate)
888 {
889         struct bam_chan *bchan = to_bam_chan(chan);
890         struct bam_async_desc *async_desc;
891         struct virt_dma_desc *vd;
892         int ret;
893         size_t residue = 0;
894         unsigned int i;
895         unsigned long flags;
896
897         ret = dma_cookie_status(chan, cookie, txstate);
898         if (ret == DMA_COMPLETE)
899                 return ret;
900
901         if (!txstate)
902                 return bchan->paused ? DMA_PAUSED : ret;
903
904         spin_lock_irqsave(&bchan->vc.lock, flags);
905         vd = vchan_find_desc(&bchan->vc, cookie);
906         if (vd) {
907                 residue = container_of(vd, struct bam_async_desc, vd)->length;
908         } else {
909                 list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
910                         if (async_desc->vd.tx.cookie != cookie)
911                                 continue;
912
913                         for (i = 0; i < async_desc->num_desc; i++)
914                                 residue += le16_to_cpu(
915                                                 async_desc->curr_desc[i].size);
916                 }
917         }
918
919         spin_unlock_irqrestore(&bchan->vc.lock, flags);
920
921         dma_set_residue(txstate, residue);
922
923         if (ret == DMA_IN_PROGRESS && bchan->paused)
924                 ret = DMA_PAUSED;
925
926         return ret;
927 }
928
929 /**
930  * bam_apply_new_config
931  * @bchan: bam dma channel
932  * @dir: DMA direction
933  */
934 static void bam_apply_new_config(struct bam_chan *bchan,
935         enum dma_transfer_direction dir)
936 {
937         struct bam_device *bdev = bchan->bdev;
938         u32 maxburst;
939
940         if (!bdev->controlled_remotely) {
941                 if (dir == DMA_DEV_TO_MEM)
942                         maxburst = bchan->slave.src_maxburst;
943                 else
944                         maxburst = bchan->slave.dst_maxburst;
945
946                 writel_relaxed(maxburst,
947                                bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
948         }
949
950         bchan->reconfigure = 0;
951 }
952
953 /**
954  * bam_start_dma - start next transaction
955  * @bchan: bam dma channel
956  */
957 static void bam_start_dma(struct bam_chan *bchan)
958 {
959         struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
960         struct bam_device *bdev = bchan->bdev;
961         struct bam_async_desc *async_desc = NULL;
962         struct bam_desc_hw *desc;
963         struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
964                                         sizeof(struct bam_desc_hw));
965         int ret;
966         unsigned int avail;
967         struct dmaengine_desc_callback cb;
968
969         lockdep_assert_held(&bchan->vc.lock);
970
971         if (!vd)
972                 return;
973
974         ret = pm_runtime_get_sync(bdev->dev);
975         if (ret < 0)
976                 return;
977
978         while (vd && !IS_BUSY(bchan)) {
979                 list_del(&vd->node);
980
981                 async_desc = container_of(vd, struct bam_async_desc, vd);
982
983                 /* on first use, initialize the channel hardware */
984                 if (!bchan->initialized)
985                         bam_chan_init_hw(bchan, async_desc->dir);
986
987                 /* apply new slave config changes, if necessary */
988                 if (bchan->reconfigure)
989                         bam_apply_new_config(bchan, async_desc->dir);
990
991                 desc = async_desc->curr_desc;
992                 avail = CIRC_SPACE(bchan->tail, bchan->head,
993                                    MAX_DESCRIPTORS + 1);
994
995                 if (async_desc->num_desc > avail)
996                         async_desc->xfer_len = avail;
997                 else
998                         async_desc->xfer_len = async_desc->num_desc;
999
1000                 /* set any special flags on the last descriptor */
1001                 if (async_desc->num_desc == async_desc->xfer_len)
1002                         desc[async_desc->xfer_len - 1].flags |=
1003                                                 cpu_to_le16(async_desc->flags);
1004
1005                 vd = vchan_next_desc(&bchan->vc);
1006
1007                 dmaengine_desc_get_callback(&async_desc->vd.tx, &cb);
1008
1009                 /*
1010                  * An interrupt is generated at this desc, if
1011                  *  - FIFO is FULL.
1012                  *  - No more descriptors to add.
1013                  *  - If a callback completion was requested for this DESC,
1014                  *     In this case, BAM will deliver the completion callback
1015                  *     for this desc and continue processing the next desc.
1016                  */
1017                 if (((avail <= async_desc->xfer_len) || !vd ||
1018                      dmaengine_desc_callback_valid(&cb)) &&
1019                     !(async_desc->flags & DESC_FLAG_EOT))
1020                         desc[async_desc->xfer_len - 1].flags |=
1021                                 cpu_to_le16(DESC_FLAG_INT);
1022
1023                 if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
1024                         u32 partial = MAX_DESCRIPTORS - bchan->tail;
1025
1026                         memcpy(&fifo[bchan->tail], desc,
1027                                partial * sizeof(struct bam_desc_hw));
1028                         memcpy(fifo, &desc[partial],
1029                                (async_desc->xfer_len - partial) *
1030                                 sizeof(struct bam_desc_hw));
1031                 } else {
1032                         memcpy(&fifo[bchan->tail], desc,
1033                                async_desc->xfer_len *
1034                                sizeof(struct bam_desc_hw));
1035                 }
1036
1037                 bchan->tail += async_desc->xfer_len;
1038                 bchan->tail %= MAX_DESCRIPTORS;
1039                 list_add_tail(&async_desc->desc_node, &bchan->desc_list);
1040         }
1041
1042         /* ensure descriptor writes and dma start not reordered */
1043         wmb();
1044         writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),
1045                         bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));
1046
1047         pm_runtime_mark_last_busy(bdev->dev);
1048         pm_runtime_put_autosuspend(bdev->dev);
1049 }
1050
1051 /**
1052  * dma_tasklet - DMA IRQ tasklet
1053  * @data: tasklet argument (bam controller structure)
1054  *
1055  * Sets up next DMA operation and then processes all completed transactions
1056  */
1057 static void dma_tasklet(unsigned long data)
1058 {
1059         struct bam_device *bdev = (struct bam_device *)data;
1060         struct bam_chan *bchan;
1061         unsigned long flags;
1062         unsigned int i;
1063
1064         /* go through the channels and kick off transactions */
1065         for (i = 0; i < bdev->num_channels; i++) {
1066                 bchan = &bdev->channels[i];
1067                 spin_lock_irqsave(&bchan->vc.lock, flags);
1068
1069                 if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
1070                         bam_start_dma(bchan);
1071                 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1072         }
1073
1074 }
1075
1076 /**
1077  * bam_issue_pending - starts pending transactions
1078  * @chan: dma channel
1079  *
1080  * Calls tasklet directly which in turn starts any pending transactions
1081  */
1082 static void bam_issue_pending(struct dma_chan *chan)
1083 {
1084         struct bam_chan *bchan = to_bam_chan(chan);
1085         unsigned long flags;
1086
1087         spin_lock_irqsave(&bchan->vc.lock, flags);
1088
1089         /* if work pending and idle, start a transaction */
1090         if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
1091                 bam_start_dma(bchan);
1092
1093         spin_unlock_irqrestore(&bchan->vc.lock, flags);
1094 }
1095
1096 /**
1097  * bam_dma_free_desc - free descriptor memory
1098  * @vd: virtual descriptor
1099  *
1100  */
1101 static void bam_dma_free_desc(struct virt_dma_desc *vd)
1102 {
1103         struct bam_async_desc *async_desc = container_of(vd,
1104                         struct bam_async_desc, vd);
1105
1106         kfree(async_desc);
1107 }
1108
1109 static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec,
1110                 struct of_dma *of)
1111 {
1112         struct bam_device *bdev = container_of(of->of_dma_data,
1113                                         struct bam_device, common);
1114         unsigned int request;
1115
1116         if (dma_spec->args_count != 1)
1117                 return NULL;
1118
1119         request = dma_spec->args[0];
1120         if (request >= bdev->num_channels)
1121                 return NULL;
1122
1123         return dma_get_slave_channel(&(bdev->channels[request].vc.chan));
1124 }
1125
1126 /**
1127  * bam_init
1128  * @bdev: bam device
1129  *
1130  * Initialization helper for global bam registers
1131  */
1132 static int bam_init(struct bam_device *bdev)
1133 {
1134         u32 val;
1135
1136         /* read revision and configuration information */
1137         if (!bdev->num_ees) {
1138                 val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION));
1139                 bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK;
1140         }
1141
1142         /* check that configured EE is within range */
1143         if (bdev->ee >= bdev->num_ees)
1144                 return -EINVAL;
1145
1146         if (!bdev->num_channels) {
1147                 val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));
1148                 bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1149         }
1150
1151         if (bdev->controlled_remotely)
1152                 return 0;
1153
1154         /* s/w reset bam */
1155         /* after reset all pipes are disabled and idle */
1156         val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
1157         val |= BAM_SW_RST;
1158         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1159         val &= ~BAM_SW_RST;
1160         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1161
1162         /* make sure previous stores are visible before enabling BAM */
1163         wmb();
1164
1165         /* enable bam */
1166         val |= BAM_EN;
1167         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1168
1169         /* set descriptor threshhold, start with 4 bytes */
1170         writel_relaxed(DEFAULT_CNT_THRSHLD,
1171                         bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1172
1173         /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
1174         writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
1175
1176         /* enable irqs for errors */
1177         writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
1178                         bam_addr(bdev, 0, BAM_IRQ_EN));
1179
1180         /* unmask global bam interrupt */
1181         writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1182
1183         return 0;
1184 }
1185
1186 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
1187         u32 index)
1188 {
1189         bchan->id = index;
1190         bchan->bdev = bdev;
1191
1192         vchan_init(&bchan->vc, &bdev->common);
1193         bchan->vc.desc_free = bam_dma_free_desc;
1194         INIT_LIST_HEAD(&bchan->desc_list);
1195 }
1196
1197 static const struct of_device_id bam_of_match[] = {
1198         { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1199         { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1200         { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1201         {}
1202 };
1203
1204 MODULE_DEVICE_TABLE(of, bam_of_match);
1205
1206 static int bam_dma_probe(struct platform_device *pdev)
1207 {
1208         struct bam_device *bdev;
1209         const struct of_device_id *match;
1210         struct resource *iores;
1211         int ret, i;
1212
1213         bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);
1214         if (!bdev)
1215                 return -ENOMEM;
1216
1217         bdev->dev = &pdev->dev;
1218
1219         match = of_match_node(bam_of_match, pdev->dev.of_node);
1220         if (!match) {
1221                 dev_err(&pdev->dev, "Unsupported BAM module\n");
1222                 return -ENODEV;
1223         }
1224
1225         bdev->layout = match->data;
1226
1227         iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1228         bdev->regs = devm_ioremap_resource(&pdev->dev, iores);
1229         if (IS_ERR(bdev->regs))
1230                 return PTR_ERR(bdev->regs);
1231
1232         bdev->irq = platform_get_irq(pdev, 0);
1233         if (bdev->irq < 0)
1234                 return bdev->irq;
1235
1236         ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);
1237         if (ret) {
1238                 dev_err(bdev->dev, "Execution environment unspecified\n");
1239                 return ret;
1240         }
1241
1242         bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1243                                                 "qcom,controlled-remotely");
1244
1245         if (bdev->controlled_remotely) {
1246                 ret = of_property_read_u32(pdev->dev.of_node, "num-channels",
1247                                            &bdev->num_channels);
1248                 if (ret)
1249                         dev_err(bdev->dev, "num-channels unspecified in dt\n");
1250
1251                 ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees",
1252                                            &bdev->num_ees);
1253                 if (ret)
1254                         dev_err(bdev->dev, "num-ees unspecified in dt\n");
1255         }
1256
1257         bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
1258         if (IS_ERR(bdev->bamclk)) {
1259                 if (!bdev->controlled_remotely)
1260                         return PTR_ERR(bdev->bamclk);
1261
1262                 bdev->bamclk = NULL;
1263         }
1264
1265         ret = clk_prepare_enable(bdev->bamclk);
1266         if (ret) {
1267                 dev_err(bdev->dev, "failed to prepare/enable clock\n");
1268                 return ret;
1269         }
1270
1271         ret = bam_init(bdev);
1272         if (ret)
1273                 goto err_disable_clk;
1274
1275         tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev);
1276
1277         bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
1278                                 sizeof(*bdev->channels), GFP_KERNEL);
1279
1280         if (!bdev->channels) {
1281                 ret = -ENOMEM;
1282                 goto err_tasklet_kill;
1283         }
1284
1285         /* allocate and initialize channels */
1286         INIT_LIST_HEAD(&bdev->common.channels);
1287
1288         for (i = 0; i < bdev->num_channels; i++)
1289                 bam_channel_init(bdev, &bdev->channels[i], i);
1290
1291         ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,
1292                         IRQF_TRIGGER_HIGH, "bam_dma", bdev);
1293         if (ret)
1294                 goto err_bam_channel_exit;
1295
1296         /* set max dma segment size */
1297         bdev->common.dev = bdev->dev;
1298         bdev->common.dev->dma_parms = &bdev->dma_parms;
1299         ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);
1300         if (ret) {
1301                 dev_err(bdev->dev, "cannot set maximum segment size\n");
1302                 goto err_bam_channel_exit;
1303         }
1304
1305         platform_set_drvdata(pdev, bdev);
1306
1307         /* set capabilities */
1308         dma_cap_zero(bdev->common.cap_mask);
1309         dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);
1310
1311         /* initialize dmaengine apis */
1312         bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1313         bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1314         bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1315         bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1316         bdev->common.device_alloc_chan_resources = bam_alloc_chan;
1317         bdev->common.device_free_chan_resources = bam_free_chan;
1318         bdev->common.device_prep_slave_sg = bam_prep_slave_sg;
1319         bdev->common.device_config = bam_slave_config;
1320         bdev->common.device_pause = bam_pause;
1321         bdev->common.device_resume = bam_resume;
1322         bdev->common.device_terminate_all = bam_dma_terminate_all;
1323         bdev->common.device_issue_pending = bam_issue_pending;
1324         bdev->common.device_tx_status = bam_tx_status;
1325         bdev->common.dev = bdev->dev;
1326
1327         ret = dma_async_device_register(&bdev->common);
1328         if (ret) {
1329                 dev_err(bdev->dev, "failed to register dma async device\n");
1330                 goto err_bam_channel_exit;
1331         }
1332
1333         ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate,
1334                                         &bdev->common);
1335         if (ret)
1336                 goto err_unregister_dma;
1337
1338         if (bdev->controlled_remotely) {
1339                 pm_runtime_disable(&pdev->dev);
1340                 return 0;
1341         }
1342
1343         pm_runtime_irq_safe(&pdev->dev);
1344         pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY);
1345         pm_runtime_use_autosuspend(&pdev->dev);
1346         pm_runtime_mark_last_busy(&pdev->dev);
1347         pm_runtime_set_active(&pdev->dev);
1348         pm_runtime_enable(&pdev->dev);
1349
1350         return 0;
1351
1352 err_unregister_dma:
1353         dma_async_device_unregister(&bdev->common);
1354 err_bam_channel_exit:
1355         for (i = 0; i < bdev->num_channels; i++)
1356                 tasklet_kill(&bdev->channels[i].vc.task);
1357 err_tasklet_kill:
1358         tasklet_kill(&bdev->task);
1359 err_disable_clk:
1360         clk_disable_unprepare(bdev->bamclk);
1361
1362         return ret;
1363 }
1364
1365 static int bam_dma_remove(struct platform_device *pdev)
1366 {
1367         struct bam_device *bdev = platform_get_drvdata(pdev);
1368         u32 i;
1369
1370         pm_runtime_force_suspend(&pdev->dev);
1371
1372         of_dma_controller_free(pdev->dev.of_node);
1373         dma_async_device_unregister(&bdev->common);
1374
1375         /* mask all interrupts for this execution environment */
1376         writel_relaxed(0, bam_addr(bdev, 0,  BAM_IRQ_SRCS_MSK_EE));
1377
1378         devm_free_irq(bdev->dev, bdev->irq, bdev);
1379
1380         for (i = 0; i < bdev->num_channels; i++) {
1381                 bam_dma_terminate_all(&bdev->channels[i].vc.chan);
1382                 tasklet_kill(&bdev->channels[i].vc.task);
1383
1384                 if (!bdev->channels[i].fifo_virt)
1385                         continue;
1386
1387                 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
1388                             bdev->channels[i].fifo_virt,
1389                             bdev->channels[i].fifo_phys);
1390         }
1391
1392         tasklet_kill(&bdev->task);
1393
1394         clk_disable_unprepare(bdev->bamclk);
1395
1396         return 0;
1397 }
1398
1399 static int __maybe_unused bam_dma_runtime_suspend(struct device *dev)
1400 {
1401         struct bam_device *bdev = dev_get_drvdata(dev);
1402
1403         clk_disable(bdev->bamclk);
1404
1405         return 0;
1406 }
1407
1408 static int __maybe_unused bam_dma_runtime_resume(struct device *dev)
1409 {
1410         struct bam_device *bdev = dev_get_drvdata(dev);
1411         int ret;
1412
1413         ret = clk_enable(bdev->bamclk);
1414         if (ret < 0) {
1415                 dev_err(dev, "clk_enable failed: %d\n", ret);
1416                 return ret;
1417         }
1418
1419         return 0;
1420 }
1421
1422 static int __maybe_unused bam_dma_suspend(struct device *dev)
1423 {
1424         struct bam_device *bdev = dev_get_drvdata(dev);
1425
1426         if (!bdev->controlled_remotely)
1427                 pm_runtime_force_suspend(dev);
1428
1429         clk_unprepare(bdev->bamclk);
1430
1431         return 0;
1432 }
1433
1434 static int __maybe_unused bam_dma_resume(struct device *dev)
1435 {
1436         struct bam_device *bdev = dev_get_drvdata(dev);
1437         int ret;
1438
1439         ret = clk_prepare(bdev->bamclk);
1440         if (ret)
1441                 return ret;
1442
1443         if (!bdev->controlled_remotely)
1444                 pm_runtime_force_resume(dev);
1445
1446         return 0;
1447 }
1448
1449 static const struct dev_pm_ops bam_dma_pm_ops = {
1450         SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume)
1451         SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume,
1452                                 NULL)
1453 };
1454
1455 static struct platform_driver bam_dma_driver = {
1456         .probe = bam_dma_probe,
1457         .remove = bam_dma_remove,
1458         .driver = {
1459                 .name = "bam-dma-engine",
1460                 .pm = &bam_dma_pm_ops,
1461                 .of_match_table = bam_of_match,
1462         },
1463 };
1464
1465 module_platform_driver(bam_dma_driver);
1466
1467 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1468 MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1469 MODULE_LICENSE("GPL v2");