Merge tag 'dmaengine-4.18-rc1' of git://git.infradead.org/users/vkoul/slave-dma
[muen/linux.git] / drivers / dma / qcom / bam_dma.c
1 /*
2  * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved.
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 and
6  * only version 2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  */
14 /*
15  * QCOM BAM DMA engine driver
16  *
17  * QCOM BAM DMA blocks are distributed amongst a number of the on-chip
18  * peripherals on the MSM 8x74.  The configuration of the channels are dependent
19  * on the way they are hard wired to that specific peripheral.  The peripheral
20  * device tree entries specify the configuration of each channel.
21  *
22  * The DMA controller requires the use of external memory for storage of the
23  * hardware descriptors for each channel.  The descriptor FIFO is accessed as a
24  * circular buffer and operations are managed according to the offset within the
25  * FIFO.  After pipe/channel reset, all of the pipe registers and internal state
26  * are back to defaults.
27  *
28  * During DMA operations, we write descriptors to the FIFO, being careful to
29  * handle wrapping and then write the last FIFO offset to that channel's
30  * P_EVNT_REG register to kick off the transaction.  The P_SW_OFSTS register
31  * indicates the current FIFO offset that is being processed, so there is some
32  * indication of where the hardware is currently working.
33  */
34
35 #include <linux/kernel.h>
36 #include <linux/io.h>
37 #include <linux/init.h>
38 #include <linux/slab.h>
39 #include <linux/module.h>
40 #include <linux/interrupt.h>
41 #include <linux/dma-mapping.h>
42 #include <linux/scatterlist.h>
43 #include <linux/device.h>
44 #include <linux/platform_device.h>
45 #include <linux/of.h>
46 #include <linux/of_address.h>
47 #include <linux/of_irq.h>
48 #include <linux/of_dma.h>
49 #include <linux/circ_buf.h>
50 #include <linux/clk.h>
51 #include <linux/dmaengine.h>
52 #include <linux/pm_runtime.h>
53
54 #include "../dmaengine.h"
55 #include "../virt-dma.h"
56
57 struct bam_desc_hw {
58         __le32 addr;            /* Buffer physical address */
59         __le16 size;            /* Buffer size in bytes */
60         __le16 flags;
61 };
62
63 #define BAM_DMA_AUTOSUSPEND_DELAY 100
64
65 #define DESC_FLAG_INT BIT(15)
66 #define DESC_FLAG_EOT BIT(14)
67 #define DESC_FLAG_EOB BIT(13)
68 #define DESC_FLAG_NWD BIT(12)
69 #define DESC_FLAG_CMD BIT(11)
70
71 struct bam_async_desc {
72         struct virt_dma_desc vd;
73
74         u32 num_desc;
75         u32 xfer_len;
76
77         /* transaction flags, EOT|EOB|NWD */
78         u16 flags;
79
80         struct bam_desc_hw *curr_desc;
81
82         /* list node for the desc in the bam_chan list of descriptors */
83         struct list_head desc_node;
84         enum dma_transfer_direction dir;
85         size_t length;
86         struct bam_desc_hw desc[0];
87 };
88
89 enum bam_reg {
90         BAM_CTRL,
91         BAM_REVISION,
92         BAM_NUM_PIPES,
93         BAM_DESC_CNT_TRSHLD,
94         BAM_IRQ_SRCS,
95         BAM_IRQ_SRCS_MSK,
96         BAM_IRQ_SRCS_UNMASKED,
97         BAM_IRQ_STTS,
98         BAM_IRQ_CLR,
99         BAM_IRQ_EN,
100         BAM_CNFG_BITS,
101         BAM_IRQ_SRCS_EE,
102         BAM_IRQ_SRCS_MSK_EE,
103         BAM_P_CTRL,
104         BAM_P_RST,
105         BAM_P_HALT,
106         BAM_P_IRQ_STTS,
107         BAM_P_IRQ_CLR,
108         BAM_P_IRQ_EN,
109         BAM_P_EVNT_DEST_ADDR,
110         BAM_P_EVNT_REG,
111         BAM_P_SW_OFSTS,
112         BAM_P_DATA_FIFO_ADDR,
113         BAM_P_DESC_FIFO_ADDR,
114         BAM_P_EVNT_GEN_TRSHLD,
115         BAM_P_FIFO_SIZES,
116 };
117
118 struct reg_offset_data {
119         u32 base_offset;
120         unsigned int pipe_mult, evnt_mult, ee_mult;
121 };
122
123 static const struct reg_offset_data bam_v1_3_reg_info[] = {
124         [BAM_CTRL]              = { 0x0F80, 0x00, 0x00, 0x00 },
125         [BAM_REVISION]          = { 0x0F84, 0x00, 0x00, 0x00 },
126         [BAM_NUM_PIPES]         = { 0x0FBC, 0x00, 0x00, 0x00 },
127         [BAM_DESC_CNT_TRSHLD]   = { 0x0F88, 0x00, 0x00, 0x00 },
128         [BAM_IRQ_SRCS]          = { 0x0F8C, 0x00, 0x00, 0x00 },
129         [BAM_IRQ_SRCS_MSK]      = { 0x0F90, 0x00, 0x00, 0x00 },
130         [BAM_IRQ_SRCS_UNMASKED] = { 0x0FB0, 0x00, 0x00, 0x00 },
131         [BAM_IRQ_STTS]          = { 0x0F94, 0x00, 0x00, 0x00 },
132         [BAM_IRQ_CLR]           = { 0x0F98, 0x00, 0x00, 0x00 },
133         [BAM_IRQ_EN]            = { 0x0F9C, 0x00, 0x00, 0x00 },
134         [BAM_CNFG_BITS]         = { 0x0FFC, 0x00, 0x00, 0x00 },
135         [BAM_IRQ_SRCS_EE]       = { 0x1800, 0x00, 0x00, 0x80 },
136         [BAM_IRQ_SRCS_MSK_EE]   = { 0x1804, 0x00, 0x00, 0x80 },
137         [BAM_P_CTRL]            = { 0x0000, 0x80, 0x00, 0x00 },
138         [BAM_P_RST]             = { 0x0004, 0x80, 0x00, 0x00 },
139         [BAM_P_HALT]            = { 0x0008, 0x80, 0x00, 0x00 },
140         [BAM_P_IRQ_STTS]        = { 0x0010, 0x80, 0x00, 0x00 },
141         [BAM_P_IRQ_CLR]         = { 0x0014, 0x80, 0x00, 0x00 },
142         [BAM_P_IRQ_EN]          = { 0x0018, 0x80, 0x00, 0x00 },
143         [BAM_P_EVNT_DEST_ADDR]  = { 0x102C, 0x00, 0x40, 0x00 },
144         [BAM_P_EVNT_REG]        = { 0x1018, 0x00, 0x40, 0x00 },
145         [BAM_P_SW_OFSTS]        = { 0x1000, 0x00, 0x40, 0x00 },
146         [BAM_P_DATA_FIFO_ADDR]  = { 0x1024, 0x00, 0x40, 0x00 },
147         [BAM_P_DESC_FIFO_ADDR]  = { 0x101C, 0x00, 0x40, 0x00 },
148         [BAM_P_EVNT_GEN_TRSHLD] = { 0x1028, 0x00, 0x40, 0x00 },
149         [BAM_P_FIFO_SIZES]      = { 0x1020, 0x00, 0x40, 0x00 },
150 };
151
152 static const struct reg_offset_data bam_v1_4_reg_info[] = {
153         [BAM_CTRL]              = { 0x0000, 0x00, 0x00, 0x00 },
154         [BAM_REVISION]          = { 0x0004, 0x00, 0x00, 0x00 },
155         [BAM_NUM_PIPES]         = { 0x003C, 0x00, 0x00, 0x00 },
156         [BAM_DESC_CNT_TRSHLD]   = { 0x0008, 0x00, 0x00, 0x00 },
157         [BAM_IRQ_SRCS]          = { 0x000C, 0x00, 0x00, 0x00 },
158         [BAM_IRQ_SRCS_MSK]      = { 0x0010, 0x00, 0x00, 0x00 },
159         [BAM_IRQ_SRCS_UNMASKED] = { 0x0030, 0x00, 0x00, 0x00 },
160         [BAM_IRQ_STTS]          = { 0x0014, 0x00, 0x00, 0x00 },
161         [BAM_IRQ_CLR]           = { 0x0018, 0x00, 0x00, 0x00 },
162         [BAM_IRQ_EN]            = { 0x001C, 0x00, 0x00, 0x00 },
163         [BAM_CNFG_BITS]         = { 0x007C, 0x00, 0x00, 0x00 },
164         [BAM_IRQ_SRCS_EE]       = { 0x0800, 0x00, 0x00, 0x80 },
165         [BAM_IRQ_SRCS_MSK_EE]   = { 0x0804, 0x00, 0x00, 0x80 },
166         [BAM_P_CTRL]            = { 0x1000, 0x1000, 0x00, 0x00 },
167         [BAM_P_RST]             = { 0x1004, 0x1000, 0x00, 0x00 },
168         [BAM_P_HALT]            = { 0x1008, 0x1000, 0x00, 0x00 },
169         [BAM_P_IRQ_STTS]        = { 0x1010, 0x1000, 0x00, 0x00 },
170         [BAM_P_IRQ_CLR]         = { 0x1014, 0x1000, 0x00, 0x00 },
171         [BAM_P_IRQ_EN]          = { 0x1018, 0x1000, 0x00, 0x00 },
172         [BAM_P_EVNT_DEST_ADDR]  = { 0x182C, 0x00, 0x1000, 0x00 },
173         [BAM_P_EVNT_REG]        = { 0x1818, 0x00, 0x1000, 0x00 },
174         [BAM_P_SW_OFSTS]        = { 0x1800, 0x00, 0x1000, 0x00 },
175         [BAM_P_DATA_FIFO_ADDR]  = { 0x1824, 0x00, 0x1000, 0x00 },
176         [BAM_P_DESC_FIFO_ADDR]  = { 0x181C, 0x00, 0x1000, 0x00 },
177         [BAM_P_EVNT_GEN_TRSHLD] = { 0x1828, 0x00, 0x1000, 0x00 },
178         [BAM_P_FIFO_SIZES]      = { 0x1820, 0x00, 0x1000, 0x00 },
179 };
180
181 static const struct reg_offset_data bam_v1_7_reg_info[] = {
182         [BAM_CTRL]              = { 0x00000, 0x00, 0x00, 0x00 },
183         [BAM_REVISION]          = { 0x01000, 0x00, 0x00, 0x00 },
184         [BAM_NUM_PIPES]         = { 0x01008, 0x00, 0x00, 0x00 },
185         [BAM_DESC_CNT_TRSHLD]   = { 0x00008, 0x00, 0x00, 0x00 },
186         [BAM_IRQ_SRCS]          = { 0x03010, 0x00, 0x00, 0x00 },
187         [BAM_IRQ_SRCS_MSK]      = { 0x03014, 0x00, 0x00, 0x00 },
188         [BAM_IRQ_SRCS_UNMASKED] = { 0x03018, 0x00, 0x00, 0x00 },
189         [BAM_IRQ_STTS]          = { 0x00014, 0x00, 0x00, 0x00 },
190         [BAM_IRQ_CLR]           = { 0x00018, 0x00, 0x00, 0x00 },
191         [BAM_IRQ_EN]            = { 0x0001C, 0x00, 0x00, 0x00 },
192         [BAM_CNFG_BITS]         = { 0x0007C, 0x00, 0x00, 0x00 },
193         [BAM_IRQ_SRCS_EE]       = { 0x03000, 0x00, 0x00, 0x1000 },
194         [BAM_IRQ_SRCS_MSK_EE]   = { 0x03004, 0x00, 0x00, 0x1000 },
195         [BAM_P_CTRL]            = { 0x13000, 0x1000, 0x00, 0x00 },
196         [BAM_P_RST]             = { 0x13004, 0x1000, 0x00, 0x00 },
197         [BAM_P_HALT]            = { 0x13008, 0x1000, 0x00, 0x00 },
198         [BAM_P_IRQ_STTS]        = { 0x13010, 0x1000, 0x00, 0x00 },
199         [BAM_P_IRQ_CLR]         = { 0x13014, 0x1000, 0x00, 0x00 },
200         [BAM_P_IRQ_EN]          = { 0x13018, 0x1000, 0x00, 0x00 },
201         [BAM_P_EVNT_DEST_ADDR]  = { 0x1382C, 0x00, 0x1000, 0x00 },
202         [BAM_P_EVNT_REG]        = { 0x13818, 0x00, 0x1000, 0x00 },
203         [BAM_P_SW_OFSTS]        = { 0x13800, 0x00, 0x1000, 0x00 },
204         [BAM_P_DATA_FIFO_ADDR]  = { 0x13824, 0x00, 0x1000, 0x00 },
205         [BAM_P_DESC_FIFO_ADDR]  = { 0x1381C, 0x00, 0x1000, 0x00 },
206         [BAM_P_EVNT_GEN_TRSHLD] = { 0x13828, 0x00, 0x1000, 0x00 },
207         [BAM_P_FIFO_SIZES]      = { 0x13820, 0x00, 0x1000, 0x00 },
208 };
209
210 /* BAM CTRL */
211 #define BAM_SW_RST                      BIT(0)
212 #define BAM_EN                          BIT(1)
213 #define BAM_EN_ACCUM                    BIT(4)
214 #define BAM_TESTBUS_SEL_SHIFT           5
215 #define BAM_TESTBUS_SEL_MASK            0x3F
216 #define BAM_DESC_CACHE_SEL_SHIFT        13
217 #define BAM_DESC_CACHE_SEL_MASK         0x3
218 #define BAM_CACHED_DESC_STORE           BIT(15)
219 #define IBC_DISABLE                     BIT(16)
220
221 /* BAM REVISION */
222 #define REVISION_SHIFT          0
223 #define REVISION_MASK           0xFF
224 #define NUM_EES_SHIFT           8
225 #define NUM_EES_MASK            0xF
226 #define CE_BUFFER_SIZE          BIT(13)
227 #define AXI_ACTIVE              BIT(14)
228 #define USE_VMIDMT              BIT(15)
229 #define SECURED                 BIT(16)
230 #define BAM_HAS_NO_BYPASS       BIT(17)
231 #define HIGH_FREQUENCY_BAM      BIT(18)
232 #define INACTIV_TMRS_EXST       BIT(19)
233 #define NUM_INACTIV_TMRS        BIT(20)
234 #define DESC_CACHE_DEPTH_SHIFT  21
235 #define DESC_CACHE_DEPTH_1      (0 << DESC_CACHE_DEPTH_SHIFT)
236 #define DESC_CACHE_DEPTH_2      (1 << DESC_CACHE_DEPTH_SHIFT)
237 #define DESC_CACHE_DEPTH_3      (2 << DESC_CACHE_DEPTH_SHIFT)
238 #define DESC_CACHE_DEPTH_4      (3 << DESC_CACHE_DEPTH_SHIFT)
239 #define CMD_DESC_EN             BIT(23)
240 #define INACTIV_TMR_BASE_SHIFT  24
241 #define INACTIV_TMR_BASE_MASK   0xFF
242
243 /* BAM NUM PIPES */
244 #define BAM_NUM_PIPES_SHIFT             0
245 #define BAM_NUM_PIPES_MASK              0xFF
246 #define PERIPH_NON_PIPE_GRP_SHIFT       16
247 #define PERIPH_NON_PIP_GRP_MASK         0xFF
248 #define BAM_NON_PIPE_GRP_SHIFT          24
249 #define BAM_NON_PIPE_GRP_MASK           0xFF
250
251 /* BAM CNFG BITS */
252 #define BAM_PIPE_CNFG           BIT(2)
253 #define BAM_FULL_PIPE           BIT(11)
254 #define BAM_NO_EXT_P_RST        BIT(12)
255 #define BAM_IBC_DISABLE         BIT(13)
256 #define BAM_SB_CLK_REQ          BIT(14)
257 #define BAM_PSM_CSW_REQ         BIT(15)
258 #define BAM_PSM_P_RES           BIT(16)
259 #define BAM_AU_P_RES            BIT(17)
260 #define BAM_SI_P_RES            BIT(18)
261 #define BAM_WB_P_RES            BIT(19)
262 #define BAM_WB_BLK_CSW          BIT(20)
263 #define BAM_WB_CSW_ACK_IDL      BIT(21)
264 #define BAM_WB_RETR_SVPNT       BIT(22)
265 #define BAM_WB_DSC_AVL_P_RST    BIT(23)
266 #define BAM_REG_P_EN            BIT(24)
267 #define BAM_PSM_P_HD_DATA       BIT(25)
268 #define BAM_AU_ACCUMED          BIT(26)
269 #define BAM_CMD_ENABLE          BIT(27)
270
271 #define BAM_CNFG_BITS_DEFAULT   (BAM_PIPE_CNFG |        \
272                                  BAM_NO_EXT_P_RST |     \
273                                  BAM_IBC_DISABLE |      \
274                                  BAM_SB_CLK_REQ |       \
275                                  BAM_PSM_CSW_REQ |      \
276                                  BAM_PSM_P_RES |        \
277                                  BAM_AU_P_RES |         \
278                                  BAM_SI_P_RES |         \
279                                  BAM_WB_P_RES |         \
280                                  BAM_WB_BLK_CSW |       \
281                                  BAM_WB_CSW_ACK_IDL |   \
282                                  BAM_WB_RETR_SVPNT |    \
283                                  BAM_WB_DSC_AVL_P_RST | \
284                                  BAM_REG_P_EN |         \
285                                  BAM_PSM_P_HD_DATA |    \
286                                  BAM_AU_ACCUMED |       \
287                                  BAM_CMD_ENABLE)
288
289 /* PIPE CTRL */
290 #define P_EN                    BIT(1)
291 #define P_DIRECTION             BIT(3)
292 #define P_SYS_STRM              BIT(4)
293 #define P_SYS_MODE              BIT(5)
294 #define P_AUTO_EOB              BIT(6)
295 #define P_AUTO_EOB_SEL_SHIFT    7
296 #define P_AUTO_EOB_SEL_512      (0 << P_AUTO_EOB_SEL_SHIFT)
297 #define P_AUTO_EOB_SEL_256      (1 << P_AUTO_EOB_SEL_SHIFT)
298 #define P_AUTO_EOB_SEL_128      (2 << P_AUTO_EOB_SEL_SHIFT)
299 #define P_AUTO_EOB_SEL_64       (3 << P_AUTO_EOB_SEL_SHIFT)
300 #define P_PREFETCH_LIMIT_SHIFT  9
301 #define P_PREFETCH_LIMIT_32     (0 << P_PREFETCH_LIMIT_SHIFT)
302 #define P_PREFETCH_LIMIT_16     (1 << P_PREFETCH_LIMIT_SHIFT)
303 #define P_PREFETCH_LIMIT_4      (2 << P_PREFETCH_LIMIT_SHIFT)
304 #define P_WRITE_NWD             BIT(11)
305 #define P_LOCK_GROUP_SHIFT      16
306 #define P_LOCK_GROUP_MASK       0x1F
307
308 /* BAM_DESC_CNT_TRSHLD */
309 #define CNT_TRSHLD              0xffff
310 #define DEFAULT_CNT_THRSHLD     0x4
311
312 /* BAM_IRQ_SRCS */
313 #define BAM_IRQ                 BIT(31)
314 #define P_IRQ                   0x7fffffff
315
316 /* BAM_IRQ_SRCS_MSK */
317 #define BAM_IRQ_MSK             BAM_IRQ
318 #define P_IRQ_MSK               P_IRQ
319
320 /* BAM_IRQ_STTS */
321 #define BAM_TIMER_IRQ           BIT(4)
322 #define BAM_EMPTY_IRQ           BIT(3)
323 #define BAM_ERROR_IRQ           BIT(2)
324 #define BAM_HRESP_ERR_IRQ       BIT(1)
325
326 /* BAM_IRQ_CLR */
327 #define BAM_TIMER_CLR           BIT(4)
328 #define BAM_EMPTY_CLR           BIT(3)
329 #define BAM_ERROR_CLR           BIT(2)
330 #define BAM_HRESP_ERR_CLR       BIT(1)
331
332 /* BAM_IRQ_EN */
333 #define BAM_TIMER_EN            BIT(4)
334 #define BAM_EMPTY_EN            BIT(3)
335 #define BAM_ERROR_EN            BIT(2)
336 #define BAM_HRESP_ERR_EN        BIT(1)
337
338 /* BAM_P_IRQ_EN */
339 #define P_PRCSD_DESC_EN         BIT(0)
340 #define P_TIMER_EN              BIT(1)
341 #define P_WAKE_EN               BIT(2)
342 #define P_OUT_OF_DESC_EN        BIT(3)
343 #define P_ERR_EN                BIT(4)
344 #define P_TRNSFR_END_EN         BIT(5)
345 #define P_DEFAULT_IRQS_EN       (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN)
346
347 /* BAM_P_SW_OFSTS */
348 #define P_SW_OFSTS_MASK         0xffff
349
350 #define BAM_DESC_FIFO_SIZE      SZ_32K
351 #define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1)
352 #define BAM_FIFO_SIZE   (SZ_32K - 8)
353 #define IS_BUSY(chan)   (CIRC_SPACE(bchan->tail, bchan->head,\
354                          MAX_DESCRIPTORS + 1) == 0)
355
356 struct bam_chan {
357         struct virt_dma_chan vc;
358
359         struct bam_device *bdev;
360
361         /* configuration from device tree */
362         u32 id;
363
364         /* runtime configuration */
365         struct dma_slave_config slave;
366
367         /* fifo storage */
368         struct bam_desc_hw *fifo_virt;
369         dma_addr_t fifo_phys;
370
371         /* fifo markers */
372         unsigned short head;            /* start of active descriptor entries */
373         unsigned short tail;            /* end of active descriptor entries */
374
375         unsigned int initialized;       /* is the channel hw initialized? */
376         unsigned int paused;            /* is the channel paused? */
377         unsigned int reconfigure;       /* new slave config? */
378         /* list of descriptors currently processed */
379         struct list_head desc_list;
380
381         struct list_head node;
382 };
383
384 static inline struct bam_chan *to_bam_chan(struct dma_chan *common)
385 {
386         return container_of(common, struct bam_chan, vc.chan);
387 }
388
389 struct bam_device {
390         void __iomem *regs;
391         struct device *dev;
392         struct dma_device common;
393         struct device_dma_parameters dma_parms;
394         struct bam_chan *channels;
395         u32 num_channels;
396         u32 num_ees;
397
398         /* execution environment ID, from DT */
399         u32 ee;
400         bool controlled_remotely;
401
402         const struct reg_offset_data *layout;
403
404         struct clk *bamclk;
405         int irq;
406
407         /* dma start transaction tasklet */
408         struct tasklet_struct task;
409 };
410
411 /**
412  * bam_addr - returns BAM register address
413  * @bdev: bam device
414  * @pipe: pipe instance (ignored when register doesn't have multiple instances)
415  * @reg:  register enum
416  */
417 static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe,
418                 enum bam_reg reg)
419 {
420         const struct reg_offset_data r = bdev->layout[reg];
421
422         return bdev->regs + r.base_offset +
423                 r.pipe_mult * pipe +
424                 r.evnt_mult * pipe +
425                 r.ee_mult * bdev->ee;
426 }
427
428 /**
429  * bam_reset_channel - Reset individual BAM DMA channel
430  * @bchan: bam channel
431  *
432  * This function resets a specific BAM channel
433  */
434 static void bam_reset_channel(struct bam_chan *bchan)
435 {
436         struct bam_device *bdev = bchan->bdev;
437
438         lockdep_assert_held(&bchan->vc.lock);
439
440         /* reset channel */
441         writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));
442         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));
443
444         /* don't allow cpu to reorder BAM register accesses done after this */
445         wmb();
446
447         /* make sure hw is initialized when channel is used the first time  */
448         bchan->initialized = 0;
449 }
450
451 /**
452  * bam_chan_init_hw - Initialize channel hardware
453  * @bchan: bam channel
454  * @dir: DMA transfer direction
455  *
456  * This function resets and initializes the BAM channel
457  */
458 static void bam_chan_init_hw(struct bam_chan *bchan,
459         enum dma_transfer_direction dir)
460 {
461         struct bam_device *bdev = bchan->bdev;
462         u32 val;
463
464         /* Reset the channel to clear internal state of the FIFO */
465         bam_reset_channel(bchan);
466
467         /*
468          * write out 8 byte aligned address.  We have enough space for this
469          * because we allocated 1 more descriptor (8 bytes) than we can use
470          */
471         writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),
472                         bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));
473         writel_relaxed(BAM_FIFO_SIZE,
474                         bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));
475
476         /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */
477         writel_relaxed(P_DEFAULT_IRQS_EN,
478                         bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
479
480         /* unmask the specific pipe and EE combo */
481         val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
482         val |= BIT(bchan->id);
483         writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
484
485         /* don't allow cpu to reorder the channel enable done below */
486         wmb();
487
488         /* set fixed direction and mode, then enable channel */
489         val = P_EN | P_SYS_MODE;
490         if (dir == DMA_DEV_TO_MEM)
491                 val |= P_DIRECTION;
492
493         writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));
494
495         bchan->initialized = 1;
496
497         /* init FIFO pointers */
498         bchan->head = 0;
499         bchan->tail = 0;
500 }
501
502 /**
503  * bam_alloc_chan - Allocate channel resources for DMA channel.
504  * @chan: specified channel
505  *
506  * This function allocates the FIFO descriptor memory
507  */
508 static int bam_alloc_chan(struct dma_chan *chan)
509 {
510         struct bam_chan *bchan = to_bam_chan(chan);
511         struct bam_device *bdev = bchan->bdev;
512
513         if (bchan->fifo_virt)
514                 return 0;
515
516         /* allocate FIFO descriptor space, but only if necessary */
517         bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
518                                         &bchan->fifo_phys, GFP_KERNEL);
519
520         if (!bchan->fifo_virt) {
521                 dev_err(bdev->dev, "Failed to allocate desc fifo\n");
522                 return -ENOMEM;
523         }
524
525         return 0;
526 }
527
528 static int bam_pm_runtime_get_sync(struct device *dev)
529 {
530         if (pm_runtime_enabled(dev))
531                 return pm_runtime_get_sync(dev);
532
533         return 0;
534 }
535
536 /**
537  * bam_free_chan - Frees dma resources associated with specific channel
538  * @chan: specified channel
539  *
540  * Free the allocated fifo descriptor memory and channel resources
541  *
542  */
543 static void bam_free_chan(struct dma_chan *chan)
544 {
545         struct bam_chan *bchan = to_bam_chan(chan);
546         struct bam_device *bdev = bchan->bdev;
547         u32 val;
548         unsigned long flags;
549         int ret;
550
551         ret = bam_pm_runtime_get_sync(bdev->dev);
552         if (ret < 0)
553                 return;
554
555         vchan_free_chan_resources(to_virt_chan(chan));
556
557         if (!list_empty(&bchan->desc_list)) {
558                 dev_err(bchan->bdev->dev, "Cannot free busy channel\n");
559                 goto err;
560         }
561
562         spin_lock_irqsave(&bchan->vc.lock, flags);
563         bam_reset_channel(bchan);
564         spin_unlock_irqrestore(&bchan->vc.lock, flags);
565
566         dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,
567                     bchan->fifo_phys);
568         bchan->fifo_virt = NULL;
569
570         /* mask irq for pipe/channel */
571         val = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
572         val &= ~BIT(bchan->id);
573         writel_relaxed(val, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
574
575         /* disable irq */
576         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));
577
578 err:
579         pm_runtime_mark_last_busy(bdev->dev);
580         pm_runtime_put_autosuspend(bdev->dev);
581 }
582
583 /**
584  * bam_slave_config - set slave configuration for channel
585  * @chan: dma channel
586  * @cfg: slave configuration
587  *
588  * Sets slave configuration for channel
589  *
590  */
591 static int bam_slave_config(struct dma_chan *chan,
592                             struct dma_slave_config *cfg)
593 {
594         struct bam_chan *bchan = to_bam_chan(chan);
595         unsigned long flag;
596
597         spin_lock_irqsave(&bchan->vc.lock, flag);
598         memcpy(&bchan->slave, cfg, sizeof(*cfg));
599         bchan->reconfigure = 1;
600         spin_unlock_irqrestore(&bchan->vc.lock, flag);
601
602         return 0;
603 }
604
605 /**
606  * bam_prep_slave_sg - Prep slave sg transaction
607  *
608  * @chan: dma channel
609  * @sgl: scatter gather list
610  * @sg_len: length of sg
611  * @direction: DMA transfer direction
612  * @flags: DMA flags
613  * @context: transfer context (unused)
614  */
615 static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan,
616         struct scatterlist *sgl, unsigned int sg_len,
617         enum dma_transfer_direction direction, unsigned long flags,
618         void *context)
619 {
620         struct bam_chan *bchan = to_bam_chan(chan);
621         struct bam_device *bdev = bchan->bdev;
622         struct bam_async_desc *async_desc;
623         struct scatterlist *sg;
624         u32 i;
625         struct bam_desc_hw *desc;
626         unsigned int num_alloc = 0;
627
628
629         if (!is_slave_direction(direction)) {
630                 dev_err(bdev->dev, "invalid dma direction\n");
631                 return NULL;
632         }
633
634         /* calculate number of required entries */
635         for_each_sg(sgl, sg, sg_len, i)
636                 num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE);
637
638         /* allocate enough room to accomodate the number of entries */
639         async_desc = kzalloc(sizeof(*async_desc) +
640                         (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT);
641
642         if (!async_desc)
643                 goto err_out;
644
645         if (flags & DMA_PREP_FENCE)
646                 async_desc->flags |= DESC_FLAG_NWD;
647
648         if (flags & DMA_PREP_INTERRUPT)
649                 async_desc->flags |= DESC_FLAG_EOT;
650
651         async_desc->num_desc = num_alloc;
652         async_desc->curr_desc = async_desc->desc;
653         async_desc->dir = direction;
654
655         /* fill in temporary descriptors */
656         desc = async_desc->desc;
657         for_each_sg(sgl, sg, sg_len, i) {
658                 unsigned int remainder = sg_dma_len(sg);
659                 unsigned int curr_offset = 0;
660
661                 do {
662                         if (flags & DMA_PREP_CMD)
663                                 desc->flags |= cpu_to_le16(DESC_FLAG_CMD);
664
665                         desc->addr = cpu_to_le32(sg_dma_address(sg) +
666                                                  curr_offset);
667
668                         if (remainder > BAM_FIFO_SIZE) {
669                                 desc->size = cpu_to_le16(BAM_FIFO_SIZE);
670                                 remainder -= BAM_FIFO_SIZE;
671                                 curr_offset += BAM_FIFO_SIZE;
672                         } else {
673                                 desc->size = cpu_to_le16(remainder);
674                                 remainder = 0;
675                         }
676
677                         async_desc->length += le16_to_cpu(desc->size);
678                         desc++;
679                 } while (remainder > 0);
680         }
681
682         return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);
683
684 err_out:
685         kfree(async_desc);
686         return NULL;
687 }
688
689 /**
690  * bam_dma_terminate_all - terminate all transactions on a channel
691  * @chan: bam dma channel
692  *
693  * Dequeues and frees all transactions
694  * No callbacks are done
695  *
696  */
697 static int bam_dma_terminate_all(struct dma_chan *chan)
698 {
699         struct bam_chan *bchan = to_bam_chan(chan);
700         struct bam_async_desc *async_desc, *tmp;
701         unsigned long flag;
702         LIST_HEAD(head);
703
704         /* remove all transactions, including active transaction */
705         spin_lock_irqsave(&bchan->vc.lock, flag);
706         list_for_each_entry_safe(async_desc, tmp,
707                                  &bchan->desc_list, desc_node) {
708                 list_add(&async_desc->vd.node, &bchan->vc.desc_issued);
709                 list_del(&async_desc->desc_node);
710         }
711
712         vchan_get_all_descriptors(&bchan->vc, &head);
713         spin_unlock_irqrestore(&bchan->vc.lock, flag);
714
715         vchan_dma_desc_free_list(&bchan->vc, &head);
716
717         return 0;
718 }
719
720 /**
721  * bam_pause - Pause DMA channel
722  * @chan: dma channel
723  *
724  */
725 static int bam_pause(struct dma_chan *chan)
726 {
727         struct bam_chan *bchan = to_bam_chan(chan);
728         struct bam_device *bdev = bchan->bdev;
729         unsigned long flag;
730         int ret;
731
732         ret = bam_pm_runtime_get_sync(bdev->dev);
733         if (ret < 0)
734                 return ret;
735
736         spin_lock_irqsave(&bchan->vc.lock, flag);
737         writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));
738         bchan->paused = 1;
739         spin_unlock_irqrestore(&bchan->vc.lock, flag);
740         pm_runtime_mark_last_busy(bdev->dev);
741         pm_runtime_put_autosuspend(bdev->dev);
742
743         return 0;
744 }
745
746 /**
747  * bam_resume - Resume DMA channel operations
748  * @chan: dma channel
749  *
750  */
751 static int bam_resume(struct dma_chan *chan)
752 {
753         struct bam_chan *bchan = to_bam_chan(chan);
754         struct bam_device *bdev = bchan->bdev;
755         unsigned long flag;
756         int ret;
757
758         ret = bam_pm_runtime_get_sync(bdev->dev);
759         if (ret < 0)
760                 return ret;
761
762         spin_lock_irqsave(&bchan->vc.lock, flag);
763         writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));
764         bchan->paused = 0;
765         spin_unlock_irqrestore(&bchan->vc.lock, flag);
766         pm_runtime_mark_last_busy(bdev->dev);
767         pm_runtime_put_autosuspend(bdev->dev);
768
769         return 0;
770 }
771
772 /**
773  * process_channel_irqs - processes the channel interrupts
774  * @bdev: bam controller
775  *
776  * This function processes the channel interrupts
777  *
778  */
779 static u32 process_channel_irqs(struct bam_device *bdev)
780 {
781         u32 i, srcs, pipe_stts, offset, avail;
782         unsigned long flags;
783         struct bam_async_desc *async_desc, *tmp;
784
785         srcs = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_SRCS_EE));
786
787         /* return early if no pipe/channel interrupts are present */
788         if (!(srcs & P_IRQ))
789                 return srcs;
790
791         for (i = 0; i < bdev->num_channels; i++) {
792                 struct bam_chan *bchan = &bdev->channels[i];
793
794                 if (!(srcs & BIT(i)))
795                         continue;
796
797                 /* clear pipe irq */
798                 pipe_stts = readl_relaxed(bam_addr(bdev, i, BAM_P_IRQ_STTS));
799
800                 writel_relaxed(pipe_stts, bam_addr(bdev, i, BAM_P_IRQ_CLR));
801
802                 spin_lock_irqsave(&bchan->vc.lock, flags);
803
804                 offset = readl_relaxed(bam_addr(bdev, i, BAM_P_SW_OFSTS)) &
805                                        P_SW_OFSTS_MASK;
806                 offset /= sizeof(struct bam_desc_hw);
807
808                 /* Number of bytes available to read */
809                 avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);
810
811                 list_for_each_entry_safe(async_desc, tmp,
812                                          &bchan->desc_list, desc_node) {
813                         /* Not enough data to read */
814                         if (avail < async_desc->xfer_len)
815                                 break;
816
817                         /* manage FIFO */
818                         bchan->head += async_desc->xfer_len;
819                         bchan->head %= MAX_DESCRIPTORS;
820
821                         async_desc->num_desc -= async_desc->xfer_len;
822                         async_desc->curr_desc += async_desc->xfer_len;
823                         avail -= async_desc->xfer_len;
824
825                         /*
826                          * if complete, process cookie. Otherwise
827                          * push back to front of desc_issued so that
828                          * it gets restarted by the tasklet
829                          */
830                         if (!async_desc->num_desc) {
831                                 vchan_cookie_complete(&async_desc->vd);
832                         } else {
833                                 list_add(&async_desc->vd.node,
834                                          &bchan->vc.desc_issued);
835                         }
836                         list_del(&async_desc->desc_node);
837                 }
838
839                 spin_unlock_irqrestore(&bchan->vc.lock, flags);
840         }
841
842         return srcs;
843 }
844
845 /**
846  * bam_dma_irq - irq handler for bam controller
847  * @irq: IRQ of interrupt
848  * @data: callback data
849  *
850  * IRQ handler for the bam controller
851  */
852 static irqreturn_t bam_dma_irq(int irq, void *data)
853 {
854         struct bam_device *bdev = data;
855         u32 clr_mask = 0, srcs = 0;
856         int ret;
857
858         srcs |= process_channel_irqs(bdev);
859
860         /* kick off tasklet to start next dma transfer */
861         if (srcs & P_IRQ)
862                 tasklet_schedule(&bdev->task);
863
864         ret = bam_pm_runtime_get_sync(bdev->dev);
865         if (ret < 0)
866                 return ret;
867
868         if (srcs & BAM_IRQ) {
869                 clr_mask = readl_relaxed(bam_addr(bdev, 0, BAM_IRQ_STTS));
870
871                 /*
872                  * don't allow reorder of the various accesses to the BAM
873                  * registers
874                  */
875                 mb();
876
877                 writel_relaxed(clr_mask, bam_addr(bdev, 0, BAM_IRQ_CLR));
878         }
879
880         pm_runtime_mark_last_busy(bdev->dev);
881         pm_runtime_put_autosuspend(bdev->dev);
882
883         return IRQ_HANDLED;
884 }
885
886 /**
887  * bam_tx_status - returns status of transaction
888  * @chan: dma channel
889  * @cookie: transaction cookie
890  * @txstate: DMA transaction state
891  *
892  * Return status of dma transaction
893  */
894 static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
895                 struct dma_tx_state *txstate)
896 {
897         struct bam_chan *bchan = to_bam_chan(chan);
898         struct bam_async_desc *async_desc;
899         struct virt_dma_desc *vd;
900         int ret;
901         size_t residue = 0;
902         unsigned int i;
903         unsigned long flags;
904
905         ret = dma_cookie_status(chan, cookie, txstate);
906         if (ret == DMA_COMPLETE)
907                 return ret;
908
909         if (!txstate)
910                 return bchan->paused ? DMA_PAUSED : ret;
911
912         spin_lock_irqsave(&bchan->vc.lock, flags);
913         vd = vchan_find_desc(&bchan->vc, cookie);
914         if (vd) {
915                 residue = container_of(vd, struct bam_async_desc, vd)->length;
916         } else {
917                 list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {
918                         if (async_desc->vd.tx.cookie != cookie)
919                                 continue;
920
921                         for (i = 0; i < async_desc->num_desc; i++)
922                                 residue += le16_to_cpu(
923                                                 async_desc->curr_desc[i].size);
924                 }
925         }
926
927         spin_unlock_irqrestore(&bchan->vc.lock, flags);
928
929         dma_set_residue(txstate, residue);
930
931         if (ret == DMA_IN_PROGRESS && bchan->paused)
932                 ret = DMA_PAUSED;
933
934         return ret;
935 }
936
937 /**
938  * bam_apply_new_config
939  * @bchan: bam dma channel
940  * @dir: DMA direction
941  */
942 static void bam_apply_new_config(struct bam_chan *bchan,
943         enum dma_transfer_direction dir)
944 {
945         struct bam_device *bdev = bchan->bdev;
946         u32 maxburst;
947
948         if (!bdev->controlled_remotely) {
949                 if (dir == DMA_DEV_TO_MEM)
950                         maxburst = bchan->slave.src_maxburst;
951                 else
952                         maxburst = bchan->slave.dst_maxburst;
953
954                 writel_relaxed(maxburst,
955                                bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
956         }
957
958         bchan->reconfigure = 0;
959 }
960
961 /**
962  * bam_start_dma - start next transaction
963  * @bchan: bam dma channel
964  */
965 static void bam_start_dma(struct bam_chan *bchan)
966 {
967         struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);
968         struct bam_device *bdev = bchan->bdev;
969         struct bam_async_desc *async_desc = NULL;
970         struct bam_desc_hw *desc;
971         struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,
972                                         sizeof(struct bam_desc_hw));
973         int ret;
974         unsigned int avail;
975         struct dmaengine_desc_callback cb;
976
977         lockdep_assert_held(&bchan->vc.lock);
978
979         if (!vd)
980                 return;
981
982         ret = bam_pm_runtime_get_sync(bdev->dev);
983         if (ret < 0)
984                 return;
985
986         while (vd && !IS_BUSY(bchan)) {
987                 list_del(&vd->node);
988
989                 async_desc = container_of(vd, struct bam_async_desc, vd);
990
991                 /* on first use, initialize the channel hardware */
992                 if (!bchan->initialized)
993                         bam_chan_init_hw(bchan, async_desc->dir);
994
995                 /* apply new slave config changes, if necessary */
996                 if (bchan->reconfigure)
997                         bam_apply_new_config(bchan, async_desc->dir);
998
999                 desc = async_desc->curr_desc;
1000                 avail = CIRC_SPACE(bchan->tail, bchan->head,
1001                                    MAX_DESCRIPTORS + 1);
1002
1003                 if (async_desc->num_desc > avail)
1004                         async_desc->xfer_len = avail;
1005                 else
1006                         async_desc->xfer_len = async_desc->num_desc;
1007
1008                 /* set any special flags on the last descriptor */
1009                 if (async_desc->num_desc == async_desc->xfer_len)
1010                         desc[async_desc->xfer_len - 1].flags |=
1011                                                 cpu_to_le16(async_desc->flags);
1012
1013                 vd = vchan_next_desc(&bchan->vc);
1014
1015                 dmaengine_desc_get_callback(&async_desc->vd.tx, &cb);
1016
1017                 /*
1018                  * An interrupt is generated at this desc, if
1019                  *  - FIFO is FULL.
1020                  *  - No more descriptors to add.
1021                  *  - If a callback completion was requested for this DESC,
1022                  *     In this case, BAM will deliver the completion callback
1023                  *     for this desc and continue processing the next desc.
1024                  */
1025                 if (((avail <= async_desc->xfer_len) || !vd ||
1026                      dmaengine_desc_callback_valid(&cb)) &&
1027                     !(async_desc->flags & DESC_FLAG_EOT))
1028                         desc[async_desc->xfer_len - 1].flags |=
1029                                 cpu_to_le16(DESC_FLAG_INT);
1030
1031                 if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {
1032                         u32 partial = MAX_DESCRIPTORS - bchan->tail;
1033
1034                         memcpy(&fifo[bchan->tail], desc,
1035                                partial * sizeof(struct bam_desc_hw));
1036                         memcpy(fifo, &desc[partial],
1037                                (async_desc->xfer_len - partial) *
1038                                 sizeof(struct bam_desc_hw));
1039                 } else {
1040                         memcpy(&fifo[bchan->tail], desc,
1041                                async_desc->xfer_len *
1042                                sizeof(struct bam_desc_hw));
1043                 }
1044
1045                 bchan->tail += async_desc->xfer_len;
1046                 bchan->tail %= MAX_DESCRIPTORS;
1047                 list_add_tail(&async_desc->desc_node, &bchan->desc_list);
1048         }
1049
1050         /* ensure descriptor writes and dma start not reordered */
1051         wmb();
1052         writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),
1053                         bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));
1054
1055         pm_runtime_mark_last_busy(bdev->dev);
1056         pm_runtime_put_autosuspend(bdev->dev);
1057 }
1058
1059 /**
1060  * dma_tasklet - DMA IRQ tasklet
1061  * @data: tasklet argument (bam controller structure)
1062  *
1063  * Sets up next DMA operation and then processes all completed transactions
1064  */
1065 static void dma_tasklet(unsigned long data)
1066 {
1067         struct bam_device *bdev = (struct bam_device *)data;
1068         struct bam_chan *bchan;
1069         unsigned long flags;
1070         unsigned int i;
1071
1072         /* go through the channels and kick off transactions */
1073         for (i = 0; i < bdev->num_channels; i++) {
1074                 bchan = &bdev->channels[i];
1075                 spin_lock_irqsave(&bchan->vc.lock, flags);
1076
1077                 if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))
1078                         bam_start_dma(bchan);
1079                 spin_unlock_irqrestore(&bchan->vc.lock, flags);
1080         }
1081
1082 }
1083
1084 /**
1085  * bam_issue_pending - starts pending transactions
1086  * @chan: dma channel
1087  *
1088  * Calls tasklet directly which in turn starts any pending transactions
1089  */
1090 static void bam_issue_pending(struct dma_chan *chan)
1091 {
1092         struct bam_chan *bchan = to_bam_chan(chan);
1093         unsigned long flags;
1094
1095         spin_lock_irqsave(&bchan->vc.lock, flags);
1096
1097         /* if work pending and idle, start a transaction */
1098         if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))
1099                 bam_start_dma(bchan);
1100
1101         spin_unlock_irqrestore(&bchan->vc.lock, flags);
1102 }
1103
1104 /**
1105  * bam_dma_free_desc - free descriptor memory
1106  * @vd: virtual descriptor
1107  *
1108  */
1109 static void bam_dma_free_desc(struct virt_dma_desc *vd)
1110 {
1111         struct bam_async_desc *async_desc = container_of(vd,
1112                         struct bam_async_desc, vd);
1113
1114         kfree(async_desc);
1115 }
1116
1117 static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec,
1118                 struct of_dma *of)
1119 {
1120         struct bam_device *bdev = container_of(of->of_dma_data,
1121                                         struct bam_device, common);
1122         unsigned int request;
1123
1124         if (dma_spec->args_count != 1)
1125                 return NULL;
1126
1127         request = dma_spec->args[0];
1128         if (request >= bdev->num_channels)
1129                 return NULL;
1130
1131         return dma_get_slave_channel(&(bdev->channels[request].vc.chan));
1132 }
1133
1134 /**
1135  * bam_init
1136  * @bdev: bam device
1137  *
1138  * Initialization helper for global bam registers
1139  */
1140 static int bam_init(struct bam_device *bdev)
1141 {
1142         u32 val;
1143
1144         /* read revision and configuration information */
1145         if (!bdev->num_ees) {
1146                 val = readl_relaxed(bam_addr(bdev, 0, BAM_REVISION));
1147                 bdev->num_ees = (val >> NUM_EES_SHIFT) & NUM_EES_MASK;
1148         }
1149
1150         /* check that configured EE is within range */
1151         if (bdev->ee >= bdev->num_ees)
1152                 return -EINVAL;
1153
1154         if (!bdev->num_channels) {
1155                 val = readl_relaxed(bam_addr(bdev, 0, BAM_NUM_PIPES));
1156                 bdev->num_channels = val & BAM_NUM_PIPES_MASK;
1157         }
1158
1159         if (bdev->controlled_remotely)
1160                 return 0;
1161
1162         /* s/w reset bam */
1163         /* after reset all pipes are disabled and idle */
1164         val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL));
1165         val |= BAM_SW_RST;
1166         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1167         val &= ~BAM_SW_RST;
1168         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1169
1170         /* make sure previous stores are visible before enabling BAM */
1171         wmb();
1172
1173         /* enable bam */
1174         val |= BAM_EN;
1175         writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL));
1176
1177         /* set descriptor threshhold, start with 4 bytes */
1178         writel_relaxed(DEFAULT_CNT_THRSHLD,
1179                         bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD));
1180
1181         /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */
1182         writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS));
1183
1184         /* enable irqs for errors */
1185         writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN,
1186                         bam_addr(bdev, 0, BAM_IRQ_EN));
1187
1188         /* unmask global bam interrupt */
1189         writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE));
1190
1191         return 0;
1192 }
1193
1194 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,
1195         u32 index)
1196 {
1197         bchan->id = index;
1198         bchan->bdev = bdev;
1199
1200         vchan_init(&bchan->vc, &bdev->common);
1201         bchan->vc.desc_free = bam_dma_free_desc;
1202         INIT_LIST_HEAD(&bchan->desc_list);
1203 }
1204
1205 static const struct of_device_id bam_of_match[] = {
1206         { .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
1207         { .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
1208         { .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
1209         {}
1210 };
1211
1212 MODULE_DEVICE_TABLE(of, bam_of_match);
1213
1214 static int bam_dma_probe(struct platform_device *pdev)
1215 {
1216         struct bam_device *bdev;
1217         const struct of_device_id *match;
1218         struct resource *iores;
1219         int ret, i;
1220
1221         bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL);
1222         if (!bdev)
1223                 return -ENOMEM;
1224
1225         bdev->dev = &pdev->dev;
1226
1227         match = of_match_node(bam_of_match, pdev->dev.of_node);
1228         if (!match) {
1229                 dev_err(&pdev->dev, "Unsupported BAM module\n");
1230                 return -ENODEV;
1231         }
1232
1233         bdev->layout = match->data;
1234
1235         iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1236         bdev->regs = devm_ioremap_resource(&pdev->dev, iores);
1237         if (IS_ERR(bdev->regs))
1238                 return PTR_ERR(bdev->regs);
1239
1240         bdev->irq = platform_get_irq(pdev, 0);
1241         if (bdev->irq < 0)
1242                 return bdev->irq;
1243
1244         ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee);
1245         if (ret) {
1246                 dev_err(bdev->dev, "Execution environment unspecified\n");
1247                 return ret;
1248         }
1249
1250         bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node,
1251                                                 "qcom,controlled-remotely");
1252
1253         if (bdev->controlled_remotely) {
1254                 ret = of_property_read_u32(pdev->dev.of_node, "num-channels",
1255                                            &bdev->num_channels);
1256                 if (ret)
1257                         dev_err(bdev->dev, "num-channels unspecified in dt\n");
1258
1259                 ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees",
1260                                            &bdev->num_ees);
1261                 if (ret)
1262                         dev_err(bdev->dev, "num-ees unspecified in dt\n");
1263         }
1264
1265         bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk");
1266         if (IS_ERR(bdev->bamclk)) {
1267                 if (!bdev->controlled_remotely)
1268                         return PTR_ERR(bdev->bamclk);
1269
1270                 bdev->bamclk = NULL;
1271         }
1272
1273         ret = clk_prepare_enable(bdev->bamclk);
1274         if (ret) {
1275                 dev_err(bdev->dev, "failed to prepare/enable clock\n");
1276                 return ret;
1277         }
1278
1279         ret = bam_init(bdev);
1280         if (ret)
1281                 goto err_disable_clk;
1282
1283         tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev);
1284
1285         bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels,
1286                                 sizeof(*bdev->channels), GFP_KERNEL);
1287
1288         if (!bdev->channels) {
1289                 ret = -ENOMEM;
1290                 goto err_tasklet_kill;
1291         }
1292
1293         /* allocate and initialize channels */
1294         INIT_LIST_HEAD(&bdev->common.channels);
1295
1296         for (i = 0; i < bdev->num_channels; i++)
1297                 bam_channel_init(bdev, &bdev->channels[i], i);
1298
1299         ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq,
1300                         IRQF_TRIGGER_HIGH, "bam_dma", bdev);
1301         if (ret)
1302                 goto err_bam_channel_exit;
1303
1304         /* set max dma segment size */
1305         bdev->common.dev = bdev->dev;
1306         bdev->common.dev->dma_parms = &bdev->dma_parms;
1307         ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE);
1308         if (ret) {
1309                 dev_err(bdev->dev, "cannot set maximum segment size\n");
1310                 goto err_bam_channel_exit;
1311         }
1312
1313         platform_set_drvdata(pdev, bdev);
1314
1315         /* set capabilities */
1316         dma_cap_zero(bdev->common.cap_mask);
1317         dma_cap_set(DMA_SLAVE, bdev->common.cap_mask);
1318
1319         /* initialize dmaengine apis */
1320         bdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1321         bdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
1322         bdev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1323         bdev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES;
1324         bdev->common.device_alloc_chan_resources = bam_alloc_chan;
1325         bdev->common.device_free_chan_resources = bam_free_chan;
1326         bdev->common.device_prep_slave_sg = bam_prep_slave_sg;
1327         bdev->common.device_config = bam_slave_config;
1328         bdev->common.device_pause = bam_pause;
1329         bdev->common.device_resume = bam_resume;
1330         bdev->common.device_terminate_all = bam_dma_terminate_all;
1331         bdev->common.device_issue_pending = bam_issue_pending;
1332         bdev->common.device_tx_status = bam_tx_status;
1333         bdev->common.dev = bdev->dev;
1334
1335         ret = dma_async_device_register(&bdev->common);
1336         if (ret) {
1337                 dev_err(bdev->dev, "failed to register dma async device\n");
1338                 goto err_bam_channel_exit;
1339         }
1340
1341         ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate,
1342                                         &bdev->common);
1343         if (ret)
1344                 goto err_unregister_dma;
1345
1346         if (bdev->controlled_remotely) {
1347                 pm_runtime_disable(&pdev->dev);
1348                 return 0;
1349         }
1350
1351         pm_runtime_irq_safe(&pdev->dev);
1352         pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY);
1353         pm_runtime_use_autosuspend(&pdev->dev);
1354         pm_runtime_mark_last_busy(&pdev->dev);
1355         pm_runtime_set_active(&pdev->dev);
1356         pm_runtime_enable(&pdev->dev);
1357
1358         return 0;
1359
1360 err_unregister_dma:
1361         dma_async_device_unregister(&bdev->common);
1362 err_bam_channel_exit:
1363         for (i = 0; i < bdev->num_channels; i++)
1364                 tasklet_kill(&bdev->channels[i].vc.task);
1365 err_tasklet_kill:
1366         tasklet_kill(&bdev->task);
1367 err_disable_clk:
1368         clk_disable_unprepare(bdev->bamclk);
1369
1370         return ret;
1371 }
1372
1373 static int bam_dma_remove(struct platform_device *pdev)
1374 {
1375         struct bam_device *bdev = platform_get_drvdata(pdev);
1376         u32 i;
1377
1378         pm_runtime_force_suspend(&pdev->dev);
1379
1380         of_dma_controller_free(pdev->dev.of_node);
1381         dma_async_device_unregister(&bdev->common);
1382
1383         /* mask all interrupts for this execution environment */
1384         writel_relaxed(0, bam_addr(bdev, 0,  BAM_IRQ_SRCS_MSK_EE));
1385
1386         devm_free_irq(bdev->dev, bdev->irq, bdev);
1387
1388         for (i = 0; i < bdev->num_channels; i++) {
1389                 bam_dma_terminate_all(&bdev->channels[i].vc.chan);
1390                 tasklet_kill(&bdev->channels[i].vc.task);
1391
1392                 if (!bdev->channels[i].fifo_virt)
1393                         continue;
1394
1395                 dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE,
1396                             bdev->channels[i].fifo_virt,
1397                             bdev->channels[i].fifo_phys);
1398         }
1399
1400         tasklet_kill(&bdev->task);
1401
1402         clk_disable_unprepare(bdev->bamclk);
1403
1404         return 0;
1405 }
1406
1407 static int __maybe_unused bam_dma_runtime_suspend(struct device *dev)
1408 {
1409         struct bam_device *bdev = dev_get_drvdata(dev);
1410
1411         clk_disable(bdev->bamclk);
1412
1413         return 0;
1414 }
1415
1416 static int __maybe_unused bam_dma_runtime_resume(struct device *dev)
1417 {
1418         struct bam_device *bdev = dev_get_drvdata(dev);
1419         int ret;
1420
1421         ret = clk_enable(bdev->bamclk);
1422         if (ret < 0) {
1423                 dev_err(dev, "clk_enable failed: %d\n", ret);
1424                 return ret;
1425         }
1426
1427         return 0;
1428 }
1429
1430 static int __maybe_unused bam_dma_suspend(struct device *dev)
1431 {
1432         struct bam_device *bdev = dev_get_drvdata(dev);
1433
1434         if (!bdev->controlled_remotely)
1435                 pm_runtime_force_suspend(dev);
1436
1437         clk_unprepare(bdev->bamclk);
1438
1439         return 0;
1440 }
1441
1442 static int __maybe_unused bam_dma_resume(struct device *dev)
1443 {
1444         struct bam_device *bdev = dev_get_drvdata(dev);
1445         int ret;
1446
1447         ret = clk_prepare(bdev->bamclk);
1448         if (ret)
1449                 return ret;
1450
1451         if (!bdev->controlled_remotely)
1452                 pm_runtime_force_resume(dev);
1453
1454         return 0;
1455 }
1456
1457 static const struct dev_pm_ops bam_dma_pm_ops = {
1458         SET_LATE_SYSTEM_SLEEP_PM_OPS(bam_dma_suspend, bam_dma_resume)
1459         SET_RUNTIME_PM_OPS(bam_dma_runtime_suspend, bam_dma_runtime_resume,
1460                                 NULL)
1461 };
1462
1463 static struct platform_driver bam_dma_driver = {
1464         .probe = bam_dma_probe,
1465         .remove = bam_dma_remove,
1466         .driver = {
1467                 .name = "bam-dma-engine",
1468                 .pm = &bam_dma_pm_ops,
1469                 .of_match_table = bam_of_match,
1470         },
1471 };
1472
1473 module_platform_driver(bam_dma_driver);
1474
1475 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
1476 MODULE_DESCRIPTION("QCOM BAM DMA engine driver");
1477 MODULE_LICENSE("GPL v2");