drm/amdgpu:Add DPG mode support for vcn 1.0
[muen/linux.git] / drivers / gpu / drm / amd / amdgpu / vcn_v1_0.c
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23
24 #include <linux/firmware.h>
25 #include <drm/drmP.h>
26 #include "amdgpu.h"
27 #include "amdgpu_vcn.h"
28 #include "soc15.h"
29 #include "soc15d.h"
30 #include "soc15_common.h"
31
32 #include "vcn/vcn_1_0_offset.h"
33 #include "vcn/vcn_1_0_sh_mask.h"
34 #include "hdp/hdp_4_0_offset.h"
35 #include "mmhub/mmhub_9_1_offset.h"
36 #include "mmhub/mmhub_9_1_sh_mask.h"
37
38 #include "ivsrcid/vcn/irqsrcs_vcn_1_0.h"
39
40 static int vcn_v1_0_stop(struct amdgpu_device *adev);
41 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev);
42 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev);
43 static void vcn_v1_0_set_jpeg_ring_funcs(struct amdgpu_device *adev);
44 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev);
45 static void vcn_v1_0_jpeg_ring_set_patch_ring(struct amdgpu_ring *ring, uint32_t ptr);
46
47 /**
48  * vcn_v1_0_early_init - set function pointers
49  *
50  * @handle: amdgpu_device pointer
51  *
52  * Set ring and irq function pointers
53  */
54 static int vcn_v1_0_early_init(void *handle)
55 {
56         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
57
58         adev->vcn.num_enc_rings = 2;
59
60         vcn_v1_0_set_dec_ring_funcs(adev);
61         vcn_v1_0_set_enc_ring_funcs(adev);
62         vcn_v1_0_set_jpeg_ring_funcs(adev);
63         vcn_v1_0_set_irq_funcs(adev);
64
65         return 0;
66 }
67
68 /**
69  * vcn_v1_0_sw_init - sw init for VCN block
70  *
71  * @handle: amdgpu_device pointer
72  *
73  * Load firmware and sw initialization
74  */
75 static int vcn_v1_0_sw_init(void *handle)
76 {
77         struct amdgpu_ring *ring;
78         int i, r;
79         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
80
81         /* VCN DEC TRAP */
82         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, VCN_1_0__SRCID__UVD_SYSTEM_MESSAGE_INTERRUPT, &adev->vcn.irq);
83         if (r)
84                 return r;
85
86         /* VCN ENC TRAP */
87         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
88                 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, i + VCN_1_0__SRCID__UVD_ENC_GENERAL_PURPOSE,
89                                         &adev->vcn.irq);
90                 if (r)
91                         return r;
92         }
93
94         /* VCN JPEG TRAP */
95         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 126, &adev->vcn.irq);
96         if (r)
97                 return r;
98
99         r = amdgpu_vcn_sw_init(adev);
100         if (r)
101                 return r;
102
103         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
104                 const struct common_firmware_header *hdr;
105                 hdr = (const struct common_firmware_header *)adev->vcn.fw->data;
106                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN;
107                 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw;
108                 adev->firmware.fw_size +=
109                         ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
110                 DRM_INFO("PSP loading VCN firmware\n");
111         }
112
113         r = amdgpu_vcn_resume(adev);
114         if (r)
115                 return r;
116
117         ring = &adev->vcn.ring_dec;
118         sprintf(ring->name, "vcn_dec");
119         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
120         if (r)
121                 return r;
122
123         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
124                 ring = &adev->vcn.ring_enc[i];
125                 sprintf(ring->name, "vcn_enc%d", i);
126                 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
127                 if (r)
128                         return r;
129         }
130
131         ring = &adev->vcn.ring_jpeg;
132         sprintf(ring->name, "vcn_jpeg");
133         r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.irq, 0);
134         if (r)
135                 return r;
136
137         return r;
138 }
139
140 /**
141  * vcn_v1_0_sw_fini - sw fini for VCN block
142  *
143  * @handle: amdgpu_device pointer
144  *
145  * VCN suspend and free up sw allocation
146  */
147 static int vcn_v1_0_sw_fini(void *handle)
148 {
149         int r;
150         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
151
152         r = amdgpu_vcn_suspend(adev);
153         if (r)
154                 return r;
155
156         r = amdgpu_vcn_sw_fini(adev);
157
158         return r;
159 }
160
161 /**
162  * vcn_v1_0_hw_init - start and test VCN block
163  *
164  * @handle: amdgpu_device pointer
165  *
166  * Initialize the hardware, boot up the VCPU and do some testing
167  */
168 static int vcn_v1_0_hw_init(void *handle)
169 {
170         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
171         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
172         int i, r;
173
174         ring->ready = true;
175         r = amdgpu_ring_test_ring(ring);
176         if (r) {
177                 ring->ready = false;
178                 goto done;
179         }
180
181         for (i = 0; i < adev->vcn.num_enc_rings; ++i) {
182                 ring = &adev->vcn.ring_enc[i];
183                 ring->ready = true;
184                 r = amdgpu_ring_test_ring(ring);
185                 if (r) {
186                         ring->ready = false;
187                         goto done;
188                 }
189         }
190
191         ring = &adev->vcn.ring_jpeg;
192         ring->ready = true;
193         r = amdgpu_ring_test_ring(ring);
194         if (r) {
195                 ring->ready = false;
196                 goto done;
197         }
198
199 done:
200         if (!r)
201                 DRM_INFO("VCN decode and encode initialized successfully(under %s).\n",
202                         (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)?"DPG Mode":"SPG Mode");
203
204         return r;
205 }
206
207 /**
208  * vcn_v1_0_hw_fini - stop the hardware block
209  *
210  * @handle: amdgpu_device pointer
211  *
212  * Stop the VCN block, mark ring as not ready any more
213  */
214 static int vcn_v1_0_hw_fini(void *handle)
215 {
216         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
217         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
218
219         if (RREG32_SOC15(VCN, 0, mmUVD_STATUS))
220                 vcn_v1_0_stop(adev);
221
222         ring->ready = false;
223
224         return 0;
225 }
226
227 /**
228  * vcn_v1_0_suspend - suspend VCN block
229  *
230  * @handle: amdgpu_device pointer
231  *
232  * HW fini and suspend VCN block
233  */
234 static int vcn_v1_0_suspend(void *handle)
235 {
236         int r;
237         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
238
239         r = vcn_v1_0_hw_fini(adev);
240         if (r)
241                 return r;
242
243         r = amdgpu_vcn_suspend(adev);
244
245         return r;
246 }
247
248 /**
249  * vcn_v1_0_resume - resume VCN block
250  *
251  * @handle: amdgpu_device pointer
252  *
253  * Resume firmware and hw init VCN block
254  */
255 static int vcn_v1_0_resume(void *handle)
256 {
257         int r;
258         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
259
260         r = amdgpu_vcn_resume(adev);
261         if (r)
262                 return r;
263
264         r = vcn_v1_0_hw_init(adev);
265
266         return r;
267 }
268
269 /**
270  * vcn_v1_0_mc_resume_spg_mode - memory controller programming
271  *
272  * @adev: amdgpu_device pointer
273  *
274  * Let the VCN memory controller know it's offsets
275  */
276 static void vcn_v1_0_mc_resume_spg_mode(struct amdgpu_device *adev)
277 {
278         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
279         uint32_t offset;
280
281         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
282                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
283                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo));
284                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
285                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi));
286                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0);
287                 offset = 0;
288         } else {
289                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
290                         lower_32_bits(adev->vcn.gpu_addr));
291                 WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
292                         upper_32_bits(adev->vcn.gpu_addr));
293                 offset = size;
294                 WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
295                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
296         }
297
298         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size);
299
300         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
301                      lower_32_bits(adev->vcn.gpu_addr + offset));
302         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
303                      upper_32_bits(adev->vcn.gpu_addr + offset));
304         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0);
305         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE);
306
307         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
308                      lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
309         WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
310                      upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE));
311         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0);
312         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
313                         AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40));
314
315         WREG32_SOC15(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
316                         adev->gfx.config.gb_addr_config);
317         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
318                         adev->gfx.config.gb_addr_config);
319         WREG32_SOC15(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
320                         adev->gfx.config.gb_addr_config);
321 }
322
323 static void vcn_v1_0_mc_resume_dpg_mode(struct amdgpu_device *adev)
324 {
325         uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
326         uint32_t offset;
327
328         if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
329                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
330                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo),
331                              0xFFFFFFFF, 0);
332                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
333                              (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi),
334                              0xFFFFFFFF, 0);
335                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0,
336                              0xFFFFFFFF, 0);
337                 offset = 0;
338         } else {
339                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW,
340                         lower_32_bits(adev->vcn.gpu_addr), 0xFFFFFFFF, 0);
341                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH,
342                         upper_32_bits(adev->vcn.gpu_addr), 0xFFFFFFFF, 0);
343                 offset = size;
344                 WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0,
345                              AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0xFFFFFFFF, 0);
346         }
347
348         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size, 0xFFFFFFFF, 0);
349
350         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW,
351                      lower_32_bits(adev->vcn.gpu_addr + offset), 0xFFFFFFFF, 0);
352         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH,
353                      upper_32_bits(adev->vcn.gpu_addr + offset), 0xFFFFFFFF, 0);
354         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0,
355                              0xFFFFFFFF, 0);
356         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE,
357                              0xFFFFFFFF, 0);
358
359         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW,
360                      lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE),
361                              0xFFFFFFFF, 0);
362         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH,
363                      upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE),
364                              0xFFFFFFFF, 0);
365         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0, 0xFFFFFFFF, 0);
366         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CACHE_SIZE2,
367                         AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40),
368                              0xFFFFFFFF, 0);
369
370         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_ADDR_CONFIG,
371                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
372         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_DB_ADDR_CONFIG,
373                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
374         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_UDEC_DBW_ADDR_CONFIG,
375                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
376         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_ADDR_CONFIG,
377                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
378         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG,
379                         adev->gfx.config.gb_addr_config, 0xFFFFFFFF, 0);
380 }
381
382 /**
383  * vcn_v1_0_disable_clock_gating - disable VCN clock gating
384  *
385  * @adev: amdgpu_device pointer
386  * @sw: enable SW clock gating
387  *
388  * Disable clock gating for VCN block
389  */
390 static void vcn_v1_0_disable_clock_gating(struct amdgpu_device *adev)
391 {
392         uint32_t data;
393
394         /* JPEG disable CGC */
395         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
396
397         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
398                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
399         else
400                 data &= ~JPEG_CGC_CTRL__DYN_CLOCK_MODE_MASK;
401
402         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
403         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
404         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
405
406         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
407         data &= ~(JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
408         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
409
410         /* UVD disable CGC */
411         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
412         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
413                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
414         else
415                 data &= ~ UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK;
416
417         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
418         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
419         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
420
421         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_GATE);
422         data &= ~(UVD_CGC_GATE__SYS_MASK
423                 | UVD_CGC_GATE__UDEC_MASK
424                 | UVD_CGC_GATE__MPEG2_MASK
425                 | UVD_CGC_GATE__REGS_MASK
426                 | UVD_CGC_GATE__RBC_MASK
427                 | UVD_CGC_GATE__LMI_MC_MASK
428                 | UVD_CGC_GATE__LMI_UMC_MASK
429                 | UVD_CGC_GATE__IDCT_MASK
430                 | UVD_CGC_GATE__MPRD_MASK
431                 | UVD_CGC_GATE__MPC_MASK
432                 | UVD_CGC_GATE__LBSI_MASK
433                 | UVD_CGC_GATE__LRBBM_MASK
434                 | UVD_CGC_GATE__UDEC_RE_MASK
435                 | UVD_CGC_GATE__UDEC_CM_MASK
436                 | UVD_CGC_GATE__UDEC_IT_MASK
437                 | UVD_CGC_GATE__UDEC_DB_MASK
438                 | UVD_CGC_GATE__UDEC_MP_MASK
439                 | UVD_CGC_GATE__WCB_MASK
440                 | UVD_CGC_GATE__VCPU_MASK
441                 | UVD_CGC_GATE__SCPU_MASK);
442         WREG32_SOC15(VCN, 0, mmUVD_CGC_GATE, data);
443
444         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
445         data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK
446                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
447                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
448                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
449                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
450                 | UVD_CGC_CTRL__SYS_MODE_MASK
451                 | UVD_CGC_CTRL__UDEC_MODE_MASK
452                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
453                 | UVD_CGC_CTRL__REGS_MODE_MASK
454                 | UVD_CGC_CTRL__RBC_MODE_MASK
455                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
456                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
457                 | UVD_CGC_CTRL__IDCT_MODE_MASK
458                 | UVD_CGC_CTRL__MPRD_MODE_MASK
459                 | UVD_CGC_CTRL__MPC_MODE_MASK
460                 | UVD_CGC_CTRL__LBSI_MODE_MASK
461                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
462                 | UVD_CGC_CTRL__WCB_MODE_MASK
463                 | UVD_CGC_CTRL__VCPU_MODE_MASK
464                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
465         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
466
467         /* turn on */
468         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE);
469         data |= (UVD_SUVD_CGC_GATE__SRE_MASK
470                 | UVD_SUVD_CGC_GATE__SIT_MASK
471                 | UVD_SUVD_CGC_GATE__SMP_MASK
472                 | UVD_SUVD_CGC_GATE__SCM_MASK
473                 | UVD_SUVD_CGC_GATE__SDB_MASK
474                 | UVD_SUVD_CGC_GATE__SRE_H264_MASK
475                 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK
476                 | UVD_SUVD_CGC_GATE__SIT_H264_MASK
477                 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK
478                 | UVD_SUVD_CGC_GATE__SCM_H264_MASK
479                 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK
480                 | UVD_SUVD_CGC_GATE__SDB_H264_MASK
481                 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK
482                 | UVD_SUVD_CGC_GATE__SCLR_MASK
483                 | UVD_SUVD_CGC_GATE__UVD_SC_MASK
484                 | UVD_SUVD_CGC_GATE__ENT_MASK
485                 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK
486                 | UVD_SUVD_CGC_GATE__SIT_HEVC_ENC_MASK
487                 | UVD_SUVD_CGC_GATE__SITE_MASK
488                 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK
489                 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK
490                 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK
491                 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK
492                 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK);
493         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_GATE, data);
494
495         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
496         data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
497                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
498                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
499                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
500                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
501                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
502                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
503                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
504                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
505                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
506         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
507 }
508
509 /**
510  * vcn_v1_0_enable_clock_gating - enable VCN clock gating
511  *
512  * @adev: amdgpu_device pointer
513  * @sw: enable SW clock gating
514  *
515  * Enable clock gating for VCN block
516  */
517 static void vcn_v1_0_enable_clock_gating(struct amdgpu_device *adev)
518 {
519         uint32_t data = 0;
520
521         /* enable JPEG CGC */
522         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL);
523         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
524                 data |= 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
525         else
526                 data |= 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
527         data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
528         data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
529         WREG32_SOC15(VCN, 0, mmJPEG_CGC_CTRL, data);
530
531         data = RREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE);
532         data |= (JPEG_CGC_GATE__JPEG_MASK | JPEG_CGC_GATE__JPEG2_MASK);
533         WREG32_SOC15(VCN, 0, mmJPEG_CGC_GATE, data);
534
535         /* enable UVD CGC */
536         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
537         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
538                 data |= 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
539         else
540                 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
541         data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
542         data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
543         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
544
545         data = RREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL);
546         data |= (UVD_CGC_CTRL__UDEC_RE_MODE_MASK
547                 | UVD_CGC_CTRL__UDEC_CM_MODE_MASK
548                 | UVD_CGC_CTRL__UDEC_IT_MODE_MASK
549                 | UVD_CGC_CTRL__UDEC_DB_MODE_MASK
550                 | UVD_CGC_CTRL__UDEC_MP_MODE_MASK
551                 | UVD_CGC_CTRL__SYS_MODE_MASK
552                 | UVD_CGC_CTRL__UDEC_MODE_MASK
553                 | UVD_CGC_CTRL__MPEG2_MODE_MASK
554                 | UVD_CGC_CTRL__REGS_MODE_MASK
555                 | UVD_CGC_CTRL__RBC_MODE_MASK
556                 | UVD_CGC_CTRL__LMI_MC_MODE_MASK
557                 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK
558                 | UVD_CGC_CTRL__IDCT_MODE_MASK
559                 | UVD_CGC_CTRL__MPRD_MODE_MASK
560                 | UVD_CGC_CTRL__MPC_MODE_MASK
561                 | UVD_CGC_CTRL__LBSI_MODE_MASK
562                 | UVD_CGC_CTRL__LRBBM_MODE_MASK
563                 | UVD_CGC_CTRL__WCB_MODE_MASK
564                 | UVD_CGC_CTRL__VCPU_MODE_MASK
565                 | UVD_CGC_CTRL__SCPU_MODE_MASK);
566         WREG32_SOC15(VCN, 0, mmUVD_CGC_CTRL, data);
567
568         data = RREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL);
569         data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK
570                 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK
571                 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK
572                 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK
573                 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK
574                 | UVD_SUVD_CGC_CTRL__SCLR_MODE_MASK
575                 | UVD_SUVD_CGC_CTRL__UVD_SC_MODE_MASK
576                 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK
577                 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK
578                 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK);
579         WREG32_SOC15(VCN, 0, mmUVD_SUVD_CGC_CTRL, data);
580 }
581
582 static void vcn_v1_0_clock_gating_dpg_mode(struct amdgpu_device *adev, uint8_t sram_sel)
583 {
584         uint32_t reg_data = 0;
585
586         /* disable JPEG CGC */
587         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
588                 reg_data = 1 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
589         else
590                 reg_data = 0 << JPEG_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
591         reg_data |= 1 << JPEG_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
592         reg_data |= 4 << JPEG_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
593         WREG32_SOC15_DPG_MODE(UVD, 0, mmJPEG_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
594
595         WREG32_SOC15_DPG_MODE(UVD, 0, mmJPEG_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
596
597         /* enable sw clock gating control */
598         if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG)
599                 reg_data = 1 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
600         else
601                 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT;
602         reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT;
603         reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT;
604         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
605
606         reg_data &= ~(UVD_CGC_CTRL__UDEC_RE_MODE_MASK |
607                  UVD_CGC_CTRL__UDEC_CM_MODE_MASK |
608                  UVD_CGC_CTRL__UDEC_IT_MODE_MASK |
609                  UVD_CGC_CTRL__UDEC_DB_MODE_MASK |
610                  UVD_CGC_CTRL__UDEC_MP_MODE_MASK |
611                  UVD_CGC_CTRL__SYS_MODE_MASK |
612                  UVD_CGC_CTRL__UDEC_MODE_MASK |
613                  UVD_CGC_CTRL__MPEG2_MODE_MASK |
614                  UVD_CGC_CTRL__REGS_MODE_MASK |
615                  UVD_CGC_CTRL__RBC_MODE_MASK |
616                  UVD_CGC_CTRL__LMI_MC_MODE_MASK |
617                  UVD_CGC_CTRL__LMI_UMC_MODE_MASK |
618                  UVD_CGC_CTRL__IDCT_MODE_MASK |
619                  UVD_CGC_CTRL__MPRD_MODE_MASK |
620                  UVD_CGC_CTRL__MPC_MODE_MASK |
621                  UVD_CGC_CTRL__LBSI_MODE_MASK |
622                  UVD_CGC_CTRL__LRBBM_MODE_MASK |
623                  UVD_CGC_CTRL__WCB_MODE_MASK |
624                  UVD_CGC_CTRL__VCPU_MODE_MASK |
625                  UVD_CGC_CTRL__SCPU_MODE_MASK);
626         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_CGC_CTRL, reg_data, 0xFFFFFFFF, sram_sel);
627
628         /* turn off clock gating */
629         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_CGC_GATE, 0, 0xFFFFFFFF, sram_sel);
630
631         /* turn on SUVD clock gating */
632         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SUVD_CGC_GATE, 1, 0xFFFFFFFF, sram_sel);
633
634         /* turn on sw mode in UVD_SUVD_CGC_CTRL */
635         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SUVD_CGC_CTRL, 0, 0xFFFFFFFF, sram_sel);
636 }
637
638 static void vcn_1_0_disable_static_power_gating(struct amdgpu_device *adev)
639 {
640         uint32_t data = 0;
641         int ret;
642
643         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
644                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
645                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
646                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
647                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
648                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
649                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
650                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
651                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
652                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
653                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
654                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
655
656                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
657                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, UVD_PGFSM_STATUS__UVDM_UVDU_PWR_ON, 0xFFFFFF, ret);
658         } else {
659                 data = (1 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
660                         | 1 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
661                         | 1 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
662                         | 1 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
663                         | 1 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
664                         | 1 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
665                         | 1 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
666                         | 1 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
667                         | 1 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
668                         | 1 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
669                         | 1 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
670                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
671                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, 0,  0xFFFFFFFF, ret);
672         }
673
674         /* polling UVD_PGFSM_STATUS to confirm UVDM_PWR_STATUS , UVDU_PWR_STATUS are 0 (power on) */
675
676         data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
677         data &= ~0x103;
678         if (adev->pg_flags & AMD_PG_SUPPORT_VCN)
679                 data |= UVD_PGFSM_CONFIG__UVDM_UVDU_PWR_ON | UVD_POWER_STATUS__UVD_PG_EN_MASK;
680
681         WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
682 }
683
684 static void vcn_1_0_enable_static_power_gating(struct amdgpu_device *adev)
685 {
686         uint32_t data = 0;
687         int ret;
688
689         if (adev->pg_flags & AMD_PG_SUPPORT_VCN) {
690                 /* Before power off, this indicator has to be turned on */
691                 data = RREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS);
692                 data &= ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK;
693                 data |= UVD_POWER_STATUS__UVD_POWER_STATUS_TILES_OFF;
694                 WREG32_SOC15(VCN, 0, mmUVD_POWER_STATUS, data);
695
696
697                 data = (2 << UVD_PGFSM_CONFIG__UVDM_PWR_CONFIG__SHIFT
698                         | 2 << UVD_PGFSM_CONFIG__UVDU_PWR_CONFIG__SHIFT
699                         | 2 << UVD_PGFSM_CONFIG__UVDF_PWR_CONFIG__SHIFT
700                         | 2 << UVD_PGFSM_CONFIG__UVDC_PWR_CONFIG__SHIFT
701                         | 2 << UVD_PGFSM_CONFIG__UVDB_PWR_CONFIG__SHIFT
702                         | 2 << UVD_PGFSM_CONFIG__UVDIL_PWR_CONFIG__SHIFT
703                         | 2 << UVD_PGFSM_CONFIG__UVDIR_PWR_CONFIG__SHIFT
704                         | 2 << UVD_PGFSM_CONFIG__UVDTD_PWR_CONFIG__SHIFT
705                         | 2 << UVD_PGFSM_CONFIG__UVDTE_PWR_CONFIG__SHIFT
706                         | 2 << UVD_PGFSM_CONFIG__UVDE_PWR_CONFIG__SHIFT
707                         | 2 << UVD_PGFSM_CONFIG__UVDW_PWR_CONFIG__SHIFT);
708
709                 WREG32_SOC15(VCN, 0, mmUVD_PGFSM_CONFIG, data);
710
711                 data = (2 << UVD_PGFSM_STATUS__UVDM_PWR_STATUS__SHIFT
712                         | 2 << UVD_PGFSM_STATUS__UVDU_PWR_STATUS__SHIFT
713                         | 2 << UVD_PGFSM_STATUS__UVDF_PWR_STATUS__SHIFT
714                         | 2 << UVD_PGFSM_STATUS__UVDC_PWR_STATUS__SHIFT
715                         | 2 << UVD_PGFSM_STATUS__UVDB_PWR_STATUS__SHIFT
716                         | 2 << UVD_PGFSM_STATUS__UVDIL_PWR_STATUS__SHIFT
717                         | 2 << UVD_PGFSM_STATUS__UVDIR_PWR_STATUS__SHIFT
718                         | 2 << UVD_PGFSM_STATUS__UVDTD_PWR_STATUS__SHIFT
719                         | 2 << UVD_PGFSM_STATUS__UVDTE_PWR_STATUS__SHIFT
720                         | 2 << UVD_PGFSM_STATUS__UVDE_PWR_STATUS__SHIFT
721                         | 2 << UVD_PGFSM_STATUS__UVDW_PWR_STATUS__SHIFT);
722                 SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_PGFSM_STATUS, data, 0xFFFFFFFF, ret);
723         }
724 }
725
726 /**
727  * vcn_v1_0_start - start VCN block
728  *
729  * @adev: amdgpu_device pointer
730  *
731  * Setup and start the VCN block
732  */
733 static int vcn_v1_0_start_spg_mode(struct amdgpu_device *adev)
734 {
735         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
736         uint32_t rb_bufsz, tmp;
737         uint32_t lmi_swap_cntl;
738         int i, j, r;
739
740         /* disable byte swapping */
741         lmi_swap_cntl = 0;
742
743         vcn_1_0_disable_static_power_gating(adev);
744         /* disable clock gating */
745         vcn_v1_0_disable_clock_gating(adev);
746
747         vcn_v1_0_mc_resume_spg_mode(adev);
748
749         /* disable interupt */
750         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN), 0,
751                         ~UVD_MASTINT_EN__VCPU_EN_MASK);
752
753         /* stall UMC and register bus before resetting VCPU */
754         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
755                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
756                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
757         mdelay(1);
758
759         /* put LMI, VCPU, RBC etc... into reset */
760         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
761                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
762                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
763                 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
764                 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
765                 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
766                 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
767                 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
768                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
769         mdelay(5);
770
771         /* initialize VCN memory controller */
772         WREG32_SOC15(UVD, 0, mmUVD_LMI_CTRL,
773                 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
774                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
775                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
776                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
777                 UVD_LMI_CTRL__REQ_MODE_MASK |
778                 0x00100000L);
779
780 #ifdef __BIG_ENDIAN
781         /* swap (8 in 32) RB and IB */
782         lmi_swap_cntl = 0xa;
783 #endif
784         WREG32_SOC15(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl);
785
786         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040);
787         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0);
788         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040);
789         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0);
790         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_ALU, 0);
791         WREG32_SOC15(UVD, 0, mmUVD_MPC_SET_MUX, 0x88);
792
793         /* take all subblocks out of reset, except VCPU */
794         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
795                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
796         mdelay(5);
797
798         /* enable VCPU clock */
799         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL,
800                         UVD_VCPU_CNTL__CLK_EN_MASK);
801
802         /* enable UMC */
803         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
804                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
805
806         /* boot up the VCPU */
807         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET, 0);
808         mdelay(10);
809
810         for (i = 0; i < 10; ++i) {
811                 uint32_t status;
812
813                 for (j = 0; j < 100; ++j) {
814                         status = RREG32_SOC15(UVD, 0, mmUVD_STATUS);
815                         if (status & 2)
816                                 break;
817                         mdelay(10);
818                 }
819                 r = 0;
820                 if (status & 2)
821                         break;
822
823                 DRM_ERROR("VCN decode not responding, trying to reset the VCPU!!!\n");
824                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET),
825                                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK,
826                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
827                 mdelay(10);
828                 WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SOFT_RESET), 0,
829                                 ~UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
830                 mdelay(10);
831                 r = -1;
832         }
833
834         if (r) {
835                 DRM_ERROR("VCN decode not responding, giving up!!!\n");
836                 return r;
837         }
838         /* enable master interrupt */
839         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_MASTINT_EN),
840                 (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
841                 ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
842
843         /* enable system interrupt for JRBC, TODO: move to set interrupt*/
844         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_SYS_INT_EN),
845                 UVD_SYS_INT_EN__UVD_JRBC_EN_MASK,
846                 ~UVD_SYS_INT_EN__UVD_JRBC_EN_MASK);
847
848         /* clear the bit 4 of VCN_STATUS */
849         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_STATUS), 0,
850                         ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
851
852         /* force RBC into idle state */
853         rb_bufsz = order_base_2(ring->ring_size);
854         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
855         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
856         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
857         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
858         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
859         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
860         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
861
862         /* set the write pointer delay */
863         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
864
865         /* set the wb address */
866         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
867                         (upper_32_bits(ring->gpu_addr) >> 2));
868
869         /* programm the RB_BASE for ring buffer */
870         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
871                         lower_32_bits(ring->gpu_addr));
872         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
873                         upper_32_bits(ring->gpu_addr));
874
875         /* Initialize the ring buffer's read and write pointers */
876         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
877
878         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
879         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
880                         lower_32_bits(ring->wptr));
881
882         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
883                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
884
885         ring = &adev->vcn.ring_enc[0];
886         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR, lower_32_bits(ring->wptr));
887         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR, lower_32_bits(ring->wptr));
888         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO, ring->gpu_addr);
889         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
890         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE, ring->ring_size / 4);
891
892         ring = &adev->vcn.ring_enc[1];
893         WREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2, lower_32_bits(ring->wptr));
894         WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2, lower_32_bits(ring->wptr));
895         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_LO2, ring->gpu_addr);
896         WREG32_SOC15(UVD, 0, mmUVD_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
897         WREG32_SOC15(UVD, 0, mmUVD_RB_SIZE2, ring->ring_size / 4);
898
899         ring = &adev->vcn.ring_jpeg;
900         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_VMID, 0);
901         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, (0x00000001L | 0x00000002L));
902         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW, lower_32_bits(ring->gpu_addr));
903         WREG32_SOC15(UVD, 0, mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH, upper_32_bits(ring->gpu_addr));
904         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR, 0);
905         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, 0);
906         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_CNTL, 0x00000002L);
907
908         /* initialize wptr */
909         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
910
911         /* copy patch commands to the jpeg ring */
912         vcn_v1_0_jpeg_ring_set_patch_ring(ring,
913                 (ring->wptr + ring->max_dw * amdgpu_sched_hw_submission));
914
915         return 0;
916 }
917
918 static int vcn_v1_0_start_dpg_mode(struct amdgpu_device *adev)
919 {
920         struct amdgpu_ring *ring = &adev->vcn.ring_dec;
921         uint32_t rb_bufsz, tmp, reg_data;
922         uint32_t lmi_swap_cntl;
923
924         /* disable byte swapping */
925         lmi_swap_cntl = 0;
926
927         vcn_1_0_enable_static_power_gating(adev);
928
929         /* enable dynamic power gating mode */
930         reg_data = RREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS);
931         reg_data |= UVD_POWER_STATUS__UVD_PG_MODE_MASK;
932         reg_data |= UVD_POWER_STATUS__UVD_PG_EN_MASK;
933         WREG32_SOC15(UVD, 0, mmUVD_POWER_STATUS, reg_data);
934
935         /* enable clock gating */
936         vcn_v1_0_clock_gating_dpg_mode(adev, 0);
937
938         /* enable VCPU clock */
939         reg_data = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT);
940         reg_data |= UVD_VCPU_CNTL__CLK_EN_MASK;
941         reg_data |= UVD_VCPU_CNTL__MIF_WR_LOW_THRESHOLD_BP_MASK;
942         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CNTL, reg_data, 0xFFFFFFFF, 0);
943
944         /* disable interupt */
945         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MASTINT_EN,
946                         0, UVD_MASTINT_EN__VCPU_EN_MASK, 0);
947
948         /* stall UMC and register bus before resetting VCPU */
949         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL2,
950                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
951
952         /* put LMI, VCPU, RBC etc... into reset */
953         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SOFT_RESET,
954                 UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
955                 UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
956                 UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
957                 UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
958                 UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
959                 UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
960                 UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
961                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK,
962                 0xFFFFFFFF, 0);
963
964         /* initialize VCN memory controller */
965         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL,
966                 (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
967                 UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
968                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
969                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
970                 UVD_LMI_CTRL__REQ_MODE_MASK |
971                 0x00100000L, 0xFFFFFFFF, 0);
972
973 #ifdef __BIG_ENDIAN
974         /* swap (8 in 32) RB and IB */
975         lmi_swap_cntl = 0xa;
976 #endif
977         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_SWAP_CNTL, lmi_swap_cntl, 0xFFFFFFFF, 0);
978
979         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXA0, 0x40c2040, 0xFFFFFFFF, 0);
980         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXA1, 0x0, 0xFFFFFFFF, 0);
981         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXB0, 0x40c2040, 0xFFFFFFFF, 0);
982         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUXB1, 0x0, 0xFFFFFFFF, 0);
983         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_ALU, 0, 0xFFFFFFFF, 0);
984         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MPC_SET_MUX, 0x88, 0xFFFFFFFF, 0);
985
986         vcn_v1_0_mc_resume_dpg_mode(adev);
987
988         /* take all subblocks out of reset, except VCPU */
989         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SOFT_RESET,
990                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK, 0xFFFFFFFF, 0);
991
992         /* enable VCPU clock */
993         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_VCPU_CNTL,
994                         UVD_VCPU_CNTL__CLK_EN_MASK, 0xFFFFFFFF, 0);
995
996         /* enable UMC */
997         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL2,
998                         0, UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, 0);
999
1000         /* boot up the VCPU */
1001         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SOFT_RESET, 0, 0xFFFFFFFF, 0);
1002
1003         /* enable master interrupt */
1004         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_MASTINT_EN,
1005                         (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
1006                         (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK), 0);
1007
1008         vcn_v1_0_clock_gating_dpg_mode(adev, 1);
1009         /* setup mmUVD_LMI_CTRL */
1010         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_LMI_CTRL,
1011                         (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
1012                                 UVD_LMI_CTRL__CRC_RESET_MASK |
1013                                 UVD_LMI_CTRL__MASK_MC_URGENT_MASK |
1014                                 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
1015                                 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
1016                                 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
1017                                 0x00100000L), 0xFFFFFFFF, 1);
1018
1019         tmp = adev->gfx.config.gb_addr_config;
1020         /* setup VCN global tiling registers */
1021         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1022         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_JPEG_UV_ADDR_CONFIG, tmp, 0xFFFFFFFF, 1);
1023
1024         /* enable System Interrupt for JRBC */
1025         WREG32_SOC15_DPG_MODE(UVD, 0, mmUVD_SYS_INT_EN,
1026                                                                         UVD_SYS_INT_EN__UVD_JRBC_EN_MASK, 0xFFFFFFFF, 1);
1027
1028         /* force RBC into idle state */
1029         rb_bufsz = order_base_2(ring->ring_size);
1030         tmp = REG_SET_FIELD(0, UVD_RBC_RB_CNTL, RB_BUFSZ, rb_bufsz);
1031         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_BLKSZ, 1);
1032         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_FETCH, 1);
1033         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_WPTR_POLL_EN, 0);
1034         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_NO_UPDATE, 1);
1035         tmp = REG_SET_FIELD(tmp, UVD_RBC_RB_CNTL, RB_RPTR_WR_EN, 1);
1036         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, tmp);
1037
1038         /* set the write pointer delay */
1039         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR_CNTL, 0);
1040
1041         /* set the wb address */
1042         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR_ADDR,
1043                                                                 (upper_32_bits(ring->gpu_addr) >> 2));
1044
1045         /* programm the RB_BASE for ring buffer */
1046         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_LOW,
1047                                                                 lower_32_bits(ring->gpu_addr));
1048         WREG32_SOC15(UVD, 0, mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH,
1049                                                                 upper_32_bits(ring->gpu_addr));
1050
1051         /* Initialize the ring buffer's read and write pointers */
1052         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR, 0);
1053
1054         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1055         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR,
1056                                                                 lower_32_bits(ring->wptr));
1057
1058         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_RB_CNTL), 0,
1059                         ~UVD_RBC_RB_CNTL__RB_NO_FETCH_MASK);
1060
1061         /* initialize wptr */
1062         ring->wptr = RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1063
1064         /* copy patch commands to the jpeg ring */
1065         vcn_v1_0_jpeg_ring_set_patch_ring(ring,
1066                 (ring->wptr + ring->max_dw * amdgpu_sched_hw_submission));
1067
1068         return 0;
1069 }
1070
1071 static int vcn_v1_0_start(struct amdgpu_device *adev)
1072 {
1073         int r;
1074
1075         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1076                 r = vcn_v1_0_start_dpg_mode(adev);
1077         else
1078                 r = vcn_v1_0_start_spg_mode(adev);
1079         return r;
1080 }
1081
1082 /**
1083  * vcn_v1_0_stop - stop VCN block
1084  *
1085  * @adev: amdgpu_device pointer
1086  *
1087  * stop the VCN block
1088  */
1089 static int vcn_v1_0_stop_spg_mode(struct amdgpu_device *adev)
1090 {
1091         /* force RBC into idle state */
1092         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_CNTL, 0x11010101);
1093
1094         /* Stall UMC and register bus before resetting VCPU */
1095         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2),
1096                         UVD_LMI_CTRL2__STALL_ARB_UMC_MASK,
1097                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1098         mdelay(1);
1099
1100         /* put VCPU into reset */
1101         WREG32_SOC15(UVD, 0, mmUVD_SOFT_RESET,
1102                         UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK);
1103         mdelay(5);
1104
1105         /* disable VCPU clock */
1106         WREG32_SOC15(UVD, 0, mmUVD_VCPU_CNTL, 0x0);
1107
1108         /* Unstall UMC and register bus */
1109         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_CTRL2), 0,
1110                         ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
1111
1112         WREG32_SOC15(VCN, 0, mmUVD_STATUS, 0);
1113
1114         vcn_v1_0_enable_clock_gating(adev);
1115         vcn_1_0_enable_static_power_gating(adev);
1116         return 0;
1117 }
1118
1119 static int vcn_v1_0_stop_dpg_mode(struct amdgpu_device *adev)
1120 {
1121         int ret_code;
1122
1123         /* Wait for power status to be 1 */
1124         SOC15_WAIT_ON_RREG(UVD, 0, mmUVD_POWER_STATUS, 0x1,
1125                         UVD_POWER_STATUS__UVD_POWER_STATUS_MASK, ret_code);
1126
1127         /* disable dynamic power gating mode */
1128         WREG32_P(SOC15_REG_OFFSET(UVD, 0, mmUVD_POWER_STATUS), 0,
1129                         ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
1130
1131         return 0;
1132 }
1133
1134 static int vcn_v1_0_stop(struct amdgpu_device *adev)
1135 {
1136         int r;
1137
1138         if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
1139                 r = vcn_v1_0_stop_dpg_mode(adev);
1140         else
1141                 r = vcn_v1_0_stop_spg_mode(adev);
1142
1143         return r;
1144 }
1145
1146 static bool vcn_v1_0_is_idle(void *handle)
1147 {
1148         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1149
1150         return (RREG32_SOC15(VCN, 0, mmUVD_STATUS) == 0x2);
1151 }
1152
1153 static int vcn_v1_0_wait_for_idle(void *handle)
1154 {
1155         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1156         int ret = 0;
1157
1158         SOC15_WAIT_ON_RREG(VCN, 0, mmUVD_STATUS, 0x2, 0x2, ret);
1159
1160         return ret;
1161 }
1162
1163 static int vcn_v1_0_set_clockgating_state(void *handle,
1164                                           enum amd_clockgating_state state)
1165 {
1166         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1167         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
1168
1169         if (enable) {
1170                 /* wait for STATUS to clear */
1171                 if (vcn_v1_0_is_idle(handle))
1172                         return -EBUSY;
1173                 vcn_v1_0_enable_clock_gating(adev);
1174         } else {
1175                 /* disable HW gating and enable Sw gating */
1176                 vcn_v1_0_disable_clock_gating(adev);
1177         }
1178         return 0;
1179 }
1180
1181 /**
1182  * vcn_v1_0_dec_ring_get_rptr - get read pointer
1183  *
1184  * @ring: amdgpu_ring pointer
1185  *
1186  * Returns the current hardware read pointer
1187  */
1188 static uint64_t vcn_v1_0_dec_ring_get_rptr(struct amdgpu_ring *ring)
1189 {
1190         struct amdgpu_device *adev = ring->adev;
1191
1192         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_RPTR);
1193 }
1194
1195 /**
1196  * vcn_v1_0_dec_ring_get_wptr - get write pointer
1197  *
1198  * @ring: amdgpu_ring pointer
1199  *
1200  * Returns the current hardware write pointer
1201  */
1202 static uint64_t vcn_v1_0_dec_ring_get_wptr(struct amdgpu_ring *ring)
1203 {
1204         struct amdgpu_device *adev = ring->adev;
1205
1206         return RREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR);
1207 }
1208
1209 /**
1210  * vcn_v1_0_dec_ring_set_wptr - set write pointer
1211  *
1212  * @ring: amdgpu_ring pointer
1213  *
1214  * Commits the write pointer to the hardware
1215  */
1216 static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
1217 {
1218         struct amdgpu_device *adev = ring->adev;
1219
1220         WREG32_SOC15(UVD, 0, mmUVD_RBC_RB_WPTR, lower_32_bits(ring->wptr));
1221 }
1222
1223 /**
1224  * vcn_v1_0_dec_ring_insert_start - insert a start command
1225  *
1226  * @ring: amdgpu_ring pointer
1227  *
1228  * Write a start command to the ring.
1229  */
1230 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
1231 {
1232         struct amdgpu_device *adev = ring->adev;
1233
1234         amdgpu_ring_write(ring,
1235                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1236         amdgpu_ring_write(ring, 0);
1237         amdgpu_ring_write(ring,
1238                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1239         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_START << 1);
1240 }
1241
1242 /**
1243  * vcn_v1_0_dec_ring_insert_end - insert a end command
1244  *
1245  * @ring: amdgpu_ring pointer
1246  *
1247  * Write a end command to the ring.
1248  */
1249 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
1250 {
1251         struct amdgpu_device *adev = ring->adev;
1252
1253         amdgpu_ring_write(ring,
1254                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1255         amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
1256 }
1257
1258 /**
1259  * vcn_v1_0_dec_ring_emit_fence - emit an fence & trap command
1260  *
1261  * @ring: amdgpu_ring pointer
1262  * @fence: fence to emit
1263  *
1264  * Write a fence and a trap command to the ring.
1265  */
1266 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1267                                      unsigned flags)
1268 {
1269         struct amdgpu_device *adev = ring->adev;
1270
1271         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1272
1273         amdgpu_ring_write(ring,
1274                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_CONTEXT_ID), 0));
1275         amdgpu_ring_write(ring, seq);
1276         amdgpu_ring_write(ring,
1277                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1278         amdgpu_ring_write(ring, addr & 0xffffffff);
1279         amdgpu_ring_write(ring,
1280                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1281         amdgpu_ring_write(ring, upper_32_bits(addr) & 0xff);
1282         amdgpu_ring_write(ring,
1283                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1284         amdgpu_ring_write(ring, VCN_DEC_CMD_FENCE << 1);
1285
1286         amdgpu_ring_write(ring,
1287                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1288         amdgpu_ring_write(ring, 0);
1289         amdgpu_ring_write(ring,
1290                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1291         amdgpu_ring_write(ring, 0);
1292         amdgpu_ring_write(ring,
1293                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1294         amdgpu_ring_write(ring, VCN_DEC_CMD_TRAP << 1);
1295 }
1296
1297 /**
1298  * vcn_v1_0_dec_ring_emit_ib - execute indirect buffer
1299  *
1300  * @ring: amdgpu_ring pointer
1301  * @ib: indirect buffer to execute
1302  *
1303  * Write ring commands to execute the indirect buffer
1304  */
1305 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
1306                                   struct amdgpu_ib *ib,
1307                                   unsigned vmid, bool ctx_switch)
1308 {
1309         struct amdgpu_device *adev = ring->adev;
1310
1311         amdgpu_ring_write(ring,
1312                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
1313         amdgpu_ring_write(ring, vmid);
1314
1315         amdgpu_ring_write(ring,
1316                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
1317         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1318         amdgpu_ring_write(ring,
1319                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH), 0));
1320         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1321         amdgpu_ring_write(ring,
1322                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_RBC_IB_SIZE), 0));
1323         amdgpu_ring_write(ring, ib->length_dw);
1324 }
1325
1326 static void vcn_v1_0_dec_ring_emit_reg_wait(struct amdgpu_ring *ring,
1327                                             uint32_t reg, uint32_t val,
1328                                             uint32_t mask)
1329 {
1330         struct amdgpu_device *adev = ring->adev;
1331
1332         amdgpu_ring_write(ring,
1333                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1334         amdgpu_ring_write(ring, reg << 2);
1335         amdgpu_ring_write(ring,
1336                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1337         amdgpu_ring_write(ring, val);
1338         amdgpu_ring_write(ring,
1339                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GP_SCRATCH8), 0));
1340         amdgpu_ring_write(ring, mask);
1341         amdgpu_ring_write(ring,
1342                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1343         amdgpu_ring_write(ring, VCN_DEC_CMD_REG_READ_COND_WAIT << 1);
1344 }
1345
1346 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
1347                                             unsigned vmid, uint64_t pd_addr)
1348 {
1349         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1350         uint32_t data0, data1, mask;
1351
1352         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1353
1354         /* wait for register write */
1355         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
1356         data1 = lower_32_bits(pd_addr);
1357         mask = 0xffffffff;
1358         vcn_v1_0_dec_ring_emit_reg_wait(ring, data0, data1, mask);
1359 }
1360
1361 static void vcn_v1_0_dec_ring_emit_wreg(struct amdgpu_ring *ring,
1362                                         uint32_t reg, uint32_t val)
1363 {
1364         struct amdgpu_device *adev = ring->adev;
1365
1366         amdgpu_ring_write(ring,
1367                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
1368         amdgpu_ring_write(ring, reg << 2);
1369         amdgpu_ring_write(ring,
1370                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA1), 0));
1371         amdgpu_ring_write(ring, val);
1372         amdgpu_ring_write(ring,
1373                 PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
1374         amdgpu_ring_write(ring, VCN_DEC_CMD_WRITE_REG << 1);
1375 }
1376
1377 /**
1378  * vcn_v1_0_enc_ring_get_rptr - get enc read pointer
1379  *
1380  * @ring: amdgpu_ring pointer
1381  *
1382  * Returns the current hardware enc read pointer
1383  */
1384 static uint64_t vcn_v1_0_enc_ring_get_rptr(struct amdgpu_ring *ring)
1385 {
1386         struct amdgpu_device *adev = ring->adev;
1387
1388         if (ring == &adev->vcn.ring_enc[0])
1389                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR);
1390         else
1391                 return RREG32_SOC15(UVD, 0, mmUVD_RB_RPTR2);
1392 }
1393
1394  /**
1395  * vcn_v1_0_enc_ring_get_wptr - get enc write pointer
1396  *
1397  * @ring: amdgpu_ring pointer
1398  *
1399  * Returns the current hardware enc write pointer
1400  */
1401 static uint64_t vcn_v1_0_enc_ring_get_wptr(struct amdgpu_ring *ring)
1402 {
1403         struct amdgpu_device *adev = ring->adev;
1404
1405         if (ring == &adev->vcn.ring_enc[0])
1406                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR);
1407         else
1408                 return RREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2);
1409 }
1410
1411  /**
1412  * vcn_v1_0_enc_ring_set_wptr - set enc write pointer
1413  *
1414  * @ring: amdgpu_ring pointer
1415  *
1416  * Commits the enc write pointer to the hardware
1417  */
1418 static void vcn_v1_0_enc_ring_set_wptr(struct amdgpu_ring *ring)
1419 {
1420         struct amdgpu_device *adev = ring->adev;
1421
1422         if (ring == &adev->vcn.ring_enc[0])
1423                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR,
1424                         lower_32_bits(ring->wptr));
1425         else
1426                 WREG32_SOC15(UVD, 0, mmUVD_RB_WPTR2,
1427                         lower_32_bits(ring->wptr));
1428 }
1429
1430 /**
1431  * vcn_v1_0_enc_ring_emit_fence - emit an enc fence & trap command
1432  *
1433  * @ring: amdgpu_ring pointer
1434  * @fence: fence to emit
1435  *
1436  * Write enc a fence and a trap command to the ring.
1437  */
1438 static void vcn_v1_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
1439                         u64 seq, unsigned flags)
1440 {
1441         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1442
1443         amdgpu_ring_write(ring, VCN_ENC_CMD_FENCE);
1444         amdgpu_ring_write(ring, addr);
1445         amdgpu_ring_write(ring, upper_32_bits(addr));
1446         amdgpu_ring_write(ring, seq);
1447         amdgpu_ring_write(ring, VCN_ENC_CMD_TRAP);
1448 }
1449
1450 static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
1451 {
1452         amdgpu_ring_write(ring, VCN_ENC_CMD_END);
1453 }
1454
1455 /**
1456  * vcn_v1_0_enc_ring_emit_ib - enc execute indirect buffer
1457  *
1458  * @ring: amdgpu_ring pointer
1459  * @ib: indirect buffer to execute
1460  *
1461  * Write enc ring commands to execute the indirect buffer
1462  */
1463 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
1464                 struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
1465 {
1466         amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
1467         amdgpu_ring_write(ring, vmid);
1468         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1469         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1470         amdgpu_ring_write(ring, ib->length_dw);
1471 }
1472
1473 static void vcn_v1_0_enc_ring_emit_reg_wait(struct amdgpu_ring *ring,
1474                                             uint32_t reg, uint32_t val,
1475                                             uint32_t mask)
1476 {
1477         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
1478         amdgpu_ring_write(ring, reg << 2);
1479         amdgpu_ring_write(ring, mask);
1480         amdgpu_ring_write(ring, val);
1481 }
1482
1483 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
1484                                             unsigned int vmid, uint64_t pd_addr)
1485 {
1486         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1487
1488         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1489
1490         /* wait for reg writes */
1491         vcn_v1_0_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
1492                                         lower_32_bits(pd_addr), 0xffffffff);
1493 }
1494
1495 static void vcn_v1_0_enc_ring_emit_wreg(struct amdgpu_ring *ring,
1496                                         uint32_t reg, uint32_t val)
1497 {
1498         amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
1499         amdgpu_ring_write(ring, reg << 2);
1500         amdgpu_ring_write(ring, val);
1501 }
1502
1503
1504 /**
1505  * vcn_v1_0_jpeg_ring_get_rptr - get read pointer
1506  *
1507  * @ring: amdgpu_ring pointer
1508  *
1509  * Returns the current hardware read pointer
1510  */
1511 static uint64_t vcn_v1_0_jpeg_ring_get_rptr(struct amdgpu_ring *ring)
1512 {
1513         struct amdgpu_device *adev = ring->adev;
1514
1515         return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_RPTR);
1516 }
1517
1518 /**
1519  * vcn_v1_0_jpeg_ring_get_wptr - get write pointer
1520  *
1521  * @ring: amdgpu_ring pointer
1522  *
1523  * Returns the current hardware write pointer
1524  */
1525 static uint64_t vcn_v1_0_jpeg_ring_get_wptr(struct amdgpu_ring *ring)
1526 {
1527         struct amdgpu_device *adev = ring->adev;
1528
1529         return RREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR);
1530 }
1531
1532 /**
1533  * vcn_v1_0_jpeg_ring_set_wptr - set write pointer
1534  *
1535  * @ring: amdgpu_ring pointer
1536  *
1537  * Commits the write pointer to the hardware
1538  */
1539 static void vcn_v1_0_jpeg_ring_set_wptr(struct amdgpu_ring *ring)
1540 {
1541         struct amdgpu_device *adev = ring->adev;
1542
1543         WREG32_SOC15(UVD, 0, mmUVD_JRBC_RB_WPTR, lower_32_bits(ring->wptr));
1544 }
1545
1546 /**
1547  * vcn_v1_0_jpeg_ring_insert_start - insert a start command
1548  *
1549  * @ring: amdgpu_ring pointer
1550  *
1551  * Write a start command to the ring.
1552  */
1553 static void vcn_v1_0_jpeg_ring_insert_start(struct amdgpu_ring *ring)
1554 {
1555         struct amdgpu_device *adev = ring->adev;
1556
1557         amdgpu_ring_write(ring,
1558                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1559         amdgpu_ring_write(ring, 0x68e04);
1560
1561         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1562         amdgpu_ring_write(ring, 0x80010000);
1563 }
1564
1565 /**
1566  * vcn_v1_0_jpeg_ring_insert_end - insert a end command
1567  *
1568  * @ring: amdgpu_ring pointer
1569  *
1570  * Write a end command to the ring.
1571  */
1572 static void vcn_v1_0_jpeg_ring_insert_end(struct amdgpu_ring *ring)
1573 {
1574         struct amdgpu_device *adev = ring->adev;
1575
1576         amdgpu_ring_write(ring,
1577                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1578         amdgpu_ring_write(ring, 0x68e04);
1579
1580         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1581         amdgpu_ring_write(ring, 0x00010000);
1582 }
1583
1584 /**
1585  * vcn_v1_0_jpeg_ring_emit_fence - emit an fence & trap command
1586  *
1587  * @ring: amdgpu_ring pointer
1588  * @fence: fence to emit
1589  *
1590  * Write a fence and a trap command to the ring.
1591  */
1592 static void vcn_v1_0_jpeg_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
1593                                      unsigned flags)
1594 {
1595         struct amdgpu_device *adev = ring->adev;
1596
1597         WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
1598
1599         amdgpu_ring_write(ring,
1600                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_DATA0), 0, 0, PACKETJ_TYPE0));
1601         amdgpu_ring_write(ring, seq);
1602
1603         amdgpu_ring_write(ring,
1604                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_DATA1), 0, 0, PACKETJ_TYPE0));
1605         amdgpu_ring_write(ring, seq);
1606
1607         amdgpu_ring_write(ring,
1608                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1609         amdgpu_ring_write(ring, lower_32_bits(addr));
1610
1611         amdgpu_ring_write(ring,
1612                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_WR_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1613         amdgpu_ring_write(ring, upper_32_bits(addr));
1614
1615         amdgpu_ring_write(ring,
1616                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_CMD), 0, 0, PACKETJ_TYPE0));
1617         amdgpu_ring_write(ring, 0x8);
1618
1619         amdgpu_ring_write(ring,
1620                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JPEG_GPCOM_CMD), 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE4));
1621         amdgpu_ring_write(ring, 0);
1622
1623         amdgpu_ring_write(ring,
1624                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1625         amdgpu_ring_write(ring, 0x01400200);
1626
1627         amdgpu_ring_write(ring,
1628                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1629         amdgpu_ring_write(ring, seq);
1630
1631         amdgpu_ring_write(ring,
1632                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1633         amdgpu_ring_write(ring, lower_32_bits(addr));
1634
1635         amdgpu_ring_write(ring,
1636                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1637         amdgpu_ring_write(ring, upper_32_bits(addr));
1638
1639         amdgpu_ring_write(ring,
1640                 PACKETJ(0, 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE2));
1641         amdgpu_ring_write(ring, 0xffffffff);
1642
1643         amdgpu_ring_write(ring,
1644                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1645         amdgpu_ring_write(ring, 0x3fbc);
1646
1647         amdgpu_ring_write(ring,
1648                 PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1649         amdgpu_ring_write(ring, 0x1);
1650
1651         /* emit trap */
1652         amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE7));
1653         amdgpu_ring_write(ring, 0);
1654 }
1655
1656 /**
1657  * vcn_v1_0_jpeg_ring_emit_ib - execute indirect buffer
1658  *
1659  * @ring: amdgpu_ring pointer
1660  * @ib: indirect buffer to execute
1661  *
1662  * Write ring commands to execute the indirect buffer.
1663  */
1664 static void vcn_v1_0_jpeg_ring_emit_ib(struct amdgpu_ring *ring,
1665                                   struct amdgpu_ib *ib,
1666                                   unsigned vmid, bool ctx_switch)
1667 {
1668         struct amdgpu_device *adev = ring->adev;
1669
1670         amdgpu_ring_write(ring,
1671                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_VMID), 0, 0, PACKETJ_TYPE0));
1672         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1673
1674         amdgpu_ring_write(ring,
1675                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JPEG_VMID), 0, 0, PACKETJ_TYPE0));
1676         amdgpu_ring_write(ring, (vmid | (vmid << 4)));
1677
1678         amdgpu_ring_write(ring,
1679                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1680         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
1681
1682         amdgpu_ring_write(ring,
1683                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1684         amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
1685
1686         amdgpu_ring_write(ring,
1687                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_IB_SIZE), 0, 0, PACKETJ_TYPE0));
1688         amdgpu_ring_write(ring, ib->length_dw);
1689
1690         amdgpu_ring_write(ring,
1691                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW), 0, 0, PACKETJ_TYPE0));
1692         amdgpu_ring_write(ring, lower_32_bits(ring->gpu_addr));
1693
1694         amdgpu_ring_write(ring,
1695                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH), 0, 0, PACKETJ_TYPE0));
1696         amdgpu_ring_write(ring, upper_32_bits(ring->gpu_addr));
1697
1698         amdgpu_ring_write(ring,
1699                 PACKETJ(0, 0, PACKETJ_CONDITION_CHECK0, PACKETJ_TYPE2));
1700         amdgpu_ring_write(ring, 0);
1701
1702         amdgpu_ring_write(ring,
1703                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1704         amdgpu_ring_write(ring, 0x01400200);
1705
1706         amdgpu_ring_write(ring,
1707                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1708         amdgpu_ring_write(ring, 0x2);
1709
1710         amdgpu_ring_write(ring,
1711                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_STATUS), 0, PACKETJ_CONDITION_CHECK3, PACKETJ_TYPE3));
1712         amdgpu_ring_write(ring, 0x2);
1713 }
1714
1715 static void vcn_v1_0_jpeg_ring_emit_reg_wait(struct amdgpu_ring *ring,
1716                                             uint32_t reg, uint32_t val,
1717                                             uint32_t mask)
1718 {
1719         struct amdgpu_device *adev = ring->adev;
1720         uint32_t reg_offset = (reg << 2);
1721
1722         amdgpu_ring_write(ring,
1723                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0));
1724         amdgpu_ring_write(ring, 0x01400200);
1725
1726         amdgpu_ring_write(ring,
1727                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0));
1728         amdgpu_ring_write(ring, val);
1729
1730         amdgpu_ring_write(ring,
1731                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1732         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1733                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1734                 amdgpu_ring_write(ring, 0);
1735                 amdgpu_ring_write(ring,
1736                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3));
1737         } else {
1738                 amdgpu_ring_write(ring, reg_offset);
1739                 amdgpu_ring_write(ring,
1740                         PACKETJ(0, 0, 0, PACKETJ_TYPE3));
1741         }
1742         amdgpu_ring_write(ring, mask);
1743 }
1744
1745 static void vcn_v1_0_jpeg_ring_emit_vm_flush(struct amdgpu_ring *ring,
1746                 unsigned vmid, uint64_t pd_addr)
1747 {
1748         struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
1749         uint32_t data0, data1, mask;
1750
1751         pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
1752
1753         /* wait for register write */
1754         data0 = hub->ctx0_ptb_addr_lo32 + vmid * 2;
1755         data1 = lower_32_bits(pd_addr);
1756         mask = 0xffffffff;
1757         vcn_v1_0_jpeg_ring_emit_reg_wait(ring, data0, data1, mask);
1758 }
1759
1760 static void vcn_v1_0_jpeg_ring_emit_wreg(struct amdgpu_ring *ring,
1761                                         uint32_t reg, uint32_t val)
1762 {
1763         struct amdgpu_device *adev = ring->adev;
1764         uint32_t reg_offset = (reg << 2);
1765
1766         amdgpu_ring_write(ring,
1767                 PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0));
1768         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1769                         ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1770                 amdgpu_ring_write(ring, 0);
1771                 amdgpu_ring_write(ring,
1772                         PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0));
1773         } else {
1774                 amdgpu_ring_write(ring, reg_offset);
1775                 amdgpu_ring_write(ring,
1776                         PACKETJ(0, 0, 0, PACKETJ_TYPE0));
1777         }
1778         amdgpu_ring_write(ring, val);
1779 }
1780
1781 static void vcn_v1_0_jpeg_ring_nop(struct amdgpu_ring *ring, uint32_t count)
1782 {
1783         int i;
1784
1785         WARN_ON(ring->wptr % 2 || count % 2);
1786
1787         for (i = 0; i < count / 2; i++) {
1788                 amdgpu_ring_write(ring, PACKETJ(0, 0, 0, PACKETJ_TYPE6));
1789                 amdgpu_ring_write(ring, 0);
1790         }
1791 }
1792
1793 static void vcn_v1_0_jpeg_ring_patch_wreg(struct amdgpu_ring *ring, uint32_t *ptr, uint32_t reg_offset, uint32_t val)
1794 {
1795         struct amdgpu_device *adev = ring->adev;
1796         ring->ring[(*ptr)++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0);
1797         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1798                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1799                 ring->ring[(*ptr)++] = 0;
1800                 ring->ring[(*ptr)++] = PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE0);
1801         } else {
1802                 ring->ring[(*ptr)++] = reg_offset;
1803                 ring->ring[(*ptr)++] = PACKETJ(0, 0, 0, PACKETJ_TYPE0);
1804         }
1805         ring->ring[(*ptr)++] = val;
1806 }
1807
1808 static void vcn_v1_0_jpeg_ring_set_patch_ring(struct amdgpu_ring *ring, uint32_t ptr)
1809 {
1810         struct amdgpu_device *adev = ring->adev;
1811
1812         uint32_t reg, reg_offset, val, mask, i;
1813
1814         // 1st: program mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW
1815         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_LOW);
1816         reg_offset = (reg << 2);
1817         val = lower_32_bits(ring->gpu_addr);
1818         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1819
1820         // 2nd: program mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH
1821         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_JRBC_RB_MEM_RD_64BIT_BAR_HIGH);
1822         reg_offset = (reg << 2);
1823         val = upper_32_bits(ring->gpu_addr);
1824         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1825
1826         // 3rd to 5th: issue MEM_READ commands
1827         for (i = 0; i <= 2; i++) {
1828                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE2);
1829                 ring->ring[ptr++] = 0;
1830         }
1831
1832         // 6th: program mmUVD_JRBC_RB_CNTL register to enable NO_FETCH and RPTR write ability
1833         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
1834         reg_offset = (reg << 2);
1835         val = 0x13;
1836         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1837
1838         // 7th: program mmUVD_JRBC_RB_REF_DATA
1839         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA);
1840         reg_offset = (reg << 2);
1841         val = 0x1;
1842         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1843
1844         // 8th: issue conditional register read mmUVD_JRBC_RB_CNTL
1845         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
1846         reg_offset = (reg << 2);
1847         val = 0x1;
1848         mask = 0x1;
1849
1850         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_COND_RD_TIMER), 0, 0, PACKETJ_TYPE0);
1851         ring->ring[ptr++] = 0x01400200;
1852         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_REF_DATA), 0, 0, PACKETJ_TYPE0);
1853         ring->ring[ptr++] = val;
1854         ring->ring[ptr++] = PACKETJ(SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_EXTERNAL_REG_BASE), 0, 0, PACKETJ_TYPE0);
1855         if (((reg_offset >= 0x1f800) && (reg_offset <= 0x21fff)) ||
1856                 ((reg_offset >= 0x1e000) && (reg_offset <= 0x1e1ff))) {
1857                 ring->ring[ptr++] = 0;
1858                 ring->ring[ptr++] = PACKETJ((reg_offset >> 2), 0, 0, PACKETJ_TYPE3);
1859         } else {
1860                 ring->ring[ptr++] = reg_offset;
1861                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE3);
1862         }
1863         ring->ring[ptr++] = mask;
1864
1865         //9th to 21st: insert no-op
1866         for (i = 0; i <= 12; i++) {
1867                 ring->ring[ptr++] = PACKETJ(0, 0, 0, PACKETJ_TYPE6);
1868                 ring->ring[ptr++] = 0;
1869         }
1870
1871         //22nd: reset mmUVD_JRBC_RB_RPTR
1872         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_RPTR);
1873         reg_offset = (reg << 2);
1874         val = 0;
1875         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1876
1877         //23rd: program mmUVD_JRBC_RB_CNTL to disable no_fetch
1878         reg = SOC15_REG_OFFSET(UVD, 0, mmUVD_JRBC_RB_CNTL);
1879         reg_offset = (reg << 2);
1880         val = 0x12;
1881         vcn_v1_0_jpeg_ring_patch_wreg(ring, &ptr, reg_offset, val);
1882 }
1883
1884 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
1885                                         struct amdgpu_irq_src *source,
1886                                         unsigned type,
1887                                         enum amdgpu_interrupt_state state)
1888 {
1889         return 0;
1890 }
1891
1892 static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
1893                                       struct amdgpu_irq_src *source,
1894                                       struct amdgpu_iv_entry *entry)
1895 {
1896         DRM_DEBUG("IH: VCN TRAP\n");
1897
1898         switch (entry->src_id) {
1899         case 124:
1900                 amdgpu_fence_process(&adev->vcn.ring_dec);
1901                 break;
1902         case 119:
1903                 amdgpu_fence_process(&adev->vcn.ring_enc[0]);
1904                 break;
1905         case 120:
1906                 amdgpu_fence_process(&adev->vcn.ring_enc[1]);
1907                 break;
1908         case 126:
1909                 amdgpu_fence_process(&adev->vcn.ring_jpeg);
1910                 break;
1911         default:
1912                 DRM_ERROR("Unhandled interrupt: %d %d\n",
1913                           entry->src_id, entry->src_data[0]);
1914                 break;
1915         }
1916
1917         return 0;
1918 }
1919
1920 static void vcn_v1_0_dec_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
1921 {
1922         struct amdgpu_device *adev = ring->adev;
1923         int i;
1924
1925         WARN_ON(ring->wptr % 2 || count % 2);
1926
1927         for (i = 0; i < count / 2; i++) {
1928                 amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
1929                 amdgpu_ring_write(ring, 0);
1930         }
1931 }
1932
1933 static int vcn_v1_0_set_powergating_state(void *handle,
1934                                           enum amd_powergating_state state)
1935 {
1936         /* This doesn't actually powergate the VCN block.
1937          * That's done in the dpm code via the SMC.  This
1938          * just re-inits the block as necessary.  The actual
1939          * gating still happens in the dpm code.  We should
1940          * revisit this when there is a cleaner line between
1941          * the smc and the hw blocks
1942          */
1943         int ret;
1944         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1945
1946         if(state == adev->vcn.cur_state)
1947                 return 0;
1948
1949         if (state == AMD_PG_STATE_GATE)
1950                 ret = vcn_v1_0_stop(adev);
1951         else
1952                 ret = vcn_v1_0_start(adev);
1953
1954         if(!ret)
1955                 adev->vcn.cur_state = state;
1956         return ret;
1957 }
1958
1959 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
1960         .name = "vcn_v1_0",
1961         .early_init = vcn_v1_0_early_init,
1962         .late_init = NULL,
1963         .sw_init = vcn_v1_0_sw_init,
1964         .sw_fini = vcn_v1_0_sw_fini,
1965         .hw_init = vcn_v1_0_hw_init,
1966         .hw_fini = vcn_v1_0_hw_fini,
1967         .suspend = vcn_v1_0_suspend,
1968         .resume = vcn_v1_0_resume,
1969         .is_idle = vcn_v1_0_is_idle,
1970         .wait_for_idle = vcn_v1_0_wait_for_idle,
1971         .check_soft_reset = NULL /* vcn_v1_0_check_soft_reset */,
1972         .pre_soft_reset = NULL /* vcn_v1_0_pre_soft_reset */,
1973         .soft_reset = NULL /* vcn_v1_0_soft_reset */,
1974         .post_soft_reset = NULL /* vcn_v1_0_post_soft_reset */,
1975         .set_clockgating_state = vcn_v1_0_set_clockgating_state,
1976         .set_powergating_state = vcn_v1_0_set_powergating_state,
1977 };
1978
1979 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
1980         .type = AMDGPU_RING_TYPE_VCN_DEC,
1981         .align_mask = 0xf,
1982         .support_64bit_ptrs = false,
1983         .vmhub = AMDGPU_MMHUB,
1984         .get_rptr = vcn_v1_0_dec_ring_get_rptr,
1985         .get_wptr = vcn_v1_0_dec_ring_get_wptr,
1986         .set_wptr = vcn_v1_0_dec_ring_set_wptr,
1987         .emit_frame_size =
1988                 6 + 6 + /* hdp invalidate / flush */
1989                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
1990                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
1991                 8 + /* vcn_v1_0_dec_ring_emit_vm_flush */
1992                 14 + 14 + /* vcn_v1_0_dec_ring_emit_fence x2 vm fence */
1993                 6,
1994         .emit_ib_size = 8, /* vcn_v1_0_dec_ring_emit_ib */
1995         .emit_ib = vcn_v1_0_dec_ring_emit_ib,
1996         .emit_fence = vcn_v1_0_dec_ring_emit_fence,
1997         .emit_vm_flush = vcn_v1_0_dec_ring_emit_vm_flush,
1998         .test_ring = amdgpu_vcn_dec_ring_test_ring,
1999         .test_ib = amdgpu_vcn_dec_ring_test_ib,
2000         .insert_nop = vcn_v1_0_dec_ring_insert_nop,
2001         .insert_start = vcn_v1_0_dec_ring_insert_start,
2002         .insert_end = vcn_v1_0_dec_ring_insert_end,
2003         .pad_ib = amdgpu_ring_generic_pad_ib,
2004         .begin_use = amdgpu_vcn_ring_begin_use,
2005         .end_use = amdgpu_vcn_ring_end_use,
2006         .emit_wreg = vcn_v1_0_dec_ring_emit_wreg,
2007         .emit_reg_wait = vcn_v1_0_dec_ring_emit_reg_wait,
2008         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2009 };
2010
2011 static const struct amdgpu_ring_funcs vcn_v1_0_enc_ring_vm_funcs = {
2012         .type = AMDGPU_RING_TYPE_VCN_ENC,
2013         .align_mask = 0x3f,
2014         .nop = VCN_ENC_CMD_NO_OP,
2015         .support_64bit_ptrs = false,
2016         .vmhub = AMDGPU_MMHUB,
2017         .get_rptr = vcn_v1_0_enc_ring_get_rptr,
2018         .get_wptr = vcn_v1_0_enc_ring_get_wptr,
2019         .set_wptr = vcn_v1_0_enc_ring_set_wptr,
2020         .emit_frame_size =
2021                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
2022                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
2023                 4 + /* vcn_v1_0_enc_ring_emit_vm_flush */
2024                 5 + 5 + /* vcn_v1_0_enc_ring_emit_fence x2 vm fence */
2025                 1, /* vcn_v1_0_enc_ring_insert_end */
2026         .emit_ib_size = 5, /* vcn_v1_0_enc_ring_emit_ib */
2027         .emit_ib = vcn_v1_0_enc_ring_emit_ib,
2028         .emit_fence = vcn_v1_0_enc_ring_emit_fence,
2029         .emit_vm_flush = vcn_v1_0_enc_ring_emit_vm_flush,
2030         .test_ring = amdgpu_vcn_enc_ring_test_ring,
2031         .test_ib = amdgpu_vcn_enc_ring_test_ib,
2032         .insert_nop = amdgpu_ring_insert_nop,
2033         .insert_end = vcn_v1_0_enc_ring_insert_end,
2034         .pad_ib = amdgpu_ring_generic_pad_ib,
2035         .begin_use = amdgpu_vcn_ring_begin_use,
2036         .end_use = amdgpu_vcn_ring_end_use,
2037         .emit_wreg = vcn_v1_0_enc_ring_emit_wreg,
2038         .emit_reg_wait = vcn_v1_0_enc_ring_emit_reg_wait,
2039         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2040 };
2041
2042 static const struct amdgpu_ring_funcs vcn_v1_0_jpeg_ring_vm_funcs = {
2043         .type = AMDGPU_RING_TYPE_VCN_JPEG,
2044         .align_mask = 0xf,
2045         .nop = PACKET0(0x81ff, 0),
2046         .support_64bit_ptrs = false,
2047         .vmhub = AMDGPU_MMHUB,
2048         .extra_dw = 64,
2049         .get_rptr = vcn_v1_0_jpeg_ring_get_rptr,
2050         .get_wptr = vcn_v1_0_jpeg_ring_get_wptr,
2051         .set_wptr = vcn_v1_0_jpeg_ring_set_wptr,
2052         .emit_frame_size =
2053                 6 + 6 + /* hdp invalidate / flush */
2054                 SOC15_FLUSH_GPU_TLB_NUM_WREG * 6 +
2055                 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 8 +
2056                 8 + /* vcn_v1_0_jpeg_ring_emit_vm_flush */
2057                 26 + 26 + /* vcn_v1_0_jpeg_ring_emit_fence x2 vm fence */
2058                 6,
2059         .emit_ib_size = 22, /* vcn_v1_0_jpeg_ring_emit_ib */
2060         .emit_ib = vcn_v1_0_jpeg_ring_emit_ib,
2061         .emit_fence = vcn_v1_0_jpeg_ring_emit_fence,
2062         .emit_vm_flush = vcn_v1_0_jpeg_ring_emit_vm_flush,
2063         .test_ring = amdgpu_vcn_jpeg_ring_test_ring,
2064         .test_ib = amdgpu_vcn_jpeg_ring_test_ib,
2065         .insert_nop = vcn_v1_0_jpeg_ring_nop,
2066         .insert_start = vcn_v1_0_jpeg_ring_insert_start,
2067         .insert_end = vcn_v1_0_jpeg_ring_insert_end,
2068         .pad_ib = amdgpu_ring_generic_pad_ib,
2069         .begin_use = amdgpu_vcn_ring_begin_use,
2070         .end_use = amdgpu_vcn_ring_end_use,
2071         .emit_wreg = vcn_v1_0_jpeg_ring_emit_wreg,
2072         .emit_reg_wait = vcn_v1_0_jpeg_ring_emit_reg_wait,
2073         .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
2074 };
2075
2076 static void vcn_v1_0_set_dec_ring_funcs(struct amdgpu_device *adev)
2077 {
2078         adev->vcn.ring_dec.funcs = &vcn_v1_0_dec_ring_vm_funcs;
2079         DRM_INFO("VCN decode is enabled in VM mode\n");
2080 }
2081
2082 static void vcn_v1_0_set_enc_ring_funcs(struct amdgpu_device *adev)
2083 {
2084         int i;
2085
2086         for (i = 0; i < adev->vcn.num_enc_rings; ++i)
2087                 adev->vcn.ring_enc[i].funcs = &vcn_v1_0_enc_ring_vm_funcs;
2088
2089         DRM_INFO("VCN encode is enabled in VM mode\n");
2090 }
2091
2092 static void vcn_v1_0_set_jpeg_ring_funcs(struct amdgpu_device *adev)
2093 {
2094         adev->vcn.ring_jpeg.funcs = &vcn_v1_0_jpeg_ring_vm_funcs;
2095         DRM_INFO("VCN jpeg decode is enabled in VM mode\n");
2096 }
2097
2098 static const struct amdgpu_irq_src_funcs vcn_v1_0_irq_funcs = {
2099         .set = vcn_v1_0_set_interrupt_state,
2100         .process = vcn_v1_0_process_interrupt,
2101 };
2102
2103 static void vcn_v1_0_set_irq_funcs(struct amdgpu_device *adev)
2104 {
2105         adev->vcn.irq.num_types = adev->vcn.num_enc_rings + 2;
2106         adev->vcn.irq.funcs = &vcn_v1_0_irq_funcs;
2107 }
2108
2109 const struct amdgpu_ip_block_version vcn_v1_0_ip_block =
2110 {
2111                 .type = AMD_IP_BLOCK_TYPE_VCN,
2112                 .major = 1,
2113                 .minor = 0,
2114                 .rev = 0,
2115                 .funcs = &vcn_v1_0_ip_funcs,
2116 };