drm/i915/gen9: Add WaClearHIZ_WM_CHICKEN3 for bxt and glk
[muen/linux.git] / drivers / gpu / drm / i915 / intel_workarounds.c
1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2018 Intel Corporation
5  */
6
7 #include "i915_drv.h"
8 #include "intel_workarounds.h"
9
10 /**
11  * DOC: Hardware workarounds
12  *
13  * This file is intended as a central place to implement most [1]_ of the
14  * required workarounds for hardware to work as originally intended. They fall
15  * in five basic categories depending on how/when they are applied:
16  *
17  * - Workarounds that touch registers that are saved/restored to/from the HW
18  *   context image. The list is emitted (via Load Register Immediate commands)
19  *   everytime a new context is created.
20  * - GT workarounds. The list of these WAs is applied whenever these registers
21  *   revert to default values (on GPU reset, suspend/resume [2]_, etc..).
22  * - Display workarounds. The list is applied during display clock-gating
23  *   initialization.
24  * - Workarounds that whitelist a privileged register, so that UMDs can manage
25  *   them directly. This is just a special case of a MMMIO workaround (as we
26  *   write the list of these to/be-whitelisted registers to some special HW
27  *   registers).
28  * - Workaround batchbuffers, that get executed automatically by the hardware
29  *   on every HW context restore.
30  *
31  * .. [1] Please notice that there are other WAs that, due to their nature,
32  *    cannot be applied from a central place. Those are peppered around the rest
33  *    of the code, as needed.
34  *
35  * .. [2] Technically, some registers are powercontext saved & restored, so they
36  *    survive a suspend/resume. In practice, writing them again is not too
37  *    costly and simplifies things. We can revisit this in the future.
38  *
39  * Layout
40  * ''''''
41  *
42  * Keep things in this file ordered by WA type, as per the above (context, GT,
43  * display, register whitelist, batchbuffer). Then, inside each type, keep the
44  * following order:
45  *
46  * - Infrastructure functions and macros
47  * - WAs per platform in standard gen/chrono order
48  * - Public functions to init or apply the given workaround type.
49  */
50
51 static int wa_add(struct drm_i915_private *dev_priv,
52                   i915_reg_t addr,
53                   const u32 mask, const u32 val)
54 {
55         const unsigned int idx = dev_priv->workarounds.count;
56
57         if (WARN_ON(idx >= I915_MAX_WA_REGS))
58                 return -ENOSPC;
59
60         dev_priv->workarounds.reg[idx].addr = addr;
61         dev_priv->workarounds.reg[idx].value = val;
62         dev_priv->workarounds.reg[idx].mask = mask;
63
64         dev_priv->workarounds.count++;
65
66         return 0;
67 }
68
69 #define WA_REG(addr, mask, val) do { \
70                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
71                 if (r) \
72                         return r; \
73         } while (0)
74
75 #define WA_SET_BIT_MASKED(addr, mask) \
76         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
77
78 #define WA_CLR_BIT_MASKED(addr, mask) \
79         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
80
81 #define WA_SET_FIELD_MASKED(addr, mask, value) \
82         WA_REG(addr, (mask), _MASKED_FIELD(mask, value))
83
84 static int gen8_ctx_workarounds_init(struct drm_i915_private *dev_priv)
85 {
86         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
87
88         /* WaDisableAsyncFlipPerfMode:bdw,chv */
89         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
90
91         /* WaDisablePartialInstShootdown:bdw,chv */
92         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
93                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
94
95         /* Use Force Non-Coherent whenever executing a 3D context. This is a
96          * workaround for for a possible hang in the unlikely event a TLB
97          * invalidation occurs during a PSD flush.
98          */
99         /* WaForceEnableNonCoherent:bdw,chv */
100         /* WaHdcDisableFetchWhenMasked:bdw,chv */
101         WA_SET_BIT_MASKED(HDC_CHICKEN0,
102                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
103                           HDC_FORCE_NON_COHERENT);
104
105         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
106          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
107          *  polygons in the same 8x4 pixel/sample area to be processed without
108          *  stalling waiting for the earlier ones to write to Hierarchical Z
109          *  buffer."
110          *
111          * This optimization is off by default for BDW and CHV; turn it on.
112          */
113         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
114
115         /* Wa4x4STCOptimizationDisable:bdw,chv */
116         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
117
118         /*
119          * BSpec recommends 8x4 when MSAA is used,
120          * however in practice 16x4 seems fastest.
121          *
122          * Note that PS/WM thread counts depend on the WIZ hashing
123          * disable bit, which we don't touch here, but it's good
124          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
125          */
126         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
127                             GEN6_WIZ_HASHING_MASK,
128                             GEN6_WIZ_HASHING_16x4);
129
130         return 0;
131 }
132
133 static int bdw_ctx_workarounds_init(struct drm_i915_private *dev_priv)
134 {
135         int ret;
136
137         ret = gen8_ctx_workarounds_init(dev_priv);
138         if (ret)
139                 return ret;
140
141         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
142         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
143
144         /* WaDisableDopClockGating:bdw
145          *
146          * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
147          * to disable EUTC clock gating.
148          */
149         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
150                           DOP_CLOCK_GATING_DISABLE);
151
152         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
153                           GEN8_SAMPLER_POWER_BYPASS_DIS);
154
155         WA_SET_BIT_MASKED(HDC_CHICKEN0,
156                           /* WaForceContextSaveRestoreNonCoherent:bdw */
157                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
158                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
159                           (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
160
161         return 0;
162 }
163
164 static int chv_ctx_workarounds_init(struct drm_i915_private *dev_priv)
165 {
166         int ret;
167
168         ret = gen8_ctx_workarounds_init(dev_priv);
169         if (ret)
170                 return ret;
171
172         /* WaDisableThreadStallDopClockGating:chv */
173         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
174
175         /* Improve HiZ throughput on CHV. */
176         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
177
178         return 0;
179 }
180
181 static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv)
182 {
183         if (HAS_LLC(dev_priv)) {
184                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
185                  *
186                  * Must match Display Engine. See
187                  * WaCompressedResourceDisplayNewHashMode.
188                  */
189                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
190                                   GEN9_PBE_COMPRESSED_HASH_SELECTION);
191                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
192                                   GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
193         }
194
195         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
196         /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
197         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
198                           FLOW_CONTROL_ENABLE |
199                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
200
201         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
202         if (!IS_COFFEELAKE(dev_priv))
203                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
204                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
205
206         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
207         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
208         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
209                           GEN9_ENABLE_YV12_BUGFIX |
210                           GEN9_ENABLE_GPGPU_PREEMPTION);
211
212         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
213         /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
214         WA_SET_BIT_MASKED(CACHE_MODE_1,
215                           GEN8_4x4_STC_OPTIMIZATION_DISABLE |
216                           GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE);
217
218         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
219         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
220                           GEN9_CCS_TLB_PREFETCH_ENABLE);
221
222         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
223         WA_SET_BIT_MASKED(HDC_CHICKEN0,
224                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
225                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
226
227         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
228          * both tied to WaForceContextSaveRestoreNonCoherent
229          * in some hsds for skl. We keep the tie for all gen9. The
230          * documentation is a bit hazy and so we want to get common behaviour,
231          * even though there is no clear evidence we would need both on kbl/bxt.
232          * This area has been source of system hangs so we play it safe
233          * and mimic the skl regardless of what bspec says.
234          *
235          * Use Force Non-Coherent whenever executing a 3D context. This
236          * is a workaround for a possible hang in the unlikely event
237          * a TLB invalidation occurs during a PSD flush.
238          */
239
240         /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
241         WA_SET_BIT_MASKED(HDC_CHICKEN0,
242                           HDC_FORCE_NON_COHERENT);
243
244         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
245         if (IS_SKYLAKE(dev_priv) ||
246             IS_KABYLAKE(dev_priv) ||
247             IS_COFFEELAKE(dev_priv))
248                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
249                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
250
251         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
252         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
253
254         /*
255          * Supporting preemption with fine-granularity requires changes in the
256          * batch buffer programming. Since we can't break old userspace, we
257          * need to set our default preemption level to safe value. Userspace is
258          * still able to use more fine-grained preemption levels, since in
259          * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
260          * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
261          * not real HW workarounds, but merely a way to start using preemption
262          * while maintaining old contract with userspace.
263          */
264
265         /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
266         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
267
268         /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
269         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
270                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
271                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
272
273         /* WaClearHIZ_WM_CHICKEN3:bxt,glk */
274         if (IS_GEN9_LP(dev_priv))
275                 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ);
276
277         return 0;
278 }
279
280 static int skl_tune_iz_hashing(struct drm_i915_private *dev_priv)
281 {
282         u8 vals[3] = { 0, 0, 0 };
283         unsigned int i;
284
285         for (i = 0; i < 3; i++) {
286                 u8 ss;
287
288                 /*
289                  * Only consider slices where one, and only one, subslice has 7
290                  * EUs
291                  */
292                 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
293                         continue;
294
295                 /*
296                  * subslice_7eu[i] != 0 (because of the check above) and
297                  * ss_max == 4 (maximum number of subslices possible per slice)
298                  *
299                  * ->    0 <= ss <= 3;
300                  */
301                 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
302                 vals[i] = 3 - ss;
303         }
304
305         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
306                 return 0;
307
308         /* Tune IZ hashing. See intel_device_info_runtime_init() */
309         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
310                             GEN9_IZ_HASHING_MASK(2) |
311                             GEN9_IZ_HASHING_MASK(1) |
312                             GEN9_IZ_HASHING_MASK(0),
313                             GEN9_IZ_HASHING(2, vals[2]) |
314                             GEN9_IZ_HASHING(1, vals[1]) |
315                             GEN9_IZ_HASHING(0, vals[0]));
316
317         return 0;
318 }
319
320 static int skl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
321 {
322         int ret;
323
324         ret = gen9_ctx_workarounds_init(dev_priv);
325         if (ret)
326                 return ret;
327
328         return skl_tune_iz_hashing(dev_priv);
329 }
330
331 static int bxt_ctx_workarounds_init(struct drm_i915_private *dev_priv)
332 {
333         int ret;
334
335         ret = gen9_ctx_workarounds_init(dev_priv);
336         if (ret)
337                 return ret;
338
339         /* WaDisableThreadStallDopClockGating:bxt */
340         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
341                           STALL_DOP_GATING_DISABLE);
342
343         /* WaToEnableHwFixForPushConstHWBug:bxt */
344         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
345                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
346
347         return 0;
348 }
349
350 static int kbl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
351 {
352         int ret;
353
354         ret = gen9_ctx_workarounds_init(dev_priv);
355         if (ret)
356                 return ret;
357
358         /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
359         if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
360                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
361                                   HDC_FENCE_DEST_SLM_DISABLE);
362
363         /* WaToEnableHwFixForPushConstHWBug:kbl */
364         if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
365                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
366                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
367
368         /* WaDisableSbeCacheDispatchPortSharing:kbl */
369         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
370                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
371
372         return 0;
373 }
374
375 static int glk_ctx_workarounds_init(struct drm_i915_private *dev_priv)
376 {
377         int ret;
378
379         ret = gen9_ctx_workarounds_init(dev_priv);
380         if (ret)
381                 return ret;
382
383         /* WaToEnableHwFixForPushConstHWBug:glk */
384         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
385                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
386
387         return 0;
388 }
389
390 static int cfl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
391 {
392         int ret;
393
394         ret = gen9_ctx_workarounds_init(dev_priv);
395         if (ret)
396                 return ret;
397
398         /* WaToEnableHwFixForPushConstHWBug:cfl */
399         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
400                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
401
402         /* WaDisableSbeCacheDispatchPortSharing:cfl */
403         WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1,
404                           GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
405
406         return 0;
407 }
408
409 static int cnl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
410 {
411         /* WaForceContextSaveRestoreNonCoherent:cnl */
412         WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
413                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
414
415         /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
416         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
417                 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
418
419         /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
420         WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
421                           GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
422
423         /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
424         if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
425                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
426                                   GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
427
428         /* WaPushConstantDereferenceHoldDisable:cnl */
429         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
430
431         /* FtrEnableFastAnisoL1BankingFix:cnl */
432         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
433
434         /* WaDisable3DMidCmdPreemption:cnl */
435         WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
436
437         /* WaDisableGPGPUMidCmdPreemption:cnl */
438         WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
439                             GEN9_PREEMPT_GPGPU_LEVEL_MASK,
440                             GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
441
442         /* WaDisableEarlyEOT:cnl */
443         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
444
445         return 0;
446 }
447
448 static int icl_ctx_workarounds_init(struct drm_i915_private *dev_priv)
449 {
450         /* Wa_1604370585:icl (pre-prod)
451          * Formerly known as WaPushConstantDereferenceHoldDisable
452          */
453         if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0))
454                 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
455                                   PUSH_CONSTANT_DEREF_DISABLE);
456
457         /* WaForceEnableNonCoherent:icl
458          * This is not the same workaround as in early Gen9 platforms, where
459          * lacking this could cause system hangs, but coherency performance
460          * overhead is high and only a few compute workloads really need it
461          * (the register is whitelisted in hardware now, so UMDs can opt in
462          * for coherency if they have a good reason).
463          */
464         WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT);
465
466         return 0;
467 }
468
469 int intel_ctx_workarounds_init(struct drm_i915_private *dev_priv)
470 {
471         int err = 0;
472
473         dev_priv->workarounds.count = 0;
474
475         if (INTEL_GEN(dev_priv) < 8)
476                 err = 0;
477         else if (IS_BROADWELL(dev_priv))
478                 err = bdw_ctx_workarounds_init(dev_priv);
479         else if (IS_CHERRYVIEW(dev_priv))
480                 err = chv_ctx_workarounds_init(dev_priv);
481         else if (IS_SKYLAKE(dev_priv))
482                 err = skl_ctx_workarounds_init(dev_priv);
483         else if (IS_BROXTON(dev_priv))
484                 err = bxt_ctx_workarounds_init(dev_priv);
485         else if (IS_KABYLAKE(dev_priv))
486                 err = kbl_ctx_workarounds_init(dev_priv);
487         else if (IS_GEMINILAKE(dev_priv))
488                 err = glk_ctx_workarounds_init(dev_priv);
489         else if (IS_COFFEELAKE(dev_priv))
490                 err = cfl_ctx_workarounds_init(dev_priv);
491         else if (IS_CANNONLAKE(dev_priv))
492                 err = cnl_ctx_workarounds_init(dev_priv);
493         else if (IS_ICELAKE(dev_priv))
494                 err = icl_ctx_workarounds_init(dev_priv);
495         else
496                 MISSING_CASE(INTEL_GEN(dev_priv));
497         if (err)
498                 return err;
499
500         DRM_DEBUG_DRIVER("Number of context specific w/a: %d\n",
501                          dev_priv->workarounds.count);
502         return 0;
503 }
504
505 int intel_ctx_workarounds_emit(struct i915_request *rq)
506 {
507         struct i915_workarounds *w = &rq->i915->workarounds;
508         u32 *cs;
509         int ret, i;
510
511         if (w->count == 0)
512                 return 0;
513
514         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
515         if (ret)
516                 return ret;
517
518         cs = intel_ring_begin(rq, (w->count * 2 + 2));
519         if (IS_ERR(cs))
520                 return PTR_ERR(cs);
521
522         *cs++ = MI_LOAD_REGISTER_IMM(w->count);
523         for (i = 0; i < w->count; i++) {
524                 *cs++ = i915_mmio_reg_offset(w->reg[i].addr);
525                 *cs++ = w->reg[i].value;
526         }
527         *cs++ = MI_NOOP;
528
529         intel_ring_advance(rq, cs);
530
531         ret = rq->engine->emit_flush(rq, EMIT_BARRIER);
532         if (ret)
533                 return ret;
534
535         return 0;
536 }
537
538 static void bdw_gt_workarounds_apply(struct drm_i915_private *dev_priv)
539 {
540 }
541
542 static void chv_gt_workarounds_apply(struct drm_i915_private *dev_priv)
543 {
544 }
545
546 static void gen9_gt_workarounds_apply(struct drm_i915_private *dev_priv)
547 {
548         /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
549         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS,
550                    _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
551
552         /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
553         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
554                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
555
556         /* WaDisableKillLogic:bxt,skl,kbl */
557         if (!IS_COFFEELAKE(dev_priv))
558                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
559                            ECOCHK_DIS_TLB);
560
561         if (HAS_LLC(dev_priv)) {
562                 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
563                  *
564                  * Must match Display Engine. See
565                  * WaCompressedResourceDisplayNewHashMode.
566                  */
567                 I915_WRITE(MMCD_MISC_CTRL,
568                            I915_READ(MMCD_MISC_CTRL) |
569                            MMCD_PCLA |
570                            MMCD_HOTSPOT_EN);
571         }
572
573         /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
574         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
575                    BDW_DISABLE_HDC_INVALIDATION);
576
577         /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
578         if (IS_GEN9_LP(dev_priv)) {
579                 u32 val = I915_READ(GEN8_L3SQCREG1);
580
581                 val &= ~L3_PRIO_CREDITS_MASK;
582                 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
583                 I915_WRITE(GEN8_L3SQCREG1, val);
584         }
585
586         /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
587         I915_WRITE(GEN8_L3SQCREG4,
588                    I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_FLUSH_COHERENT_LINES);
589
590         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
591         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
592                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
593 }
594
595 static void skl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
596 {
597         gen9_gt_workarounds_apply(dev_priv);
598
599         /* WaEnableGapsTsvCreditFix:skl */
600         I915_WRITE(GEN8_GARBCNTL,
601                    I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
602
603         /* WaDisableGafsUnitClkGating:skl */
604         I915_WRITE(GEN7_UCGCTL4,
605                    I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
606
607         /* WaInPlaceDecompressionHang:skl */
608         if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
609                 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
610                            I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
611                            GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
612 }
613
614 static void bxt_gt_workarounds_apply(struct drm_i915_private *dev_priv)
615 {
616         gen9_gt_workarounds_apply(dev_priv);
617
618         /* WaDisablePooledEuLoadBalancingFix:bxt */
619         I915_WRITE(FF_SLICE_CS_CHICKEN2,
620                    _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
621
622         /* WaInPlaceDecompressionHang:bxt */
623         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
624                    I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
625                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
626 }
627
628 static void kbl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
629 {
630         gen9_gt_workarounds_apply(dev_priv);
631
632         /* WaEnableGapsTsvCreditFix:kbl */
633         I915_WRITE(GEN8_GARBCNTL,
634                    I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
635
636         /* WaDisableDynamicCreditSharing:kbl */
637         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
638                 I915_WRITE(GAMT_CHKN_BIT_REG,
639                            I915_READ(GAMT_CHKN_BIT_REG) |
640                            GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
641
642         /* WaDisableGafsUnitClkGating:kbl */
643         I915_WRITE(GEN7_UCGCTL4,
644                    I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
645
646         /* WaInPlaceDecompressionHang:kbl */
647         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
648                    I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
649                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
650 }
651
652 static void glk_gt_workarounds_apply(struct drm_i915_private *dev_priv)
653 {
654         gen9_gt_workarounds_apply(dev_priv);
655 }
656
657 static void cfl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
658 {
659         gen9_gt_workarounds_apply(dev_priv);
660
661         /* WaEnableGapsTsvCreditFix:cfl */
662         I915_WRITE(GEN8_GARBCNTL,
663                    I915_READ(GEN8_GARBCNTL) | GEN9_GAPS_TSV_CREDIT_DISABLE);
664
665         /* WaDisableGafsUnitClkGating:cfl */
666         I915_WRITE(GEN7_UCGCTL4,
667                    I915_READ(GEN7_UCGCTL4) | GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
668
669         /* WaInPlaceDecompressionHang:cfl */
670         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
671                    I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
672                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
673 }
674
675 static void cnl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
676 {
677         /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
678         if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
679                 I915_WRITE(GAMT_CHKN_BIT_REG,
680                            I915_READ(GAMT_CHKN_BIT_REG) |
681                            GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT);
682
683         /* WaInPlaceDecompressionHang:cnl */
684         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
685                    I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
686                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
687
688         /* WaEnablePreemptionGranularityControlByUMD:cnl */
689         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
690                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
691 }
692
693 static void icl_gt_workarounds_apply(struct drm_i915_private *dev_priv)
694 {
695         /* This is not an Wa. Enable for better image quality */
696         I915_WRITE(_3D_CHICKEN3,
697                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE));
698
699         /* WaInPlaceDecompressionHang:icl */
700         I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA, I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
701                                             GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
702
703         /* WaPipelineFlushCoherentLines:icl */
704         I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
705                                    GEN8_LQSC_FLUSH_COHERENT_LINES);
706
707         /* Wa_1405543622:icl
708          * Formerly known as WaGAPZPriorityScheme
709          */
710         I915_WRITE(GEN8_GARBCNTL, I915_READ(GEN8_GARBCNTL) |
711                                   GEN11_ARBITRATION_PRIO_ORDER_MASK);
712
713         /* Wa_1604223664:icl
714          * Formerly known as WaL3BankAddressHashing
715          */
716         I915_WRITE(GEN8_GARBCNTL,
717                    (I915_READ(GEN8_GARBCNTL) & ~GEN11_HASH_CTRL_EXCL_MASK) |
718                    GEN11_HASH_CTRL_EXCL_BIT0);
719         I915_WRITE(GEN11_GLBLINVL,
720                    (I915_READ(GEN11_GLBLINVL) & ~GEN11_BANK_HASH_ADDR_EXCL_MASK) |
721                    GEN11_BANK_HASH_ADDR_EXCL_BIT0);
722
723         /* WaModifyGamTlbPartitioning:icl */
724         I915_WRITE(GEN11_GACB_PERF_CTRL,
725                    (I915_READ(GEN11_GACB_PERF_CTRL) & ~GEN11_HASH_CTRL_MASK) |
726                    GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4);
727
728         /* Wa_1405733216:icl
729          * Formerly known as WaDisableCleanEvicts
730          */
731         I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
732                                    GEN11_LQSC_CLEAN_EVICT_DISABLE);
733
734         /* Wa_1405766107:icl
735          * Formerly known as WaCL2SFHalfMaxAlloc
736          */
737         I915_WRITE(GEN11_LSN_UNSLCVC, I915_READ(GEN11_LSN_UNSLCVC) |
738                                       GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC |
739                                       GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC);
740
741         /* Wa_220166154:icl
742          * Formerly known as WaDisCtxReload
743          */
744         I915_WRITE(GAMW_ECO_DEV_RW_IA_REG, I915_READ(GAMW_ECO_DEV_RW_IA_REG) |
745                                            GAMW_ECO_DEV_CTX_RELOAD_DISABLE);
746
747         /* Wa_1405779004:icl (pre-prod) */
748         if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_A0))
749                 I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE,
750                            I915_READ(SLICE_UNIT_LEVEL_CLKGATE) |
751                            MSCUNIT_CLKGATE_DIS);
752
753         /* Wa_1406680159:icl */
754         I915_WRITE(SUBSLICE_UNIT_LEVEL_CLKGATE,
755                    I915_READ(SUBSLICE_UNIT_LEVEL_CLKGATE) |
756                    GWUNIT_CLKGATE_DIS);
757
758         /* Wa_1604302699:icl */
759         I915_WRITE(GEN10_L3_CHICKEN_MODE_REGISTER,
760                    I915_READ(GEN10_L3_CHICKEN_MODE_REGISTER) |
761                    GEN11_I2M_WRITE_DISABLE);
762
763         /* Wa_1406838659:icl (pre-prod) */
764         if (IS_ICL_REVID(dev_priv, ICL_REVID_A0, ICL_REVID_B0))
765                 I915_WRITE(INF_UNIT_LEVEL_CLKGATE,
766                            I915_READ(INF_UNIT_LEVEL_CLKGATE) |
767                            CGPSF_CLKGATE_DIS);
768
769         /* WaForwardProgressSoftReset:icl */
770         I915_WRITE(GEN10_SCRATCH_LNCF2,
771                    I915_READ(GEN10_SCRATCH_LNCF2) |
772                    PMFLUSHDONE_LNICRSDROP |
773                    PMFLUSH_GAPL3UNBLOCK |
774                    PMFLUSHDONE_LNEBLK);
775 }
776
777 void intel_gt_workarounds_apply(struct drm_i915_private *dev_priv)
778 {
779         if (INTEL_GEN(dev_priv) < 8)
780                 return;
781         else if (IS_BROADWELL(dev_priv))
782                 bdw_gt_workarounds_apply(dev_priv);
783         else if (IS_CHERRYVIEW(dev_priv))
784                 chv_gt_workarounds_apply(dev_priv);
785         else if (IS_SKYLAKE(dev_priv))
786                 skl_gt_workarounds_apply(dev_priv);
787         else if (IS_BROXTON(dev_priv))
788                 bxt_gt_workarounds_apply(dev_priv);
789         else if (IS_KABYLAKE(dev_priv))
790                 kbl_gt_workarounds_apply(dev_priv);
791         else if (IS_GEMINILAKE(dev_priv))
792                 glk_gt_workarounds_apply(dev_priv);
793         else if (IS_COFFEELAKE(dev_priv))
794                 cfl_gt_workarounds_apply(dev_priv);
795         else if (IS_CANNONLAKE(dev_priv))
796                 cnl_gt_workarounds_apply(dev_priv);
797         else if (IS_ICELAKE(dev_priv))
798                 icl_gt_workarounds_apply(dev_priv);
799         else
800                 MISSING_CASE(INTEL_GEN(dev_priv));
801 }
802
803 struct whitelist {
804         i915_reg_t reg[RING_MAX_NONPRIV_SLOTS];
805         unsigned int count;
806         u32 nopid;
807 };
808
809 static void whitelist_reg(struct whitelist *w, i915_reg_t reg)
810 {
811         if (GEM_WARN_ON(w->count >= RING_MAX_NONPRIV_SLOTS))
812                 return;
813
814         w->reg[w->count++] = reg;
815 }
816
817 static void bdw_whitelist_build(struct whitelist *w)
818 {
819 }
820
821 static void chv_whitelist_build(struct whitelist *w)
822 {
823 }
824
825 static void gen9_whitelist_build(struct whitelist *w)
826 {
827         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
828         whitelist_reg(w, GEN9_CTX_PREEMPT_REG);
829
830         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
831         whitelist_reg(w, GEN8_CS_CHICKEN1);
832
833         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
834         whitelist_reg(w, GEN8_HDC_CHICKEN1);
835 }
836
837 static void skl_whitelist_build(struct whitelist *w)
838 {
839         gen9_whitelist_build(w);
840
841         /* WaDisableLSQCROPERFforOCL:skl */
842         whitelist_reg(w, GEN8_L3SQCREG4);
843 }
844
845 static void bxt_whitelist_build(struct whitelist *w)
846 {
847         gen9_whitelist_build(w);
848 }
849
850 static void kbl_whitelist_build(struct whitelist *w)
851 {
852         gen9_whitelist_build(w);
853
854         /* WaDisableLSQCROPERFforOCL:kbl */
855         whitelist_reg(w, GEN8_L3SQCREG4);
856 }
857
858 static void glk_whitelist_build(struct whitelist *w)
859 {
860         gen9_whitelist_build(w);
861
862         /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */
863         whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
864 }
865
866 static void cfl_whitelist_build(struct whitelist *w)
867 {
868         gen9_whitelist_build(w);
869 }
870
871 static void cnl_whitelist_build(struct whitelist *w)
872 {
873         /* WaEnablePreemptionGranularityControlByUMD:cnl */
874         whitelist_reg(w, GEN8_CS_CHICKEN1);
875 }
876
877 static void icl_whitelist_build(struct whitelist *w)
878 {
879 }
880
881 static struct whitelist *whitelist_build(struct intel_engine_cs *engine,
882                                          struct whitelist *w)
883 {
884         struct drm_i915_private *i915 = engine->i915;
885
886         GEM_BUG_ON(engine->id != RCS);
887
888         w->count = 0;
889         w->nopid = i915_mmio_reg_offset(RING_NOPID(engine->mmio_base));
890
891         if (INTEL_GEN(i915) < 8)
892                 return NULL;
893         else if (IS_BROADWELL(i915))
894                 bdw_whitelist_build(w);
895         else if (IS_CHERRYVIEW(i915))
896                 chv_whitelist_build(w);
897         else if (IS_SKYLAKE(i915))
898                 skl_whitelist_build(w);
899         else if (IS_BROXTON(i915))
900                 bxt_whitelist_build(w);
901         else if (IS_KABYLAKE(i915))
902                 kbl_whitelist_build(w);
903         else if (IS_GEMINILAKE(i915))
904                 glk_whitelist_build(w);
905         else if (IS_COFFEELAKE(i915))
906                 cfl_whitelist_build(w);
907         else if (IS_CANNONLAKE(i915))
908                 cnl_whitelist_build(w);
909         else if (IS_ICELAKE(i915))
910                 icl_whitelist_build(w);
911         else
912                 MISSING_CASE(INTEL_GEN(i915));
913
914         return w;
915 }
916
917 static void whitelist_apply(struct intel_engine_cs *engine,
918                             const struct whitelist *w)
919 {
920         struct drm_i915_private *dev_priv = engine->i915;
921         const u32 base = engine->mmio_base;
922         unsigned int i;
923
924         if (!w)
925                 return;
926
927         intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
928
929         for (i = 0; i < w->count; i++)
930                 I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i),
931                               i915_mmio_reg_offset(w->reg[i]));
932
933         /* And clear the rest just in case of garbage */
934         for (; i < RING_MAX_NONPRIV_SLOTS; i++)
935                 I915_WRITE_FW(RING_FORCE_TO_NONPRIV(base, i), w->nopid);
936
937         intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
938 }
939
940 void intel_whitelist_workarounds_apply(struct intel_engine_cs *engine)
941 {
942         struct whitelist w;
943
944         whitelist_apply(engine, whitelist_build(engine, &w));
945 }
946
947 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
948 #include "selftests/intel_workarounds.c"
949 #endif