Merge tag 'drm-intel-next-2017-09-07' of git://anongit.freedesktop.org/git/drm-intel...
[muen/linux.git] / drivers / gpu / drm / i915 / i915_drv.h
1 /* i915_drv.h -- Private header for the I915 driver -*- linux-c -*-
2  */
3 /*
4  *
5  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
6  * All Rights Reserved.
7  *
8  * Permission is hereby granted, free of charge, to any person obtaining a
9  * copy of this software and associated documentation files (the
10  * "Software"), to deal in the Software without restriction, including
11  * without limitation the rights to use, copy, modify, merge, publish,
12  * distribute, sub license, and/or sell copies of the Software, and to
13  * permit persons to whom the Software is furnished to do so, subject to
14  * the following conditions:
15  *
16  * The above copyright notice and this permission notice (including the
17  * next paragraph) shall be included in all copies or substantial portions
18  * of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
23  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
24  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
25  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
26  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
27  *
28  */
29
30 #ifndef _I915_DRV_H_
31 #define _I915_DRV_H_
32
33 #include <uapi/drm/i915_drm.h>
34 #include <uapi/drm/drm_fourcc.h>
35
36 #include <linux/io-mapping.h>
37 #include <linux/i2c.h>
38 #include <linux/i2c-algo-bit.h>
39 #include <linux/backlight.h>
40 #include <linux/hash.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/kref.h>
43 #include <linux/pm_qos.h>
44 #include <linux/reservation.h>
45 #include <linux/shmem_fs.h>
46
47 #include <drm/drmP.h>
48 #include <drm/intel-gtt.h>
49 #include <drm/drm_legacy.h> /* for struct drm_dma_handle */
50 #include <drm/drm_gem.h>
51 #include <drm/drm_auth.h>
52 #include <drm/drm_cache.h>
53
54 #include "i915_params.h"
55 #include "i915_reg.h"
56 #include "i915_utils.h"
57
58 #include "intel_uncore.h"
59 #include "intel_bios.h"
60 #include "intel_dpll_mgr.h"
61 #include "intel_uc.h"
62 #include "intel_lrc.h"
63 #include "intel_ringbuffer.h"
64
65 #include "i915_gem.h"
66 #include "i915_gem_context.h"
67 #include "i915_gem_fence_reg.h"
68 #include "i915_gem_object.h"
69 #include "i915_gem_gtt.h"
70 #include "i915_gem_render_state.h"
71 #include "i915_gem_request.h"
72 #include "i915_gem_timeline.h"
73
74 #include "i915_vma.h"
75
76 #include "intel_gvt.h"
77
78 /* General customization:
79  */
80
81 #define DRIVER_NAME             "i915"
82 #define DRIVER_DESC             "Intel Graphics"
83 #define DRIVER_DATE             "20170907"
84 #define DRIVER_TIMESTAMP        1504772900
85
86 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
87  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
88  * which may not necessarily be a user visible problem.  This will either
89  * WARN() or DRM_ERROR() depending on the verbose_checks moduleparam, to
90  * enable distros and users to tailor their preferred amount of i915 abrt
91  * spam.
92  */
93 #define I915_STATE_WARN(condition, format...) ({                        \
94         int __ret_warn_on = !!(condition);                              \
95         if (unlikely(__ret_warn_on))                                    \
96                 if (!WARN(i915.verbose_state_checks, format))           \
97                         DRM_ERROR(format);                              \
98         unlikely(__ret_warn_on);                                        \
99 })
100
101 #define I915_STATE_WARN_ON(x)                                           \
102         I915_STATE_WARN((x), "%s", "WARN_ON(" __stringify(x) ")")
103
104 bool __i915_inject_load_failure(const char *func, int line);
105 #define i915_inject_load_failure() \
106         __i915_inject_load_failure(__func__, __LINE__)
107
108 typedef struct {
109         uint32_t val;
110 } uint_fixed_16_16_t;
111
112 #define FP_16_16_MAX ({ \
113         uint_fixed_16_16_t fp; \
114         fp.val = UINT_MAX; \
115         fp; \
116 })
117
118 static inline bool is_fixed16_zero(uint_fixed_16_16_t val)
119 {
120         if (val.val == 0)
121                 return true;
122         return false;
123 }
124
125 static inline uint_fixed_16_16_t u32_to_fixed16(uint32_t val)
126 {
127         uint_fixed_16_16_t fp;
128
129         WARN_ON(val >> 16);
130
131         fp.val = val << 16;
132         return fp;
133 }
134
135 static inline uint32_t fixed16_to_u32_round_up(uint_fixed_16_16_t fp)
136 {
137         return DIV_ROUND_UP(fp.val, 1 << 16);
138 }
139
140 static inline uint32_t fixed16_to_u32(uint_fixed_16_16_t fp)
141 {
142         return fp.val >> 16;
143 }
144
145 static inline uint_fixed_16_16_t min_fixed16(uint_fixed_16_16_t min1,
146                                                  uint_fixed_16_16_t min2)
147 {
148         uint_fixed_16_16_t min;
149
150         min.val = min(min1.val, min2.val);
151         return min;
152 }
153
154 static inline uint_fixed_16_16_t max_fixed16(uint_fixed_16_16_t max1,
155                                                  uint_fixed_16_16_t max2)
156 {
157         uint_fixed_16_16_t max;
158
159         max.val = max(max1.val, max2.val);
160         return max;
161 }
162
163 static inline uint_fixed_16_16_t clamp_u64_to_fixed16(uint64_t val)
164 {
165         uint_fixed_16_16_t fp;
166         WARN_ON(val >> 32);
167         fp.val = clamp_t(uint32_t, val, 0, ~0);
168         return fp;
169 }
170
171 static inline uint32_t div_round_up_fixed16(uint_fixed_16_16_t val,
172                                             uint_fixed_16_16_t d)
173 {
174         return DIV_ROUND_UP(val.val, d.val);
175 }
176
177 static inline uint32_t mul_round_up_u32_fixed16(uint32_t val,
178                                                 uint_fixed_16_16_t mul)
179 {
180         uint64_t intermediate_val;
181
182         intermediate_val = (uint64_t) val * mul.val;
183         intermediate_val = DIV_ROUND_UP_ULL(intermediate_val, 1 << 16);
184         WARN_ON(intermediate_val >> 32);
185         return clamp_t(uint32_t, intermediate_val, 0, ~0);
186 }
187
188 static inline uint_fixed_16_16_t mul_fixed16(uint_fixed_16_16_t val,
189                                              uint_fixed_16_16_t mul)
190 {
191         uint64_t intermediate_val;
192
193         intermediate_val = (uint64_t) val.val * mul.val;
194         intermediate_val = intermediate_val >> 16;
195         return clamp_u64_to_fixed16(intermediate_val);
196 }
197
198 static inline uint_fixed_16_16_t div_fixed16(uint32_t val, uint32_t d)
199 {
200         uint64_t interm_val;
201
202         interm_val = (uint64_t)val << 16;
203         interm_val = DIV_ROUND_UP_ULL(interm_val, d);
204         return clamp_u64_to_fixed16(interm_val);
205 }
206
207 static inline uint32_t div_round_up_u32_fixed16(uint32_t val,
208                                                 uint_fixed_16_16_t d)
209 {
210         uint64_t interm_val;
211
212         interm_val = (uint64_t)val << 16;
213         interm_val = DIV_ROUND_UP_ULL(interm_val, d.val);
214         WARN_ON(interm_val >> 32);
215         return clamp_t(uint32_t, interm_val, 0, ~0);
216 }
217
218 static inline uint_fixed_16_16_t mul_u32_fixed16(uint32_t val,
219                                                      uint_fixed_16_16_t mul)
220 {
221         uint64_t intermediate_val;
222
223         intermediate_val = (uint64_t) val * mul.val;
224         return clamp_u64_to_fixed16(intermediate_val);
225 }
226
227 static inline uint_fixed_16_16_t add_fixed16(uint_fixed_16_16_t add1,
228                                              uint_fixed_16_16_t add2)
229 {
230         uint64_t interm_sum;
231
232         interm_sum = (uint64_t) add1.val + add2.val;
233         return clamp_u64_to_fixed16(interm_sum);
234 }
235
236 static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1,
237                                                  uint32_t add2)
238 {
239         uint64_t interm_sum;
240         uint_fixed_16_16_t interm_add2 = u32_to_fixed16(add2);
241
242         interm_sum = (uint64_t) add1.val + interm_add2.val;
243         return clamp_u64_to_fixed16(interm_sum);
244 }
245
246 static inline const char *yesno(bool v)
247 {
248         return v ? "yes" : "no";
249 }
250
251 static inline const char *onoff(bool v)
252 {
253         return v ? "on" : "off";
254 }
255
256 static inline const char *enableddisabled(bool v)
257 {
258         return v ? "enabled" : "disabled";
259 }
260
261 enum pipe {
262         INVALID_PIPE = -1,
263         PIPE_A = 0,
264         PIPE_B,
265         PIPE_C,
266         _PIPE_EDP,
267         I915_MAX_PIPES = _PIPE_EDP
268 };
269 #define pipe_name(p) ((p) + 'A')
270
271 enum transcoder {
272         TRANSCODER_A = 0,
273         TRANSCODER_B,
274         TRANSCODER_C,
275         TRANSCODER_EDP,
276         TRANSCODER_DSI_A,
277         TRANSCODER_DSI_C,
278         I915_MAX_TRANSCODERS
279 };
280
281 static inline const char *transcoder_name(enum transcoder transcoder)
282 {
283         switch (transcoder) {
284         case TRANSCODER_A:
285                 return "A";
286         case TRANSCODER_B:
287                 return "B";
288         case TRANSCODER_C:
289                 return "C";
290         case TRANSCODER_EDP:
291                 return "EDP";
292         case TRANSCODER_DSI_A:
293                 return "DSI A";
294         case TRANSCODER_DSI_C:
295                 return "DSI C";
296         default:
297                 return "<invalid>";
298         }
299 }
300
301 static inline bool transcoder_is_dsi(enum transcoder transcoder)
302 {
303         return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C;
304 }
305
306 /*
307  * Global legacy plane identifier. Valid only for primary/sprite
308  * planes on pre-g4x, and only for primary planes on g4x+.
309  */
310 enum plane {
311         PLANE_A,
312         PLANE_B,
313         PLANE_C,
314 };
315 #define plane_name(p) ((p) + 'A')
316
317 #define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
318
319 /*
320  * Per-pipe plane identifier.
321  * I915_MAX_PLANES in the enum below is the maximum (across all platforms)
322  * number of planes per CRTC.  Not all platforms really have this many planes,
323  * which means some arrays of size I915_MAX_PLANES may have unused entries
324  * between the topmost sprite plane and the cursor plane.
325  *
326  * This is expected to be passed to various register macros
327  * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care.
328  */
329 enum plane_id {
330         PLANE_PRIMARY,
331         PLANE_SPRITE0,
332         PLANE_SPRITE1,
333         PLANE_SPRITE2,
334         PLANE_CURSOR,
335         I915_MAX_PLANES,
336 };
337
338 #define for_each_plane_id_on_crtc(__crtc, __p) \
339         for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \
340                 for_each_if ((__crtc)->plane_ids_mask & BIT(__p))
341
342 enum port {
343         PORT_NONE = -1,
344         PORT_A = 0,
345         PORT_B,
346         PORT_C,
347         PORT_D,
348         PORT_E,
349         I915_MAX_PORTS
350 };
351 #define port_name(p) ((p) + 'A')
352
353 #define I915_NUM_PHYS_VLV 2
354
355 enum dpio_channel {
356         DPIO_CH0,
357         DPIO_CH1
358 };
359
360 enum dpio_phy {
361         DPIO_PHY0,
362         DPIO_PHY1,
363         DPIO_PHY2,
364 };
365
366 enum intel_display_power_domain {
367         POWER_DOMAIN_PIPE_A,
368         POWER_DOMAIN_PIPE_B,
369         POWER_DOMAIN_PIPE_C,
370         POWER_DOMAIN_PIPE_A_PANEL_FITTER,
371         POWER_DOMAIN_PIPE_B_PANEL_FITTER,
372         POWER_DOMAIN_PIPE_C_PANEL_FITTER,
373         POWER_DOMAIN_TRANSCODER_A,
374         POWER_DOMAIN_TRANSCODER_B,
375         POWER_DOMAIN_TRANSCODER_C,
376         POWER_DOMAIN_TRANSCODER_EDP,
377         POWER_DOMAIN_TRANSCODER_DSI_A,
378         POWER_DOMAIN_TRANSCODER_DSI_C,
379         POWER_DOMAIN_PORT_DDI_A_LANES,
380         POWER_DOMAIN_PORT_DDI_B_LANES,
381         POWER_DOMAIN_PORT_DDI_C_LANES,
382         POWER_DOMAIN_PORT_DDI_D_LANES,
383         POWER_DOMAIN_PORT_DDI_E_LANES,
384         POWER_DOMAIN_PORT_DDI_A_IO,
385         POWER_DOMAIN_PORT_DDI_B_IO,
386         POWER_DOMAIN_PORT_DDI_C_IO,
387         POWER_DOMAIN_PORT_DDI_D_IO,
388         POWER_DOMAIN_PORT_DDI_E_IO,
389         POWER_DOMAIN_PORT_DSI,
390         POWER_DOMAIN_PORT_CRT,
391         POWER_DOMAIN_PORT_OTHER,
392         POWER_DOMAIN_VGA,
393         POWER_DOMAIN_AUDIO,
394         POWER_DOMAIN_PLLS,
395         POWER_DOMAIN_AUX_A,
396         POWER_DOMAIN_AUX_B,
397         POWER_DOMAIN_AUX_C,
398         POWER_DOMAIN_AUX_D,
399         POWER_DOMAIN_GMBUS,
400         POWER_DOMAIN_MODESET,
401         POWER_DOMAIN_INIT,
402
403         POWER_DOMAIN_NUM,
404 };
405
406 #define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A)
407 #define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \
408                 ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER)
409 #define POWER_DOMAIN_TRANSCODER(tran) \
410         ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \
411          (tran) + POWER_DOMAIN_TRANSCODER_A)
412
413 enum hpd_pin {
414         HPD_NONE = 0,
415         HPD_TV = HPD_NONE,     /* TV is known to be unreliable */
416         HPD_CRT,
417         HPD_SDVO_B,
418         HPD_SDVO_C,
419         HPD_PORT_A,
420         HPD_PORT_B,
421         HPD_PORT_C,
422         HPD_PORT_D,
423         HPD_PORT_E,
424         HPD_NUM_PINS
425 };
426
427 #define for_each_hpd_pin(__pin) \
428         for ((__pin) = (HPD_NONE + 1); (__pin) < HPD_NUM_PINS; (__pin)++)
429
430 #define HPD_STORM_DEFAULT_THRESHOLD 5
431
432 struct i915_hotplug {
433         struct work_struct hotplug_work;
434
435         struct {
436                 unsigned long last_jiffies;
437                 int count;
438                 enum {
439                         HPD_ENABLED = 0,
440                         HPD_DISABLED = 1,
441                         HPD_MARK_DISABLED = 2
442                 } state;
443         } stats[HPD_NUM_PINS];
444         u32 event_bits;
445         struct delayed_work reenable_work;
446
447         struct intel_digital_port *irq_port[I915_MAX_PORTS];
448         u32 long_port_mask;
449         u32 short_port_mask;
450         struct work_struct dig_port_work;
451
452         struct work_struct poll_init_work;
453         bool poll_enabled;
454
455         unsigned int hpd_storm_threshold;
456
457         /*
458          * if we get a HPD irq from DP and a HPD irq from non-DP
459          * the non-DP HPD could block the workqueue on a mode config
460          * mutex getting, that userspace may have taken. However
461          * userspace is waiting on the DP workqueue to run which is
462          * blocked behind the non-DP one.
463          */
464         struct workqueue_struct *dp_wq;
465 };
466
467 #define I915_GEM_GPU_DOMAINS \
468         (I915_GEM_DOMAIN_RENDER | \
469          I915_GEM_DOMAIN_SAMPLER | \
470          I915_GEM_DOMAIN_COMMAND | \
471          I915_GEM_DOMAIN_INSTRUCTION | \
472          I915_GEM_DOMAIN_VERTEX)
473
474 #define for_each_pipe(__dev_priv, __p) \
475         for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++)
476 #define for_each_pipe_masked(__dev_priv, __p, __mask) \
477         for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \
478                 for_each_if ((__mask) & (1 << (__p)))
479 #define for_each_universal_plane(__dev_priv, __pipe, __p)               \
480         for ((__p) = 0;                                                 \
481              (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \
482              (__p)++)
483 #define for_each_sprite(__dev_priv, __p, __s)                           \
484         for ((__s) = 0;                                                 \
485              (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)];        \
486              (__s)++)
487
488 #define for_each_port_masked(__port, __ports_mask) \
489         for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++)  \
490                 for_each_if ((__ports_mask) & (1 << (__port)))
491
492 #define for_each_crtc(dev, crtc) \
493         list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head)
494
495 #define for_each_intel_plane(dev, intel_plane) \
496         list_for_each_entry(intel_plane,                        \
497                             &(dev)->mode_config.plane_list,     \
498                             base.head)
499
500 #define for_each_intel_plane_mask(dev, intel_plane, plane_mask)         \
501         list_for_each_entry(intel_plane,                                \
502                             &(dev)->mode_config.plane_list,             \
503                             base.head)                                  \
504                 for_each_if ((plane_mask) &                             \
505                              (1 << drm_plane_index(&intel_plane->base)))
506
507 #define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane)      \
508         list_for_each_entry(intel_plane,                                \
509                             &(dev)->mode_config.plane_list,             \
510                             base.head)                                  \
511                 for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe)
512
513 #define for_each_intel_crtc(dev, intel_crtc)                            \
514         list_for_each_entry(intel_crtc,                                 \
515                             &(dev)->mode_config.crtc_list,              \
516                             base.head)
517
518 #define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask)            \
519         list_for_each_entry(intel_crtc,                                 \
520                             &(dev)->mode_config.crtc_list,              \
521                             base.head)                                  \
522                 for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base)))
523
524 #define for_each_intel_encoder(dev, intel_encoder)              \
525         list_for_each_entry(intel_encoder,                      \
526                             &(dev)->mode_config.encoder_list,   \
527                             base.head)
528
529 #define for_each_intel_connector_iter(intel_connector, iter) \
530         while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter))))
531
532 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
533         list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
534                 for_each_if ((intel_encoder)->base.crtc == (__crtc))
535
536 #define for_each_connector_on_encoder(dev, __encoder, intel_connector) \
537         list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \
538                 for_each_if ((intel_connector)->base.encoder == (__encoder))
539
540 #define for_each_power_domain(domain, mask)                             \
541         for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
542                 for_each_if (BIT_ULL(domain) & (mask))
543
544 #define for_each_power_well(__dev_priv, __power_well)                           \
545         for ((__power_well) = (__dev_priv)->power_domains.power_wells;  \
546              (__power_well) - (__dev_priv)->power_domains.power_wells < \
547                 (__dev_priv)->power_domains.power_well_count;           \
548              (__power_well)++)
549
550 #define for_each_power_well_rev(__dev_priv, __power_well)                       \
551         for ((__power_well) = (__dev_priv)->power_domains.power_wells +         \
552                               (__dev_priv)->power_domains.power_well_count - 1; \
553              (__power_well) - (__dev_priv)->power_domains.power_wells >= 0;     \
554              (__power_well)--)
555
556 #define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask)     \
557         for_each_power_well(__dev_priv, __power_well)                           \
558                 for_each_if ((__power_well)->domains & (__domain_mask))
559
560 #define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \
561         for_each_power_well_rev(__dev_priv, __power_well)                       \
562                 for_each_if ((__power_well)->domains & (__domain_mask))
563
564 #define for_each_intel_plane_in_state(__state, plane, plane_state, __i) \
565         for ((__i) = 0; \
566              (__i) < (__state)->base.dev->mode_config.num_total_plane && \
567                      ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
568                       (plane_state) = to_intel_plane_state((__state)->base.planes[__i].state), 1); \
569              (__i)++) \
570                 for_each_if (plane_state)
571
572 #define for_each_new_intel_crtc_in_state(__state, crtc, new_crtc_state, __i) \
573         for ((__i) = 0; \
574              (__i) < (__state)->base.dev->mode_config.num_crtc && \
575                      ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \
576                       (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \
577              (__i)++) \
578                 for_each_if (crtc)
579
580
581 #define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \
582         for ((__i) = 0; \
583              (__i) < (__state)->base.dev->mode_config.num_total_plane && \
584                      ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
585                       (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), \
586                       (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \
587              (__i)++) \
588                 for_each_if (plane)
589
590 struct drm_i915_private;
591 struct i915_mm_struct;
592 struct i915_mmu_object;
593
594 struct drm_i915_file_private {
595         struct drm_i915_private *dev_priv;
596         struct drm_file *file;
597
598         struct {
599                 spinlock_t lock;
600                 struct list_head request_list;
601 /* 20ms is a fairly arbitrary limit (greater than the average frame time)
602  * chosen to prevent the CPU getting more than a frame ahead of the GPU
603  * (when using lax throttling for the frontbuffer). We also use it to
604  * offer free GPU waitboosts for severely congested workloads.
605  */
606 #define DRM_I915_THROTTLE_JIFFIES msecs_to_jiffies(20)
607         } mm;
608         struct idr context_idr;
609
610         struct intel_rps_client {
611                 atomic_t boosts;
612         } rps;
613
614         unsigned int bsd_engine;
615
616 /* Client can have a maximum of 3 contexts banned before
617  * it is denied of creating new contexts. As one context
618  * ban needs 4 consecutive hangs, and more if there is
619  * progress in between, this is a last resort stop gap measure
620  * to limit the badly behaving clients access to gpu.
621  */
622 #define I915_MAX_CLIENT_CONTEXT_BANS 3
623         atomic_t context_bans;
624 };
625
626 /* Used by dp and fdi links */
627 struct intel_link_m_n {
628         uint32_t        tu;
629         uint32_t        gmch_m;
630         uint32_t        gmch_n;
631         uint32_t        link_m;
632         uint32_t        link_n;
633 };
634
635 void intel_link_compute_m_n(int bpp, int nlanes,
636                             int pixel_clock, int link_clock,
637                             struct intel_link_m_n *m_n,
638                             bool reduce_m_n);
639
640 /* Interface history:
641  *
642  * 1.1: Original.
643  * 1.2: Add Power Management
644  * 1.3: Add vblank support
645  * 1.4: Fix cmdbuffer path, add heap destroy
646  * 1.5: Add vblank pipe configuration
647  * 1.6: - New ioctl for scheduling buffer swaps on vertical blank
648  *      - Support vertical blank on secondary display pipe
649  */
650 #define DRIVER_MAJOR            1
651 #define DRIVER_MINOR            6
652 #define DRIVER_PATCHLEVEL       0
653
654 struct opregion_header;
655 struct opregion_acpi;
656 struct opregion_swsci;
657 struct opregion_asle;
658
659 struct intel_opregion {
660         struct opregion_header *header;
661         struct opregion_acpi *acpi;
662         struct opregion_swsci *swsci;
663         u32 swsci_gbda_sub_functions;
664         u32 swsci_sbcb_sub_functions;
665         struct opregion_asle *asle;
666         void *rvda;
667         void *vbt_firmware;
668         const void *vbt;
669         u32 vbt_size;
670         u32 *lid_state;
671         struct work_struct asle_work;
672 };
673 #define OPREGION_SIZE            (8*1024)
674
675 struct intel_overlay;
676 struct intel_overlay_error_state;
677
678 struct sdvo_device_mapping {
679         u8 initialized;
680         u8 dvo_port;
681         u8 slave_addr;
682         u8 dvo_wiring;
683         u8 i2c_pin;
684         u8 ddc_pin;
685 };
686
687 struct intel_connector;
688 struct intel_encoder;
689 struct intel_atomic_state;
690 struct intel_crtc_state;
691 struct intel_initial_plane_config;
692 struct intel_crtc;
693 struct intel_limit;
694 struct dpll;
695 struct intel_cdclk_state;
696
697 struct drm_i915_display_funcs {
698         void (*get_cdclk)(struct drm_i915_private *dev_priv,
699                           struct intel_cdclk_state *cdclk_state);
700         void (*set_cdclk)(struct drm_i915_private *dev_priv,
701                           const struct intel_cdclk_state *cdclk_state);
702         int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane);
703         int (*compute_pipe_wm)(struct intel_crtc_state *cstate);
704         int (*compute_intermediate_wm)(struct drm_device *dev,
705                                        struct intel_crtc *intel_crtc,
706                                        struct intel_crtc_state *newstate);
707         void (*initial_watermarks)(struct intel_atomic_state *state,
708                                    struct intel_crtc_state *cstate);
709         void (*atomic_update_watermarks)(struct intel_atomic_state *state,
710                                          struct intel_crtc_state *cstate);
711         void (*optimize_watermarks)(struct intel_atomic_state *state,
712                                     struct intel_crtc_state *cstate);
713         int (*compute_global_watermarks)(struct drm_atomic_state *state);
714         void (*update_wm)(struct intel_crtc *crtc);
715         int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
716         /* Returns the active state of the crtc, and if the crtc is active,
717          * fills out the pipe-config with the hw state. */
718         bool (*get_pipe_config)(struct intel_crtc *,
719                                 struct intel_crtc_state *);
720         void (*get_initial_plane_config)(struct intel_crtc *,
721                                          struct intel_initial_plane_config *);
722         int (*crtc_compute_clock)(struct intel_crtc *crtc,
723                                   struct intel_crtc_state *crtc_state);
724         void (*crtc_enable)(struct intel_crtc_state *pipe_config,
725                             struct drm_atomic_state *old_state);
726         void (*crtc_disable)(struct intel_crtc_state *old_crtc_state,
727                              struct drm_atomic_state *old_state);
728         void (*update_crtcs)(struct drm_atomic_state *state);
729         void (*audio_codec_enable)(struct drm_connector *connector,
730                                    struct intel_encoder *encoder,
731                                    const struct drm_display_mode *adjusted_mode);
732         void (*audio_codec_disable)(struct intel_encoder *encoder);
733         void (*fdi_link_train)(struct intel_crtc *crtc,
734                                const struct intel_crtc_state *crtc_state);
735         void (*init_clock_gating)(struct drm_i915_private *dev_priv);
736         void (*hpd_irq_setup)(struct drm_i915_private *dev_priv);
737         /* clock updates for mode set */
738         /* cursor updates */
739         /* render clock increase/decrease */
740         /* display clock increase/decrease */
741         /* pll clock increase/decrease */
742
743         void (*load_csc_matrix)(struct drm_crtc_state *crtc_state);
744         void (*load_luts)(struct drm_crtc_state *crtc_state);
745 };
746
747 #define CSR_VERSION(major, minor)       ((major) << 16 | (minor))
748 #define CSR_VERSION_MAJOR(version)      ((version) >> 16)
749 #define CSR_VERSION_MINOR(version)      ((version) & 0xffff)
750
751 struct intel_csr {
752         struct work_struct work;
753         const char *fw_path;
754         uint32_t *dmc_payload;
755         uint32_t dmc_fw_size;
756         uint32_t version;
757         uint32_t mmio_count;
758         i915_reg_t mmioaddr[8];
759         uint32_t mmiodata[8];
760         uint32_t dc_state;
761         uint32_t allowed_dc_mask;
762 };
763
764 #define DEV_INFO_FOR_EACH_FLAG(func) \
765         func(is_mobile); \
766         func(is_lp); \
767         func(is_alpha_support); \
768         /* Keep has_* in alphabetical order */ \
769         func(has_64bit_reloc); \
770         func(has_aliasing_ppgtt); \
771         func(has_csr); \
772         func(has_ddi); \
773         func(has_dp_mst); \
774         func(has_reset_engine); \
775         func(has_fbc); \
776         func(has_fpga_dbg); \
777         func(has_full_ppgtt); \
778         func(has_full_48bit_ppgtt); \
779         func(has_gmbus_irq); \
780         func(has_gmch_display); \
781         func(has_guc); \
782         func(has_guc_ct); \
783         func(has_hotplug); \
784         func(has_l3_dpf); \
785         func(has_llc); \
786         func(has_logical_ring_contexts); \
787         func(has_overlay); \
788         func(has_pipe_cxsr); \
789         func(has_pooled_eu); \
790         func(has_psr); \
791         func(has_rc6); \
792         func(has_rc6p); \
793         func(has_resource_streamer); \
794         func(has_runtime_pm); \
795         func(has_snoop); \
796         func(unfenced_needs_alignment); \
797         func(cursor_needs_physical); \
798         func(hws_needs_physical); \
799         func(overlay_needs_physical); \
800         func(supports_tv);
801
802 struct sseu_dev_info {
803         u8 slice_mask;
804         u8 subslice_mask;
805         u8 eu_total;
806         u8 eu_per_subslice;
807         u8 min_eu_in_pool;
808         /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
809         u8 subslice_7eu[3];
810         u8 has_slice_pg:1;
811         u8 has_subslice_pg:1;
812         u8 has_eu_pg:1;
813 };
814
815 static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
816 {
817         return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask);
818 }
819
820 /* Keep in gen based order, and chronological order within a gen */
821 enum intel_platform {
822         INTEL_PLATFORM_UNINITIALIZED = 0,
823         INTEL_I830,
824         INTEL_I845G,
825         INTEL_I85X,
826         INTEL_I865G,
827         INTEL_I915G,
828         INTEL_I915GM,
829         INTEL_I945G,
830         INTEL_I945GM,
831         INTEL_G33,
832         INTEL_PINEVIEW,
833         INTEL_I965G,
834         INTEL_I965GM,
835         INTEL_G45,
836         INTEL_GM45,
837         INTEL_IRONLAKE,
838         INTEL_SANDYBRIDGE,
839         INTEL_IVYBRIDGE,
840         INTEL_VALLEYVIEW,
841         INTEL_HASWELL,
842         INTEL_BROADWELL,
843         INTEL_CHERRYVIEW,
844         INTEL_SKYLAKE,
845         INTEL_BROXTON,
846         INTEL_KABYLAKE,
847         INTEL_GEMINILAKE,
848         INTEL_COFFEELAKE,
849         INTEL_CANNONLAKE,
850         INTEL_MAX_PLATFORMS
851 };
852
853 struct intel_device_info {
854         u32 display_mmio_offset;
855         u16 device_id;
856         u8 num_pipes;
857         u8 num_sprites[I915_MAX_PIPES];
858         u8 num_scalers[I915_MAX_PIPES];
859         u8 gen;
860         u16 gen_mask;
861         enum intel_platform platform;
862         u8 gt; /* GT number, 0 if undefined */
863         u8 ring_mask; /* Rings supported by the HW */
864         u8 num_rings;
865 #define DEFINE_FLAG(name) u8 name:1
866         DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
867 #undef DEFINE_FLAG
868         u16 ddb_size; /* in blocks */
869         /* Register offsets for the various display pipes and transcoders */
870         int pipe_offsets[I915_MAX_TRANSCODERS];
871         int trans_offsets[I915_MAX_TRANSCODERS];
872         int palette_offsets[I915_MAX_PIPES];
873         int cursor_offsets[I915_MAX_PIPES];
874
875         /* Slice/subslice/EU info */
876         struct sseu_dev_info sseu;
877
878         struct color_luts {
879                 u16 degamma_lut_size;
880                 u16 gamma_lut_size;
881         } color;
882 };
883
884 struct intel_display_error_state;
885
886 struct i915_gpu_state {
887         struct kref ref;
888         struct timeval time;
889         struct timeval boottime;
890         struct timeval uptime;
891
892         struct drm_i915_private *i915;
893
894         char error_msg[128];
895         bool simulated;
896         bool awake;
897         bool wakelock;
898         bool suspended;
899         int iommu;
900         u32 reset_count;
901         u32 suspend_count;
902         struct intel_device_info device_info;
903         struct i915_params params;
904
905         /* Generic register state */
906         u32 eir;
907         u32 pgtbl_er;
908         u32 ier;
909         u32 gtier[4], ngtier;
910         u32 ccid;
911         u32 derrmr;
912         u32 forcewake;
913         u32 error; /* gen6+ */
914         u32 err_int; /* gen7 */
915         u32 fault_data0; /* gen8, gen9 */
916         u32 fault_data1; /* gen8, gen9 */
917         u32 done_reg;
918         u32 gac_eco;
919         u32 gam_ecochk;
920         u32 gab_ctl;
921         u32 gfx_mode;
922
923         u32 nfence;
924         u64 fence[I915_MAX_NUM_FENCES];
925         struct intel_overlay_error_state *overlay;
926         struct intel_display_error_state *display;
927         struct drm_i915_error_object *semaphore;
928         struct drm_i915_error_object *guc_log;
929
930         struct drm_i915_error_engine {
931                 int engine_id;
932                 /* Software tracked state */
933                 bool waiting;
934                 int num_waiters;
935                 unsigned long hangcheck_timestamp;
936                 bool hangcheck_stalled;
937                 enum intel_engine_hangcheck_action hangcheck_action;
938                 struct i915_address_space *vm;
939                 int num_requests;
940                 u32 reset_count;
941
942                 /* position of active request inside the ring */
943                 u32 rq_head, rq_post, rq_tail;
944
945                 /* our own tracking of ring head and tail */
946                 u32 cpu_ring_head;
947                 u32 cpu_ring_tail;
948
949                 u32 last_seqno;
950
951                 /* Register state */
952                 u32 start;
953                 u32 tail;
954                 u32 head;
955                 u32 ctl;
956                 u32 mode;
957                 u32 hws;
958                 u32 ipeir;
959                 u32 ipehr;
960                 u32 bbstate;
961                 u32 instpm;
962                 u32 instps;
963                 u32 seqno;
964                 u64 bbaddr;
965                 u64 acthd;
966                 u32 fault_reg;
967                 u64 faddr;
968                 u32 rc_psmi; /* sleep state */
969                 u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
970                 struct intel_instdone instdone;
971
972                 struct drm_i915_error_context {
973                         char comm[TASK_COMM_LEN];
974                         pid_t pid;
975                         u32 handle;
976                         u32 hw_id;
977                         int ban_score;
978                         int active;
979                         int guilty;
980                 } context;
981
982                 struct drm_i915_error_object {
983                         u64 gtt_offset;
984                         u64 gtt_size;
985                         int page_count;
986                         int unused;
987                         u32 *pages[0];
988                 } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
989
990                 struct drm_i915_error_object **user_bo;
991                 long user_bo_count;
992
993                 struct drm_i915_error_object *wa_ctx;
994
995                 struct drm_i915_error_request {
996                         long jiffies;
997                         pid_t pid;
998                         u32 context;
999                         int ban_score;
1000                         u32 seqno;
1001                         u32 head;
1002                         u32 tail;
1003                 } *requests, execlist[2];
1004
1005                 struct drm_i915_error_waiter {
1006                         char comm[TASK_COMM_LEN];
1007                         pid_t pid;
1008                         u32 seqno;
1009                 } *waiters;
1010
1011                 struct {
1012                         u32 gfx_mode;
1013                         union {
1014                                 u64 pdp[4];
1015                                 u32 pp_dir_base;
1016                         };
1017                 } vm_info;
1018         } engine[I915_NUM_ENGINES];
1019
1020         struct drm_i915_error_buffer {
1021                 u32 size;
1022                 u32 name;
1023                 u32 rseqno[I915_NUM_ENGINES], wseqno;
1024                 u64 gtt_offset;
1025                 u32 read_domains;
1026                 u32 write_domain;
1027                 s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
1028                 u32 tiling:2;
1029                 u32 dirty:1;
1030                 u32 purgeable:1;
1031                 u32 userptr:1;
1032                 s32 engine:4;
1033                 u32 cache_level:3;
1034         } *active_bo[I915_NUM_ENGINES], *pinned_bo;
1035         u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
1036         struct i915_address_space *active_vm[I915_NUM_ENGINES];
1037 };
1038
1039 enum i915_cache_level {
1040         I915_CACHE_NONE = 0,
1041         I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */
1042         I915_CACHE_L3_LLC, /* gen7+, L3 sits between the domain specifc
1043                               caches, eg sampler/render caches, and the
1044                               large Last-Level-Cache. LLC is coherent with
1045                               the CPU, but L3 is only visible to the GPU. */
1046         I915_CACHE_WT, /* hsw:gt3e WriteThrough for scanouts */
1047 };
1048
1049 #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */
1050
1051 enum fb_op_origin {
1052         ORIGIN_GTT,
1053         ORIGIN_CPU,
1054         ORIGIN_CS,
1055         ORIGIN_FLIP,
1056         ORIGIN_DIRTYFB,
1057 };
1058
1059 struct intel_fbc {
1060         /* This is always the inner lock when overlapping with struct_mutex and
1061          * it's the outer lock when overlapping with stolen_lock. */
1062         struct mutex lock;
1063         unsigned threshold;
1064         unsigned int possible_framebuffer_bits;
1065         unsigned int busy_bits;
1066         unsigned int visible_pipes_mask;
1067         struct intel_crtc *crtc;
1068
1069         struct drm_mm_node compressed_fb;
1070         struct drm_mm_node *compressed_llb;
1071
1072         bool false_color;
1073
1074         bool enabled;
1075         bool active;
1076
1077         bool underrun_detected;
1078         struct work_struct underrun_work;
1079
1080         /*
1081          * Due to the atomic rules we can't access some structures without the
1082          * appropriate locking, so we cache information here in order to avoid
1083          * these problems.
1084          */
1085         struct intel_fbc_state_cache {
1086                 struct i915_vma *vma;
1087
1088                 struct {
1089                         unsigned int mode_flags;
1090                         uint32_t hsw_bdw_pixel_rate;
1091                 } crtc;
1092
1093                 struct {
1094                         unsigned int rotation;
1095                         int src_w;
1096                         int src_h;
1097                         bool visible;
1098                 } plane;
1099
1100                 struct {
1101                         const struct drm_format_info *format;
1102                         unsigned int stride;
1103                 } fb;
1104         } state_cache;
1105
1106         /*
1107          * This structure contains everything that's relevant to program the
1108          * hardware registers. When we want to figure out if we need to disable
1109          * and re-enable FBC for a new configuration we just check if there's
1110          * something different in the struct. The genx_fbc_activate functions
1111          * are supposed to read from it in order to program the registers.
1112          */
1113         struct intel_fbc_reg_params {
1114                 struct i915_vma *vma;
1115
1116                 struct {
1117                         enum pipe pipe;
1118                         enum plane plane;
1119                         unsigned int fence_y_offset;
1120                 } crtc;
1121
1122                 struct {
1123                         const struct drm_format_info *format;
1124                         unsigned int stride;
1125                 } fb;
1126
1127                 int cfb_size;
1128                 unsigned int gen9_wa_cfb_stride;
1129         } params;
1130
1131         struct intel_fbc_work {
1132                 bool scheduled;
1133                 u32 scheduled_vblank;
1134                 struct work_struct work;
1135         } work;
1136
1137         const char *no_fbc_reason;
1138 };
1139
1140 /*
1141  * HIGH_RR is the highest eDP panel refresh rate read from EDID
1142  * LOW_RR is the lowest eDP panel refresh rate found from EDID
1143  * parsing for same resolution.
1144  */
1145 enum drrs_refresh_rate_type {
1146         DRRS_HIGH_RR,
1147         DRRS_LOW_RR,
1148         DRRS_MAX_RR, /* RR count */
1149 };
1150
1151 enum drrs_support_type {
1152         DRRS_NOT_SUPPORTED = 0,
1153         STATIC_DRRS_SUPPORT = 1,
1154         SEAMLESS_DRRS_SUPPORT = 2
1155 };
1156
1157 struct intel_dp;
1158 struct i915_drrs {
1159         struct mutex mutex;
1160         struct delayed_work work;
1161         struct intel_dp *dp;
1162         unsigned busy_frontbuffer_bits;
1163         enum drrs_refresh_rate_type refresh_rate_type;
1164         enum drrs_support_type type;
1165 };
1166
1167 struct i915_psr {
1168         struct mutex lock;
1169         bool sink_support;
1170         bool source_ok;
1171         struct intel_dp *enabled;
1172         bool active;
1173         struct delayed_work work;
1174         unsigned busy_frontbuffer_bits;
1175         bool psr2_support;
1176         bool aux_frame_sync;
1177         bool link_standby;
1178         bool y_cord_support;
1179         bool colorimetry_support;
1180         bool alpm;
1181 };
1182
1183 enum intel_pch {
1184         PCH_NONE = 0,   /* No PCH present */
1185         PCH_IBX,        /* Ibexpeak PCH */
1186         PCH_CPT,        /* Cougarpoint/Pantherpoint PCH */
1187         PCH_LPT,        /* Lynxpoint/Wildcatpoint PCH */
1188         PCH_SPT,        /* Sunrisepoint PCH */
1189         PCH_KBP,        /* Kaby Lake PCH */
1190         PCH_CNP,        /* Cannon Lake PCH */
1191         PCH_NOP,
1192 };
1193
1194 enum intel_sbi_destination {
1195         SBI_ICLK,
1196         SBI_MPHY,
1197 };
1198
1199 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
1200 #define QUIRK_INVERT_BRIGHTNESS (1<<2)
1201 #define QUIRK_BACKLIGHT_PRESENT (1<<3)
1202 #define QUIRK_PIN_SWIZZLED_PAGES (1<<5)
1203 #define QUIRK_INCREASE_T12_DELAY (1<<6)
1204
1205 struct intel_fbdev;
1206 struct intel_fbc_work;
1207
1208 struct intel_gmbus {
1209         struct i2c_adapter adapter;
1210 #define GMBUS_FORCE_BIT_RETRY (1U << 31)
1211         u32 force_bit;
1212         u32 reg0;
1213         i915_reg_t gpio_reg;
1214         struct i2c_algo_bit_data bit_algo;
1215         struct drm_i915_private *dev_priv;
1216 };
1217
1218 struct i915_suspend_saved_registers {
1219         u32 saveDSPARB;
1220         u32 saveFBC_CONTROL;
1221         u32 saveCACHE_MODE_0;
1222         u32 saveMI_ARB_STATE;
1223         u32 saveSWF0[16];
1224         u32 saveSWF1[16];
1225         u32 saveSWF3[3];
1226         uint64_t saveFENCE[I915_MAX_NUM_FENCES];
1227         u32 savePCH_PORT_HOTPLUG;
1228         u16 saveGCDGMBUS;
1229 };
1230
1231 struct vlv_s0ix_state {
1232         /* GAM */
1233         u32 wr_watermark;
1234         u32 gfx_prio_ctrl;
1235         u32 arb_mode;
1236         u32 gfx_pend_tlb0;
1237         u32 gfx_pend_tlb1;
1238         u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM];
1239         u32 media_max_req_count;
1240         u32 gfx_max_req_count;
1241         u32 render_hwsp;
1242         u32 ecochk;
1243         u32 bsd_hwsp;
1244         u32 blt_hwsp;
1245         u32 tlb_rd_addr;
1246
1247         /* MBC */
1248         u32 g3dctl;
1249         u32 gsckgctl;
1250         u32 mbctl;
1251
1252         /* GCP */
1253         u32 ucgctl1;
1254         u32 ucgctl3;
1255         u32 rcgctl1;
1256         u32 rcgctl2;
1257         u32 rstctl;
1258         u32 misccpctl;
1259
1260         /* GPM */
1261         u32 gfxpause;
1262         u32 rpdeuhwtc;
1263         u32 rpdeuc;
1264         u32 ecobus;
1265         u32 pwrdwnupctl;
1266         u32 rp_down_timeout;
1267         u32 rp_deucsw;
1268         u32 rcubmabdtmr;
1269         u32 rcedata;
1270         u32 spare2gh;
1271
1272         /* Display 1 CZ domain */
1273         u32 gt_imr;
1274         u32 gt_ier;
1275         u32 pm_imr;
1276         u32 pm_ier;
1277         u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM];
1278
1279         /* GT SA CZ domain */
1280         u32 tilectl;
1281         u32 gt_fifoctl;
1282         u32 gtlc_wake_ctrl;
1283         u32 gtlc_survive;
1284         u32 pmwgicz;
1285
1286         /* Display 2 CZ domain */
1287         u32 gu_ctl0;
1288         u32 gu_ctl1;
1289         u32 pcbr;
1290         u32 clock_gate_dis2;
1291 };
1292
1293 struct intel_rps_ei {
1294         ktime_t ktime;
1295         u32 render_c0;
1296         u32 media_c0;
1297 };
1298
1299 struct intel_gen6_power_mgmt {
1300         /*
1301          * work, interrupts_enabled and pm_iir are protected by
1302          * dev_priv->irq_lock
1303          */
1304         struct work_struct work;
1305         bool interrupts_enabled;
1306         u32 pm_iir;
1307
1308         /* PM interrupt bits that should never be masked */
1309         u32 pm_intrmsk_mbz;
1310
1311         /* Frequencies are stored in potentially platform dependent multiples.
1312          * In other words, *_freq needs to be multiplied by X to be interesting.
1313          * Soft limits are those which are used for the dynamic reclocking done
1314          * by the driver (raise frequencies under heavy loads, and lower for
1315          * lighter loads). Hard limits are those imposed by the hardware.
1316          *
1317          * A distinction is made for overclocking, which is never enabled by
1318          * default, and is considered to be above the hard limit if it's
1319          * possible at all.
1320          */
1321         u8 cur_freq;            /* Current frequency (cached, may not == HW) */
1322         u8 min_freq_softlimit;  /* Minimum frequency permitted by the driver */
1323         u8 max_freq_softlimit;  /* Max frequency permitted by the driver */
1324         u8 max_freq;            /* Maximum frequency, RP0 if not overclocking */
1325         u8 min_freq;            /* AKA RPn. Minimum frequency */
1326         u8 boost_freq;          /* Frequency to request when wait boosting */
1327         u8 idle_freq;           /* Frequency to request when we are idle */
1328         u8 efficient_freq;      /* AKA RPe. Pre-determined balanced frequency */
1329         u8 rp1_freq;            /* "less than" RP0 power/freqency */
1330         u8 rp0_freq;            /* Non-overclocked max frequency. */
1331         u16 gpll_ref_freq;      /* vlv/chv GPLL reference frequency */
1332
1333         u8 up_threshold; /* Current %busy required to uplock */
1334         u8 down_threshold; /* Current %busy required to downclock */
1335
1336         int last_adj;
1337         enum { LOW_POWER, BETWEEN, HIGH_POWER } power;
1338
1339         bool enabled;
1340         struct delayed_work autoenable_work;
1341         atomic_t num_waiters;
1342         atomic_t boosts;
1343
1344         /* manual wa residency calculations */
1345         struct intel_rps_ei ei;
1346
1347         /*
1348          * Protects RPS/RC6 register access and PCU communication.
1349          * Must be taken after struct_mutex if nested. Note that
1350          * this lock may be held for long periods of time when
1351          * talking to hw - so only take it when talking to hw!
1352          */
1353         struct mutex hw_lock;
1354 };
1355
1356 /* defined intel_pm.c */
1357 extern spinlock_t mchdev_lock;
1358
1359 struct intel_ilk_power_mgmt {
1360         u8 cur_delay;
1361         u8 min_delay;
1362         u8 max_delay;
1363         u8 fmax;
1364         u8 fstart;
1365
1366         u64 last_count1;
1367         unsigned long last_time1;
1368         unsigned long chipset_power;
1369         u64 last_count2;
1370         u64 last_time2;
1371         unsigned long gfx_power;
1372         u8 corr;
1373
1374         int c_m;
1375         int r_t;
1376 };
1377
1378 struct drm_i915_private;
1379 struct i915_power_well;
1380
1381 struct i915_power_well_ops {
1382         /*
1383          * Synchronize the well's hw state to match the current sw state, for
1384          * example enable/disable it based on the current refcount. Called
1385          * during driver init and resume time, possibly after first calling
1386          * the enable/disable handlers.
1387          */
1388         void (*sync_hw)(struct drm_i915_private *dev_priv,
1389                         struct i915_power_well *power_well);
1390         /*
1391          * Enable the well and resources that depend on it (for example
1392          * interrupts located on the well). Called after the 0->1 refcount
1393          * transition.
1394          */
1395         void (*enable)(struct drm_i915_private *dev_priv,
1396                        struct i915_power_well *power_well);
1397         /*
1398          * Disable the well and resources that depend on it. Called after
1399          * the 1->0 refcount transition.
1400          */
1401         void (*disable)(struct drm_i915_private *dev_priv,
1402                         struct i915_power_well *power_well);
1403         /* Returns the hw enabled state. */
1404         bool (*is_enabled)(struct drm_i915_private *dev_priv,
1405                            struct i915_power_well *power_well);
1406 };
1407
1408 /* Power well structure for haswell */
1409 struct i915_power_well {
1410         const char *name;
1411         bool always_on;
1412         /* power well enable/disable usage count */
1413         int count;
1414         /* cached hw enabled state */
1415         bool hw_enabled;
1416         u64 domains;
1417         /* unique identifier for this power well */
1418         enum i915_power_well_id id;
1419         /*
1420          * Arbitraty data associated with this power well. Platform and power
1421          * well specific.
1422          */
1423         union {
1424                 struct {
1425                         enum dpio_phy phy;
1426                 } bxt;
1427                 struct {
1428                         /* Mask of pipes whose IRQ logic is backed by the pw */
1429                         u8 irq_pipe_mask;
1430                         /* The pw is backing the VGA functionality */
1431                         bool has_vga:1;
1432                         bool has_fuses:1;
1433                 } hsw;
1434         };
1435         const struct i915_power_well_ops *ops;
1436 };
1437
1438 struct i915_power_domains {
1439         /*
1440          * Power wells needed for initialization at driver init and suspend
1441          * time are on. They are kept on until after the first modeset.
1442          */
1443         bool init_power_on;
1444         bool initializing;
1445         int power_well_count;
1446
1447         struct mutex lock;
1448         int domain_use_count[POWER_DOMAIN_NUM];
1449         struct i915_power_well *power_wells;
1450 };
1451
1452 #define MAX_L3_SLICES 2
1453 struct intel_l3_parity {
1454         u32 *remap_info[MAX_L3_SLICES];
1455         struct work_struct error_work;
1456         int which_slice;
1457 };
1458
1459 struct i915_gem_mm {
1460         /** Memory allocator for GTT stolen memory */
1461         struct drm_mm stolen;
1462         /** Protects the usage of the GTT stolen memory allocator. This is
1463          * always the inner lock when overlapping with struct_mutex. */
1464         struct mutex stolen_lock;
1465
1466         /** List of all objects in gtt_space. Used to restore gtt
1467          * mappings on resume */
1468         struct list_head bound_list;
1469         /**
1470          * List of objects which are not bound to the GTT (thus
1471          * are idle and not used by the GPU). These objects may or may
1472          * not actually have any pages attached.
1473          */
1474         struct list_head unbound_list;
1475
1476         /** List of all objects in gtt_space, currently mmaped by userspace.
1477          * All objects within this list must also be on bound_list.
1478          */
1479         struct list_head userfault_list;
1480
1481         /**
1482          * List of objects which are pending destruction.
1483          */
1484         struct llist_head free_list;
1485         struct work_struct free_work;
1486
1487         /**
1488          * Small stash of WC pages
1489          */
1490         struct pagevec wc_stash;
1491
1492         /** Usable portion of the GTT for GEM */
1493         dma_addr_t stolen_base; /* limited to low memory (32-bit) */
1494
1495         /** PPGTT used for aliasing the PPGTT with the GTT */
1496         struct i915_hw_ppgtt *aliasing_ppgtt;
1497
1498         struct notifier_block oom_notifier;
1499         struct notifier_block vmap_notifier;
1500         struct shrinker shrinker;
1501
1502         /** LRU list of objects with fence regs on them. */
1503         struct list_head fence_list;
1504
1505         /**
1506          * Workqueue to fault in userptr pages, flushed by the execbuf
1507          * when required but otherwise left to userspace to try again
1508          * on EAGAIN.
1509          */
1510         struct workqueue_struct *userptr_wq;
1511
1512         u64 unordered_timeline;
1513
1514         /* the indicator for dispatch video commands on two BSD rings */
1515         atomic_t bsd_engine_dispatch_index;
1516
1517         /** Bit 6 swizzling required for X tiling */
1518         uint32_t bit_6_swizzle_x;
1519         /** Bit 6 swizzling required for Y tiling */
1520         uint32_t bit_6_swizzle_y;
1521
1522         /* accounting, useful for userland debugging */
1523         spinlock_t object_stat_lock;
1524         u64 object_memory;
1525         u32 object_count;
1526 };
1527
1528 struct drm_i915_error_state_buf {
1529         struct drm_i915_private *i915;
1530         unsigned bytes;
1531         unsigned size;
1532         int err;
1533         u8 *buf;
1534         loff_t start;
1535         loff_t pos;
1536 };
1537
1538 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
1539 #define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */
1540
1541 #define I915_ENGINE_DEAD_TIMEOUT  (4 * HZ)  /* Seqno, head and subunits dead */
1542 #define I915_SEQNO_DEAD_TIMEOUT   (12 * HZ) /* Seqno dead with active head */
1543
1544 struct i915_gpu_error {
1545         /* For hangcheck timer */
1546 #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
1547 #define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
1548
1549         struct delayed_work hangcheck_work;
1550
1551         /* For reset and error_state handling. */
1552         spinlock_t lock;
1553         /* Protected by the above dev->gpu_error.lock. */
1554         struct i915_gpu_state *first_error;
1555
1556         atomic_t pending_fb_pin;
1557
1558         unsigned long missed_irq_rings;
1559
1560         /**
1561          * State variable controlling the reset flow and count
1562          *
1563          * This is a counter which gets incremented when reset is triggered,
1564          *
1565          * Before the reset commences, the I915_RESET_BACKOFF bit is set
1566          * meaning that any waiters holding onto the struct_mutex should
1567          * relinquish the lock immediately in order for the reset to start.
1568          *
1569          * If reset is not completed succesfully, the I915_WEDGE bit is
1570          * set meaning that hardware is terminally sour and there is no
1571          * recovery. All waiters on the reset_queue will be woken when
1572          * that happens.
1573          *
1574          * This counter is used by the wait_seqno code to notice that reset
1575          * event happened and it needs to restart the entire ioctl (since most
1576          * likely the seqno it waited for won't ever signal anytime soon).
1577          *
1578          * This is important for lock-free wait paths, where no contended lock
1579          * naturally enforces the correct ordering between the bail-out of the
1580          * waiter and the gpu reset work code.
1581          */
1582         unsigned long reset_count;
1583
1584         /**
1585          * flags: Control various stages of the GPU reset
1586          *
1587          * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
1588          * other users acquiring the struct_mutex. To do this we set the
1589          * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
1590          * and then check for that bit before acquiring the struct_mutex (in
1591          * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
1592          * secondary role in preventing two concurrent global reset attempts.
1593          *
1594          * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the
1595          * struct_mutex. We try to acquire the struct_mutex in the reset worker,
1596          * but it may be held by some long running waiter (that we cannot
1597          * interrupt without causing trouble). Once we are ready to do the GPU
1598          * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If
1599          * they already hold the struct_mutex and want to participate they can
1600          * inspect the bit and do the reset directly, otherwise the worker
1601          * waits for the struct_mutex.
1602          *
1603          * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
1604          * acquire the struct_mutex to reset an engine, we need an explicit
1605          * flag to prevent two concurrent reset attempts in the same engine.
1606          * As the number of engines continues to grow, allocate the flags from
1607          * the most significant bits.
1608          *
1609          * #I915_WEDGED - If reset fails and we can no longer use the GPU,
1610          * we set the #I915_WEDGED bit. Prior to command submission, e.g.
1611          * i915_gem_request_alloc(), this bit is checked and the sequence
1612          * aborted (with -EIO reported to userspace) if set.
1613          */
1614         unsigned long flags;
1615 #define I915_RESET_BACKOFF      0
1616 #define I915_RESET_HANDOFF      1
1617 #define I915_RESET_MODESET      2
1618 #define I915_WEDGED             (BITS_PER_LONG - 1)
1619 #define I915_RESET_ENGINE       (I915_WEDGED - I915_NUM_ENGINES)
1620
1621         /** Number of times an engine has been reset */
1622         u32 reset_engine_count[I915_NUM_ENGINES];
1623
1624         /**
1625          * Waitqueue to signal when a hang is detected. Used to for waiters
1626          * to release the struct_mutex for the reset to procede.
1627          */
1628         wait_queue_head_t wait_queue;
1629
1630         /**
1631          * Waitqueue to signal when the reset has completed. Used by clients
1632          * that wait for dev_priv->mm.wedged to settle.
1633          */
1634         wait_queue_head_t reset_queue;
1635
1636         /* For missed irq/seqno simulation. */
1637         unsigned long test_irq_rings;
1638 };
1639
1640 enum modeset_restore {
1641         MODESET_ON_LID_OPEN,
1642         MODESET_DONE,
1643         MODESET_SUSPENDED,
1644 };
1645
1646 #define DP_AUX_A 0x40
1647 #define DP_AUX_B 0x10
1648 #define DP_AUX_C 0x20
1649 #define DP_AUX_D 0x30
1650
1651 #define DDC_PIN_B  0x05
1652 #define DDC_PIN_C  0x04
1653 #define DDC_PIN_D  0x06
1654
1655 struct ddi_vbt_port_info {
1656         /*
1657          * This is an index in the HDMI/DVI DDI buffer translation table.
1658          * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't
1659          * populate this field.
1660          */
1661 #define HDMI_LEVEL_SHIFT_UNKNOWN        0xff
1662         uint8_t hdmi_level_shift;
1663
1664         uint8_t supports_dvi:1;
1665         uint8_t supports_hdmi:1;
1666         uint8_t supports_dp:1;
1667         uint8_t supports_edp:1;
1668
1669         uint8_t alternate_aux_channel;
1670         uint8_t alternate_ddc_pin;
1671
1672         uint8_t dp_boost_level;
1673         uint8_t hdmi_boost_level;
1674 };
1675
1676 enum psr_lines_to_wait {
1677         PSR_0_LINES_TO_WAIT = 0,
1678         PSR_1_LINE_TO_WAIT,
1679         PSR_4_LINES_TO_WAIT,
1680         PSR_8_LINES_TO_WAIT
1681 };
1682
1683 struct intel_vbt_data {
1684         struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */
1685         struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */
1686
1687         /* Feature bits */
1688         unsigned int int_tv_support:1;
1689         unsigned int lvds_dither:1;
1690         unsigned int lvds_vbt:1;
1691         unsigned int int_crt_support:1;
1692         unsigned int lvds_use_ssc:1;
1693         unsigned int display_clock_mode:1;
1694         unsigned int fdi_rx_polarity_inverted:1;
1695         unsigned int panel_type:4;
1696         int lvds_ssc_freq;
1697         unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */
1698
1699         enum drrs_support_type drrs_type;
1700
1701         struct {
1702                 int rate;
1703                 int lanes;
1704                 int preemphasis;
1705                 int vswing;
1706                 bool low_vswing;
1707                 bool initialized;
1708                 bool support;
1709                 int bpp;
1710                 struct edp_power_seq pps;
1711         } edp;
1712
1713         struct {
1714                 bool full_link;
1715                 bool require_aux_wakeup;
1716                 int idle_frames;
1717                 enum psr_lines_to_wait lines_to_wait;
1718                 int tp1_wakeup_time;
1719                 int tp2_tp3_wakeup_time;
1720         } psr;
1721
1722         struct {
1723                 u16 pwm_freq_hz;
1724                 bool present;
1725                 bool active_low_pwm;
1726                 u8 min_brightness;      /* min_brightness/255 of max */
1727                 u8 controller;          /* brightness controller number */
1728                 enum intel_backlight_type type;
1729         } backlight;
1730
1731         /* MIPI DSI */
1732         struct {
1733                 u16 panel_id;
1734                 struct mipi_config *config;
1735                 struct mipi_pps_data *pps;
1736                 u8 seq_version;
1737                 u32 size;
1738                 u8 *data;
1739                 const u8 *sequence[MIPI_SEQ_MAX];
1740         } dsi;
1741
1742         int crt_ddc_pin;
1743
1744         int child_dev_num;
1745         struct child_device_config *child_dev;
1746
1747         struct ddi_vbt_port_info ddi_port_info[I915_MAX_PORTS];
1748         struct sdvo_device_mapping sdvo_mappings[2];
1749 };
1750
1751 enum intel_ddb_partitioning {
1752         INTEL_DDB_PART_1_2,
1753         INTEL_DDB_PART_5_6, /* IVB+ */
1754 };
1755
1756 struct intel_wm_level {
1757         bool enable;
1758         uint32_t pri_val;
1759         uint32_t spr_val;
1760         uint32_t cur_val;
1761         uint32_t fbc_val;
1762 };
1763
1764 struct ilk_wm_values {
1765         uint32_t wm_pipe[3];
1766         uint32_t wm_lp[3];
1767         uint32_t wm_lp_spr[3];
1768         uint32_t wm_linetime[3];
1769         bool enable_fbc_wm;
1770         enum intel_ddb_partitioning partitioning;
1771 };
1772
1773 struct g4x_pipe_wm {
1774         uint16_t plane[I915_MAX_PLANES];
1775         uint16_t fbc;
1776 };
1777
1778 struct g4x_sr_wm {
1779         uint16_t plane;
1780         uint16_t cursor;
1781         uint16_t fbc;
1782 };
1783
1784 struct vlv_wm_ddl_values {
1785         uint8_t plane[I915_MAX_PLANES];
1786 };
1787
1788 struct vlv_wm_values {
1789         struct g4x_pipe_wm pipe[3];
1790         struct g4x_sr_wm sr;
1791         struct vlv_wm_ddl_values ddl[3];
1792         uint8_t level;
1793         bool cxsr;
1794 };
1795
1796 struct g4x_wm_values {
1797         struct g4x_pipe_wm pipe[2];
1798         struct g4x_sr_wm sr;
1799         struct g4x_sr_wm hpll;
1800         bool cxsr;
1801         bool hpll_en;
1802         bool fbc_en;
1803 };
1804
1805 struct skl_ddb_entry {
1806         uint16_t start, end;    /* in number of blocks, 'end' is exclusive */
1807 };
1808
1809 static inline uint16_t skl_ddb_entry_size(const struct skl_ddb_entry *entry)
1810 {
1811         return entry->end - entry->start;
1812 }
1813
1814 static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
1815                                        const struct skl_ddb_entry *e2)
1816 {
1817         if (e1->start == e2->start && e1->end == e2->end)
1818                 return true;
1819
1820         return false;
1821 }
1822
1823 struct skl_ddb_allocation {
1824         struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
1825         struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
1826 };
1827
1828 struct skl_wm_values {
1829         unsigned dirty_pipes;
1830         struct skl_ddb_allocation ddb;
1831 };
1832
1833 struct skl_wm_level {
1834         bool plane_en;
1835         uint16_t plane_res_b;
1836         uint8_t plane_res_l;
1837 };
1838
1839 /*
1840  * This struct helps tracking the state needed for runtime PM, which puts the
1841  * device in PCI D3 state. Notice that when this happens, nothing on the
1842  * graphics device works, even register access, so we don't get interrupts nor
1843  * anything else.
1844  *
1845  * Every piece of our code that needs to actually touch the hardware needs to
1846  * either call intel_runtime_pm_get or call intel_display_power_get with the
1847  * appropriate power domain.
1848  *
1849  * Our driver uses the autosuspend delay feature, which means we'll only really
1850  * suspend if we stay with zero refcount for a certain amount of time. The
1851  * default value is currently very conservative (see intel_runtime_pm_enable), but
1852  * it can be changed with the standard runtime PM files from sysfs.
1853  *
1854  * The irqs_disabled variable becomes true exactly after we disable the IRQs and
1855  * goes back to false exactly before we reenable the IRQs. We use this variable
1856  * to check if someone is trying to enable/disable IRQs while they're supposed
1857  * to be disabled. This shouldn't happen and we'll print some error messages in
1858  * case it happens.
1859  *
1860  * For more, read the Documentation/power/runtime_pm.txt.
1861  */
1862 struct i915_runtime_pm {
1863         atomic_t wakeref_count;
1864         bool suspended;
1865         bool irqs_enabled;
1866 };
1867
1868 enum intel_pipe_crc_source {
1869         INTEL_PIPE_CRC_SOURCE_NONE,
1870         INTEL_PIPE_CRC_SOURCE_PLANE1,
1871         INTEL_PIPE_CRC_SOURCE_PLANE2,
1872         INTEL_PIPE_CRC_SOURCE_PF,
1873         INTEL_PIPE_CRC_SOURCE_PIPE,
1874         /* TV/DP on pre-gen5/vlv can't use the pipe source. */
1875         INTEL_PIPE_CRC_SOURCE_TV,
1876         INTEL_PIPE_CRC_SOURCE_DP_B,
1877         INTEL_PIPE_CRC_SOURCE_DP_C,
1878         INTEL_PIPE_CRC_SOURCE_DP_D,
1879         INTEL_PIPE_CRC_SOURCE_AUTO,
1880         INTEL_PIPE_CRC_SOURCE_MAX,
1881 };
1882
1883 struct intel_pipe_crc_entry {
1884         uint32_t frame;
1885         uint32_t crc[5];
1886 };
1887
1888 #define INTEL_PIPE_CRC_ENTRIES_NR       128
1889 struct intel_pipe_crc {
1890         spinlock_t lock;
1891         bool opened;            /* exclusive access to the result file */
1892         struct intel_pipe_crc_entry *entries;
1893         enum intel_pipe_crc_source source;
1894         int head, tail;
1895         wait_queue_head_t wq;
1896         int skipped;
1897 };
1898
1899 struct i915_frontbuffer_tracking {
1900         spinlock_t lock;
1901
1902         /*
1903          * Tracking bits for delayed frontbuffer flushing du to gpu activity or
1904          * scheduled flips.
1905          */
1906         unsigned busy_bits;
1907         unsigned flip_bits;
1908 };
1909
1910 struct i915_wa_reg {
1911         i915_reg_t addr;
1912         u32 value;
1913         /* bitmask representing WA bits */
1914         u32 mask;
1915 };
1916
1917 /*
1918  * RING_MAX_NONPRIV_SLOTS is per-engine but at this point we are only
1919  * allowing it for RCS as we don't foresee any requirement of having
1920  * a whitelist for other engines. When it is really required for
1921  * other engines then the limit need to be increased.
1922  */
1923 #define I915_MAX_WA_REGS (16 + RING_MAX_NONPRIV_SLOTS)
1924
1925 struct i915_workarounds {
1926         struct i915_wa_reg reg[I915_MAX_WA_REGS];
1927         u32 count;
1928         u32 hw_whitelist_count[I915_NUM_ENGINES];
1929 };
1930
1931 struct i915_virtual_gpu {
1932         bool active;
1933         u32 caps;
1934 };
1935
1936 /* used in computing the new watermarks state */
1937 struct intel_wm_config {
1938         unsigned int num_pipes_active;
1939         bool sprites_enabled;
1940         bool sprites_scaled;
1941 };
1942
1943 struct i915_oa_format {
1944         u32 format;
1945         int size;
1946 };
1947
1948 struct i915_oa_reg {
1949         i915_reg_t addr;
1950         u32 value;
1951 };
1952
1953 struct i915_oa_config {
1954         char uuid[UUID_STRING_LEN + 1];
1955         int id;
1956
1957         const struct i915_oa_reg *mux_regs;
1958         u32 mux_regs_len;
1959         const struct i915_oa_reg *b_counter_regs;
1960         u32 b_counter_regs_len;
1961         const struct i915_oa_reg *flex_regs;
1962         u32 flex_regs_len;
1963
1964         struct attribute_group sysfs_metric;
1965         struct attribute *attrs[2];
1966         struct device_attribute sysfs_metric_id;
1967
1968         atomic_t ref_count;
1969 };
1970
1971 struct i915_perf_stream;
1972
1973 /**
1974  * struct i915_perf_stream_ops - the OPs to support a specific stream type
1975  */
1976 struct i915_perf_stream_ops {
1977         /**
1978          * @enable: Enables the collection of HW samples, either in response to
1979          * `I915_PERF_IOCTL_ENABLE` or implicitly called when stream is opened
1980          * without `I915_PERF_FLAG_DISABLED`.
1981          */
1982         void (*enable)(struct i915_perf_stream *stream);
1983
1984         /**
1985          * @disable: Disables the collection of HW samples, either in response
1986          * to `I915_PERF_IOCTL_DISABLE` or implicitly called before destroying
1987          * the stream.
1988          */
1989         void (*disable)(struct i915_perf_stream *stream);
1990
1991         /**
1992          * @poll_wait: Call poll_wait, passing a wait queue that will be woken
1993          * once there is something ready to read() for the stream
1994          */
1995         void (*poll_wait)(struct i915_perf_stream *stream,
1996                           struct file *file,
1997                           poll_table *wait);
1998
1999         /**
2000          * @wait_unlocked: For handling a blocking read, wait until there is
2001          * something to ready to read() for the stream. E.g. wait on the same
2002          * wait queue that would be passed to poll_wait().
2003          */
2004         int (*wait_unlocked)(struct i915_perf_stream *stream);
2005
2006         /**
2007          * @read: Copy buffered metrics as records to userspace
2008          * **buf**: the userspace, destination buffer
2009          * **count**: the number of bytes to copy, requested by userspace
2010          * **offset**: zero at the start of the read, updated as the read
2011          * proceeds, it represents how many bytes have been copied so far and
2012          * the buffer offset for copying the next record.
2013          *
2014          * Copy as many buffered i915 perf samples and records for this stream
2015          * to userspace as will fit in the given buffer.
2016          *
2017          * Only write complete records; returning -%ENOSPC if there isn't room
2018          * for a complete record.
2019          *
2020          * Return any error condition that results in a short read such as
2021          * -%ENOSPC or -%EFAULT, even though these may be squashed before
2022          * returning to userspace.
2023          */
2024         int (*read)(struct i915_perf_stream *stream,
2025                     char __user *buf,
2026                     size_t count,
2027                     size_t *offset);
2028
2029         /**
2030          * @destroy: Cleanup any stream specific resources.
2031          *
2032          * The stream will always be disabled before this is called.
2033          */
2034         void (*destroy)(struct i915_perf_stream *stream);
2035 };
2036
2037 /**
2038  * struct i915_perf_stream - state for a single open stream FD
2039  */
2040 struct i915_perf_stream {
2041         /**
2042          * @dev_priv: i915 drm device
2043          */
2044         struct drm_i915_private *dev_priv;
2045
2046         /**
2047          * @link: Links the stream into ``&drm_i915_private->streams``
2048          */
2049         struct list_head link;
2050
2051         /**
2052          * @sample_flags: Flags representing the `DRM_I915_PERF_PROP_SAMPLE_*`
2053          * properties given when opening a stream, representing the contents
2054          * of a single sample as read() by userspace.
2055          */
2056         u32 sample_flags;
2057
2058         /**
2059          * @sample_size: Considering the configured contents of a sample
2060          * combined with the required header size, this is the total size
2061          * of a single sample record.
2062          */
2063         int sample_size;
2064
2065         /**
2066          * @ctx: %NULL if measuring system-wide across all contexts or a
2067          * specific context that is being monitored.
2068          */
2069         struct i915_gem_context *ctx;
2070
2071         /**
2072          * @enabled: Whether the stream is currently enabled, considering
2073          * whether the stream was opened in a disabled state and based
2074          * on `I915_PERF_IOCTL_ENABLE` and `I915_PERF_IOCTL_DISABLE` calls.
2075          */
2076         bool enabled;
2077
2078         /**
2079          * @ops: The callbacks providing the implementation of this specific
2080          * type of configured stream.
2081          */
2082         const struct i915_perf_stream_ops *ops;
2083
2084         /**
2085          * @oa_config: The OA configuration used by the stream.
2086          */
2087         struct i915_oa_config *oa_config;
2088 };
2089
2090 /**
2091  * struct i915_oa_ops - Gen specific implementation of an OA unit stream
2092  */
2093 struct i915_oa_ops {
2094         /**
2095          * @is_valid_b_counter_reg: Validates register's address for
2096          * programming boolean counters for a particular platform.
2097          */
2098         bool (*is_valid_b_counter_reg)(struct drm_i915_private *dev_priv,
2099                                        u32 addr);
2100
2101         /**
2102          * @is_valid_mux_reg: Validates register's address for programming mux
2103          * for a particular platform.
2104          */
2105         bool (*is_valid_mux_reg)(struct drm_i915_private *dev_priv, u32 addr);
2106
2107         /**
2108          * @is_valid_flex_reg: Validates register's address for programming
2109          * flex EU filtering for a particular platform.
2110          */
2111         bool (*is_valid_flex_reg)(struct drm_i915_private *dev_priv, u32 addr);
2112
2113         /**
2114          * @init_oa_buffer: Resets the head and tail pointers of the
2115          * circular buffer for periodic OA reports.
2116          *
2117          * Called when first opening a stream for OA metrics, but also may be
2118          * called in response to an OA buffer overflow or other error
2119          * condition.
2120          *
2121          * Note it may be necessary to clear the full OA buffer here as part of
2122          * maintaining the invariable that new reports must be written to
2123          * zeroed memory for us to be able to reliable detect if an expected
2124          * report has not yet landed in memory.  (At least on Haswell the OA
2125          * buffer tail pointer is not synchronized with reports being visible
2126          * to the CPU)
2127          */
2128         void (*init_oa_buffer)(struct drm_i915_private *dev_priv);
2129
2130         /**
2131          * @enable_metric_set: Selects and applies any MUX configuration to set
2132          * up the Boolean and Custom (B/C) counters that are part of the
2133          * counter reports being sampled. May apply system constraints such as
2134          * disabling EU clock gating as required.
2135          */
2136         int (*enable_metric_set)(struct drm_i915_private *dev_priv,
2137                                  const struct i915_oa_config *oa_config);
2138
2139         /**
2140          * @disable_metric_set: Remove system constraints associated with using
2141          * the OA unit.
2142          */
2143         void (*disable_metric_set)(struct drm_i915_private *dev_priv);
2144
2145         /**
2146          * @oa_enable: Enable periodic sampling
2147          */
2148         void (*oa_enable)(struct drm_i915_private *dev_priv);
2149
2150         /**
2151          * @oa_disable: Disable periodic sampling
2152          */
2153         void (*oa_disable)(struct drm_i915_private *dev_priv);
2154
2155         /**
2156          * @read: Copy data from the circular OA buffer into a given userspace
2157          * buffer.
2158          */
2159         int (*read)(struct i915_perf_stream *stream,
2160                     char __user *buf,
2161                     size_t count,
2162                     size_t *offset);
2163
2164         /**
2165          * @oa_hw_tail_read: read the OA tail pointer register
2166          *
2167          * In particular this enables us to share all the fiddly code for
2168          * handling the OA unit tail pointer race that affects multiple
2169          * generations.
2170          */
2171         u32 (*oa_hw_tail_read)(struct drm_i915_private *dev_priv);
2172 };
2173
2174 struct intel_cdclk_state {
2175         unsigned int cdclk, vco, ref;
2176 };
2177
2178 struct drm_i915_private {
2179         struct drm_device drm;
2180
2181         struct kmem_cache *objects;
2182         struct kmem_cache *vmas;
2183         struct kmem_cache *luts;
2184         struct kmem_cache *requests;
2185         struct kmem_cache *dependencies;
2186         struct kmem_cache *priorities;
2187
2188         const struct intel_device_info info;
2189
2190         void __iomem *regs;
2191
2192         struct intel_uncore uncore;
2193
2194         struct i915_virtual_gpu vgpu;
2195
2196         struct intel_gvt *gvt;
2197
2198         struct intel_huc huc;
2199         struct intel_guc guc;
2200
2201         struct intel_csr csr;
2202
2203         struct intel_gmbus gmbus[GMBUS_NUM_PINS];
2204
2205         /** gmbus_mutex protects against concurrent usage of the single hw gmbus
2206          * controller on different i2c buses. */
2207         struct mutex gmbus_mutex;
2208
2209         /**
2210          * Base address of the gmbus and gpio block.
2211          */
2212         uint32_t gpio_mmio_base;
2213
2214         /* MMIO base address for MIPI regs */
2215         uint32_t mipi_mmio_base;
2216
2217         uint32_t psr_mmio_base;
2218
2219         uint32_t pps_mmio_base;
2220
2221         wait_queue_head_t gmbus_wait_queue;
2222
2223         struct pci_dev *bridge_dev;
2224         struct i915_gem_context *kernel_context;
2225         struct intel_engine_cs *engine[I915_NUM_ENGINES];
2226         struct i915_vma *semaphore;
2227
2228         struct drm_dma_handle *status_page_dmah;
2229         struct resource mch_res;
2230
2231         /* protects the irq masks */
2232         spinlock_t irq_lock;
2233
2234         bool display_irqs_enabled;
2235
2236         /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */
2237         struct pm_qos_request pm_qos;
2238
2239         /* Sideband mailbox protection */
2240         struct mutex sb_lock;
2241
2242         /** Cached value of IMR to avoid reads in updating the bitfield */
2243         union {
2244                 u32 irq_mask;
2245                 u32 de_irq_mask[I915_MAX_PIPES];
2246         };
2247         u32 gt_irq_mask;
2248         u32 pm_imr;
2249         u32 pm_ier;
2250         u32 pm_rps_events;
2251         u32 pm_guc_events;
2252         u32 pipestat_irq_mask[I915_MAX_PIPES];
2253
2254         struct i915_hotplug hotplug;
2255         struct intel_fbc fbc;
2256         struct i915_drrs drrs;
2257         struct intel_opregion opregion;
2258         struct intel_vbt_data vbt;
2259
2260         bool preserve_bios_swizzle;
2261
2262         /* overlay */
2263         struct intel_overlay *overlay;
2264
2265         /* backlight registers and fields in struct intel_panel */
2266         struct mutex backlight_lock;
2267
2268         /* LVDS info */
2269         bool no_aux_handshake;
2270
2271         /* protects panel power sequencer state */
2272         struct mutex pps_mutex;
2273
2274         struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
2275         int num_fence_regs; /* 8 on pre-965, 16 otherwise */
2276
2277         unsigned int fsb_freq, mem_freq, is_ddr3;
2278         unsigned int skl_preferred_vco_freq;
2279         unsigned int max_cdclk_freq;
2280
2281         unsigned int max_dotclk_freq;
2282         unsigned int rawclk_freq;
2283         unsigned int hpll_freq;
2284         unsigned int czclk_freq;
2285
2286         struct {
2287                 /*
2288                  * The current logical cdclk state.
2289                  * See intel_atomic_state.cdclk.logical
2290                  *
2291                  * For reading holding any crtc lock is sufficient,
2292                  * for writing must hold all of them.
2293                  */
2294                 struct intel_cdclk_state logical;
2295                 /*
2296                  * The current actual cdclk state.
2297                  * See intel_atomic_state.cdclk.actual
2298                  */
2299                 struct intel_cdclk_state actual;
2300                 /* The current hardware cdclk state */
2301                 struct intel_cdclk_state hw;
2302         } cdclk;
2303
2304         /**
2305          * wq - Driver workqueue for GEM.
2306          *
2307          * NOTE: Work items scheduled here are not allowed to grab any modeset
2308          * locks, for otherwise the flushing done in the pageflip code will
2309          * result in deadlocks.
2310          */
2311         struct workqueue_struct *wq;
2312
2313         /* Display functions */
2314         struct drm_i915_display_funcs display;
2315
2316         /* PCH chipset type */
2317         enum intel_pch pch_type;
2318         unsigned short pch_id;
2319
2320         unsigned long quirks;
2321
2322         enum modeset_restore modeset_restore;
2323         struct mutex modeset_restore_lock;
2324         struct drm_atomic_state *modeset_restore_state;
2325         struct drm_modeset_acquire_ctx reset_ctx;
2326
2327         struct list_head vm_list; /* Global list of all address spaces */
2328         struct i915_ggtt ggtt; /* VM representing the global address space */
2329
2330         struct i915_gem_mm mm;
2331         DECLARE_HASHTABLE(mm_structs, 7);
2332         struct mutex mm_lock;
2333
2334         /* Kernel Modesetting */
2335
2336         struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
2337         struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES];
2338
2339 #ifdef CONFIG_DEBUG_FS
2340         struct intel_pipe_crc pipe_crc[I915_MAX_PIPES];
2341 #endif
2342
2343         /* dpll and cdclk state is protected by connection_mutex */
2344         int num_shared_dpll;
2345         struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
2346         const struct intel_dpll_mgr *dpll_mgr;
2347
2348         /*
2349          * dpll_lock serializes intel_{prepare,enable,disable}_shared_dpll.
2350          * Must be global rather than per dpll, because on some platforms
2351          * plls share registers.
2352          */
2353         struct mutex dpll_lock;
2354
2355         unsigned int active_crtcs;
2356         /* minimum acceptable cdclk for each pipe */
2357         int min_cdclk[I915_MAX_PIPES];
2358
2359         int dpio_phy_iosf_port[I915_NUM_PHYS_VLV];
2360
2361         struct i915_workarounds workarounds;
2362
2363         struct i915_frontbuffer_tracking fb_tracking;
2364
2365         struct intel_atomic_helper {
2366                 struct llist_head free_list;
2367                 struct work_struct free_work;
2368         } atomic_helper;
2369
2370         u16 orig_clock;
2371
2372         bool mchbar_need_disable;
2373
2374         struct intel_l3_parity l3_parity;
2375
2376         /* Cannot be determined by PCIID. You must always read a register. */
2377         u32 edram_cap;
2378
2379         /* gen6+ rps state */
2380         struct intel_gen6_power_mgmt rps;
2381
2382         /* ilk-only ips/rps state. Everything in here is protected by the global
2383          * mchdev_lock in intel_pm.c */
2384         struct intel_ilk_power_mgmt ips;
2385
2386         struct i915_power_domains power_domains;
2387
2388         struct i915_psr psr;
2389
2390         struct i915_gpu_error gpu_error;
2391
2392         struct drm_i915_gem_object *vlv_pctx;
2393
2394         /* list of fbdev register on this device */
2395         struct intel_fbdev *fbdev;
2396         struct work_struct fbdev_suspend_work;
2397
2398         struct drm_property *broadcast_rgb_property;
2399         struct drm_property *force_audio_property;
2400
2401         /* hda/i915 audio component */
2402         struct i915_audio_component *audio_component;
2403         bool audio_component_registered;
2404         /**
2405          * av_mutex - mutex for audio/video sync
2406          *
2407          */
2408         struct mutex av_mutex;
2409
2410         struct {
2411                 struct list_head list;
2412                 struct llist_head free_list;
2413                 struct work_struct free_work;
2414
2415                 /* The hw wants to have a stable context identifier for the
2416                  * lifetime of the context (for OA, PASID, faults, etc).
2417                  * This is limited in execlists to 21 bits.
2418                  */
2419                 struct ida hw_ida;
2420 #define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */
2421         } contexts;
2422
2423         u32 fdi_rx_config;
2424
2425         /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */
2426         u32 chv_phy_control;
2427         /*
2428          * Shadows for CHV DPLL_MD regs to keep the state
2429          * checker somewhat working in the presence hardware
2430          * crappiness (can't read out DPLL_MD for pipes B & C).
2431          */
2432         u32 chv_dpll_md[I915_MAX_PIPES];
2433         u32 bxt_phy_grc;
2434
2435         u32 suspend_count;
2436         bool suspended_to_idle;
2437         struct i915_suspend_saved_registers regfile;
2438         struct vlv_s0ix_state vlv_s0ix_state;
2439
2440         enum {
2441                 I915_SAGV_UNKNOWN = 0,
2442                 I915_SAGV_DISABLED,
2443                 I915_SAGV_ENABLED,
2444                 I915_SAGV_NOT_CONTROLLED
2445         } sagv_status;
2446
2447         struct {
2448                 /*
2449                  * Raw watermark latency values:
2450                  * in 0.1us units for WM0,
2451                  * in 0.5us units for WM1+.
2452                  */
2453                 /* primary */
2454                 uint16_t pri_latency[5];
2455                 /* sprite */
2456                 uint16_t spr_latency[5];
2457                 /* cursor */
2458                 uint16_t cur_latency[5];
2459                 /*
2460                  * Raw watermark memory latency values
2461                  * for SKL for all 8 levels
2462                  * in 1us units.
2463                  */
2464                 uint16_t skl_latency[8];
2465
2466                 /* current hardware state */
2467                 union {
2468                         struct ilk_wm_values hw;
2469                         struct skl_wm_values skl_hw;
2470                         struct vlv_wm_values vlv;
2471                         struct g4x_wm_values g4x;
2472                 };
2473
2474                 uint8_t max_level;
2475
2476                 /*
2477                  * Should be held around atomic WM register writing; also
2478                  * protects * intel_crtc->wm.active and
2479                  * cstate->wm.need_postvbl_update.
2480                  */
2481                 struct mutex wm_mutex;
2482
2483                 /*
2484                  * Set during HW readout of watermarks/DDB.  Some platforms
2485                  * need to know when we're still using BIOS-provided values
2486                  * (which we don't fully trust).
2487                  */
2488                 bool distrust_bios_wm;
2489         } wm;
2490
2491         struct i915_runtime_pm pm;
2492
2493         struct {
2494                 bool initialized;
2495
2496                 struct kobject *metrics_kobj;
2497                 struct ctl_table_header *sysctl_header;
2498
2499                 /*
2500                  * Lock associated with adding/modifying/removing OA configs
2501                  * in dev_priv->perf.metrics_idr.
2502                  */
2503                 struct mutex metrics_lock;
2504
2505                 /*
2506                  * List of dynamic configurations, you need to hold
2507                  * dev_priv->perf.metrics_lock to access it.
2508                  */
2509                 struct idr metrics_idr;
2510
2511                 /*
2512                  * Lock associated with anything below within this structure
2513                  * except exclusive_stream.
2514                  */
2515                 struct mutex lock;
2516                 struct list_head streams;
2517
2518                 struct {
2519                         /*
2520                          * The stream currently using the OA unit. If accessed
2521                          * outside a syscall associated to its file
2522                          * descriptor, you need to hold
2523                          * dev_priv->drm.struct_mutex.
2524                          */
2525                         struct i915_perf_stream *exclusive_stream;
2526
2527                         u32 specific_ctx_id;
2528
2529                         struct hrtimer poll_check_timer;
2530                         wait_queue_head_t poll_wq;
2531                         bool pollin;
2532
2533                         /**
2534                          * For rate limiting any notifications of spurious
2535                          * invalid OA reports
2536                          */
2537                         struct ratelimit_state spurious_report_rs;
2538
2539                         bool periodic;
2540                         int period_exponent;
2541                         int timestamp_frequency;
2542
2543                         struct i915_oa_config test_config;
2544
2545                         struct {
2546                                 struct i915_vma *vma;
2547                                 u8 *vaddr;
2548                                 u32 last_ctx_id;
2549                                 int format;
2550                                 int format_size;
2551
2552                                 /**
2553                                  * Locks reads and writes to all head/tail state
2554                                  *
2555                                  * Consider: the head and tail pointer state
2556                                  * needs to be read consistently from a hrtimer
2557                                  * callback (atomic context) and read() fop
2558                                  * (user context) with tail pointer updates
2559                                  * happening in atomic context and head updates
2560                                  * in user context and the (unlikely)
2561                                  * possibility of read() errors needing to
2562                                  * reset all head/tail state.
2563                                  *
2564                                  * Note: Contention or performance aren't
2565                                  * currently a significant concern here
2566                                  * considering the relatively low frequency of
2567                                  * hrtimer callbacks (5ms period) and that
2568                                  * reads typically only happen in response to a
2569                                  * hrtimer event and likely complete before the
2570                                  * next callback.
2571                                  *
2572                                  * Note: This lock is not held *while* reading
2573                                  * and copying data to userspace so the value
2574                                  * of head observed in htrimer callbacks won't
2575                                  * represent any partial consumption of data.
2576                                  */
2577                                 spinlock_t ptr_lock;
2578
2579                                 /**
2580                                  * One 'aging' tail pointer and one 'aged'
2581                                  * tail pointer ready to used for reading.
2582                                  *
2583                                  * Initial values of 0xffffffff are invalid
2584                                  * and imply that an update is required
2585                                  * (and should be ignored by an attempted
2586                                  * read)
2587                                  */
2588                                 struct {
2589                                         u32 offset;
2590                                 } tails[2];
2591
2592                                 /**
2593                                  * Index for the aged tail ready to read()
2594                                  * data up to.
2595                                  */
2596                                 unsigned int aged_tail_idx;
2597
2598                                 /**
2599                                  * A monotonic timestamp for when the current
2600                                  * aging tail pointer was read; used to
2601                                  * determine when it is old enough to trust.
2602                                  */
2603                                 u64 aging_timestamp;
2604
2605                                 /**
2606                                  * Although we can always read back the head
2607                                  * pointer register, we prefer to avoid
2608                                  * trusting the HW state, just to avoid any
2609                                  * risk that some hardware condition could
2610                                  * somehow bump the head pointer unpredictably
2611                                  * and cause us to forward the wrong OA buffer
2612                                  * data to userspace.
2613                                  */
2614                                 u32 head;
2615                         } oa_buffer;
2616
2617                         u32 gen7_latched_oastatus1;
2618                         u32 ctx_oactxctrl_offset;
2619                         u32 ctx_flexeu0_offset;
2620
2621                         /**
2622                          * The RPT_ID/reason field for Gen8+ includes a bit
2623                          * to determine if the CTX ID in the report is valid
2624                          * but the specific bit differs between Gen 8 and 9
2625                          */
2626                         u32 gen8_valid_ctx_bit;
2627
2628                         struct i915_oa_ops ops;
2629                         const struct i915_oa_format *oa_formats;
2630                 } oa;
2631         } perf;
2632
2633         /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
2634         struct {
2635                 void (*resume)(struct drm_i915_private *);
2636                 void (*cleanup_engine)(struct intel_engine_cs *engine);
2637
2638                 struct list_head timelines;
2639                 struct i915_gem_timeline global_timeline;
2640                 u32 active_requests;
2641
2642                 /**
2643                  * Is the GPU currently considered idle, or busy executing
2644                  * userspace requests? Whilst idle, we allow runtime power
2645                  * management to power down the hardware and display clocks.
2646                  * In order to reduce the effect on performance, there
2647                  * is a slight delay before we do so.
2648                  */
2649                 bool awake;
2650
2651                 /**
2652                  * We leave the user IRQ off as much as possible,
2653                  * but this means that requests will finish and never
2654                  * be retired once the system goes idle. Set a timer to
2655                  * fire periodically while the ring is running. When it
2656                  * fires, go retire requests.
2657                  */
2658                 struct delayed_work retire_work;
2659
2660                 /**
2661                  * When we detect an idle GPU, we want to turn on
2662                  * powersaving features. So once we see that there
2663                  * are no more requests outstanding and no more
2664                  * arrive within a small period of time, we fire
2665                  * off the idle_work.
2666                  */
2667                 struct delayed_work idle_work;
2668
2669                 ktime_t last_init_time;
2670         } gt;
2671
2672         /* perform PHY state sanity checks? */
2673         bool chv_phy_assert[2];
2674
2675         bool ipc_enabled;
2676
2677         /* Used to save the pipe-to-encoder mapping for audio */
2678         struct intel_encoder *av_enc_map[I915_MAX_PIPES];
2679
2680         /* necessary resource sharing with HDMI LPE audio driver. */
2681         struct {
2682                 struct platform_device *platdev;
2683                 int     irq;
2684         } lpe_audio;
2685
2686         /*
2687          * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
2688          * will be rejected. Instead look for a better place.
2689          */
2690 };
2691
2692 static inline struct drm_i915_private *to_i915(const struct drm_device *dev)
2693 {
2694         return container_of(dev, struct drm_i915_private, drm);
2695 }
2696
2697 static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
2698 {
2699         return to_i915(dev_get_drvdata(kdev));
2700 }
2701
2702 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
2703 {
2704         return container_of(guc, struct drm_i915_private, guc);
2705 }
2706
2707 static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
2708 {
2709         return container_of(huc, struct drm_i915_private, huc);
2710 }
2711
2712 /* Simple iterator over all initialised engines */
2713 #define for_each_engine(engine__, dev_priv__, id__) \
2714         for ((id__) = 0; \
2715              (id__) < I915_NUM_ENGINES; \
2716              (id__)++) \
2717                 for_each_if ((engine__) = (dev_priv__)->engine[(id__)])
2718
2719 /* Iterator over subset of engines selected by mask */
2720 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
2721         for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;        \
2722              tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
2723
2724 enum hdmi_force_audio {
2725         HDMI_AUDIO_OFF_DVI = -2,        /* no aux data for HDMI-DVI converter */
2726         HDMI_AUDIO_OFF,                 /* force turn off HDMI audio */
2727         HDMI_AUDIO_AUTO,                /* trust EDID */
2728         HDMI_AUDIO_ON,                  /* force turn on HDMI audio */
2729 };
2730
2731 #define I915_GTT_OFFSET_NONE ((u32)-1)
2732
2733 /*
2734  * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
2735  * considered to be the frontbuffer for the given plane interface-wise. This
2736  * doesn't mean that the hw necessarily already scans it out, but that any
2737  * rendering (by the cpu or gpu) will land in the frontbuffer eventually.
2738  *
2739  * We have one bit per pipe and per scanout plane type.
2740  */
2741 #define INTEL_MAX_SPRITE_BITS_PER_PIPE 5
2742 #define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
2743 #define INTEL_FRONTBUFFER_PRIMARY(pipe) \
2744         (1 << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
2745 #define INTEL_FRONTBUFFER_CURSOR(pipe) \
2746         (1 << (1 + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
2747 #define INTEL_FRONTBUFFER_SPRITE(pipe, plane) \
2748         (1 << (2 + plane + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
2749 #define INTEL_FRONTBUFFER_OVERLAY(pipe) \
2750         (1 << (2 + INTEL_MAX_SPRITE_BITS_PER_PIPE + (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))))
2751 #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
2752         (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)))
2753
2754 /*
2755  * Optimised SGL iterator for GEM objects
2756  */
2757 static __always_inline struct sgt_iter {
2758         struct scatterlist *sgp;
2759         union {
2760                 unsigned long pfn;
2761                 dma_addr_t dma;
2762         };
2763         unsigned int curr;
2764         unsigned int max;
2765 } __sgt_iter(struct scatterlist *sgl, bool dma) {
2766         struct sgt_iter s = { .sgp = sgl };
2767
2768         if (s.sgp) {
2769                 s.max = s.curr = s.sgp->offset;
2770                 s.max += s.sgp->length;
2771                 if (dma)
2772                         s.dma = sg_dma_address(s.sgp);
2773                 else
2774                         s.pfn = page_to_pfn(sg_page(s.sgp));
2775         }
2776
2777         return s;
2778 }
2779
2780 static inline struct scatterlist *____sg_next(struct scatterlist *sg)
2781 {
2782         ++sg;
2783         if (unlikely(sg_is_chain(sg)))
2784                 sg = sg_chain_ptr(sg);
2785         return sg;
2786 }
2787
2788 /**
2789  * __sg_next - return the next scatterlist entry in a list
2790  * @sg:         The current sg entry
2791  *
2792  * Description:
2793  *   If the entry is the last, return NULL; otherwise, step to the next
2794  *   element in the array (@sg@+1). If that's a chain pointer, follow it;
2795  *   otherwise just return the pointer to the current element.
2796  **/
2797 static inline struct scatterlist *__sg_next(struct scatterlist *sg)
2798 {
2799 #ifdef CONFIG_DEBUG_SG
2800         BUG_ON(sg->sg_magic != SG_MAGIC);
2801 #endif
2802         return sg_is_last(sg) ? NULL : ____sg_next(sg);
2803 }
2804
2805 /**
2806  * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table
2807  * @__dmap:     DMA address (output)
2808  * @__iter:     'struct sgt_iter' (iterator state, internal)
2809  * @__sgt:      sg_table to iterate over (input)
2810  */
2811 #define for_each_sgt_dma(__dmap, __iter, __sgt)                         \
2812         for ((__iter) = __sgt_iter((__sgt)->sgl, true);                 \
2813              ((__dmap) = (__iter).dma + (__iter).curr);                 \
2814              (((__iter).curr += PAGE_SIZE) < (__iter).max) ||           \
2815              ((__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0))
2816
2817 /**
2818  * for_each_sgt_page - iterate over the pages of the given sg_table
2819  * @__pp:       page pointer (output)
2820  * @__iter:     'struct sgt_iter' (iterator state, internal)
2821  * @__sgt:      sg_table to iterate over (input)
2822  */
2823 #define for_each_sgt_page(__pp, __iter, __sgt)                          \
2824         for ((__iter) = __sgt_iter((__sgt)->sgl, false);                \
2825              ((__pp) = (__iter).pfn == 0 ? NULL :                       \
2826               pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \
2827              (((__iter).curr += PAGE_SIZE) < (__iter).max) ||           \
2828              ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
2829
2830 static inline const struct intel_device_info *
2831 intel_info(const struct drm_i915_private *dev_priv)
2832 {
2833         return &dev_priv->info;
2834 }
2835
2836 #define INTEL_INFO(dev_priv)    intel_info((dev_priv))
2837
2838 #define INTEL_GEN(dev_priv)     ((dev_priv)->info.gen)
2839 #define INTEL_DEVID(dev_priv)   ((dev_priv)->info.device_id)
2840
2841 #define REVID_FOREVER           0xff
2842 #define INTEL_REVID(dev_priv)   ((dev_priv)->drm.pdev->revision)
2843
2844 #define GEN_FOREVER (0)
2845 /*
2846  * Returns true if Gen is in inclusive range [Start, End].
2847  *
2848  * Use GEN_FOREVER for unbound start and or end.
2849  */
2850 #define IS_GEN(dev_priv, s, e) ({ \
2851         unsigned int __s = (s), __e = (e); \
2852         BUILD_BUG_ON(!__builtin_constant_p(s)); \
2853         BUILD_BUG_ON(!__builtin_constant_p(e)); \
2854         if ((__s) != GEN_FOREVER) \
2855                 __s = (s) - 1; \
2856         if ((__e) == GEN_FOREVER) \
2857                 __e = BITS_PER_LONG - 1; \
2858         else \
2859                 __e = (e) - 1; \
2860         !!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \
2861 })
2862
2863 /*
2864  * Return true if revision is in range [since,until] inclusive.
2865  *
2866  * Use 0 for open-ended since, and REVID_FOREVER for open-ended until.
2867  */
2868 #define IS_REVID(p, since, until) \
2869         (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
2870
2871 #define IS_I830(dev_priv)       ((dev_priv)->info.platform == INTEL_I830)
2872 #define IS_I845G(dev_priv)      ((dev_priv)->info.platform == INTEL_I845G)
2873 #define IS_I85X(dev_priv)       ((dev_priv)->info.platform == INTEL_I85X)
2874 #define IS_I865G(dev_priv)      ((dev_priv)->info.platform == INTEL_I865G)
2875 #define IS_I915G(dev_priv)      ((dev_priv)->info.platform == INTEL_I915G)
2876 #define IS_I915GM(dev_priv)     ((dev_priv)->info.platform == INTEL_I915GM)
2877 #define IS_I945G(dev_priv)      ((dev_priv)->info.platform == INTEL_I945G)
2878 #define IS_I945GM(dev_priv)     ((dev_priv)->info.platform == INTEL_I945GM)
2879 #define IS_I965G(dev_priv)      ((dev_priv)->info.platform == INTEL_I965G)
2880 #define IS_I965GM(dev_priv)     ((dev_priv)->info.platform == INTEL_I965GM)
2881 #define IS_G45(dev_priv)        ((dev_priv)->info.platform == INTEL_G45)
2882 #define IS_GM45(dev_priv)       ((dev_priv)->info.platform == INTEL_GM45)
2883 #define IS_G4X(dev_priv)        (IS_G45(dev_priv) || IS_GM45(dev_priv))
2884 #define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001)
2885 #define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011)
2886 #define IS_PINEVIEW(dev_priv)   ((dev_priv)->info.platform == INTEL_PINEVIEW)
2887 #define IS_G33(dev_priv)        ((dev_priv)->info.platform == INTEL_G33)
2888 #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046)
2889 #define IS_IVYBRIDGE(dev_priv)  ((dev_priv)->info.platform == INTEL_IVYBRIDGE)
2890 #define IS_IVB_GT1(dev_priv)    (IS_IVYBRIDGE(dev_priv) && \
2891                                  (dev_priv)->info.gt == 1)
2892 #define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_VALLEYVIEW)
2893 #define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.platform == INTEL_CHERRYVIEW)
2894 #define IS_HASWELL(dev_priv)    ((dev_priv)->info.platform == INTEL_HASWELL)
2895 #define IS_BROADWELL(dev_priv)  ((dev_priv)->info.platform == INTEL_BROADWELL)
2896 #define IS_SKYLAKE(dev_priv)    ((dev_priv)->info.platform == INTEL_SKYLAKE)
2897 #define IS_BROXTON(dev_priv)    ((dev_priv)->info.platform == INTEL_BROXTON)
2898 #define IS_KABYLAKE(dev_priv)   ((dev_priv)->info.platform == INTEL_KABYLAKE)
2899 #define IS_GEMINILAKE(dev_priv) ((dev_priv)->info.platform == INTEL_GEMINILAKE)
2900 #define IS_COFFEELAKE(dev_priv) ((dev_priv)->info.platform == INTEL_COFFEELAKE)
2901 #define IS_CANNONLAKE(dev_priv) ((dev_priv)->info.platform == INTEL_CANNONLAKE)
2902 #define IS_MOBILE(dev_priv)     ((dev_priv)->info.is_mobile)
2903 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
2904                                     (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
2905 #define IS_BDW_ULT(dev_priv)    (IS_BROADWELL(dev_priv) && \
2906                                  ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 ||       \
2907                                  (INTEL_DEVID(dev_priv) & 0xf) == 0xb ||        \
2908                                  (INTEL_DEVID(dev_priv) & 0xf) == 0xe))
2909 /* ULX machines are also considered ULT. */
2910 #define IS_BDW_ULX(dev_priv)    (IS_BROADWELL(dev_priv) && \
2911                                  (INTEL_DEVID(dev_priv) & 0xf) == 0xe)
2912 #define IS_BDW_GT3(dev_priv)    (IS_BROADWELL(dev_priv) && \
2913                                  (dev_priv)->info.gt == 3)
2914 #define IS_HSW_ULT(dev_priv)    (IS_HASWELL(dev_priv) && \
2915                                  (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00)
2916 #define IS_HSW_GT3(dev_priv)    (IS_HASWELL(dev_priv) && \
2917                                  (dev_priv)->info.gt == 3)
2918 /* ULX machines are also considered ULT. */
2919 #define IS_HSW_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x0A0E || \
2920                                  INTEL_DEVID(dev_priv) == 0x0A1E)
2921 #define IS_SKL_ULT(dev_priv)    (INTEL_DEVID(dev_priv) == 0x1906 || \
2922                                  INTEL_DEVID(dev_priv) == 0x1913 || \
2923                                  INTEL_DEVID(dev_priv) == 0x1916 || \
2924                                  INTEL_DEVID(dev_priv) == 0x1921 || \
2925                                  INTEL_DEVID(dev_priv) == 0x1926)
2926 #define IS_SKL_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x190E || \
2927                                  INTEL_DEVID(dev_priv) == 0x1915 || \
2928                                  INTEL_DEVID(dev_priv) == 0x191E)
2929 #define IS_KBL_ULT(dev_priv)    (INTEL_DEVID(dev_priv) == 0x5906 || \
2930                                  INTEL_DEVID(dev_priv) == 0x5913 || \
2931                                  INTEL_DEVID(dev_priv) == 0x5916 || \
2932                                  INTEL_DEVID(dev_priv) == 0x5921 || \
2933                                  INTEL_DEVID(dev_priv) == 0x5926)
2934 #define IS_KBL_ULX(dev_priv)    (INTEL_DEVID(dev_priv) == 0x590E || \
2935                                  INTEL_DEVID(dev_priv) == 0x5915 || \
2936                                  INTEL_DEVID(dev_priv) == 0x591E)
2937 #define IS_SKL_GT2(dev_priv)    (IS_SKYLAKE(dev_priv) && \
2938                                  (dev_priv)->info.gt == 2)
2939 #define IS_SKL_GT3(dev_priv)    (IS_SKYLAKE(dev_priv) && \
2940                                  (dev_priv)->info.gt == 3)
2941 #define IS_SKL_GT4(dev_priv)    (IS_SKYLAKE(dev_priv) && \
2942                                  (dev_priv)->info.gt == 4)
2943 #define IS_KBL_GT2(dev_priv)    (IS_KABYLAKE(dev_priv) && \
2944                                  (dev_priv)->info.gt == 2)
2945 #define IS_KBL_GT3(dev_priv)    (IS_KABYLAKE(dev_priv) && \
2946                                  (dev_priv)->info.gt == 3)
2947 #define IS_CFL_ULT(dev_priv)    (IS_COFFEELAKE(dev_priv) && \
2948                                  (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
2949
2950 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
2951
2952 #define SKL_REVID_A0            0x0
2953 #define SKL_REVID_B0            0x1
2954 #define SKL_REVID_C0            0x2
2955 #define SKL_REVID_D0            0x3
2956 #define SKL_REVID_E0            0x4
2957 #define SKL_REVID_F0            0x5
2958 #define SKL_REVID_G0            0x6
2959 #define SKL_REVID_H0            0x7
2960
2961 #define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until))
2962
2963 #define BXT_REVID_A0            0x0
2964 #define BXT_REVID_A1            0x1
2965 #define BXT_REVID_B0            0x3
2966 #define BXT_REVID_B_LAST        0x8
2967 #define BXT_REVID_C0            0x9
2968
2969 #define IS_BXT_REVID(dev_priv, since, until) \
2970         (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until))
2971
2972 #define KBL_REVID_A0            0x0
2973 #define KBL_REVID_B0            0x1
2974 #define KBL_REVID_C0            0x2
2975 #define KBL_REVID_D0            0x3
2976 #define KBL_REVID_E0            0x4
2977
2978 #define IS_KBL_REVID(dev_priv, since, until) \
2979         (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until))
2980
2981 #define GLK_REVID_A0            0x0
2982 #define GLK_REVID_A1            0x1
2983
2984 #define IS_GLK_REVID(dev_priv, since, until) \
2985         (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until))
2986
2987 #define CNL_REVID_A0            0x0
2988 #define CNL_REVID_B0            0x1
2989
2990 #define IS_CNL_REVID(p, since, until) \
2991         (IS_CANNONLAKE(p) && IS_REVID(p, since, until))
2992
2993 /*
2994  * The genX designation typically refers to the render engine, so render
2995  * capability related checks should use IS_GEN, while display and other checks
2996  * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
2997  * chips, etc.).
2998  */
2999 #define IS_GEN2(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(1)))
3000 #define IS_GEN3(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(2)))
3001 #define IS_GEN4(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(3)))
3002 #define IS_GEN5(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(4)))
3003 #define IS_GEN6(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(5)))
3004 #define IS_GEN7(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(6)))
3005 #define IS_GEN8(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(7)))
3006 #define IS_GEN9(dev_priv)       (!!((dev_priv)->info.gen_mask & BIT(8)))
3007 #define IS_GEN10(dev_priv)      (!!((dev_priv)->info.gen_mask & BIT(9)))
3008
3009 #define IS_LP(dev_priv) (INTEL_INFO(dev_priv)->is_lp)
3010 #define IS_GEN9_LP(dev_priv)    (IS_GEN9(dev_priv) && IS_LP(dev_priv))
3011 #define IS_GEN9_BC(dev_priv)    (IS_GEN9(dev_priv) && !IS_LP(dev_priv))
3012
3013 #define ENGINE_MASK(id) BIT(id)
3014 #define RENDER_RING     ENGINE_MASK(RCS)
3015 #define BSD_RING        ENGINE_MASK(VCS)
3016 #define BLT_RING        ENGINE_MASK(BCS)
3017 #define VEBOX_RING      ENGINE_MASK(VECS)
3018 #define BSD2_RING       ENGINE_MASK(VCS2)
3019 #define ALL_ENGINES     (~0)
3020
3021 #define HAS_ENGINE(dev_priv, id) \
3022         (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id)))
3023
3024 #define HAS_BSD(dev_priv)       HAS_ENGINE(dev_priv, VCS)
3025 #define HAS_BSD2(dev_priv)      HAS_ENGINE(dev_priv, VCS2)
3026 #define HAS_BLT(dev_priv)       HAS_ENGINE(dev_priv, BCS)
3027 #define HAS_VEBOX(dev_priv)     HAS_ENGINE(dev_priv, VECS)
3028
3029 #define HAS_LLC(dev_priv)       ((dev_priv)->info.has_llc)
3030 #define HAS_SNOOP(dev_priv)     ((dev_priv)->info.has_snoop)
3031 #define HAS_EDRAM(dev_priv)     (!!((dev_priv)->edram_cap & EDRAM_ENABLED))
3032 #define HAS_WT(dev_priv)        ((IS_HASWELL(dev_priv) || \
3033                                  IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
3034
3035 #define HWS_NEEDS_PHYSICAL(dev_priv)    ((dev_priv)->info.hws_needs_physical)
3036
3037 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
3038                 ((dev_priv)->info.has_logical_ring_contexts)
3039 #define USES_PPGTT(dev_priv)            (i915.enable_ppgtt)
3040 #define USES_FULL_PPGTT(dev_priv)       (i915.enable_ppgtt >= 2)
3041 #define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3)
3042
3043 #define HAS_OVERLAY(dev_priv)            ((dev_priv)->info.has_overlay)
3044 #define OVERLAY_NEEDS_PHYSICAL(dev_priv) \
3045                 ((dev_priv)->info.overlay_needs_physical)
3046
3047 /* Early gen2 have a totally busted CS tlb and require pinned batches. */
3048 #define HAS_BROKEN_CS_TLB(dev_priv)     (IS_I830(dev_priv) || IS_I845G(dev_priv))
3049
3050 /* WaRsDisableCoarsePowerGating:skl,bxt */
3051 #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
3052         (IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
3053
3054 /*
3055  * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
3056  * even when in MSI mode. This results in spurious interrupt warnings if the
3057  * legacy irq no. is shared with another device. The kernel then disables that
3058  * interrupt source and so prevents the other device from working properly.
3059  */
3060 #define HAS_AUX_IRQ(dev_priv)   ((dev_priv)->info.gen >= 5)
3061 #define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq)
3062
3063 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
3064  * rows, which changed the alignment requirements and fence programming.
3065  */
3066 #define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \
3067                                          !(IS_I915G(dev_priv) || \
3068                                          IS_I915GM(dev_priv)))
3069 #define SUPPORTS_TV(dev_priv)           ((dev_priv)->info.supports_tv)
3070 #define I915_HAS_HOTPLUG(dev_priv)      ((dev_priv)->info.has_hotplug)
3071
3072 #define HAS_FW_BLC(dev_priv)    (INTEL_GEN(dev_priv) > 2)
3073 #define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr)
3074 #define HAS_FBC(dev_priv)       ((dev_priv)->info.has_fbc)
3075 #define HAS_CUR_FBC(dev_priv)   (!HAS_GMCH_DISPLAY(dev_priv) && INTEL_INFO(dev_priv)->gen >= 7)
3076
3077 #define HAS_IPS(dev_priv)       (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
3078
3079 #define HAS_DP_MST(dev_priv)    ((dev_priv)->info.has_dp_mst)
3080
3081 #define HAS_DDI(dev_priv)                ((dev_priv)->info.has_ddi)
3082 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg)
3083 #define HAS_PSR(dev_priv)                ((dev_priv)->info.has_psr)
3084 #define HAS_RC6(dev_priv)                ((dev_priv)->info.has_rc6)
3085 #define HAS_RC6p(dev_priv)               ((dev_priv)->info.has_rc6p)
3086
3087 #define HAS_CSR(dev_priv)       ((dev_priv)->info.has_csr)
3088
3089 #define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm)
3090 #define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc)
3091
3092 /*
3093  * For now, anything with a GuC requires uCode loading, and then supports
3094  * command submission once loaded. But these are logically independent
3095  * properties, so we have separate macros to test them.
3096  */
3097 #define HAS_GUC(dev_priv)       ((dev_priv)->info.has_guc)
3098 #define HAS_GUC_CT(dev_priv)    ((dev_priv)->info.has_guc_ct)
3099 #define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv))
3100 #define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv))
3101 #define HAS_HUC_UCODE(dev_priv) (HAS_GUC(dev_priv))
3102
3103 #define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer)
3104
3105 #define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu)
3106
3107 #define INTEL_PCH_DEVICE_ID_MASK                0xff80
3108 #define INTEL_PCH_IBX_DEVICE_ID_TYPE            0x3b00
3109 #define INTEL_PCH_CPT_DEVICE_ID_TYPE            0x1c00
3110 #define INTEL_PCH_PPT_DEVICE_ID_TYPE            0x1e00
3111 #define INTEL_PCH_LPT_DEVICE_ID_TYPE            0x8c00
3112 #define INTEL_PCH_LPT_LP_DEVICE_ID_TYPE         0x9c00
3113 #define INTEL_PCH_WPT_DEVICE_ID_TYPE            0x8c80
3114 #define INTEL_PCH_WPT_LP_DEVICE_ID_TYPE         0x9c80
3115 #define INTEL_PCH_SPT_DEVICE_ID_TYPE            0xA100
3116 #define INTEL_PCH_SPT_LP_DEVICE_ID_TYPE         0x9D00
3117 #define INTEL_PCH_KBP_DEVICE_ID_TYPE            0xA280
3118 #define INTEL_PCH_CNP_DEVICE_ID_TYPE            0xA300
3119 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE         0x9D80
3120 #define INTEL_PCH_P2X_DEVICE_ID_TYPE            0x7100
3121 #define INTEL_PCH_P3X_DEVICE_ID_TYPE            0x7000
3122 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE           0x2900 /* qemu q35 has 2918 */
3123
3124 #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type)
3125 #define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP)
3126 #define HAS_PCH_CNP_LP(dev_priv) \
3127         ((dev_priv)->pch_id == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE)
3128 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
3129 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
3130 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
3131 #define HAS_PCH_LPT_LP(dev_priv) \
3132         ((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE || \
3133          (dev_priv)->pch_id == INTEL_PCH_WPT_LP_DEVICE_ID_TYPE)
3134 #define HAS_PCH_LPT_H(dev_priv) \
3135         ((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE || \
3136          (dev_priv)->pch_id == INTEL_PCH_WPT_DEVICE_ID_TYPE)
3137 #define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT)
3138 #define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX)
3139 #define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP)
3140 #define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE)
3141
3142 #define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display)
3143
3144 #define HAS_LSPCON(dev_priv) (INTEL_GEN(dev_priv) >= 9)
3145
3146 /* DPF == dynamic parity feature */
3147 #define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf)
3148 #define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \
3149                                  2 : HAS_L3_DPF(dev_priv))
3150
3151 #define GT_FREQUENCY_MULTIPLIER 50
3152 #define GEN9_FREQ_SCALER 3
3153
3154 #include "i915_trace.h"
3155
3156 static inline bool intel_vtd_active(void)
3157 {
3158 #ifdef CONFIG_INTEL_IOMMU
3159         if (intel_iommu_gfx_mapped)
3160                 return true;
3161 #endif
3162         return false;
3163 }
3164
3165 static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
3166 {
3167         return INTEL_GEN(dev_priv) >= 6 && intel_vtd_active();
3168 }
3169
3170 static inline bool
3171 intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv)
3172 {
3173         return IS_BROXTON(dev_priv) && intel_vtd_active();
3174 }
3175
3176 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
3177                                 int enable_ppgtt);
3178
3179 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value);
3180
3181 /* i915_drv.c */
3182 void __printf(3, 4)
3183 __i915_printk(struct drm_i915_private *dev_priv, const char *level,
3184               const char *fmt, ...);
3185
3186 #define i915_report_error(dev_priv, fmt, ...)                              \
3187         __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__)
3188
3189 #ifdef CONFIG_COMPAT
3190 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
3191                               unsigned long arg);
3192 #else
3193 #define i915_compat_ioctl NULL
3194 #endif
3195 extern const struct dev_pm_ops i915_pm_ops;
3196
3197 extern int i915_driver_load(struct pci_dev *pdev,
3198                             const struct pci_device_id *ent);
3199 extern void i915_driver_unload(struct drm_device *dev);
3200 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
3201 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
3202
3203 #define I915_RESET_QUIET BIT(0)
3204 extern void i915_reset(struct drm_i915_private *i915, unsigned int flags);
3205 extern int i915_reset_engine(struct intel_engine_cs *engine,
3206                              unsigned int flags);
3207
3208 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
3209 extern int intel_guc_reset(struct drm_i915_private *dev_priv);
3210 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
3211 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv);
3212 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
3213 extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv);
3214 extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv);
3215 extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
3216 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
3217
3218 int intel_engines_init_mmio(struct drm_i915_private *dev_priv);
3219 int intel_engines_init(struct drm_i915_private *dev_priv);
3220
3221 /* intel_hotplug.c */
3222 void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
3223                            u32 pin_mask, u32 long_mask);
3224 void intel_hpd_init(struct drm_i915_private *dev_priv);
3225 void intel_hpd_init_work(struct drm_i915_private *dev_priv);
3226 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
3227 enum port intel_hpd_pin_to_port(enum hpd_pin pin);
3228 enum hpd_pin intel_hpd_pin(enum port port);
3229 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
3230 void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
3231
3232 /* i915_irq.c */
3233 static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv)
3234 {
3235         unsigned long delay;
3236
3237         if (unlikely(!i915.enable_hangcheck))
3238                 return;
3239
3240         /* Don't continually defer the hangcheck so that it is always run at
3241          * least once after work has been scheduled on any ring. Otherwise,
3242          * we will ignore a hung ring if a second ring is kept busy.
3243          */
3244
3245         delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES);
3246         queue_delayed_work(system_long_wq,
3247                            &dev_priv->gpu_error.hangcheck_work, delay);
3248 }
3249
3250 __printf(3, 4)
3251 void i915_handle_error(struct drm_i915_private *dev_priv,
3252                        u32 engine_mask,
3253                        const char *fmt, ...);
3254
3255 extern void intel_irq_init(struct drm_i915_private *dev_priv);
3256 extern void intel_irq_fini(struct drm_i915_private *dev_priv);
3257 int intel_irq_install(struct drm_i915_private *dev_priv);
3258 void intel_irq_uninstall(struct drm_i915_private *dev_priv);
3259
3260 static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
3261 {
3262         return dev_priv->gvt;
3263 }
3264
3265 static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv)
3266 {
3267         return dev_priv->vgpu.active;
3268 }
3269
3270 void
3271 i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
3272                      u32 status_mask);
3273
3274 void
3275 i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
3276                       u32 status_mask);
3277
3278 void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv);
3279 void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv);
3280 void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
3281                                    uint32_t mask,
3282                                    uint32_t bits);
3283 void ilk_update_display_irq(struct drm_i915_private *dev_priv,
3284                             uint32_t interrupt_mask,
3285                             uint32_t enabled_irq_mask);
3286 static inline void
3287 ilk_enable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits)
3288 {
3289         ilk_update_display_irq(dev_priv, bits, bits);
3290 }
3291 static inline void
3292 ilk_disable_display_irq(struct drm_i915_private *dev_priv, uint32_t bits)
3293 {
3294         ilk_update_display_irq(dev_priv, bits, 0);
3295 }
3296 void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
3297                          enum pipe pipe,
3298                          uint32_t interrupt_mask,
3299                          uint32_t enabled_irq_mask);
3300 static inline void bdw_enable_pipe_irq(struct drm_i915_private *dev_priv,
3301                                        enum pipe pipe, uint32_t bits)
3302 {
3303         bdw_update_pipe_irq(dev_priv, pipe, bits, bits);
3304 }
3305 static inline void bdw_disable_pipe_irq(struct drm_i915_private *dev_priv,
3306                                         enum pipe pipe, uint32_t bits)
3307 {
3308         bdw_update_pipe_irq(dev_priv, pipe, bits, 0);
3309 }
3310 void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
3311                                   uint32_t interrupt_mask,
3312                                   uint32_t enabled_irq_mask);
3313 static inline void
3314 ibx_enable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits)
3315 {
3316         ibx_display_interrupt_update(dev_priv, bits, bits);
3317 }
3318 static inline void
3319 ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits)
3320 {
3321         ibx_display_interrupt_update(dev_priv, bits, 0);
3322 }
3323
3324 /* i915_gem.c */
3325 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
3326                           struct drm_file *file_priv);
3327 int i915_gem_pread_ioctl(struct drm_device *dev, void *data,
3328                          struct drm_file *file_priv);
3329 int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
3330                           struct drm_file *file_priv);
3331 int i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
3332                         struct drm_file *file_priv);
3333 int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
3334                         struct drm_file *file_priv);
3335 int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
3336                               struct drm_file *file_priv);
3337 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
3338                              struct drm_file *file_priv);
3339 int i915_gem_execbuffer(struct drm_device *dev, void *data,
3340                         struct drm_file *file_priv);
3341 int i915_gem_execbuffer2(struct drm_device *dev, void *data,
3342                          struct drm_file *file_priv);
3343 int i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3344                         struct drm_file *file_priv);
3345 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3346                                struct drm_file *file);
3347 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3348                                struct drm_file *file);
3349 int i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3350                             struct drm_file *file_priv);
3351 int i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3352                            struct drm_file *file_priv);
3353 int i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
3354                               struct drm_file *file_priv);
3355 int i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
3356                               struct drm_file *file_priv);
3357 int i915_gem_init_userptr(struct drm_i915_private *dev_priv);
3358 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv);
3359 int i915_gem_userptr_ioctl(struct drm_device *dev, void *data,
3360                            struct drm_file *file);
3361 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
3362                                 struct drm_file *file_priv);
3363 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
3364                         struct drm_file *file_priv);
3365 void i915_gem_sanitize(struct drm_i915_private *i915);
3366 int i915_gem_load_init(struct drm_i915_private *dev_priv);
3367 void i915_gem_load_cleanup(struct drm_i915_private *dev_priv);
3368 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
3369 int i915_gem_freeze(struct drm_i915_private *dev_priv);
3370 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
3371
3372 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv);
3373 void i915_gem_object_free(struct drm_i915_gem_object *obj);
3374 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3375                          const struct drm_i915_gem_object_ops *ops);
3376 struct drm_i915_gem_object *
3377 i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size);
3378 struct drm_i915_gem_object *
3379 i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
3380                                  const void *data, size_t size);
3381 void i915_gem_close_object(