Merge drm/drm-next into drm-intel-next-queued
authorJani Nikula <jani.nikula@intel.com>
Wed, 2 May 2018 09:20:32 +0000 (12:20 +0300)
committerJani Nikula <jani.nikula@intel.com>
Wed, 2 May 2018 09:20:32 +0000 (12:20 +0300)
Need d224985a5e31 ("sched/wait, drivers/drm: Convert wait_on_atomic_t()
usage to the new wait_var_event() API") in dinq to be able to fix
https://bugs.freedesktop.org/show_bug.cgi?id=106085.

Signed-off-by: Jani Nikula <jani.nikula@intel.com>
109 files changed:
Documentation/gpu/i915.rst
drivers/gpu/drm/drm_atomic.c
drivers/gpu/drm/i915/Kconfig.debug
drivers/gpu/drm/i915/Makefile
drivers/gpu/drm/i915/gvt/cmd_parser.c
drivers/gpu/drm/i915/gvt/debugfs.c
drivers/gpu/drm/i915/gvt/gvt.h
drivers/gpu/drm/i915/gvt/mmio_context.c
drivers/gpu/drm/i915/gvt/sched_policy.c
drivers/gpu/drm/i915/gvt/scheduler.c
drivers/gpu/drm/i915/gvt/scheduler.h
drivers/gpu/drm/i915/gvt/trace.h
drivers/gpu/drm/i915/i915_debugfs.c
drivers/gpu/drm/i915/i915_drv.c
drivers/gpu/drm/i915/i915_drv.h
drivers/gpu/drm/i915/i915_gem.c
drivers/gpu/drm/i915/i915_gem.h
drivers/gpu/drm/i915/i915_gem_batch_pool.c
drivers/gpu/drm/i915/i915_gem_batch_pool.h
drivers/gpu/drm/i915/i915_gem_context.c
drivers/gpu/drm/i915/i915_gem_context.h
drivers/gpu/drm/i915/i915_gem_execbuffer.c
drivers/gpu/drm/i915/i915_gem_stolen.c
drivers/gpu/drm/i915/i915_gem_timeline.h
drivers/gpu/drm/i915/i915_gpu_error.c
drivers/gpu/drm/i915/i915_gpu_error.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_irq.c
drivers/gpu/drm/i915/i915_oa_icl.c [new file with mode: 0644]
drivers/gpu/drm/i915/i915_oa_icl.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_params.h
drivers/gpu/drm/i915/i915_pci.c
drivers/gpu/drm/i915/i915_perf.c
drivers/gpu/drm/i915/i915_pmu.c
drivers/gpu/drm/i915/i915_pmu.h
drivers/gpu/drm/i915/i915_reg.h
drivers/gpu/drm/i915/i915_request.c
drivers/gpu/drm/i915/i915_request.h
drivers/gpu/drm/i915/i915_scheduler.h [new file with mode: 0644]
drivers/gpu/drm/i915/i915_utils.h
drivers/gpu/drm/i915/intel_atomic.c
drivers/gpu/drm/i915/intel_bios.c
drivers/gpu/drm/i915/intel_breadcrumbs.c
drivers/gpu/drm/i915/intel_csr.c
drivers/gpu/drm/i915/intel_ddi.c
drivers/gpu/drm/i915/intel_device_info.c
drivers/gpu/drm/i915/intel_device_info.h
drivers/gpu/drm/i915/intel_display.c
drivers/gpu/drm/i915/intel_display.h
drivers/gpu/drm/i915/intel_dp.c
drivers/gpu/drm/i915/intel_dp_link_training.c
drivers/gpu/drm/i915/intel_dp_mst.c
drivers/gpu/drm/i915/intel_dpio_phy.c
drivers/gpu/drm/i915/intel_dpll_mgr.c
drivers/gpu/drm/i915/intel_dpll_mgr.h
drivers/gpu/drm/i915/intel_drv.h
drivers/gpu/drm/i915/intel_dsi_vbt.c
drivers/gpu/drm/i915/intel_engine_cs.c
drivers/gpu/drm/i915/intel_fbc.c
drivers/gpu/drm/i915/intel_fbdev.c
drivers/gpu/drm/i915/intel_frontbuffer.c
drivers/gpu/drm/i915/intel_gpu_commands.h [new file with mode: 0644]
drivers/gpu/drm/i915/intel_guc.c
drivers/gpu/drm/i915/intel_guc.h
drivers/gpu/drm/i915/intel_guc_ads.c
drivers/gpu/drm/i915/intel_guc_ct.c
drivers/gpu/drm/i915/intel_guc_ct.h
drivers/gpu/drm/i915/intel_guc_fw.c
drivers/gpu/drm/i915/intel_guc_fwif.h
drivers/gpu/drm/i915/intel_guc_log.c
drivers/gpu/drm/i915/intel_guc_log.h
drivers/gpu/drm/i915/intel_guc_reg.h
drivers/gpu/drm/i915/intel_guc_submission.c
drivers/gpu/drm/i915/intel_hangcheck.c
drivers/gpu/drm/i915/intel_hdcp.c
drivers/gpu/drm/i915/intel_hdmi.c
drivers/gpu/drm/i915/intel_hotplug.c
drivers/gpu/drm/i915/intel_huc.c
drivers/gpu/drm/i915/intel_huc.h
drivers/gpu/drm/i915/intel_huc_fw.c
drivers/gpu/drm/i915/intel_lrc.c
drivers/gpu/drm/i915/intel_lrc.h
drivers/gpu/drm/i915/intel_overlay.c
drivers/gpu/drm/i915/intel_pipe_crc.c
drivers/gpu/drm/i915/intel_pm.c
drivers/gpu/drm/i915/intel_psr.c
drivers/gpu/drm/i915/intel_ringbuffer.c
drivers/gpu/drm/i915/intel_ringbuffer.h
drivers/gpu/drm/i915/intel_runtime_pm.c
drivers/gpu/drm/i915/intel_sprite.c
drivers/gpu/drm/i915/intel_uc.c
drivers/gpu/drm/i915/intel_uc.h
drivers/gpu/drm/i915/intel_uc_fw.c
drivers/gpu/drm/i915/intel_uc_fw.h
drivers/gpu/drm/i915/intel_uncore.c
drivers/gpu/drm/i915/intel_uncore.h
drivers/gpu/drm/i915/intel_wopcm.c [new file with mode: 0644]
drivers/gpu/drm/i915/intel_wopcm.h [new file with mode: 0644]
drivers/gpu/drm/i915/intel_workarounds.c [new file with mode: 0644]
drivers/gpu/drm/i915/intel_workarounds.h [new file with mode: 0644]
drivers/gpu/drm/i915/selftests/i915_live_selftests.h
drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
drivers/gpu/drm/i915/selftests/intel_engine_cs.c [new file with mode: 0644]
drivers/gpu/drm/i915/selftests/intel_hangcheck.c
drivers/gpu/drm/i915/selftests/intel_lrc.c [new file with mode: 0644]
drivers/gpu/drm/i915/selftests/intel_workarounds.c [new file with mode: 0644]
drivers/gpu/drm/i915/selftests/mock_engine.c
drivers/gpu/drm/i915/selftests/mock_gem_device.c
include/drm/drm_dp_helper.h
include/drm/i915_pciids.h

index 41dc881..055df45 100644 (file)
@@ -58,6 +58,12 @@ Intel GVT-g Host Support(vGPU device model)
 .. kernel-doc:: drivers/gpu/drm/i915/intel_gvt.c
    :internal:
 
+Workarounds
+-----------
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_workarounds.c
+   :doc: Hardware workarounds
+
 Display Hardware Handling
 =========================
 
@@ -249,6 +255,103 @@ Memory Management and Command Submission
 This sections covers all things related to the GEM implementation in the
 i915 driver.
 
+Intel GPU Basics
+----------------
+
+An Intel GPU has multiple engines. There are several engine types.
+
+- RCS engine is for rendering 3D and performing compute, this is named
+  `I915_EXEC_RENDER` in user space.
+- BCS is a blitting (copy) engine, this is named `I915_EXEC_BLT` in user
+  space.
+- VCS is a video encode and decode engine, this is named `I915_EXEC_BSD`
+  in user space
+- VECS is video enhancement engine, this is named `I915_EXEC_VEBOX` in user
+  space.
+- The enumeration `I915_EXEC_DEFAULT` does not refer to specific engine;
+  instead it is to be used by user space to specify a default rendering
+  engine (for 3D) that may or may not be the same as RCS.
+
+The Intel GPU family is a family of integrated GPU's using Unified
+Memory Access. For having the GPU "do work", user space will feed the
+GPU batch buffers via one of the ioctls `DRM_IOCTL_I915_GEM_EXECBUFFER2`
+or `DRM_IOCTL_I915_GEM_EXECBUFFER2_WR`. Most such batchbuffers will
+instruct the GPU to perform work (for example rendering) and that work
+needs memory from which to read and memory to which to write. All memory
+is encapsulated within GEM buffer objects (usually created with the ioctl
+`DRM_IOCTL_I915_GEM_CREATE`). An ioctl providing a batchbuffer for the GPU
+to create will also list all GEM buffer objects that the batchbuffer reads
+and/or writes. For implementation details of memory management see
+`GEM BO Management Implementation Details`_.
+
+The i915 driver allows user space to create a context via the ioctl
+`DRM_IOCTL_I915_GEM_CONTEXT_CREATE` which is identified by a 32-bit
+integer. Such a context should be viewed by user-space as -loosely-
+analogous to the idea of a CPU process of an operating system. The i915
+driver guarantees that commands issued to a fixed context are to be
+executed so that writes of a previously issued command are seen by
+reads of following commands. Actions issued between different contexts
+(even if from the same file descriptor) are NOT given that guarantee
+and the only way to synchronize across contexts (even from the same
+file descriptor) is through the use of fences. At least as far back as
+Gen4, also have that a context carries with it a GPU HW context;
+the HW context is essentially (most of atleast) the state of a GPU.
+In addition to the ordering guarantees, the kernel will restore GPU
+state via HW context when commands are issued to a context, this saves
+user space the need to restore (most of atleast) the GPU state at the
+start of each batchbuffer. The non-deprecated ioctls to submit batchbuffer
+work can pass that ID (in the lower bits of drm_i915_gem_execbuffer2::rsvd1)
+to identify what context to use with the command.
+
+The GPU has its own memory management and address space. The kernel
+driver maintains the memory translation table for the GPU. For older
+GPUs (i.e. those before Gen8), there is a single global such translation
+table, a global Graphics Translation Table (GTT). For newer generation
+GPUs each context has its own translation table, called Per-Process
+Graphics Translation Table (PPGTT). Of important note, is that although
+PPGTT is named per-process it is actually per context. When user space
+submits a batchbuffer, the kernel walks the list of GEM buffer objects
+used by the batchbuffer and guarantees that not only is the memory of
+each such GEM buffer object resident but it is also present in the
+(PP)GTT. If the GEM buffer object is not yet placed in the (PP)GTT,
+then it is given an address. Two consequences of this are: the kernel
+needs to edit the batchbuffer submitted to write the correct value of
+the GPU address when a GEM BO is assigned a GPU address and the kernel
+might evict a different GEM BO from the (PP)GTT to make address room
+for another GEM BO. Consequently, the ioctls submitting a batchbuffer
+for execution also include a list of all locations within buffers that
+refer to GPU-addresses so that the kernel can edit the buffer correctly.
+This process is dubbed relocation.
+
+GEM BO Management Implementation Details
+----------------------------------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_vma.h
+   :doc: Virtual Memory Address
+
+Buffer Object Eviction
+----------------------
+
+This section documents the interface functions for evicting buffer
+objects to make space available in the virtual gpu address spaces. Note
+that this is mostly orthogonal to shrinking buffer objects caches, which
+has the goal to make main memory (shared with the gpu through the
+unified memory architecture) available.
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_evict.c
+   :internal:
+
+Buffer Object Memory Shrinking
+------------------------------
+
+This section documents the interface function for shrinking memory usage
+of buffer object caches. Shrinking is used to make main memory
+available. Note that this is mostly orthogonal to evicting buffer
+objects, which has the goal to make space in gpu virtual address spaces.
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_shrinker.c
+   :internal:
+
 Batchbuffer Parsing
 -------------------
 
@@ -267,6 +370,12 @@ Batchbuffer Pools
 .. kernel-doc:: drivers/gpu/drm/i915/i915_gem_batch_pool.c
    :internal:
 
+User Batchbuffer Execution
+--------------------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_execbuffer.c
+   :doc: User command execution
+
 Logical Rings, Logical Ring Contexts and Execlists
 --------------------------------------------------
 
@@ -312,28 +421,14 @@ Object Tiling IOCTLs
 .. kernel-doc:: drivers/gpu/drm/i915/i915_gem_tiling.c
    :doc: buffer object tiling
 
-Buffer Object Eviction
-----------------------
-
-This section documents the interface functions for evicting buffer
-objects to make space available in the virtual gpu address spaces. Note
-that this is mostly orthogonal to shrinking buffer objects caches, which
-has the goal to make main memory (shared with the gpu through the
-unified memory architecture) available.
-
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_evict.c
-   :internal:
-
-Buffer Object Memory Shrinking
-------------------------------
+WOPCM
+=====
 
-This section documents the interface function for shrinking memory usage
-of buffer object caches. Shrinking is used to make main memory
-available. Note that this is mostly orthogonal to evicting buffer
-objects, which has the goal to make space in gpu virtual address spaces.
+WOPCM Layout
+------------
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_gem_shrinker.c
-   :internal:
+.. kernel-doc:: drivers/gpu/drm/i915/intel_wopcm.c
+   :doc: WOPCM Layout
 
 GuC
 ===
@@ -359,6 +454,12 @@ GuC Firmware Layout
 .. kernel-doc:: drivers/gpu/drm/i915/intel_guc_fwif.h
    :doc: GuC Firmware Layout
 
+GuC Address Space
+-----------------
+
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc.c
+   :doc: GuC Address Space
+
 Tracing
 =======
 
index 9bdd677..3c181d2 100644 (file)
@@ -1714,11 +1714,15 @@ int drm_atomic_check_only(struct drm_atomic_state *state)
                }
        }
 
-       if (config->funcs->atomic_check)
+       if (config->funcs->atomic_check) {
                ret = config->funcs->atomic_check(state->dev, state);
 
-       if (ret)
-               return ret;
+               if (ret) {
+                       DRM_DEBUG_ATOMIC("atomic driver check for %p failed: %d\n",
+                                        state, ret);
+                       return ret;
+               }
+       }
 
        if (!state->allow_modeset) {
                for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
index 108d21f..80efee1 100644 (file)
@@ -25,6 +25,7 @@ config DRM_I915_DEBUG
         select X86_MSR # used by igt/pm_rpm
         select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
         select DRM_DEBUG_MM if DRM=y
+        select STACKDEPOT if DRM=y # for DRM_DEBUG_MM
        select DRM_DEBUG_MM_SELFTEST
        select SW_SYNC # signaling validation framework (igt/syncobj*)
        select DRM_I915_SW_FENCE_DEBUG_OBJECTS
@@ -89,6 +90,18 @@ config DRM_I915_SW_FENCE_CHECK_DAG
 
           If in doubt, say "N".
 
+config DRM_I915_DEBUG_GUC
+        bool "Enable additional driver debugging for GuC"
+        depends on DRM_I915
+        default n
+        help
+          Choose this option to turn on extra driver debugging that may affect
+          performance but will help resolve GuC related issues.
+
+          Recommended for driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_SELFTEST
        bool "Enable selftests upon driver load"
        depends on DRM_I915
index 4eee91a..9bee52a 100644 (file)
@@ -12,7 +12,7 @@
 # Note the danger in using -Wall -Wextra is that when CI updates gcc we
 # will most likely get a sudden build breakage... Hopefully we will fix
 # new warnings before CI updates!
-subdir-ccflags-y := -Wall -Wextra
+subdir-ccflags-y := -Wall -Wextra -Wvla
 subdir-ccflags-y += $(call cc-disable-warning, unused-parameter)
 subdir-ccflags-y += $(call cc-disable-warning, type-limits)
 subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers)
@@ -43,7 +43,8 @@ i915-y := i915_drv.o \
          intel_csr.o \
          intel_device_info.o \
          intel_pm.o \
-         intel_runtime_pm.o
+         intel_runtime_pm.o \
+         intel_workarounds.o
 
 i915-$(CONFIG_COMPAT)   += i915_ioc32.o
 i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o
@@ -79,7 +80,8 @@ i915-y += i915_cmd_parser.o \
          intel_lrc.o \
          intel_mocs.o \
          intel_ringbuffer.o \
-         intel_uncore.o
+         intel_uncore.o \
+         intel_wopcm.o
 
 # general-purpose microcontroller (GuC) support
 i915-y += intel_uc.o \
@@ -171,7 +173,8 @@ i915-y += i915_perf.o \
          i915_oa_glk.o \
          i915_oa_cflgt2.o \
          i915_oa_cflgt3.o \
-         i915_oa_cnl.o
+         i915_oa_cnl.o \
+         i915_oa_icl.o
 
 ifeq ($(CONFIG_DRM_I915_GVT),y)
 i915-y += intel_gvt.o
index d85939b..a294427 100644 (file)
@@ -1604,7 +1604,8 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s)
        if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)
                || IS_KABYLAKE(gvt->dev_priv)) {
                /* BDW decides privilege based on address space */
-               if (cmd_val(s, 0) & (1 << 8))
+               if (cmd_val(s, 0) & (1 << 8) &&
+                       !(s->vgpu->scan_nonprivbb & (1 << s->ring_id)))
                        return 0;
        }
        return 1;
@@ -1618,6 +1619,8 @@ static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
        bool bb_end = false;
        struct intel_vgpu *vgpu = s->vgpu;
        u32 cmd;
+       struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
+               s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
 
        *bb_size = 0;
 
@@ -1629,18 +1632,22 @@ static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
        cmd = cmd_val(s, 0);
        info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
        if (info == NULL) {
-               gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-                               cmd, get_opcode(cmd, s->ring_id));
+               gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+                               cmd, get_opcode(cmd, s->ring_id),
+                               (s->buf_addr_type == PPGTT_BUFFER) ?
+                               "ppgtt" : "ggtt", s->ring_id, s->workload);
                return -EBADRQC;
        }
        do {
-               if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+               if (copy_gma_to_hva(s->vgpu, mm,
                                gma, gma + 4, &cmd) < 0)
                        return -EFAULT;
                info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
                if (info == NULL) {
-                       gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-                               cmd, get_opcode(cmd, s->ring_id));
+                       gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+                               cmd, get_opcode(cmd, s->ring_id),
+                               (s->buf_addr_type == PPGTT_BUFFER) ?
+                               "ppgtt" : "ggtt", s->ring_id, s->workload);
                        return -EBADRQC;
                }
 
@@ -1666,6 +1673,9 @@ static int perform_bb_shadow(struct parser_exec_state *s)
        unsigned long gma = 0;
        unsigned long bb_size;
        int ret = 0;
+       struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ?
+               s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm;
+       unsigned long gma_start_offset = 0;
 
        /* get the start gm address of the batch buffer */
        gma = get_gma_bb_from_cmd(s, 1);
@@ -1680,8 +1690,24 @@ static int perform_bb_shadow(struct parser_exec_state *s)
        if (!bb)
                return -ENOMEM;
 
+       bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true;
+
+       /* the gma_start_offset stores the batch buffer's start gma's
+        * offset relative to page boundary. so for non-privileged batch
+        * buffer, the shadowed gem object holds exactly the same page
+        * layout as original gem object. This is for the convience of
+        * replacing the whole non-privilged batch buffer page to this
+        * shadowed one in PPGTT at the same gma address. (this replacing
+        * action is not implemented yet now, but may be necessary in
+        * future).
+        * for prileged batch buffer, we just change start gma address to
+        * that of shadowed page.
+        */
+       if (bb->ppgtt)
+               gma_start_offset = gma & ~I915_GTT_PAGE_MASK;
+
        bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv,
-                                        roundup(bb_size, PAGE_SIZE));
+                        roundup(bb_size + gma_start_offset, PAGE_SIZE));
        if (IS_ERR(bb->obj)) {
                ret = PTR_ERR(bb->obj);
                goto err_free_bb;
@@ -1702,9 +1728,9 @@ static int perform_bb_shadow(struct parser_exec_state *s)
                bb->clflush &= ~CLFLUSH_BEFORE;
        }
 
-       ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+       ret = copy_gma_to_hva(s->vgpu, mm,
                              gma, gma + bb_size,
-                             bb->va);
+                             bb->va + gma_start_offset);
        if (ret < 0) {
                gvt_vgpu_err("fail to copy guest ring buffer\n");
                ret = -EFAULT;
@@ -1730,7 +1756,7 @@ static int perform_bb_shadow(struct parser_exec_state *s)
         * buffer's gma in pair. After all, we don't want to pin the shadow
         * buffer here (too early).
         */
-       s->ip_va = bb->va;
+       s->ip_va = bb->va + gma_start_offset;
        s->ip_gma = gma;
        return 0;
 err_unmap:
@@ -2469,15 +2495,18 @@ static int cmd_parser_exec(struct parser_exec_state *s)
 
        info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
        if (info == NULL) {
-               gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
-                               cmd, get_opcode(cmd, s->ring_id));
+               gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n",
+                               cmd, get_opcode(cmd, s->ring_id),
+                               (s->buf_addr_type == PPGTT_BUFFER) ?
+                               "ppgtt" : "ggtt", s->ring_id, s->workload);
                return -EBADRQC;
        }
 
        s->info = info;
 
        trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va,
-                         cmd_length(s), s->buf_type);
+                         cmd_length(s), s->buf_type, s->buf_addr_type,
+                         s->workload, info->name);
 
        if (info->handler) {
                ret = info->handler(s);
index 32a66df..2ec89bc 100644 (file)
@@ -122,18 +122,69 @@ static int vgpu_mmio_diff_show(struct seq_file *s, void *unused)
        seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff);
        return 0;
 }
+DEFINE_SHOW_ATTRIBUTE(vgpu_mmio_diff);
 
-static int vgpu_mmio_diff_open(struct inode *inode, struct file *file)
+static int
+vgpu_scan_nonprivbb_get(void *data, u64 *val)
 {
-       return single_open(file, vgpu_mmio_diff_show, inode->i_private);
+       struct intel_vgpu *vgpu = (struct intel_vgpu *)data;
+       *val = vgpu->scan_nonprivbb;
+       return 0;
 }
 
-static const struct file_operations vgpu_mmio_diff_fops = {
-       .open           = vgpu_mmio_diff_open,
-       .read           = seq_read,
-       .llseek         = seq_lseek,
-       .release        = single_release,
-};
+/*
+ * set/unset bit engine_id of vgpu->scan_nonprivbb to turn on/off scanning
+ * of non-privileged batch buffer. e.g.
+ * if vgpu->scan_nonprivbb=3, then it will scan non-privileged batch buffer
+ * on engine 0 and 1.
+ */
+static int
+vgpu_scan_nonprivbb_set(void *data, u64 val)
+{
+       struct intel_vgpu *vgpu = (struct intel_vgpu *)data;
+       struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+       enum intel_engine_id id;
+       char buf[128], *s;
+       int len;
+
+       val &= (1 << I915_NUM_ENGINES) - 1;
+
+       if (vgpu->scan_nonprivbb == val)
+               return 0;
+
+       if (!val)
+               goto done;
+
+       len = sprintf(buf,
+               "gvt: vgpu %d turns on non-privileged batch buffers scanning on Engines:",
+               vgpu->id);
+
+       s = buf + len;
+
+       for (id = 0; id < I915_NUM_ENGINES; id++) {
+               struct intel_engine_cs *engine;
+
+               engine = dev_priv->engine[id];
+               if (engine && (val & (1 << id))) {
+                       len = snprintf(s, 4, "%d, ", engine->id);
+                       s += len;
+               } else
+                       val &=  ~(1 << id);
+       }
+
+       if (val)
+               sprintf(s, "low performance expected.");
+
+       pr_warn("%s\n", buf);
+
+done:
+       vgpu->scan_nonprivbb = val;
+       return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vgpu_scan_nonprivbb_fops,
+                       vgpu_scan_nonprivbb_get, vgpu_scan_nonprivbb_set,
+                       "0x%llx\n");
 
 /**
  * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU
@@ -162,6 +213,11 @@ int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu)
        if (!ent)
                return -ENOMEM;
 
+       ent = debugfs_create_file("scan_nonprivbb", 0644, vgpu->debugfs,
+                                vgpu, &vgpu_scan_nonprivbb_fops);
+       if (!ent)
+               return -ENOMEM;
+
        return 0;
 }
 
index efacd8a..6ec8888 100644 (file)
@@ -226,6 +226,7 @@ struct intel_vgpu {
 
        struct completion vblank_done;
 
+       u32 scan_nonprivbb;
 };
 
 /* validating GM healthy status*/
index a5bac83..0f94955 100644 (file)
@@ -448,7 +448,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
 
 bool is_inhibit_context(struct i915_gem_context *ctx, int ring_id)
 {
-       u32 *reg_state = ctx->engine[ring_id].lrc_reg_state;
+       u32 *reg_state = ctx->__engine[ring_id].lrc_reg_state;
        u32 inhibit_mask =
                _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
index 75b7bc7..d053cbe 100644 (file)
@@ -53,7 +53,6 @@ struct vgpu_sched_data {
        bool active;
 
        ktime_t sched_in_time;
-       ktime_t sched_out_time;
        ktime_t sched_time;
        ktime_t left_ts;
        ktime_t allocated_ts;
@@ -66,17 +65,22 @@ struct gvt_sched_data {
        struct hrtimer timer;
        unsigned long period;
        struct list_head lru_runq_head;
+       ktime_t expire_time;
 };
 
-static void vgpu_update_timeslice(struct intel_vgpu *pre_vgpu)
+static void vgpu_update_timeslice(struct intel_vgpu *vgpu, ktime_t cur_time)
 {
        ktime_t delta_ts;
-       struct vgpu_sched_data *vgpu_data = pre_vgpu->sched_data;
+       struct vgpu_sched_data *vgpu_data;
 
-       delta_ts = vgpu_data->sched_out_time - vgpu_data->sched_in_time;
+       if (!vgpu || vgpu == vgpu->gvt->idle_vgpu)
+               return;
 
-       vgpu_data->sched_time += delta_ts;
-       vgpu_data->left_ts -= delta_ts;
+       vgpu_data = vgpu->sched_data;
+       delta_ts = ktime_sub(cur_time, vgpu_data->sched_in_time);
+       vgpu_data->sched_time = ktime_add(vgpu_data->sched_time, delta_ts);
+       vgpu_data->left_ts = ktime_sub(vgpu_data->left_ts, delta_ts);
+       vgpu_data->sched_in_time = cur_time;
 }
 
 #define GVT_TS_BALANCE_PERIOD_MS 100
@@ -150,11 +154,7 @@ static void try_to_schedule_next_vgpu(struct intel_gvt *gvt)
        }
 
        cur_time = ktime_get();
-       if (scheduler->current_vgpu) {
-               vgpu_data = scheduler->current_vgpu->sched_data;
-               vgpu_data->sched_out_time = cur_time;
-               vgpu_update_timeslice(scheduler->current_vgpu);
-       }
+       vgpu_update_timeslice(scheduler->current_vgpu, cur_time);
        vgpu_data = scheduler->next_vgpu->sched_data;
        vgpu_data->sched_in_time = cur_time;
 
@@ -226,17 +226,22 @@ out:
 void intel_gvt_schedule(struct intel_gvt *gvt)
 {
        struct gvt_sched_data *sched_data = gvt->scheduler.sched_data;
-       static uint64_t timer_check;
+       ktime_t cur_time;
 
        mutex_lock(&gvt->lock);
+       cur_time = ktime_get();
 
        if (test_and_clear_bit(INTEL_GVT_REQUEST_SCHED,
                                (void *)&gvt->service_request)) {
-               if (!(timer_check++ % GVT_TS_BALANCE_PERIOD_MS))
+               if (cur_time >= sched_data->expire_time) {
                        gvt_balance_timeslice(sched_data);
+                       sched_data->expire_time = ktime_add_ms(
+                               cur_time, GVT_TS_BALANCE_PERIOD_MS);
+               }
        }
        clear_bit(INTEL_GVT_REQUEST_EVENT_SCHED, (void *)&gvt->service_request);
 
+       vgpu_update_timeslice(gvt->scheduler.current_vgpu, cur_time);
        tbs_sched_func(sched_data);
 
        mutex_unlock(&gvt->lock);
index 638abe8..ffb45a9 100644 (file)
@@ -58,7 +58,7 @@ static void update_shadow_pdps(struct intel_vgpu_workload *workload)
        int ring_id = workload->ring_id;
        struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
        struct drm_i915_gem_object *ctx_obj =
-               shadow_ctx->engine[ring_id].state->obj;
+               shadow_ctx->__engine[ring_id].state->obj;
        struct execlist_ring_context *shadow_ring_context;
        struct page *page;
 
@@ -97,7 +97,7 @@ static void sr_oa_regs(struct intel_vgpu_workload *workload,
                i915_mmio_reg_offset(EU_PERF_CNTL6),
        };
 
-       if (!workload || !reg_state || workload->ring_id != RCS)
+       if (workload->ring_id != RCS)
                return;
 
        if (save) {
@@ -130,7 +130,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
        int ring_id = workload->ring_id;
        struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
        struct drm_i915_gem_object *ctx_obj =
-               shadow_ctx->engine[ring_id].state->obj;
+               shadow_ctx->__engine[ring_id].state->obj;
        struct execlist_ring_context *shadow_ring_context;
        struct page *page;
        void *dst;
@@ -283,7 +283,7 @@ static int shadow_context_status_change(struct notifier_block *nb,
 static void shadow_context_descriptor_update(struct i915_gem_context *ctx,
                struct intel_engine_cs *engine)
 {
-       struct intel_context *ce = &ctx->engine[engine->id];
+       struct intel_context *ce = to_intel_context(ctx, engine);
        u64 desc = 0;
 
        desc = ce->lrc_desc;
@@ -389,7 +389,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
         * shadow_ctx pages invalid. So gvt need to pin itself. After update
         * the guest context, gvt can unpin the shadow_ctx safely.
         */
-       ring = engine->context_pin(engine, shadow_ctx);
+       ring = intel_context_pin(shadow_ctx, engine);
        if (IS_ERR(ring)) {
                ret = PTR_ERR(ring);
                gvt_vgpu_err("fail to pin shadow context\n");
@@ -403,7 +403,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
        return 0;
 
 err_unpin:
-       engine->context_unpin(engine, shadow_ctx);
+       intel_context_unpin(shadow_ctx, engine);
 err_shadow:
        release_shadow_wa_ctx(&workload->wa_ctx);
 err_scan:
@@ -437,7 +437,7 @@ static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
        return 0;
 
 err_unpin:
-       engine->context_unpin(engine, shadow_ctx);
+       intel_context_unpin(shadow_ctx, engine);
        release_shadow_wa_ctx(&workload->wa_ctx);
        return ret;
 }
@@ -452,12 +452,6 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
        int ret;
 
        list_for_each_entry(bb, &workload->shadow_bb, list) {
-               bb->vma = i915_gem_object_ggtt_pin(bb->obj, NULL, 0, 0, 0);
-               if (IS_ERR(bb->vma)) {
-                       ret = PTR_ERR(bb->vma);
-                       goto err;
-               }
-
                /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
                 * is only updated into ring_scan_buffer, not real ring address
                 * allocated in later copy_workload_to_ring_buffer. pls be noted
@@ -469,25 +463,53 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
                        bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
                                + bb->bb_offset;
 
-               /* relocate shadow batch buffer */
-               bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
-               if (gmadr_bytes == 8)
-                       bb->bb_start_cmd_va[2] = 0;
+               if (bb->ppgtt) {
+                       /* for non-priv bb, scan&shadow is only for
+                        * debugging purpose, so the content of shadow bb
+                        * is the same as original bb. Therefore,
+                        * here, rather than switch to shadow bb's gma
+                        * address, we directly use original batch buffer's
+                        * gma address, and send original bb to hardware
+                        * directly
+                        */
+                       if (bb->clflush & CLFLUSH_AFTER) {
+                               drm_clflush_virt_range(bb->va,
+                                               bb->obj->base.size);
+                               bb->clflush &= ~CLFLUSH_AFTER;
+                       }
+                       i915_gem_obj_finish_shmem_access(bb->obj);
+                       bb->accessing = false;
+
+               } else {
+                       bb->vma = i915_gem_object_ggtt_pin(bb->obj,
+                                       NULL, 0, 0, 0);
+                       if (IS_ERR(bb->vma)) {
+                               ret = PTR_ERR(bb->vma);
+                               goto err;
+                       }
 
-               /* No one is going to touch shadow bb from now on. */
-               if (bb->clflush & CLFLUSH_AFTER) {
-                       drm_clflush_virt_range(bb->va, bb->obj->base.size);
-                       bb->clflush &= ~CLFLUSH_AFTER;
-               }
+                       /* relocate shadow batch buffer */
+                       bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
+                       if (gmadr_bytes == 8)
+                               bb->bb_start_cmd_va[2] = 0;
 
-               ret = i915_gem_object_set_to_gtt_domain(bb->obj, false);
-               if (ret)
-                       goto err;
+                       /* No one is going to touch shadow bb from now on. */
+                       if (bb->clflush & CLFLUSH_AFTER) {
+                               drm_clflush_virt_range(bb->va,
+                                               bb->obj->base.size);
+                               bb->clflush &= ~CLFLUSH_AFTER;
+                       }
 
-               i915_gem_obj_finish_shmem_access(bb->obj);
-               bb->accessing = false;
+                       ret = i915_gem_object_set_to_gtt_domain(bb->obj,
+                                       false);
+                       if (ret)
+                               goto err;
 
-               i915_vma_move_to_active(bb->vma, workload->req, 0);
+                       i915_gem_obj_finish_shmem_access(bb->obj);
+                       bb->accessing = false;
+
+                       i915_vma_move_to_active(bb->vma, workload->req, 0);
+               }
        }
        return 0;
 err:
@@ -504,7 +526,7 @@ static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
        struct intel_vgpu_submission *s = &workload->vgpu->submission;
        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
        struct drm_i915_gem_object *ctx_obj =
-               shadow_ctx->engine[ring_id].state->obj;
+               shadow_ctx->__engine[ring_id].state->obj;
        struct execlist_ring_context *shadow_ring_context;
        struct page *page;
 
@@ -666,7 +688,7 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 
        ret = prepare_workload(workload);
        if (ret) {
-               engine->context_unpin(engine, shadow_ctx);
+               intel_context_unpin(shadow_ctx, engine);
                goto out;
        }
 
@@ -749,7 +771,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
        int ring_id = workload->ring_id;
        struct drm_i915_gem_object *ctx_obj =
-               shadow_ctx->engine[ring_id].state->obj;
+               shadow_ctx->__engine[ring_id].state->obj;
        struct execlist_ring_context *shadow_ring_context;
        struct page *page;
        void *src;
@@ -876,7 +898,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
                }
                mutex_lock(&dev_priv->drm.struct_mutex);
                /* unpin shadow ctx as the shadow_ctx update is done */
-               engine->context_unpin(engine, s->shadow_ctx);
+               intel_context_unpin(s->shadow_ctx, engine);
                mutex_unlock(&dev_priv->drm.struct_mutex);
        }
 
@@ -1135,7 +1157,7 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
                return PTR_ERR(s->shadow_ctx);
 
        if (HAS_LOGICAL_RING_PREEMPTION(vgpu->gvt->dev_priv))
-               s->shadow_ctx->priority = INT_MAX;
+               s->shadow_ctx->sched.priority = INT_MAX;
 
        bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
 
index 486ed57..6c64478 100644 (file)
@@ -125,6 +125,7 @@ struct intel_vgpu_shadow_bb {
        unsigned int clflush;
        bool accessing;
        unsigned long bb_offset;
+       bool ppgtt;
 };
 
 #define workload_q_head(vgpu, ring_id) \
index 82093f1..1fd6420 100644 (file)
@@ -224,19 +224,25 @@ TRACE_EVENT(oos_sync,
        TP_printk("%s", __entry->buf)
 );
 
+#define GVT_CMD_STR_LEN 40
 TRACE_EVENT(gvt_command,
-       TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va, u32 cmd_len,
-                u32 buf_type),
+       TP_PROTO(u8 vgpu_id, u8 ring_id, u32 ip_gma, u32 *cmd_va,
+               u32 cmd_len,  u32 buf_type, u32 buf_addr_type,
+               void *workload, char *cmd_name),
 
-       TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type),
+       TP_ARGS(vgpu_id, ring_id, ip_gma, cmd_va, cmd_len, buf_type,
+               buf_addr_type, workload, cmd_name),
 
        TP_STRUCT__entry(
                __field(u8, vgpu_id)
                __field(u8, ring_id)
                __field(u32, ip_gma)
                __field(u32, buf_type)
+               __field(u32, buf_addr_type)
                __field(u32, cmd_len)
+               __field(void*, workload)
                __dynamic_array(u32, raw_cmd, cmd_len)
+               __array(char, cmd_name, GVT_CMD_STR_LEN)
        ),
 
        TP_fast_assign(
@@ -244,17 +250,25 @@ TRACE_EVENT(gvt_command,
                __entry->ring_id = ring_id;
                __entry->ip_gma = ip_gma;
                __entry->buf_type = buf_type;
+               __entry->buf_addr_type = buf_addr_type;
                __entry->cmd_len = cmd_len;
+               __entry->workload = workload;
+               snprintf(__entry->cmd_name, GVT_CMD_STR_LEN, "%s", cmd_name);
                memcpy(__get_dynamic_array(raw_cmd), cmd_va, cmd_len * sizeof(*cmd_va));
        ),
 
 
-       TP_printk("vgpu%d ring %d: buf_type %u, ip_gma %08x, raw cmd %s",
+       TP_printk("vgpu%d ring %d: address_type %u, buf_type %u, ip_gma %08x,cmd (name=%s,len=%u,raw cmd=%s), workload=%p\n",
                __entry->vgpu_id,
                __entry->ring_id,
+               __entry->buf_addr_type,
                __entry->buf_type,
                __entry->ip_gma,
-               __print_array(__get_dynamic_array(raw_cmd), __entry->cmd_len, 4))
+               __entry->cmd_name,
+               __entry->cmd_len,
+               __print_array(__get_dynamic_array(raw_cmd),
+                       __entry->cmd_len, 4),
+               __entry->workload)
 );
 
 #define GVT_TEMP_STR_LEN 10
index 89f7ff2..85911bc 100644 (file)
@@ -377,16 +377,19 @@ static void print_batch_pool_stats(struct seq_file *m,
        print_file_stats(m, "[k]batch pool", stats);
 }
 
-static int per_file_ctx_stats(int id, void *ptr, void *data)
+static int per_file_ctx_stats(int idx, void *ptr, void *data)
 {
        struct i915_gem_context *ctx = ptr;
-       int n;
+       struct intel_engine_cs *engine;
+       enum intel_engine_id id;
+
+       for_each_engine(engine, ctx->i915, id) {
+               struct intel_context *ce = to_intel_context(ctx, engine);
 
-       for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) {
-               if (ctx->engine[n].state)
-                       per_file_stats(0, ctx->engine[n].state->obj, data);
-               if (ctx->engine[n].ring)
-                       per_file_stats(0, ctx->engine[n].ring->vma->obj, data);
+               if (ce->state)
+                       per_file_stats(0, ce->state->obj, data);
+               if (ce->ring)
+                       per_file_stats(0, ce->ring->vma->obj, data);
        }
 
        return 0;
@@ -1215,20 +1218,20 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
                max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 0 :
                            rp_state_cap >> 16) & 0xff;
                max_freq *= (IS_GEN9_BC(dev_priv) ||
-                            IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+                            INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
                seq_printf(m, "Lowest (RPN) frequency: %dMHz\n",
                           intel_gpu_freq(dev_priv, max_freq));
 
                max_freq = (rp_state_cap & 0xff00) >> 8;
                max_freq *= (IS_GEN9_BC(dev_priv) ||
-                            IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+                            INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
                seq_printf(m, "Nominal (RP1) frequency: %dMHz\n",
                           intel_gpu_freq(dev_priv, max_freq));
 
                max_freq = (IS_GEN9_LP(dev_priv) ? rp_state_cap >> 16 :
                            rp_state_cap >> 0) & 0xff;
                max_freq *= (IS_GEN9_BC(dev_priv) ||
-                            IS_CANNONLAKE(dev_priv) ? GEN9_FREQ_SCALER : 1);
+                            INTEL_GEN(dev_priv) >= 10 ? GEN9_FREQ_SCALER : 1);
                seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n",
                           intel_gpu_freq(dev_priv, max_freq));
                seq_printf(m, "Max overclocked frequency: %dMHz\n",
@@ -1340,10 +1343,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
                struct rb_node *rb;
 
                seq_printf(m, "%s:\n", engine->name);
-               seq_printf(m, "\tseqno = %x [current %x, last %x], inflight %d\n",
+               seq_printf(m, "\tseqno = %x [current %x, last %x]\n",
                           engine->hangcheck.seqno, seqno[id],
-                          intel_engine_last_submit(engine),
-                          engine->timeline->inflight_seqnos);
+                          intel_engine_last_submit(engine));
                seq_printf(m, "\twaiters? %s, fake irq active? %s, stalled? %s\n",
                           yesno(intel_engine_has_waiter(engine)),
                           yesno(test_bit(engine->id,
@@ -1796,9 +1798,9 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
        struct intel_rps *rps = &dev_priv->gt_pm.rps;
-       int ret = 0;
-       int gpu_freq, ia_freq;
        unsigned int max_gpu_freq, min_gpu_freq;
+       int gpu_freq, ia_freq;
+       int ret;
 
        if (!HAS_LLC(dev_priv))
                return -ENODEV;
@@ -1809,13 +1811,12 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
        if (ret)
                goto out;
 
-       if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+       min_gpu_freq = rps->min_freq;
+       max_gpu_freq = rps->max_freq;
+       if (IS_GEN9_BC(dev_priv) || INTEL_GEN(dev_priv) >= 10) {
                /* Convert GT frequency to 50 HZ units */
-               min_gpu_freq = rps->min_freq_softlimit / GEN9_FREQ_SCALER;
-               max_gpu_freq = rps->max_freq_softlimit / GEN9_FREQ_SCALER;
-       } else {
-               min_gpu_freq = rps->min_freq_softlimit;
-               max_gpu_freq = rps->max_freq_softlimit;
+               min_gpu_freq /= GEN9_FREQ_SCALER;
+               max_gpu_freq /= GEN9_FREQ_SCALER;
        }
 
        seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n");
@@ -1828,7 +1829,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
                seq_printf(m, "%d\t\t%d\t\t\t\t%d\n",
                           intel_gpu_freq(dev_priv, (gpu_freq *
                                                     (IS_GEN9_BC(dev_priv) ||
-                                                     IS_CANNONLAKE(dev_priv) ?
+                                                     INTEL_GEN(dev_priv) >= 10 ?
                                                      GEN9_FREQ_SCALER : 1))),
                           ((ia_freq >> 0) & 0xff) * 100,
                           ((ia_freq >> 8) & 0xff) * 100);
@@ -1923,8 +1924,8 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 
 static void describe_ctx_ring(struct seq_file *m, struct intel_ring *ring)
 {
-       seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u)",
-                  ring->space, ring->head, ring->tail);
+       seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, emit: %u)",
+                  ring->space, ring->head, ring->tail, ring->emit);
 }
 
 static int i915_context_status(struct seq_file *m, void *unused)
@@ -1961,7 +1962,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
                seq_putc(m, '\n');
 
                for_each_engine(engine, dev_priv, id) {
-                       struct intel_context *ce = &ctx->engine[engine->id];
+                       struct intel_context *ce =
+                               to_intel_context(ctx, engine);
 
                        seq_printf(m, "%s: ", engine->name);
                        if (ce->state)
@@ -2326,30 +2328,45 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data)
        return 0;
 }
 
-static void i915_guc_log_info(struct seq_file *m,
-                             struct drm_i915_private *dev_priv)
+static const char *
+stringify_guc_log_type(enum guc_log_buffer_type type)
 {
-       struct intel_guc *guc = &dev_priv->guc;
+       switch (type) {
+       case GUC_ISR_LOG_BUFFER:
+               return "ISR";
+       case GUC_DPC_LOG_BUFFER:
+               return "DPC";
+       case GUC_CRASH_DUMP_LOG_BUFFER:
+               return "CRASH";
+       default:
+               MISSING_CASE(type);
+       }
 
-       seq_puts(m, "\nGuC logging stats:\n");
+       return "";
+}
 
-       seq_printf(m, "\tISR:   flush count %10u, overflow count %10u\n",
-                  guc->log.flush_count[GUC_ISR_LOG_BUFFER],
-                  guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]);
+static void i915_guc_log_info(struct seq_file *m,
+                             struct drm_i915_private *dev_priv)
+{
+       struct intel_guc_log *log = &dev_priv->guc.log;
+       enum guc_log_buffer_type type;
 
-       seq_printf(m, "\tDPC:   flush count %10u, overflow count %10u\n",
-                  guc->log.flush_count[GUC_DPC_LOG_BUFFER],
-                  guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]);
+       if (!intel_guc_log_relay_enabled(log)) {
+               seq_puts(m, "GuC log relay disabled\n");
+               return;
+       }
 
-       seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n",
-                  guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER],
-                  guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]);
+       seq_puts(m, "GuC logging stats:\n");
 
-       seq_printf(m, "\tTotal flush interrupt count: %u\n",
-                  guc->log.flush_interrupt_count);
+       seq_printf(m, "\tRelay full count: %u\n",
+                  log->relay.full_count);
 
-       seq_printf(m, "\tCapture miss count: %u\n",
-                  guc->log.capture_miss_count);
+       for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) {
+               seq_printf(m, "\t%s:\tflush count %10u, overflow count %10u\n",
+                          stringify_guc_log_type(type),
+                          log->stats[type].flush,
+                          log->stats[type].sampled_overflow);
+       }
 }
 
 static void i915_guc_client_info(struct seq_file *m,
@@ -2379,14 +2396,19 @@ static int i915_guc_info(struct seq_file *m, void *data)
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
        const struct intel_guc *guc = &dev_priv->guc;
 
-       if (!USES_GUC_SUBMISSION(dev_priv))
+       if (!USES_GUC(dev_priv))
                return -ENODEV;
 
+       i915_guc_log_info(m, dev_priv);
+
+       if (!USES_GUC_SUBMISSION(dev_priv))
+               return 0;
+
        GEM_BUG_ON(!guc->execbuf_client);
 
-       seq_printf(m, "Doorbell map:\n");
+       seq_printf(m, "\nDoorbell map:\n");
        seq_printf(m, "\t%*pb\n", GUC_NUM_DOORBELLS, guc->doorbell_bitmap);
-       seq_printf(m, "Doorbell next cacheline: 0x%x\n\n", guc->db_cacheline);
+       seq_printf(m, "Doorbell next cacheline: 0x%x\n", guc->db_cacheline);
 
        seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client);
        i915_guc_client_info(m, dev_priv, guc->execbuf_client);
@@ -2396,8 +2418,6 @@ static int i915_guc_info(struct seq_file *m, void *data)
                i915_guc_client_info(m, dev_priv, guc->preempt_client);
        }
 
-       i915_guc_log_info(m, dev_priv);
-
        /* Add more as required ... */
 
        return 0;
@@ -2496,35 +2516,73 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
        return 0;
 }
 
-static int i915_guc_log_control_get(void *data, u64 *val)
+static int i915_guc_log_level_get(void *data, u64 *val)
 {
        struct drm_i915_private *dev_priv = data;
 
-       if (!HAS_GUC(dev_priv))
+       if (!USES_GUC(dev_priv))
                return -ENODEV;
 
-       if (!dev_priv->guc.log.vma)
-               return -EINVAL;
-
-       *val = i915_modparams.guc_log_level;
+       *val = intel_guc_log_level_get(&dev_priv->guc.log);
 
        return 0;
 }
 
-static int i915_guc_log_control_set(void *data, u64 val)
+static int i915_guc_log_level_set(void *data, u64 val)
 {
        struct drm_i915_private *dev_priv = data;
 
-       if (!HAS_GUC(dev_priv))
+       if (!USES_GUC(dev_priv))
                return -ENODEV;
 
-       return intel_guc_log_control(&dev_priv->guc, val);
+       return intel_guc_log_level_set(&dev_priv->guc.log, val);
 }
 
-DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops,
-                       i915_guc_log_control_get, i915_guc_log_control_set,
+DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_level_fops,
+                       i915_guc_log_level_get, i915_guc_log_level_set,
                        "%lld\n");
 
+static int i915_guc_log_relay_open(struct inode *inode, struct file *file)
+{
+       struct drm_i915_private *dev_priv = inode->i_private;
+
+       if (!USES_GUC(dev_priv))
+               return -ENODEV;
+
+       file->private_data = &dev_priv->guc.log;
+
+       return intel_guc_log_relay_open(&dev_priv->guc.log);
+}
+
+static ssize_t
+i915_guc_log_relay_write(struct file *filp,
+                        const char __user *ubuf,
+                        size_t cnt,
+                        loff_t *ppos)
+{
+       struct intel_guc_log *log = filp->private_data;
+
+       intel_guc_log_relay_flush(log);
+
+       return cnt;
+}
+
+static int i915_guc_log_relay_release(struct inode *inode, struct file *file)
+{
+       struct drm_i915_private *dev_priv = inode->i_private;
+
+       intel_guc_log_relay_close(&dev_priv->guc.log);
+
+       return 0;
+}
+
+static const struct file_operations i915_guc_log_relay_fops = {
+       .owner = THIS_MODULE,
+       .open = i915_guc_log_relay_open,
+       .write = i915_guc_log_relay_write,
+       .release = i915_guc_log_relay_release,
+};
+
 static const char *psr2_live_status(u32 val)
 {
        static const char * const live_status[] = {
@@ -2548,6 +2606,26 @@ static const char *psr2_live_status(u32 val)
        return "unknown";
 }
 
+static const char *psr_sink_status(u8 val)
+{
+       static const char * const sink_status[] = {
+               "inactive",
+               "transition to active, capture and display",
+               "active, display from RFB",
+               "active, capture and display on sink device timings",
+               "transition to inactive, capture and display, timing re-sync",
+               "reserved",
+               "reserved",
+               "sink internal error"
+       };
+
+       val &= DP_PSR_SINK_STATE_MASK;
+       if (val < ARRAY_SIZE(sink_status))
+               return sink_status[val];
+
+       return "unknown";
+}
+
 static int i915_edp_psr_status(struct seq_file *m, void *data)
 {
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -2569,14 +2647,13 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
        mutex_lock(&dev_priv->psr.lock);
        seq_printf(m, "Enabled: %s\n", yesno((bool)dev_priv->psr.enabled));
-       seq_printf(m, "Active: %s\n", yesno(dev_priv->psr.active));
        seq_printf(m, "Busy frontbuffer bits: 0x%03x\n",
                   dev_priv->psr.busy_frontbuffer_bits);
        seq_printf(m, "Re-enable work scheduled: %s\n",
                   yesno(work_busy(&dev_priv->psr.work.work)));
 
        if (HAS_DDI(dev_priv)) {
-               if (dev_priv->psr.psr2_support)
+               if (dev_priv->psr.psr2_enabled)
                        enabled = I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE;
                else
                        enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE;
@@ -2624,18 +2701,67 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
                seq_printf(m, "Performance_Counter: %u\n", psrperf);
        }
-       if (dev_priv->psr.psr2_support) {
+       if (dev_priv->psr.psr2_enabled) {
                u32 psr2 = I915_READ(EDP_PSR2_STATUS);
 
                seq_printf(m, "EDP_PSR2_STATUS: %x [%s]\n",
                           psr2, psr2_live_status(psr2));
        }
+
+       if (dev_priv->psr.enabled) {
+               struct drm_dp_aux *aux = &dev_priv->psr.enabled->aux;
+               u8 val;
+
+               if (drm_dp_dpcd_readb(aux, DP_PSR_STATUS, &val) == 1)
+                       seq_printf(m, "Sink PSR status: 0x%x [%s]\n", val,
+                                  psr_sink_status(val));
+       }
        mutex_unlock(&dev_priv->psr.lock);
 
+       if (READ_ONCE(dev_priv->psr.debug)) {
+               seq_printf(m, "Last attempted entry at: %lld\n",
+                          dev_priv->psr.last_entry_attempt);
+               seq_printf(m, "Last exit at: %lld\n",
+                          dev_priv->psr.last_exit);
+       }
+
        intel_runtime_pm_put(dev_priv);
        return 0;
 }
 
+static int
+i915_edp_psr_debug_set(void *data, u64 val)
+{
+       struct drm_i915_private *dev_priv = data;
+
+       if (!CAN_PSR(dev_priv))
+               return -ENODEV;
+
+       DRM_DEBUG_KMS("PSR debug %s\n", enableddisabled(val));
+
+       intel_runtime_pm_get(dev_priv);
+       intel_psr_irq_control(dev_priv, !!val);
+       intel_runtime_pm_put(dev_priv);
+
+       return 0;
+}
+
+static int
+i915_edp_psr_debug_get(void *data, u64 *val)
+{
+       struct drm_i915_private *dev_priv = data;
+
+       if (!CAN_PSR(dev_priv))
+               return -ENODEV;
+
+       *val = READ_ONCE(dev_priv->psr.debug);
+       return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(i915_edp_psr_debug_fops,
+                       i915_edp_psr_debug_get, i915_edp_psr_debug_set,
+                       "%llu\n");
+
 static int i915_sink_crc(struct seq_file *m, void *data)
 {
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -3231,7 +3357,8 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
        for (i = 0; i < dev_priv->num_shared_dpll; i++) {
                struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
 
-               seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->name, pll->id);
+               seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name,
+                          pll->info->id);
                seq_printf(m, " crtc_mask: 0x%08x, active: 0x%x, on: %s\n",
                           pll->state.crtc_mask, pll->active_mask, yesno(pll->on));
                seq_printf(m, " tracked hardware state:\n");
@@ -3249,24 +3376,13 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 
 static int i915_wa_registers(struct seq_file *m, void *unused)
 {
-       int i;
-       int ret;
-       struct intel_engine_cs *engine;
        struct drm_i915_private *dev_priv = node_to_i915(m->private);
-       struct drm_device *dev = &dev_priv->drm;
        struct i915_workarounds *workarounds = &dev_priv->workarounds;
-       enum intel_engine_id id;
-
-       ret = mutex_lock_interruptible(&dev->struct_mutex);
-       if (ret)
-               return ret;
+       int i;
 
        intel_runtime_pm_get(dev_priv);
 
        seq_printf(m, "Workarounds applied: %d\n", workarounds->count);
-       for_each_engine(engine, dev_priv, id)
-               seq_printf(m, "HW whitelist count for %s: %d\n",
-                          engine->name, workarounds->hw_whitelist_count[id]);
        for (i = 0; i < workarounds->count; ++i) {
                i915_reg_t addr;
                u32 mask, value, read;
@@ -3282,7 +3398,6 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
        }
 
        intel_runtime_pm_put(dev_priv);
-       mutex_unlock(&dev->struct_mutex);
 
        return 0;
 }
@@ -3567,7 +3682,8 @@ static ssize_t i915_displayport_test_active_write(struct file *file,
 
 static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 {
-       struct drm_device *dev = m->private;
+       struct drm_i915_private *dev_priv = m->private;
+       struct drm_device *dev = &dev_priv->drm;
        struct drm_connector *connector;
        struct drm_connector_list_iter conn_iter;
        struct intel_dp *intel_dp;
@@ -3601,10 +3717,8 @@ static int i915_displayport_test_active_show(struct seq_file *m, void *data)
 static int i915_displayport_test_active_open(struct inode *inode,
                                             struct file *file)
 {
-       struct drm_i915_private *dev_priv = inode->i_private;
-
        return single_open(file, i915_displayport_test_active_show,
-                          &dev_priv->drm);
+                          inode->i_private);
 }
 
 static const struct file_operations i915_displayport_test_active_fops = {
@@ -3618,7 +3732,8 @@ static const struct file_operations i915_displayport_test_active_fops = {
 
 static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 {
-       struct drm_device *dev = m->private;
+       struct drm_i915_private *dev_priv = m->private;
+       struct drm_device *dev = &dev_priv->drm;
        struct drm_connector *connector;
        struct drm_connector_list_iter conn_iter;
        struct intel_dp *intel_dp;
@@ -3657,26 +3772,12 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data)
 
        return 0;
 }
-static int i915_displayport_test_data_open(struct inode *inode,
-                                          struct file *file)
-{
-       struct drm_i915_private *dev_priv = inode->i_private;
-
-       return single_open(file, i915_displayport_test_data_show,
-                          &dev_priv->drm);
-}
-
-static const struct file_operations i915_displayport_test_data_fops = {
-       .owner = THIS_MODULE,
-       .open = i915_displayport_test_data_open,
-       .read = seq_read,
-       .llseek = seq_lseek,
-       .release = single_release
-};
+DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_data);
 
 static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 {
-       struct drm_device *dev = m->private;
+       struct drm_i915_private *dev_priv = m->private;
+       struct drm_device *dev = &dev_priv->drm;
        struct drm_connector *connector;
        struct drm_connector_list_iter conn_iter;
        struct intel_dp *intel_dp;
@@ -3703,23 +3804,7 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data)
 
        return 0;
 }
-
-static int i915_displayport_test_type_open(struct inode *inode,
-                                      struct file *file)
-{
-       struct drm_i915_private *dev_priv = inode->i_private;
-
-       return single_open(file, i915_displayport_test_type_show,
-                          &dev_priv->drm);
-}
-
-static const struct file_operations i915_displayport_test_type_fops = {
-       .owner = THIS_MODULE,
-       .open = i915_displayport_test_type_open,
-       .read = seq_read,
-       .llseek = seq_lseek,
-       .release = single_release
-};
+DEFINE_SHOW_ATTRIBUTE(i915_displayport_test_type);
 
 static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 {
@@ -3987,8 +4072,8 @@ i915_wedged_set(void *data, u64 val)
                engine->hangcheck.stalled = true;
        }
 
-       i915_handle_error(i915, val, "Manually set wedged engine mask = %llx",
-                         val);
+       i915_handle_error(i915, val, I915_ERROR_CAPTURE,
+                         "Manually set wedged engine mask = %llx", val);
 
        wait_on_bit(&i915->gpu_error.flags,
                    I915_RESET_HANDOFF,
@@ -4151,119 +4236,6 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops,
                        i915_drop_caches_get, i915_drop_caches_set,
                        "0x%08llx\n");
 
-static int
-i915_max_freq_get(void *data, u64 *val)
-{
-       struct drm_i915_private *dev_priv = data;
-
-       if (INTEL_GEN(dev_priv) < 6)
-               return -ENODEV;
-
-       *val = intel_gpu_freq(dev_priv, dev_priv->gt_pm.rps.max_freq_softlimit);
-       return 0;
-}
-
-static int
-i915_max_freq_set(void *data, u64 val)
-{
-       struct drm_i915_private *dev_priv = data;
-       struct intel_rps *rps = &dev_priv->gt_pm.rps;
-       u32 hw_max, hw_min;
-       int ret;
-
-       if (INTEL_GEN(dev_priv) < 6)
-               return -ENODEV;
-
-       DRM_DEBUG_DRIVER("Manually setting max freq to %llu\n", val);
-
-       ret = mutex_lock_interruptible(&dev_priv->pcu_lock);
-       if (ret)
-               return ret;
-
-       /*
-        * Turbo will still be enabled, but won't go above the set value.
-        */
-       val = intel_freq_opcode(dev_priv, val);
-
-       hw_max = rps->max_freq;
-       hw_min = rps->min_freq;
-
-       if (val < hw_min || val > hw_max || val < rps->min_freq_softlimit) {
-               mutex_unlock(&dev_priv->pcu_lock);
-               return -EINVAL;
-       }
-
-       rps->max_freq_softlimit = val;
-
-       if (intel_set_rps(dev_priv, val))
-               DRM_DEBUG_DRIVER("failed to update RPS to new softlimit\n");
-
-       mutex_unlock(&dev_priv->pcu_lock);
-
-       return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_max_freq_fops,
-                       i915_max_freq_get, i915_max_freq_set,
-                       "%llu\n");
-
-static int
-i915_min_freq_get(void *data, u64 *val)
-{
-       struct drm_i915_private *dev_priv = data;
-
-       if (INTEL_GEN(dev_priv) < 6)
-               return -ENODEV;
-
-       *val = intel_gpu_freq(dev_priv, dev_priv->gt_pm.rps.min_freq_softlimit);
-       return 0;
-}
-
-static int
-i915_min_freq_set(void *data, u64 val)
-{
-       struct drm_i915_private *dev_priv = data;
-       struct intel_rps *rps = &dev_priv->gt_pm.rps;
-       u32 hw_max, hw_min;
-       int ret;
-
-       if (INTEL_GEN(dev_priv) < 6)
-               return -ENODEV;
-
-       DRM_DEBUG_DRIVER("Manually setting min freq to %llu\n", val);
-
-       ret = mutex_lock_interruptible(&dev_priv->pcu_lock);
-       if (ret)
-               return ret;
-
-       /*
-        * Turbo will still be enabled, but won't go below the set value.
-        */
-       val = intel_freq_opcode(dev_priv, val);
-
-       hw_max = rps->max_freq;
-       hw_min = rps->min_freq;
-
-       if (val < hw_min ||
-           val > hw_max || val > rps->max_freq_softlimit) {
-               mutex_unlock(&dev_priv->pcu_lock);
-               return -EINVAL;
-       }
-
-       rps->min_freq_softlimit = val;
-
-       if (intel_set_rps(dev_priv, val))
-               DRM_DEBUG_DRIVER("failed to update RPS to new softlimit\n");
-
-       mutex_unlock(&dev_priv->pcu_lock);
-
-       return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_min_freq_fops,
-                       i915_min_freq_get, i915_min_freq_set,
-                       "%llu\n");
-
 static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
@@ -4316,9 +4288,10 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_cache_sharing_fops,
 static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
                                          struct sseu_dev_info *sseu)
 {
-       int ss_max = 2;
+#define SS_MAX 2
+       const int ss_max = SS_MAX;
+       u32 sig1[SS_MAX], sig2[SS_MAX];
        int ss;
-       u32 sig1[ss_max], sig2[ss_max];
 
        sig1[0] = I915_READ(CHV_POWER_SS0_SIG1);
        sig1[1] = I915_READ(CHV_POWER_SS1_SIG1);
@@ -4342,15 +4315,16 @@ static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
                sseu->eu_per_subslice = max_t(unsigned int,
                                              sseu->eu_per_subslice, eu_cnt);
        }
+#undef SS_MAX
 }
 
 static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
                                     struct sseu_dev_info *sseu)
 {
+#define SS_MAX 6
        const struct intel_device_info *info = INTEL_INFO(dev_priv);
+       u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2];
        int s, ss;
-       u32 s_reg[info->sseu.max_slices];
-       u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
 
        for (s = 0; s < info->sseu.max_slices; s++) {
                /*
@@ -4397,15 +4371,16 @@ static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
                                                      eu_cnt);
                }
        }
+#undef SS_MAX
 }
 
 static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
                                    struct sseu_dev_info *sseu)
 {
+#define SS_MAX 3
        const struct intel_device_info *info = INTEL_INFO(dev_priv);
+       u32 s_reg[SS_MAX], eu_reg[2 * SS_MAX], eu_mask[2];
        int s, ss;
-       u32 s_reg[info->sseu.max_slices];
-       u32 eu_reg[2 * info->sseu.max_subslices], eu_mask[2];
 
        for (s = 0; s < info->sseu.max_slices; s++) {
                s_reg[s] = I915_READ(GEN9_SLICE_PGCTL_ACK(s));
@@ -4452,6 +4427,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
                                                      eu_cnt);
                }
        }
+#undef SS_MAX
 }
 
 static void broadwell_sseu_device_status(struct drm_i915_private *dev_priv,
@@ -4703,6 +4679,67 @@ static int i915_drrs_ctl_set(void *data, u64 val)
 
 DEFINE_SIMPLE_ATTRIBUTE(i915_drrs_ctl_fops, NULL, i915_drrs_ctl_set, "%llu\n");
 
+static ssize_t
+i915_fifo_underrun_reset_write(struct file *filp,
+                              const char __user *ubuf,
+                              size_t cnt, loff_t *ppos)
+{
+       struct drm_i915_private *dev_priv = filp->private_data;
+       struct intel_crtc *intel_crtc;
+       struct drm_device *dev = &dev_priv->drm;
+       int ret;
+       bool reset;
+
+       ret = kstrtobool_from_user(ubuf, cnt, &reset);
+       if (ret)
+               return ret;
+
+       if (!reset)
+               return cnt;
+
+       for_each_intel_crtc(dev, intel_crtc) {
+               struct drm_crtc_commit *commit;
+               struct intel_crtc_state *crtc_state;
+
+               ret = drm_modeset_lock_single_interruptible(&intel_crtc->base.mutex);
+               if (ret)
+                       return ret;
+
+               crtc_state = to_intel_crtc_state(intel_crtc->base.state);
+               commit = crtc_state->base.commit;
+               if (commit) {
+                       ret = wait_for_completion_interruptible(&commit->hw_done);
+                       if (!ret)
+                               ret = wait_for_completion_interruptible(&commit->flip_done);
+               }
+
+               if (!ret && crtc_state->base.active) {
+                       DRM_DEBUG_KMS("Re-arming FIFO underruns on pipe %c\n",
+                                     pipe_name(intel_crtc->pipe));
+
+                       intel_crtc_arm_fifo_underrun(intel_crtc, crtc_state);
+               }
+
+               drm_modeset_unlock(&intel_crtc->base.mutex);
+
+               if (ret)
+                       return ret;
+       }
+
+       ret = intel_fbc_reset_underrun(dev_priv);
+       if (ret)
+               return ret;
+
+       return cnt;
+}
+
+static const struct file_operations i915_fifo_underrun_reset_ops = {
+       .owner = THIS_MODULE,
+       .open = simple_open,
+       .write = i915_fifo_underrun_reset_write,
+       .llseek = default_llseek,
+};
+
 static const struct drm_info_list i915_debugfs_list[] = {
        {"i915_capabilities", i915_capabilities, 0},
        {"i915_gem_objects", i915_gem_object_info, 0},
@@ -4760,8 +4797,6 @@ static const struct i915_debugfs_files {
        const struct file_operations *fops;
 } i915_debugfs_files[] = {
        {"i915_wedged", &i915_wedged_fops},
-       {"i915_max_freq", &i915_max_freq_fops},
-       {"i915_min_freq", &i915_min_freq_fops},
        {"i915_cache_sharing", &i915_cache_sharing_fops},
        {"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
        {"i915_ring_test_irq", &i915_ring_test_irq_fops},
@@ -4770,6 +4805,7 @@ static const struct i915_debugfs_files {
        {"i915_error_state", &i915_error_state_fops},
        {"i915_gpu_info", &i915_gpu_info_fops},
 #endif
+       {"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops},
        {"i915_next_seqno", &i915_next_seqno_fops},
        {"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
        {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
@@ -4779,10 +4815,12 @@ static const struct i915_debugfs_files {
        {"i915_dp_test_data", &i915_displayport_test_data_fops},
        {"i915_dp_test_type", &i915_displayport_test_type_fops},
        {"i915_dp_test_active", &i915_displayport_test_active_fops},
-       {"i915_guc_log_control", &i915_guc_log_control_fops},
+       {"i915_guc_log_level", &i915_guc_log_level_fops},
+       {"i915_guc_log_relay", &i915_guc_log_relay_fops},
        {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops},
        {"i915_ipc_status", &i915_ipc_status_fops},
-       {"i915_drrs_ctl", &i915_drrs_ctl_fops}
+       {"i915_drrs_ctl", &i915_drrs_ctl_fops},
+       {"i915_edp_psr_debug", &i915_edp_psr_debug_fops}
 };
 
 int i915_debugfs_register(struct drm_i915_private *dev_priv)
@@ -4876,19 +4914,7 @@ static int i915_dpcd_show(struct seq_file *m, void *data)
 
        return 0;
 }
-
-static int i915_dpcd_open(struct inode *inode, struct file *file)
-{
-       return single_open(file, i915_dpcd_show, inode->i_private);
-}
-
-static const struct file_operations i915_dpcd_fops = {
-       .owner = THIS_MODULE,
-       .open = i915_dpcd_open,
-       .read = seq_read,
-       .llseek = seq_lseek,
-       .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(i915_dpcd);
 
 static int i915_panel_show(struct seq_file *m, void *data)
 {
@@ -4910,19 +4936,7 @@ static int i915_panel_show(struct seq_file *m, void *data)
 
        return 0;
 }
-
-static int i915_panel_open(struct inode *inode, struct file *file)
-{
-       return single_open(file, i915_panel_show, inode->i_private);
-}
-
-static const struct file_operations i915_panel_fops = {
-       .owner = THIS_MODULE,
-       .open = i915_panel_open,
-       .read = seq_read,
-       .llseek = seq_lseek,
-       .release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(i915_panel);
 
 /**
  * i915_debugfs_connector_add - add i915 specific connector debugfs files
index 3b4daaf..b7dbeba 100644 (file)
@@ -377,9 +377,9 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
                value = INTEL_INFO(dev_priv)->sseu.min_eu_in_pool;
                break;
        case I915_PARAM_HUC_STATUS:
-               intel_runtime_pm_get(dev_priv);
-               value = I915_READ(HUC_STATUS2) & HUC_FW_VERIFIED;
-               intel_runtime_pm_put(dev_priv);
+               value = intel_huc_check_status(&dev_priv->huc);
+               if (value < 0)
+                       return value;
                break;
        case I915_PARAM_MMAP_GTT_VERSION:
                /* Though we've started our numbering from 1, and so class all
@@ -695,11 +695,9 @@ static int i915_load_modeset_init(struct drm_device *dev)
        if (ret)
                goto cleanup_irq;
 
-       intel_uc_init_fw(dev_priv);
-
        ret = i915_gem_init(dev_priv);
        if (ret)
-               goto cleanup_uc;
+               goto cleanup_irq;
 
        intel_setup_overlay(dev_priv);
 
@@ -719,8 +717,6 @@ cleanup_gem:
        if (i915_gem_suspend(dev_priv))
                DRM_ERROR("failed to idle hardware; continuing to unload!\n");
        i915_gem_fini(dev_priv);
-cleanup_uc:
-       intel_uc_fini_fw(dev_priv);
 cleanup_irq:
        drm_irq_uninstall(dev);
        intel_teardown_gmbus(dev_priv);
@@ -922,16 +918,21 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
        mutex_init(&dev_priv->wm.wm_mutex);
        mutex_init(&dev_priv->pps_mutex);
 
-       intel_uc_init_early(dev_priv);
        i915_memcpy_init_early(dev_priv);
 
        ret = i915_workqueues_init(dev_priv);
        if (ret < 0)
                goto err_engines;
 
+       ret = i915_gem_init_early(dev_priv);
+       if (ret < 0)
+               goto err_workqueues;
+
        /* This must be called before any calls to HAS_PCH_* */
        intel_detect_pch(dev_priv);
 
+       intel_wopcm_init_early(&dev_priv->wopcm);
+       intel_uc_init_early(dev_priv);
        intel_pm_setup(dev_priv);
        intel_init_dpio(dev_priv);
        intel_power_domains_init(dev_priv);
@@ -940,18 +941,13 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
        intel_init_display_hooks(dev_priv);
        intel_init_clock_gating_hooks(dev_priv);
        intel_init_audio_hooks(dev_priv);
-       ret = i915_gem_load_init(dev_priv);
-       if (ret < 0)
-               goto err_irq;
-
        intel_display_crc_init(dev_priv);
 
        intel_detect_preproduction_hw(dev_priv);
 
        return 0;
 
-err_irq:
-       intel_irq_fini(dev_priv);
+err_workqueues:
        i915_workqueues_cleanup(dev_priv);
 err_engines:
        i915_engines_cleanup(dev_priv);
@@ -964,8 +960,9 @@ err_engines:
  */
 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 {
-       i915_gem_load_cleanup(dev_priv);
        intel_irq_fini(dev_priv);
+       intel_uc_cleanup_early(dev_priv);
+       i915_gem_cleanup_early(dev_priv);
        i915_workqueues_cleanup(dev_priv);
        i915_engines_cleanup(dev_priv);
 }
@@ -1035,6 +1032,10 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
 
        intel_uncore_init(dev_priv);
 
+       intel_device_info_init_mmio(dev_priv);
+
+       intel_uncore_prune(dev_priv);
+
        intel_uc_init_mmio(dev_priv);
 
        ret = intel_engines_init_mmio(dev_priv);
@@ -1077,8 +1078,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
                                            i915_modparams.enable_ppgtt);
        DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt);
 
-       intel_uc_sanitize_options(dev_priv);
-
        intel_gvt_sanitize_options(dev_priv);
 }
 
@@ -1244,7 +1243,6 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
        /* Reveal our presence to userspace */
        if (drm_dev_register(dev, 0) == 0) {
                i915_debugfs_register(dev_priv);
-               i915_guc_log_register(dev_priv);
                i915_setup_sysfs(dev_priv);
 
                /* Depends on sysfs having been initialized */
@@ -1304,7 +1302,6 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
        i915_pmu_unregister(dev_priv);
 
        i915_teardown_sysfs(dev_priv);
-       i915_guc_log_unregister(dev_priv);
        drm_dev_unregister(&dev_priv->drm);
 
        i915_gem_shrinker_unregister(dev_priv);
@@ -1463,7 +1460,6 @@ void i915_driver_unload(struct drm_device *dev)
        i915_reset_error_state(dev_priv);
 
        i915_gem_fini(dev_priv);
-       intel_uc_fini_fw(dev_priv);
        intel_fbc_cleanup_cfb(dev_priv);
 
        intel_power_domains_fini(dev_priv);
@@ -1876,7 +1872,8 @@ static int i915_resume_switcheroo(struct drm_device *dev)
 /**
  * i915_reset - reset chip after a hang
  * @i915: #drm_i915_private to reset
- * @flags: Instructions
+ * @stalled_mask: mask of the stalled engines with the guilty requests
+ * @reason: user error message for why we are resetting
  *
  * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
  * on failure.
@@ -1891,12 +1888,16 @@ static int i915_resume_switcheroo(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-void i915_reset(struct drm_i915_private *i915, unsigned int flags)
+void i915_reset(struct drm_i915_private *i915,
+               unsigned int stalled_mask,
+               const char *reason)
 {
        struct i915_gpu_error *error = &i915->gpu_error;
        int ret;
        int i;
 
+       GEM_TRACE("flags=%lx\n", error->flags);
+
        might_sleep();
        lockdep_assert_held(&i915->drm.struct_mutex);
        GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
@@ -1908,8 +1909,8 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
        if (!i915_gem_unset_wedged(i915))
                goto wakeup;
 
-       if (!(flags & I915_RESET_QUIET))
-               dev_notice(i915->drm.dev, "Resetting chip after gpu hang\n");
+       if (reason)
+               dev_notice(i915->drm.dev, "Resetting chip for %s\n", reason);
        error->reset_count++;
 
        disable_irq(i915->drm.irq);
@@ -1952,7 +1953,7 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
                goto error;
        }
 
-       i915_gem_reset(i915);
+       i915_gem_reset(i915, stalled_mask);
        intel_overlay_reset(i915);
 
        /*
@@ -1998,7 +1999,6 @@ taint:
 error:
        i915_gem_set_wedged(i915);
        i915_retire_requests(i915);
-       intel_gpu_reset(i915, ALL_ENGINES);
        goto finish;
 }
 
@@ -2011,7 +2011,7 @@ static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
 /**
  * i915_reset_engine - reset GPU engine to recover from a hang
  * @engine: engine to reset
- * @flags: options
+ * @msg: reason for GPU reset; or NULL for no dev_notice()
  *
  * Reset a specific GPU engine. Useful if a hang is detected.
  * Returns zero on successful reset or otherwise an error code.
@@ -2021,12 +2021,13 @@ static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
  *  - reset engine (which will force the engine to idle)
  *  - re-init/configure engine
  */
-int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
+int i915_reset_engine(struct intel_engine_cs *engine, const char *msg)
 {
        struct i915_gpu_error *error = &engine->i915->gpu_error;
        struct i915_request *active_request;
        int ret;
 
+       GEM_TRACE("%s flags=%lx\n", engine->name, error->flags);
        GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
 
        active_request = i915_gem_reset_prepare_engine(engine);
@@ -2036,10 +2037,9 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
                goto out;
        }
 
-       if (!(flags & I915_RESET_QUIET)) {
+       if (msg)
                dev_notice(engine->i915->drm.dev,
-                          "Resetting %s after gpu hang\n", engine->name);
-       }
+                          "Resetting %s for %s\n", engine->name, msg);
        error->reset_engine_count[engine->id]++;
 
        if (!engine->i915->guc.execbuf_client)
@@ -2059,7 +2059,7 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
         * active request and can drop it, adjust head to skip the offending
         * request to resume executing remaining requests in the queue.
         */
-       i915_gem_reset_engine(engine, active_request);
+       i915_gem_reset_engine(engine, active_request, true);
 
        /*
         * The engine and its registers (and workarounds in case of render)
index ce18b6c..6268a51 100644 (file)
@@ -64,6 +64,7 @@
 #include "intel_opregion.h"
 #include "intel_ringbuffer.h"
 #include "intel_uncore.h"
+#include "intel_wopcm.h"
 #include "intel_uc.h"
 
 #include "i915_gem.h"
@@ -72,8 +73,9 @@
 #include "i915_gem_object.h"
 #include "i915_gem_gtt.h"
 #include "i915_gem_timeline.h"
-
+#include "i915_gpu_error.h"
 #include "i915_request.h"
+#include "i915_scheduler.h"
 #include "i915_vma.h"
 
 #include "intel_gvt.h"
@@ -83,8 +85,8 @@
 
 #define DRIVER_NAME            "i915"
 #define DRIVER_DESC            "Intel Graphics"
-#define DRIVER_DATE            "20180308"
-#define DRIVER_TIMESTAMP       1520513379
+#define DRIVER_DATE            "20180413"
+#define DRIVER_TIMESTAMP       1523611258
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -261,6 +263,7 @@ enum hpd_pin {
        HPD_PORT_C,
        HPD_PORT_D,
        HPD_PORT_E,
+       HPD_PORT_F,
        HPD_NUM_PINS
 };
 
@@ -453,172 +456,6 @@ struct intel_csr {
        uint32_t allowed_dc_mask;
 };
 
-struct intel_display_error_state;
-
-struct i915_gpu_state {
-       struct kref ref;
-       ktime_t time;
-       ktime_t boottime;
-       ktime_t uptime;
-
-       struct drm_i915_private *i915;
-
-       char error_msg[128];
-       bool simulated;
-       bool awake;
-       bool wakelock;
-       bool suspended;
-       int iommu;
-       u32 reset_count;
-       u32 suspend_count;
-       struct intel_device_info device_info;
-       struct intel_driver_caps driver_caps;
-       struct i915_params params;
-
-       struct i915_error_uc {
-               struct intel_uc_fw guc_fw;
-               struct intel_uc_fw huc_fw;
-               struct drm_i915_error_object *guc_log;
-       } uc;
-
-       /* Generic register state */
-       u32 eir;
-       u32 pgtbl_er;
-       u32 ier;
-       u32 gtier[4], ngtier;
-       u32 ccid;
-       u32 derrmr;
-       u32 forcewake;
-       u32 error; /* gen6+ */
-       u32 err_int; /* gen7 */
-       u32 fault_data0; /* gen8, gen9 */
-       u32 fault_data1; /* gen8, gen9 */
-       u32 done_reg;
-       u32 gac_eco;
-       u32 gam_ecochk;
-       u32 gab_ctl;
-       u32 gfx_mode;
-
-       u32 nfence;
-       u64 fence[I915_MAX_NUM_FENCES];
-       struct intel_overlay_error_state *overlay;
-       struct intel_display_error_state *display;
-
-       struct drm_i915_error_engine {
-               int engine_id;
-               /* Software tracked state */
-               bool idle;
-               bool waiting;
-               int num_waiters;
-               unsigned long hangcheck_timestamp;
-               bool hangcheck_stalled;
-               enum intel_engine_hangcheck_action hangcheck_action;
-               struct i915_address_space *vm;
-               int num_requests;
-               u32 reset_count;
-
-               /* position of active request inside the ring */
-               u32 rq_head, rq_post, rq_tail;
-
-               /* our own tracking of ring head and tail */
-               u32 cpu_ring_head;
-               u32 cpu_ring_tail;
-
-               u32 last_seqno;
-
-               /* Register state */
-               u32 start;
-               u32 tail;
-               u32 head;
-               u32 ctl;
-               u32 mode;
-               u32 hws;
-               u32 ipeir;
-               u32 ipehr;
-               u32 bbstate;
-               u32 instpm;
-               u32 instps;
-               u32 seqno;
-               u64 bbaddr;
-               u64 acthd;
-               u32 fault_reg;
-               u64 faddr;
-               u32 rc_psmi; /* sleep state */
-               u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
-               struct intel_instdone instdone;
-
-               struct drm_i915_error_context {
-                       char comm[TASK_COMM_LEN];
-                       pid_t pid;
-                       u32 handle;
-                       u32 hw_id;
-                       int priority;
-                       int ban_score;
-                       int active;
-                       int guilty;
-                       bool bannable;
-               } context;
-
-               struct drm_i915_error_object {
-                       u64 gtt_offset;
-                       u64 gtt_size;
-                       int page_count;
-                       int unused;
-                       u32 *pages[0];
-               } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
-
-               struct drm_i915_error_object **user_bo;
-               long user_bo_count;
-
-               struct drm_i915_error_object *wa_ctx;
-               struct drm_i915_error_object *default_state;
-
-               struct drm_i915_error_request {
-                       long jiffies;
-                       pid_t pid;
-                       u32 context;
-                       int priority;
-                       int ban_score;
-                       u32 seqno;
-                       u32 head;
-                       u32 tail;
-               } *requests, execlist[EXECLIST_MAX_PORTS];
-               unsigned int num_ports;
-
-               struct drm_i915_error_waiter {
-                       char comm[TASK_COMM_LEN];
-                       pid_t pid;
-                       u32 seqno;
-               } *waiters;
-
-               struct {
-                       u32 gfx_mode;
-                       union {
-                               u64 pdp[4];
-                               u32 pp_dir_base;
-                       };
-               } vm_info;
-       } engine[I915_NUM_ENGINES];
-
-       struct drm_i915_error_buffer {
-               u32 size;
-               u32 name;
-               u32 rseqno[I915_NUM_ENGINES], wseqno;
-               u64 gtt_offset;
-               u32 read_domains;
-               u32 write_domain;
-               s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
-               u32 tiling:2;
-               u32 dirty:1;
-               u32 purgeable:1;
-               u32 userptr:1;
-               s32 engine:4;
-               u32 cache_level:3;
-       } *active_bo[I915_NUM_ENGINES], *pinned_bo;
-       u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
-       struct i915_address_space *active_vm[I915_NUM_ENGINES];
-};
-
 enum i915_cache_level {
        I915_CACHE_NONE = 0,
        I915_CACHE_LLC, /* also used for snoopable memory on non-LLC */
@@ -766,12 +603,16 @@ struct i915_psr {
        bool active;
        struct delayed_work work;
        unsigned busy_frontbuffer_bits;
-       bool psr2_support;
-       bool aux_frame_sync;
+       bool sink_psr2_support;
        bool link_standby;
-       bool y_cord_support;
        bool colorimetry_support;
        bool alpm;
+       bool has_hw_tracking;
+       bool psr2_enabled;
+       u8 sink_sync_latency;
+       bool debug;
+       ktime_t last_entry_attempt;
+       ktime_t last_exit;
 
        void (*enable_source)(struct intel_dp *,
                              const struct intel_crtc_state *);
@@ -1146,16 +987,6 @@ struct i915_gem_mm {
        u32 object_count;
 };
 
-struct drm_i915_error_state_buf {
-       struct drm_i915_private *i915;
-       unsigned bytes;
-       unsigned size;
-       int err;
-       u8 *buf;
-       loff_t start;
-       loff_t pos;
-};
-
 #define I915_IDLE_ENGINES_TIMEOUT (200) /* in ms */
 
 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
@@ -1164,102 +995,6 @@ struct drm_i915_error_state_buf {
 #define I915_ENGINE_DEAD_TIMEOUT  (4 * HZ)  /* Seqno, head and subunits dead */
 #define I915_SEQNO_DEAD_TIMEOUT   (12 * HZ) /* Seqno dead with active head */
 
-struct i915_gpu_error {
-       /* For hangcheck timer */
-#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
-#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
-
-       struct delayed_work hangcheck_work;
-
-       /* For reset and error_state handling. */
-       spinlock_t lock;
-       /* Protected by the above dev->gpu_error.lock. */
-       struct i915_gpu_state *first_error;
-
-       atomic_t pending_fb_pin;
-
-       unsigned long missed_irq_rings;
-
-       /**
-        * State variable controlling the reset flow and count
-        *
-        * This is a counter which gets incremented when reset is triggered,
-        *
-        * Before the reset commences, the I915_RESET_BACKOFF bit is set
-        * meaning that any waiters holding onto the struct_mutex should
-        * relinquish the lock immediately in order for the reset to start.
-        *
-        * If reset is not completed succesfully, the I915_WEDGE bit is
-        * set meaning that hardware is terminally sour and there is no
-        * recovery. All waiters on the reset_queue will be woken when
-        * that happens.
-        *
-        * This counter is used by the wait_seqno code to notice that reset
-        * event happened and it needs to restart the entire ioctl (since most
-        * likely the seqno it waited for won't ever signal anytime soon).
-        *
-        * This is important for lock-free wait paths, where no contended lock
-        * naturally enforces the correct ordering between the bail-out of the
-        * waiter and the gpu reset work code.
-        */
-       unsigned long reset_count;
-
-       /**
-        * flags: Control various stages of the GPU reset
-        *
-        * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
-        * other users acquiring the struct_mutex. To do this we set the
-        * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
-        * and then check for that bit before acquiring the struct_mutex (in
-        * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
-        * secondary role in preventing two concurrent global reset attempts.
-        *
-        * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the
-        * struct_mutex. We try to acquire the struct_mutex in the reset worker,
-        * but it may be held by some long running waiter (that we cannot
-        * interrupt without causing trouble). Once we are ready to do the GPU
-        * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If
-        * they already hold the struct_mutex and want to participate they can
-        * inspect the bit and do the reset directly, otherwise the worker
-        * waits for the struct_mutex.
-        *
-        * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
-        * acquire the struct_mutex to reset an engine, we need an explicit
-        * flag to prevent two concurrent reset attempts in the same engine.
-        * As the number of engines continues to grow, allocate the flags from
-        * the most significant bits.
-        *
-        * #I915_WEDGED - If reset fails and we can no longer use the GPU,
-        * we set the #I915_WEDGED bit. Prior to command submission, e.g.
-        * i915_request_alloc(), this bit is checked and the sequence
-        * aborted (with -EIO reported to userspace) if set.
-        */
-       unsigned long flags;
-#define I915_RESET_BACKOFF     0
-#define I915_RESET_HANDOFF     1
-#define I915_RESET_MODESET     2
-#define I915_WEDGED            (BITS_PER_LONG - 1)
-#define I915_RESET_ENGINE      (I915_WEDGED - I915_NUM_ENGINES)
-
-       /** Number of times an engine has been reset */
-       u32 reset_engine_count[I915_NUM_ENGINES];
-
-       /**
-        * Waitqueue to signal when a hang is detected. Used to for waiters
-        * to release the struct_mutex for the reset to procede.
-        */
-       wait_queue_head_t wait_queue;
-
-       /**
-        * Waitqueue to signal when the reset has completed. Used by clients
-        * that wait for dev_priv->mm.wedged to settle.
-        */
-       wait_queue_head_t reset_queue;
-
-       /* For missed irq/seqno simulation. */
-       unsigned long test_irq_rings;
-};
-
 enum modeset_restore {
        MODESET_ON_LID_OPEN,
        MODESET_DONE,
@@ -1451,11 +1186,13 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1,
 }
 
 struct skl_ddb_allocation {
-       struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */
-       struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES];
+       /* packed/y */
+       struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES];
+       struct skl_ddb_entry uv_plane[I915_MAX_PIPES][I915_MAX_PLANES];
+       u8 enabled_slices; /* GEN11 has configurable 2 slices */
 };
 
-struct skl_wm_values {
+struct skl_ddb_values {
        unsigned dirty_pipes;
        struct skl_ddb_allocation ddb;
 };
@@ -1470,6 +1207,7 @@ struct skl_wm_level {
 struct skl_wm_params {
        bool x_tiled, y_tiled;
        bool rc_surface;
+       bool is_planar;
        uint32_t width;
        uint8_t cpp;
        uint32_t plane_pixel_rate;
@@ -1564,7 +1302,6 @@ struct i915_wa_reg {
 struct i915_workarounds {
        struct i915_wa_reg reg[I915_MAX_WA_REGS];
        u32 count;
-       u32 hw_whitelist_count[I915_NUM_ENGINES];
 };
 
 struct i915_virtual_gpu {
@@ -1860,6 +1597,8 @@ struct drm_i915_private {
 
        struct intel_gvt *gvt;
 
+       struct intel_wopcm wopcm;
+
        struct intel_huc huc;
        struct intel_guc guc;
 
@@ -2152,7 +1891,7 @@ struct drm_i915_private {
                /* current hardware state */
                union {
                        struct ilk_wm_values hw;
-                       struct skl_wm_values skl_hw;
+                       struct skl_ddb_values skl_hw;
                        struct vlv_wm_values vlv;
                        struct g4x_wm_values g4x;
                };
@@ -2320,9 +2059,12 @@ struct drm_i915_private {
                void (*resume)(struct drm_i915_private *);
                void (*cleanup_engine)(struct intel_engine_cs *engine);
 
-               struct list_head timelines;
                struct i915_gem_timeline global_timeline;
+               struct list_head timelines;
+
+               struct list_head active_rings;
                u32 active_requests;
+               u32 request_serial;
 
                /**
                 * Is the GPU currently considered idle, or busy executing
@@ -2392,6 +2134,11 @@ static inline struct drm_i915_private *kdev_to_i915(struct device *kdev)
        return to_i915(dev_get_drvdata(kdev));
 }
 
+static inline struct drm_i915_private *wopcm_to_i915(struct intel_wopcm *wopcm)
+{
+       return container_of(wopcm, struct drm_i915_private, wopcm);
+}
+
 static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc)
 {
        return container_of(guc, struct drm_i915_private, guc);
@@ -2411,8 +2158,10 @@ static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
 
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
-       for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask;        \
-            tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; )
+       for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->ring_mask; \
+            (tmp__) ? \
+            ((engine__) = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : \
+            0;)
 
 enum hdmi_force_audio {
        HDMI_AUDIO_OFF_DVI = -2,        /* no aux data for HDMI-DVI converter */
@@ -2963,10 +2712,11 @@ extern void i915_driver_unload(struct drm_device *dev);
 extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
 extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
 
-#define I915_RESET_QUIET BIT(0)
-extern void i915_reset(struct drm_i915_private *i915, unsigned int flags);
+extern void i915_reset(struct drm_i915_private *i915,
+                      unsigned int stalled_mask,
+                      const char *reason);
 extern int i915_reset_engine(struct intel_engine_cs *engine,
-                            unsigned int flags);
+                            const char *reason);
 
 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
 extern int intel_reset_guc(struct drm_i915_private *dev_priv);
@@ -3014,10 +2764,12 @@ static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv)
                           &dev_priv->gpu_error.hangcheck_work, delay);
 }
 
-__printf(3, 4)
+__printf(4, 5)
 void i915_handle_error(struct drm_i915_private *dev_priv,
                       u32 engine_mask,
+                      unsigned long flags,
                       const char *fmt, ...);
+#define I915_ERROR_CAPTURE BIT(0)
 
 extern void intel_irq_init(struct drm_i915_private *dev_priv);
 extern void intel_irq_fini(struct drm_i915_private *dev_priv);
@@ -3132,8 +2884,8 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv);
 void i915_gem_sanitize(struct drm_i915_private *i915);
-int i915_gem_load_init(struct drm_i915_private *dev_priv);
-void i915_gem_load_cleanup(struct drm_i915_private *dev_priv);
+int i915_gem_init_early(struct drm_i915_private *dev_priv);
+void i915_gem_cleanup_early(struct drm_i915_private *dev_priv);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
 int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
@@ -3388,13 +3140,15 @@ static inline u32 i915_reset_engine_count(struct i915_gpu_error *error,
 struct i915_request *
 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine);
 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv);
-void i915_gem_reset(struct drm_i915_private *dev_priv);
+void i915_gem_reset(struct drm_i915_private *dev_priv,
+                   unsigned int stalled_mask);
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine);
 void i915_gem_reset_finish(struct drm_i915_private *dev_priv);
 void i915_gem_set_wedged(struct drm_i915_private *dev_priv);
 bool i915_gem_unset_wedged(struct drm_i915_private *dev_priv);
 void i915_gem_reset_engine(struct intel_engine_cs *engine,
-                          struct i915_request *request);
+                          struct i915_request *request,
+                          bool stalled);
 
 void i915_gem_init_mmio(struct drm_i915_private *i915);
 int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
@@ -3412,7 +3166,7 @@ int i915_gem_object_wait(struct drm_i915_gem_object *obj,
                         struct intel_rps_client *rps);
 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
                                  unsigned int flags,
-                                 int priority);
+                                 const struct i915_sched_attr *attr);
 #define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX
 
 int __must_check
@@ -3589,64 +3343,6 @@ static inline int i915_debugfs_connector_add(struct drm_connector *connector)
 static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
-/* i915_gpu_error.c */
-#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
-
-__printf(2, 3)
-void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
-int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
-                           const struct i915_gpu_state *gpu);
-int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
-                             struct drm_i915_private *i915,
-                             size_t count, loff_t pos);
-static inline void i915_error_state_buf_release(
-       struct drm_i915_error_state_buf *eb)
-{
-       kfree(eb->buf);
-}
-
-struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
-void i915_capture_error_state(struct drm_i915_private *dev_priv,
-                             u32 engine_mask,
-                             const char *error_msg);
-
-static inline struct i915_gpu_state *
-i915_gpu_state_get(struct i915_gpu_state *gpu)
-{
-       kref_get(&gpu->ref);
-       return gpu;
-}
-
-void __i915_gpu_state_free(struct kref *kref);
-static inline void i915_gpu_state_put(struct i915_gpu_state *gpu)
-{
-       if (gpu)
-               kref_put(&gpu->ref, __i915_gpu_state_free);
-}
-
-struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915);
-void i915_reset_error_state(struct drm_i915_private *i915);
-
-#else
-
-static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
-                                           u32 engine_mask,
-                                           const char *error_msg)
-{
-}
-
-static inline struct i915_gpu_state *
-i915_first_error_state(struct drm_i915_private *i915)
-{
-       return NULL;
-}
-
-static inline void i915_reset_error_state(struct drm_i915_private *i915)
-{
-}
-
-#endif
-
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
index 7b5a9d7..fa1d94a 100644 (file)
@@ -35,6 +35,7 @@
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
 #include "intel_mocs.h"
+#include "intel_workarounds.h"
 #include "i915_gemfs.h"
 #include <linux/dma-fence-array.h>
 #include <linux/kthread.h>
@@ -136,6 +137,101 @@ int i915_mutex_lock_interruptible(struct drm_device *dev)
        return 0;
 }
 
+static u32 __i915_gem_park(struct drm_i915_private *i915)
+{
+       lockdep_assert_held(&i915->drm.struct_mutex);
+       GEM_BUG_ON(i915->gt.active_requests);
+       GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
+
+       if (!i915->gt.awake)
+               return I915_EPOCH_INVALID;
+
+       GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
+
+       /*
+        * Be paranoid and flush a concurrent interrupt to make sure
+        * we don't reactivate any irq tasklets after parking.
+        *
+        * FIXME: Note that even though we have waited for execlists to be idle,
+        * there may still be an in-flight interrupt even though the CSB
+        * is now empty. synchronize_irq() makes sure that a residual interrupt
+        * is completed before we continue, but it doesn't prevent the HW from
+        * raising a spurious interrupt later. To complete the shield we should
+        * coordinate disabling the CS irq with flushing the interrupts.
+        */
+       synchronize_irq(i915->drm.irq);
+
+       intel_engines_park(i915);
+       i915_gem_timelines_park(i915);
+
+       i915_pmu_gt_parked(i915);
+
+       i915->gt.awake = false;
+
+       if (INTEL_GEN(i915) >= 6)
+               gen6_rps_idle(i915);
+
+       intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ);
+
+       intel_runtime_pm_put(i915);
+
+       return i915->gt.epoch;
+}
+
+void i915_gem_park(struct drm_i915_private *i915)
+{
+       lockdep_assert_held(&i915->drm.struct_mutex);
+       GEM_BUG_ON(i915->gt.active_requests);
+
+       if (!i915->gt.awake)
+               return;
+
+       /* Defer the actual call to __i915_gem_park() to prevent ping-pongs */
+       mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100));
+}
+
+void i915_gem_unpark(struct drm_i915_private *i915)
+{
+       lockdep_assert_held(&i915->drm.struct_mutex);
+       GEM_BUG_ON(!i915->gt.active_requests);
+
+       if (i915->gt.awake)
+               return;
+
+       intel_runtime_pm_get_noresume(i915);
+
+       /*
+        * It seems that the DMC likes to transition between the DC states a lot
+        * when there are no connected displays (no active power domains) during
+        * command submission.
+        *
+        * This activity has negative impact on the performance of the chip with
+        * huge latencies observed in the interrupt handler and elsewhere.
+        *
+        * Work around it by grabbing a GT IRQ power domain whilst there is any
+        * GT activity, preventing any DC state transitions.
+        */
+       intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
+
+       i915->gt.awake = true;
+       if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
+               i915->gt.epoch = 1;
+
+       intel_enable_gt_powersave(i915);
+       i915_update_gfx_val(i915);
+       if (INTEL_GEN(i915) >= 6)
+               gen6_rps_busy(i915);
+       i915_pmu_gt_unparked(i915);
+
+       intel_engines_unpark(i915);
+
+       i915_queue_hangcheck(i915);
+
+       queue_delayed_work(i915->wq,
+                          &i915->gt.retire_work,
+                          round_jiffies_up_relative(HZ));
+}
+
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
                            struct drm_file *file)
@@ -469,7 +565,8 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
        return timeout;
 }
 
-static void __fence_set_priority(struct dma_fence *fence, int prio)
+static void __fence_set_priority(struct dma_fence *fence,
+                                const struct i915_sched_attr *attr)
 {
        struct i915_request *rq;
        struct intel_engine_cs *engine;
@@ -482,11 +579,12 @@ static void __fence_set_priority(struct dma_fence *fence, int prio)
 
        rcu_read_lock();
        if (engine->schedule)
-               engine->schedule(rq, prio);
+               engine->schedule(rq, attr);
        rcu_read_unlock();
 }
 
-static void fence_set_priority(struct dma_fence *fence, int prio)
+static void fence_set_priority(struct dma_fence *fence,
+                              const struct i915_sched_attr *attr)
 {
        /* Recurse once into a fence-array */
        if (dma_fence_is_array(fence)) {
@@ -494,16 +592,16 @@ static void fence_set_priority(struct dma_fence *fence, int prio)
                int i;
 
                for (i = 0; i < array->num_fences; i++)
-                       __fence_set_priority(array->fences[i], prio);
+                       __fence_set_priority(array->fences[i], attr);
        } else {
-               __fence_set_priority(fence, prio);
+               __fence_set_priority(fence, attr);
        }
 }
 
 int
 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
                              unsigned int flags,
-                             int prio)
+                             const struct i915_sched_attr *attr)
 {
        struct dma_fence *excl;
 
@@ -518,7 +616,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
                        return ret;
 
                for (i = 0; i < count; i++) {
-                       fence_set_priority(shared[i], prio);
+                       fence_set_priority(shared[i], attr);
                        dma_fence_put(shared[i]);
                }
 
@@ -528,7 +626,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
        }
 
        if (excl) {
-               fence_set_priority(excl, prio);
+               fence_set_priority(excl, attr);
                dma_fence_put(excl);
        }
        return 0;
@@ -2896,20 +2994,6 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
        return active;
 }
 
-static bool engine_stalled(struct intel_engine_cs *engine)
-{
-       if (!engine->hangcheck.stalled)
-               return false;
-
-       /* Check for possible seqno movement after hang declaration */
-       if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) {
-               DRM_DEBUG_DRIVER("%s pardoned\n", engine->name);
-               return false;
-       }
-
-       return true;
-}
-
 /*
  * Ensure irq handler finishes, and not run again.
  * Also return the active request so that we only search for it once.
@@ -2998,6 +3082,7 @@ int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
        }
 
        i915_gem_revoke_fences(dev_priv);
+       intel_uc_sanitize(dev_priv);
 
        return err;
 }
@@ -3047,7 +3132,8 @@ static void engine_skip_context(struct i915_request *request)
 /* Returns the request if it was guilty of the hang */
 static struct i915_request *
 i915_gem_reset_request(struct intel_engine_cs *engine,
-                      struct i915_request *request)
+                      struct i915_request *request,
+                      bool stalled)
 {
        /* The guilty request will get skipped on a hung engine.
         *
@@ -3070,7 +3156,15 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
         * subsequent hangs.
         */
 
-       if (engine_stalled(engine)) {
+       if (i915_request_completed(request)) {
+               GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n",
+                         engine->name, request->global_seqno,
+                         request->fence.context, request->fence.seqno,
+                         intel_engine_get_seqno(engine));
+               stalled = false;
+       }
+
+       if (stalled) {
                i915_gem_context_mark_guilty(request->ctx);
                skip_request(request);
 
@@ -3101,7 +3195,8 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 }
 
 void i915_gem_reset_engine(struct intel_engine_cs *engine,
-                          struct i915_request *request)
+                          struct i915_request *request,
+                          bool stalled)
 {
        /*
         * Make sure this write is visible before we re-enable the interrupt
@@ -3111,7 +3206,7 @@ void i915_gem_reset_engine(struct intel_engine_cs *engine,
        smp_store_mb(engine->irq_posted, 0);
 
        if (request)
-               request = i915_gem_reset_request(engine, request);
+               request = i915_gem_reset_request(engine, request, stalled);
 
        if (request) {
                DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
@@ -3122,7 +3217,8 @@ void i915_gem_reset_engine(struct intel_engine_cs *engine,
        engine->reset_hw(engine, request);
 }
 
-void i915_gem_reset(struct drm_i915_private *dev_priv)
+void i915_gem_reset(struct drm_i915_private *dev_priv,
+                   unsigned int stalled_mask)
 {
        struct intel_engine_cs *engine;
        enum intel_engine_id id;
@@ -3134,10 +3230,12 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
        for_each_engine(engine, dev_priv, id) {
                struct i915_gem_context *ctx;
 
-               i915_gem_reset_engine(engine, engine->hangcheck.active_request);
+               i915_gem_reset_engine(engine,
+                                     engine->hangcheck.active_request,
+                                     stalled_mask & ENGINE_MASK(id));
                ctx = fetch_and_zero(&engine->last_retired_context);
                if (ctx)
-                       engine->context_unpin(engine, ctx);
+                       intel_context_unpin(ctx, engine);
 
                /*
                 * Ostensibily, we always want a context loaded for powersaving,
@@ -3160,13 +3258,6 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
        }
 
        i915_gem_restore_fences(dev_priv);
-
-       if (dev_priv->gt.awake) {
-               intel_sanitize_gt_powersave(dev_priv);
-               intel_enable_gt_powersave(dev_priv);
-               if (INTEL_GEN(dev_priv) >= 6)
-                       gen6_rps_busy(dev_priv);
-       }
 }
 
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
@@ -3192,6 +3283,9 @@ void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
 
 static void nop_submit_request(struct i915_request *request)
 {
+       GEM_TRACE("%s fence %llx:%d -> -EIO\n",
+                 request->engine->name,
+                 request->fence.context, request->fence.seqno);
        dma_fence_set_error(&request->fence, -EIO);
 
        i915_request_submit(request);
@@ -3201,6 +3295,9 @@ static void nop_complete_submit_request(struct i915_request *request)
 {
        unsigned long flags;
 
+       GEM_TRACE("%s fence %llx:%d -> -EIO\n",
+                 request->engine->name,
+                 request->fence.context, request->fence.seqno);
        dma_fence_set_error(&request->fence, -EIO);
 
        spin_lock_irqsave(&request->engine->timeline->lock, flags);
@@ -3214,7 +3311,9 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
        struct intel_engine_cs *engine;
        enum intel_engine_id id;
 
-       if (drm_debug & DRM_UT_DRIVER) {
+       GEM_TRACE("start\n");
+
+       if (GEM_SHOW_DEBUG()) {
                struct drm_printer p = drm_debug_printer(__func__);
 
                for_each_engine(engine, i915, id)
@@ -3237,6 +3336,9 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
        }
        i915->caps.scheduler = 0;
 
+       /* Even if the GPU reset fails, it should still stop the engines */
+       intel_gpu_reset(i915, ALL_ENGINES);
+
        /*
         * Make sure no one is running the old callback before we proceed with
         * cancelling requests and resetting the completion tracking. Otherwise
@@ -3278,6 +3380,8 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
                i915_gem_reset_finish_engine(engine);
        }
 
+       GEM_TRACE("end\n");
+
        wake_up_all(&i915->gpu_error.reset_queue);
 }
 
@@ -3290,7 +3394,10 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
        if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
                return true;
 
-       /* Before unwedging, make sure that all pending operations
+       GEM_TRACE("start\n");
+
+       /*
+        * Before unwedging, make sure that all pending operations
         * are flushed and errored out - we may have requests waiting upon
         * third party fences. We marked all inflight requests as EIO, and
         * every execbuf since returned EIO, for consistency we want all
@@ -3308,7 +3415,8 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
                        if (!rq)
                                continue;
 
-                       /* We can't use our normal waiter as we want to
+                       /*
+                        * We can't use our normal waiter as we want to
                         * avoid recursively trying to handle the current
                         * reset. The basic dma_fence_default_wait() installs
                         * a callback for dma_fence_signal(), which is
@@ -3323,8 +3431,11 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
                                return false;
                }
        }
+       i915_retire_requests(i915);
+       GEM_BUG_ON(i915->gt.active_requests);
 
-       /* Undo nop_submit_request. We prevent all new i915 requests from
+       /*
+        * Undo nop_submit_request. We prevent all new i915 requests from
         * being queued (by disallowing execbuf whilst wedged) so having
         * waited for all active requests above, we know the system is idle
         * and do not have to worry about a thread being inside
@@ -3335,6 +3446,8 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
        intel_engines_reset_default_submission(i915);
        i915_gem_contexts_lost(i915);
 
+       GEM_TRACE("end\n");
+
        smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
        clear_bit(I915_WEDGED, &i915->gpu_error.flags);
 
@@ -3473,36 +3586,9 @@ i915_gem_idle_work_handler(struct work_struct *work)
        if (new_requests_since_last_retire(dev_priv))
                goto out_unlock;
 
-       /*
-        * Be paranoid and flush a concurrent interrupt to make sure
-        * we don't reactivate any irq tasklets after parking.
-        *
-        * FIXME: Note that even though we have waited for execlists to be idle,
-        * there may still be an in-flight interrupt even though the CSB
-        * is now empty. synchronize_irq() makes sure that a residual interrupt
-        * is completed before we continue, but it doesn't prevent the HW from
-        * raising a spurious interrupt later. To complete the shield we should
-        * coordinate disabling the CS irq with flushing the interrupts.
-        */
-       synchronize_irq(dev_priv->drm.irq);
-
-       intel_engines_park(dev_priv);
-       i915_gem_timelines_park(dev_priv);
-
-       i915_pmu_gt_parked(dev_priv);
+       epoch = __i915_gem_park(dev_priv);
 
-       GEM_BUG_ON(!dev_priv->gt.awake);
-       dev_priv->gt.awake = false;
-       epoch = dev_priv->gt.epoch;
-       GEM_BUG_ON(epoch == I915_EPOCH_INVALID);
        rearm_hangcheck = false;
-
-       if (INTEL_GEN(dev_priv) >= 6)
-               gen6_rps_idle(dev_priv);
-
-       intel_display_power_put(dev_priv, POWER_DOMAIN_GT_IRQ);
-
-       intel_runtime_pm_put(dev_priv);
 out_unlock:
        mutex_unlock(&dev_priv->drm.struct_mutex);
 
@@ -3666,16 +3752,7 @@ static int wait_for_engines(struct drm_i915_private *i915)
        if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
                dev_err(i915->drm.dev,
                        "Failed to idle engines, declaring wedged!\n");
-               if (drm_debug & DRM_UT_DRIVER) {
-                       struct drm_printer p = drm_debug_printer(__func__);
-                       struct intel_engine_cs *engine;
-                       enum intel_engine_id id;
-
-                       for_each_engine(engine, i915, id)
-                               intel_engine_dump(engine, &p,
-                                                 "%s\n", engine->name);
-               }
-
+               GEM_TRACE_DUMP();
                i915_gem_set_wedged(i915);
                return -EIO;
        }
@@ -4088,9 +4165,10 @@ out:
 }
 
 /*
- * Prepare buffer for display plane (scanout, cursors, etc).
- * Can be called from an uninterruptible phase (modesetting) and allows
- * any flushes to be pipelined (for pageflips).
+ * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
+ * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
+ * (for pageflips). We only flush the caches while preparing the buffer for
+ * display, the callers are responsible for frontbuffer flush.
  */
 struct i915_vma *
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
@@ -4146,9 +4224,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 
        vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 
-       /* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
        __i915_gem_object_flush_for_display(obj);
-       intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
 
        /* It should now be out of any other write domains, and we can update
         * the domain values for our changes.
@@ -4973,6 +5049,7 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
         * machines is a good idea, we don't - just in case it leaves the
         * machine in an unusable condition.
         */
+       intel_uc_sanitize(dev_priv);
        i915_gem_sanitize(dev_priv);
 
        intel_runtime_pm_put(dev_priv);
@@ -5118,6 +5195,8 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
                }
        }
 
+       intel_gt_workarounds_apply(dev_priv);
+
        i915_gem_init_swizzling(dev_priv);
 
        /*
@@ -5140,6 +5219,12 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
                goto out;
        }
 
+       ret = intel_wopcm_init_hw(&dev_priv->wopcm);
+       if (ret) {
+               DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
+               goto out;
+       }
+
        /* We can't enable contexts until all firmware is loaded */
        ret = intel_uc_init_hw(dev_priv);
        if (ret) {
@@ -5207,7 +5292,7 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
        for_each_engine(engine, i915, id) {
                struct i915_vma *state;
 
-               state = ctx->engine[id].state;
+               state = to_intel_context(ctx, engine)->state;
                if (!state)
                        continue;
 
@@ -5297,6 +5382,10 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
        if (ret)
                return ret;
 
+       ret = intel_wopcm_init(&dev_priv->wopcm);
+       if (ret)
+               return ret;
+
        ret = intel_uc_init_misc(dev_priv);
        if (ret)
                return ret;
@@ -5478,8 +5567,7 @@ static void i915_gem_init__mm(struct drm_i915_private *i915)
        INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
 }
 
-int
-i915_gem_load_init(struct drm_i915_private *dev_priv)
+int i915_gem_init_early(struct drm_i915_private *dev_priv)
 {
        int err = -ENOMEM;
 
@@ -5512,8 +5600,10 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
        if (!dev_priv->priorities)
                goto err_dependencies;
 
-       mutex_lock(&dev_priv->drm.struct_mutex);
        INIT_LIST_HEAD(&dev_priv->gt.timelines);
+       INIT_LIST_HEAD(&dev_priv->gt.active_rings);
+
+       mutex_lock(&dev_priv->drm.struct_mutex);
        err = i915_gem_timeline_init__global(dev_priv);
        mutex_unlock(&dev_priv->drm.struct_mutex);
        if (err)
@@ -5554,7 +5644,7 @@ err_out:
        return err;
 }
 
-void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
+void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
 {
        i915_gem_drain_freed_objects(dev_priv);
        GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
index f54c4ff..5259204 100644 (file)
 
 #include <linux/bug.h>
 
+struct drm_i915_private;
+
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
+
+#define GEM_SHOW_DEBUG() (drm_debug & DRM_UT_DRIVER)
+
 #define GEM_BUG_ON(condition) do { if (unlikely((condition))) {        \
                pr_err("%s:%d GEM_BUG_ON(%s)\n", \
                       __func__, __LINE__, __stringify(condition)); \
@@ -43,6 +48,9 @@
 #define GEM_DEBUG_BUG_ON(expr) GEM_BUG_ON(expr)
 
 #else
+
+#define GEM_SHOW_DEBUG() (0)
+
 #define GEM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
 #define GEM_WARN_ON(expr) (BUILD_BUG_ON_INVALID(expr), 0)
 
 
 #if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM)
 #define GEM_TRACE(...) trace_printk(__VA_ARGS__)
+#define GEM_TRACE_DUMP() ftrace_dump(DUMP_ALL)
 #else
 #define GEM_TRACE(...) do { } while (0)
+#define GEM_TRACE_DUMP() do { } while (0)
 #endif
 
 #define I915_NUM_ENGINES 8
 
+void i915_gem_park(struct drm_i915_private *i915);
+void i915_gem_unpark(struct drm_i915_private *i915);
+
 #endif /* __I915_GEM_H__ */
index d3cbe84..f3890b6 100644 (file)
@@ -1,29 +1,11 @@
 /*
- * Copyright © 2014 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2014-2018 Intel Corporation
  */
 
-#include "i915_drv.h"
 #include "i915_gem_batch_pool.h"
+#include "i915_drv.h"
 
 /**
  * DOC: batch pool
 
 /**
  * i915_gem_batch_pool_init() - initialize a batch buffer pool
- * @engine: the associated request submission engine
  * @pool: the batch buffer pool
+ * @engine: the associated request submission engine
  */
-void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
-                             struct i915_gem_batch_pool *pool)
+void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool,
+                             struct intel_engine_cs *engine)
 {
        int n;
 
index 10d5ac4..56947da 100644 (file)
@@ -1,31 +1,13 @@
 /*
- * Copyright © 2014 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2014-2018 Intel Corporation
  */
 
 #ifndef I915_GEM_BATCH_POOL_H
 #define I915_GEM_BATCH_POOL_H
 
-#include "i915_drv.h"
+#include <linux/types.h>
 
 struct intel_engine_cs;
 
@@ -34,9 +16,8 @@ struct i915_gem_batch_pool {
        struct list_head cache_list[4];
 };
 
-/* i915_gem_batch_pool.c */
-void i915_gem_batch_pool_init(struct intel_engine_cs *engine,
-                             struct i915_gem_batch_pool *pool);
+void i915_gem_batch_pool_init(struct i915_gem_batch_pool *pool,
+                             struct intel_engine_cs *engine);
 void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool);
 struct drm_i915_gem_object*
 i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size);
index f2cbea7..59d4bd4 100644 (file)
@@ -90,6 +90,7 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
+#include "intel_workarounds.h"
 
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
@@ -116,15 +117,15 @@ static void lut_close(struct i915_gem_context *ctx)
 
 static void i915_gem_context_free(struct i915_gem_context *ctx)
 {
-       int i;
+       unsigned int n;
 
        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
        GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
 
        i915_ppgtt_put(ctx->ppgtt);
 
-       for (i = 0; i < I915_NUM_ENGINES; i++) {
-               struct intel_context *ce = &ctx->engine[i];
+       for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) {
+               struct intel_context *ce = &ctx->__engine[n];
 
                if (!ce->state)
                        continue;
@@ -280,7 +281,7 @@ __create_hw_context(struct drm_i915_private *dev_priv,
        kref_init(&ctx->ref);
        list_add_tail(&ctx->link, &dev_priv->contexts.list);
        ctx->i915 = dev_priv;
-       ctx->priority = I915_PRIORITY_NORMAL;
+       ctx->sched.priority = I915_PRIORITY_NORMAL;
 
        INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
        INIT_LIST_HEAD(&ctx->handles_list);
@@ -318,12 +319,13 @@ __create_hw_context(struct drm_i915_private *dev_priv,
        ctx->desc_template =
                default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt);
 
-       /* GuC requires the ring to be placed above GUC_WOPCM_TOP. If GuC is not
+       /*
+        * GuC requires the ring to be placed in Non-WOPCM memory. If GuC is not
         * present or not in use we still need a small bias as ring wraparound
         * at offset 0 sometimes hangs. No idea why.
         */
        if (USES_GUC(dev_priv))
-               ctx->ggtt_offset_bias = GUC_WOPCM_TOP;
+               ctx->ggtt_offset_bias = dev_priv->guc.ggtt_pin_bias;
        else
                ctx->ggtt_offset_bias = I915_GTT_PAGE_SIZE;
 
@@ -429,7 +431,7 @@ i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio)
                return ctx;
 
        i915_gem_context_clear_bannable(ctx);
-       ctx->priority = prio;
+       ctx->sched.priority = prio;
        ctx->ring_size = PAGE_SIZE;
 
        GEM_BUG_ON(!i915_gem_context_is_kernel(ctx));
@@ -458,11 +460,16 @@ static bool needs_preempt_context(struct drm_i915_private *i915)
 int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 {
        struct i915_gem_context *ctx;
+       int ret;
 
        /* Reassure ourselves we are only called once */
        GEM_BUG_ON(dev_priv->kernel_context);
        GEM_BUG_ON(dev_priv->preempt_context);
 
+       ret = intel_ctx_workarounds_init(dev_priv);
+       if (ret)
+               return ret;
+
        INIT_LIST_HEAD(&dev_priv->contexts.list);
        INIT_WORK(&dev_priv->contexts.free_work, contexts_free_worker);
        init_llist_head(&dev_priv->contexts.free_list);
@@ -514,7 +521,7 @@ void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
                if (!engine->last_retired_context)
                        continue;
 
-               engine->context_unpin(engine, engine->last_retired_context);
+               intel_context_unpin(engine->last_retired_context, engine);
                engine->last_retired_context = NULL;
        }
 }
@@ -746,7 +753,7 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
                args->value = i915_gem_context_is_bannable(ctx);
                break;
        case I915_CONTEXT_PARAM_PRIORITY:
-               args->value = ctx->priority;
+               args->value = ctx->sched.priority;
                break;
        default:
                ret = -EINVAL;
@@ -819,7 +826,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
                                 !capable(CAP_SYS_NICE))
                                ret = -EPERM;
                        else
-                               ctx->priority = priority;
+                               ctx->sched.priority = priority;
                }
                break;
 
index 7854262..ace3b12 100644 (file)
@@ -137,18 +137,7 @@ struct i915_gem_context {
         */
        u32 user_handle;
 
-       /**
-        * @priority: execution and service priority
-        *
-        * All clients are equal, but some are more equal than others!
-        *
-        * Requests from a context with a greater (more positive) value of
-        * @priority will be executed before those with a lower @priority
-        * value, forming a simple QoS.
-        *
-        * The &drm_i915_private.kernel_context is assigned the lowest priority.
-        */
-       int priority;
+       struct i915_sched_attr sched;
 
        /** ggtt_offset_bias: placement restriction for context objects */
        u32 ggtt_offset_bias;
@@ -160,7 +149,7 @@ struct i915_gem_context {
                u32 *lrc_reg_state;
                u64 lrc_desc;
                int pin_count;
-       } engine[I915_NUM_ENGINES];
+       } __engine[I915_NUM_ENGINES];
 
        /** ring_size: size for allocating the per-engine ring buffer */
        u32 ring_size;
@@ -267,6 +256,34 @@ static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
        return !ctx->file_priv;
 }
 
+static inline struct intel_context *
+to_intel_context(struct i915_gem_context *ctx,
+                const struct intel_engine_cs *engine)
+{
+       return &ctx->__engine[engine->id];
+}
+
+static inline struct intel_ring *
+intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
+{
+       return engine->context_pin(engine, ctx);
+}
+
+static inline void __intel_context_pin(struct i915_gem_context *ctx,
+                                      const struct intel_engine_cs *engine)
+{
+       struct intel_context *ce = to_intel_context(ctx, engine);
+
+       GEM_BUG_ON(!ce->pin_count);
+       ce->pin_count++;
+}
+
+static inline void intel_context_unpin(struct i915_gem_context *ctx,
+                                      struct intel_engine_cs *engine)
+{
+       engine->context_unpin(engine, ctx);
+}
+
 /* i915_gem_context.c */
 int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
 void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
index 0414228..c74f5df 100644 (file)
@@ -81,6 +81,35 @@ enum {
  * but this remains just a hint as the kernel may choose a new location for
  * any object in the future.
  *
+ * At the level of talking to the hardware, submitting a batchbuffer for the
+ * GPU to execute is to add content to a buffer from which the HW
+ * command streamer is reading.
+ *
+ * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
+ *    Execlists, this command is not placed on the same buffer as the
+ *    remaining items.
+ *
+ * 2. Add a command to invalidate caches to the buffer.
+ *
+ * 3. Add a batchbuffer start command to the buffer; the start command is
+ *    essentially a token together with the GPU address of the batchbuffer
+ *    to be executed.
+ *
+ * 4. Add a pipeline flush to the buffer.
+ *
+ * 5. Add a memory write command to the buffer to record when the GPU
+ *    is done executing the batchbuffer. The memory write writes the
+ *    global sequence number of the request, ``i915_request::global_seqno``;
+ *    the i915 driver uses the current value in the register to determine
+ *    if the GPU has completed the batchbuffer.
+ *
+ * 6. Add a user interrupt command to the buffer. This command instructs
+ *    the GPU to issue an interrupt when the command, pipeline flush and
+ *    memory write are completed.
+ *
+ * 7. Inform the hardware of the additional commands added to the buffer
+ *    (by updating the tail pointer).
+ *
  * Processing an execbuf ioctl is conceptually split up into a few phases.
  *
  * 1. Validation - Ensure all the pointers, handles and flags are valid.
index 62aa679..ad949cc 100644 (file)
@@ -51,6 +51,10 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv,
        if (!drm_mm_initialized(&dev_priv->mm.stolen))
                return -ENODEV;
 
+       /* WaSkipStolenMemoryFirstPage:bdw+ */
+       if (INTEL_GEN(dev_priv) >= 8 && start < 4096)
+               start = 4096;
+
        mutex_lock(&dev_priv->mm.stolen_lock);
        ret = drm_mm_insert_node_in_range(&dev_priv->mm.stolen, node,
                                          size, alignment, 0,
@@ -121,8 +125,8 @@ static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
 
                if (stolen[0].start != stolen[1].start ||
                    stolen[0].end != stolen[1].end) {
-                       DRM_DEBUG_KMS("GTT within stolen memory at %pR\n", &ggtt_res);
-                       DRM_DEBUG_KMS("Stolen memory adjusted to %pR\n", dsm);
+                       DRM_DEBUG_DRIVER("GTT within stolen memory at %pR\n", &ggtt_res);
+                       DRM_DEBUG_DRIVER("Stolen memory adjusted to %pR\n", dsm);
                }
        }
 
@@ -174,18 +178,19 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 }
 
 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
-                                   resource_size_t *base, resource_size_t *size)
+                                   resource_size_t *base,
+                                   resource_size_t *size)
 {
-       uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ?
-                                    CTG_STOLEN_RESERVED :
-                                    ELK_STOLEN_RESERVED);
+       u32 reg_val = I915_READ(IS_GM45(dev_priv) ?
+                               CTG_STOLEN_RESERVED :
+                               ELK_STOLEN_RESERVED);
        resource_size_t stolen_top = dev_priv->dsm.end + 1;
 
-       if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) {
-               *base = 0;
-               *size = 0;
+       DRM_DEBUG_DRIVER("%s_STOLEN_RESERVED = %08x\n",
+                        IS_GM45(dev_priv) ? "CTG" : "ELK", reg_val);
+
+       if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0)
                return;
-       }
 
        /*
         * Whether ILK really reuses the ELK register for this is unclear.
@@ -193,30 +198,25 @@ static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
         */
        WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 
-       *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
+       if (!(reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK))
+               return;
 
+       *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
        WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
 
-       /* On these platforms, the register doesn't have a size field, so the
-        * size is the distance between the base and the top of the stolen
-        * memory. We also have the genuine case where base is zero and there's
-        * nothing reserved. */
-       if (*base == 0)
-               *size = 0;
-       else
-               *size = stolen_top - *base;
+       *size = stolen_top - *base;
 }
 
 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
-                                    resource_size_t *base, resource_size_t *size)
+                                    resource_size_t *base,
+                                    resource_size_t *size)
 {
-       uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+       u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+
+       DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 
-       if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-               *base = 0;
-               *size = 0;
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
                return;
-       }
 
        *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
@@ -239,17 +239,44 @@ static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
        }
 }
 
-static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
-                                    resource_size_t *base, resource_size_t *size)
+static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
+                                   resource_size_t *base,
+                                   resource_size_t *size)
 {
-       uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+       u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+       resource_size_t stolen_top = dev_priv->dsm.end + 1;
+
+       DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
 
-       if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-               *base = 0;
-               *size = 0;
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
                return;
+
+       switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
+       default:
+               MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
+       case GEN7_STOLEN_RESERVED_1M:
+               *size = 1024 * 1024;
+               break;
        }
 
+       /*
+        * On vlv, the ADDR_MASK portion is left as 0 and HW deduces the
+        * reserved location as (top - size).
+        */
+       *base = stolen_top - *size;
+}
+
+static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
+                                    resource_size_t *base,
+                                    resource_size_t *size)
+{
+       u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+
+       DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
+               return;
+
        *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 
        switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
@@ -266,15 +293,15 @@ static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
-                                   resource_size_t *base, resource_size_t *size)
+                                   resource_size_t *base,
+                                   resource_size_t *size)
 {
-       uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+       u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
-       if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-               *base = 0;
-               *size = 0;
+       DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
                return;
-       }
 
        *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
@@ -298,36 +325,28 @@ static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
-                                   resource_size_t *base, resource_size_t *size)
+                                   resource_size_t *base,
+                                   resource_size_t *size)
 {
-       uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
-       resource_size_t stolen_top;
+       u32 reg_val = I915_READ(GEN6_STOLEN_RESERVED);
+       resource_size_t stolen_top = dev_priv->dsm.end + 1;
 
-       if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
-               *base = 0;
-               *size = 0;
+       DRM_DEBUG_DRIVER("GEN6_STOLEN_RESERVED = %08x\n", reg_val);
+
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ENABLE))
                return;
-       }
 
-       stolen_top = dev_priv->dsm.end + 1;
+       if (!(reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK))
+               return;
 
        *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
-
-       /* On these platforms, the register doesn't have a size field, so the
-        * size is the distance between the base and the top of the stolen
-        * memory. We also have the genuine case where base is zero and there's
-        * nothing reserved. */
-       if (*base == 0)
-               *size = 0;
-       else
-               *size = stolen_top - *base;
+       *size = stolen_top - *base;
 }
 
 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 {
        resource_size_t reserved_base, stolen_top;
        resource_size_t reserved_total, reserved_size;
-       resource_size_t stolen_usable_start;
 
        mutex_init(&dev_priv->mm.stolen_lock);
 
@@ -353,7 +372,7 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
        GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
 
        stolen_top = dev_priv->dsm.end + 1;
-       reserved_base = 0;
+       reserved_base = stolen_top;
        reserved_size = 0;
 
        switch (INTEL_GEN(dev_priv)) {
@@ -373,8 +392,12 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
                                         &reserved_base, &reserved_size);
                break;
        case 7:
-               gen7_get_stolen_reserved(dev_priv,
-                                        &reserved_base, &reserved_size);
+               if (IS_VALLEYVIEW(dev_priv))
+                       vlv_get_stolen_reserved(dev_priv,
+                                               &reserved_base, &reserved_size);
+               else
+                       gen7_get_stolen_reserved(dev_priv,
+                                                &reserved_base, &reserved_size);
                break;
        default:
                if (IS_LP(dev_priv))
@@ -386,11 +409,16 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
                break;
        }
 
-       /* It is possible for the reserved base to be zero, but the register
-        * field for size doesn't have a zero option. */
-       if (reserved_base == 0) {
-               reserved_size = 0;
+       /*
+        * Our expectation is that the reserved space is at the top of the
+        * stolen region and *never* at the bottom. If we see !reserved_base,
+        * it likely means we failed to read the registers correctly.
+        */
+       if (!reserved_base) {
+               DRM_ERROR("inconsistent reservation %pa + %pa; ignoring\n",
+                         &reserved_base, &reserved_size);
                reserved_base = stolen_top;
+               reserved_size = 0;
        }
 
        dev_priv->dsm_reserved =
@@ -406,21 +434,15 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
         * memory, so just consider the start. */
        reserved_total = stolen_top - reserved_base;
 
-       DRM_DEBUG_KMS("Memory reserved for graphics device: %lluK, usable: %lluK\n",
-                     (u64)resource_size(&dev_priv->dsm) >> 10,
-                     ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
-
-       stolen_usable_start = 0;
-       /* WaSkipStolenMemoryFirstPage:bdw+ */
-       if (INTEL_GEN(dev_priv) >= 8)
-               stolen_usable_start = 4096;
+       DRM_DEBUG_DRIVER("Memory reserved for graphics device: %lluK, usable: %lluK\n",
+                        (u64)resource_size(&dev_priv->dsm) >> 10,
+                        ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 
        dev_priv->stolen_usable_size =
-               resource_size(&dev_priv->dsm) - reserved_total - stolen_usable_start;
+               resource_size(&dev_priv->dsm) - reserved_total;
 
        /* Basic memrange allocator for stolen space. */
-       drm_mm_init(&dev_priv->mm.stolen, stolen_usable_start,
-                   dev_priv->stolen_usable_size);
+       drm_mm_init(&dev_priv->mm.stolen, 0, dev_priv->stolen_usable_size);
 
        return 0;
 }
@@ -580,8 +602,8 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
 
        lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-       DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
-                       &stolen_offset, &gtt_offset, &size);
+       DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
+                        &stolen_offset, &gtt_offset, &size);
 
        /* KISS and expect everything to be page-aligned */
        if (WARN_ON(size == 0) ||
@@ -599,14 +621,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
        ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen);
        mutex_unlock(&dev_priv->mm.stolen_lock);
        if (ret) {
-               DRM_DEBUG_KMS("failed to allocate stolen space\n");
+               DRM_DEBUG_DRIVER("failed to allocate stolen space\n");
                kfree(stolen);
                return NULL;
        }
 
        obj = _i915_gem_object_create_stolen(dev_priv, stolen);
        if (obj == NULL) {
-               DRM_DEBUG_KMS("failed to allocate stolen object\n");
+               DRM_DEBUG_DRIVER("failed to allocate stolen object\n");
                i915_gem_stolen_remove_node(dev_priv, stolen);
                kfree(stolen);
                return NULL;
@@ -635,7 +657,7 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
                                   size, gtt_offset, obj->cache_level,
                                   0);
        if (ret) {
-               DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
+               DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n");
                goto err_pages;
        }
 
index 33e01bf..6e82119 100644 (file)
@@ -37,12 +37,6 @@ struct intel_timeline {
        u64 fence_context;
        u32 seqno;
 
-       /**
-        * Count of outstanding requests, from the time they are constructed
-        * to the moment they are retired. Loosely coupled to hardware.
-        */
-       u32 inflight_seqnos;
-
        spinlock_t lock;
 
        /**
index f89ac7a..1176d06 100644 (file)
@@ -32,6 +32,7 @@
 #include <linux/zlib.h>
 #include <drm/drm_print.h>
 
+#include "i915_gpu_error.h"
 #include "i915_drv.h"
 
 static inline const struct intel_engine_cs *
@@ -403,15 +404,16 @@ static const char *bannable(const struct drm_i915_error_context *ctx)
 
 static void error_print_request(struct drm_i915_error_state_buf *m,
                                const char *prefix,
-                               const struct drm_i915_error_request *erq)
+                               const struct drm_i915_error_request *erq,
+                               const unsigned long epoch)
 {
        if (!erq->seqno)
                return;
 
-       err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms ago, head %08x, tail %08x\n",
+       err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms, head %08x, tail %08x\n",
                   prefix, erq->pid, erq->ban_score,
-                  erq->context, erq->seqno, erq->priority,
-                  jiffies_to_msecs(jiffies - erq->jiffies),
+                  erq->context, erq->seqno, erq->sched_attr.priority,
+                  jiffies_to_msecs(erq->jiffies - epoch),
                   erq->head, erq->tail);
 }
 
@@ -421,12 +423,13 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
 {
        err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, ban score %d%s guilty %d active %d\n",
                   header, ctx->comm, ctx->pid, ctx->handle, ctx->hw_id,
-                  ctx->priority, ctx->ban_score, bannable(ctx),
+                  ctx->sched_attr.priority, ctx->ban_score, bannable(ctx),
                   ctx->guilty, ctx->active);
 }
 
 static void error_print_engine(struct drm_i915_error_state_buf *m,
-                              const struct drm_i915_error_engine *ee)
+                              const struct drm_i915_error_engine *ee,
+                              const unsigned long epoch)
 {
        int n;
 
@@ -496,14 +499,15 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
        err_printf(m, "  hangcheck stall: %s\n", yesno(ee->hangcheck_stalled));
        err_printf(m, "  hangcheck action: %s\n",
                   hangcheck_action_to_str(ee->hangcheck_action));
-       err_printf(m, "  hangcheck action timestamp: %lu, %u ms ago\n",
+       err_printf(m, "  hangcheck action timestamp: %dms (%lu%s)\n",
+                  jiffies_to_msecs(ee->hangcheck_timestamp - epoch),
                   ee->hangcheck_timestamp,
-                  jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));
+                  ee->hangcheck_timestamp == epoch ? "; epoch" : "");
        err_printf(m, "  engine reset count: %u\n", ee->reset_count);
 
        for (n = 0; n < ee->num_ports; n++) {
                err_printf(m, "  ELSP[%d]:", n);
-               error_print_request(m, " ", &ee->execlist[n]);
+               error_print_request(m, " ", &ee->execlist[n], epoch);
        }
 
        error_print_context(m, "  Active context: ", &ee->context);
@@ -649,6 +653,11 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
        ts = ktime_to_timespec64(error->uptime);
        err_printf(m, "Uptime: %lld s %ld us\n",
                   (s64)ts.tv_sec, ts.tv_nsec / NSEC_PER_USEC);
+       err_printf(m, "Epoch: %lu jiffies (%u HZ)\n", error->epoch, HZ);
+       err_printf(m, "Capture: %lu jiffies; %d ms ago, %d ms after epoch\n",
+                  error->capture,
+                  jiffies_to_msecs(jiffies - error->capture),
+                  jiffies_to_msecs(error->capture - error->epoch));
 
        for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
                if (error->engine[i].hangcheck_stalled &&
@@ -709,7 +718,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
        for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
                if (error->engine[i].engine_id != -1)
-                       error_print_engine(m, &error->engine[i]);
+                       error_print_engine(m, &error->engine[i], error->epoch);
        }
 
        for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) {
@@ -768,7 +777,9 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                                   dev_priv->engine[i]->name,
                                   ee->num_requests);
                        for (j = 0; j < ee->num_requests; j++)
-                               error_print_request(m, " ", &ee->requests[j]);
+                               error_print_request(m, " ",
+                                                   &ee->requests[j],
+                                                   error->epoch);
                }
 
                if (IS_ERR(ee->waiters)) {
@@ -1277,7 +1288,7 @@ static void record_request(struct i915_request *request,
                           struct drm_i915_error_request *erq)
 {
        erq->context = request->ctx->hw_id;
-       erq->priority = request->priotree.priority;
+       erq->sched_attr = request->sched.attr;
        erq->ban_score = atomic_read(&request->ctx->ban_score);
        erq->seqno = request->global_seqno;
        erq->jiffies = request->emitted_jiffies;
@@ -1371,7 +1382,7 @@ static void record_context(struct drm_i915_error_context *e,
 
        e->handle = ctx->user_handle;
        e->hw_id = ctx->hw_id;
-       e->priority = ctx->priority;
+       e->sched_attr = ctx->sched;
        e->ban_score = atomic_read(&ctx->ban_score);
        e->bannable = i915_gem_context_is_bannable(ctx);
        e->guilty = atomic_read(&ctx->guilty_count);
@@ -1471,7 +1482,8 @@ static void gem_record_rings(struct i915_gpu_state *error)
 
                        ee->ctx =
                                i915_error_object_create(i915,
-                                                        request->ctx->engine[i].state);
+                                                        to_intel_context(request->ctx,
+                                                                         engine)->state);
 
                        error->simulated |=
                                i915_gem_context_no_error_capture(request->ctx);
@@ -1734,6 +1746,22 @@ static void capture_params(struct i915_gpu_state *error)
 #undef DUP
 }
 
+static unsigned long capture_find_epoch(const struct i915_gpu_state *error)
+{
+       unsigned long epoch = error->capture;
+       int i;
+
+       for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
+               const struct drm_i915_error_engine *ee = &error->engine[i];
+
+               if (ee->hangcheck_stalled &&
+                   time_before(ee->hangcheck_timestamp, epoch))
+                       epoch = ee->hangcheck_timestamp;
+       }
+
+       return epoch;
+}
+
 static int capture(void *data)
 {
        struct i915_gpu_state *error = data;
@@ -1742,6 +1770,7 @@ static int capture(void *data)
        error->boottime = ktime_get_boottime();
        error->uptime = ktime_sub(ktime_get(),
                                  error->i915->gt.last_init_time);
+       error->capture = jiffies;
 
        capture_params(error);
        capture_gen_state(error);
@@ -1755,6 +1784,8 @@ static int capture(void *data)
        error->overlay = intel_overlay_capture_error_state(error->i915);
        error->display = intel_display_capture_error_state(error->i915);
 
+       error->epoch = capture_find_epoch(error);
+
        return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
new file mode 100644 (file)
index 0000000..0accd2e
--- /dev/null
@@ -0,0 +1,365 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright � 2008-2018 Intel Corporation
+ */
+
+#ifndef _I915_GPU_ERROR_H_
+#define _I915_GPU_ERROR_H_
+
+#include <linux/kref.h>
+#include <linux/ktime.h>
+#include <linux/sched.h>
+
+#include <drm/drm_mm.h>
+
+#include "intel_device_info.h"
+#include "intel_ringbuffer.h"
+#include "intel_uc_fw.h"
+
+#include "i915_gem.h"
+#include "i915_gem_gtt.h"
+#include "i915_params.h"
+#include "i915_scheduler.h"
+
+struct drm_i915_private;
+struct intel_overlay_error_state;
+struct intel_display_error_state;
+
+struct i915_gpu_state {
+       struct kref ref;
+       ktime_t time;
+       ktime_t boottime;
+       ktime_t uptime;
+       unsigned long capture;
+       unsigned long epoch;
+
+       struct drm_i915_private *i915;
+
+       char error_msg[128];
+       bool simulated;
+       bool awake;
+       bool wakelock;
+       bool suspended;
+       int iommu;
+       u32 reset_count;
+       u32 suspend_count;
+       struct intel_device_info device_info;
+       struct intel_driver_caps driver_caps;
+       struct i915_params params;
+
+       struct i915_error_uc {
+               struct intel_uc_fw guc_fw;
+               struct intel_uc_fw huc_fw;
+               struct drm_i915_error_object *guc_log;
+       } uc;
+
+       /* Generic register state */
+       u32 eir;
+       u32 pgtbl_er;
+       u32 ier;
+       u32 gtier[4], ngtier;
+       u32 ccid;
+       u32 derrmr;
+       u32 forcewake;
+       u32 error; /* gen6+ */
+       u32 err_int; /* gen7 */
+       u32 fault_data0; /* gen8, gen9 */
+       u32 fault_data1; /* gen8, gen9 */
+       u32 done_reg;
+       u32 gac_eco;
+       u32 gam_ecochk;
+       u32 gab_ctl;
+       u32 gfx_mode;
+
+       u32 nfence;
+       u64 fence[I915_MAX_NUM_FENCES];
+       struct intel_overlay_error_state *overlay;
+       struct intel_display_error_state *display;
+
+       struct drm_i915_error_engine {
+               int engine_id;
+               /* Software tracked state */
+               bool idle;
+               bool waiting;
+               int num_waiters;
+               unsigned long hangcheck_timestamp;
+               bool hangcheck_stalled;
+               enum intel_engine_hangcheck_action hangcheck_action;
+               struct i915_address_space *vm;
+               int num_requests;
+               u32 reset_count;
+
+               /* position of active request inside the ring */
+               u32 rq_head, rq_post, rq_tail;
+
+               /* our own tracking of ring head and tail */
+               u32 cpu_ring_head;
+               u32 cpu_ring_tail;
+
+               u32 last_seqno;
+
+               /* Register state */
+               u32 start;
+               u32 tail;
+               u32 head;
+               u32 ctl;
+               u32 mode;
+               u32 hws;
+               u32 ipeir;
+               u32 ipehr;
+               u32 bbstate;
+               u32 instpm;
+               u32 instps;
+               u32 seqno;
+               u64 bbaddr;
+               u64 acthd;
+               u32 fault_reg;
+               u64 faddr;
+               u32 rc_psmi; /* sleep state */
+               u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
+               struct intel_instdone instdone;
+
+               struct drm_i915_error_context {
+                       char comm[TASK_COMM_LEN];
+                       pid_t pid;
+                       u32 handle;
+                       u32 hw_id;
+                       int ban_score;
+                       int active;
+                       int guilty;
+                       bool bannable;
+                       struct i915_sched_attr sched_attr;
+               } context;
+
+               struct drm_i915_error_object {
+                       u64 gtt_offset;
+                       u64 gtt_size;
+                       int page_count;
+                       int unused;
+                       u32 *pages[0];
+               } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
+
+               struct drm_i915_error_object **user_bo;
+               long user_bo_count;
+
+               struct drm_i915_error_object *wa_ctx;
+               struct drm_i915_error_object *default_state;
+
+               struct drm_i915_error_request {
+                       long jiffies;
+                       pid_t pid;
+                       u32 context;
+                       int ban_score;
+                       u32 seqno;
+                       u32 head;
+                       u32 tail;
+                       struct i915_sched_attr sched_attr;
+               } *requests, execlist[EXECLIST_MAX_PORTS];
+               unsigned int num_ports;
+
+               struct drm_i915_error_waiter {
+                       char comm[TASK_COMM_LEN];
+                       pid_t pid;
+                       u32 seqno;
+               } *waiters;
+
+               struct {
+                       u32 gfx_mode;
+                       union {
+                               u64 pdp[4];
+                               u32 pp_dir_base;
+                       };
+               } vm_info;
+       } engine[I915_NUM_ENGINES];
+
+       struct drm_i915_error_buffer {
+               u32 size;
+               u32 name;
+               u32 rseqno[I915_NUM_ENGINES], wseqno;
+               u64 gtt_offset;
+               u32 read_domains;
+               u32 write_domain;
+               s32 fence_reg:I915_MAX_NUM_FENCE_BITS;
+               u32 tiling:2;
+               u32 dirty:1;
+               u32 purgeable:1;
+               u32 userptr:1;
+               s32 engine:4;
+               u32 cache_level:3;
+       } *active_bo[I915_NUM_ENGINES], *pinned_bo;
+       u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
+       struct i915_address_space *active_vm[I915_NUM_ENGINES];
+};
+
+struct i915_gpu_error {
+       /* For hangcheck timer */
+#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */
+#define DRM_I915_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)
+
+       struct delayed_work hangcheck_work;
+
+       /* For reset and error_state handling. */
+       spinlock_t lock;
+       /* Protected by the above dev->gpu_error.lock. */
+       struct i915_gpu_state *first_error;
+
+       atomic_t pending_fb_pin;
+
+       unsigned long missed_irq_rings;
+
+       /**
+        * State variable controlling the reset flow and count
+        *
+        * This is a counter which gets incremented when reset is triggered,
+        *
+        * Before the reset commences, the I915_RESET_BACKOFF bit is set
+        * meaning that any waiters holding onto the struct_mutex should
+        * relinquish the lock immediately in order for the reset to start.
+        *
+        * If reset is not completed successfully, the I915_WEDGE bit is
+        * set meaning that hardware is terminally sour and there is no
+        * recovery. All waiters on the reset_queue will be woken when
+        * that happens.
+        *
+        * This counter is used by the wait_seqno code to notice that reset
+        * event happened and it needs to restart the entire ioctl (since most
+        * likely the seqno it waited for won't ever signal anytime soon).
+        *
+        * This is important for lock-free wait paths, where no contended lock
+        * naturally enforces the correct ordering between the bail-out of the
+        * waiter and the gpu reset work code.
+        */
+       unsigned long reset_count;
+
+       /**
+        * flags: Control various stages of the GPU reset
+        *
+        * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
+        * other users acquiring the struct_mutex. To do this we set the
+        * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
+        * and then check for that bit before acquiring the struct_mutex (in
+        * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
+        * secondary role in preventing two concurrent global reset attempts.
+        *
+        * #I915_RESET_HANDOFF - To perform the actual GPU reset, we need the
+        * struct_mutex. We try to acquire the struct_mutex in the reset worker,
+        * but it may be held by some long running waiter (that we cannot
+        * interrupt without causing trouble). Once we are ready to do the GPU
+        * reset, we set the I915_RESET_HANDOFF bit and wakeup any waiters. If
+        * they already hold the struct_mutex and want to participate they can
+        * inspect the bit and do the reset directly, otherwise the worker
+        * waits for the struct_mutex.
+        *
+        * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
+        * acquire the struct_mutex to reset an engine, we need an explicit
+        * flag to prevent two concurrent reset attempts in the same engine.
+        * As the number of engines continues to grow, allocate the flags from
+        * the most significant bits.
+        *
+        * #I915_WEDGED - If reset fails and we can no longer use the GPU,
+        * we set the #I915_WEDGED bit. Prior to command submission, e.g.
+        * i915_request_alloc(), this bit is checked and the sequence
+        * aborted (with -EIO reported to userspace) if set.
+        */
+       unsigned long flags;
+#define I915_RESET_BACKOFF     0
+#define I915_RESET_HANDOFF     1
+#define I915_RESET_MODESET     2
+#define I915_WEDGED            (BITS_PER_LONG - 1)
+#define I915_RESET_ENGINE      (I915_WEDGED - I915_NUM_ENGINES)
+
+       /** Number of times an engine has been reset */
+       u32 reset_engine_count[I915_NUM_ENGINES];
+
+       /** Set of stalled engines with guilty requests, in the current reset */
+       u32 stalled_mask;
+
+       /** Reason for the current *global* reset */
+       const char *reason;
+
+       /**
+        * Waitqueue to signal when a hang is detected. Used to for waiters
+        * to release the struct_mutex for the reset to procede.
+        */
+       wait_queue_head_t wait_queue;
+
+       /**
+        * Waitqueue to signal when the reset has completed. Used by clients
+        * that wait for dev_priv->mm.wedged to settle.
+        */
+       wait_queue_head_t reset_queue;
+
+       /* For missed irq/seqno simulation. */
+       unsigned long test_irq_rings;
+};
+
+struct drm_i915_error_state_buf {
+       struct drm_i915_private *i915;
+       unsigned int bytes;
+       unsigned int size;
+       int err;
+       u8 *buf;
+       loff_t start;
+       loff_t pos;
+};
+
+#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
+
+__printf(2, 3)
+void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
+int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
+                           const struct i915_gpu_state *gpu);
+int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
+                             struct drm_i915_private *i915,
+                             size_t count, loff_t pos);
+
+static inline void
+i915_error_state_buf_release(struct drm_i915_error_state_buf *eb)
+{
+       kfree(eb->buf);
+}
+
+struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
+void i915_capture_error_state(struct drm_i915_private *dev_priv,
+                             u32 engine_mask,
+                             const char *error_msg);
+
+static inline struct i915_gpu_state *
+i915_gpu_state_get(struct i915_gpu_state *gpu)
+{
+       kref_get(&gpu->ref);
+       return gpu;
+}
+
+void __i915_gpu_state_free(struct kref *kref);
+static inline void i915_gpu_state_put(struct i915_gpu_state *gpu)
+{
+       if (gpu)
+               kref_put(&gpu->ref, __i915_gpu_state_free);
+}
+
+struct i915_gpu_state *i915_first_error_state(struct drm_i915_private *i915);
+void i915_reset_error_state(struct drm_i915_private *i915);
+
+#else
+
+static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
+                                           u32 engine_mask,
+                                           const char *error_msg)
+{
+}
+
+static inline struct i915_gpu_state *
+i915_first_error_state(struct drm_i915_private *i915)
+{
+       return NULL;
+}
+
+static inline void i915_reset_error_state(struct drm_i915_private *i915)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) */
+
+#endif /* _I915_GPU_ERROR_H_ */
index 633c187..f9bc3aa 100644 (file)
@@ -243,6 +243,41 @@ void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
        spin_unlock_irq(&dev_priv->irq_lock);
 }
 
+static u32
+gen11_gt_engine_identity(struct drm_i915_private * const i915,
+                        const unsigned int bank, const unsigned int bit);
+
+bool gen11_reset_one_iir(struct drm_i915_private * const i915,
+                        const unsigned int bank,
+                        const unsigned int bit)
+{
+       void __iomem * const regs = i915->regs;
+       u32 dw;
+
+       lockdep_assert_held(&i915->irq_lock);
+
+       dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
+       if (dw & BIT(bit)) {
+               /*
+                * According to the BSpec, DW_IIR bits cannot be cleared without
+                * first servicing the Selector & Shared IIR registers.
+                */
+               gen11_gt_engine_identity(i915, bank, bit);
+
+               /*
+                * We locked GT INT DW by reading it. If we want to (try
+                * to) recover from this succesfully, we need to clear
+                * our bit, otherwise we are locking the register for
+                * everybody.
+                */
+               raw_reg_write(regs, GEN11_GT_INTR_DW(bank), BIT(bit));
+
+               return true;
+       }
+
+       return false;
+}
+
 /**
  * ilk_update_display_irq - update DEIMR
  * @dev_priv: driver private
@@ -308,17 +343,29 @@ void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 
 static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 {
+       WARN_ON_ONCE(INTEL_GEN(dev_priv) >= 11);
+
        return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 }
 
 static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
 {
-       return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
+       if (INTEL_GEN(dev_priv) >= 11)
+               return GEN11_GPM_WGBOXPERF_INTR_MASK;
+       else if (INTEL_GEN(dev_priv) >= 8)
+               return GEN8_GT_IMR(2);
+       else
+               return GEN6_PMIMR;
 }
 
 static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
 {
-       return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
+       if (INTEL_GEN(dev_priv) >= 11)
+               return GEN11_GPM_WGBOXPERF_INTR_ENABLE;
+       else if (INTEL_GEN(dev_priv) >= 8)
+               return GEN8_GT_IER(2);
+       else
+               return GEN6_PMIER;
 }
 
 /**
@@ -400,6 +447,18 @@ static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_m
        /* though a barrier is missing here, but don't really need a one */
 }
 
+void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv)
+{
+       spin_lock_irq(&dev_priv->irq_lock);
+
+       while (gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM))
+               ;
+
+       dev_priv->gt_pm.rps.pm_iir = 0;
+
+       spin_unlock_irq(&dev_priv->irq_lock);
+}
+
 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
 {
        spin_lock_irq(&dev_priv->irq_lock);
@@ -415,12 +474,14 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
        if (READ_ONCE(rps->interrupts_enabled))
                return;
 
-       if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
-               return;
-
        spin_lock_irq(&dev_priv->irq_lock);
        WARN_ON_ONCE(rps->pm_iir);
-       WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
+
+       if (INTEL_GEN(dev_priv) >= 11)
+               WARN_ON_ONCE(gen11_reset_one_iir(dev_priv, 0, GEN11_GTPM));
+       else
+               WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
+
        rps->interrupts_enabled = true;
        gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
 
@@ -434,9 +495,6 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
        if (!READ_ONCE(rps->interrupts_enabled))
                return;
 
-       if (WARN_ON_ONCE(IS_GEN11(dev_priv)))
-               return;
-
        spin_lock_irq(&dev_priv->irq_lock);
        rps->interrupts_enabled = false;
 
@@ -453,7 +511,10 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
         * state of the worker can be discarded.
         */
        cancel_work_sync(&rps->work);
-       gen6_reset_rps_interrupts(dev_priv);
+       if (INTEL_GEN(dev_priv) >= 11)
+               gen11_reset_rps_interrupts(dev_priv);
+       else
+               gen6_reset_rps_interrupts(dev_priv);
 }
 
 void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
@@ -1399,19 +1460,18 @@ static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 }
 
 static void
-gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
+gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 {
        struct intel_engine_execlists * const execlists = &engine->execlists;
        bool tasklet = false;
 
-       if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
-               if (READ_ONCE(engine->execlists.active)) {
-                       __set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-                       tasklet = true;
-               }
+       if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
+               if (READ_ONCE(engine->execlists.active))
+                       tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
+                                                   &engine->irq_posted);
        }
 
-       if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
+       if (iir & GT_RENDER_USER_INTERRUPT) {
                notify_ring(engine);
                tasklet |= USES_GUC_SUBMISSION(engine->i915);
        }
@@ -1466,21 +1526,21 @@ static void gen8_gt_irq_handler(struct drm_i915_private *i915,
 {
        if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
                gen8_cs_irq_handler(i915->engine[RCS],
-                                   gt_iir[0], GEN8_RCS_IRQ_SHIFT);
+                                   gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
                gen8_cs_irq_handler(i915->engine[BCS],
-                                   gt_iir[0], GEN8_BCS_IRQ_SHIFT);
+                                   gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
        }
 
        if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
                gen8_cs_irq_handler(i915->engine[VCS],
-                                   gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
+                                   gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
                gen8_cs_irq_handler(i915->engine[VCS2],
-                                   gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
+                                   gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
        }
 
        if (master_ctl & GEN8_GT_VECS_IRQ) {
                gen8_cs_irq_handler(i915->engine[VECS],
-                                   gt_iir[3], GEN8_VECS_IRQ_SHIFT);
+                                   gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
        }
 
        if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
@@ -1627,7 +1687,7 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
        int head, tail;
 
        spin_lock(&pipe_crc->lock);
-       if (pipe_crc->source) {
+       if (pipe_crc->source && !crtc->base.crc.opened) {
                if (!pipe_crc->entries) {
                        spin_unlock(&pipe_crc->lock);
                        DRM_DEBUG_KMS("spurious interrupt\n");
@@ -1667,7 +1727,7 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
                 * On GEN8+ sometimes the second CRC is bonkers as well, so
                 * don't trust that one either.
                 */
-               if (pipe_crc->skipped == 0 ||
+               if (pipe_crc->skipped <= 0 ||
                    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
                        pipe_crc->skipped++;
                        spin_unlock(&pipe_crc->lock);
@@ -1766,37 +1826,8 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 
 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
 {
-       if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
-               /* Sample the log buffer flush related bits & clear them out now
-                * itself from the message identity register to minimize the
-                * probability of losing a flush interrupt, when there are back
-                * to back flush interrupts.
-                * There can be a new flush interrupt, for different log buffer
-                * type (like for ISR), whilst Host is handling one (for DPC).
-                * Since same bit is used in message register for ISR & DPC, it
-                * could happen that GuC sets the bit for 2nd interrupt but Host
-                * clears out the bit on handling the 1st interrupt.
-                */
-               u32 msg, flush;
-
-               msg = I915_READ(SOFT_SCRATCH(15));
-               flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
-                              INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
-               if (flush) {
-                       /* Clear the message bits that are handled */
-                       I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);
-
-                       /* Handle flush interrupt in bottom half */
-                       queue_work(dev_priv->guc.log.runtime.flush_wq,
-                                  &dev_priv->guc.log.runtime.flush_work);
-
-                       dev_priv->guc.log.flush_interrupt_count++;
-               } else {
-                       /* Not clearing of unhandled event bits won't result in
-                        * re-triggering of the interrupt.
-                        */
-               }
-       }
+       if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT)
+               intel_guc_to_host_event_handler(&dev_priv->guc);
 }
 
 static void i9xx_pipestat_irq_reset(struct drm_i915_private *dev_priv)
@@ -2433,6 +2464,13 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
        if (de_iir & DE_ERR_INT_IVB)
                ivb_err_int_handler(dev_priv);
 
+       if (de_iir & DE_EDP_PSR_INT_HSW) {
+               u32 psr_iir = I915_READ(EDP_PSR_IIR);
+
+               intel_psr_irq_handler(dev_priv, psr_iir);
+               I915_WRITE(EDP_PSR_IIR, psr_iir);
+       }
+
        if (de_iir & DE_AUX_CHANNEL_A_IVB)
                dp_aux_irq_handler(dev_priv);
 
@@ -2562,11 +2600,25 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
        if (master_ctl & GEN8_DE_MISC_IRQ) {
                iir = I915_READ(GEN8_DE_MISC_IIR);
                if (iir) {
+                       bool found = false;
+
                        I915_WRITE(GEN8_DE_MISC_IIR, iir);
                        ret = IRQ_HANDLED;
-                       if (iir & GEN8_DE_MISC_GSE)
+
+                       if (iir & GEN8_DE_MISC_GSE) {
                                intel_opregion_asle_intr(dev_priv);
-                       else
+                               found = true;
+                       }
+
+                       if (iir & GEN8_DE_EDP_PSR) {
+                               u32 psr_iir = I915_READ(EDP_PSR_IIR);
+
+                               intel_psr_irq_handler(dev_priv, psr_iir);
+                               I915_WRITE(EDP_PSR_IIR, psr_iir);
+                               found = true;
+                       }
+
+                       if (!found)
                                DRM_ERROR("Unexpected DE Misc interrupt\n");
                }
                else
@@ -2762,58 +2814,16 @@ static void __fini_wedge(struct wedge_me *w)
             (W)->i915;                                                 \
             __fini_wedge((W)))
 
-static __always_inline void
-gen11_cs_irq_handler(struct intel_engine_cs * const engine, const u32 iir)
-{
-       gen8_cs_irq_handler(engine, iir, 0);
-}
-
-static void
-gen11_gt_engine_irq_handler(struct drm_i915_private * const i915,
-                           const unsigned int bank,
-                           const unsigned int engine_n,
-                           const u16 iir)
-{
-       struct intel_engine_cs ** const engine = i915->engine;
-
-       switch (bank) {
-       case 0:
-               switch (engine_n) {
-
-               case GEN11_RCS0:
-                       return gen11_cs_irq_handler(engine[RCS], iir);
-
-               case GEN11_BCS:
-                       return gen11_cs_irq_handler(engine[BCS], iir);
-               }
-       case 1:
-               switch (engine_n) {
-
-               case GEN11_VCS(0):
-                       return gen11_cs_irq_handler(engine[_VCS(0)], iir);
-               case GEN11_VCS(1):
-                       return gen11_cs_irq_handler(engine[_VCS(1)], iir);
-               case GEN11_VCS(2):
-                       return gen11_cs_irq_handler(engine[_VCS(2)], iir);
-               case GEN11_VCS(3):
-                       return gen11_cs_irq_handler(engine[_VCS(3)], iir);
-
-               case GEN11_VECS(0):
-                       return gen11_cs_irq_handler(engine[_VECS(0)], iir);
-               case GEN11_VECS(1):
-                       return gen11_cs_irq_handler(engine[_VECS(1)], iir);
-               }
-       }
-}
-
 static u32
-gen11_gt_engine_intr(struct drm_i915_private * const i915,
-                    const unsigned int bank, const unsigned int bit)
+gen11_gt_engine_identity(struct drm_i915_private * const i915,
+                        const unsigned int bank, const unsigned int bit)
 {
        void __iomem * const regs = i915->regs;
        u32 timeout_ts;
        u32 ident;
 
+       lockdep_assert_held(&i915->irq_lock);
+
        raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
 
        /*
@@ -2835,42 +2845,101 @@ gen11_gt_engine_intr(struct drm_i915_private * const i915,
        raw_reg_write(regs, GEN11_INTR_IDENTITY_REG(bank),
                      GEN11_INTR_DATA_VALID);
 
-       return ident & GEN11_INTR_ENGINE_MASK;
+       return ident;
 }
 
 static void
-gen11_gt_irq_handler(struct drm_i915_private * const i915,
-                    const u32 master_ctl)
+gen11_other_irq_handler(struct drm_i915_private * const i915,
+                       const u8 instance, const u16 iir)
+{
+       if (instance == OTHER_GTPM_INSTANCE)
+               return gen6_rps_irq_handler(i915, iir);
+
+       WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
+                 instance, iir);
+}
+
+static void
+gen11_engine_irq_handler(struct drm_i915_private * const i915,
+                        const u8 class, const u8 instance, const u16 iir)
+{
+       struct intel_engine_cs *engine;
+
+       if (instance <= MAX_ENGINE_INSTANCE)
+               engine = i915->engine_class[class][instance];
+       else
+               engine = NULL;
+
+       if (likely(engine))
+               return gen8_cs_irq_handler(engine, iir);
+
+       WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
+                 class, instance);
+}
+
+static void
+gen11_gt_identity_handler(struct drm_i915_private * const i915,
+                         const u32 identity)
+{
+       const u8 class = GEN11_INTR_ENGINE_CLASS(identity);
+       const u8 instance = GEN11_INTR_ENGINE_INSTANCE(identity);
+       const u16 intr = GEN11_INTR_ENGINE_INTR(identity);
+
+       if (unlikely(!intr))
+               return;
+
+       if (class <= COPY_ENGINE_CLASS)
+               return gen11_engine_irq_handler(i915, class, instance, intr);
+
+       if (class == OTHER_CLASS)
+               return gen11_other_irq_handler(i915, instance, intr);
+
+       WARN_ONCE(1, "unknown interrupt class=0x%x, instance=0x%x, intr=0x%x\n",
+                 class, instance, intr);
+}
+
+static void
+gen11_gt_bank_handler(struct drm_i915_private * const i915,
+                     const unsigned int bank)
 {
        void __iomem * const regs = i915->regs;
-       unsigned int bank;
+       unsigned long intr_dw;
+       unsigned int bit;
 
-       for (bank = 0; bank < 2; bank++) {
-               unsigned long intr_dw;
-               unsigned int bit;
+       lockdep_assert_held(&i915->irq_lock);
 
-               if (!(master_ctl & GEN11_GT_DW_IRQ(bank)))
-                       continue;
+       intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 
-               intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
+       if (unlikely(!intr_dw)) {
+               DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
+               return;
+       }
 
-               if (unlikely(!intr_dw)) {
-                       DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
-                       continue;
-               }
+       for_each_set_bit(bit, &intr_dw, 32) {
+               const u32 ident = gen11_gt_engine_identity(i915,
+                                                          bank, bit);
 
-               for_each_set_bit(bit, &intr_dw, 32) {
-                       const u16 iir = gen11_gt_engine_intr(i915, bank, bit);
+               gen11_gt_identity_handler(i915, ident);
+       }
 
-                       if (unlikely(!iir))
-                               continue;
+       /* Clear must be after shared has been served for engine */
+       raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
+}
 
-                       gen11_gt_engine_irq_handler(i915, bank, bit, iir);
-               }
+static void
+gen11_gt_irq_handler(struct drm_i915_private * const i915,
+                    const u32 master_ctl)
+{
+       unsigned int bank;
 
-               /* Clear must be after shared has been served for engine */
-               raw_reg_write(regs, GEN11_GT_INTR_DW(bank), intr_dw);
+       spin_lock(&i915->irq_lock);
+
+       for (bank = 0; bank < 2; bank++) {
+               if (master_ctl & GEN11_GT_DW_IRQ(bank))
+                       gen11_gt_bank_handler(i915, bank);
        }
+
+       spin_unlock(&i915->irq_lock);
 }
 
 static irqreturn_t gen11_irq_handler(int irq, void *arg)
@@ -2912,15 +2981,11 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg)
        return IRQ_HANDLED;
 }
 
-/**
- * i915_reset_device - do process context error handling work
- * @dev_priv: i915 device private
- *
- * Fire an error uevent so userspace can see that a hang or error
- * was detected.
- */
-static void i915_reset_device(struct drm_i915_private *dev_priv)
+static void i915_reset_device(struct drm_i915_private *dev_priv,
+                             u32 engine_mask,
+                             const char *reason)
 {
+       struct i915_gpu_error *error = &dev_priv->gpu_error;
        struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
        char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
        char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
@@ -2936,29 +3001,35 @@ static void i915_reset_device(struct drm_i915_private *dev_priv)
        i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
                intel_prepare_reset(dev_priv);
 
+               error->reason = reason;
+               error->stalled_mask = engine_mask;
+
                /* Signal that locked waiters should reset the GPU */
-               set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
-               wake_up_all(&dev_priv->gpu_error.wait_queue);
+               smp_mb__before_atomic();
+               set_bit(I915_RESET_HANDOFF, &error->flags);
+               wake_up_all(&error->wait_queue);
 
                /* Wait for anyone holding the lock to wakeup, without
                 * blocking indefinitely on struct_mutex.
                 */
                do {
                        if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
-                               i915_reset(dev_priv, 0);
+                               i915_reset(dev_priv, engine_mask, reason);
                                mutex_unlock(&dev_priv->drm.struct_mutex);
                        }
-               } while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
+               } while (wait_on_bit_timeout(&error->flags,
                                             I915_RESET_HANDOFF,
                                             TASK_UNINTERRUPTIBLE,
                                             1));
 
+               error->stalled_mask = 0;
+               error->reason = NULL;
+
                intel_finish_reset(dev_priv);
        }
 
-       if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
-               kobject_uevent_env(kobj,
-                                  KOBJ_CHANGE, reset_done_event);
+       if (!test_bit(I915_WEDGED, &error->flags))
+               kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
 }
 
 static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
@@ -2990,6 +3061,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  * i915_handle_error - handle a gpu error
  * @dev_priv: i915 device private
  * @engine_mask: mask representing engines that are hung
+ * @flags: control flags
  * @fmt: Error message format string
  *
  * Do some basic checking of register state at error time and
@@ -3000,16 +3072,23 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  */
 void i915_handle_error(struct drm_i915_private *dev_priv,
                       u32 engine_mask,
+                      unsigned long flags,
                       const char *fmt, ...)
 {
        struct intel_engine_cs *engine;
        unsigned int tmp;
-       va_list args;
        char error_msg[80];
+       char *msg = NULL;
 
-       va_start(args, fmt);
-       vscnprintf(error_msg, sizeof(error_msg), fmt, args);
-       va_end(args);
+       if (fmt) {
+               va_list args;
+
+               va_start(args, fmt);
+               vscnprintf(error_msg, sizeof(error_msg), fmt, args);
+               va_end(args);
+
+               msg = error_msg;
+       }
 
        /*
         * In most cases it's guaranteed that we get here with an RPM
@@ -3020,8 +3099,12 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
         */
        intel_runtime_pm_get(dev_priv);
 
-       i915_capture_error_state(dev_priv, engine_mask, error_msg);
-       i915_clear_error_registers(dev_priv);
+       engine_mask &= INTEL_INFO(dev_priv)->ring_mask;
+
+       if (flags & I915_ERROR_CAPTURE) {
+               i915_capture_error_state(dev_priv, engine_mask, msg);
+               i915_clear_error_registers(dev_priv);
+       }
 
        /*
         * Try engine reset when available. We fall back to full reset if
@@ -3034,7 +3117,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
                                             &dev_priv->gpu_error.flags))
                                continue;
 
-                       if (i915_reset_engine(engine, 0) == 0)
+                       if (i915_reset_engine(engine, msg) == 0)
                                engine_mask &= ~intel_engine_flag(engine);
 
                        clear_bit(I915_RESET_ENGINE + engine->id,
@@ -3064,7 +3147,7 @@ void i915_handle_error(struct drm_i915_private *dev_priv,
                                    TASK_UNINTERRUPTIBLE);
        }
 
-       i915_reset_device(dev_priv);
+       i915_reset_device(dev_priv, engine_mask, msg);
 
        for_each_engine(engine, dev_priv, tmp) {
                clear_bit(I915_RESET_ENGINE + engine->id,
@@ -3286,6 +3369,11 @@ static void ironlake_irq_reset(struct drm_device *dev)
        if (IS_GEN7(dev_priv))
                I915_WRITE(GEN7_ERR_INT, 0xffffffff);
 
+       if (IS_HASWELL(dev_priv)) {
+               I915_WRITE(EDP_PSR_IMR, 0xffffffff);
+               I915_WRITE(EDP_PSR_IIR, 0xffffffff);
+       }
+
        gen5_gt_irq_reset(dev_priv);
 
        ibx_irq_reset(dev_priv);
@@ -3324,6 +3412,9 @@ static void gen8_irq_reset(struct drm_device *dev)
 
        gen8_gt_irq_reset(dev_priv);
 
+       I915_WRITE(EDP_PSR_IMR, 0xffffffff);
+       I915_WRITE(EDP_PSR_IIR, 0xffffffff);
+
        for_each_pipe(dev_priv, pipe)
                if (intel_display_power_is_enabled(dev_priv,
                                                   POWER_DOMAIN_PIPE(pipe)))
@@ -3349,6 +3440,9 @@ static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
        I915_WRITE(GEN11_VCS0_VCS1_INTR_MASK,   ~0);
        I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,   ~0);
        I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~0);
+
+       I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
+       I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
 static void gen11_irq_reset(struct drm_device *dev)
@@ -3697,6 +3791,12 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
                              DE_DP_A_HOTPLUG);
        }
 
+       if (IS_HASWELL(dev_priv)) {
+               gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+               intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
+               display_mask |= DE_EDP_PSR_INT_HSW;
+       }
+
        dev_priv->irq_mask = ~display_mask;
 
        ibx_irq_pre_postinstall(dev);
@@ -3807,7 +3907,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
        uint32_t de_pipe_enables;
        u32 de_port_masked = GEN8_AUX_CHANNEL_A;
        u32 de_port_enables;
-       u32 de_misc_masked = GEN8_DE_MISC_GSE;
+       u32 de_misc_masked = GEN8_DE_MISC_GSE | GEN8_DE_EDP_PSR;
        enum pipe pipe;
 
        if (INTEL_GEN(dev_priv) >= 9) {
@@ -3832,6 +3932,9 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
        else if (IS_BROADWELL(dev_priv))
                de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
 
+       gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+       intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
+
        for_each_pipe(dev_priv, pipe) {
                dev_priv->de_irq_mask[pipe] = ~de_pipe_masked;
 
@@ -3887,7 +3990,14 @@ static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
        I915_WRITE(GEN11_VCS2_VCS3_INTR_MASK,   ~(irqs | irqs << 16));
        I915_WRITE(GEN11_VECS0_VECS1_INTR_MASK, ~(irqs | irqs << 16));
 
-       dev_priv->pm_imr = 0xffffffff; /* TODO */
+       /*
+        * RPS interrupts will get enabled/disabled on demand when RPS itself
+        * is enabled/disabled.
+        */
+       dev_priv->pm_ier = 0x0;
+       dev_priv->pm_imr = ~dev_priv->pm_ier;
+       I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0);
+       I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
 static int gen11_irq_postinstall(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_oa_icl.c b/drivers/gpu/drm/i915/i915_oa_icl.c
new file mode 100644 (file)
index 0000000..a566792
--- /dev/null
@@ -0,0 +1,118 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_icl.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+       { _MMIO(0x2740), 0x00000000 },
+       { _MMIO(0x2710), 0x00000000 },
+       { _MMIO(0x2714), 0xf0800000 },
+       { _MMIO(0x2720), 0x00000000 },
+       { _MMIO(0x2724), 0xf0800000 },
+       { _MMIO(0x2770), 0x00000004 },
+       { _MMIO(0x2774), 0x0000ffff },
+       { _MMIO(0x2778), 0x00000003 },
+       { _MMIO(0x277c), 0x0000ffff },
+       { _MMIO(0x2780), 0x00000007 },
+       { _MMIO(0x2784), 0x0000ffff },
+       { _MMIO(0x2788), 0x00100002 },
+       { _MMIO(0x278c), 0x0000fff7 },
+       { _MMIO(0x2790), 0x00100002 },
+       { _MMIO(0x2794), 0x0000ffcf },
+       { _MMIO(0x2798), 0x00100082 },
+       { _MMIO(0x279c), 0x0000ffef },
+       { _MMIO(0x27a0), 0x001000c2 },
+       { _MMIO(0x27a4), 0x0000ffe7 },
+       { _MMIO(0x27a8), 0x00100001 },
+       { _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+       { _MMIO(0xd04), 0x00000200 },
+       { _MMIO(0x9840), 0x00000000 },
+       { _MMIO(0x9884), 0x00000000 },
+       { _MMIO(0x9888), 0x10060000 },
+       { _MMIO(0x9888), 0x22060000 },
+       { _MMIO(0x9888), 0x16060000 },
+       { _MMIO(0x9888), 0x24060000 },
+       { _MMIO(0x9888), 0x18060000 },
+       { _MMIO(0x9888), 0x1a060000 },
+       { _MMIO(0x9888), 0x12060000 },
+       { _MMIO(0x9888), 0x14060000 },
+       { _MMIO(0x9888), 0x10060000 },
+       { _MMIO(0x9888), 0x22060000 },
+       { _MMIO(0x9884), 0x00000003 },
+       { _MMIO(0x9888), 0x16130000 },
+       { _MMIO(0x9888), 0x24000001 },
+       { _MMIO(0x9888), 0x0e130056 },
+       { _MMIO(0x9888), 0x10130000 },
+       { _MMIO(0x9888), 0x1a130000 },
+       { _MMIO(0x9888), 0x541f0001 },
+       { _MMIO(0x9888), 0x181f0000 },
+       { _MMIO(0x9888), 0x4c1f0000 },
+       { _MMIO(0x9888), 0x301f0000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+       return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv)
+{
+       strlcpy(dev_priv->perf.oa.test_config.uuid,
+               "a291665e-244b-4b76-9b9a-01de9d3c8068",
+               sizeof(dev_priv->perf.oa.test_config.uuid));
+       dev_priv->perf.oa.test_config.id = 1;
+
+       dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+       dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+       dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+       dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+       dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+       dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+       dev_priv->perf.oa.test_config.sysfs_metric.name = "a291665e-244b-4b76-9b9a-01de9d3c8068";
+       dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+       dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+       dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+       dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+       dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_icl.h b/drivers/gpu/drm/i915/i915_oa_icl.h
new file mode 100644 (file)
index 0000000..ae1c24a
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_ICL_H__
+#define __I915_OA_ICL_H__
+
+extern void i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv);
+
+#endif
index 430f5f9..c963603 100644 (file)
@@ -48,7 +48,7 @@ struct drm_printer;
        param(int, enable_ips, 1) \
        param(int, invert_brightness, 0) \
        param(int, enable_guc, 0) \
-       param(int, guc_log_level, 0) \
+       param(int, guc_log_level, -1) \
        param(char *, guc_firmware_path, NULL) \
        param(char *, huc_firmware_path, NULL) \
        param(int, mmio_debug, 0) \
index 062e91b..4364922 100644 (file)
@@ -602,6 +602,7 @@ static const struct intel_device_info intel_icelake_11_info = {
        PLATFORM(INTEL_ICELAKE),
        .is_alpha_support = 1,
        .has_resource_streamer = 0,
+       .ring_mask = RENDER_RING | BLT_RING | VEBOX_RING | BSD_RING | BSD3_RING,
 };
 
 #undef GEN
index abaca6e..4b1da01 100644 (file)
 #include "i915_oa_cflgt2.h"
 #include "i915_oa_cflgt3.h"
 #include "i915_oa_cnl.h"
+#include "i915_oa_icl.h"
 
 /* HW requires this to be a power of two, between 128k and 16M, though driver
  * is currently generally designed assuming the largest 16M size is used such
@@ -1042,7 +1043,7 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream,
 
                I915_WRITE(GEN7_OASTATUS2,
                           ((head & GEN7_OASTATUS2_HEAD_MASK) |
-                           OA_MEM_SELECT_GGTT));
+                           GEN7_OASTATUS2_MEM_SELECT_GGTT));
                dev_priv->perf.oa.oa_buffer.head = head;
 
                spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags);
@@ -1233,7 +1234,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
                 *
                 * NB: implied RCS engine...
                 */
-               ring = engine->context_pin(engine, stream->ctx);
+               ring = intel_context_pin(stream->ctx, engine);
                mutex_unlock(&dev_priv->drm.struct_mutex);
                if (IS_ERR(ring))
                        return PTR_ERR(ring);
@@ -1245,7 +1246,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
                 * with gen8+ and execlists
                 */
                dev_priv->perf.oa.specific_ctx_id =
-                       i915_ggtt_offset(stream->ctx->engine[engine->id].state);
+                       i915_ggtt_offset(to_intel_context(stream->ctx, engine)->state);
        }
 
        return 0;
@@ -1270,7 +1271,7 @@ static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
                mutex_lock(&dev_priv->drm.struct_mutex);
 
                dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
-               engine->context_unpin(engine, stream->ctx);
+               intel_context_unpin(stream->ctx, engine);
 
                mutex_unlock(&dev_priv->drm.struct_mutex);
        }
@@ -1332,7 +1333,8 @@ static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv)
        /* Pre-DevBDW: OABUFFER must be set with counters off,
         * before OASTATUS1, but after OASTATUS2
         */
-       I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */
+       I915_WRITE(GEN7_OASTATUS2,
+                  gtt_offset | GEN7_OASTATUS2_MEM_SELECT_GGTT); /* head */
        dev_priv->perf.oa.oa_buffer.head = gtt_offset;
 
        I915_WRITE(GEN7_OABUFFER, gtt_offset);
@@ -1392,7 +1394,7 @@ static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv)
         *  bit."
         */
        I915_WRITE(GEN8_OABUFFER, gtt_offset |
-                  OABUFFER_SIZE_16M | OA_MEM_SELECT_GGTT);
+                  OABUFFER_SIZE_16M | GEN8_OABUFFER_MEM_SELECT_GGTT);
        I915_WRITE(GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK);
 
        /* Mark that we need updated tail pointers to read from... */
@@ -1757,6 +1759,7 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
                                       const struct i915_oa_config *oa_config)
 {
+       struct intel_engine_cs *engine = dev_priv->engine[RCS];
        struct i915_gem_context *ctx;
        int ret;
        unsigned int wait_flags = I915_WAIT_LOCKED;
@@ -1787,7 +1790,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 
        /* Update all contexts now that we've stalled the submission. */
        list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
-               struct intel_context *ce = &ctx->engine[RCS];
+               struct intel_context *ce = to_intel_context(ctx, engine);
                u32 *regs;
 
                /* OA settings will be set upon first use */
@@ -1840,7 +1843,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
         * be read back from automatically triggered reports, as part of the
         * RPT_ID field.
         */
-       if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) {
+       if (IS_GEN(dev_priv, 9, 11)) {
                I915_WRITE(GEN8_OA_DEBUG,
                           _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
                                              GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
@@ -1870,7 +1873,6 @@ static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
 
        I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
                                      ~GT_NOA_ENABLE));
-
 }
 
 static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
@@ -1885,6 +1887,13 @@ static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
 
 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 {
+       struct i915_gem_context *ctx =
+                       dev_priv->perf.oa.exclusive_stream->ctx;
+       u32 ctx_id = dev_priv->perf.oa.specific_ctx_id;
+       bool periodic = dev_priv->perf.oa.periodic;
+       u32 period_exponent = dev_priv->perf.oa.period_exponent;
+       u32 report_format = dev_priv->perf.oa.oa_buffer.format;
+
        /*
         * Reset buf pointers so we don't forward reports from before now.
         *
@@ -1896,25 +1905,14 @@ static void gen7_oa_enable(struct drm_i915_private *dev_priv)
         */
        gen7_init_oa_buffer(dev_priv);
 
-       if (dev_priv->perf.oa.exclusive_stream->enabled) {
-               struct i915_gem_context *ctx =
-                       dev_priv->perf.oa.exclusive_stream->ctx;
-               u32 ctx_id = dev_priv->perf.oa.specific_ctx_id;
-
-               bool periodic = dev_priv->perf.oa.periodic;
-               u32 period_exponent = dev_priv->perf.oa.period_exponent;
-               u32 report_format = dev_priv->perf.oa.oa_buffer.format;
-
-               I915_WRITE(GEN7_OACONTROL,
-                          (ctx_id & GEN7_OACONTROL_CTX_MASK) |
-                          (period_exponent <<
-                           GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
-                          (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
-                          (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
-                          (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
-                          GEN7_OACONTROL_ENABLE);
-       } else
-               I915_WRITE(GEN7_OACONTROL, 0);
+       I915_WRITE(GEN7_OACONTROL,
+                  (ctx_id & GEN7_OACONTROL_CTX_MASK) |
+                  (period_exponent <<
+                   GEN7_OACONTROL_TIMER_PERIOD_SHIFT) |
+                  (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) |
+                  (report_format << GEN7_OACONTROL_FORMAT_SHIFT) |
+                  (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) |
+                  GEN7_OACONTROL_ENABLE);
 }
 
 static void gen8_oa_enable(struct drm_i915_private *dev_priv)
@@ -2099,13 +2097,17 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 
        if (stream->ctx) {
                ret = oa_get_render_ctx_id(stream);
-               if (ret)
+               if (ret) {
+                       DRM_DEBUG("Invalid context id to filter with\n");
                        return ret;
+               }
        }
 
        ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config);
-       if (ret)
+       if (ret) {
+               DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set);
                goto err_config;
+       }
 
        /* PRM - observability performance counters:
         *
@@ -2132,8 +2134,10 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 
        ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv,
                                                      stream->oa_config);
-       if (ret)
+       if (ret) {
+               DRM_DEBUG("Unable to enable metric set\n");
                goto err_enable;
+       }
 
        stream->ops = &i915_oa_stream_ops;
 
@@ -2745,7 +2749,8 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv,
                        props->ctx_handle = value;
                        break;
                case DRM_I915_PERF_PROP_SAMPLE_OA:
-                       props->sample_flags |= SAMPLE_OA_REPORT;
+                       if (value)
+                               props->sample_flags |= SAMPLE_OA_REPORT;
                        break;
                case DRM_I915_PERF_PROP_OA_METRICS_SET:
                        if (value == 0) {
@@ -2935,6 +2940,8 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
                        i915_perf_load_test_config_cflgt3(dev_priv);
        } else if (IS_CANNONLAKE(dev_priv)) {
                i915_perf_load_test_config_cnl(dev_priv);
+       } else if (IS_ICELAKE(dev_priv)) {
+               i915_perf_load_test_config_icl(dev_priv);
        }
 
        if (dev_priv->perf.oa.test_config.id == 0)
@@ -3292,6 +3299,8 @@ int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
 
        mutex_unlock(&dev_priv->perf.metrics_lock);
 
+       DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id);
+
        return oa_config->id;
 
 sysfs_err:
@@ -3348,6 +3357,9 @@ int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
                           &oa_config->sysfs_metric);
 
        idr_remove(&dev_priv->perf.metrics_idr, *arg);
+
+       DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
+
        put_oa_config(dev_priv, oa_config);
 
 config_err:
@@ -3467,7 +3479,7 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 
                                dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
                        }
-               } else if (IS_GEN10(dev_priv)) {
+               } else if (IS_GEN(dev_priv, 10, 11)) {
                        dev_priv->perf.oa.ops.is_valid_b_counter_reg =
                                gen7_is_valid_b_counter_addr;
                        dev_priv->perf.oa.ops.is_valid_mux_reg =
index f0519e3..dc87797 100644 (file)
@@ -1,33 +1,12 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
 
-#include <linux/perf_event.h>
-#include <linux/pm_runtime.h>
-
-#include "i915_drv.h"
 #include "i915_pmu.h"
 #include "intel_ringbuffer.h"
+#include "i915_drv.h"
 
 /* Frequency for the sampling timer for events which need it. */
 #define FREQUENCY 200
index aa1b1a9..2ba7352 100644 (file)
@@ -1,29 +1,19 @@
 /*
- * Copyright © 2017 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  *
+ * Copyright © 2017-2018 Intel Corporation
  */
+
 #ifndef __I915_PMU_H__
 #define __I915_PMU_H__
 
+#include <linux/hrtimer.h>
+#include <linux/perf_event.h>
+#include <linux/spinlock_types.h>
+#include <drm/i915_drm.h>
+
+struct drm_i915_private;
+
 enum {
        __I915_SAMPLE_FREQ_ACT = 0,
        __I915_SAMPLE_FREQ_REQ,
index e6a8c0e..197c966 100644 (file)
@@ -153,9 +153,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
 #define _PLL(pll, a, b) ((a) + (pll)*((b)-(a)))
 #define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b))
-#define _MMIO_PORT6(port, a, b, c, d, e, f) _MMIO(_PICK(port, a, b, c, d, e, f))
-#define _MMIO_PORT6_LN(port, ln, a0, a1, b, c, d, e, f)                        \
-       _MMIO(_PICK(port, a0, b, c, d, e, f) + (ln * (a1 - a0)))
 #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__)
 #define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c))
 
@@ -191,6 +188,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OTHER_CLASS            4
 #define MAX_ENGINE_CLASS       4
 
+#define OTHER_GTPM_INSTANCE    1
 #define MAX_ENGINE_INSTANCE    3
 
 /* PCI config space */
@@ -304,6 +302,17 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN6_GRDOM_VECS               (1 << 4)
 #define  GEN9_GRDOM_GUC                        (1 << 5)
 #define  GEN8_GRDOM_MEDIA2             (1 << 7)
+/* GEN11 changed all bit defs except for FULL & RENDER */
+#define  GEN11_GRDOM_FULL              GEN6_GRDOM_FULL
+#define  GEN11_GRDOM_RENDER            GEN6_GRDOM_RENDER
+#define  GEN11_GRDOM_BLT               (1 << 2)
+#define  GEN11_GRDOM_GUC               (1 << 3)
+#define  GEN11_GRDOM_MEDIA             (1 << 5)
+#define  GEN11_GRDOM_MEDIA2            (1 << 6)
+#define  GEN11_GRDOM_MEDIA3            (1 << 7)
+#define  GEN11_GRDOM_MEDIA4            (1 << 8)
+#define  GEN11_GRDOM_VECS              (1 << 13)
+#define  GEN11_GRDOM_VECS2             (1 << 14)
 
 #define RING_PP_DIR_BASE(engine)       _MMIO((engine)->mmio_base+0x228)
 #define RING_PP_DIR_BASE_READ(engine)  _MMIO((engine)->mmio_base+0x518)
@@ -430,145 +439,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define VGA_CR_INDEX_CGA 0x3d4
 #define VGA_CR_DATA_CGA 0x3d5
 
-/*
- * Instruction field definitions used by the command parser
- */
-#define INSTR_CLIENT_SHIFT      29
-#define   INSTR_MI_CLIENT       0x0
-#define   INSTR_BC_CLIENT       0x2
-#define   INSTR_RC_CLIENT       0x3
-#define INSTR_SUBCLIENT_SHIFT   27
-#define INSTR_SUBCLIENT_MASK    0x18000000
-#define   INSTR_MEDIA_SUBCLIENT 0x2
-#define INSTR_26_TO_24_MASK    0x7000000
-#define   INSTR_26_TO_24_SHIFT 24
-
-/*
- * Memory interface instructions used by the kernel
- */
-#define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags))
-/* Many MI commands use bit 22 of the header dword for GGTT vs PPGTT */
-#define  MI_GLOBAL_GTT    (1<<22)
-
-#define MI_NOOP                        MI_INSTR(0, 0)
-#define MI_USER_INTERRUPT      MI_INSTR(0x02, 0)
-#define MI_WAIT_FOR_EVENT       MI_INSTR(0x03, 0)
-#define   MI_WAIT_FOR_OVERLAY_FLIP     (1<<16)
-#define   MI_WAIT_FOR_PLANE_B_FLIP      (1<<6)
-#define   MI_WAIT_FOR_PLANE_A_FLIP      (1<<2)
-#define   MI_WAIT_FOR_PLANE_A_SCANLINES (1<<1)
-#define MI_FLUSH               MI_INSTR(0x04, 0)
-#define   MI_READ_FLUSH                (1 << 0)
-#define   MI_EXE_FLUSH         (1 << 1)
-#define   MI_NO_WRITE_FLUSH    (1 << 2)
-#define   MI_SCENE_COUNT       (1 << 3) /* just increment scene count */
-#define   MI_END_SCENE         (1 << 4) /* flush binner and incr scene count */
-#define   MI_INVALIDATE_ISP    (1 << 5) /* invalidate indirect state pointers */
-#define MI_REPORT_HEAD         MI_INSTR(0x07, 0)
-#define MI_ARB_ON_OFF          MI_INSTR(0x08, 0)
-#define   MI_ARB_ENABLE                        (1<<0)
-#define   MI_ARB_DISABLE               (0<<0)
-#define MI_BATCH_BUFFER_END    MI_INSTR(0x0a, 0)
-#define MI_SUSPEND_FLUSH       MI_INSTR(0x0b, 0)
-#define   MI_SUSPEND_FLUSH_EN  (1<<0)
-#define MI_SET_APPID           MI_INSTR(0x0e, 0)
-#define MI_OVERLAY_FLIP                MI_INSTR(0x11, 0)
-#define   MI_OVERLAY_CONTINUE  (0x0<<21)
-#define   MI_OVERLAY_ON                (0x1<<21)
-#define   MI_OVERLAY_OFF       (0x2<<21)
-#define MI_LOAD_SCAN_LINES_INCL MI_INSTR(0x12, 0)
-#define MI_DISPLAY_FLIP                MI_INSTR(0x14, 2)
-#define MI_DISPLAY_FLIP_I915   MI_INSTR(0x14, 1)
-#define   MI_DISPLAY_FLIP_PLANE(n) ((n) << 20)
-/* IVB has funny definitions for which plane to flip. */
-#define   MI_DISPLAY_FLIP_IVB_PLANE_A  (0 << 19)
-#define   MI_DISPLAY_FLIP_IVB_PLANE_B  (1 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_A (2 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_B (3 << 19)
-#define   MI_DISPLAY_FLIP_IVB_PLANE_C  (4 << 19)
-#define   MI_DISPLAY_FLIP_IVB_SPRITE_C (5 << 19)
-/* SKL ones */
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_A        (0 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_B        (1 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_1_C        (2 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_A        (4 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_B        (5 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_2_C        (6 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_A        (7 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_B        (8 << 8)
-#define   MI_DISPLAY_FLIP_SKL_PLANE_3_C        (9 << 8)
-#define MI_SEMAPHORE_MBOX      MI_INSTR(0x16, 1) /* gen6, gen7 */
-#define   MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
-#define   MI_SEMAPHORE_UPDATE      (1<<21)
-#define   MI_SEMAPHORE_COMPARE     (1<<20)
-#define   MI_SEMAPHORE_REGISTER            (1<<18)
-#define   MI_SEMAPHORE_SYNC_VR     (0<<16) /* RCS  wait for VCS  (RVSYNC) */
-#define   MI_SEMAPHORE_SYNC_VER            (1<<16) /* RCS  wait for VECS (RVESYNC) */
-#define   MI_SEMAPHORE_SYNC_BR     (2<<16) /* RCS  wait for BCS  (RBSYNC) */
-#define   MI_SEMAPHORE_SYNC_BV     (0<<16) /* VCS  wait for BCS  (VBSYNC) */
-#define   MI_SEMAPHORE_SYNC_VEV            (1<<16) /* VCS  wait for VECS (VVESYNC) */
-#define   MI_SEMAPHORE_SYNC_RV     (2<<16) /* VCS  wait for RCS  (VRSYNC) */
-#define   MI_SEMAPHORE_SYNC_RB     (0<<16) /* BCS  wait for RCS  (BRSYNC) */
-#define   MI_SEMAPHORE_SYNC_VEB            (1<<16) /* BCS  wait for VECS (BVESYNC) */
-#define   MI_SEMAPHORE_SYNC_VB     (2<<16) /* BCS  wait for VCS  (BVSYNC) */
-#define   MI_SEMAPHORE_SYNC_BVE            (0<<16) /* VECS wait for BCS  (VEBSYNC) */
-#define   MI_SEMAPHORE_SYNC_VVE            (1<<16) /* VECS wait for VCS  (VEVSYNC) */
-#define   MI_SEMAPHORE_SYNC_RVE            (2<<16) /* VECS wait for RCS  (VERSYNC) */
-#define   MI_SEMAPHORE_SYNC_INVALID (3<<16)
-#define   MI_SEMAPHORE_SYNC_MASK    (3<<16)
-#define MI_SET_CONTEXT         MI_INSTR(0x18, 0)
-#define   MI_MM_SPACE_GTT              (1<<8)
-#define   MI_MM_SPACE_PHYSICAL         (0<<8)
-#define   MI_SAVE_EXT_STATE_EN         (1<<3)
-#define   MI_RESTORE_EXT_STATE_EN      (1<<2)
-#define   MI_FORCE_RESTORE             (1<<1)
-#define   MI_RESTORE_INHIBIT           (1<<0)
-#define   HSW_MI_RS_SAVE_STATE_EN       (1<<3)
-#define   HSW_MI_RS_RESTORE_STATE_EN    (1<<2)
-#define MI_SEMAPHORE_SIGNAL    MI_INSTR(0x1b, 0) /* GEN8+ */
-#define   MI_SEMAPHORE_TARGET(engine)  ((engine)<<15)
-#define MI_SEMAPHORE_WAIT      MI_INSTR(0x1c, 2) /* GEN8+ */
-#define   MI_SEMAPHORE_POLL            (1<<15)
-#define   MI_SEMAPHORE_SAD_GTE_SDD     (1<<12)
-#define MI_STORE_DWORD_IMM     MI_INSTR(0x20, 1)
-#define MI_STORE_DWORD_IMM_GEN4        MI_INSTR(0x20, 2)
-#define   MI_MEM_VIRTUAL       (1 << 22) /* 945,g33,965 */
-#define   MI_USE_GGTT          (1 << 22) /* g4x+ */
-#define MI_STORE_DWORD_INDEX   MI_INSTR(0x21, 1)
-#define   MI_STORE_DWORD_INDEX_SHIFT 2
-/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM:
- * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw
- *   simply ignores the register load under certain conditions.
- * - One can actually load arbitrary many arbitrary registers: Simply issue x
- *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
- */
-#define MI_LOAD_REGISTER_IMM(x)        MI_INSTR(0x22, 2*(x)-1)
-#define   MI_LRI_FORCE_POSTED          (1<<12)
-#define MI_STORE_REGISTER_MEM        MI_INSTR(0x24, 1)
-#define MI_STORE_REGISTER_MEM_GEN8   MI_INSTR(0x24, 2)
-#define   MI_SRM_LRM_GLOBAL_GTT                (1<<22)
-#define MI_FLUSH_DW            MI_INSTR(0x26, 1) /* for GEN6 */
-#define   MI_FLUSH_DW_STORE_INDEX      (1<<21)
-#define   MI_INVALIDATE_TLB            (1<<18)
-#define   MI_FLUSH_DW_OP_STOREDW       (1<<14)
-#define   MI_FLUSH_DW_OP_MASK          (3<<14)
-#define   MI_FLUSH_DW_NOTIFY           (1<<8)
-#define   MI_INVALIDATE_BSD            (1<<7)
-#define   MI_FLUSH_DW_USE_GTT          (1<<2)
-#define   MI_FLUSH_DW_USE_PPGTT                (0<<2)
-#define MI_LOAD_REGISTER_MEM      MI_INSTR(0x29, 1)
-#define MI_LOAD_REGISTER_MEM_GEN8  MI_INSTR(0x29, 2)
-#define MI_BATCH_BUFFER                MI_INSTR(0x30, 1)
-#define   MI_BATCH_NON_SECURE          (1)
-/* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */
-#define   MI_BATCH_NON_SECURE_I965     (1<<8)
-#define   MI_BATCH_PPGTT_HSW           (1<<8)
-#define   MI_BATCH_NON_SECURE_HSW      (1<<13)
-#define MI_BATCH_BUFFER_START  MI_INSTR(0x31, 0)
-#define   MI_BATCH_GTT             (2<<6) /* aliased with (1<<7) on gen4 */
-#define MI_BATCH_BUFFER_START_GEN8     MI_INSTR(0x31, 1)
-#define   MI_BATCH_RESOURCE_STREAMER (1<<10)
-
 #define MI_PREDICATE_SRC0      _MMIO(0x2400)
 #define MI_PREDICATE_SRC0_UDW  _MMIO(0x2400 + 4)
 #define MI_PREDICATE_SRC1      _MMIO(0x2408)
@@ -578,130 +448,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  LOWER_SLICE_ENABLED   (1<<0)
 #define  LOWER_SLICE_DISABLED  (0<<0)
 
-/*
- * 3D instructions used by the kernel
- */
-#define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags))
-
-#define GEN9_MEDIA_POOL_STATE     ((0x3 << 29) | (0x2 << 27) | (0x5 << 16) | 4)
-#define   GEN9_MEDIA_POOL_ENABLE  (1 << 31)
-#define GFX_OP_RASTER_RULES    ((0x3<<29)|(0x7<<24))
-#define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define   SC_UPDATE_SCISSOR       (0x1<<1)
-#define   SC_ENABLE_MASK          (0x1<<0)
-#define   SC_ENABLE               (0x1<<0)
-#define GFX_OP_LOAD_INDIRECT   ((0x3<<29)|(0x1d<<24)|(0x7<<16))
-#define GFX_OP_SCISSOR_INFO    ((0x3<<29)|(0x1d<<24)|(0x81<<16)|(0x1))
-#define   SCI_YMIN_MASK      (0xffff<<16)
-#define   SCI_XMIN_MASK      (0xffff<<0)
-#define   SCI_YMAX_MASK      (0xffff<<16)
-#define   SCI_XMAX_MASK      (0xffff<<0)
-#define GFX_OP_SCISSOR_ENABLE   ((0x3<<29)|(0x1c<<24)|(0x10<<19))
-#define GFX_OP_SCISSOR_RECT     ((0x3<<29)|(0x1d<<24)|(0x81<<16)|1)
-#define GFX_OP_COLOR_FACTOR      ((0x3<<29)|(0x1d<<24)|(0x1<<16)|0x0)
-#define GFX_OP_STIPPLE           ((0x3<<29)|(0x1d<<24)|(0x83<<16))
-#define GFX_OP_MAP_INFO          ((0x3<<29)|(0x1d<<24)|0x4)
-#define GFX_OP_DESTBUFFER_VARS   ((0x3<<29)|(0x1d<<24)|(0x85<<16)|0x0)
-#define GFX_OP_DESTBUFFER_INFO  ((0x3<<29)|(0x1d<<24)|(0x8e<<16)|1)
-#define GFX_OP_DRAWRECT_INFO     ((0x3<<29)|(0x1d<<24)|(0x80<<16)|(0x3))
-#define GFX_OP_DRAWRECT_INFO_I965  ((0x7900<<16)|0x2)
-
-#define COLOR_BLT_CMD                  (2<<29 | 0x40<<22 | (5-2))
-#define SRC_COPY_BLT_CMD               ((2<<29)|(0x43<<22)|4)
-#define XY_SRC_COPY_BLT_CMD            ((2<<29)|(0x53<<22)|6)
-#define XY_MONO_SRC_COPY_IMM_BLT       ((2<<29)|(0x71<<22)|5)
-#define   BLT_WRITE_A                  (2<<20)
-#define   BLT_WRITE_RGB                        (1<<20)
-#define   BLT_WRITE_RGBA               (BLT_WRITE_RGB | BLT_WRITE_A)
-#define   BLT_DEPTH_8                  (0<<24)
-#define   BLT_DEPTH_16_565             (1<<24)
-#define   BLT_DEPTH_16_1555            (2<<24)
-#define   BLT_DEPTH_32                 (3<<24)
-#define   BLT_ROP_SRC_COPY             (0xcc<<16)
-#define   BLT_ROP_COLOR_COPY           (0xf0<<16)
-#define XY_SRC_COPY_BLT_SRC_TILED      (1<<15) /* 965+ only */
-#define XY_SRC_COPY_BLT_DST_TILED      (1<<11) /* 965+ only */
-#define CMD_OP_DISPLAYBUFFER_INFO ((0x0<<29)|(0x14<<23)|2)
-#define   ASYNC_FLIP                (1<<22)
-#define   DISPLAY_PLANE_A           (0<<20)
-#define   DISPLAY_PLANE_B           (1<<20)
-#define GFX_OP_PIPE_CONTROL(len)       ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2))
-#define   PIPE_CONTROL_FLUSH_L3                                (1<<27)
-#define   PIPE_CONTROL_GLOBAL_GTT_IVB                  (1<<24) /* gen7+ */
-#define   PIPE_CONTROL_MMIO_WRITE                      (1<<23)
-#define   PIPE_CONTROL_STORE_DATA_INDEX                        (1<<21)
-#define   PIPE_CONTROL_CS_STALL                                (1<<20)
-#define   PIPE_CONTROL_TLB_INVALIDATE                  (1<<18)
-#define   PIPE_CONTROL_MEDIA_STATE_CLEAR               (1<<16)
-#define   PIPE_CONTROL_QW_WRITE                                (1<<14)
-#define   PIPE_CONTROL_POST_SYNC_OP_MASK                (3<<14)
-#define   PIPE_CONTROL_DEPTH_STALL                     (1<<13)
-#define   PIPE_CONTROL_WRITE_FLUSH                     (1<<12)
-#define   PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH       (1<<12) /* gen6+ */
-#define   PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE    (1<<11) /* MBZ on Ironlake */
-#define   PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE                (1<<10) /* GM45+ only */
-#define   PIPE_CONTROL_INDIRECT_STATE_DISABLE          (1<<9)
-#define   PIPE_CONTROL_NOTIFY                          (1<<8)
-#define   PIPE_CONTROL_FLUSH_ENABLE                    (1<<7) /* gen7+ */
-#define   PIPE_CONTROL_DC_FLUSH_ENABLE                 (1<<5)
-#define   PIPE_CONTROL_VF_CACHE_INVALIDATE             (1<<4)
-#define   PIPE_CONTROL_CONST_CACHE_INVALIDATE          (1<<3)
-#define   PIPE_CONTROL_STATE_CACHE_INVALIDATE          (1<<2)
-#define   PIPE_CONTROL_STALL_AT_SCOREBOARD             (1<<1)
-#define   PIPE_CONTROL_DEPTH_CACHE_FLUSH               (1<<0)
-#define   PIPE_CONTROL_GLOBAL_GTT (1<<2) /* in addr dword */
-
-/*
- * Commands used only by the command parser
- */
-#define MI_SET_PREDICATE        MI_INSTR(0x01, 0)
-#define MI_ARB_CHECK            MI_INSTR(0x05, 0)
-#define MI_RS_CONTROL           MI_INSTR(0x06, 0)
-#define MI_URB_ATOMIC_ALLOC     MI_INSTR(0x09, 0)
-#define MI_PREDICATE            MI_INSTR(0x0C, 0)
-#define MI_RS_CONTEXT           MI_INSTR(0x0F, 0)
-#define MI_TOPOLOGY_FILTER      MI_INSTR(0x0D, 0)
-#define MI_LOAD_SCAN_LINES_EXCL MI_INSTR(0x13, 0)
-#define MI_URB_CLEAR            MI_INSTR(0x19, 0)
-#define MI_UPDATE_GTT           MI_INSTR(0x23, 0)
-#define MI_CLFLUSH              MI_INSTR(0x27, 0)
-#define MI_REPORT_PERF_COUNT    MI_INSTR(0x28, 0)
-#define   MI_REPORT_PERF_COUNT_GGTT (1<<0)
-#define MI_LOAD_REGISTER_REG    MI_INSTR(0x2A, 0)
-#define MI_RS_STORE_DATA_IMM    MI_INSTR(0x2B, 0)
-#define MI_LOAD_URB_MEM         MI_INSTR(0x2C, 0)
-#define MI_STORE_URB_MEM        MI_INSTR(0x2D, 0)
-#define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0)
-
-#define PIPELINE_SELECT                ((0x3<<29)|(0x1<<27)|(0x1<<24)|(0x4<<16))
-#define GFX_OP_3DSTATE_VF_STATISTICS   ((0x3<<29)|(0x1<<27)|(0x0<<24)|(0xB<<16))
-#define MEDIA_VFE_STATE                ((0x3<<29)|(0x2<<27)|(0x0<<24)|(0x0<<16))
-#define  MEDIA_VFE_STATE_MMIO_ACCESS_MASK (0x18)
-#define GPGPU_OBJECT                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x4<<16))
-#define GPGPU_WALKER                   ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x5<<16))
-#define GFX_OP_3DSTATE_DX9_CONSTANTF_VS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x39<<16))
-#define GFX_OP_3DSTATE_DX9_CONSTANTF_PS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x3A<<16))
-#define GFX_OP_3DSTATE_SO_DECL_LIST \
-       ((0x3<<29)|(0x3<<27)|(0x1<<24)|(0x17<<16))
-
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_VS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x43<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_GS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x44<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_HS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x45<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_DS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x46<<16))
-#define GFX_OP_3DSTATE_BINDING_TABLE_EDIT_PS \
-       ((0x3<<29)|(0x3<<27)|(0x0<<24)|(0x47<<16))
-
-#define MFX_WAIT  ((0x3<<29)|(0x1<<27)|(0x0<<16))
-
-#define COLOR_BLT     ((0x2<<29)|(0x40<<22))
-#define SRC_COPY_BLT  ((0x2<<29)|(0x43<<22))
-
 /*
  * Registers used only by the command parser
  */
@@ -802,6 +548,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 
 #define GEN8_OABUFFER_UDW _MMIO(0x23b4)
 #define GEN8_OABUFFER _MMIO(0x2b14)
+#define  GEN8_OABUFFER_MEM_SELECT_GGTT      (1 << 0)  /* 0: PPGTT, 1: GGTT */
 
 #define GEN7_OASTATUS1 _MMIO(0x2364)
 #define  GEN7_OASTATUS1_TAIL_MASK          0xffffffc0
@@ -810,7 +557,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN7_OASTATUS1_REPORT_LOST        (1<<0)
 
 #define GEN7_OASTATUS2 _MMIO(0x2368)
-#define GEN7_OASTATUS2_HEAD_MASK    0xffffffc0
+#define  GEN7_OASTATUS2_HEAD_MASK           0xffffffc0
+#define  GEN7_OASTATUS2_MEM_SELECT_GGTT     (1 << 0) /* 0: PPGTT, 1: GGTT */
 
 #define GEN8_OASTATUS _MMIO(0x2b08)
 #define  GEN8_OASTATUS_OVERRUN_STATUS      (1<<3)
@@ -832,8 +580,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OABUFFER_SIZE_8M    (6<<3)
 #define OABUFFER_SIZE_16M   (7<<3)
 
-#define OA_MEM_SELECT_GGTT  (1<<0)
-
 /*
  * Flexible, Aggregate EU Counter Registers.
  * Note: these aren't contiguous
@@ -1127,6 +873,12 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK      (1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ  0
 #define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ    1
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT    3
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK     (0x7 << GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ   0
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ 1
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ 2
+#define  GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ   3
 #define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT   1
 #define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK    (0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT)
 
@@ -1948,79 +1700,100 @@ enum i915_power_well_id {
 #define _CNL_PORT_PCS_DW1_LN0_C                0x162C04
 #define _CNL_PORT_PCS_DW1_LN0_D                0x162E04
 #define _CNL_PORT_PCS_DW1_LN0_F                0x162804
-#define CNL_PORT_PCS_DW1_GRP(port)     _MMIO_PORT6(port, \
+#define CNL_PORT_PCS_DW1_GRP(port)     _MMIO(_PICK(port, \
                                                    _CNL_PORT_PCS_DW1_GRP_AE, \
                                                    _CNL_PORT_PCS_DW1_GRP_B, \
                                                    _CNL_PORT_PCS_DW1_GRP_C, \
                                                    _CNL_PORT_PCS_DW1_GRP_D, \
                                                    _CNL_PORT_PCS_DW1_GRP_AE, \
-                                                   _CNL_PORT_PCS_DW1_GRP_F)
-#define CNL_PORT_PCS_DW1_LN0(port)     _MMIO_PORT6(port, \
+                                                   _CNL_PORT_PCS_DW1_GRP_F))
+
+#define CNL_PORT_PCS_DW1_LN0(port)     _MMIO(_PICK(port, \
                                                    _CNL_PORT_PCS_DW1_LN0_AE, \
                                                    _CNL_PORT_PCS_DW1_LN0_B, \
                                                    _CNL_PORT_PCS_DW1_LN0_C, \
                                                    _CNL_PORT_PCS_DW1_LN0_D, \
                                                    _CNL_PORT_PCS_DW1_LN0_AE, \
-                                                   _CNL_PORT_PCS_DW1_LN0_F)
+                                                   _CNL_PORT_PCS_DW1_LN0_F))
+#define _ICL_PORT_PCS_DW1_GRP_A                0x162604
+#define _ICL_PORT_PCS_DW1_GRP_B                0x6C604
+#define _ICL_PORT_PCS_DW1_LN0_A                0x162804
+#define _ICL_PORT_PCS_DW1_LN0_B                0x6C804
+#define ICL_PORT_PCS_DW1_GRP(port)     _MMIO_PORT(port,\
+                                                  _ICL_PORT_PCS_DW1_GRP_A, \
+                                                  _ICL_PORT_PCS_DW1_GRP_B)
+#define ICL_PORT_PCS_DW1_LN0(port)     _MMIO_PORT(port, \
+                                                  _ICL_PORT_PCS_DW1_LN0_A, \
+                                                  _ICL_PORT_PCS_DW1_LN0_B)
 #define   COMMON_KEEPER_EN             (1 << 26)
 
-#define _CNL_PORT_TX_DW2_GRP_AE                0x162348
-#define _CNL_PORT_TX_DW2_GRP_B         0x1623C8
-#define _CNL_PORT_TX_DW2_GRP_C         0x162B48
-#define _CNL_PORT_TX_DW2_GRP_D         0x162BC8
-#define _CNL_PORT_TX_DW2_GRP_F         0x162A48
-#define _CNL_PORT_TX_DW2_LN0_AE                0x162448
-#define _CNL_PORT_TX_DW2_LN0_B         0x162648
-#define _CNL_PORT_TX_DW2_LN0_C         0x162C48
-#define _CNL_PORT_TX_DW2_LN0_D         0x162E48
-#define _CNL_PORT_TX_DW2_LN0_F         0x162848
-#define CNL_PORT_TX_DW2_GRP(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW2_GRP_AE, \
-                                                   _CNL_PORT_TX_DW2_GRP_B, \
-                                                   _CNL_PORT_TX_DW2_GRP_C, \
-                                                   _CNL_PORT_TX_DW2_GRP_D, \
-                                                   _CNL_PORT_TX_DW2_GRP_AE, \
-                                                   _CNL_PORT_TX_DW2_GRP_F)
-#define CNL_PORT_TX_DW2_LN0(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW2_LN0_AE, \
-                                                   _CNL_PORT_TX_DW2_LN0_B, \
-                                                   _CNL_PORT_TX_DW2_LN0_C, \
-                                                   _CNL_PORT_TX_DW2_LN0_D, \
-                                                   _CNL_PORT_TX_DW2_LN0_AE, \
-                                                   _CNL_PORT_TX_DW2_LN0_F)
-#define   SWING_SEL_UPPER(x)           ((x >> 3) << 15)
+/* CNL Port TX registers */
+#define _CNL_PORT_TX_AE_GRP_OFFSET             0x162340
+#define _CNL_PORT_TX_B_GRP_OFFSET              0x1623C0
+#define _CNL_PORT_TX_C_GRP_OFFSET              0x162B40
+#define _CNL_PORT_TX_D_GRP_OFFSET              0x162BC0
+#define _CNL_PORT_TX_F_GRP_OFFSET              0x162A40
+#define _CNL_PORT_TX_AE_LN0_OFFSET             0x162440
+#define _CNL_PORT_TX_B_LN0_OFFSET              0x162640
+#define _CNL_PORT_TX_C_LN0_OFFSET              0x162C40
+#define _CNL_PORT_TX_D_LN0_OFFSET              0x162E40
+#define _CNL_PORT_TX_F_LN0_OFFSET              0x162840
+#define _CNL_PORT_TX_DW_GRP(port, dw)  (_PICK((port), \
+                                              _CNL_PORT_TX_AE_GRP_OFFSET, \
+                                              _CNL_PORT_TX_B_GRP_OFFSET, \
+                                              _CNL_PORT_TX_B_GRP_OFFSET, \
+                                              _CNL_PORT_TX_D_GRP_OFFSET, \
+                                              _CNL_PORT_TX_AE_GRP_OFFSET, \
+                                              _CNL_PORT_TX_F_GRP_OFFSET) + \
+                                              4*(dw))
+#define _CNL_PORT_TX_DW_LN0(port, dw)  (_PICK((port), \
+                                              _CNL_PORT_TX_AE_LN0_OFFSET, \
+                                              _CNL_PORT_TX_B_LN0_OFFSET, \
+                                              _CNL_PORT_TX_B_LN0_OFFSET, \
+                                              _CNL_PORT_TX_D_LN0_OFFSET, \
+                                              _CNL_PORT_TX_AE_LN0_OFFSET, \
+                                              _CNL_PORT_TX_F_LN0_OFFSET) + \
+                                              4*(dw))
+
+#define CNL_PORT_TX_DW2_GRP(port)      _MMIO(_CNL_PORT_TX_DW_GRP((port), 2))
+#define CNL_PORT_TX_DW2_LN0(port)      _MMIO(_CNL_PORT_TX_DW_LN0((port), 2))
+#define _ICL_PORT_TX_DW2_GRP_A         0x162688
+#define _ICL_PORT_TX_DW2_GRP_B         0x6C688
+#define _ICL_PORT_TX_DW2_LN0_A         0x162888
+#define _ICL_PORT_TX_DW2_LN0_B         0x6C888
+#define ICL_PORT_TX_DW2_GRP(port)      _MMIO_PORT(port, \
+                                                  _ICL_PORT_TX_DW2_GRP_A, \
+                                                  _ICL_PORT_TX_DW2_GRP_B)
+#define ICL_PORT_TX_DW2_LN0(port)      _MMIO_PORT(port, \
+                                                  _ICL_PORT_TX_DW2_LN0_A, \
+                                                  _ICL_PORT_TX_DW2_LN0_B)
+#define   SWING_SEL_UPPER(x)           (((x) >> 3) << 15)
 #define   SWING_SEL_UPPER_MASK         (1 << 15)
-#define   SWING_SEL_LOWER(x)           ((x & 0x7) << 11)
+#define   SWING_SEL_LOWER(x)           (((x) & 0x7) << 11)
 #define   SWING_SEL_LOWER_MASK         (0x7 << 11)
 #define   RCOMP_SCALAR(x)              ((x) << 0)
 #define   RCOMP_SCALAR_MASK            (0xFF << 0)
 
-#define _CNL_PORT_TX_DW4_GRP_AE                0x162350
-#define _CNL_PORT_TX_DW4_GRP_B         0x1623D0
-#define _CNL_PORT_TX_DW4_GRP_C         0x162B50
-#define _CNL_PORT_TX_DW4_GRP_D         0x162BD0
-#define _CNL_PORT_TX_DW4_GRP_F         0x162A50
 #define _CNL_PORT_TX_DW4_LN0_AE                0x162450
 #define _CNL_PORT_TX_DW4_LN1_AE                0x1624D0
-#define _CNL_PORT_TX_DW4_LN0_B         0x162650
-#define _CNL_PORT_TX_DW4_LN0_C         0x162C50
-#define _CNL_PORT_TX_DW4_LN0_D         0x162E50
-#define _CNL_PORT_TX_DW4_LN0_F         0x162850
-#define CNL_PORT_TX_DW4_GRP(port)       _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW4_GRP_AE, \
-                                                   _CNL_PORT_TX_DW4_GRP_B, \
-                                                   _CNL_PORT_TX_DW4_GRP_C, \
-                                                   _CNL_PORT_TX_DW4_GRP_D, \
-                                                   _CNL_PORT_TX_DW4_GRP_AE, \
-                                                   _CNL_PORT_TX_DW4_GRP_F)
-#define CNL_PORT_TX_DW4_LN(port, ln)       _MMIO_PORT6_LN(port, ln,    \
-                                                   _CNL_PORT_TX_DW4_LN0_AE, \
-                                                   _CNL_PORT_TX_DW4_LN1_AE, \
-                                                   _CNL_PORT_TX_DW4_LN0_B, \
-                                                   _CNL_PORT_TX_DW4_LN0_C, \
-                                                   _CNL_PORT_TX_DW4_LN0_D, \
-                                                   _CNL_PORT_TX_DW4_LN0_AE, \
-                                                   _CNL_PORT_TX_DW4_LN0_F)
+#define CNL_PORT_TX_DW4_GRP(port)      _MMIO(_CNL_PORT_TX_DW_GRP((port), 4))
+#define CNL_PORT_TX_DW4_LN0(port)      _MMIO(_CNL_PORT_TX_DW_LN0((port), 4))
+#define CNL_PORT_TX_DW4_LN(port, ln)   _MMIO(_CNL_PORT_TX_DW_LN0((port), 4) + \
+                                            (ln * (_CNL_PORT_TX_DW4_LN1_AE - \
+                                                   _CNL_PORT_TX_DW4_LN0_AE)))
+#define _ICL_PORT_TX_DW4_GRP_A         0x162690
+#define _ICL_PORT_TX_DW4_GRP_B         0x6C690
+#define _ICL_PORT_TX_DW4_LN0_A         0x162890
+#define _ICL_PORT_TX_DW4_LN1_A         0x162990
+#define _ICL_PORT_TX_DW4_LN0_B         0x6C890
+#define ICL_PORT_TX_DW4_GRP(port)      _MMIO_PORT(port, \
+                                                  _ICL_PORT_TX_DW4_GRP_A, \
+                                                  _ICL_PORT_TX_DW4_GRP_B)
+#define ICL_PORT_TX_DW4_LN(port, ln)   _MMIO(_PORT(port, \
+                                                  _ICL_PORT_TX_DW4_LN0_A, \
+                                                  _ICL_PORT_TX_DW4_LN0_B) + \
+                                             (ln * (_ICL_PORT_TX_DW4_LN1_A - \
+                                                    _ICL_PORT_TX_DW4_LN0_A)))
 #define   LOADGEN_SELECT               (1 << 31)
 #define   POST_CURSOR_1(x)             ((x) << 12)
 #define   POST_CURSOR_1_MASK           (0x3F << 12)
@@ -2029,64 +1802,147 @@ enum i915_power_well_id {
 #define   CURSOR_COEFF(x)              ((x) << 0)
 #define   CURSOR_COEFF_MASK            (0x3F << 0)
 
-#define _CNL_PORT_TX_DW5_GRP_AE                0x162354
-#define _CNL_PORT_TX_DW5_GRP_B         0x1623D4
-#define _CNL_PORT_TX_DW5_GRP_C         0x162B54
-#define _CNL_PORT_TX_DW5_GRP_D         0x162BD4
-#define _CNL_PORT_TX_DW5_GRP_F         0x162A54
-#define _CNL_PORT_TX_DW5_LN0_AE                0x162454
-#define _CNL_PORT_TX_DW5_LN0_B         0x162654
-#define _CNL_PORT_TX_DW5_LN0_C         0x162C54
-#define _CNL_PORT_TX_DW5_LN0_D         0x162E54
-#define _CNL_PORT_TX_DW5_LN0_F         0x162854
-#define CNL_PORT_TX_DW5_GRP(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW5_GRP_AE, \
-                                                   _CNL_PORT_TX_DW5_GRP_B, \
-                                                   _CNL_PORT_TX_DW5_GRP_C, \
-                                                   _CNL_PORT_TX_DW5_GRP_D, \
-                                                   _CNL_PORT_TX_DW5_GRP_AE, \
-                                                   _CNL_PORT_TX_DW5_GRP_F)
-#define CNL_PORT_TX_DW5_LN0(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW5_LN0_AE, \
-                                                   _CNL_PORT_TX_DW5_LN0_B, \
-                                                   _CNL_PORT_TX_DW5_LN0_C, \
-                                                   _CNL_PORT_TX_DW5_LN0_D, \
-                                                   _CNL_PORT_TX_DW5_LN0_AE, \
-                                                   _CNL_PORT_TX_DW5_LN0_F)
+#define CNL_PORT_TX_DW5_GRP(port)      _MMIO(_CNL_PORT_TX_DW_GRP((port), 5))
+#define CNL_PORT_TX_DW5_LN0(port)      _MMIO(_CNL_PORT_TX_DW_LN0((port), 5))
+#define _ICL_PORT_TX_DW5_GRP_A         0x162694
+#define _ICL_PORT_TX_DW5_GRP_B         0x6C694
+#define _ICL_PORT_TX_DW5_LN0_A         0x162894
+#define _ICL_PORT_TX_DW5_LN0_B         0x6C894
+#define ICL_PORT_TX_DW5_GRP(port)      _MMIO_PORT(port, \
+                                                  _ICL_PORT_TX_DW5_GRP_A, \
+                                                  _ICL_PORT_TX_DW5_GRP_B)
+#define ICL_PORT_TX_DW5_LN0(port)      _MMIO_PORT(port, \
+                                                  _ICL_PORT_TX_DW5_LN0_A, \
+                                                  _ICL_PORT_TX_DW5_LN0_B)
 #define   TX_TRAINING_EN               (1 << 31)
+#define   TAP2_DISABLE                 (1 << 30)
 #define   TAP3_DISABLE                 (1 << 29)
 #define   SCALING_MODE_SEL(x)          ((x) << 18)
 #define   SCALING_MODE_SEL_MASK                (0x7 << 18)
 #define   RTERM_SELECT(x)              ((x) << 3)
 #define   RTERM_SELECT_MASK            (0x7 << 3)
 
-#define _CNL_PORT_TX_DW7_GRP_AE                0x16235C
-#define _CNL_PORT_TX_DW7_GRP_B         0x1623DC
-#define _CNL_PORT_TX_DW7_GRP_C         0x162B5C
-#define _CNL_PORT_TX_DW7_GRP_D         0x162BDC
-#define _CNL_PORT_TX_DW7_GRP_F         0x162A5C
-#define _CNL_PORT_TX_DW7_LN0_AE                0x16245C
-#define _CNL_PORT_TX_DW7_LN0_B         0x16265C
-#define _CNL_PORT_TX_DW7_LN0_C         0x162C5C
-#define _CNL_PORT_TX_DW7_LN0_D         0x162E5C
-#define _CNL_PORT_TX_DW7_LN0_F         0x16285C
-#define CNL_PORT_TX_DW7_GRP(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW7_GRP_AE, \
-                                                   _CNL_PORT_TX_DW7_GRP_B, \
-                                                   _CNL_PORT_TX_DW7_GRP_C, \
-                                                   _CNL_PORT_TX_DW7_GRP_D, \
-                                                   _CNL_PORT_TX_DW7_GRP_AE, \
-                                                   _CNL_PORT_TX_DW7_GRP_F)
-#define CNL_PORT_TX_DW7_LN0(port)      _MMIO_PORT6(port, \
-                                                   _CNL_PORT_TX_DW7_LN0_AE, \
-                                                   _CNL_PORT_TX_DW7_LN0_B, \
-                                                   _CNL_PORT_TX_DW7_LN0_C, \
-                                                   _CNL_PORT_TX_DW7_LN0_D, \
-                                                   _CNL_PORT_TX_DW7_LN0_AE, \
-                                                   _CNL_PORT_TX_DW7_LN0_F)
+#define CNL_PORT_TX_DW7_GRP(port)      _MMIO(_CNL_PORT_TX_DW_GRP((port), 7))
+#define CNL_PORT_TX_DW7_LN0(port)      _MMIO(_CNL_PORT_TX_DW_LN0((port), 7))
 #define   N_SCALAR(x)                  ((x) << 24)
 #define   N_SCALAR_MASK                        (0x7F << 24)
 
+#define _ICL_MG_PHY_PORT_LN(port, ln, ln0p1, ln0p2, ln1p1) \
+       _MMIO(_PORT((port) - PORT_C, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1)))
+
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT1            0x16812C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT1            0x16852C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT2            0x16912C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT2            0x16952C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT3            0x16A12C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT3            0x16A52C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT4            0x16B12C
+#define _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT4            0x16B52C
+#define ICL_PORT_MG_TX1_LINK_PARAMS(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT1, \
+                                     _ICL_MG_TX_LINK_PARAMS_TX1LN0_PORT2, \
+                                     _ICL_MG_TX_LINK_PARAMS_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT1            0x1680AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT1            0x1684AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT2            0x1690AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT2            0x1694AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT3            0x16A0AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT3            0x16A4AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT4            0x16B0AC
+#define _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT4            0x16B4AC
+#define ICL_PORT_MG_TX2_LINK_PARAMS(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT1, \
+                                     _ICL_MG_TX_LINK_PARAMS_TX2LN0_PORT2, \
+                                     _ICL_MG_TX_LINK_PARAMS_TX2LN1_PORT1)
+#define CRI_USE_FS32                   (1 << 5)
+
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT1          0x16814C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT1          0x16854C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT2          0x16914C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT2          0x16954C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT3          0x16A14C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT3          0x16A54C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT4          0x16B14C
+#define _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT4          0x16B54C
+#define ICL_PORT_MG_TX1_PISO_READLOAD(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT1, \
+                                     _ICL_MG_TX_PISO_READLOAD_TX1LN0_PORT2, \
+                                     _ICL_MG_TX_PISO_READLOAD_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT1          0x1680CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT1          0x1684CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT2          0x1690CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT2          0x1694CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT3          0x16A0CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT3          0x16A4CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT4          0x16B0CC
+#define _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT4          0x16B4CC
+#define ICL_PORT_MG_TX2_PISO_READLOAD(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT1, \
+                                     _ICL_MG_TX_PISO_READLOAD_TX2LN0_PORT2, \
+                                     _ICL_MG_TX_PISO_READLOAD_TX2LN1_PORT1)
+#define CRI_CALCINIT                                   (1 << 1)
+
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT1              0x168148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT1              0x168548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT2              0x169148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT2              0x169548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT3              0x16A148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT3              0x16A548
+#define _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT4              0x16B148
+#define _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT4              0x16B548
+#define ICL_PORT_MG_TX1_SWINGCTRL(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT1, \
+                                     _ICL_MG_TX_SWINGCTRL_TX1LN0_PORT2, \
+                                     _ICL_MG_TX_SWINGCTRL_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT1              0x1680C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT1              0x1684C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT2              0x1690C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT2              0x1694C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT3              0x16A0C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT3              0x16A4C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT4              0x16B0C8
+#define _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT4              0x16B4C8
+#define ICL_PORT_MG_TX2_SWINGCTRL(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT1, \
+                                     _ICL_MG_TX_SWINGCTRL_TX2LN0_PORT2, \
+                                     _ICL_MG_TX_SWINGCTRL_TX2LN1_PORT1)
+#define CRI_TXDEEMPH_OVERRIDE_17_12(x)                 ((x) << 0)
+#define CRI_TXDEEMPH_OVERRIDE_17_12_MASK               (0x3F << 0)
+
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT1                        0x168144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT1                        0x168544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT2                        0x169144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT2                        0x169544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT3                        0x16A144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT3                        0x16A544
+#define _ICL_MG_TX_DRVCTRL_TX1LN0_PORT4                        0x16B144
+#define _ICL_MG_TX_DRVCTRL_TX1LN1_PORT4                        0x16B544
+#define ICL_PORT_MG_TX1_DRVCTRL(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_DRVCTRL_TX1LN0_PORT1, \
+                                     _ICL_MG_TX_DRVCTRL_TX1LN0_PORT2, \
+                                     _ICL_MG_TX_DRVCTRL_TX1LN1_PORT1)
+
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT1                        0x1680C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT1                        0x1684C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT2                        0x1690C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT2                        0x1694C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT3                        0x16A0C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT3                        0x16A4C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN0_PORT4                        0x16B0C4
+#define _ICL_MG_TX_DRVCTRL_TX2LN1_PORT4                        0x16B4C4
+#define ICL_PORT_MG_TX2_DRVCTRL(port, ln) \
+       _ICL_MG_PHY_PORT_LN(port, ln, _ICL_MG_TX_DRVCTRL_TX2LN0_PORT1, \
+                                     _ICL_MG_TX_DRVCTRL_TX2LN0_PORT2, \
+                                     _ICL_MG_TX_DRVCTRL_TX2LN1_PORT1)
+#define CRI_TXDEEMPH_OVERRIDE_11_6(x)                  ((x) << 24)
+#define CRI_TXDEEMPH_OVERRIDE_11_6_MASK                        (0x3F << 24)
+#define CRI_TXDEEMPH_OVERRIDE_EN                       (1 << 22)
+#define CRI_TXDEEMPH_OVERRIDE_5_0(x)                   ((x) << 16)
+#define CRI_TXDEEMPH_OVERRIDE_5_0_MASK                 (0x3F << 16)
+
 /* The spec defines this only for BXT PHY0, but lets assume that this
  * would exist for PHY1 too if it had a second channel.
  */
@@ -2473,6 +2329,10 @@ enum i915_power_well_id {
 #define   GEN8_MCR_SLICE_MASK          GEN8_MCR_SLICE(3)
 #define   GEN8_MCR_SUBSLICE(subslice)  (((subslice) & 3) << 24)
 #define   GEN8_MCR_SUBSLICE_MASK       GEN8_MCR_SUBSLICE(3)
+#define   GEN11_MCR_SLICE(slice)       (((slice) & 0xf) << 27)
+#define   GEN11_MCR_SLICE_MASK         GEN11_MCR_SLICE(0xf)
+#define   GEN11_MCR_SUBSLICE(subslice) (((subslice) & 0x7) << 24)
+#define   GEN11_MCR_SUBSLICE_MASK      GEN11_MCR_SUBSLICE(0x7)
 #define RING_IPEIR(base)       _MMIO((base)+0x64)
 #define RING_IPEHR(base)       _MMIO((base)+0x68)
 /*
@@ -2867,6 +2727,19 @@ enum i915_power_well_id {
 #define GEN10_EU_DISABLE3              _MMIO(0x9140)
 #define   GEN10_EU_DIS_SS_MASK         0xff
 
+#define GEN11_GT_VEBOX_VDBOX_DISABLE   _MMIO(0x9140)
+#define   GEN11_GT_VDBOX_DISABLE_MASK  0xff
+#define   GEN11_GT_VEBOX_DISABLE_SHIFT 16
+#define   GEN11_GT_VEBOX_DISABLE_MASK  (0xff << GEN11_GT_VEBOX_DISABLE_SHIFT)
+
+#define GEN11_EU_DISABLE _MMIO(0x9134)
+#define GEN11_EU_DIS_MASK 0xFF
+
+#define GEN11_GT_SLICE_ENABLE _MMIO(0x9138)
+#define GEN11_GT_S_ENA_MASK 0xFF
+
+#define GEN11_GT_SUBSLICE_DISABLE _MMIO(0x913C)
+
 #define GEN6_BSD_SLEEP_PSMI_CONTROL    _MMIO(0x12050)
 #define   GEN6_BSD_SLEEP_MSG_DISABLE   (1 << 0)
 #define   GEN6_BSD_SLEEP_FLUSH_DISABLE (1 << 2)
@@ -3951,6 +3824,9 @@ enum {
 #define _CLKGATE_DIS_PSL_A             0x46520
 #define _CLKGATE_DIS_PSL_B             0x46524
 #define _CLKGATE_DIS_PSL_C             0x46528
+#define   DUPS1_GATING_DIS             (1 << 15)
+#define   DUPS2_GATING_DIS             (1 << 19)
+#define   DUPS3_GATING_DIS             (1 << 23)
 #define   DPF_GATING_DIS               (1 << 10)
 #define   DPF_RAM_GATING_DIS           (1 << 9)
 #define   DPFR_GATING_DIS              (1 << 8)
@@ -4150,7 +4026,20 @@ enum {
 #define   EDP_PSR_TP1_TIME_0us                 (3<<4)
 #define   EDP_PSR_IDLE_FRAME_SHIFT             0
 
+/* Bspec claims those aren't shifted but stay at 0x64800 */
+#define EDP_PSR_IMR                            _MMIO(0x64834)
+#define EDP_PSR_IIR                            _MMIO(0x64838)
+#define   EDP_PSR_ERROR(trans)                 (1 << (((trans) * 8 + 10) & 31))
+#define   EDP_PSR_POST_EXIT(trans)             (1 << (((trans) * 8 + 9) & 31))
+#define   EDP_PSR_PRE_ENTRY(trans)             (1 << (((trans) * 8 + 8) & 31))
+
 #define EDP_PSR_AUX_CTL                                _MMIO(dev_priv->psr_mmio_base + 0x10)
+#define   EDP_PSR_AUX_CTL_TIME_OUT_MASK                (3 << 26)
+#define   EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK    (0x1f << 20)
+#define   EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK   (0xf << 16)
+#define   EDP_PSR_AUX_CTL_ERROR_INTERRUPT      (1 << 11)
+#define   EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK    (0x7ff)
+
 #define EDP_PSR_AUX_DATA(i)                    _MMIO(dev_priv->psr_mmio_base + 0x14 + (i) * 4) /* 5 registers */
 
 #define EDP_PSR_STATUS                         _MMIO(dev_priv->psr_mmio_base + 0x40)
@@ -4180,17 +4069,19 @@ enum {
 #define EDP_PSR_PERF_CNT               _MMIO(dev_priv->psr_mmio_base + 0x44)
 #define   EDP_PSR_PERF_CNT_MASK                0xffffff
 
-#define EDP_PSR_DEBUG                          _MMIO(dev_priv->psr_mmio_base + 0x60)
+#define EDP_PSR_DEBUG                          _MMIO(dev_priv->psr_mmio_base + 0x60) /* PSR_MASK on SKL+ */
 #define   EDP_PSR_DEBUG_MASK_MAX_SLEEP         (1<<28)
 #define   EDP_PSR_DEBUG_MASK_LPSP              (1<<27)
 #define   EDP_PSR_DEBUG_MASK_MEMUP             (1<<26)
 #define   EDP_PSR_DEBUG_MASK_HPD               (1<<25)
 #define   EDP_PSR_DEBUG_MASK_DISP_REG_WRITE    (1<<16)
-#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1<<15)
+#define   EDP_PSR_DEBUG_EXIT_ON_PIXEL_UNDERRUN (1<<15) /* SKL+ */
 
 #define EDP_PSR2_CTL                   _MMIO(0x6f900)
 #define   EDP_PSR2_ENABLE              (1<<31)
 #define   EDP_SU_TRACK_ENABLE          (1<<30)
+#define   EDP_Y_COORDINATE_VALID       (1<<26) /* GLK and CNL+ */
+#define   EDP_Y_COORDINATE_ENABLE      (1<<25) /* GLK and CNL+ */
 #define   EDP_MAX_SU_DISABLE_TIME(t)   ((t)<<20)
 #define   EDP_MAX_SU_DISABLE_TIME_MASK (0x1f<<20)
 #define   EDP_PSR2_TP2_TIME_500                (0<<8)
@@ -4200,8 +4091,32 @@ enum {
 #define   EDP_PSR2_TP2_TIME_MASK       (3<<8)
 #define   EDP_PSR2_FRAME_BEFORE_SU_SHIFT 4
 #define   EDP_PSR2_FRAME_BEFORE_SU_MASK        (0xf<<4)
-#define   EDP_PSR2_IDLE_MASK           0xf
 #define   EDP_PSR2_FRAME_BEFORE_SU(a)  ((a)<<4)
+#define   EDP_PSR2_IDLE_FRAME_MASK     0xf
+#define   EDP_PSR2_IDLE_FRAME_SHIFT    0
+
+#define _PSR_EVENT_TRANS_A                     0x60848
+#define _PSR_EVENT_TRANS_B                     0x61848
+#define _PSR_EVENT_TRANS_C                     0x62848
+#define _PSR_EVENT_TRANS_D                     0x63848
+#define _PSR_EVENT_TRANS_EDP                   0x6F848
+#define PSR_EVENT(trans)                       _MMIO_TRANS2(trans, _PSR_EVENT_TRANS_A)
+#define  PSR_EVENT_PSR2_WD_TIMER_EXPIRE                (1 << 17)
+#define  PSR_EVENT_PSR2_DISABLED               (1 << 16)
+#define  PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN      (1 << 15)
+#define  PSR_EVENT_SU_CRC_FIFO_UNDERRUN                (1 << 14)
+#define  PSR_EVENT_GRAPHICS_RESET              (1 << 12)
+#define  PSR_EVENT_PCH_INTERRUPT               (1 << 11)
+#define  PSR_EVENT_MEMORY_UP                   (1 << 10)
+#define  PSR_EVENT_FRONT_BUFFER_MODIFY         (1 << 9)
+#define  PSR_EVENT_WD_TIMER_EXPIRE             (1 << 8)
+#define  PSR_EVENT_PIPE_REGISTERS_UPDATE       (1 << 6)
+#define  PSR_EVENT_REGISTER_UPDATE             (1 << 5)
+#define  PSR_EVENT_HDCP_ENABLE                 (1 << 4)
+#define  PSR_EVENT_KVMR_SESSION_ENABLE         (1 << 3)
+#define  PSR_EVENT_VBI_ENABLE                  (1 << 2)
+#define  PSR_EVENT_LPSP_MODE_EXIT              (1 << 1)
+#define  PSR_EVENT_PSR_DISABLE                 (1 << 0)
 
 #define EDP_PSR2_STATUS                        _MMIO(0x6f940)
 #define EDP_PSR2_STATUS_STATE_MASK     (0xf<<28)
@@ -5265,8 +5180,6 @@ enum {
 #define   DP_LINK_TRAIN_OFF            (3 << 28)
 #define   DP_LINK_TRAIN_MASK           (3 << 28)
 #define   DP_LINK_TRAIN_SHIFT          28
-#define   DP_LINK_TRAIN_PAT_3_CHV      (1 << 14)
-#define   DP_LINK_TRAIN_MASK_CHV       ((3 << 28)|(1<<14))
 
 /* CPT Link training mode */
 #def