1a88c1972416ddc99fafba74b5d86513a154b703
[muen/linux.git] / drivers / gpu / drm / i915 / gvt / scheduler.c
1 /*
2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  *
23  * Authors:
24  *    Zhi Wang <zhi.a.wang@intel.com>
25  *
26  * Contributors:
27  *    Ping Gao <ping.a.gao@intel.com>
28  *    Tina Zhang <tina.zhang@intel.com>
29  *    Chanbin Du <changbin.du@intel.com>
30  *    Min He <min.he@intel.com>
31  *    Bing Niu <bing.niu@intel.com>
32  *    Zhenyu Wang <zhenyuw@linux.intel.com>
33  *
34  */
35
36 #include <linux/kthread.h>
37
38 #include "i915_drv.h"
39 #include "gvt.h"
40
41 #define RING_CTX_OFF(x) \
42         offsetof(struct execlist_ring_context, x)
43
44 static void set_context_pdp_root_pointer(
45                 struct execlist_ring_context *ring_context,
46                 u32 pdp[8])
47 {
48         int i;
49
50         for (i = 0; i < 8; i++)
51                 ring_context->pdps[i].val = pdp[7 - i];
52 }
53
54 static void update_shadow_pdps(struct intel_vgpu_workload *workload)
55 {
56         struct drm_i915_gem_object *ctx_obj =
57                 workload->req->hw_context->state->obj;
58         struct execlist_ring_context *shadow_ring_context;
59         struct page *page;
60
61         if (WARN_ON(!workload->shadow_mm))
62                 return;
63
64         if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount)))
65                 return;
66
67         page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
68         shadow_ring_context = kmap(page);
69         set_context_pdp_root_pointer(shadow_ring_context,
70                         (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);
71         kunmap(page);
72 }
73
74 /*
75  * when populating shadow ctx from guest, we should not overrride oa related
76  * registers, so that they will not be overlapped by guest oa configs. Thus
77  * made it possible to capture oa data from host for both host and guests.
78  */
79 static void sr_oa_regs(struct intel_vgpu_workload *workload,
80                 u32 *reg_state, bool save)
81 {
82         struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
83         u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
84         u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
85         int i = 0;
86         u32 flex_mmio[] = {
87                 i915_mmio_reg_offset(EU_PERF_CNTL0),
88                 i915_mmio_reg_offset(EU_PERF_CNTL1),
89                 i915_mmio_reg_offset(EU_PERF_CNTL2),
90                 i915_mmio_reg_offset(EU_PERF_CNTL3),
91                 i915_mmio_reg_offset(EU_PERF_CNTL4),
92                 i915_mmio_reg_offset(EU_PERF_CNTL5),
93                 i915_mmio_reg_offset(EU_PERF_CNTL6),
94         };
95
96         if (workload->ring_id != RCS)
97                 return;
98
99         if (save) {
100                 workload->oactxctrl = reg_state[ctx_oactxctrl + 1];
101
102                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
103                         u32 state_offset = ctx_flexeu0 + i * 2;
104
105                         workload->flex_mmio[i] = reg_state[state_offset + 1];
106                 }
107         } else {
108                 reg_state[ctx_oactxctrl] =
109                         i915_mmio_reg_offset(GEN8_OACTXCONTROL);
110                 reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;
111
112                 for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
113                         u32 state_offset = ctx_flexeu0 + i * 2;
114                         u32 mmio = flex_mmio[i];
115
116                         reg_state[state_offset] = mmio;
117                         reg_state[state_offset + 1] = workload->flex_mmio[i];
118                 }
119         }
120 }
121
122 static int populate_shadow_context(struct intel_vgpu_workload *workload)
123 {
124         struct intel_vgpu *vgpu = workload->vgpu;
125         struct intel_gvt *gvt = vgpu->gvt;
126         int ring_id = workload->ring_id;
127         struct drm_i915_gem_object *ctx_obj =
128                 workload->req->hw_context->state->obj;
129         struct execlist_ring_context *shadow_ring_context;
130         struct page *page;
131         void *dst;
132         unsigned long context_gpa, context_page_num;
133         int i;
134
135         page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
136         shadow_ring_context = kmap(page);
137
138         sr_oa_regs(workload, (u32 *)shadow_ring_context, true);
139 #define COPY_REG(name) \
140         intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
141                 + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
142 #define COPY_REG_MASKED(name) {\
143                 intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
144                                               + RING_CTX_OFF(name.val),\
145                                               &shadow_ring_context->name.val, 4);\
146                 shadow_ring_context->name.val |= 0xffff << 16;\
147         }
148
149         COPY_REG_MASKED(ctx_ctrl);
150         COPY_REG(ctx_timestamp);
151
152         if (ring_id == RCS) {
153                 COPY_REG(bb_per_ctx_ptr);
154                 COPY_REG(rcs_indirect_ctx);
155                 COPY_REG(rcs_indirect_ctx_offset);
156         }
157 #undef COPY_REG
158 #undef COPY_REG_MASKED
159
160         intel_gvt_hypervisor_read_gpa(vgpu,
161                         workload->ring_context_gpa +
162                         sizeof(*shadow_ring_context),
163                         (void *)shadow_ring_context +
164                         sizeof(*shadow_ring_context),
165                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
166
167         sr_oa_regs(workload, (u32 *)shadow_ring_context, false);
168         kunmap(page);
169
170         if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val))
171                 return 0;
172
173         gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
174                         workload->ctx_desc.lrca);
175
176         context_page_num = gvt->dev_priv->engine[ring_id]->context_size;
177
178         context_page_num = context_page_num >> PAGE_SHIFT;
179
180         if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
181                 context_page_num = 19;
182
183         i = 2;
184         while (i < context_page_num) {
185                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
186                                 (u32)((workload->ctx_desc.lrca + i) <<
187                                 I915_GTT_PAGE_SHIFT));
188                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
189                         gvt_vgpu_err("Invalid guest context descriptor\n");
190                         return -EFAULT;
191                 }
192
193                 page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
194                 dst = kmap(page);
195                 intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
196                                 I915_GTT_PAGE_SIZE);
197                 kunmap(page);
198                 i++;
199         }
200         return 0;
201 }
202
203 static inline bool is_gvt_request(struct i915_request *req)
204 {
205         return i915_gem_context_force_single_submission(req->gem_context);
206 }
207
208 static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id)
209 {
210         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
211         u32 ring_base = dev_priv->engine[ring_id]->mmio_base;
212         i915_reg_t reg;
213
214         reg = RING_INSTDONE(ring_base);
215         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
216         reg = RING_ACTHD(ring_base);
217         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
218         reg = RING_ACTHD_UDW(ring_base);
219         vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
220 }
221
222 static int shadow_context_status_change(struct notifier_block *nb,
223                 unsigned long action, void *data)
224 {
225         struct i915_request *req = data;
226         struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
227                                 shadow_ctx_notifier_block[req->engine->id]);
228         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
229         enum intel_engine_id ring_id = req->engine->id;
230         struct intel_vgpu_workload *workload;
231         unsigned long flags;
232
233         if (!is_gvt_request(req)) {
234                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
235                 if (action == INTEL_CONTEXT_SCHEDULE_IN &&
236                     scheduler->engine_owner[ring_id]) {
237                         /* Switch ring from vGPU to host. */
238                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
239                                               NULL, ring_id);
240                         scheduler->engine_owner[ring_id] = NULL;
241                 }
242                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
243
244                 return NOTIFY_OK;
245         }
246
247         workload = scheduler->current_workload[ring_id];
248         if (unlikely(!workload))
249                 return NOTIFY_OK;
250
251         switch (action) {
252         case INTEL_CONTEXT_SCHEDULE_IN:
253                 spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
254                 if (workload->vgpu != scheduler->engine_owner[ring_id]) {
255                         /* Switch ring from host to vGPU or vGPU to vGPU. */
256                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
257                                               workload->vgpu, ring_id);
258                         scheduler->engine_owner[ring_id] = workload->vgpu;
259                 } else
260                         gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
261                                       ring_id, workload->vgpu->id);
262                 spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
263                 atomic_set(&workload->shadow_ctx_active, 1);
264                 break;
265         case INTEL_CONTEXT_SCHEDULE_OUT:
266                 save_ring_hw_state(workload->vgpu, ring_id);
267                 atomic_set(&workload->shadow_ctx_active, 0);
268                 break;
269         case INTEL_CONTEXT_SCHEDULE_PREEMPTED:
270                 save_ring_hw_state(workload->vgpu, ring_id);
271                 break;
272         default:
273                 WARN_ON(1);
274                 return NOTIFY_OK;
275         }
276         wake_up(&workload->shadow_ctx_status_wq);
277         return NOTIFY_OK;
278 }
279
280 static void shadow_context_descriptor_update(struct intel_context *ce)
281 {
282         u64 desc = 0;
283
284         desc = ce->lrc_desc;
285
286         /* Update bits 0-11 of the context descriptor which includes flags
287          * like GEN8_CTX_* cached in desc_template
288          */
289         desc &= U64_MAX << 12;
290         desc |= ce->gem_context->desc_template & ((1ULL << 12) - 1);
291
292         ce->lrc_desc = desc;
293 }
294
295 static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
296 {
297         struct intel_vgpu *vgpu = workload->vgpu;
298         struct i915_request *req = workload->req;
299         void *shadow_ring_buffer_va;
300         u32 *cs;
301
302         if ((IS_KABYLAKE(req->i915) || IS_BROXTON(req->i915))
303                 && is_inhibit_context(req->hw_context))
304                 intel_vgpu_restore_inhibit_context(vgpu, req);
305
306         /* allocate shadow ring buffer */
307         cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
308         if (IS_ERR(cs)) {
309                 gvt_vgpu_err("fail to alloc size =%ld shadow  ring buffer\n",
310                         workload->rb_len);
311                 return PTR_ERR(cs);
312         }
313
314         shadow_ring_buffer_va = workload->shadow_ring_buffer_va;
315
316         /* get shadow ring buffer va */
317         workload->shadow_ring_buffer_va = cs;
318
319         memcpy(cs, shadow_ring_buffer_va,
320                         workload->rb_len);
321
322         cs += workload->rb_len / sizeof(u32);
323         intel_ring_advance(workload->req, cs);
324
325         return 0;
326 }
327
328 static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
329 {
330         if (!wa_ctx->indirect_ctx.obj)
331                 return;
332
333         i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
334         i915_gem_object_put(wa_ctx->indirect_ctx.obj);
335 }
336
337 /**
338  * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
339  * shadow it as well, include ringbuffer,wa_ctx and ctx.
340  * @workload: an abstract entity for each execlist submission.
341  *
342  * This function is called before the workload submitting to i915, to make
343  * sure the content of the workload is valid.
344  */
345 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
346 {
347         struct intel_vgpu *vgpu = workload->vgpu;
348         struct intel_vgpu_submission *s = &vgpu->submission;
349         struct i915_gem_context *shadow_ctx = s->shadow_ctx;
350         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
351         struct intel_engine_cs *engine = dev_priv->engine[workload->ring_id];
352         struct intel_context *ce;
353         struct i915_request *rq;
354         int ret;
355
356         lockdep_assert_held(&dev_priv->drm.struct_mutex);
357
358         if (workload->req)
359                 return 0;
360
361         /* pin shadow context by gvt even the shadow context will be pinned
362          * when i915 alloc request. That is because gvt will update the guest
363          * context from shadow context when workload is completed, and at that
364          * moment, i915 may already unpined the shadow context to make the
365          * shadow_ctx pages invalid. So gvt need to pin itself. After update
366          * the guest context, gvt can unpin the shadow_ctx safely.
367          */
368         ce = intel_context_pin(shadow_ctx, engine);
369         if (IS_ERR(ce)) {
370                 gvt_vgpu_err("fail to pin shadow context\n");
371                 return PTR_ERR(ce);
372         }
373
374         shadow_ctx->desc_template &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT);
375         shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode <<
376                                     GEN8_CTX_ADDRESSING_MODE_SHIFT;
377
378         if (!test_and_set_bit(workload->ring_id, s->shadow_ctx_desc_updated))
379                 shadow_context_descriptor_update(ce);
380
381         ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
382         if (ret)
383                 goto err_unpin;
384
385         if ((workload->ring_id == RCS) &&
386             (workload->wa_ctx.indirect_ctx.size != 0)) {
387                 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
388                 if (ret)
389                         goto err_shadow;
390         }
391
392         rq = i915_request_alloc(engine, shadow_ctx);
393         if (IS_ERR(rq)) {
394                 gvt_vgpu_err("fail to allocate gem request\n");
395                 ret = PTR_ERR(rq);
396                 goto err_shadow;
397         }
398         workload->req = i915_request_get(rq);
399
400         ret = populate_shadow_context(workload);
401         if (ret)
402                 goto err_req;
403
404         return 0;
405 err_req:
406         rq = fetch_and_zero(&workload->req);
407         i915_request_put(rq);
408 err_shadow:
409         release_shadow_wa_ctx(&workload->wa_ctx);
410 err_unpin:
411         intel_context_unpin(ce);
412         return ret;
413 }
414
415 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
416
417 static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
418 {
419         struct intel_gvt *gvt = workload->vgpu->gvt;
420         const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
421         struct intel_vgpu_shadow_bb *bb;
422         int ret;
423
424         list_for_each_entry(bb, &workload->shadow_bb, list) {
425                 /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
426                  * is only updated into ring_scan_buffer, not real ring address
427                  * allocated in later copy_workload_to_ring_buffer. pls be noted
428                  * shadow_ring_buffer_va is now pointed to real ring buffer va
429                  * in copy_workload_to_ring_buffer.
430                  */
431
432                 if (bb->bb_offset)
433                         bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
434                                 + bb->bb_offset;
435
436                 if (bb->ppgtt) {
437                         /* for non-priv bb, scan&shadow is only for
438                          * debugging purpose, so the content of shadow bb
439                          * is the same as original bb. Therefore,
440                          * here, rather than switch to shadow bb's gma
441                          * address, we directly use original batch buffer's
442                          * gma address, and send original bb to hardware
443                          * directly
444                          */
445                         if (bb->clflush & CLFLUSH_AFTER) {
446                                 drm_clflush_virt_range(bb->va,
447                                                 bb->obj->base.size);
448                                 bb->clflush &= ~CLFLUSH_AFTER;
449                         }
450                         i915_gem_obj_finish_shmem_access(bb->obj);
451                         bb->accessing = false;
452
453                 } else {
454                         bb->vma = i915_gem_object_ggtt_pin(bb->obj,
455                                         NULL, 0, 0, 0);
456                         if (IS_ERR(bb->vma)) {
457                                 ret = PTR_ERR(bb->vma);
458                                 goto err;
459                         }
460
461                         /* relocate shadow batch buffer */
462                         bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
463                         if (gmadr_bytes == 8)
464                                 bb->bb_start_cmd_va[2] = 0;
465
466                         /* No one is going to touch shadow bb from now on. */
467                         if (bb->clflush & CLFLUSH_AFTER) {
468                                 drm_clflush_virt_range(bb->va,
469                                                 bb->obj->base.size);
470                                 bb->clflush &= ~CLFLUSH_AFTER;
471                         }
472
473                         ret = i915_gem_object_set_to_gtt_domain(bb->obj,
474                                         false);
475                         if (ret)
476                                 goto err;
477
478                         i915_gem_obj_finish_shmem_access(bb->obj);
479                         bb->accessing = false;
480
481                         ret = i915_vma_move_to_active(bb->vma,
482                                                       workload->req,
483                                                       0);
484                         if (ret)
485                                 goto err;
486                 }
487         }
488         return 0;
489 err:
490         release_shadow_batch_buffer(workload);
491         return ret;
492 }
493
494 static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
495 {
496         struct intel_vgpu_workload *workload =
497                 container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx);
498         struct i915_request *rq = workload->req;
499         struct execlist_ring_context *shadow_ring_context =
500                 (struct execlist_ring_context *)rq->hw_context->lrc_reg_state;
501
502         shadow_ring_context->bb_per_ctx_ptr.val =
503                 (shadow_ring_context->bb_per_ctx_ptr.val &
504                 (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
505         shadow_ring_context->rcs_indirect_ctx.val =
506                 (shadow_ring_context->rcs_indirect_ctx.val &
507                 (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
508 }
509
510 static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
511 {
512         struct i915_vma *vma;
513         unsigned char *per_ctx_va =
514                 (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
515                 wa_ctx->indirect_ctx.size;
516
517         if (wa_ctx->indirect_ctx.size == 0)
518                 return 0;
519
520         vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
521                                        0, CACHELINE_BYTES, 0);
522         if (IS_ERR(vma))
523                 return PTR_ERR(vma);
524
525         /* FIXME: we are not tracking our pinned VMA leaving it
526          * up to the core to fix up the stray pin_count upon
527          * free.
528          */
529
530         wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
531
532         wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
533         memset(per_ctx_va, 0, CACHELINE_BYTES);
534
535         update_wa_ctx_2_shadow_ctx(wa_ctx);
536         return 0;
537 }
538
539 static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
540 {
541         struct intel_vgpu *vgpu = workload->vgpu;
542         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
543         struct intel_vgpu_shadow_bb *bb, *pos;
544
545         if (list_empty(&workload->shadow_bb))
546                 return;
547
548         bb = list_first_entry(&workload->shadow_bb,
549                         struct intel_vgpu_shadow_bb, list);
550
551         mutex_lock(&dev_priv->drm.struct_mutex);
552
553         list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
554                 if (bb->obj) {
555                         if (bb->accessing)
556                                 i915_gem_obj_finish_shmem_access(bb->obj);
557
558                         if (bb->va && !IS_ERR(bb->va))
559                                 i915_gem_object_unpin_map(bb->obj);
560
561                         if (bb->vma && !IS_ERR(bb->vma)) {
562                                 i915_vma_unpin(bb->vma);
563                                 i915_vma_close(bb->vma);
564                         }
565                         __i915_gem_object_release_unless_active(bb->obj);
566                 }
567                 list_del(&bb->list);
568                 kfree(bb);
569         }
570
571         mutex_unlock(&dev_priv->drm.struct_mutex);
572 }
573
574 static int prepare_workload(struct intel_vgpu_workload *workload)
575 {
576         struct intel_vgpu *vgpu = workload->vgpu;
577         int ret = 0;
578
579         ret = intel_vgpu_pin_mm(workload->shadow_mm);
580         if (ret) {
581                 gvt_vgpu_err("fail to vgpu pin mm\n");
582                 return ret;
583         }
584
585         update_shadow_pdps(workload);
586
587         ret = intel_vgpu_sync_oos_pages(workload->vgpu);
588         if (ret) {
589                 gvt_vgpu_err("fail to vgpu sync oos pages\n");
590                 goto err_unpin_mm;
591         }
592
593         ret = intel_vgpu_flush_post_shadow(workload->vgpu);
594         if (ret) {
595                 gvt_vgpu_err("fail to flush post shadow\n");
596                 goto err_unpin_mm;
597         }
598
599         ret = copy_workload_to_ring_buffer(workload);
600         if (ret) {
601                 gvt_vgpu_err("fail to generate request\n");
602                 goto err_unpin_mm;
603         }
604
605         ret = prepare_shadow_batch_buffer(workload);
606         if (ret) {
607                 gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
608                 goto err_unpin_mm;
609         }
610
611         ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
612         if (ret) {
613                 gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
614                 goto err_shadow_batch;
615         }
616
617         if (workload->prepare) {
618                 ret = workload->prepare(workload);
619                 if (ret)
620                         goto err_shadow_wa_ctx;
621         }
622
623         return 0;
624 err_shadow_wa_ctx:
625         release_shadow_wa_ctx(&workload->wa_ctx);
626 err_shadow_batch:
627         release_shadow_batch_buffer(workload);
628 err_unpin_mm:
629         intel_vgpu_unpin_mm(workload->shadow_mm);
630         return ret;
631 }
632
633 static int dispatch_workload(struct intel_vgpu_workload *workload)
634 {
635         struct intel_vgpu *vgpu = workload->vgpu;
636         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
637         int ring_id = workload->ring_id;
638         int ret;
639
640         gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
641                 ring_id, workload);
642
643         mutex_lock(&vgpu->vgpu_lock);
644         mutex_lock(&dev_priv->drm.struct_mutex);
645
646         ret = intel_gvt_scan_and_shadow_workload(workload);
647         if (ret)
648                 goto out;
649
650         ret = prepare_workload(workload);
651
652 out:
653         if (ret)
654                 workload->status = ret;
655
656         if (!IS_ERR_OR_NULL(workload->req)) {
657                 gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
658                                 ring_id, workload->req);
659                 i915_request_add(workload->req);
660                 workload->dispatched = true;
661         }
662
663         mutex_unlock(&dev_priv->drm.struct_mutex);
664         mutex_unlock(&vgpu->vgpu_lock);
665         return ret;
666 }
667
668 static struct intel_vgpu_workload *pick_next_workload(
669                 struct intel_gvt *gvt, int ring_id)
670 {
671         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
672         struct intel_vgpu_workload *workload = NULL;
673
674         mutex_lock(&gvt->sched_lock);
675
676         /*
677          * no current vgpu / will be scheduled out / no workload
678          * bail out
679          */
680         if (!scheduler->current_vgpu) {
681                 gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id);
682                 goto out;
683         }
684
685         if (scheduler->need_reschedule) {
686                 gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id);
687                 goto out;
688         }
689
690         if (list_empty(workload_q_head(scheduler->current_vgpu, ring_id)))
691                 goto out;
692
693         /*
694          * still have current workload, maybe the workload disptacher
695          * fail to submit it for some reason, resubmit it.
696          */
697         if (scheduler->current_workload[ring_id]) {
698                 workload = scheduler->current_workload[ring_id];
699                 gvt_dbg_sched("ring id %d still have current workload %p\n",
700                                 ring_id, workload);
701                 goto out;
702         }
703
704         /*
705          * pick a workload as current workload
706          * once current workload is set, schedule policy routines
707          * will wait the current workload is finished when trying to
708          * schedule out a vgpu.
709          */
710         scheduler->current_workload[ring_id] = container_of(
711                         workload_q_head(scheduler->current_vgpu, ring_id)->next,
712                         struct intel_vgpu_workload, list);
713
714         workload = scheduler->current_workload[ring_id];
715
716         gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
717
718         atomic_inc(&workload->vgpu->submission.running_workload_num);
719 out:
720         mutex_unlock(&gvt->sched_lock);
721         return workload;
722 }
723
724 static void update_guest_context(struct intel_vgpu_workload *workload)
725 {
726         struct i915_request *rq = workload->req;
727         struct intel_vgpu *vgpu = workload->vgpu;
728         struct intel_gvt *gvt = vgpu->gvt;
729         struct drm_i915_gem_object *ctx_obj = rq->hw_context->state->obj;
730         struct execlist_ring_context *shadow_ring_context;
731         struct page *page;
732         void *src;
733         unsigned long context_gpa, context_page_num;
734         int i;
735
736         gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id,
737                       workload->ctx_desc.lrca);
738
739         context_page_num = rq->engine->context_size;
740         context_page_num = context_page_num >> PAGE_SHIFT;
741
742         if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS)
743                 context_page_num = 19;
744
745         i = 2;
746
747         while (i < context_page_num) {
748                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
749                                 (u32)((workload->ctx_desc.lrca + i) <<
750                                         I915_GTT_PAGE_SHIFT));
751                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
752                         gvt_vgpu_err("invalid guest context descriptor\n");
753                         return;
754                 }
755
756                 page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
757                 src = kmap(page);
758                 intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
759                                 I915_GTT_PAGE_SIZE);
760                 kunmap(page);
761                 i++;
762         }
763
764         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
765                 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
766
767         page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
768         shadow_ring_context = kmap(page);
769
770 #define COPY_REG(name) \
771         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
772                 RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
773
774         COPY_REG(ctx_ctrl);
775         COPY_REG(ctx_timestamp);
776
777 #undef COPY_REG
778
779         intel_gvt_hypervisor_write_gpa(vgpu,
780                         workload->ring_context_gpa +
781                         sizeof(*shadow_ring_context),
782                         (void *)shadow_ring_context +
783                         sizeof(*shadow_ring_context),
784                         I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
785
786         kunmap(page);
787 }
788
789 static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
790 {
791         struct intel_vgpu_submission *s = &vgpu->submission;
792         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
793         struct intel_engine_cs *engine;
794         struct intel_vgpu_workload *pos, *n;
795         unsigned int tmp;
796
797         /* free the unsubmited workloads in the queues. */
798         for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
799                 list_for_each_entry_safe(pos, n,
800                         &s->workload_q_head[engine->id], list) {
801                         list_del_init(&pos->list);
802                         intel_vgpu_destroy_workload(pos);
803                 }
804                 clear_bit(engine->id, s->shadow_ctx_desc_updated);
805         }
806 }
807
808 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
809 {
810         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
811         struct intel_vgpu_workload *workload =
812                 scheduler->current_workload[ring_id];
813         struct intel_vgpu *vgpu = workload->vgpu;
814         struct intel_vgpu_submission *s = &vgpu->submission;
815         struct i915_request *rq = workload->req;
816         int event;
817
818         mutex_lock(&vgpu->vgpu_lock);
819         mutex_lock(&gvt->sched_lock);
820
821         /* For the workload w/ request, needs to wait for the context
822          * switch to make sure request is completed.
823          * For the workload w/o request, directly complete the workload.
824          */
825         if (rq) {
826                 wait_event(workload->shadow_ctx_status_wq,
827                            !atomic_read(&workload->shadow_ctx_active));
828
829                 /* If this request caused GPU hang, req->fence.error will
830                  * be set to -EIO. Use -EIO to set workload status so
831                  * that when this request caused GPU hang, didn't trigger
832                  * context switch interrupt to guest.
833                  */
834                 if (likely(workload->status == -EINPROGRESS)) {
835                         if (workload->req->fence.error == -EIO)
836                                 workload->status = -EIO;
837                         else
838                                 workload->status = 0;
839                 }
840
841                 if (!workload->status && !(vgpu->resetting_eng &
842                                            ENGINE_MASK(ring_id))) {
843                         update_guest_context(workload);
844
845                         for_each_set_bit(event, workload->pending_events,
846                                          INTEL_GVT_EVENT_MAX)
847                                 intel_vgpu_trigger_virtual_event(vgpu, event);
848                 }
849
850                 /* unpin shadow ctx as the shadow_ctx update is done */
851                 mutex_lock(&rq->i915->drm.struct_mutex);
852                 intel_context_unpin(rq->hw_context);
853                 mutex_unlock(&rq->i915->drm.struct_mutex);
854
855                 i915_request_put(fetch_and_zero(&workload->req));
856         }
857
858         gvt_dbg_sched("ring id %d complete workload %p status %d\n",
859                         ring_id, workload, workload->status);
860
861         scheduler->current_workload[ring_id] = NULL;
862
863         list_del_init(&workload->list);
864
865         if (!workload->status) {
866                 release_shadow_batch_buffer(workload);
867                 release_shadow_wa_ctx(&workload->wa_ctx);
868         }
869
870         if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
871                 /* if workload->status is not successful means HW GPU
872                  * has occurred GPU hang or something wrong with i915/GVT,
873                  * and GVT won't inject context switch interrupt to guest.
874                  * So this error is a vGPU hang actually to the guest.
875                  * According to this we should emunlate a vGPU hang. If
876                  * there are pending workloads which are already submitted
877                  * from guest, we should clean them up like HW GPU does.
878                  *
879                  * if it is in middle of engine resetting, the pending
880                  * workloads won't be submitted to HW GPU and will be
881                  * cleaned up during the resetting process later, so doing
882                  * the workload clean up here doesn't have any impact.
883                  **/
884                 clean_workloads(vgpu, ENGINE_MASK(ring_id));
885         }
886
887         workload->complete(workload);
888
889         atomic_dec(&s->running_workload_num);
890         wake_up(&scheduler->workload_complete_wq);
891
892         if (gvt->scheduler.need_reschedule)
893                 intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
894
895         mutex_unlock(&gvt->sched_lock);
896         mutex_unlock(&vgpu->vgpu_lock);
897 }
898
899 struct workload_thread_param {
900         struct intel_gvt *gvt;
901         int ring_id;
902 };
903
904 static int workload_thread(void *priv)
905 {
906         struct workload_thread_param *p = (struct workload_thread_param *)priv;
907         struct intel_gvt *gvt = p->gvt;
908         int ring_id = p->ring_id;
909         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
910         struct intel_vgpu_workload *workload = NULL;
911         struct intel_vgpu *vgpu = NULL;
912         int ret;
913         bool need_force_wake = IS_SKYLAKE(gvt->dev_priv)
914                         || IS_KABYLAKE(gvt->dev_priv)
915                         || IS_BROXTON(gvt->dev_priv);
916         DEFINE_WAIT_FUNC(wait, woken_wake_function);
917
918         kfree(p);
919
920         gvt_dbg_core("workload thread for ring %d started\n", ring_id);
921
922         while (!kthread_should_stop()) {
923                 add_wait_queue(&scheduler->waitq[ring_id], &wait);
924                 do {
925                         workload = pick_next_workload(gvt, ring_id);
926                         if (workload)
927                                 break;
928                         wait_woken(&wait, TASK_INTERRUPTIBLE,
929                                    MAX_SCHEDULE_TIMEOUT);
930                 } while (!kthread_should_stop());
931                 remove_wait_queue(&scheduler->waitq[ring_id], &wait);
932
933                 if (!workload)
934                         break;
935
936                 gvt_dbg_sched("ring id %d next workload %p vgpu %d\n",
937                                 workload->ring_id, workload,
938                                 workload->vgpu->id);
939
940                 intel_runtime_pm_get(gvt->dev_priv);
941
942                 gvt_dbg_sched("ring id %d will dispatch workload %p\n",
943                                 workload->ring_id, workload);
944
945                 if (need_force_wake)
946                         intel_uncore_forcewake_get(gvt->dev_priv,
947                                         FORCEWAKE_ALL);
948
949                 ret = dispatch_workload(workload);
950
951                 if (ret) {
952                         vgpu = workload->vgpu;
953                         gvt_vgpu_err("fail to dispatch workload, skip\n");
954                         goto complete;
955                 }
956
957                 gvt_dbg_sched("ring id %d wait workload %p\n",
958                                 workload->ring_id, workload);
959                 i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
960
961 complete:
962                 gvt_dbg_sched("will complete workload %p, status: %d\n",
963                                 workload, workload->status);
964
965                 complete_current_workload(gvt, ring_id);
966
967                 if (need_force_wake)
968                         intel_uncore_forcewake_put(gvt->dev_priv,
969                                         FORCEWAKE_ALL);
970
971                 intel_runtime_pm_put(gvt->dev_priv);
972                 if (ret && (vgpu_is_vm_unhealthy(ret)))
973                         enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
974         }
975         return 0;
976 }
977
978 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
979 {
980         struct intel_vgpu_submission *s = &vgpu->submission;
981         struct intel_gvt *gvt = vgpu->gvt;
982         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
983
984         if (atomic_read(&s->running_workload_num)) {
985                 gvt_dbg_sched("wait vgpu idle\n");
986
987                 wait_event(scheduler->workload_complete_wq,
988                                 !atomic_read(&s->running_workload_num));
989         }
990 }
991
992 void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
993 {
994         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
995         struct intel_engine_cs *engine;
996         enum intel_engine_id i;
997
998         gvt_dbg_core("clean workload scheduler\n");
999
1000         for_each_engine(engine, gvt->dev_priv, i) {
1001                 atomic_notifier_chain_unregister(
1002                                         &engine->context_status_notifier,
1003                                         &gvt->shadow_ctx_notifier_block[i]);
1004                 kthread_stop(scheduler->thread[i]);
1005         }
1006 }
1007
1008 int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
1009 {
1010         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1011         struct workload_thread_param *param = NULL;
1012         struct intel_engine_cs *engine;
1013         enum intel_engine_id i;
1014         int ret;
1015
1016         gvt_dbg_core("init workload scheduler\n");
1017
1018         init_waitqueue_head(&scheduler->workload_complete_wq);
1019
1020         for_each_engine(engine, gvt->dev_priv, i) {
1021                 init_waitqueue_head(&scheduler->waitq[i]);
1022
1023                 param = kzalloc(sizeof(*param), GFP_KERNEL);
1024                 if (!param) {
1025                         ret = -ENOMEM;
1026                         goto err;
1027                 }
1028
1029                 param->gvt = gvt;
1030                 param->ring_id = i;
1031
1032                 scheduler->thread[i] = kthread_run(workload_thread, param,
1033                         "gvt workload %d", i);
1034                 if (IS_ERR(scheduler->thread[i])) {
1035                         gvt_err("fail to create workload thread\n");
1036                         ret = PTR_ERR(scheduler->thread[i]);
1037                         goto err;
1038                 }
1039
1040                 gvt->shadow_ctx_notifier_block[i].notifier_call =
1041                                         shadow_context_status_change;
1042                 atomic_notifier_chain_register(&engine->context_status_notifier,
1043                                         &gvt->shadow_ctx_notifier_block[i]);
1044         }
1045         return 0;
1046 err:
1047         intel_gvt_clean_workload_scheduler(gvt);
1048         kfree(param);
1049         param = NULL;
1050         return ret;
1051 }
1052
1053 /**
1054  * intel_vgpu_clean_submission - free submission-related resource for vGPU
1055  * @vgpu: a vGPU
1056  *
1057  * This function is called when a vGPU is being destroyed.
1058  *
1059  */
1060 void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
1061 {
1062         struct intel_vgpu_submission *s = &vgpu->submission;
1063
1064         intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
1065         i915_gem_context_put(s->shadow_ctx);
1066         kmem_cache_destroy(s->workloads);
1067 }
1068
1069
1070 /**
1071  * intel_vgpu_reset_submission - reset submission-related resource for vGPU
1072  * @vgpu: a vGPU
1073  * @engine_mask: engines expected to be reset
1074  *
1075  * This function is called when a vGPU is being destroyed.
1076  *
1077  */
1078 void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
1079                 unsigned long engine_mask)
1080 {
1081         struct intel_vgpu_submission *s = &vgpu->submission;
1082
1083         if (!s->active)
1084                 return;
1085
1086         clean_workloads(vgpu, engine_mask);
1087         s->ops->reset(vgpu, engine_mask);
1088 }
1089
1090 /**
1091  * intel_vgpu_setup_submission - setup submission-related resource for vGPU
1092  * @vgpu: a vGPU
1093  *
1094  * This function is called when a vGPU is being created.
1095  *
1096  * Returns:
1097  * Zero on success, negative error code if failed.
1098  *
1099  */
1100 int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
1101 {
1102         struct intel_vgpu_submission *s = &vgpu->submission;
1103         enum intel_engine_id i;
1104         struct intel_engine_cs *engine;
1105         int ret;
1106
1107         s->shadow_ctx = i915_gem_context_create_gvt(
1108                         &vgpu->gvt->dev_priv->drm);
1109         if (IS_ERR(s->shadow_ctx))
1110                 return PTR_ERR(s->shadow_ctx);
1111
1112         bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
1113
1114         s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
1115                                                   sizeof(struct intel_vgpu_workload), 0,
1116                                                   SLAB_HWCACHE_ALIGN,
1117                                                   offsetof(struct intel_vgpu_workload, rb_tail),
1118                                                   sizeof_field(struct intel_vgpu_workload, rb_tail),
1119                                                   NULL);
1120
1121         if (!s->workloads) {
1122                 ret = -ENOMEM;
1123                 goto out_shadow_ctx;
1124         }
1125
1126         for_each_engine(engine, vgpu->gvt->dev_priv, i)
1127                 INIT_LIST_HEAD(&s->workload_q_head[i]);
1128
1129         atomic_set(&s->running_workload_num, 0);
1130         bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
1131
1132         return 0;
1133
1134 out_shadow_ctx:
1135         i915_gem_context_put(s->shadow_ctx);
1136         return ret;
1137 }
1138
1139 /**
1140  * intel_vgpu_select_submission_ops - select virtual submission interface
1141  * @vgpu: a vGPU
1142  * @engine_mask: either ALL_ENGINES or target engine mask
1143  * @interface: expected vGPU virtual submission interface
1144  *
1145  * This function is called when guest configures submission interface.
1146  *
1147  * Returns:
1148  * Zero on success, negative error code if failed.
1149  *
1150  */
1151 int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
1152                                      unsigned long engine_mask,
1153                                      unsigned int interface)
1154 {
1155         struct intel_vgpu_submission *s = &vgpu->submission;
1156         const struct intel_vgpu_submission_ops *ops[] = {
1157                 [INTEL_VGPU_EXECLIST_SUBMISSION] =
1158                         &intel_vgpu_execlist_submission_ops,
1159         };
1160         int ret;
1161
1162         if (WARN_ON(interface >= ARRAY_SIZE(ops)))
1163                 return -EINVAL;
1164
1165         if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES))
1166                 return -EINVAL;
1167
1168         if (s->active)
1169                 s->ops->clean(vgpu, engine_mask);
1170
1171         if (interface == 0) {
1172                 s->ops = NULL;
1173                 s->virtual_submission_interface = 0;
1174                 s->active = false;
1175                 gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id);
1176                 return 0;
1177         }
1178
1179         ret = ops[interface]->init(vgpu, engine_mask);
1180         if (ret)
1181                 return ret;
1182
1183         s->ops = ops[interface];
1184         s->virtual_submission_interface = interface;
1185         s->active = true;
1186
1187         gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
1188                         vgpu->id, s->ops->name);
1189
1190         return 0;
1191 }
1192
1193 /**
1194  * intel_vgpu_destroy_workload - destroy a vGPU workload
1195  * @workload: workload to destroy
1196  *
1197  * This function is called when destroy a vGPU workload.
1198  *
1199  */
1200 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
1201 {
1202         struct intel_vgpu_submission *s = &workload->vgpu->submission;
1203
1204         if (workload->shadow_mm)
1205                 intel_vgpu_mm_put(workload->shadow_mm);
1206
1207         kmem_cache_free(s->workloads, workload);
1208 }
1209
1210 static struct intel_vgpu_workload *
1211 alloc_workload(struct intel_vgpu *vgpu)
1212 {
1213         struct intel_vgpu_submission *s = &vgpu->submission;
1214         struct intel_vgpu_workload *workload;
1215
1216         workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
1217         if (!workload)
1218                 return ERR_PTR(-ENOMEM);
1219
1220         INIT_LIST_HEAD(&workload->list);
1221         INIT_LIST_HEAD(&workload->shadow_bb);
1222
1223         init_waitqueue_head(&workload->shadow_ctx_status_wq);
1224         atomic_set(&workload->shadow_ctx_active, 0);
1225
1226         workload->status = -EINPROGRESS;
1227         workload->vgpu = vgpu;
1228
1229         return workload;
1230 }
1231
1232 #define RING_CTX_OFF(x) \
1233         offsetof(struct execlist_ring_context, x)
1234
1235 static void read_guest_pdps(struct intel_vgpu *vgpu,
1236                 u64 ring_context_gpa, u32 pdp[8])
1237 {
1238         u64 gpa;
1239         int i;
1240
1241         gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
1242
1243         for (i = 0; i < 8; i++)
1244                 intel_gvt_hypervisor_read_gpa(vgpu,
1245                                 gpa + i * 8, &pdp[7 - i], 4);
1246 }
1247
1248 static int prepare_mm(struct intel_vgpu_workload *workload)
1249 {
1250         struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
1251         struct intel_vgpu_mm *mm;
1252         struct intel_vgpu *vgpu = workload->vgpu;
1253         intel_gvt_gtt_type_t root_entry_type;
1254         u64 pdps[GVT_RING_CTX_NR_PDPS];
1255
1256         switch (desc->addressing_mode) {
1257         case 1: /* legacy 32-bit */
1258                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1259                 break;
1260         case 3: /* legacy 64-bit */
1261                 root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1262                 break;
1263         default:
1264                 gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
1265                 return -EINVAL;
1266         }
1267
1268         read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);
1269
1270         mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);
1271         if (IS_ERR(mm))
1272                 return PTR_ERR(mm);
1273
1274         workload->shadow_mm = mm;
1275         return 0;
1276 }
1277
1278 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
1279                 ((a)->lrca == (b)->lrca))
1280
1281 #define get_last_workload(q) \
1282         (list_empty(q) ? NULL : container_of(q->prev, \
1283         struct intel_vgpu_workload, list))
1284 /**
1285  * intel_vgpu_create_workload - create a vGPU workload
1286  * @vgpu: a vGPU
1287  * @ring_id: ring index
1288  * @desc: a guest context descriptor
1289  *
1290  * This function is called when creating a vGPU workload.
1291  *
1292  * Returns:
1293  * struct intel_vgpu_workload * on success, negative error code in
1294  * pointer if failed.
1295  *
1296  */
1297 struct intel_vgpu_workload *
1298 intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
1299                            struct execlist_ctx_descriptor_format *desc)
1300 {
1301         struct intel_vgpu_submission *s = &vgpu->submission;
1302         struct list_head *q = workload_q_head(vgpu, ring_id);
1303         struct intel_vgpu_workload *last_workload = get_last_workload(q);
1304         struct intel_vgpu_workload *workload = NULL;
1305         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
1306         u64 ring_context_gpa;
1307         u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
1308         int ret;
1309
1310         ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
1311                         (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
1312         if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
1313                 gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
1314                 return ERR_PTR(-EINVAL);
1315         }
1316
1317         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1318                         RING_CTX_OFF(ring_header.val), &head, 4);
1319
1320         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1321                         RING_CTX_OFF(ring_tail.val), &tail, 4);
1322
1323         head &= RB_HEAD_OFF_MASK;
1324         tail &= RB_TAIL_OFF_MASK;
1325
1326         if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
1327                 gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
1328                 gvt_dbg_el("ctx head %x real head %lx\n", head,
1329                                 last_workload->rb_tail);
1330                 /*
1331                  * cannot use guest context head pointer here,
1332                  * as it might not be updated at this time
1333                  */
1334                 head = last_workload->rb_tail;
1335         }
1336
1337         gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
1338
1339         /* record some ring buffer register values for scan and shadow */
1340         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1341                         RING_CTX_OFF(rb_start.val), &start, 4);
1342         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1343                         RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
1344         intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1345                         RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
1346
1347         workload = alloc_workload(vgpu);
1348         if (IS_ERR(workload))
1349                 return workload;
1350
1351         workload->ring_id = ring_id;
1352         workload->ctx_desc = *desc;
1353         workload->ring_context_gpa = ring_context_gpa;
1354         workload->rb_head = head;
1355         workload->rb_tail = tail;
1356         workload->rb_start = start;
1357         workload->rb_ctl = ctl;
1358
1359         if (ring_id == RCS) {
1360                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1361                         RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
1362                 intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
1363                         RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
1364
1365                 workload->wa_ctx.indirect_ctx.guest_gma =
1366                         indirect_ctx & INDIRECT_CTX_ADDR_MASK;
1367                 workload->wa_ctx.indirect_ctx.size =
1368                         (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
1369                         CACHELINE_BYTES;
1370                 workload->wa_ctx.per_ctx.guest_gma =
1371                         per_ctx & PER_CTX_ADDR_MASK;
1372                 workload->wa_ctx.per_ctx.valid = per_ctx & 1;
1373         }
1374
1375         gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
1376                         workload, ring_id, head, tail, start, ctl);
1377
1378         ret = prepare_mm(workload);
1379         if (ret) {
1380                 kmem_cache_free(s->workloads, workload);
1381                 return ERR_PTR(ret);
1382         }
1383
1384         /* Only scan and shadow the first workload in the queue
1385          * as there is only one pre-allocated buf-obj for shadow.
1386          */
1387         if (list_empty(workload_q_head(vgpu, ring_id))) {
1388                 intel_runtime_pm_get(dev_priv);
1389                 mutex_lock(&dev_priv->drm.struct_mutex);
1390                 ret = intel_gvt_scan_and_shadow_workload(workload);
1391                 mutex_unlock(&dev_priv->drm.struct_mutex);
1392                 intel_runtime_pm_put(dev_priv);
1393         }
1394
1395         if (ret && (vgpu_is_vm_unhealthy(ret))) {
1396                 enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1397                 intel_vgpu_destroy_workload(workload);
1398                 return ERR_PTR(ret);
1399         }
1400
1401         return workload;
1402 }
1403
1404 /**
1405  * intel_vgpu_queue_workload - Qeue a vGPU workload
1406  * @workload: the workload to queue in
1407  */
1408 void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)
1409 {
1410         list_add_tail(&workload->list,
1411                 workload_q_head(workload->vgpu, workload->ring_id));
1412         intel_gvt_kick_schedule(workload->vgpu->gvt);
1413         wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]);
1414 }