Merge tag 'drm-intel-next-2018-07-09' of git://anongit.freedesktop.org/drm/drm-intel...
[muen/linux.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
1 /*
2  * Copyright © 2010 Daniel Vetter
3  * Copyright © 2011-2014 Intel Corporation
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice (including the next
13  * paragraph) shall be included in all copies or substantial portions of the
14  * Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22  * IN THE SOFTWARE.
23  *
24  */
25
26 #include <linux/slab.h> /* fault-inject.h is not standalone! */
27
28 #include <linux/fault-inject.h>
29 #include <linux/log2.h>
30 #include <linux/random.h>
31 #include <linux/seq_file.h>
32 #include <linux/stop_machine.h>
33
34 #include <asm/set_memory.h>
35
36 #include <drm/drmP.h>
37 #include <drm/i915_drm.h>
38
39 #include "i915_drv.h"
40 #include "i915_vgpu.h"
41 #include "i915_trace.h"
42 #include "intel_drv.h"
43 #include "intel_frontbuffer.h"
44
45 #define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
46
47 /**
48  * DOC: Global GTT views
49  *
50  * Background and previous state
51  *
52  * Historically objects could exists (be bound) in global GTT space only as
53  * singular instances with a view representing all of the object's backing pages
54  * in a linear fashion. This view will be called a normal view.
55  *
56  * To support multiple views of the same object, where the number of mapped
57  * pages is not equal to the backing store, or where the layout of the pages
58  * is not linear, concept of a GGTT view was added.
59  *
60  * One example of an alternative view is a stereo display driven by a single
61  * image. In this case we would have a framebuffer looking like this
62  * (2x2 pages):
63  *
64  *    12
65  *    34
66  *
67  * Above would represent a normal GGTT view as normally mapped for GPU or CPU
68  * rendering. In contrast, fed to the display engine would be an alternative
69  * view which could look something like this:
70  *
71  *   1212
72  *   3434
73  *
74  * In this example both the size and layout of pages in the alternative view is
75  * different from the normal view.
76  *
77  * Implementation and usage
78  *
79  * GGTT views are implemented using VMAs and are distinguished via enum
80  * i915_ggtt_view_type and struct i915_ggtt_view.
81  *
82  * A new flavour of core GEM functions which work with GGTT bound objects were
83  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
84  * renaming  in large amounts of code. They take the struct i915_ggtt_view
85  * parameter encapsulating all metadata required to implement a view.
86  *
87  * As a helper for callers which are only interested in the normal view,
88  * globally const i915_ggtt_view_normal singleton instance exists. All old core
89  * GEM API functions, the ones not taking the view parameter, are operating on,
90  * or with the normal GGTT view.
91  *
92  * Code wanting to add or use a new GGTT view needs to:
93  *
94  * 1. Add a new enum with a suitable name.
95  * 2. Extend the metadata in the i915_ggtt_view structure if required.
96  * 3. Add support to i915_get_vma_pages().
97  *
98  * New views are required to build a scatter-gather table from within the
99  * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
100  * exists for the lifetime of an VMA.
101  *
102  * Core API is designed to have copy semantics which means that passed in
103  * struct i915_ggtt_view does not need to be persistent (left around after
104  * calling the core API functions).
105  *
106  */
107
108 static int
109 i915_get_ggtt_vma_pages(struct i915_vma *vma);
110
111 static void gen6_ggtt_invalidate(struct drm_i915_private *dev_priv)
112 {
113         /*
114          * Note that as an uncached mmio write, this will flush the
115          * WCB of the writes into the GGTT before it triggers the invalidate.
116          */
117         I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
118 }
119
120 static void guc_ggtt_invalidate(struct drm_i915_private *dev_priv)
121 {
122         gen6_ggtt_invalidate(dev_priv);
123         I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
124 }
125
126 static void gmch_ggtt_invalidate(struct drm_i915_private *dev_priv)
127 {
128         intel_gtt_chipset_flush();
129 }
130
131 static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
132 {
133         i915->ggtt.invalidate(i915);
134 }
135
136 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
137                                 int enable_ppgtt)
138 {
139         bool has_full_ppgtt;
140         bool has_full_48bit_ppgtt;
141
142         if (!dev_priv->info.has_aliasing_ppgtt)
143                 return 0;
144
145         has_full_ppgtt = dev_priv->info.has_full_ppgtt;
146         has_full_48bit_ppgtt = dev_priv->info.has_full_48bit_ppgtt;
147
148         if (intel_vgpu_active(dev_priv)) {
149                 /* GVT-g has no support for 32bit ppgtt */
150                 has_full_ppgtt = false;
151                 has_full_48bit_ppgtt = intel_vgpu_has_full_48bit_ppgtt(dev_priv);
152         }
153
154         /*
155          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
156          * execlists, the sole mechanism available to submit work.
157          */
158         if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
159                 return 0;
160
161         if (enable_ppgtt == 1)
162                 return 1;
163
164         if (enable_ppgtt == 2 && has_full_ppgtt)
165                 return 2;
166
167         if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
168                 return 3;
169
170         /* Disable ppgtt on SNB if VT-d is on. */
171         if (IS_GEN6(dev_priv) && intel_vtd_active()) {
172                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
173                 return 0;
174         }
175
176         /* Early VLV doesn't have this */
177         if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
178                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
179                 return 0;
180         }
181
182         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
183                 if (has_full_48bit_ppgtt)
184                         return 3;
185
186                 if (has_full_ppgtt)
187                         return 2;
188         }
189
190         return 1;
191 }
192
193 static int ppgtt_bind_vma(struct i915_vma *vma,
194                           enum i915_cache_level cache_level,
195                           u32 unused)
196 {
197         u32 pte_flags;
198         int err;
199
200         if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
201                 err = vma->vm->allocate_va_range(vma->vm,
202                                                  vma->node.start, vma->size);
203                 if (err)
204                         return err;
205         }
206
207         /* Currently applicable only to VLV */
208         pte_flags = 0;
209         if (vma->obj->gt_ro)
210                 pte_flags |= PTE_READ_ONLY;
211
212         vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
213
214         return 0;
215 }
216
217 static void ppgtt_unbind_vma(struct i915_vma *vma)
218 {
219         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
220 }
221
222 static int ppgtt_set_pages(struct i915_vma *vma)
223 {
224         GEM_BUG_ON(vma->pages);
225
226         vma->pages = vma->obj->mm.pages;
227
228         vma->page_sizes = vma->obj->mm.page_sizes;
229
230         return 0;
231 }
232
233 static void clear_pages(struct i915_vma *vma)
234 {
235         GEM_BUG_ON(!vma->pages);
236
237         if (vma->pages != vma->obj->mm.pages) {
238                 sg_free_table(vma->pages);
239                 kfree(vma->pages);
240         }
241         vma->pages = NULL;
242
243         memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
244 }
245
246 static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
247                                   enum i915_cache_level level)
248 {
249         gen8_pte_t pte = _PAGE_PRESENT | _PAGE_RW;
250         pte |= addr;
251
252         switch (level) {
253         case I915_CACHE_NONE:
254                 pte |= PPAT_UNCACHED;
255                 break;
256         case I915_CACHE_WT:
257                 pte |= PPAT_DISPLAY_ELLC;
258                 break;
259         default:
260                 pte |= PPAT_CACHED;
261                 break;
262         }
263
264         return pte;
265 }
266
267 static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
268                                   const enum i915_cache_level level)
269 {
270         gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
271         pde |= addr;
272         if (level != I915_CACHE_NONE)
273                 pde |= PPAT_CACHED_PDE;
274         else
275                 pde |= PPAT_UNCACHED;
276         return pde;
277 }
278
279 #define gen8_pdpe_encode gen8_pde_encode
280 #define gen8_pml4e_encode gen8_pde_encode
281
282 static gen6_pte_t snb_pte_encode(dma_addr_t addr,
283                                  enum i915_cache_level level,
284                                  u32 unused)
285 {
286         gen6_pte_t pte = GEN6_PTE_VALID;
287         pte |= GEN6_PTE_ADDR_ENCODE(addr);
288
289         switch (level) {
290         case I915_CACHE_L3_LLC:
291         case I915_CACHE_LLC:
292                 pte |= GEN6_PTE_CACHE_LLC;
293                 break;
294         case I915_CACHE_NONE:
295                 pte |= GEN6_PTE_UNCACHED;
296                 break;
297         default:
298                 MISSING_CASE(level);
299         }
300
301         return pte;
302 }
303
304 static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
305                                  enum i915_cache_level level,
306                                  u32 unused)
307 {
308         gen6_pte_t pte = GEN6_PTE_VALID;
309         pte |= GEN6_PTE_ADDR_ENCODE(addr);
310
311         switch (level) {
312         case I915_CACHE_L3_LLC:
313                 pte |= GEN7_PTE_CACHE_L3_LLC;
314                 break;
315         case I915_CACHE_LLC:
316                 pte |= GEN6_PTE_CACHE_LLC;
317                 break;
318         case I915_CACHE_NONE:
319                 pte |= GEN6_PTE_UNCACHED;
320                 break;
321         default:
322                 MISSING_CASE(level);
323         }
324
325         return pte;
326 }
327
328 static gen6_pte_t byt_pte_encode(dma_addr_t addr,
329                                  enum i915_cache_level level,
330                                  u32 flags)
331 {
332         gen6_pte_t pte = GEN6_PTE_VALID;
333         pte |= GEN6_PTE_ADDR_ENCODE(addr);
334
335         if (!(flags & PTE_READ_ONLY))
336                 pte |= BYT_PTE_WRITEABLE;
337
338         if (level != I915_CACHE_NONE)
339                 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
340
341         return pte;
342 }
343
344 static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
345                                  enum i915_cache_level level,
346                                  u32 unused)
347 {
348         gen6_pte_t pte = GEN6_PTE_VALID;
349         pte |= HSW_PTE_ADDR_ENCODE(addr);
350
351         if (level != I915_CACHE_NONE)
352                 pte |= HSW_WB_LLC_AGE3;
353
354         return pte;
355 }
356
357 static gen6_pte_t iris_pte_encode(dma_addr_t addr,
358                                   enum i915_cache_level level,
359                                   u32 unused)
360 {
361         gen6_pte_t pte = GEN6_PTE_VALID;
362         pte |= HSW_PTE_ADDR_ENCODE(addr);
363
364         switch (level) {
365         case I915_CACHE_NONE:
366                 break;
367         case I915_CACHE_WT:
368                 pte |= HSW_WT_ELLC_LLC_AGE3;
369                 break;
370         default:
371                 pte |= HSW_WB_ELLC_LLC_AGE3;
372                 break;
373         }
374
375         return pte;
376 }
377
378 static void stash_init(struct pagestash *stash)
379 {
380         pagevec_init(&stash->pvec);
381         spin_lock_init(&stash->lock);
382 }
383
384 static struct page *stash_pop_page(struct pagestash *stash)
385 {
386         struct page *page = NULL;
387
388         spin_lock(&stash->lock);
389         if (likely(stash->pvec.nr))
390                 page = stash->pvec.pages[--stash->pvec.nr];
391         spin_unlock(&stash->lock);
392
393         return page;
394 }
395
396 static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec)
397 {
398         int nr;
399
400         spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING);
401
402         nr = min_t(int, pvec->nr, pagevec_space(&stash->pvec));
403         memcpy(stash->pvec.pages + stash->pvec.nr,
404                pvec->pages + pvec->nr - nr,
405                sizeof(pvec->pages[0]) * nr);
406         stash->pvec.nr += nr;
407
408         spin_unlock(&stash->lock);
409
410         pvec->nr -= nr;
411 }
412
413 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
414 {
415         struct pagevec stack;
416         struct page *page;
417
418         if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
419                 i915_gem_shrink_all(vm->i915);
420
421         page = stash_pop_page(&vm->free_pages);
422         if (page)
423                 return page;
424
425         if (!vm->pt_kmap_wc)
426                 return alloc_page(gfp);
427
428         /* Look in our global stash of WC pages... */
429         page = stash_pop_page(&vm->i915->mm.wc_stash);
430         if (page)
431                 return page;
432
433         /*
434          * Otherwise batch allocate pages to amortize cost of set_pages_wc.
435          *
436          * We have to be careful as page allocation may trigger the shrinker
437          * (via direct reclaim) which will fill up the WC stash underneath us.
438          * So we add our WB pages into a temporary pvec on the stack and merge
439          * them into the WC stash after all the allocations are complete.
440          */
441         pagevec_init(&stack);
442         do {
443                 struct page *page;
444
445                 page = alloc_page(gfp);
446                 if (unlikely(!page))
447                         break;
448
449                 stack.pages[stack.nr++] = page;
450         } while (pagevec_space(&stack));
451
452         if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) {
453                 page = stack.pages[--stack.nr];
454
455                 /* Merge spare WC pages to the global stash */
456                 stash_push_pagevec(&vm->i915->mm.wc_stash, &stack);
457
458                 /* Push any surplus WC pages onto the local VM stash */
459                 if (stack.nr)
460                         stash_push_pagevec(&vm->free_pages, &stack);
461         }
462
463         /* Return unwanted leftovers */
464         if (unlikely(stack.nr)) {
465                 WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr));
466                 __pagevec_release(&stack);
467         }
468
469         return page;
470 }
471
472 static void vm_free_pages_release(struct i915_address_space *vm,
473                                   bool immediate)
474 {
475         struct pagevec *pvec = &vm->free_pages.pvec;
476         struct pagevec stack;
477
478         lockdep_assert_held(&vm->free_pages.lock);
479         GEM_BUG_ON(!pagevec_count(pvec));
480
481         if (vm->pt_kmap_wc) {
482                 /*
483                  * When we use WC, first fill up the global stash and then
484                  * only if full immediately free the overflow.
485                  */
486                 stash_push_pagevec(&vm->i915->mm.wc_stash, pvec);
487
488                 /*
489                  * As we have made some room in the VM's free_pages,
490                  * we can wait for it to fill again. Unless we are
491                  * inside i915_address_space_fini() and must
492                  * immediately release the pages!
493                  */
494                 if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1))
495                         return;
496
497                 /*
498                  * We have to drop the lock to allow ourselves to sleep,
499                  * so take a copy of the pvec and clear the stash for
500                  * others to use it as we sleep.
501                  */
502                 stack = *pvec;
503                 pagevec_reinit(pvec);
504                 spin_unlock(&vm->free_pages.lock);
505
506                 pvec = &stack;
507                 set_pages_array_wb(pvec->pages, pvec->nr);
508
509                 spin_lock(&vm->free_pages.lock);
510         }
511
512         __pagevec_release(pvec);
513 }
514
515 static void vm_free_page(struct i915_address_space *vm, struct page *page)
516 {
517         /*
518          * On !llc, we need to change the pages back to WB. We only do so
519          * in bulk, so we rarely need to change the page attributes here,
520          * but doing so requires a stop_machine() from deep inside arch/x86/mm.
521          * To make detection of the possible sleep more likely, use an
522          * unconditional might_sleep() for everybody.
523          */
524         might_sleep();
525         spin_lock(&vm->free_pages.lock);
526         if (!pagevec_add(&vm->free_pages.pvec, page))
527                 vm_free_pages_release(vm, false);
528         spin_unlock(&vm->free_pages.lock);
529 }
530
531 static void i915_address_space_init(struct i915_address_space *vm,
532                                     struct drm_i915_private *dev_priv)
533 {
534         GEM_BUG_ON(!vm->total);
535         drm_mm_init(&vm->mm, 0, vm->total);
536         vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
537
538         stash_init(&vm->free_pages);
539
540         INIT_LIST_HEAD(&vm->active_list);
541         INIT_LIST_HEAD(&vm->inactive_list);
542         INIT_LIST_HEAD(&vm->unbound_list);
543 }
544
545 static void i915_address_space_fini(struct i915_address_space *vm)
546 {
547         spin_lock(&vm->free_pages.lock);
548         if (pagevec_count(&vm->free_pages.pvec))
549                 vm_free_pages_release(vm, true);
550         GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec));
551         spin_unlock(&vm->free_pages.lock);
552
553         drm_mm_takedown(&vm->mm);
554 }
555
556 static int __setup_page_dma(struct i915_address_space *vm,
557                             struct i915_page_dma *p,
558                             gfp_t gfp)
559 {
560         p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL);
561         if (unlikely(!p->page))
562                 return -ENOMEM;
563
564         p->daddr = dma_map_page_attrs(vm->dma,
565                                       p->page, 0, PAGE_SIZE,
566                                       PCI_DMA_BIDIRECTIONAL,
567                                       DMA_ATTR_SKIP_CPU_SYNC |
568                                       DMA_ATTR_NO_WARN);
569         if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
570                 vm_free_page(vm, p->page);
571                 return -ENOMEM;
572         }
573
574         return 0;
575 }
576
577 static int setup_page_dma(struct i915_address_space *vm,
578                           struct i915_page_dma *p)
579 {
580         return __setup_page_dma(vm, p, __GFP_HIGHMEM);
581 }
582
583 static void cleanup_page_dma(struct i915_address_space *vm,
584                              struct i915_page_dma *p)
585 {
586         dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
587         vm_free_page(vm, p->page);
588 }
589
590 #define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
591
592 #define setup_px(vm, px) setup_page_dma((vm), px_base(px))
593 #define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
594 #define fill_px(vm, px, v) fill_page_dma((vm), px_base(px), (v))
595 #define fill32_px(vm, px, v) fill_page_dma_32((vm), px_base(px), (v))
596
597 static void fill_page_dma(struct i915_address_space *vm,
598                           struct i915_page_dma *p,
599                           const u64 val)
600 {
601         u64 * const vaddr = kmap_atomic(p->page);
602
603         memset64(vaddr, val, PAGE_SIZE / sizeof(val));
604
605         kunmap_atomic(vaddr);
606 }
607
608 static void fill_page_dma_32(struct i915_address_space *vm,
609                              struct i915_page_dma *p,
610                              const u32 v)
611 {
612         fill_page_dma(vm, p, (u64)v << 32 | v);
613 }
614
615 static int
616 setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
617 {
618         unsigned long size;
619
620         /*
621          * In order to utilize 64K pages for an object with a size < 2M, we will
622          * need to support a 64K scratch page, given that every 16th entry for a
623          * page-table operating in 64K mode must point to a properly aligned 64K
624          * region, including any PTEs which happen to point to scratch.
625          *
626          * This is only relevant for the 48b PPGTT where we support
627          * huge-gtt-pages, see also i915_vma_insert().
628          *
629          * TODO: we should really consider write-protecting the scratch-page and
630          * sharing between ppgtt
631          */
632         size = I915_GTT_PAGE_SIZE_4K;
633         if (i915_vm_is_48bit(vm) &&
634             HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
635                 size = I915_GTT_PAGE_SIZE_64K;
636                 gfp |= __GFP_NOWARN;
637         }
638         gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;
639
640         do {
641                 int order = get_order(size);
642                 struct page *page;
643                 dma_addr_t addr;
644
645                 page = alloc_pages(gfp, order);
646                 if (unlikely(!page))
647                         goto skip;
648
649                 addr = dma_map_page_attrs(vm->dma,
650                                           page, 0, size,
651                                           PCI_DMA_BIDIRECTIONAL,
652                                           DMA_ATTR_SKIP_CPU_SYNC |
653                                           DMA_ATTR_NO_WARN);
654                 if (unlikely(dma_mapping_error(vm->dma, addr)))
655                         goto free_page;
656
657                 if (unlikely(!IS_ALIGNED(addr, size)))
658                         goto unmap_page;
659
660                 vm->scratch_page.page = page;
661                 vm->scratch_page.daddr = addr;
662                 vm->scratch_page.order = order;
663                 return 0;
664
665 unmap_page:
666                 dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL);
667 free_page:
668                 __free_pages(page, order);
669 skip:
670                 if (size == I915_GTT_PAGE_SIZE_4K)
671                         return -ENOMEM;
672
673                 size = I915_GTT_PAGE_SIZE_4K;
674                 gfp &= ~__GFP_NOWARN;
675         } while (1);
676 }
677
678 static void cleanup_scratch_page(struct i915_address_space *vm)
679 {
680         struct i915_page_dma *p = &vm->scratch_page;
681
682         dma_unmap_page(vm->dma, p->daddr, BIT(p->order) << PAGE_SHIFT,
683                        PCI_DMA_BIDIRECTIONAL);
684         __free_pages(p->page, p->order);
685 }
686
687 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
688 {
689         struct i915_page_table *pt;
690
691         pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL);
692         if (unlikely(!pt))
693                 return ERR_PTR(-ENOMEM);
694
695         if (unlikely(setup_px(vm, pt))) {
696                 kfree(pt);
697                 return ERR_PTR(-ENOMEM);
698         }
699
700         pt->used_ptes = 0;
701         return pt;
702 }
703
704 static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
705 {
706         cleanup_px(vm, pt);
707         kfree(pt);
708 }
709
710 static void gen8_initialize_pt(struct i915_address_space *vm,
711                                struct i915_page_table *pt)
712 {
713         fill_px(vm, pt,
714                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC));
715 }
716
717 static void gen6_initialize_pt(struct gen6_hw_ppgtt *ppgtt,
718                                struct i915_page_table *pt)
719 {
720         fill32_px(&ppgtt->base.vm, pt, ppgtt->scratch_pte);
721 }
722
723 static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
724 {
725         struct i915_page_directory *pd;
726
727         pd = kzalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
728         if (unlikely(!pd))
729                 return ERR_PTR(-ENOMEM);
730
731         if (unlikely(setup_px(vm, pd))) {
732                 kfree(pd);
733                 return ERR_PTR(-ENOMEM);
734         }
735
736         pd->used_pdes = 0;
737         return pd;
738 }
739
740 static void free_pd(struct i915_address_space *vm,
741                     struct i915_page_directory *pd)
742 {
743         cleanup_px(vm, pd);
744         kfree(pd);
745 }
746
747 static void gen8_initialize_pd(struct i915_address_space *vm,
748                                struct i915_page_directory *pd)
749 {
750         fill_px(vm, pd,
751                 gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC));
752         memset_p((void **)pd->page_table, vm->scratch_pt, I915_PDES);
753 }
754
755 static int __pdp_init(struct i915_address_space *vm,
756                       struct i915_page_directory_pointer *pdp)
757 {
758         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
759
760         pdp->page_directory = kmalloc_array(pdpes, sizeof(*pdp->page_directory),
761                                             I915_GFP_ALLOW_FAIL);
762         if (unlikely(!pdp->page_directory))
763                 return -ENOMEM;
764
765         memset_p((void **)pdp->page_directory, vm->scratch_pd, pdpes);
766
767         return 0;
768 }
769
770 static void __pdp_fini(struct i915_page_directory_pointer *pdp)
771 {
772         kfree(pdp->page_directory);
773         pdp->page_directory = NULL;
774 }
775
776 static inline bool use_4lvl(const struct i915_address_space *vm)
777 {
778         return i915_vm_is_48bit(vm);
779 }
780
781 static struct i915_page_directory_pointer *
782 alloc_pdp(struct i915_address_space *vm)
783 {
784         struct i915_page_directory_pointer *pdp;
785         int ret = -ENOMEM;
786
787         GEM_BUG_ON(!use_4lvl(vm));
788
789         pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
790         if (!pdp)
791                 return ERR_PTR(-ENOMEM);
792
793         ret = __pdp_init(vm, pdp);
794         if (ret)
795                 goto fail_bitmap;
796
797         ret = setup_px(vm, pdp);
798         if (ret)
799                 goto fail_page_m;
800
801         return pdp;
802
803 fail_page_m:
804         __pdp_fini(pdp);
805 fail_bitmap:
806         kfree(pdp);
807
808         return ERR_PTR(ret);
809 }
810
811 static void free_pdp(struct i915_address_space *vm,
812                      struct i915_page_directory_pointer *pdp)
813 {
814         __pdp_fini(pdp);
815
816         if (!use_4lvl(vm))
817                 return;
818
819         cleanup_px(vm, pdp);
820         kfree(pdp);
821 }
822
823 static void gen8_initialize_pdp(struct i915_address_space *vm,
824                                 struct i915_page_directory_pointer *pdp)
825 {
826         gen8_ppgtt_pdpe_t scratch_pdpe;
827
828         scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
829
830         fill_px(vm, pdp, scratch_pdpe);
831 }
832
833 static void gen8_initialize_pml4(struct i915_address_space *vm,
834                                  struct i915_pml4 *pml4)
835 {
836         fill_px(vm, pml4,
837                 gen8_pml4e_encode(px_dma(vm->scratch_pdp), I915_CACHE_LLC));
838         memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
839 }
840
841 /* PDE TLBs are a pain to invalidate on GEN8+. When we modify
842  * the page table structures, we mark them dirty so that
843  * context switching/execlist queuing code takes extra steps
844  * to ensure that tlbs are flushed.
845  */
846 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
847 {
848         ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->vm.i915)->ring_mask;
849 }
850
851 /* Removes entries from a single page table, releasing it if it's empty.
852  * Caller can use the return value to update higher-level entries.
853  */
854 static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm,
855                                 struct i915_page_table *pt,
856                                 u64 start, u64 length)
857 {
858         unsigned int num_entries = gen8_pte_count(start, length);
859         unsigned int pte = gen8_pte_index(start);
860         unsigned int pte_end = pte + num_entries;
861         const gen8_pte_t scratch_pte =
862                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
863         gen8_pte_t *vaddr;
864
865         GEM_BUG_ON(num_entries > pt->used_ptes);
866
867         pt->used_ptes -= num_entries;
868         if (!pt->used_ptes)
869                 return true;
870
871         vaddr = kmap_atomic_px(pt);
872         while (pte < pte_end)
873                 vaddr[pte++] = scratch_pte;
874         kunmap_atomic(vaddr);
875
876         return false;
877 }
878
879 static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
880                                struct i915_page_directory *pd,
881                                struct i915_page_table *pt,
882                                unsigned int pde)
883 {
884         gen8_pde_t *vaddr;
885
886         pd->page_table[pde] = pt;
887
888         vaddr = kmap_atomic_px(pd);
889         vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
890         kunmap_atomic(vaddr);
891 }
892
893 static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
894                                 struct i915_page_directory *pd,
895                                 u64 start, u64 length)
896 {
897         struct i915_page_table *pt;
898         u32 pde;
899
900         gen8_for_each_pde(pt, pd, start, length, pde) {
901                 GEM_BUG_ON(pt == vm->scratch_pt);
902
903                 if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
904                         continue;
905
906                 gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
907                 GEM_BUG_ON(!pd->used_pdes);
908                 pd->used_pdes--;
909
910                 free_pt(vm, pt);
911         }
912
913         return !pd->used_pdes;
914 }
915
916 static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
917                                 struct i915_page_directory_pointer *pdp,
918                                 struct i915_page_directory *pd,
919                                 unsigned int pdpe)
920 {
921         gen8_ppgtt_pdpe_t *vaddr;
922
923         pdp->page_directory[pdpe] = pd;
924         if (!use_4lvl(vm))
925                 return;
926
927         vaddr = kmap_atomic_px(pdp);
928         vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
929         kunmap_atomic(vaddr);
930 }
931
932 /* Removes entries from a single page dir pointer, releasing it if it's empty.
933  * Caller can use the return value to update higher-level entries
934  */
935 static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
936                                  struct i915_page_directory_pointer *pdp,
937                                  u64 start, u64 length)
938 {
939         struct i915_page_directory *pd;
940         unsigned int pdpe;
941
942         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
943                 GEM_BUG_ON(pd == vm->scratch_pd);
944
945                 if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
946                         continue;
947
948                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
949                 GEM_BUG_ON(!pdp->used_pdpes);
950                 pdp->used_pdpes--;
951
952                 free_pd(vm, pd);
953         }
954
955         return !pdp->used_pdpes;
956 }
957
958 static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
959                                   u64 start, u64 length)
960 {
961         gen8_ppgtt_clear_pdp(vm, &i915_vm_to_ppgtt(vm)->pdp, start, length);
962 }
963
964 static void gen8_ppgtt_set_pml4e(struct i915_pml4 *pml4,
965                                  struct i915_page_directory_pointer *pdp,
966                                  unsigned int pml4e)
967 {
968         gen8_ppgtt_pml4e_t *vaddr;
969
970         pml4->pdps[pml4e] = pdp;
971
972         vaddr = kmap_atomic_px(pml4);
973         vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
974         kunmap_atomic(vaddr);
975 }
976
977 /* Removes entries from a single pml4.
978  * This is the top-level structure in 4-level page tables used on gen8+.
979  * Empty entries are always scratch pml4e.
980  */
981 static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
982                                   u64 start, u64 length)
983 {
984         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
985         struct i915_pml4 *pml4 = &ppgtt->pml4;
986         struct i915_page_directory_pointer *pdp;
987         unsigned int pml4e;
988
989         GEM_BUG_ON(!use_4lvl(vm));
990
991         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
992                 GEM_BUG_ON(pdp == vm->scratch_pdp);
993
994                 if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
995                         continue;
996
997                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
998
999                 free_pdp(vm, pdp);
1000         }
1001 }
1002
1003 static inline struct sgt_dma {
1004         struct scatterlist *sg;
1005         dma_addr_t dma, max;
1006 } sgt_dma(struct i915_vma *vma) {
1007         struct scatterlist *sg = vma->pages->sgl;
1008         dma_addr_t addr = sg_dma_address(sg);
1009         return (struct sgt_dma) { sg, addr, addr + sg->length };
1010 }
1011
1012 struct gen8_insert_pte {
1013         u16 pml4e;
1014         u16 pdpe;
1015         u16 pde;
1016         u16 pte;
1017 };
1018
1019 static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
1020 {
1021         return (struct gen8_insert_pte) {
1022                  gen8_pml4e_index(start),
1023                  gen8_pdpe_index(start),
1024                  gen8_pde_index(start),
1025                  gen8_pte_index(start),
1026         };
1027 }
1028
1029 static __always_inline bool
1030 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
1031                               struct i915_page_directory_pointer *pdp,
1032                               struct sgt_dma *iter,
1033                               struct gen8_insert_pte *idx,
1034                               enum i915_cache_level cache_level)
1035 {
1036         struct i915_page_directory *pd;
1037         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1038         gen8_pte_t *vaddr;
1039         bool ret;
1040
1041         GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
1042         pd = pdp->page_directory[idx->pdpe];
1043         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1044         do {
1045                 vaddr[idx->pte] = pte_encode | iter->dma;
1046
1047                 iter->dma += PAGE_SIZE;
1048                 if (iter->dma >= iter->max) {
1049                         iter->sg = __sg_next(iter->sg);
1050                         if (!iter->sg) {
1051                                 ret = false;
1052                                 break;
1053                         }
1054
1055                         iter->dma = sg_dma_address(iter->sg);
1056                         iter->max = iter->dma + iter->sg->length;
1057                 }
1058
1059                 if (++idx->pte == GEN8_PTES) {
1060                         idx->pte = 0;
1061
1062                         if (++idx->pde == I915_PDES) {
1063                                 idx->pde = 0;
1064
1065                                 /* Limited by sg length for 3lvl */
1066                                 if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
1067                                         idx->pdpe = 0;
1068                                         ret = true;
1069                                         break;
1070                                 }
1071
1072                                 GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
1073                                 pd = pdp->page_directory[idx->pdpe];
1074                         }
1075
1076                         kunmap_atomic(vaddr);
1077                         vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
1078                 }
1079         } while (1);
1080         kunmap_atomic(vaddr);
1081
1082         return ret;
1083 }
1084
1085 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
1086                                    struct i915_vma *vma,
1087                                    enum i915_cache_level cache_level,
1088                                    u32 unused)
1089 {
1090         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1091         struct sgt_dma iter = sgt_dma(vma);
1092         struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1093
1094         gen8_ppgtt_insert_pte_entries(ppgtt, &ppgtt->pdp, &iter, &idx,
1095                                       cache_level);
1096
1097         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1098 }
1099
1100 static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
1101                                            struct i915_page_directory_pointer **pdps,
1102                                            struct sgt_dma *iter,
1103                                            enum i915_cache_level cache_level)
1104 {
1105         const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
1106         u64 start = vma->node.start;
1107         dma_addr_t rem = iter->sg->length;
1108
1109         do {
1110                 struct gen8_insert_pte idx = gen8_insert_pte(start);
1111                 struct i915_page_directory_pointer *pdp = pdps[idx.pml4e];
1112                 struct i915_page_directory *pd = pdp->page_directory[idx.pdpe];
1113                 unsigned int page_size;
1114                 bool maybe_64K = false;
1115                 gen8_pte_t encode = pte_encode;
1116                 gen8_pte_t *vaddr;
1117                 u16 index, max;
1118
1119                 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
1120                     IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
1121                     rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) {
1122                         index = idx.pde;
1123                         max = I915_PDES;
1124                         page_size = I915_GTT_PAGE_SIZE_2M;
1125
1126                         encode |= GEN8_PDE_PS_2M;
1127
1128                         vaddr = kmap_atomic_px(pd);
1129                 } else {
1130                         struct i915_page_table *pt = pd->page_table[idx.pde];
1131
1132                         index = idx.pte;
1133                         max = GEN8_PTES;
1134                         page_size = I915_GTT_PAGE_SIZE;
1135
1136                         if (!index &&
1137                             vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
1138                             IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1139                             (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1140                              rem >= (max - index) << PAGE_SHIFT))
1141                                 maybe_64K = true;
1142
1143                         vaddr = kmap_atomic_px(pt);
1144                 }
1145
1146                 do {
1147                         GEM_BUG_ON(iter->sg->length < page_size);
1148                         vaddr[index++] = encode | iter->dma;
1149
1150                         start += page_size;
1151                         iter->dma += page_size;
1152                         rem -= page_size;
1153                         if (iter->dma >= iter->max) {
1154                                 iter->sg = __sg_next(iter->sg);
1155                                 if (!iter->sg)
1156                                         break;
1157
1158                                 rem = iter->sg->length;
1159                                 iter->dma = sg_dma_address(iter->sg);
1160                                 iter->max = iter->dma + rem;
1161
1162                                 if (maybe_64K && index < max &&
1163                                     !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
1164                                       (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1165                                        rem >= (max - index) << PAGE_SHIFT)))
1166                                         maybe_64K = false;
1167
1168                                 if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
1169                                         break;
1170                         }
1171                 } while (rem >= page_size && index < max);
1172
1173                 kunmap_atomic(vaddr);
1174
1175                 /*
1176                  * Is it safe to mark the 2M block as 64K? -- Either we have
1177                  * filled whole page-table with 64K entries, or filled part of
1178                  * it and have reached the end of the sg table and we have
1179                  * enough padding.
1180                  */
1181                 if (maybe_64K &&
1182                     (index == max ||
1183                      (i915_vm_has_scratch_64K(vma->vm) &&
1184                       !iter->sg && IS_ALIGNED(vma->node.start +
1185                                               vma->node.size,
1186                                               I915_GTT_PAGE_SIZE_2M)))) {
1187                         vaddr = kmap_atomic_px(pd);
1188                         vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
1189                         kunmap_atomic(vaddr);
1190                         page_size = I915_GTT_PAGE_SIZE_64K;
1191
1192                         /*
1193                          * We write all 4K page entries, even when using 64K
1194                          * pages. In order to verify that the HW isn't cheating
1195                          * by using the 4K PTE instead of the 64K PTE, we want
1196                          * to remove all the surplus entries. If the HW skipped
1197                          * the 64K PTE, it will read/write into the scratch page
1198                          * instead - which we detect as missing results during
1199                          * selftests.
1200                          */
1201                         if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {
1202                                 u16 i;
1203
1204                                 encode = pte_encode | vma->vm->scratch_page.daddr;
1205                                 vaddr = kmap_atomic_px(pd->page_table[idx.pde]);
1206
1207                                 for (i = 1; i < index; i += 16)
1208                                         memset64(vaddr + i, encode, 15);
1209
1210                                 kunmap_atomic(vaddr);
1211                         }
1212                 }
1213
1214                 vma->page_sizes.gtt |= page_size;
1215         } while (iter->sg);
1216 }
1217
1218 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
1219                                    struct i915_vma *vma,
1220                                    enum i915_cache_level cache_level,
1221                                    u32 unused)
1222 {
1223         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1224         struct sgt_dma iter = sgt_dma(vma);
1225         struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
1226
1227         if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
1228                 gen8_ppgtt_insert_huge_entries(vma, pdps, &iter, cache_level);
1229         } else {
1230                 struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1231
1232                 while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++],
1233                                                      &iter, &idx, cache_level))
1234                         GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
1235
1236                 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1237         }
1238 }
1239
1240 static void gen8_free_page_tables(struct i915_address_space *vm,
1241                                   struct i915_page_directory *pd)
1242 {
1243         int i;
1244
1245         if (!px_page(pd))
1246                 return;
1247
1248         for (i = 0; i < I915_PDES; i++) {
1249                 if (pd->page_table[i] != vm->scratch_pt)
1250                         free_pt(vm, pd->page_table[i]);
1251         }
1252 }
1253
1254 static int gen8_init_scratch(struct i915_address_space *vm)
1255 {
1256         int ret;
1257
1258         ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1259         if (ret)
1260                 return ret;
1261
1262         vm->scratch_pt = alloc_pt(vm);
1263         if (IS_ERR(vm->scratch_pt)) {
1264                 ret = PTR_ERR(vm->scratch_pt);
1265                 goto free_scratch_page;
1266         }
1267
1268         vm->scratch_pd = alloc_pd(vm);
1269         if (IS_ERR(vm->scratch_pd)) {
1270                 ret = PTR_ERR(vm->scratch_pd);
1271                 goto free_pt;
1272         }
1273
1274         if (use_4lvl(vm)) {
1275                 vm->scratch_pdp = alloc_pdp(vm);
1276                 if (IS_ERR(vm->scratch_pdp)) {
1277                         ret = PTR_ERR(vm->scratch_pdp);
1278                         goto free_pd;
1279                 }
1280         }
1281
1282         gen8_initialize_pt(vm, vm->scratch_pt);
1283         gen8_initialize_pd(vm, vm->scratch_pd);
1284         if (use_4lvl(vm))
1285                 gen8_initialize_pdp(vm, vm->scratch_pdp);
1286
1287         return 0;
1288
1289 free_pd:
1290         free_pd(vm, vm->scratch_pd);
1291 free_pt:
1292         free_pt(vm, vm->scratch_pt);
1293 free_scratch_page:
1294         cleanup_scratch_page(vm);
1295
1296         return ret;
1297 }
1298
1299 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
1300 {
1301         struct i915_address_space *vm = &ppgtt->vm;
1302         struct drm_i915_private *dev_priv = vm->i915;
1303         enum vgt_g2v_type msg;
1304         int i;
1305
1306         if (use_4lvl(vm)) {
1307                 const u64 daddr = px_dma(&ppgtt->pml4);
1308
1309                 I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
1310                 I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1311
1312                 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
1313                                 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
1314         } else {
1315                 for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1316                         const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1317
1318                         I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
1319                         I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1320                 }
1321
1322                 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
1323                                 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
1324         }
1325
1326         I915_WRITE(vgtif_reg(g2v_notify), msg);
1327
1328         return 0;
1329 }
1330
1331 static void gen8_free_scratch(struct i915_address_space *vm)
1332 {
1333         if (use_4lvl(vm))
1334                 free_pdp(vm, vm->scratch_pdp);
1335         free_pd(vm, vm->scratch_pd);
1336         free_pt(vm, vm->scratch_pt);
1337         cleanup_scratch_page(vm);
1338 }
1339
1340 static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1341                                     struct i915_page_directory_pointer *pdp)
1342 {
1343         const unsigned int pdpes = i915_pdpes_per_pdp(vm);
1344         int i;
1345
1346         for (i = 0; i < pdpes; i++) {
1347                 if (pdp->page_directory[i] == vm->scratch_pd)
1348                         continue;
1349
1350                 gen8_free_page_tables(vm, pdp->page_directory[i]);
1351                 free_pd(vm, pdp->page_directory[i]);
1352         }
1353
1354         free_pdp(vm, pdp);
1355 }
1356
1357 static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
1358 {
1359         int i;
1360
1361         for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
1362                 if (ppgtt->pml4.pdps[i] == ppgtt->vm.scratch_pdp)
1363                         continue;
1364
1365                 gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pml4.pdps[i]);
1366         }
1367
1368         cleanup_px(&ppgtt->vm, &ppgtt->pml4);
1369 }
1370
1371 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
1372 {
1373         struct drm_i915_private *dev_priv = vm->i915;
1374         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1375
1376         if (intel_vgpu_active(dev_priv))
1377                 gen8_ppgtt_notify_vgt(ppgtt, false);
1378
1379         if (use_4lvl(vm))
1380                 gen8_ppgtt_cleanup_4lvl(ppgtt);
1381         else
1382                 gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);
1383
1384         gen8_free_scratch(vm);
1385 }
1386
1387 static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
1388                                struct i915_page_directory *pd,
1389                                u64 start, u64 length)
1390 {
1391         struct i915_page_table *pt;
1392         u64 from = start;
1393         unsigned int pde;
1394
1395         gen8_for_each_pde(pt, pd, start, length, pde) {
1396                 int count = gen8_pte_count(start, length);
1397
1398                 if (pt == vm->scratch_pt) {
1399                         pd->used_pdes++;
1400
1401                         pt = alloc_pt(vm);
1402                         if (IS_ERR(pt)) {
1403                                 pd->used_pdes--;
1404                                 goto unwind;
1405                         }
1406
1407                         if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
1408                                 gen8_initialize_pt(vm, pt);
1409
1410                         gen8_ppgtt_set_pde(vm, pd, pt, pde);
1411                         GEM_BUG_ON(pd->used_pdes > I915_PDES);
1412                 }
1413
1414                 pt->used_ptes += count;
1415         }
1416         return 0;
1417
1418 unwind:
1419         gen8_ppgtt_clear_pd(vm, pd, from, start - from);
1420         return -ENOMEM;
1421 }
1422
1423 static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
1424                                 struct i915_page_directory_pointer *pdp,
1425                                 u64 start, u64 length)
1426 {
1427         struct i915_page_directory *pd;
1428         u64 from = start;
1429         unsigned int pdpe;
1430         int ret;
1431
1432         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1433                 if (pd == vm->scratch_pd) {
1434                         pdp->used_pdpes++;
1435
1436                         pd = alloc_pd(vm);
1437                         if (IS_ERR(pd)) {
1438                                 pdp->used_pdpes--;
1439                                 goto unwind;
1440                         }
1441
1442                         gen8_initialize_pd(vm, pd);
1443                         gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1444                         GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
1445
1446                         mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
1447                 }
1448
1449                 ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
1450                 if (unlikely(ret))
1451                         goto unwind_pd;
1452         }
1453
1454         return 0;
1455
1456 unwind_pd:
1457         if (!pd->used_pdes) {
1458                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1459                 GEM_BUG_ON(!pdp->used_pdpes);
1460                 pdp->used_pdpes--;
1461                 free_pd(vm, pd);
1462         }
1463 unwind:
1464         gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
1465         return -ENOMEM;
1466 }
1467
1468 static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
1469                                  u64 start, u64 length)
1470 {
1471         return gen8_ppgtt_alloc_pdp(vm,
1472                                     &i915_vm_to_ppgtt(vm)->pdp, start, length);
1473 }
1474
1475 static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
1476                                  u64 start, u64 length)
1477 {
1478         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1479         struct i915_pml4 *pml4 = &ppgtt->pml4;
1480         struct i915_page_directory_pointer *pdp;
1481         u64 from = start;
1482         u32 pml4e;
1483         int ret;
1484
1485         gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1486                 if (pml4->pdps[pml4e] == vm->scratch_pdp) {
1487                         pdp = alloc_pdp(vm);
1488                         if (IS_ERR(pdp))
1489                                 goto unwind;
1490
1491                         gen8_initialize_pdp(vm, pdp);
1492                         gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
1493                 }
1494
1495                 ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
1496                 if (unlikely(ret))
1497                         goto unwind_pdp;
1498         }
1499
1500         return 0;
1501
1502 unwind_pdp:
1503         if (!pdp->used_pdpes) {
1504                 gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
1505                 free_pdp(vm, pdp);
1506         }
1507 unwind:
1508         gen8_ppgtt_clear_4lvl(vm, from, start - from);
1509         return -ENOMEM;
1510 }
1511
1512 static void gen8_dump_pdp(struct i915_hw_ppgtt *ppgtt,
1513                           struct i915_page_directory_pointer *pdp,
1514                           u64 start, u64 length,
1515                           gen8_pte_t scratch_pte,
1516                           struct seq_file *m)
1517 {
1518         struct i915_address_space *vm = &ppgtt->vm;
1519         struct i915_page_directory *pd;
1520         u32 pdpe;
1521
1522         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1523                 struct i915_page_table *pt;
1524                 u64 pd_len = length;
1525                 u64 pd_start = start;
1526                 u32 pde;
1527
1528                 if (pdp->page_directory[pdpe] == ppgtt->vm.scratch_pd)
1529                         continue;
1530
1531                 seq_printf(m, "\tPDPE #%d\n", pdpe);
1532                 gen8_for_each_pde(pt, pd, pd_start, pd_len, pde) {
1533                         u32 pte;
1534                         gen8_pte_t *pt_vaddr;
1535
1536                         if (pd->page_table[pde] == ppgtt->vm.scratch_pt)
1537                                 continue;
1538
1539                         pt_vaddr = kmap_atomic_px(pt);
1540                         for (pte = 0; pte < GEN8_PTES; pte += 4) {
1541                                 u64 va = (pdpe << GEN8_PDPE_SHIFT |
1542                                           pde << GEN8_PDE_SHIFT |
1543                                           pte << GEN8_PTE_SHIFT);
1544                                 int i;
1545                                 bool found = false;
1546
1547                                 for (i = 0; i < 4; i++)
1548                                         if (pt_vaddr[pte + i] != scratch_pte)
1549                                                 found = true;
1550                                 if (!found)
1551                                         continue;
1552
1553                                 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1554                                 for (i = 0; i < 4; i++) {
1555                                         if (pt_vaddr[pte + i] != scratch_pte)
1556                                                 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1557                                         else
1558                                                 seq_puts(m, "  SCRATCH ");
1559                                 }
1560                                 seq_puts(m, "\n");
1561                         }
1562                         kunmap_atomic(pt_vaddr);
1563                 }
1564         }
1565 }
1566
1567 static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1568 {
1569         struct i915_address_space *vm = &ppgtt->vm;
1570         const gen8_pte_t scratch_pte =
1571                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
1572         u64 start = 0, length = ppgtt->vm.total;
1573
1574         if (use_4lvl(vm)) {
1575                 u64 pml4e;
1576                 struct i915_pml4 *pml4 = &ppgtt->pml4;
1577                 struct i915_page_directory_pointer *pdp;
1578
1579                 gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1580                         if (pml4->pdps[pml4e] == ppgtt->vm.scratch_pdp)
1581                                 continue;
1582
1583                         seq_printf(m, "    PML4E #%llu\n", pml4e);
1584                         gen8_dump_pdp(ppgtt, pdp, start, length, scratch_pte, m);
1585                 }
1586         } else {
1587                 gen8_dump_pdp(ppgtt, &ppgtt->pdp, start, length, scratch_pte, m);
1588         }
1589 }
1590
1591 static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
1592 {
1593         struct i915_address_space *vm = &ppgtt->vm;
1594         struct i915_page_directory_pointer *pdp = &ppgtt->pdp;
1595         struct i915_page_directory *pd;
1596         u64 start = 0, length = ppgtt->vm.total;
1597         u64 from = start;
1598         unsigned int pdpe;
1599
1600         gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1601                 pd = alloc_pd(vm);
1602                 if (IS_ERR(pd))
1603                         goto unwind;
1604
1605                 gen8_initialize_pd(vm, pd);
1606                 gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
1607                 pdp->used_pdpes++;
1608         }
1609
1610         pdp->used_pdpes++; /* never remove */
1611         return 0;
1612
1613 unwind:
1614         start -= from;
1615         gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
1616                 gen8_ppgtt_set_pdpe(vm, pdp, vm->scratch_pd, pdpe);
1617                 free_pd(vm, pd);
1618         }
1619         pdp->used_pdpes = 0;
1620         return -ENOMEM;
1621 }
1622
1623 /*
1624  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1625  * with a net effect resembling a 2-level page table in normal x86 terms. Each
1626  * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1627  * space.
1628  *
1629  */
1630 static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
1631 {
1632         struct i915_hw_ppgtt *ppgtt;
1633         int err;
1634
1635         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1636         if (!ppgtt)
1637                 return ERR_PTR(-ENOMEM);
1638
1639         kref_init(&ppgtt->ref);
1640
1641         ppgtt->vm.i915 = i915;
1642         ppgtt->vm.dma = &i915->drm.pdev->dev;
1643
1644         ppgtt->vm.total = USES_FULL_48BIT_PPGTT(i915) ?
1645                 1ULL << 48 :
1646                 1ULL << 32;
1647
1648         i915_address_space_init(&ppgtt->vm, i915);
1649
1650         /* There are only few exceptions for gen >=6. chv and bxt.
1651          * And we are not sure about the latter so play safe for now.
1652          */
1653         if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915))
1654                 ppgtt->vm.pt_kmap_wc = true;
1655
1656         err = gen8_init_scratch(&ppgtt->vm);
1657         if (err)
1658                 goto err_free;
1659
1660         if (use_4lvl(&ppgtt->vm)) {
1661                 err = setup_px(&ppgtt->vm, &ppgtt->pml4);
1662                 if (err)
1663                         goto err_scratch;
1664
1665                 gen8_initialize_pml4(&ppgtt->vm, &ppgtt->pml4);
1666
1667                 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
1668                 ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
1669                 ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;
1670         } else {
1671                 err = __pdp_init(&ppgtt->vm, &ppgtt->pdp);
1672                 if (err)
1673                         goto err_scratch;
1674
1675                 if (intel_vgpu_active(i915)) {
1676                         err = gen8_preallocate_top_level_pdp(ppgtt);
1677                         if (err) {
1678                                 __pdp_fini(&ppgtt->pdp);
1679                                 goto err_scratch;
1680                         }
1681                 }
1682
1683                 ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
1684                 ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;
1685                 ppgtt->vm.clear_range = gen8_ppgtt_clear_3lvl;
1686         }
1687
1688         if (intel_vgpu_active(i915))
1689                 gen8_ppgtt_notify_vgt(ppgtt, true);
1690
1691         ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
1692         ppgtt->debug_dump = gen8_dump_ppgtt;
1693
1694         ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
1695         ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
1696         ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
1697         ppgtt->vm.vma_ops.clear_pages = clear_pages;
1698
1699         return ppgtt;
1700
1701 err_scratch:
1702         gen8_free_scratch(&ppgtt->vm);
1703 err_free:
1704         kfree(ppgtt);
1705         return ERR_PTR(err);
1706 }
1707
1708 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *base, struct seq_file *m)
1709 {
1710         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
1711         const gen6_pte_t scratch_pte = ppgtt->scratch_pte;
1712         struct i915_page_table *pt;
1713         u32 pte, pde;
1714
1715         gen6_for_all_pdes(pt, &base->pd, pde) {
1716                 gen6_pte_t *vaddr;
1717
1718                 if (pt == base->vm.scratch_pt)
1719                         continue;
1720
1721                 if (i915_vma_is_bound(ppgtt->vma, I915_VMA_GLOBAL_BIND)) {
1722                         u32 expected =
1723                                 GEN6_PDE_ADDR_ENCODE(px_dma(pt)) |
1724                                 GEN6_PDE_VALID;
1725                         u32 pd_entry = readl(ppgtt->pd_addr + pde);
1726
1727                         if (pd_entry != expected)
1728                                 seq_printf(m,
1729                                            "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1730                                            pde,
1731                                            pd_entry,
1732                                            expected);
1733
1734                         seq_printf(m, "\tPDE: %x\n", pd_entry);
1735                 }
1736
1737                 vaddr = kmap_atomic_px(base->pd.page_table[pde]);
1738                 for (pte = 0; pte < GEN6_PTES; pte += 4) {
1739                         int i;
1740
1741                         for (i = 0; i < 4; i++)
1742                                 if (vaddr[pte + i] != scratch_pte)
1743                                         break;
1744                         if (i == 4)
1745                                 continue;
1746
1747                         seq_printf(m, "\t\t(%03d, %04d) %08lx: ",
1748                                    pde, pte,
1749                                    (pde * GEN6_PTES + pte) * PAGE_SIZE);
1750                         for (i = 0; i < 4; i++) {
1751                                 if (vaddr[pte + i] != scratch_pte)
1752                                         seq_printf(m, " %08x", vaddr[pte + i]);
1753                                 else
1754                                         seq_puts(m, "  SCRATCH");
1755                         }
1756                         seq_puts(m, "\n");
1757                 }
1758                 kunmap_atomic(vaddr);
1759         }
1760 }
1761
1762 /* Write pde (index) from the page directory @pd to the page table @pt */
1763 static inline void gen6_write_pde(const struct gen6_hw_ppgtt *ppgtt,
1764                                   const unsigned int pde,
1765                                   const struct i915_page_table *pt)
1766 {
1767         /* Caller needs to make sure the write completes if necessary */
1768         iowrite32(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
1769                   ppgtt->pd_addr + pde);
1770 }
1771
1772 static void gen8_ppgtt_enable(struct drm_i915_private *dev_priv)
1773 {
1774         struct intel_engine_cs *engine;
1775         enum intel_engine_id id;
1776
1777         for_each_engine(engine, dev_priv, id) {
1778                 u32 four_level = USES_FULL_48BIT_PPGTT(dev_priv) ?
1779                                  GEN8_GFX_PPGTT_48B : 0;
1780                 I915_WRITE(RING_MODE_GEN7(engine),
1781                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
1782         }
1783 }
1784
1785 static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
1786 {
1787         struct intel_engine_cs *engine;
1788         u32 ecochk, ecobits;
1789         enum intel_engine_id id;
1790
1791         ecobits = I915_READ(GAC_ECO_BITS);
1792         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1793
1794         ecochk = I915_READ(GAM_ECOCHK);
1795         if (IS_HASWELL(dev_priv)) {
1796                 ecochk |= ECOCHK_PPGTT_WB_HSW;
1797         } else {
1798                 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1799                 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1800         }
1801         I915_WRITE(GAM_ECOCHK, ecochk);
1802
1803         for_each_engine(engine, dev_priv, id) {
1804                 /* GFX_MODE is per-ring on gen7+ */
1805                 I915_WRITE(RING_MODE_GEN7(engine),
1806                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1807         }
1808 }
1809
1810 static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1811 {
1812         u32 ecochk, gab_ctl, ecobits;
1813
1814         ecobits = I915_READ(GAC_ECO_BITS);
1815         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1816                    ECOBITS_PPGTT_CACHE64B);
1817
1818         gab_ctl = I915_READ(GAB_CTL);
1819         I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1820
1821         ecochk = I915_READ(GAM_ECOCHK);
1822         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1823
1824         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1825 }
1826
1827 /* PPGTT support for Sandybdrige/Gen6 and later */
1828 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1829                                    u64 start, u64 length)
1830 {
1831         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1832         unsigned int first_entry = start >> PAGE_SHIFT;
1833         unsigned int pde = first_entry / GEN6_PTES;
1834         unsigned int pte = first_entry % GEN6_PTES;
1835         unsigned int num_entries = length >> PAGE_SHIFT;
1836         const gen6_pte_t scratch_pte = ppgtt->scratch_pte;
1837
1838         while (num_entries) {
1839                 struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];
1840                 const unsigned int end = min(pte + num_entries, GEN6_PTES);
1841                 const unsigned int count = end - pte;
1842                 gen6_pte_t *vaddr;
1843
1844                 GEM_BUG_ON(pt == vm->scratch_pt);
1845
1846                 num_entries -= count;
1847
1848                 GEM_BUG_ON(count > pt->used_ptes);
1849                 pt->used_ptes -= count;
1850                 if (!pt->used_ptes)
1851                         ppgtt->scan_for_unused_pt = true;
1852
1853                 /*
1854                  * Note that the hw doesn't support removing PDE on the fly
1855                  * (they are cached inside the context with no means to
1856                  * invalidate the cache), so we can only reset the PTE
1857                  * entries back to scratch.
1858                  */
1859
1860                 vaddr = kmap_atomic_px(pt);
1861                 do {
1862                         vaddr[pte++] = scratch_pte;
1863                 } while (pte < end);
1864                 kunmap_atomic(vaddr);
1865
1866                 pte = 0;
1867         }
1868 }
1869
1870 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1871                                       struct i915_vma *vma,
1872                                       enum i915_cache_level cache_level,
1873                                       u32 flags)
1874 {
1875         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1876         unsigned first_entry = vma->node.start >> PAGE_SHIFT;
1877         unsigned act_pt = first_entry / GEN6_PTES;
1878         unsigned act_pte = first_entry % GEN6_PTES;
1879         const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1880         struct sgt_dma iter = sgt_dma(vma);
1881         gen6_pte_t *vaddr;
1882
1883         GEM_BUG_ON(ppgtt->pd.page_table[act_pt] == vm->scratch_pt);
1884
1885         vaddr = kmap_atomic_px(ppgtt->pd.page_table[act_pt]);
1886         do {
1887                 vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1888
1889                 iter.dma += PAGE_SIZE;
1890                 if (iter.dma == iter.max) {
1891                         iter.sg = __sg_next(iter.sg);
1892                         if (!iter.sg)
1893                                 break;
1894
1895                         iter.dma = sg_dma_address(iter.sg);
1896                         iter.max = iter.dma + iter.sg->length;
1897                 }
1898
1899                 if (++act_pte == GEN6_PTES) {
1900                         kunmap_atomic(vaddr);
1901                         vaddr = kmap_atomic_px(ppgtt->pd.page_table[++act_pt]);
1902                         act_pte = 0;
1903                 }
1904         } while (1);
1905         kunmap_atomic(vaddr);
1906
1907         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1908 }
1909
1910 static int gen6_alloc_va_range(struct i915_address_space *vm,
1911                                u64 start, u64 length)
1912 {
1913         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1914         struct i915_page_table *pt;
1915         u64 from = start;
1916         unsigned int pde;
1917         bool flush = false;
1918
1919         gen6_for_each_pde(pt, &ppgtt->base.pd, start, length, pde) {
1920                 const unsigned int count = gen6_pte_count(start, length);
1921
1922                 if (pt == vm->scratch_pt) {
1923                         pt = alloc_pt(vm);
1924                         if (IS_ERR(pt))
1925                                 goto unwind_out;
1926
1927                         gen6_initialize_pt(ppgtt, pt);
1928                         ppgtt->base.pd.page_table[pde] = pt;
1929
1930                         if (i915_vma_is_bound(ppgtt->vma,
1931                                               I915_VMA_GLOBAL_BIND)) {
1932                                 gen6_write_pde(ppgtt, pde, pt);
1933                                 flush = true;
1934                         }
1935
1936                         GEM_BUG_ON(pt->used_ptes);
1937                 }
1938
1939                 pt->used_ptes += count;
1940         }
1941
1942         if (flush) {
1943                 mark_tlbs_dirty(&ppgtt->base);
1944                 gen6_ggtt_invalidate(ppgtt->base.vm.i915);
1945         }
1946
1947         return 0;
1948
1949 unwind_out:
1950         gen6_ppgtt_clear_range(vm, from, start - from);
1951         return -ENOMEM;
1952 }
1953
1954 static int gen6_ppgtt_init_scratch(struct gen6_hw_ppgtt *ppgtt)
1955 {
1956         struct i915_address_space * const vm = &ppgtt->base.vm;
1957         struct i915_page_table *unused;
1958         u32 pde;
1959         int ret;
1960
1961         ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1962         if (ret)
1963                 return ret;
1964
1965         ppgtt->scratch_pte =
1966                 vm->pte_encode(vm->scratch_page.daddr,
1967                                I915_CACHE_NONE, PTE_READ_ONLY);
1968
1969         vm->scratch_pt = alloc_pt(vm);
1970         if (IS_ERR(vm->scratch_pt)) {
1971                 cleanup_scratch_page(vm);
1972                 return PTR_ERR(vm->scratch_pt);
1973         }
1974
1975         gen6_initialize_pt(ppgtt, vm->scratch_pt);
1976         gen6_for_all_pdes(unused, &ppgtt->base.pd, pde)
1977                 ppgtt->base.pd.page_table[pde] = vm->scratch_pt;
1978
1979         return 0;
1980 }
1981
1982 static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
1983 {
1984         free_pt(vm, vm->scratch_pt);
1985         cleanup_scratch_page(vm);
1986 }
1987
1988 static void gen6_ppgtt_free_pd(struct gen6_hw_ppgtt *ppgtt)
1989 {
1990         struct i915_page_table *pt;
1991         u32 pde;
1992
1993         gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
1994                 if (pt != ppgtt->base.vm.scratch_pt)
1995                         free_pt(&ppgtt->base.vm, pt);
1996 }
1997
1998 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1999 {
2000         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
2001
2002         i915_vma_destroy(ppgtt->vma);
2003
2004         gen6_ppgtt_free_pd(ppgtt);
2005         gen6_ppgtt_free_scratch(vm);
2006 }
2007
2008 static int pd_vma_set_pages(struct i915_vma *vma)
2009 {
2010         vma->pages = ERR_PTR(-ENODEV);
2011         return 0;
2012 }
2013
2014 static void pd_vma_clear_pages(struct i915_vma *vma)
2015 {
2016         GEM_BUG_ON(!vma->pages);
2017
2018         vma->pages = NULL;
2019 }
2020
2021 static int pd_vma_bind(struct i915_vma *vma,
2022                        enum i915_cache_level cache_level,
2023                        u32 unused)
2024 {
2025         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
2026         struct gen6_hw_ppgtt *ppgtt = vma->private;
2027         u32 ggtt_offset = i915_ggtt_offset(vma) / PAGE_SIZE;
2028         struct i915_page_table *pt;
2029         unsigned int pde;
2030
2031         ppgtt->base.pd.base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
2032         ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
2033
2034         gen6_for_all_pdes(pt, &ppgtt->base.pd, pde)
2035                 gen6_write_pde(ppgtt, pde, pt);
2036
2037         mark_tlbs_dirty(&ppgtt->base);
2038         gen6_ggtt_invalidate(ppgtt->base.vm.i915);
2039
2040         return 0;
2041 }
2042
2043 static void pd_vma_unbind(struct i915_vma *vma)
2044 {
2045         struct gen6_hw_ppgtt *ppgtt = vma->private;
2046         struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
2047         struct i915_page_table *pt;
2048         unsigned int pde;
2049
2050         if (!ppgtt->scan_for_unused_pt)
2051                 return;
2052
2053         /* Free all no longer used page tables */
2054         gen6_for_all_pdes(pt, &ppgtt->base.pd, pde) {
2055                 if (pt->used_ptes || pt == scratch_pt)
2056                         continue;
2057
2058                 free_pt(&ppgtt->base.vm, pt);
2059                 ppgtt->base.pd.page_table[pde] = scratch_pt;
2060         }
2061
2062         ppgtt->scan_for_unused_pt = false;
2063 }
2064
2065 static const struct i915_vma_ops pd_vma_ops = {
2066         .set_pages = pd_vma_set_pages,
2067         .clear_pages = pd_vma_clear_pages,
2068         .bind_vma = pd_vma_bind,
2069         .unbind_vma = pd_vma_unbind,
2070 };
2071
2072 static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
2073 {
2074         struct drm_i915_private *i915 = ppgtt->base.vm.i915;
2075         struct i915_ggtt *ggtt = &i915->ggtt;
2076         struct i915_vma *vma;
2077
2078         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
2079         GEM_BUG_ON(size > ggtt->vm.total);
2080
2081         vma = kmem_cache_zalloc(i915->vmas, GFP_KERNEL);
2082         if (!vma)
2083                 return ERR_PTR(-ENOMEM);
2084
2085         init_request_active(&vma->last_fence, NULL);
2086
2087         vma->vm = &ggtt->vm;
2088         vma->ops = &pd_vma_ops;
2089         vma->private = ppgtt;
2090
2091         vma->active = RB_ROOT;
2092
2093         vma->size = size;
2094         vma->fence_size = size;
2095         vma->flags = I915_VMA_GGTT;
2096         vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */
2097
2098         INIT_LIST_HEAD(&vma->obj_link);
2099         list_add(&vma->vm_link, &vma->vm->unbound_list);
2100
2101         return vma;
2102 }
2103
2104 int gen6_ppgtt_pin(struct i915_hw_ppgtt *base)
2105 {
2106         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
2107
2108         /*
2109          * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
2110          * which will be pinned into every active context.
2111          * (When vma->pin_count becomes atomic, I expect we will naturally
2112          * need a larger, unpacked, type and kill this redundancy.)
2113          */
2114         if (ppgtt->pin_count++)
2115                 return 0;
2116
2117         /*
2118          * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2119          * allocator works in address space sizes, so it's multiplied by page
2120          * size. We allocate at the top of the GTT to avoid fragmentation.
2121          */
2122         return i915_vma_pin(ppgtt->vma,
2123                             0, GEN6_PD_ALIGN,
2124                             PIN_GLOBAL | PIN_HIGH);
2125 }
2126
2127 void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base)
2128 {
2129         struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
2130
2131         GEM_BUG_ON(!ppgtt->pin_count);
2132         if (--ppgtt->pin_count)
2133                 return;
2134
2135         i915_vma_unpin(ppgtt->vma);
2136 }
2137
2138 static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
2139 {
2140         struct i915_ggtt * const ggtt = &i915->ggtt;
2141         struct gen6_hw_ppgtt *ppgtt;
2142         int err;
2143
2144         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2145         if (!ppgtt)
2146                 return ERR_PTR(-ENOMEM);
2147
2148         kref_init(&ppgtt->base.ref);
2149
2150         ppgtt->base.vm.i915 = i915;
2151         ppgtt->base.vm.dma = &i915->drm.pdev->dev;
2152
2153         ppgtt->base.vm.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
2154
2155         i915_address_space_init(&ppgtt->base.vm, i915);
2156
2157         ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
2158         ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
2159         ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
2160         ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
2161         ppgtt->base.debug_dump = gen6_dump_ppgtt;
2162
2163         ppgtt->base.vm.vma_ops.bind_vma    = ppgtt_bind_vma;
2164         ppgtt->base.vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
2165         ppgtt->base.vm.vma_ops.set_pages   = ppgtt_set_pages;
2166         ppgtt->base.vm.vma_ops.clear_pages = clear_pages;
2167
2168         ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
2169
2170         err = gen6_ppgtt_init_scratch(ppgtt);
2171         if (err)
2172                 goto err_free;
2173
2174         ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
2175         if (IS_ERR(ppgtt->vma)) {
2176                 err = PTR_ERR(ppgtt->vma);
2177                 goto err_scratch;
2178         }
2179
2180         return &ppgtt->base;
2181
2182 err_scratch:
2183         gen6_ppgtt_free_scratch(&ppgtt->base.vm);
2184 err_free:
2185         kfree(ppgtt);
2186         return ERR_PTR(err);
2187 }
2188
2189 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2190 {
2191         /* This function is for gtt related workarounds. This function is
2192          * called on driver load and after a GPU reset, so you can place
2193          * workarounds here even if they get overwritten by GPU reset.
2194          */
2195         /* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
2196         if (IS_BROADWELL(dev_priv))
2197                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2198         else if (IS_CHERRYVIEW(dev_priv))
2199                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2200         else if (IS_GEN9_LP(dev_priv))
2201                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2202         else if (INTEL_GEN(dev_priv) >= 9)
2203                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2204
2205         /*
2206          * To support 64K PTEs we need to first enable the use of the
2207          * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
2208          * mmio, otherwise the page-walker will simply ignore the IPS bit. This
2209          * shouldn't be needed after GEN10.
2210          *
2211          * 64K pages were first introduced from BDW+, although technically they
2212          * only *work* from gen9+. For pre-BDW we instead have the option for
2213          * 32K pages, but we don't currently have any support for it in our
2214          * driver.
2215          */
2216         if (HAS_PAGE_SIZES(dev_priv, I915_GTT_PAGE_SIZE_64K) &&
2217             INTEL_GEN(dev_priv) <= 10)
2218                 I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA,
2219                            I915_READ(GEN8_GAMW_ECO_DEV_RW_IA) |
2220                            GAMW_ECO_ENABLE_64K_IPS_FIELD);
2221 }
2222
2223 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2224 {
2225         gtt_write_workarounds(dev_priv);
2226
2227         /* In the case of execlists, PPGTT is enabled by the context descriptor
2228          * and the PDPs are contained within the context itself.  We don't
2229          * need to do anything here. */
2230         if (HAS_LOGICAL_RING_CONTEXTS(dev_priv))
2231                 return 0;
2232
2233         if (!USES_PPGTT(dev_priv))
2234                 return 0;
2235
2236         if (IS_GEN6(dev_priv))
2237                 gen6_ppgtt_enable(dev_priv);
2238         else if (IS_GEN7(dev_priv))
2239                 gen7_ppgtt_enable(dev_priv);
2240         else if (INTEL_GEN(dev_priv) >= 8)
2241                 gen8_ppgtt_enable(dev_priv);
2242         else
2243                 MISSING_CASE(INTEL_GEN(dev_priv));
2244
2245         return 0;
2246 }
2247
2248 static struct i915_hw_ppgtt *
2249 __hw_ppgtt_create(struct drm_i915_private *i915)
2250 {
2251         if (INTEL_GEN(i915) < 8)
2252                 return gen6_ppgtt_create(i915);
2253         else
2254                 return gen8_ppgtt_create(i915);
2255 }
2256
2257 struct i915_hw_ppgtt *
2258 i915_ppgtt_create(struct drm_i915_private *i915,
2259                   struct drm_i915_file_private *fpriv)
2260 {
2261         struct i915_hw_ppgtt *ppgtt;
2262
2263         ppgtt = __hw_ppgtt_create(i915);
2264         if (IS_ERR(ppgtt))
2265                 return ppgtt;
2266
2267         ppgtt->vm.file = fpriv;
2268
2269         trace_i915_ppgtt_create(&ppgtt->vm);
2270
2271         return ppgtt;
2272 }
2273
2274 void i915_ppgtt_close(struct i915_address_space *vm)
2275 {
2276         GEM_BUG_ON(vm->closed);
2277         vm->closed = true;
2278 }
2279
2280 static void ppgtt_destroy_vma(struct i915_address_space *vm)
2281 {
2282         struct list_head *phases[] = {
2283                 &vm->active_list,
2284                 &vm->inactive_list,
2285                 &vm->unbound_list,
2286                 NULL,
2287         }, **phase;
2288
2289         vm->closed = true;
2290         for (phase = phases; *phase; phase++) {
2291                 struct i915_vma *vma, *vn;
2292
2293                 list_for_each_entry_safe(vma, vn, *phase, vm_link)
2294                         i915_vma_destroy(vma);
2295         }
2296 }
2297
2298 void i915_ppgtt_release(struct kref *kref)
2299 {
2300         struct i915_hw_ppgtt *ppgtt =
2301                 container_of(kref, struct i915_hw_ppgtt, ref);
2302
2303         trace_i915_ppgtt_release(&ppgtt->vm);
2304
2305         ppgtt_destroy_vma(&ppgtt->vm);
2306
2307         GEM_BUG_ON(!list_empty(&ppgtt->vm.active_list));
2308         GEM_BUG_ON(!list_empty(&ppgtt->vm.inactive_list));
2309         GEM_BUG_ON(!list_empty(&ppgtt->vm.unbound_list));
2310
2311         ppgtt->vm.cleanup(&ppgtt->vm);
2312         i915_address_space_fini(&ppgtt->vm);
2313         kfree(ppgtt);
2314 }
2315
2316 /* Certain Gen5 chipsets require require idling the GPU before
2317  * unmapping anything from the GTT when VT-d is enabled.
2318  */
2319 static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2320 {
2321         /* Query intel_iommu to see if we need the workaround. Presumably that
2322          * was loaded first.
2323          */
2324         return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
2325 }
2326
2327 static void gen6_check_and_clear_faults(struct drm_i915_private *dev_priv)
2328 {
2329         struct intel_engine_cs *engine;
2330         enum intel_engine_id id;
2331         u32 fault;
2332
2333         for_each_engine(engine, dev_priv, id) {
2334                 fault = I915_READ(RING_FAULT_REG(engine));
2335                 if (fault & RING_FAULT_VALID) {
2336                         DRM_DEBUG_DRIVER("Unexpected fault\n"
2337                                          "\tAddr: 0x%08lx\n"
2338                                          "\tAddress space: %s\n"
2339                                          "\tSource ID: %d\n"
2340                                          "\tType: %d\n",
2341                                          fault & PAGE_MASK,
2342                                          fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2343                                          RING_FAULT_SRCID(fault),
2344                                          RING_FAULT_FAULT_TYPE(fault));
2345                         I915_WRITE(RING_FAULT_REG(engine),
2346                                    fault & ~RING_FAULT_VALID);
2347                 }
2348         }
2349
2350         POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
2351 }
2352
2353 static void gen8_check_and_clear_faults(struct drm_i915_private *dev_priv)
2354 {
2355         u32 fault = I915_READ(GEN8_RING_FAULT_REG);
2356
2357         if (fault & RING_FAULT_VALID) {
2358                 u32 fault_data0, fault_data1;
2359                 u64 fault_addr;
2360
2361                 fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
2362                 fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
2363                 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
2364                              ((u64)fault_data0 << 12);
2365
2366                 DRM_DEBUG_DRIVER("Unexpected fault\n"
2367                                  "\tAddr: 0x%08x_%08x\n"
2368                                  "\tAddress space: %s\n"
2369                                  "\tEngine ID: %d\n"
2370                                  "\tSource ID: %d\n"
2371                                  "\tType: %d\n",
2372                                  upper_32_bits(fault_addr),
2373                                  lower_32_bits(fault_addr),
2374                                  fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
2375                                  GEN8_RING_FAULT_ENGINE_ID(fault),
2376                                  RING_FAULT_SRCID(fault),
2377                                  RING_FAULT_FAULT_TYPE(fault));
2378                 I915_WRITE(GEN8_RING_FAULT_REG,
2379                            fault & ~RING_FAULT_VALID);
2380         }
2381
2382         POSTING_READ(GEN8_RING_FAULT_REG);
2383 }
2384
2385 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
2386 {
2387         /* From GEN8 onwards we only have one 'All Engine Fault Register' */
2388         if (INTEL_GEN(dev_priv) >= 8)
2389                 gen8_check_and_clear_faults(dev_priv);
2390         else if (INTEL_GEN(dev_priv) >= 6)
2391                 gen6_check_and_clear_faults(dev_priv);
2392         else
2393                 return;
2394 }
2395
2396 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2397 {
2398         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2399
2400         /* Don't bother messing with faults pre GEN6 as we have little
2401          * documentation supporting that it's a good idea.
2402          */
2403         if (INTEL_GEN(dev_priv) < 6)
2404                 return;
2405
2406         i915_check_and_clear_faults(dev_priv);
2407
2408         ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
2409
2410         i915_ggtt_invalidate(dev_priv);
2411 }
2412
2413 int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
2414                                struct sg_table *pages)
2415 {
2416         do {
2417                 if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
2418                                      pages->sgl, pages->nents,
2419                                      PCI_DMA_BIDIRECTIONAL,
2420                                      DMA_ATTR_NO_WARN))
2421                         return 0;
2422
2423                 /* If the DMA remap fails, one cause can be that we have
2424                  * too many objects pinned in a small remapping table,
2425                  * such as swiotlb. Incrementally purge all other objects and
2426                  * try again - if there are no more pages to remove from
2427                  * the DMA remapper, i915_gem_shrink will return 0.
2428                  */
2429                 GEM_BUG_ON(obj->mm.pages == pages);
2430         } while (i915_gem_shrink(to_i915(obj->base.dev),
2431                                  obj->base.size >> PAGE_SHIFT, NULL,
2432                                  I915_SHRINK_BOUND |
2433                                  I915_SHRINK_UNBOUND |
2434                                  I915_SHRINK_ACTIVE));
2435
2436         return -ENOSPC;
2437 }
2438
2439 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
2440 {
2441         writeq(pte, addr);
2442 }
2443
2444 static void gen8_ggtt_insert_page(struct i915_address_space *vm,
2445                                   dma_addr_t addr,
2446                                   u64 offset,
2447                                   enum i915_cache_level level,
2448                                   u32 unused)
2449 {
2450         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2451         gen8_pte_t __iomem *pte =
2452                 (gen8_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2453
2454         gen8_set_pte(pte, gen8_pte_encode(addr, level));
2455
2456         ggtt->invalidate(vm->i915);
2457 }
2458
2459 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2460                                      struct i915_vma *vma,
2461                                      enum i915_cache_level level,
2462                                      u32 unused)
2463 {
2464         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2465         struct sgt_iter sgt_iter;
2466         gen8_pte_t __iomem *gtt_entries;
2467         const gen8_pte_t pte_encode = gen8_pte_encode(0, level);
2468         dma_addr_t addr;
2469
2470         gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2471         gtt_entries += vma->node.start >> PAGE_SHIFT;
2472         for_each_sgt_dma(addr, sgt_iter, vma->pages)
2473                 gen8_set_pte(gtt_entries++, pte_encode | addr);
2474
2475         /*
2476          * We want to flush the TLBs only after we're certain all the PTE
2477          * updates have finished.
2478          */
2479         ggtt->invalidate(vm->i915);
2480 }
2481
2482 static void gen6_ggtt_insert_page(struct i915_address_space *vm,
2483                                   dma_addr_t addr,
2484                                   u64 offset,
2485                                   enum i915_cache_level level,
2486                                   u32 flags)
2487 {
2488         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2489         gen6_pte_t __iomem *pte =
2490                 (gen6_pte_t __iomem *)ggtt->gsm + (offset >> PAGE_SHIFT);
2491
2492         iowrite32(vm->pte_encode(addr, level, flags), pte);
2493
2494         ggtt->invalidate(vm->i915);
2495 }
2496
2497 /*
2498  * Binds an object into the global gtt with the specified cache level. The object
2499  * will be accessible to the GPU via commands whose operands reference offsets
2500  * within the global GTT as well as accessible by the GPU through the GMADR
2501  * mapped BAR (dev_priv->mm.gtt->gtt).
2502  */
2503 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2504                                      struct i915_vma *vma,
2505                                      enum i915_cache_level level,
2506                                      u32 flags)
2507 {
2508         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2509         gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2510         unsigned int i = vma->node.start >> PAGE_SHIFT;
2511         struct sgt_iter iter;
2512         dma_addr_t addr;
2513         for_each_sgt_dma(addr, iter, vma->pages)
2514                 iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2515
2516         /*
2517          * We want to flush the TLBs only after we're certain all the PTE
2518          * updates have finished.
2519          */
2520         ggtt->invalidate(vm->i915);
2521 }
2522
2523 static void nop_clear_range(struct i915_address_space *vm,
2524                             u64 start, u64 length)
2525 {
2526 }
2527
2528 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2529                                   u64 start, u64 length)
2530 {
2531         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2532         unsigned first_entry = start >> PAGE_SHIFT;
2533         unsigned num_entries = length >> PAGE_SHIFT;
2534         const gen8_pte_t scratch_pte =
2535                 gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC);
2536         gen8_pte_t __iomem *gtt_base =
2537                 (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
2538         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2539         int i;
2540
2541         if (WARN(num_entries > max_entries,
2542                  "First entry = %d; Num entries = %d (max=%d)\n",
2543                  first_entry, num_entries, max_entries))
2544                 num_entries = max_entries;
2545
2546         for (i = 0; i < num_entries; i++)
2547                 gen8_set_pte(&gtt_base[i], scratch_pte);
2548 }
2549
2550 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
2551 {
2552         struct drm_i915_private *dev_priv = vm->i915;
2553
2554         /*
2555          * Make sure the internal GAM fifo has been cleared of all GTT
2556          * writes before exiting stop_machine(). This guarantees that
2557          * any aperture accesses waiting to start in another process
2558          * cannot back up behind the GTT writes causing a hang.
2559          * The register can be any arbitrary GAM register.
2560          */
2561         POSTING_READ(GFX_FLSH_CNTL_GEN6);
2562 }
2563
2564 struct insert_page {
2565         struct i915_address_space *vm;
2566         dma_addr_t addr;
2567         u64 offset;
2568         enum i915_cache_level level;
2569 };
2570
2571 static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
2572 {
2573         struct insert_page *arg = _arg;
2574
2575         gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
2576         bxt_vtd_ggtt_wa(arg->vm);
2577
2578         return 0;
2579 }
2580
2581 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
2582                                           dma_addr_t addr,
2583                                           u64 offset,
2584                                           enum i915_cache_level level,
2585                                           u32 unused)
2586 {
2587         struct insert_page arg = { vm, addr, offset, level };
2588
2589         stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
2590 }
2591
2592 struct insert_entries {
2593         struct i915_address_space *vm;
2594         struct i915_vma *vma;
2595         enum i915_cache_level level;
2596 };
2597
2598 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
2599 {
2600         struct insert_entries *arg = _arg;
2601
2602         gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, 0);
2603         bxt_vtd_ggtt_wa(arg->vm);
2604
2605         return 0;
2606 }
2607
2608 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2609                                              struct i915_vma *vma,
2610                                              enum i915_cache_level level,
2611                                              u32 unused)
2612 {
2613         struct insert_entries arg = { vm, vma, level };
2614
2615         stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
2616 }
2617
2618 struct clear_range {
2619         struct i915_address_space *vm;
2620         u64 start;
2621         u64 length;
2622 };
2623
2624 static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
2625 {
2626         struct clear_range *arg = _arg;
2627
2628         gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
2629         bxt_vtd_ggtt_wa(arg->vm);
2630
2631         return 0;
2632 }
2633
2634 static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm,
2635                                           u64 start,
2636                                           u64 length)
2637 {
2638         struct clear_range arg = { vm, start, length };
2639
2640         stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL);
2641 }
2642
2643 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2644                                   u64 start, u64 length)
2645 {
2646         struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2647         unsigned first_entry = start >> PAGE_SHIFT;
2648         unsigned num_entries = length >> PAGE_SHIFT;
2649         gen6_pte_t scratch_pte, __iomem *gtt_base =
2650                 (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
2651         const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2652         int i;
2653
2654         if (WARN(num_entries > max_entries,
2655                  "First entry = %d; Num entries = %d (max=%d)\n",
2656                  first_entry, num_entries, max_entries))
2657                 num_entries = max_entries;
2658
2659         scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
2660                                      I915_CACHE_LLC, 0);
2661
2662         for (i = 0; i < num_entries; i++)
2663                 iowrite32(scratch_pte, &gtt_base[i]);
2664 }
2665
2666 static void i915_ggtt_insert_page(struct i915_address_space *vm,
2667                                   dma_addr_t addr,
2668                                   u64 offset,
2669                                   enum i915_cache_level cache_level,
2670                                   u32 unused)
2671 {
2672         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2673                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2674
2675         intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
2676 }
2677
2678 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2679                                      struct i915_vma *vma,
2680                                      enum i915_cache_level cache_level,
2681                                      u32 unused)
2682 {
2683         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2684                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2685
2686         intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
2687                                     flags);
2688 }
2689
2690 static void i915_ggtt_clear_range(struct i915_address_space *vm,
2691                                   u64 start, u64 length)
2692 {
2693         intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2694 }
2695
2696 static int ggtt_bind_vma(struct i915_vma *vma,
2697                          enum i915_cache_level cache_level,
2698                          u32 flags)
2699 {
2700         struct drm_i915_private *i915 = vma->vm->i915;
2701         struct drm_i915_gem_object *obj = vma->obj;
2702         u32 pte_flags;
2703
2704         /* Currently applicable only to VLV */
2705         pte_flags = 0;
2706         if (obj->gt_ro)
2707                 pte_flags |= PTE_READ_ONLY;
2708
2709         intel_runtime_pm_get(i915);
2710         vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2711         intel_runtime_pm_put(i915);
2712
2713         vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
2714
2715         /*
2716          * Without aliasing PPGTT there's no difference between
2717          * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2718          * upgrade to both bound if we bind either to avoid double-binding.
2719          */
2720         vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2721
2722         return 0;
2723 }
2724
2725 static void ggtt_unbind_vma(struct i915_vma *vma)
2726 {
2727         struct drm_i915_private *i915 = vma->vm->i915;
2728
2729         intel_runtime_pm_get(i915);
2730         vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2731         intel_runtime_pm_put(i915);
2732 }
2733
2734 static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2735                                  enum i915_cache_level cache_level,
2736                                  u32 flags)
2737 {
2738         struct drm_i915_private *i915 = vma->vm->i915;
2739         u32 pte_flags;
2740         int ret;
2741
2742         /* Currently applicable only to VLV */
2743         pte_flags = 0;
2744         if (vma->obj->gt_ro)
2745                 pte_flags |= PTE_READ_ONLY;
2746
2747         if (flags & I915_VMA_LOCAL_BIND) {
2748                 struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2749
2750                 if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
2751                         ret = appgtt->vm.allocate_va_range(&appgtt->vm,
2752                                                            vma->node.start,
2753                                                            vma->size);
2754                         if (ret)
2755                                 return ret;
2756                 }
2757
2758                 appgtt->vm.insert_entries(&appgtt->vm, vma, cache_level,
2759                                           pte_flags);
2760         }
2761
2762         if (flags & I915_VMA_GLOBAL_BIND) {
2763                 intel_runtime_pm_get(i915);
2764                 vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2765                 intel_runtime_pm_put(i915);
2766         }
2767
2768         return 0;
2769 }
2770
2771 static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2772 {
2773         struct drm_i915_private *i915 = vma->vm->i915;
2774
2775         if (vma->flags & I915_VMA_GLOBAL_BIND) {
2776                 intel_runtime_pm_get(i915);
2777                 vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2778                 intel_runtime_pm_put(i915);
2779         }
2780
2781         if (vma->flags & I915_VMA_LOCAL_BIND) {
2782                 struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->vm;
2783
2784                 vm->clear_range(vm, vma->node.start, vma->size);
2785         }
2786 }
2787
2788 void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
2789                                struct sg_table *pages)
2790 {
2791         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2792         struct device *kdev = &dev_priv->drm.pdev->dev;
2793         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2794
2795         if (unlikely(ggtt->do_idle_maps)) {
2796                 if (i915_gem_wait_for_idle(dev_priv, 0, MAX_SCHEDULE_TIMEOUT)) {
2797                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
2798                         /* Wait a bit, in hopes it avoids the hang */
2799                         udelay(10);
2800                 }
2801         }
2802
2803         dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2804 }
2805
2806 static int ggtt_set_pages(struct i915_vma *vma)
2807 {
2808         int ret;
2809
2810         GEM_BUG_ON(vma->pages);
2811
2812         ret = i915_get_ggtt_vma_pages(vma);
2813         if (ret)
2814                 return ret;
2815
2816         vma->page_sizes = vma->obj->mm.page_sizes;
2817
2818         return 0;
2819 }
2820
2821 static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2822                                   unsigned long color,
2823                                   u64 *start,
2824                                   u64 *end)
2825 {
2826         if (node->allocated && node->color != color)
2827                 *start += I915_GTT_PAGE_SIZE;
2828
2829         /* Also leave a space between the unallocated reserved node after the
2830          * GTT and any objects within the GTT, i.e. we use the color adjustment
2831          * to insert a guard page to prevent prefetches crossing over the
2832          * GTT boundary.
2833          */
2834         node = list_next_entry(node, node_list);
2835         if (node->color != color)
2836                 *end -= I915_GTT_PAGE_SIZE;
2837 }
2838
2839 int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
2840 {
2841         struct i915_ggtt *ggtt = &i915->ggtt;
2842         struct i915_hw_ppgtt *ppgtt;
2843         int err;
2844
2845         ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM));
2846         if (IS_ERR(ppgtt))
2847                 return PTR_ERR(ppgtt);
2848
2849         if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
2850                 err = -ENODEV;
2851                 goto err_ppgtt;
2852         }
2853
2854         /*
2855          * Note we only pre-allocate as far as the end of the global
2856          * GTT. On 48b / 4-level page-tables, the difference is very,
2857          * very significant! We have to preallocate as GVT/vgpu does
2858          * not like the page directory disappearing.
2859          */
2860         err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total);
2861         if (err)
2862                 goto err_ppgtt;
2863
2864         i915->mm.aliasing_ppgtt = ppgtt;
2865
2866         GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
2867         ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
2868
2869         GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma);
2870         ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
2871
2872         return 0;
2873
2874 err_ppgtt:
2875         i915_ppgtt_put(ppgtt);
2876         return err;
2877 }
2878
2879 void i915_gem_fini_aliasing_ppgtt(struct drm_i915_private *i915)
2880 {
2881         struct i915_ggtt *ggtt = &i915->ggtt;
2882         struct i915_hw_ppgtt *ppgtt;
2883
2884         ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
2885         if (!ppgtt)
2886                 return;
2887
2888         i915_ppgtt_put(ppgtt);
2889
2890         ggtt->vm.vma_ops.bind_vma   = ggtt_bind_vma;
2891         ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
2892 }
2893
2894 int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2895 {
2896         /* Let GEM Manage all of the aperture.
2897          *
2898          * However, leave one page at the end still bound to the scratch page.
2899          * There are a number of places where the hardware apparently prefetches
2900          * past the end of the object, and we've seen multiple hangs with the
2901          * GPU head pointer stuck in a batchbuffer bound at the last page of the
2902          * aperture.  One page should be enough to keep any prefetching inside
2903          * of the aperture.
2904          */
2905         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2906         unsigned long hole_start, hole_end;
2907         struct drm_mm_node *entry;
2908         int ret;
2909
2910         ret = intel_vgt_balloon(dev_priv);
2911         if (ret)
2912                 return ret;
2913
2914         /* Reserve a mappable slot for our lockless error capture */
2915         ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, &ggtt->error_capture,
2916                                           PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
2917                                           0, ggtt->mappable_end,
2918                                           DRM_MM_INSERT_LOW);
2919         if (ret)
2920                 return ret;
2921
2922         /* Clear any non-preallocated blocks */
2923         drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
2924                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2925                               hole_start, hole_end);
2926                 ggtt->vm.clear_range(&ggtt->vm, hole_start,
2927                                      hole_end - hole_start);
2928         }
2929
2930         /* And finally clear the reserved guard page */
2931         ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
2932
2933         if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
2934                 ret = i915_gem_init_aliasing_ppgtt(dev_priv);
2935                 if (ret)
2936                         goto err;
2937         }
2938
2939         return 0;
2940
2941 err:
2942         drm_mm_remove_node(&ggtt->error_capture);
2943         return ret;
2944 }
2945
2946 /**
2947  * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2948  * @dev_priv: i915 device
2949  */
2950 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2951 {
2952         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2953         struct i915_vma *vma, *vn;
2954         struct pagevec *pvec;
2955
2956         ggtt->vm.closed = true;
2957
2958         mutex_lock(&dev_priv->drm.struct_mutex);
2959         i915_gem_fini_aliasing_ppgtt(dev_priv);
2960
2961         GEM_BUG_ON(!list_empty(&ggtt->vm.active_list));
2962         list_for_each_entry_safe(vma, vn, &ggtt->vm.inactive_list, vm_link)
2963                 WARN_ON(i915_vma_unbind(vma));
2964
2965         if (drm_mm_node_allocated(&ggtt->error_capture))
2966                 drm_mm_remove_node(&ggtt->error_capture);
2967
2968         if (drm_mm_initialized(&ggtt->vm.mm)) {
2969                 intel_vgt_deballoon(dev_priv);
2970                 i915_address_space_fini(&ggtt->vm);
2971         }
2972
2973         ggtt->vm.cleanup(&ggtt->vm);
2974
2975         pvec = &dev_priv->mm.wc_stash.pvec;
2976         if (pvec->nr) {
2977                 set_pages_array_wb(pvec->pages, pvec->nr);
2978                 __pagevec_release(pvec);
2979         }
2980
2981         mutex_unlock(&dev_priv->drm.struct_mutex);
2982
2983         arch_phys_wc_del(ggtt->mtrr);
2984         io_mapping_fini(&ggtt->iomap);
2985
2986         i915_gem_cleanup_stolen(&dev_priv->drm);
2987 }
2988
2989 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2990 {
2991         snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2992         snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2993         return snb_gmch_ctl << 20;
2994 }
2995
2996 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2997 {
2998         bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2999         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
3000         if (bdw_gmch_ctl)
3001                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
3002
3003 #ifdef CONFIG_X86_32
3004         /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
3005         if (bdw_gmch_ctl > 4)
3006                 bdw_gmch_ctl = 4;
3007 #endif
3008
3009         return bdw_gmch_ctl << 20;
3010 }
3011
3012 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
3013 {
3014         gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
3015         gmch_ctrl &= SNB_GMCH_GGMS_MASK;
3016
3017         if (gmch_ctrl)
3018                 return 1 << (20 + gmch_ctrl);
3019
3020         return 0;
3021 }
3022
3023 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
3024 {
3025         struct drm_i915_private *dev_priv = ggtt->vm.i915;
3026         struct pci_dev *pdev = dev_priv->drm.pdev;
3027         phys_addr_t phys_addr;
3028         int ret;
3029
3030         /* For Modern GENs the PTEs and register space are split in the BAR */
3031         phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
3032
3033         /*
3034          * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
3035          * will be dropped. For WC mappings in general we have 64 byte burst
3036          * writes when the WC buffer is flushed, so we can't use it, but have to
3037          * resort to an uncached mapping. The WC issue is easily caught by the
3038          * readback check when writing GTT PTE entries.
3039          */
3040         if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10)
3041                 ggtt->gsm = ioremap_nocache(phys_addr, size);
3042         else
3043                 ggtt->gsm = ioremap_wc(phys_addr, size);
3044         if (!ggtt->gsm) {
3045                 DRM_ERROR("Failed to map the ggtt page table\n");
3046                 return -ENOMEM;
3047         }
3048
3049         ret = setup_scratch_page(&ggtt->vm, GFP_DMA32);
3050         if (ret) {
3051                 DRM_ERROR("Scratch setup failed\n");
3052                 /* iounmap will also get called at remove, but meh */
3053                 iounmap(ggtt->gsm);
3054                 return ret;
3055         }
3056
3057         return 0;
3058 }
3059
3060 static struct intel_ppat_entry *
3061 __alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value)
3062 {
3063         struct intel_ppat_entry *entry = &ppat->entries[index];
3064
3065         GEM_BUG_ON(index >= ppat->max_entries);
3066         GEM_BUG_ON(test_bit(index, ppat->used));
3067
3068         entry->ppat = ppat;
3069         entry->value = value;
3070         kref_init(&entry->ref);
3071         set_bit(index, ppat->used);
3072         set_bit(index, ppat->dirty);
3073
3074         return entry;
3075 }
3076
3077 static void __free_ppat_entry(struct intel_ppat_entry *entry)
3078 {
3079         struct intel_ppat *ppat = entry->ppat;
3080         unsigned int index = entry - ppat->entries;
3081
3082         GEM_BUG_ON(index >= ppat->max_entries);
3083         GEM_BUG_ON(!test_bit(index, ppat->used));
3084
3085         entry->value = ppat->clear_value;
3086         clear_bit(index, ppat->used);
3087         set_bit(index, ppat->dirty);
3088 }
3089
3090 /**
3091  * intel_ppat_get - get a usable PPAT entry
3092  * @i915: i915 device instance
3093  * @value: the PPAT value required by the caller
3094  *
3095  * The function tries to search if there is an existing PPAT entry which
3096  * matches with the required value. If perfectly matched, the existing PPAT
3097  * entry will be used. If only partially matched, it will try to check if
3098  * there is any available PPAT index. If yes, it will allocate a new PPAT
3099  * index for the required entry and update the HW. If not, the partially
3100  * matched entry will be used.
3101  */
3102 const struct intel_ppat_entry *
3103 intel_ppat_get(struct drm_i915_private *i915, u8 value)
3104 {
3105         struct intel_ppat *ppat = &i915->ppat;
3106         struct intel_ppat_entry *entry = NULL;
3107         unsigned int scanned, best_score;
3108         int i;
3109
3110         GEM_BUG_ON(!ppat->max_entries);
3111
3112         scanned = best_score = 0;
3113         for_each_set_bit(i, ppat->used, ppat->max_entries) {
3114                 unsigned int score;
3115
3116                 score = ppat->match(ppat->entries[i].value, value);
3117                 if (score > best_score) {
3118                         entry = &ppat->entries[i];
3119                         if (score == INTEL_PPAT_PERFECT_MATCH) {
3120                                 kref_get(&entry->ref);
3121                                 return entry;
3122                         }
3123                         best_score = score;
3124                 }
3125                 scanned++;
3126         }
3127
3128         if (scanned == ppat->max_entries) {
3129                 if (!entry)
3130                         return ERR_PTR(-ENOSPC);
3131
3132                 kref_get(&entry->ref);
3133                 return entry;
3134         }
3135
3136         i = find_first_zero_bit(ppat->used, ppat->max_entries);
3137         entry = __alloc_ppat_entry(ppat, i, value);
3138         ppat->update_hw(i915);
3139         return entry;
3140 }
3141
3142 static void release_ppat(struct kref *kref)
3143 {
3144         struct intel_ppat_entry *entry =
3145                 container_of(kref, struct intel_ppat_entry, ref);
3146         struct drm_i915_private *i915 = entry->ppat->i915;
3147
3148         __free_ppat_entry(entry);
3149         entry->ppat->update_hw(i915);
3150 }
3151
3152 /**
3153  * intel_ppat_put - put back the PPAT entry got from intel_ppat_get()
3154  * @entry: an intel PPAT entry
3155  *
3156  * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the
3157  * entry is dynamically allocated, its reference count will be decreased. Once
3158  * the reference count becomes into zero, the PPAT index becomes free again.
3159  */
3160 void intel_ppat_put(const struct intel_ppat_entry *entry)
3161 {
3162         struct intel_ppat *ppat = entry->ppat;
3163         unsigned int index = entry - ppat->entries;
3164
3165         GEM_BUG_ON(!ppat->max_entries);
3166
3167         kref_put(&ppat->entries[index].ref, release_ppat);
3168 }
3169
3170 static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv)
3171 {
3172         struct intel_ppat *ppat = &dev_priv->ppat;
3173         int i;
3174
3175         for_each_set_bit(i, ppat->dirty, ppat->max_entries) {
3176                 I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value);
3177                 clear_bit(i, ppat->dirty);
3178         }
3179 }
3180
3181 static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv)
3182 {
3183         struct intel_ppat *ppat = &dev_priv->ppat;
3184         u64 pat = 0;
3185         int i;
3186
3187         for (i = 0; i < ppat->max_entries; i++)
3188                 pat |= GEN8_PPAT(i, ppat->entries[i].value);
3189
3190         bitmap_clear(ppat->dirty, 0, ppat->max_entries);
3191
3192         I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
3193         I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
3194 }
3195
3196 static unsigned int bdw_private_pat_match(u8 src, u8 dst)
3197 {
3198         unsigned int score = 0;
3199         enum {
3200                 AGE_MATCH = BIT(0),
3201                 TC_MATCH = BIT(1),
3202                 CA_MATCH = BIT(2),
3203         };
3204
3205         /* Cache attribute has to be matched. */
3206         if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst))
3207                 return 0;
3208
3209         score |= CA_MATCH;
3210
3211         if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst))
3212                 score |= TC_MATCH;
3213
3214         if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst))
3215                 score |= AGE_MATCH;
3216
3217         if (score == (AGE_MATCH | TC_MATCH | CA_MATCH))
3218                 return INTEL_PPAT_PERFECT_MATCH;
3219
3220         return score;
3221 }
3222
3223 static unsigned int chv_private_pat_match(u8 src, u8 dst)
3224 {
3225         return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ?
3226                 INTEL_PPAT_PERFECT_MATCH : 0;
3227 }
3228
3229 static void cnl_setup_private_ppat(struct intel_ppat *ppat)
3230 {
3231         ppat->max_entries = 8;
3232         ppat->update_hw = cnl_private_pat_update_hw;
3233         ppat->match = bdw_private_pat_match;
3234         ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3235
3236         __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
3237         __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
3238         __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
3239         __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
3240         __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3241         __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3242         __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3243         __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3244 }
3245
3246 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
3247  * bits. When using advanced contexts each context stores its own PAT, but
3248  * writing this data shouldn't be harmful even in those cases. */
3249 static void bdw_setup_private_ppat(struct intel_ppat *ppat)
3250 {
3251         ppat->max_entries = 8;
3252         ppat->update_hw = bdw_private_pat_update_hw;
3253         ppat->match = bdw_private_pat_match;
3254         ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
3255
3256         if (!USES_PPGTT(ppat->i915)) {
3257                 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
3258                  * so RTL will always use the value corresponding to
3259                  * pat_sel = 000".
3260                  * So let's disable cache for GGTT to avoid screen corruptions.
3261                  * MOCS still can be used though.
3262                  * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
3263                  * before this patch, i.e. the same uncached + snooping access
3264                  * like on gen6/7 seems to be in effect.
3265                  * - So this just fixes blitter/render access. Again it looks
3266                  * like it's not just uncached access, but uncached + snooping.
3267                  * So we can still hold onto all our assumptions wrt cpu
3268                  * clflushing on LLC machines.
3269                  */
3270                 __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
3271                 return;
3272         }
3273
3274         __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);      /* for normal objects, no eLLC */
3275         __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);  /* for something pointing to ptes? */
3276         __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);  /* for scanout with eLLC */
3277         __alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);                      /* Uncached objects, mostly for scanout */
3278         __alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
3279         __alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
3280         __alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
3281         __alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
3282 }
3283
3284 static void chv_setup_private_ppat(struct intel_ppat *ppat)
3285 {
3286         ppat->max_entries = 8;
3287         ppat->update_hw = bdw_private_pat_update_hw;
3288         ppat->match = chv_private_pat_match;
3289         ppat->clear_value = CHV_PPAT_SNOOP;
3290
3291         /*
3292          * Map WB on BDW to snooped on CHV.
3293          *
3294          * Only the snoop bit has meaning for CHV, the rest is
3295          * ignored.
3296          *
3297          * The hardware will never snoop for certain types of accesses:
3298          * - CPU GTT (GMADR->GGTT->no snoop->memory)
3299          * - PPGTT page tables
3300          * - some other special cycles
3301          *
3302          * As with BDW, we also need to consider the following for GT accesses:
3303          * "For GGTT, there is NO pat_sel[2:0] from the entry,
3304          * so RTL will always use the value corresponding to
3305          * pat_sel = 000".
3306          * Which means we must set the snoop bit in PAT entry 0
3307          * in order to keep the global status page working.
3308          */
3309
3310         __alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP);
3311         __alloc_ppat_entry(ppat, 1, 0);
3312         __alloc_ppat_entry(ppat, 2, 0);
3313         __alloc_ppat_entry(ppat, 3, 0);
3314         __alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP);
3315         __alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP);