Merge tag 'drm-intel-next-2017-09-07' of git://anongit.freedesktop.org/git/drm-intel...
[muen/linux.git] / drivers / gpu / drm / i915 / i915_gem.c
1 /*
2  * Copyright © 2008-2015 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  * Authors:
24  *    Eric Anholt <eric@anholt.net>
25  *
26  */
27
28 #include <drm/drmP.h>
29 #include <drm/drm_vma_manager.h>
30 #include <drm/i915_drm.h>
31 #include "i915_drv.h"
32 #include "i915_gem_clflush.h"
33 #include "i915_vgpu.h"
34 #include "i915_trace.h"
35 #include "intel_drv.h"
36 #include "intel_frontbuffer.h"
37 #include "intel_mocs.h"
38 #include <linux/dma-fence-array.h>
39 #include <linux/kthread.h>
40 #include <linux/reservation.h>
41 #include <linux/shmem_fs.h>
42 #include <linux/slab.h>
43 #include <linux/stop_machine.h>
44 #include <linux/swap.h>
45 #include <linux/pci.h>
46 #include <linux/dma-buf.h>
47
48 static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
49
50 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
51 {
52         if (obj->cache_dirty)
53                 return false;
54
55         if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
56                 return true;
57
58         return obj->pin_display;
59 }
60
61 static int
62 insert_mappable_node(struct i915_ggtt *ggtt,
63                      struct drm_mm_node *node, u32 size)
64 {
65         memset(node, 0, sizeof(*node));
66         return drm_mm_insert_node_in_range(&ggtt->base.mm, node,
67                                            size, 0, I915_COLOR_UNEVICTABLE,
68                                            0, ggtt->mappable_end,
69                                            DRM_MM_INSERT_LOW);
70 }
71
72 static void
73 remove_mappable_node(struct drm_mm_node *node)
74 {
75         drm_mm_remove_node(node);
76 }
77
78 /* some bookkeeping */
79 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
80                                   u64 size)
81 {
82         spin_lock(&dev_priv->mm.object_stat_lock);
83         dev_priv->mm.object_count++;
84         dev_priv->mm.object_memory += size;
85         spin_unlock(&dev_priv->mm.object_stat_lock);
86 }
87
88 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
89                                      u64 size)
90 {
91         spin_lock(&dev_priv->mm.object_stat_lock);
92         dev_priv->mm.object_count--;
93         dev_priv->mm.object_memory -= size;
94         spin_unlock(&dev_priv->mm.object_stat_lock);
95 }
96
97 static int
98 i915_gem_wait_for_error(struct i915_gpu_error *error)
99 {
100         int ret;
101
102         might_sleep();
103
104         /*
105          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
106          * userspace. If it takes that long something really bad is going on and
107          * we should simply try to bail out and fail as gracefully as possible.
108          */
109         ret = wait_event_interruptible_timeout(error->reset_queue,
110                                                !i915_reset_backoff(error),
111                                                I915_RESET_TIMEOUT);
112         if (ret == 0) {
113                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
114                 return -EIO;
115         } else if (ret < 0) {
116                 return ret;
117         } else {
118                 return 0;
119         }
120 }
121
122 int i915_mutex_lock_interruptible(struct drm_device *dev)
123 {
124         struct drm_i915_private *dev_priv = to_i915(dev);
125         int ret;
126
127         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
128         if (ret)
129                 return ret;
130
131         ret = mutex_lock_interruptible(&dev->struct_mutex);
132         if (ret)
133                 return ret;
134
135         return 0;
136 }
137
138 int
139 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
140                             struct drm_file *file)
141 {
142         struct drm_i915_private *dev_priv = to_i915(dev);
143         struct i915_ggtt *ggtt = &dev_priv->ggtt;
144         struct drm_i915_gem_get_aperture *args = data;
145         struct i915_vma *vma;
146         u64 pinned;
147
148         pinned = ggtt->base.reserved;
149         mutex_lock(&dev->struct_mutex);
150         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
151                 if (i915_vma_is_pinned(vma))
152                         pinned += vma->node.size;
153         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
154                 if (i915_vma_is_pinned(vma))
155                         pinned += vma->node.size;
156         mutex_unlock(&dev->struct_mutex);
157
158         args->aper_size = ggtt->base.total;
159         args->aper_available_size = args->aper_size - pinned;
160
161         return 0;
162 }
163
164 static struct sg_table *
165 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
166 {
167         struct address_space *mapping = obj->base.filp->f_mapping;
168         drm_dma_handle_t *phys;
169         struct sg_table *st;
170         struct scatterlist *sg;
171         char *vaddr;
172         int i;
173
174         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
175                 return ERR_PTR(-EINVAL);
176
177         /* Always aligning to the object size, allows a single allocation
178          * to handle all possible callers, and given typical object sizes,
179          * the alignment of the buddy allocation will naturally match.
180          */
181         phys = drm_pci_alloc(obj->base.dev,
182                              obj->base.size,
183                              roundup_pow_of_two(obj->base.size));
184         if (!phys)
185                 return ERR_PTR(-ENOMEM);
186
187         vaddr = phys->vaddr;
188         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
189                 struct page *page;
190                 char *src;
191
192                 page = shmem_read_mapping_page(mapping, i);
193                 if (IS_ERR(page)) {
194                         st = ERR_CAST(page);
195                         goto err_phys;
196                 }
197
198                 src = kmap_atomic(page);
199                 memcpy(vaddr, src, PAGE_SIZE);
200                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
201                 kunmap_atomic(src);
202
203                 put_page(page);
204                 vaddr += PAGE_SIZE;
205         }
206
207         i915_gem_chipset_flush(to_i915(obj->base.dev));
208
209         st = kmalloc(sizeof(*st), GFP_KERNEL);
210         if (!st) {
211                 st = ERR_PTR(-ENOMEM);
212                 goto err_phys;
213         }
214
215         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
216                 kfree(st);
217                 st = ERR_PTR(-ENOMEM);
218                 goto err_phys;
219         }
220
221         sg = st->sgl;
222         sg->offset = 0;
223         sg->length = obj->base.size;
224
225         sg_dma_address(sg) = phys->busaddr;
226         sg_dma_len(sg) = obj->base.size;
227
228         obj->phys_handle = phys;
229         return st;
230
231 err_phys:
232         drm_pci_free(obj->base.dev, phys);
233         return st;
234 }
235
236 static void __start_cpu_write(struct drm_i915_gem_object *obj)
237 {
238         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
239         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
240         if (cpu_write_needs_clflush(obj))
241                 obj->cache_dirty = true;
242 }
243
244 static void
245 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
246                                 struct sg_table *pages,
247                                 bool needs_clflush)
248 {
249         GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
250
251         if (obj->mm.madv == I915_MADV_DONTNEED)
252                 obj->mm.dirty = false;
253
254         if (needs_clflush &&
255             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
256             !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
257                 drm_clflush_sg(pages);
258
259         __start_cpu_write(obj);
260 }
261
262 static void
263 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
264                                struct sg_table *pages)
265 {
266         __i915_gem_object_release_shmem(obj, pages, false);
267
268         if (obj->mm.dirty) {
269                 struct address_space *mapping = obj->base.filp->f_mapping;
270                 char *vaddr = obj->phys_handle->vaddr;
271                 int i;
272
273                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
274                         struct page *page;
275                         char *dst;
276
277                         page = shmem_read_mapping_page(mapping, i);
278                         if (IS_ERR(page))
279                                 continue;
280
281                         dst = kmap_atomic(page);
282                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
283                         memcpy(dst, vaddr, PAGE_SIZE);
284                         kunmap_atomic(dst);
285
286                         set_page_dirty(page);
287                         if (obj->mm.madv == I915_MADV_WILLNEED)
288                                 mark_page_accessed(page);
289                         put_page(page);
290                         vaddr += PAGE_SIZE;
291                 }
292                 obj->mm.dirty = false;
293         }
294
295         sg_free_table(pages);
296         kfree(pages);
297
298         drm_pci_free(obj->base.dev, obj->phys_handle);
299 }
300
301 static void
302 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
303 {
304         i915_gem_object_unpin_pages(obj);
305 }
306
307 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
308         .get_pages = i915_gem_object_get_pages_phys,
309         .put_pages = i915_gem_object_put_pages_phys,
310         .release = i915_gem_object_release_phys,
311 };
312
313 static const struct drm_i915_gem_object_ops i915_gem_object_ops;
314
315 int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
316 {
317         struct i915_vma *vma;
318         LIST_HEAD(still_in_list);
319         int ret;
320
321         lockdep_assert_held(&obj->base.dev->struct_mutex);
322
323         /* Closed vma are removed from the obj->vma_list - but they may
324          * still have an active binding on the object. To remove those we
325          * must wait for all rendering to complete to the object (as unbinding
326          * must anyway), and retire the requests.
327          */
328         ret = i915_gem_object_wait(obj,
329                                    I915_WAIT_INTERRUPTIBLE |
330                                    I915_WAIT_LOCKED |
331                                    I915_WAIT_ALL,
332                                    MAX_SCHEDULE_TIMEOUT,
333                                    NULL);
334         if (ret)
335                 return ret;
336
337         i915_gem_retire_requests(to_i915(obj->base.dev));
338
339         while ((vma = list_first_entry_or_null(&obj->vma_list,
340                                                struct i915_vma,
341                                                obj_link))) {
342                 list_move_tail(&vma->obj_link, &still_in_list);
343                 ret = i915_vma_unbind(vma);
344                 if (ret)
345                         break;
346         }
347         list_splice(&still_in_list, &obj->vma_list);
348
349         return ret;
350 }
351
352 static long
353 i915_gem_object_wait_fence(struct dma_fence *fence,
354                            unsigned int flags,
355                            long timeout,
356                            struct intel_rps_client *rps)
357 {
358         struct drm_i915_gem_request *rq;
359
360         BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
361
362         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
363                 return timeout;
364
365         if (!dma_fence_is_i915(fence))
366                 return dma_fence_wait_timeout(fence,
367                                               flags & I915_WAIT_INTERRUPTIBLE,
368                                               timeout);
369
370         rq = to_request(fence);
371         if (i915_gem_request_completed(rq))
372                 goto out;
373
374         /* This client is about to stall waiting for the GPU. In many cases
375          * this is undesirable and limits the throughput of the system, as
376          * many clients cannot continue processing user input/output whilst
377          * blocked. RPS autotuning may take tens of milliseconds to respond
378          * to the GPU load and thus incurs additional latency for the client.
379          * We can circumvent that by promoting the GPU frequency to maximum
380          * before we wait. This makes the GPU throttle up much more quickly
381          * (good for benchmarks and user experience, e.g. window animations),
382          * but at a cost of spending more power processing the workload
383          * (bad for battery). Not all clients even want their results
384          * immediately and for them we should just let the GPU select its own
385          * frequency to maximise efficiency. To prevent a single client from
386          * forcing the clocks too high for the whole system, we only allow
387          * each client to waitboost once in a busy period.
388          */
389         if (rps) {
390                 if (INTEL_GEN(rq->i915) >= 6)
391                         gen6_rps_boost(rq, rps);
392                 else
393                         rps = NULL;
394         }
395
396         timeout = i915_wait_request(rq, flags, timeout);
397
398 out:
399         if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
400                 i915_gem_request_retire_upto(rq);
401
402         return timeout;
403 }
404
405 static long
406 i915_gem_object_wait_reservation(struct reservation_object *resv,
407                                  unsigned int flags,
408                                  long timeout,
409                                  struct intel_rps_client *rps)
410 {
411         unsigned int seq = __read_seqcount_begin(&resv->seq);
412         struct dma_fence *excl;
413         bool prune_fences = false;
414
415         if (flags & I915_WAIT_ALL) {
416                 struct dma_fence **shared;
417                 unsigned int count, i;
418                 int ret;
419
420                 ret = reservation_object_get_fences_rcu(resv,
421                                                         &excl, &count, &shared);
422                 if (ret)
423                         return ret;
424
425                 for (i = 0; i < count; i++) {
426                         timeout = i915_gem_object_wait_fence(shared[i],
427                                                              flags, timeout,
428                                                              rps);
429                         if (timeout < 0)
430                                 break;
431
432                         dma_fence_put(shared[i]);
433                 }
434
435                 for (; i < count; i++)
436                         dma_fence_put(shared[i]);
437                 kfree(shared);
438
439                 prune_fences = count && timeout >= 0;
440         } else {
441                 excl = reservation_object_get_excl_rcu(resv);
442         }
443
444         if (excl && timeout >= 0) {
445                 timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps);
446                 prune_fences = timeout >= 0;
447         }
448
449         dma_fence_put(excl);
450
451         /* Oportunistically prune the fences iff we know they have *all* been
452          * signaled and that the reservation object has not been changed (i.e.
453          * no new fences have been added).
454          */
455         if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
456                 if (reservation_object_trylock(resv)) {
457                         if (!__read_seqcount_retry(&resv->seq, seq))
458                                 reservation_object_add_excl_fence(resv, NULL);
459                         reservation_object_unlock(resv);
460                 }
461         }
462
463         return timeout;
464 }
465
466 static void __fence_set_priority(struct dma_fence *fence, int prio)
467 {
468         struct drm_i915_gem_request *rq;
469         struct intel_engine_cs *engine;
470
471         if (!dma_fence_is_i915(fence))
472                 return;
473
474         rq = to_request(fence);
475         engine = rq->engine;
476         if (!engine->schedule)
477                 return;
478
479         engine->schedule(rq, prio);
480 }
481
482 static void fence_set_priority(struct dma_fence *fence, int prio)
483 {
484         /* Recurse once into a fence-array */
485         if (dma_fence_is_array(fence)) {
486                 struct dma_fence_array *array = to_dma_fence_array(fence);
487                 int i;
488
489                 for (i = 0; i < array->num_fences; i++)
490                         __fence_set_priority(array->fences[i], prio);
491         } else {
492                 __fence_set_priority(fence, prio);
493         }
494 }
495
496 int
497 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
498                               unsigned int flags,
499                               int prio)
500 {
501         struct dma_fence *excl;
502
503         if (flags & I915_WAIT_ALL) {
504                 struct dma_fence **shared;
505                 unsigned int count, i;
506                 int ret;
507
508                 ret = reservation_object_get_fences_rcu(obj->resv,
509                                                         &excl, &count, &shared);
510                 if (ret)
511                         return ret;
512
513                 for (i = 0; i < count; i++) {
514                         fence_set_priority(shared[i], prio);
515                         dma_fence_put(shared[i]);
516                 }
517
518                 kfree(shared);
519         } else {
520                 excl = reservation_object_get_excl_rcu(obj->resv);
521         }
522
523         if (excl) {
524                 fence_set_priority(excl, prio);
525                 dma_fence_put(excl);
526         }
527         return 0;
528 }
529
530 /**
531  * Waits for rendering to the object to be completed
532  * @obj: i915 gem object
533  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
534  * @timeout: how long to wait
535  * @rps: client (user process) to charge for any waitboosting
536  */
537 int
538 i915_gem_object_wait(struct drm_i915_gem_object *obj,
539                      unsigned int flags,
540                      long timeout,
541                      struct intel_rps_client *rps)
542 {
543         might_sleep();
544 #if IS_ENABLED(CONFIG_LOCKDEP)
545         GEM_BUG_ON(debug_locks &&
546                    !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
547                    !!(flags & I915_WAIT_LOCKED));
548 #endif
549         GEM_BUG_ON(timeout < 0);
550
551         timeout = i915_gem_object_wait_reservation(obj->resv,
552                                                    flags, timeout,
553                                                    rps);
554         return timeout < 0 ? timeout : 0;
555 }
556
557 static struct intel_rps_client *to_rps_client(struct drm_file *file)
558 {
559         struct drm_i915_file_private *fpriv = file->driver_priv;
560
561         return &fpriv->rps;
562 }
563
564 static int
565 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
566                      struct drm_i915_gem_pwrite *args,
567                      struct drm_file *file)
568 {
569         void *vaddr = obj->phys_handle->vaddr + args->offset;
570         char __user *user_data = u64_to_user_ptr(args->data_ptr);
571
572         /* We manually control the domain here and pretend that it
573          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
574          */
575         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
576         if (copy_from_user(vaddr, user_data, args->size))
577                 return -EFAULT;
578
579         drm_clflush_virt_range(vaddr, args->size);
580         i915_gem_chipset_flush(to_i915(obj->base.dev));
581
582         intel_fb_obj_flush(obj, ORIGIN_CPU);
583         return 0;
584 }
585
586 void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
587 {
588         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
589 }
590
591 void i915_gem_object_free(struct drm_i915_gem_object *obj)
592 {
593         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
594         kmem_cache_free(dev_priv->objects, obj);
595 }
596
597 static int
598 i915_gem_create(struct drm_file *file,
599                 struct drm_i915_private *dev_priv,
600                 uint64_t size,
601                 uint32_t *handle_p)
602 {
603         struct drm_i915_gem_object *obj;
604         int ret;
605         u32 handle;
606
607         size = roundup(size, PAGE_SIZE);
608         if (size == 0)
609                 return -EINVAL;
610
611         /* Allocate the new object */
612         obj = i915_gem_object_create(dev_priv, size);
613         if (IS_ERR(obj))
614                 return PTR_ERR(obj);
615
616         ret = drm_gem_handle_create(file, &obj->base, &handle);
617         /* drop reference from allocate - handle holds it now */
618         i915_gem_object_put(obj);
619         if (ret)
620                 return ret;
621
622         *handle_p = handle;
623         return 0;
624 }
625
626 int
627 i915_gem_dumb_create(struct drm_file *file,
628                      struct drm_device *dev,
629                      struct drm_mode_create_dumb *args)
630 {
631         /* have to work out size/pitch and return them */
632         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
633         args->size = args->pitch * args->height;
634         return i915_gem_create(file, to_i915(dev),
635                                args->size, &args->handle);
636 }
637
638 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
639 {
640         return !(obj->cache_level == I915_CACHE_NONE ||
641                  obj->cache_level == I915_CACHE_WT);
642 }
643
644 /**
645  * Creates a new mm object and returns a handle to it.
646  * @dev: drm device pointer
647  * @data: ioctl data blob
648  * @file: drm file pointer
649  */
650 int
651 i915_gem_create_ioctl(struct drm_device *dev, void *data,
652                       struct drm_file *file)
653 {
654         struct drm_i915_private *dev_priv = to_i915(dev);
655         struct drm_i915_gem_create *args = data;
656
657         i915_gem_flush_free_objects(dev_priv);
658
659         return i915_gem_create(file, dev_priv,
660                                args->size, &args->handle);
661 }
662
663 static inline enum fb_op_origin
664 fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
665 {
666         return (domain == I915_GEM_DOMAIN_GTT ?
667                 obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
668 }
669
670 static void
671 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
672 {
673         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
674
675         if (!(obj->base.write_domain & flush_domains))
676                 return;
677
678         /* No actual flushing is required for the GTT write domain.  Writes
679          * to it "immediately" go to main memory as far as we know, so there's
680          * no chipset flush.  It also doesn't land in render cache.
681          *
682          * However, we do have to enforce the order so that all writes through
683          * the GTT land before any writes to the device, such as updates to
684          * the GATT itself.
685          *
686          * We also have to wait a bit for the writes to land from the GTT.
687          * An uncached read (i.e. mmio) seems to be ideal for the round-trip
688          * timing. This issue has only been observed when switching quickly
689          * between GTT writes and CPU reads from inside the kernel on recent hw,
690          * and it appears to only affect discrete GTT blocks (i.e. on LLC
691          * system agents we cannot reproduce this behaviour).
692          */
693         wmb();
694
695         switch (obj->base.write_domain) {
696         case I915_GEM_DOMAIN_GTT:
697                 if (INTEL_GEN(dev_priv) >= 6 && !HAS_LLC(dev_priv)) {
698                         intel_runtime_pm_get(dev_priv);
699                         spin_lock_irq(&dev_priv->uncore.lock);
700                         POSTING_READ_FW(RING_ACTHD(dev_priv->engine[RCS]->mmio_base));
701                         spin_unlock_irq(&dev_priv->uncore.lock);
702                         intel_runtime_pm_put(dev_priv);
703                 }
704
705                 intel_fb_obj_flush(obj,
706                                    fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
707                 break;
708
709         case I915_GEM_DOMAIN_CPU:
710                 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
711                 break;
712
713         case I915_GEM_DOMAIN_RENDER:
714                 if (gpu_write_needs_clflush(obj))
715                         obj->cache_dirty = true;
716                 break;
717         }
718
719         obj->base.write_domain = 0;
720 }
721
722 static inline int
723 __copy_to_user_swizzled(char __user *cpu_vaddr,
724                         const char *gpu_vaddr, int gpu_offset,
725                         int length)
726 {
727         int ret, cpu_offset = 0;
728
729         while (length > 0) {
730                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
731                 int this_length = min(cacheline_end - gpu_offset, length);
732                 int swizzled_gpu_offset = gpu_offset ^ 64;
733
734                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
735                                      gpu_vaddr + swizzled_gpu_offset,
736                                      this_length);
737                 if (ret)
738                         return ret + length;
739
740                 cpu_offset += this_length;
741                 gpu_offset += this_length;
742                 length -= this_length;
743         }
744
745         return 0;
746 }
747
748 static inline int
749 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
750                           const char __user *cpu_vaddr,
751                           int length)
752 {
753         int ret, cpu_offset = 0;
754
755         while (length > 0) {
756                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
757                 int this_length = min(cacheline_end - gpu_offset, length);
758                 int swizzled_gpu_offset = gpu_offset ^ 64;
759
760                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
761                                        cpu_vaddr + cpu_offset,
762                                        this_length);
763                 if (ret)
764                         return ret + length;
765
766                 cpu_offset += this_length;
767                 gpu_offset += this_length;
768                 length -= this_length;
769         }
770
771         return 0;
772 }
773
774 /*
775  * Pins the specified object's pages and synchronizes the object with
776  * GPU accesses. Sets needs_clflush to non-zero if the caller should
777  * flush the object from the CPU cache.
778  */
779 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
780                                     unsigned int *needs_clflush)
781 {
782         int ret;
783
784         lockdep_assert_held(&obj->base.dev->struct_mutex);
785
786         *needs_clflush = 0;
787         if (!i915_gem_object_has_struct_page(obj))
788                 return -ENODEV;
789
790         ret = i915_gem_object_wait(obj,
791                                    I915_WAIT_INTERRUPTIBLE |
792                                    I915_WAIT_LOCKED,
793                                    MAX_SCHEDULE_TIMEOUT,
794                                    NULL);
795         if (ret)
796                 return ret;
797
798         ret = i915_gem_object_pin_pages(obj);
799         if (ret)
800                 return ret;
801
802         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
803             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
804                 ret = i915_gem_object_set_to_cpu_domain(obj, false);
805                 if (ret)
806                         goto err_unpin;
807                 else
808                         goto out;
809         }
810
811         flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
812
813         /* If we're not in the cpu read domain, set ourself into the gtt
814          * read domain and manually flush cachelines (if required). This
815          * optimizes for the case when the gpu will dirty the data
816          * anyway again before the next pread happens.
817          */
818         if (!obj->cache_dirty &&
819             !(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
820                 *needs_clflush = CLFLUSH_BEFORE;
821
822 out:
823         /* return with the pages pinned */
824         return 0;
825
826 err_unpin:
827         i915_gem_object_unpin_pages(obj);
828         return ret;
829 }
830
831 int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
832                                      unsigned int *needs_clflush)
833 {
834         int ret;
835
836         lockdep_assert_held(&obj->base.dev->struct_mutex);
837
838         *needs_clflush = 0;
839         if (!i915_gem_object_has_struct_page(obj))
840                 return -ENODEV;
841
842         ret = i915_gem_object_wait(obj,
843                                    I915_WAIT_INTERRUPTIBLE |
844                                    I915_WAIT_LOCKED |
845                                    I915_WAIT_ALL,
846                                    MAX_SCHEDULE_TIMEOUT,
847                                    NULL);
848         if (ret)
849                 return ret;
850
851         ret = i915_gem_object_pin_pages(obj);
852         if (ret)
853                 return ret;
854
855         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
856             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
857                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
858                 if (ret)
859                         goto err_unpin;
860                 else
861                         goto out;
862         }
863
864         flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
865
866         /* If we're not in the cpu write domain, set ourself into the
867          * gtt write domain and manually flush cachelines (as required).
868          * This optimizes for the case when the gpu will use the data
869          * right away and we therefore have to clflush anyway.
870          */
871         if (!obj->cache_dirty) {
872                 *needs_clflush |= CLFLUSH_AFTER;
873
874                 /*
875                  * Same trick applies to invalidate partially written
876                  * cachelines read before writing.
877                  */
878                 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
879                         *needs_clflush |= CLFLUSH_BEFORE;
880         }
881
882 out:
883         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
884         obj->mm.dirty = true;
885         /* return with the pages pinned */
886         return 0;
887
888 err_unpin:
889         i915_gem_object_unpin_pages(obj);
890         return ret;
891 }
892
893 static void
894 shmem_clflush_swizzled_range(char *addr, unsigned long length,
895                              bool swizzled)
896 {
897         if (unlikely(swizzled)) {
898                 unsigned long start = (unsigned long) addr;
899                 unsigned long end = (unsigned long) addr + length;
900
901                 /* For swizzling simply ensure that we always flush both
902                  * channels. Lame, but simple and it works. Swizzled
903                  * pwrite/pread is far from a hotpath - current userspace
904                  * doesn't use it at all. */
905                 start = round_down(start, 128);
906                 end = round_up(end, 128);
907
908                 drm_clflush_virt_range((void *)start, end - start);
909         } else {
910                 drm_clflush_virt_range(addr, length);
911         }
912
913 }
914
915 /* Only difference to the fast-path function is that this can handle bit17
916  * and uses non-atomic copy and kmap functions. */
917 static int
918 shmem_pread_slow(struct page *page, int offset, int length,
919                  char __user *user_data,
920                  bool page_do_bit17_swizzling, bool needs_clflush)
921 {
922         char *vaddr;
923         int ret;
924
925         vaddr = kmap(page);
926         if (needs_clflush)
927                 shmem_clflush_swizzled_range(vaddr + offset, length,
928                                              page_do_bit17_swizzling);
929
930         if (page_do_bit17_swizzling)
931                 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
932         else
933                 ret = __copy_to_user(user_data, vaddr + offset, length);
934         kunmap(page);
935
936         return ret ? - EFAULT : 0;
937 }
938
939 static int
940 shmem_pread(struct page *page, int offset, int length, char __user *user_data,
941             bool page_do_bit17_swizzling, bool needs_clflush)
942 {
943         int ret;
944
945         ret = -ENODEV;
946         if (!page_do_bit17_swizzling) {
947                 char *vaddr = kmap_atomic(page);
948
949                 if (needs_clflush)
950                         drm_clflush_virt_range(vaddr + offset, length);
951                 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
952                 kunmap_atomic(vaddr);
953         }
954         if (ret == 0)
955                 return 0;
956
957         return shmem_pread_slow(page, offset, length, user_data,
958                                 page_do_bit17_swizzling, needs_clflush);
959 }
960
961 static int
962 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
963                      struct drm_i915_gem_pread *args)
964 {
965         char __user *user_data;
966         u64 remain;
967         unsigned int obj_do_bit17_swizzling;
968         unsigned int needs_clflush;
969         unsigned int idx, offset;
970         int ret;
971
972         obj_do_bit17_swizzling = 0;
973         if (i915_gem_object_needs_bit17_swizzle(obj))
974                 obj_do_bit17_swizzling = BIT(17);
975
976         ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
977         if (ret)
978                 return ret;
979
980         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
981         mutex_unlock(&obj->base.dev->struct_mutex);
982         if (ret)
983                 return ret;
984
985         remain = args->size;
986         user_data = u64_to_user_ptr(args->data_ptr);
987         offset = offset_in_page(args->offset);
988         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
989                 struct page *page = i915_gem_object_get_page(obj, idx);
990                 int length;
991
992                 length = remain;
993                 if (offset + length > PAGE_SIZE)
994                         length = PAGE_SIZE - offset;
995
996                 ret = shmem_pread(page, offset, length, user_data,
997                                   page_to_phys(page) & obj_do_bit17_swizzling,
998                                   needs_clflush);
999                 if (ret)
1000                         break;
1001
1002                 remain -= length;
1003                 user_data += length;
1004                 offset = 0;
1005         }
1006
1007         i915_gem_obj_finish_shmem_access(obj);
1008         return ret;
1009 }
1010
1011 static inline bool
1012 gtt_user_read(struct io_mapping *mapping,
1013               loff_t base, int offset,
1014               char __user *user_data, int length)
1015 {
1016         void __iomem *vaddr;
1017         unsigned long unwritten;
1018
1019         /* We can use the cpu mem copy function because this is X86. */
1020         vaddr = io_mapping_map_atomic_wc(mapping, base);
1021         unwritten = __copy_to_user_inatomic(user_data,
1022                                             (void __force *)vaddr + offset,
1023                                             length);
1024         io_mapping_unmap_atomic(vaddr);
1025         if (unwritten) {
1026                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1027                 unwritten = copy_to_user(user_data,
1028                                          (void __force *)vaddr + offset,
1029                                          length);
1030                 io_mapping_unmap(vaddr);
1031         }
1032         return unwritten;
1033 }
1034
1035 static int
1036 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
1037                    const struct drm_i915_gem_pread *args)
1038 {
1039         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1040         struct i915_ggtt *ggtt = &i915->ggtt;
1041         struct drm_mm_node node;
1042         struct i915_vma *vma;
1043         void __user *user_data;
1044         u64 remain, offset;
1045         int ret;
1046
1047         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1048         if (ret)
1049                 return ret;
1050
1051         intel_runtime_pm_get(i915);
1052         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1053                                        PIN_MAPPABLE | PIN_NONBLOCK);
1054         if (!IS_ERR(vma)) {
1055                 node.start = i915_ggtt_offset(vma);
1056                 node.allocated = false;
1057                 ret = i915_vma_put_fence(vma);
1058                 if (ret) {
1059                         i915_vma_unpin(vma);
1060                         vma = ERR_PTR(ret);
1061                 }
1062         }
1063         if (IS_ERR(vma)) {
1064                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1065                 if (ret)
1066                         goto out_unlock;
1067                 GEM_BUG_ON(!node.allocated);
1068         }
1069
1070         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1071         if (ret)
1072                 goto out_unpin;
1073
1074         mutex_unlock(&i915->drm.struct_mutex);
1075
1076         user_data = u64_to_user_ptr(args->data_ptr);
1077         remain = args->size;
1078         offset = args->offset;
1079
1080         while (remain > 0) {
1081                 /* Operation in this page
1082                  *
1083                  * page_base = page offset within aperture
1084                  * page_offset = offset within page
1085                  * page_length = bytes to copy for this page
1086                  */
1087                 u32 page_base = node.start;
1088                 unsigned page_offset = offset_in_page(offset);
1089                 unsigned page_length = PAGE_SIZE - page_offset;
1090                 page_length = remain < page_length ? remain : page_length;
1091                 if (node.allocated) {
1092                         wmb();
1093                         ggtt->base.insert_page(&ggtt->base,
1094                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1095                                                node.start, I915_CACHE_NONE, 0);
1096                         wmb();
1097                 } else {
1098                         page_base += offset & PAGE_MASK;
1099                 }
1100
1101                 if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
1102                                   user_data, page_length)) {
1103                         ret = -EFAULT;
1104                         break;
1105                 }
1106
1107                 remain -= page_length;
1108                 user_data += page_length;
1109                 offset += page_length;
1110         }
1111
1112         mutex_lock(&i915->drm.struct_mutex);
1113 out_unpin:
1114         if (node.allocated) {
1115                 wmb();
1116                 ggtt->base.clear_range(&ggtt->base,
1117                                        node.start, node.size);
1118                 remove_mappable_node(&node);
1119         } else {
1120                 i915_vma_unpin(vma);
1121         }
1122 out_unlock:
1123         intel_runtime_pm_put(i915);
1124         mutex_unlock(&i915->drm.struct_mutex);
1125
1126         return ret;
1127 }
1128
1129 /**
1130  * Reads data from the object referenced by handle.
1131  * @dev: drm device pointer
1132  * @data: ioctl data blob
1133  * @file: drm file pointer
1134  *
1135  * On error, the contents of *data are undefined.
1136  */
1137 int
1138 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1139                      struct drm_file *file)
1140 {
1141         struct drm_i915_gem_pread *args = data;
1142         struct drm_i915_gem_object *obj;
1143         int ret;
1144
1145         if (args->size == 0)
1146                 return 0;
1147
1148         if (!access_ok(VERIFY_WRITE,
1149                        u64_to_user_ptr(args->data_ptr),
1150                        args->size))
1151                 return -EFAULT;
1152
1153         obj = i915_gem_object_lookup(file, args->handle);
1154         if (!obj)
1155                 return -ENOENT;
1156
1157         /* Bounds check source.  */
1158         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1159                 ret = -EINVAL;
1160                 goto out;
1161         }
1162
1163         trace_i915_gem_object_pread(obj, args->offset, args->size);
1164
1165         ret = i915_gem_object_wait(obj,
1166                                    I915_WAIT_INTERRUPTIBLE,
1167                                    MAX_SCHEDULE_TIMEOUT,
1168                                    to_rps_client(file));
1169         if (ret)
1170                 goto out;
1171
1172         ret = i915_gem_object_pin_pages(obj);
1173         if (ret)
1174                 goto out;
1175
1176         ret = i915_gem_shmem_pread(obj, args);
1177         if (ret == -EFAULT || ret == -ENODEV)
1178                 ret = i915_gem_gtt_pread(obj, args);
1179
1180         i915_gem_object_unpin_pages(obj);
1181 out:
1182         i915_gem_object_put(obj);
1183         return ret;
1184 }
1185
1186 /* This is the fast write path which cannot handle
1187  * page faults in the source data
1188  */
1189
1190 static inline bool
1191 ggtt_write(struct io_mapping *mapping,
1192            loff_t base, int offset,
1193            char __user *user_data, int length)
1194 {
1195         void __iomem *vaddr;
1196         unsigned long unwritten;
1197
1198         /* We can use the cpu mem copy function because this is X86. */
1199         vaddr = io_mapping_map_atomic_wc(mapping, base);
1200         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
1201                                                       user_data, length);
1202         io_mapping_unmap_atomic(vaddr);
1203         if (unwritten) {
1204                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1205                 unwritten = copy_from_user((void __force *)vaddr + offset,
1206                                            user_data, length);
1207                 io_mapping_unmap(vaddr);
1208         }
1209
1210         return unwritten;
1211 }
1212
1213 /**
1214  * This is the fast pwrite path, where we copy the data directly from the
1215  * user into the GTT, uncached.
1216  * @obj: i915 GEM object
1217  * @args: pwrite arguments structure
1218  */
1219 static int
1220 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
1221                          const struct drm_i915_gem_pwrite *args)
1222 {
1223         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1224         struct i915_ggtt *ggtt = &i915->ggtt;
1225         struct drm_mm_node node;
1226         struct i915_vma *vma;
1227         u64 remain, offset;
1228         void __user *user_data;
1229         int ret;
1230
1231         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1232         if (ret)
1233                 return ret;
1234
1235         intel_runtime_pm_get(i915);
1236         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1237                                        PIN_MAPPABLE | PIN_NONBLOCK);
1238         if (!IS_ERR(vma)) {
1239                 node.start = i915_ggtt_offset(vma);
1240                 node.allocated = false;
1241                 ret = i915_vma_put_fence(vma);
1242                 if (ret) {
1243                         i915_vma_unpin(vma);
1244                         vma = ERR_PTR(ret);
1245                 }
1246         }
1247         if (IS_ERR(vma)) {
1248                 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1249                 if (ret)
1250                         goto out_unlock;
1251                 GEM_BUG_ON(!node.allocated);
1252         }
1253
1254         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1255         if (ret)
1256                 goto out_unpin;
1257
1258         mutex_unlock(&i915->drm.struct_mutex);
1259
1260         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1261
1262         user_data = u64_to_user_ptr(args->data_ptr);
1263         offset = args->offset;
1264         remain = args->size;
1265         while (remain) {
1266                 /* Operation in this page
1267                  *
1268                  * page_base = page offset within aperture
1269                  * page_offset = offset within page
1270                  * page_length = bytes to copy for this page
1271                  */
1272                 u32 page_base = node.start;
1273                 unsigned int page_offset = offset_in_page(offset);
1274                 unsigned int page_length = PAGE_SIZE - page_offset;
1275                 page_length = remain < page_length ? remain : page_length;
1276                 if (node.allocated) {
1277                         wmb(); /* flush the write before we modify the GGTT */
1278                         ggtt->base.insert_page(&ggtt->base,
1279                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1280                                                node.start, I915_CACHE_NONE, 0);
1281                         wmb(); /* flush modifications to the GGTT (insert_page) */
1282                 } else {
1283                         page_base += offset & PAGE_MASK;
1284                 }
1285                 /* If we get a fault while copying data, then (presumably) our
1286                  * source page isn't available.  Return the error and we'll
1287                  * retry in the slow path.
1288                  * If the object is non-shmem backed, we retry again with the
1289                  * path that handles page fault.
1290                  */
1291                 if (ggtt_write(&ggtt->mappable, page_base, page_offset,
1292                                user_data, page_length)) {
1293                         ret = -EFAULT;
1294                         break;
1295                 }
1296
1297                 remain -= page_length;
1298                 user_data += page_length;
1299                 offset += page_length;
1300         }
1301         intel_fb_obj_flush(obj, ORIGIN_CPU);
1302
1303         mutex_lock(&i915->drm.struct_mutex);
1304 out_unpin:
1305         if (node.allocated) {
1306                 wmb();
1307                 ggtt->base.clear_range(&ggtt->base,
1308                                        node.start, node.size);
1309                 remove_mappable_node(&node);
1310         } else {
1311                 i915_vma_unpin(vma);
1312         }
1313 out_unlock:
1314         intel_runtime_pm_put(i915);
1315         mutex_unlock(&i915->drm.struct_mutex);
1316         return ret;
1317 }
1318
1319 static int
1320 shmem_pwrite_slow(struct page *page, int offset, int length,
1321                   char __user *user_data,
1322                   bool page_do_bit17_swizzling,
1323                   bool needs_clflush_before,
1324                   bool needs_clflush_after)
1325 {
1326         char *vaddr;
1327         int ret;
1328
1329         vaddr = kmap(page);
1330         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1331                 shmem_clflush_swizzled_range(vaddr + offset, length,
1332                                              page_do_bit17_swizzling);
1333         if (page_do_bit17_swizzling)
1334                 ret = __copy_from_user_swizzled(vaddr, offset, user_data,
1335                                                 length);
1336         else
1337                 ret = __copy_from_user(vaddr + offset, user_data, length);
1338         if (needs_clflush_after)
1339                 shmem_clflush_swizzled_range(vaddr + offset, length,
1340                                              page_do_bit17_swizzling);
1341         kunmap(page);
1342
1343         return ret ? -EFAULT : 0;
1344 }
1345
1346 /* Per-page copy function for the shmem pwrite fastpath.
1347  * Flushes invalid cachelines before writing to the target if
1348  * needs_clflush_before is set and flushes out any written cachelines after
1349  * writing if needs_clflush is set.
1350  */
1351 static int
1352 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1353              bool page_do_bit17_swizzling,
1354              bool needs_clflush_before,
1355              bool needs_clflush_after)
1356 {
1357         int ret;
1358
1359         ret = -ENODEV;
1360         if (!page_do_bit17_swizzling) {
1361                 char *vaddr = kmap_atomic(page);
1362
1363                 if (needs_clflush_before)
1364                         drm_clflush_virt_range(vaddr + offset, len);
1365                 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
1366                 if (needs_clflush_after)
1367                         drm_clflush_virt_range(vaddr + offset, len);
1368
1369                 kunmap_atomic(vaddr);
1370         }
1371         if (ret == 0)
1372                 return ret;
1373
1374         return shmem_pwrite_slow(page, offset, len, user_data,
1375                                  page_do_bit17_swizzling,
1376                                  needs_clflush_before,
1377                                  needs_clflush_after);
1378 }
1379
1380 static int
1381 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1382                       const struct drm_i915_gem_pwrite *args)
1383 {
1384         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1385         void __user *user_data;
1386         u64 remain;
1387         unsigned int obj_do_bit17_swizzling;
1388         unsigned int partial_cacheline_write;
1389         unsigned int needs_clflush;
1390         unsigned int offset, idx;
1391         int ret;
1392
1393         ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1394         if (ret)
1395                 return ret;
1396
1397         ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1398         mutex_unlock(&i915->drm.struct_mutex);
1399         if (ret)
1400                 return ret;
1401
1402         obj_do_bit17_swizzling = 0;
1403         if (i915_gem_object_needs_bit17_swizzle(obj))
1404                 obj_do_bit17_swizzling = BIT(17);
1405
1406         /* If we don't overwrite a cacheline completely we need to be
1407          * careful to have up-to-date data by first clflushing. Don't
1408          * overcomplicate things and flush the entire patch.
1409          */
1410         partial_cacheline_write = 0;
1411         if (needs_clflush & CLFLUSH_BEFORE)
1412                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1413
1414         user_data = u64_to_user_ptr(args->data_ptr);
1415         remain = args->size;
1416         offset = offset_in_page(args->offset);
1417         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1418                 struct page *page = i915_gem_object_get_page(obj, idx);
1419                 int length;
1420
1421                 length = remain;
1422                 if (offset + length > PAGE_SIZE)
1423                         length = PAGE_SIZE - offset;
1424
1425                 ret = shmem_pwrite(page, offset, length, user_data,
1426                                    page_to_phys(page) & obj_do_bit17_swizzling,
1427                                    (offset | length) & partial_cacheline_write,
1428                                    needs_clflush & CLFLUSH_AFTER);
1429                 if (ret)
1430                         break;
1431
1432                 remain -= length;
1433                 user_data += length;
1434                 offset = 0;
1435         }
1436
1437         intel_fb_obj_flush(obj, ORIGIN_CPU);
1438         i915_gem_obj_finish_shmem_access(obj);
1439         return ret;
1440 }
1441
1442 /**
1443  * Writes data to the object referenced by handle.
1444  * @dev: drm device
1445  * @data: ioctl data blob
1446  * @file: drm file
1447  *
1448  * On error, the contents of the buffer that were to be modified are undefined.
1449  */
1450 int
1451 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1452                       struct drm_file *file)
1453 {
1454         struct drm_i915_gem_pwrite *args = data;
1455         struct drm_i915_gem_object *obj;
1456         int ret;
1457
1458         if (args->size == 0)
1459                 return 0;
1460
1461         if (!access_ok(VERIFY_READ,
1462                        u64_to_user_ptr(args->data_ptr),
1463                        args->size))
1464                 return -EFAULT;
1465
1466         obj = i915_gem_object_lookup(file, args->handle);
1467         if (!obj)
1468                 return -ENOENT;
1469
1470         /* Bounds check destination. */
1471         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
1472                 ret = -EINVAL;
1473                 goto err;
1474         }
1475
1476         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1477
1478         ret = -ENODEV;
1479         if (obj->ops->pwrite)
1480                 ret = obj->ops->pwrite(obj, args);
1481         if (ret != -ENODEV)
1482                 goto err;
1483
1484         ret = i915_gem_object_wait(obj,
1485                                    I915_WAIT_INTERRUPTIBLE |
1486                                    I915_WAIT_ALL,
1487                                    MAX_SCHEDULE_TIMEOUT,
1488                                    to_rps_client(file));
1489         if (ret)
1490                 goto err;
1491
1492         ret = i915_gem_object_pin_pages(obj);
1493         if (ret)
1494                 goto err;
1495
1496         ret = -EFAULT;
1497         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1498          * it would end up going through the fenced access, and we'll get
1499          * different detiling behavior between reading and writing.
1500          * pread/pwrite currently are reading and writing from the CPU
1501          * perspective, requiring manual detiling by the client.
1502          */
1503         if (!i915_gem_object_has_struct_page(obj) ||
1504             cpu_write_needs_clflush(obj))
1505                 /* Note that the gtt paths might fail with non-page-backed user
1506                  * pointers (e.g. gtt mappings when moving data between
1507                  * textures). Fallback to the shmem path in that case.
1508                  */
1509                 ret = i915_gem_gtt_pwrite_fast(obj, args);
1510
1511         if (ret == -EFAULT || ret == -ENOSPC) {
1512                 if (obj->phys_handle)
1513                         ret = i915_gem_phys_pwrite(obj, args, file);
1514                 else
1515                         ret = i915_gem_shmem_pwrite(obj, args);
1516         }
1517
1518         i915_gem_object_unpin_pages(obj);
1519 err:
1520         i915_gem_object_put(obj);
1521         return ret;
1522 }
1523
1524 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1525 {
1526         struct drm_i915_private *i915;
1527         struct list_head *list;
1528         struct i915_vma *vma;
1529
1530         list_for_each_entry(vma, &obj->vma_list, obj_link) {
1531                 if (!i915_vma_is_ggtt(vma))
1532                         break;
1533
1534                 if (i915_vma_is_active(vma))
1535                         continue;
1536
1537                 if (!drm_mm_node_allocated(&vma->node))
1538                         continue;
1539
1540                 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1541         }
1542
1543         i915 = to_i915(obj->base.dev);
1544         list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1545         list_move_tail(&obj->global_link, list);
1546 }
1547
1548 /**
1549  * Called when user space prepares to use an object with the CPU, either
1550  * through the mmap ioctl's mapping or a GTT mapping.
1551  * @dev: drm device
1552  * @data: ioctl data blob
1553  * @file: drm file
1554  */
1555 int
1556 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1557                           struct drm_file *file)
1558 {
1559         struct drm_i915_gem_set_domain *args = data;
1560         struct drm_i915_gem_object *obj;
1561         uint32_t read_domains = args->read_domains;
1562         uint32_t write_domain = args->write_domain;
1563         int err;
1564
1565         /* Only handle setting domains to types used by the CPU. */
1566         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1567                 return -EINVAL;
1568
1569         /* Having something in the write domain implies it's in the read
1570          * domain, and only that read domain.  Enforce that in the request.
1571          */
1572         if (write_domain != 0 && read_domains != write_domain)
1573                 return -EINVAL;
1574
1575         obj = i915_gem_object_lookup(file, args->handle);
1576         if (!obj)
1577                 return -ENOENT;
1578
1579         /* Try to flush the object off the GPU without holding the lock.
1580          * We will repeat the flush holding the lock in the normal manner
1581          * to catch cases where we are gazumped.
1582          */
1583         err = i915_gem_object_wait(obj,
1584                                    I915_WAIT_INTERRUPTIBLE |
1585                                    (write_domain ? I915_WAIT_ALL : 0),
1586                                    MAX_SCHEDULE_TIMEOUT,
1587                                    to_rps_client(file));
1588         if (err)
1589                 goto out;
1590
1591         /* Flush and acquire obj->pages so that we are coherent through
1592          * direct access in memory with previous cached writes through
1593          * shmemfs and that our cache domain tracking remains valid.
1594          * For example, if the obj->filp was moved to swap without us
1595          * being notified and releasing the pages, we would mistakenly
1596          * continue to assume that the obj remained out of the CPU cached
1597          * domain.
1598          */
1599         err = i915_gem_object_pin_pages(obj);
1600         if (err)
1601                 goto out;
1602
1603         err = i915_mutex_lock_interruptible(dev);
1604         if (err)
1605                 goto out_unpin;
1606
1607         if (read_domains & I915_GEM_DOMAIN_WC)
1608                 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
1609         else if (read_domains & I915_GEM_DOMAIN_GTT)
1610                 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
1611         else
1612                 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
1613
1614         /* And bump the LRU for this access */
1615         i915_gem_object_bump_inactive_ggtt(obj);
1616
1617         mutex_unlock(&dev->struct_mutex);
1618
1619         if (write_domain != 0)
1620                 intel_fb_obj_invalidate(obj,
1621                                         fb_write_origin(obj, write_domain));
1622
1623 out_unpin:
1624         i915_gem_object_unpin_pages(obj);
1625 out:
1626         i915_gem_object_put(obj);
1627         return err;
1628 }
1629
1630 /**
1631  * Called when user space has done writes to this buffer
1632  * @dev: drm device
1633  * @data: ioctl data blob
1634  * @file: drm file
1635  */
1636 int
1637 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1638                          struct drm_file *file)
1639 {
1640         struct drm_i915_gem_sw_finish *args = data;
1641         struct drm_i915_gem_object *obj;
1642
1643         obj = i915_gem_object_lookup(file, args->handle);
1644         if (!obj)
1645                 return -ENOENT;
1646
1647         /* Pinned buffers may be scanout, so flush the cache */
1648         i915_gem_object_flush_if_display(obj);
1649         i915_gem_object_put(obj);
1650
1651         return 0;
1652 }
1653
1654 /**
1655  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1656  *                       it is mapped to.
1657  * @dev: drm device
1658  * @data: ioctl data blob
1659  * @file: drm file
1660  *
1661  * While the mapping holds a reference on the contents of the object, it doesn't
1662  * imply a ref on the object itself.
1663  *
1664  * IMPORTANT:
1665  *
1666  * DRM driver writers who look a this function as an example for how to do GEM
1667  * mmap support, please don't implement mmap support like here. The modern way
1668  * to implement DRM mmap support is with an mmap offset ioctl (like
1669  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1670  * That way debug tooling like valgrind will understand what's going on, hiding
1671  * the mmap call in a driver private ioctl will break that. The i915 driver only
1672  * does cpu mmaps this way because we didn't know better.
1673  */
1674 int
1675 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1676                     struct drm_file *file)
1677 {
1678         struct drm_i915_gem_mmap *args = data;
1679         struct drm_i915_gem_object *obj;
1680         unsigned long addr;
1681
1682         if (args->flags & ~(I915_MMAP_WC))
1683                 return -EINVAL;
1684
1685         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1686                 return -ENODEV;
1687
1688         obj = i915_gem_object_lookup(file, args->handle);
1689         if (!obj)
1690                 return -ENOENT;
1691
1692         /* prime objects have no backing filp to GEM mmap
1693          * pages from.
1694          */
1695         if (!obj->base.filp) {
1696                 i915_gem_object_put(obj);
1697                 return -EINVAL;
1698         }
1699
1700         addr = vm_mmap(obj->base.filp, 0, args->size,
1701                        PROT_READ | PROT_WRITE, MAP_SHARED,
1702                        args->offset);
1703         if (args->flags & I915_MMAP_WC) {
1704                 struct mm_struct *mm = current->mm;
1705                 struct vm_area_struct *vma;
1706
1707                 if (down_write_killable(&mm->mmap_sem)) {
1708                         i915_gem_object_put(obj);
1709                         return -EINTR;
1710                 }
1711                 vma = find_vma(mm, addr);
1712                 if (vma)
1713                         vma->vm_page_prot =
1714                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1715                 else
1716                         addr = -ENOMEM;
1717                 up_write(&mm->mmap_sem);
1718
1719                 /* This may race, but that's ok, it only gets set */
1720                 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1721         }
1722         i915_gem_object_put(obj);
1723         if (IS_ERR((void *)addr))
1724                 return addr;
1725
1726         args->addr_ptr = (uint64_t) addr;
1727
1728         return 0;
1729 }
1730
1731 static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
1732 {
1733         return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
1734 }
1735
1736 /**
1737  * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1738  *
1739  * A history of the GTT mmap interface:
1740  *
1741  * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1742  *     aligned and suitable for fencing, and still fit into the available
1743  *     mappable space left by the pinned display objects. A classic problem
1744  *     we called the page-fault-of-doom where we would ping-pong between
1745  *     two objects that could not fit inside the GTT and so the memcpy
1746  *     would page one object in at the expense of the other between every
1747  *     single byte.
1748  *
1749  * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1750  *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1751  *     object is too large for the available space (or simply too large
1752  *     for the mappable aperture!), a view is created instead and faulted
1753  *     into userspace. (This view is aligned and sized appropriately for
1754  *     fenced access.)
1755  *
1756  * 2 - Recognise WC as a separate cache domain so that we can flush the
1757  *     delayed writes via GTT before performing direct access via WC.
1758  *
1759  * Restrictions:
1760  *
1761  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
1762  *    hangs on some architectures, corruption on others. An attempt to service
1763  *    a GTT page fault from a snoopable object will generate a SIGBUS.
1764  *
1765  *  * the object must be able to fit into RAM (physical memory, though no
1766  *    limited to the mappable aperture).
1767  *
1768  *
1769  * Caveats:
1770  *
1771  *  * a new GTT page fault will synchronize rendering from the GPU and flush
1772  *    all data to system memory. Subsequent access will not be synchronized.
1773  *
1774  *  * all mappings are revoked on runtime device suspend.
1775  *
1776  *  * there are only 8, 16 or 32 fence registers to share between all users
1777  *    (older machines require fence register for display and blitter access
1778  *    as well). Contention of the fence registers will cause the previous users
1779  *    to be unmapped and any new access will generate new page faults.
1780  *
1781  *  * running out of memory while servicing a fault may generate a SIGBUS,
1782  *    rather than the expected SIGSEGV.
1783  */
1784 int i915_gem_mmap_gtt_version(void)
1785 {
1786         return 2;
1787 }
1788
1789 static inline struct i915_ggtt_view
1790 compute_partial_view(struct drm_i915_gem_object *obj,
1791                      pgoff_t page_offset,
1792                      unsigned int chunk)
1793 {
1794         struct i915_ggtt_view view;
1795
1796         if (i915_gem_object_is_tiled(obj))
1797                 chunk = roundup(chunk, tile_row_pages(obj));
1798
1799         view.type = I915_GGTT_VIEW_PARTIAL;
1800         view.partial.offset = rounddown(page_offset, chunk);
1801         view.partial.size =
1802                 min_t(unsigned int, chunk,
1803                       (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
1804
1805         /* If the partial covers the entire object, just create a normal VMA. */
1806         if (chunk >= obj->base.size >> PAGE_SHIFT)
1807                 view.type = I915_GGTT_VIEW_NORMAL;
1808
1809         return view;
1810 }
1811
1812 /**
1813  * i915_gem_fault - fault a page into the GTT
1814  * @vmf: fault info
1815  *
1816  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1817  * from userspace.  The fault handler takes care of binding the object to
1818  * the GTT (if needed), allocating and programming a fence register (again,
1819  * only if needed based on whether the old reg is still valid or the object
1820  * is tiled) and inserting a new PTE into the faulting process.
1821  *
1822  * Note that the faulting process may involve evicting existing objects
1823  * from the GTT and/or fence registers to make room.  So performance may
1824  * suffer if the GTT working set is large or there are few fence registers
1825  * left.
1826  *
1827  * The current feature set supported by i915_gem_fault() and thus GTT mmaps
1828  * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
1829  */
1830 int i915_gem_fault(struct vm_fault *vmf)
1831 {
1832 #define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
1833         struct vm_area_struct *area = vmf->vma;
1834         struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
1835         struct drm_device *dev = obj->base.dev;
1836         struct drm_i915_private *dev_priv = to_i915(dev);
1837         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1838         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1839         struct i915_vma *vma;
1840         pgoff_t page_offset;
1841         unsigned int flags;
1842         int ret;
1843
1844         /* We don't use vmf->pgoff since that has the fake offset */
1845         page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
1846
1847         trace_i915_gem_object_fault(obj, page_offset, true, write);
1848
1849         /* Try to flush the object off the GPU first without holding the lock.
1850          * Upon acquiring the lock, we will perform our sanity checks and then
1851          * repeat the flush holding the lock in the normal manner to catch cases
1852          * where we are gazumped.
1853          */
1854         ret = i915_gem_object_wait(obj,
1855                                    I915_WAIT_INTERRUPTIBLE,
1856                                    MAX_SCHEDULE_TIMEOUT,
1857                                    NULL);
1858         if (ret)
1859                 goto err;
1860
1861         ret = i915_gem_object_pin_pages(obj);
1862         if (ret)
1863                 goto err;
1864
1865         intel_runtime_pm_get(dev_priv);
1866
1867         ret = i915_mutex_lock_interruptible(dev);
1868         if (ret)
1869                 goto err_rpm;
1870
1871         /* Access to snoopable pages through the GTT is incoherent. */
1872         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
1873                 ret = -EFAULT;
1874                 goto err_unlock;
1875         }
1876
1877         /* If the object is smaller than a couple of partial vma, it is
1878          * not worth only creating a single partial vma - we may as well
1879          * clear enough space for the full object.
1880          */
1881         flags = PIN_MAPPABLE;
1882         if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
1883                 flags |= PIN_NONBLOCK | PIN_NONFAULT;
1884
1885         /* Now pin it into the GTT as needed */
1886         vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
1887         if (IS_ERR(vma)) {
1888                 /* Use a partial view if it is bigger than available space */
1889                 struct i915_ggtt_view view =
1890                         compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
1891
1892                 /* Userspace is now writing through an untracked VMA, abandon
1893                  * all hope that the hardware is able to track future writes.
1894                  */
1895                 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
1896
1897                 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1898         }
1899         if (IS_ERR(vma)) {
1900                 ret = PTR_ERR(vma);
1901                 goto err_unlock;
1902         }
1903
1904         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1905         if (ret)
1906                 goto err_unpin;
1907
1908         ret = i915_vma_get_fence(vma);
1909         if (ret)
1910                 goto err_unpin;
1911
1912         /* Mark as being mmapped into userspace for later revocation */
1913         assert_rpm_wakelock_held(dev_priv);
1914         if (list_empty(&obj->userfault_link))
1915                 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
1916
1917         /* Finally, remap it using the new GTT offset */
1918         ret = remap_io_mapping(area,
1919                                area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
1920                                (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
1921                                min_t(u64, vma->size, area->vm_end - area->vm_start),
1922                                &ggtt->mappable);
1923
1924 err_unpin:
1925         __i915_vma_unpin(vma);
1926 err_unlock:
1927         mutex_unlock(&dev->struct_mutex);
1928 err_rpm:
1929         intel_runtime_pm_put(dev_priv);
1930         i915_gem_object_unpin_pages(obj);
1931 err:
1932         switch (ret) {
1933         case -EIO:
1934                 /*
1935                  * We eat errors when the gpu is terminally wedged to avoid
1936                  * userspace unduly crashing (gl has no provisions for mmaps to
1937                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1938                  * and so needs to be reported.
1939                  */
1940                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1941                         ret = VM_FAULT_SIGBUS;
1942                         break;
1943                 }
1944         case -EAGAIN:
1945                 /*
1946                  * EAGAIN means the gpu is hung and we'll wait for the error
1947                  * handler to reset everything when re-faulting in
1948                  * i915_mutex_lock_interruptible.
1949                  */
1950         case 0:
1951         case -ERESTARTSYS:
1952         case -EINTR:
1953         case -EBUSY:
1954                 /*
1955                  * EBUSY is ok: this just means that another thread
1956                  * already did the job.
1957                  */
1958                 ret = VM_FAULT_NOPAGE;
1959                 break;
1960         case -ENOMEM:
1961                 ret = VM_FAULT_OOM;
1962                 break;
1963         case -ENOSPC:
1964         case -EFAULT:
1965                 ret = VM_FAULT_SIGBUS;
1966                 break;
1967         default:
1968                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1969                 ret = VM_FAULT_SIGBUS;
1970                 break;
1971         }
1972         return ret;
1973 }
1974
1975 /**
1976  * i915_gem_release_mmap - remove physical page mappings
1977  * @obj: obj in question
1978  *
1979  * Preserve the reservation of the mmapping with the DRM core code, but
1980  * relinquish ownership of the pages back to the system.
1981  *
1982  * It is vital that we remove the page mapping if we have mapped a tiled
1983  * object through the GTT and then lose the fence register due to
1984  * resource pressure. Similarly if the object has been moved out of the
1985  * aperture, than pages mapped into userspace must be revoked. Removing the
1986  * mapping will then trigger a page fault on the next user access, allowing
1987  * fixup by i915_gem_fault().
1988  */
1989 void
1990 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1991 {
1992         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1993
1994         /* Serialisation between user GTT access and our code depends upon
1995          * revoking the CPU's PTE whilst the mutex is held. The next user
1996          * pagefault then has to wait until we release the mutex.
1997          *
1998          * Note that RPM complicates somewhat by adding an additional
1999          * requirement that operations to the GGTT be made holding the RPM
2000          * wakeref.
2001          */
2002         lockdep_assert_held(&i915->drm.struct_mutex);
2003         intel_runtime_pm_get(i915);
2004
2005         if (list_empty(&obj->userfault_link))
2006                 goto out;
2007
2008         list_del_init(&obj->userfault_link);
2009         drm_vma_node_unmap(&obj->base.vma_node,
2010                            obj->base.dev->anon_inode->i_mapping);
2011
2012         /* Ensure that the CPU's PTE are revoked and there are not outstanding
2013          * memory transactions from userspace before we return. The TLB
2014          * flushing implied above by changing the PTE above *should* be
2015          * sufficient, an extra barrier here just provides us with a bit
2016          * of paranoid documentation about our requirement to serialise
2017          * memory writes before touching registers / GSM.
2018          */
2019         wmb();
2020
2021 out:
2022         intel_runtime_pm_put(i915);
2023 }
2024
2025 void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
2026 {
2027         struct drm_i915_gem_object *obj, *on;
2028         int i;
2029
2030         /*
2031          * Only called during RPM suspend. All users of the userfault_list
2032          * must be holding an RPM wakeref to ensure that this can not
2033          * run concurrently with themselves (and use the struct_mutex for
2034          * protection between themselves).
2035          */
2036
2037         list_for_each_entry_safe(obj, on,
2038                                  &dev_priv->mm.userfault_list, userfault_link) {
2039                 list_del_init(&obj->userfault_link);
2040                 drm_vma_node_unmap(&obj->base.vma_node,
2041                                    obj->base.dev->anon_inode->i_mapping);
2042         }
2043
2044         /* The fence will be lost when the device powers down. If any were
2045          * in use by hardware (i.e. they are pinned), we should not be powering
2046          * down! All other fences will be reacquired by the user upon waking.
2047          */
2048         for (i = 0; i < dev_priv->num_fence_regs; i++) {
2049                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2050
2051                 /* Ideally we want to assert that the fence register is not
2052                  * live at this point (i.e. that no piece of code will be
2053                  * trying to write through fence + GTT, as that both violates
2054                  * our tracking of activity and associated locking/barriers,
2055                  * but also is illegal given that the hw is powered down).
2056                  *
2057                  * Previously we used reg->pin_count as a "liveness" indicator.
2058                  * That is not sufficient, and we need a more fine-grained
2059                  * tool if we want to have a sanity check here.
2060                  */
2061
2062                 if (!reg->vma)
2063                         continue;
2064
2065                 GEM_BUG_ON(!list_empty(&reg->vma->obj->userfault_link));
2066                 reg->dirty = true;
2067         }
2068 }
2069
2070 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2071 {
2072         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2073         int err;
2074
2075         err = drm_gem_create_mmap_offset(&obj->base);
2076         if (likely(!err))
2077                 return 0;
2078
2079         /* Attempt to reap some mmap space from dead objects */
2080         do {
2081                 err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
2082                 if (err)
2083                         break;
2084
2085                 i915_gem_drain_freed_objects(dev_priv);
2086                 err = drm_gem_create_mmap_offset(&obj->base);
2087                 if (!err)
2088                         break;
2089
2090         } while (flush_delayed_work(&dev_priv->gt.retire_work));
2091
2092         return err;
2093 }
2094
2095 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2096 {
2097         drm_gem_free_mmap_offset(&obj->base);
2098 }
2099
2100 int
2101 i915_gem_mmap_gtt(struct drm_file *file,
2102                   struct drm_device *dev,
2103                   uint32_t handle,
2104                   uint64_t *offset)
2105 {
2106         struct drm_i915_gem_object *obj;
2107         int ret;
2108
2109         obj = i915_gem_object_lookup(file, handle);
2110         if (!obj)
2111                 return -ENOENT;
2112
2113         ret = i915_gem_object_create_mmap_offset(obj);
2114         if (ret == 0)
2115                 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2116
2117         i915_gem_object_put(obj);
2118         return ret;
2119 }
2120
2121 /**
2122  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2123  * @dev: DRM device
2124  * @data: GTT mapping ioctl data
2125  * @file: GEM object info
2126  *
2127  * Simply returns the fake offset to userspace so it can mmap it.
2128  * The mmap call will end up in drm_gem_mmap(), which will set things
2129  * up so we can get faults in the handler above.
2130  *
2131  * The fault handler will take care of binding the object into the GTT
2132  * (since it may have been evicted to make room for something), allocating
2133  * a fence register, and mapping the appropriate aperture address into
2134  * userspace.
2135  */
2136 int
2137 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2138                         struct drm_file *file)
2139 {
2140         struct drm_i915_gem_mmap_gtt *args = data;
2141
2142         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2143 }
2144
2145 /* Immediately discard the backing storage */
2146 static void
2147 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2148 {
2149         i915_gem_object_free_mmap_offset(obj);
2150
2151         if (obj->base.filp == NULL)
2152                 return;
2153
2154         /* Our goal here is to return as much of the memory as
2155          * is possible back to the system as we are called from OOM.
2156          * To do this we must instruct the shmfs to drop all of its
2157          * backing pages, *now*.
2158          */
2159         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2160         obj->mm.madv = __I915_MADV_PURGED;
2161         obj->mm.pages = ERR_PTR(-EFAULT);
2162 }
2163
2164 /* Try to discard unwanted pages */
2165 void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2166 {
2167         struct address_space *mapping;
2168
2169         lockdep_assert_held(&obj->mm.lock);
2170         GEM_BUG_ON(obj->mm.pages);
2171
2172         switch (obj->mm.madv) {
2173         case I915_MADV_DONTNEED:
2174                 i915_gem_object_truncate(obj);
2175         case __I915_MADV_PURGED:
2176                 return;
2177         }
2178
2179         if (obj->base.filp == NULL)
2180                 return;
2181
2182         mapping = obj->base.filp->f_mapping,
2183         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2184 }
2185
2186 static void
2187 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2188                               struct sg_table *pages)
2189 {
2190         struct sgt_iter sgt_iter;
2191         struct page *page;
2192
2193         __i915_gem_object_release_shmem(obj, pages, true);
2194
2195         i915_gem_gtt_finish_pages(obj, pages);
2196
2197         if (i915_gem_object_needs_bit17_swizzle(obj))
2198                 i915_gem_object_save_bit_17_swizzle(obj, pages);
2199
2200         for_each_sgt_page(page, sgt_iter, pages) {
2201                 if (obj->mm.dirty)
2202                         set_page_dirty(page);
2203
2204                 if (obj->mm.madv == I915_MADV_WILLNEED)
2205                         mark_page_accessed(page);
2206
2207                 put_page(page);
2208         }
2209         obj->mm.dirty = false;
2210
2211         sg_free_table(pages);
2212         kfree(pages);
2213 }
2214
2215 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2216 {
2217         struct radix_tree_iter iter;
2218         void __rcu **slot;
2219
2220         radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2221                 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2222 }
2223
2224 void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2225                                  enum i915_mm_subclass subclass)
2226 {
2227         struct sg_table *pages;
2228
2229         if (i915_gem_object_has_pinned_pages(obj))
2230                 return;
2231
2232         GEM_BUG_ON(obj->bind_count);
2233         if (!READ_ONCE(obj->mm.pages))
2234                 return;
2235
2236         /* May be called by shrinker from within get_pages() (on another bo) */
2237         mutex_lock_nested(&obj->mm.lock, subclass);
2238         if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
2239                 goto unlock;
2240
2241         /* ->put_pages might need to allocate memory for the bit17 swizzle
2242          * array, hence protect them from being reaped by removing them from gtt
2243          * lists early. */
2244         pages = fetch_and_zero(&obj->mm.pages);
2245         GEM_BUG_ON(!pages);
2246
2247         if (obj->mm.mapping) {
2248                 void *ptr;
2249
2250                 ptr = page_mask_bits(obj->mm.mapping);
2251                 if (is_vmalloc_addr(ptr))
2252                         vunmap(ptr);
2253                 else
2254                         kunmap(kmap_to_page(ptr));
2255
2256                 obj->mm.mapping = NULL;
2257         }
2258
2259         __i915_gem_object_reset_page_iter(obj);
2260
2261         if (!IS_ERR(pages))
2262                 obj->ops->put_pages(obj, pages);
2263
2264 unlock:
2265         mutex_unlock(&obj->mm.lock);
2266 }
2267
2268 static bool i915_sg_trim(struct sg_table *orig_st)
2269 {
2270         struct sg_table new_st;
2271         struct scatterlist *sg, *new_sg;
2272         unsigned int i;
2273
2274         if (orig_st->nents == orig_st->orig_nents)
2275                 return false;
2276
2277         if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2278                 return false;
2279
2280         new_sg = new_st.sgl;
2281         for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2282                 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2283                 /* called before being DMA mapped, no need to copy sg->dma_* */
2284                 new_sg = sg_next(new_sg);
2285         }
2286         GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
2287
2288         sg_free_table(orig_st);
2289
2290         *orig_st = new_st;
2291         return true;
2292 }
2293
2294 static struct sg_table *
2295 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2296 {
2297         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2298         const unsigned long page_count = obj->base.size / PAGE_SIZE;
2299         unsigned long i;
2300         struct address_space *mapping;
2301         struct sg_table *st;
2302         struct scatterlist *sg;
2303         struct sgt_iter sgt_iter;
2304         struct page *page;
2305         unsigned long last_pfn = 0;     /* suppress gcc warning */
2306         unsigned int max_segment;
2307         gfp_t noreclaim;
2308         int ret;
2309
2310         /* Assert that the object is not currently in any GPU domain. As it
2311          * wasn't in the GTT, there shouldn't be any way it could have been in
2312          * a GPU cache
2313          */
2314         GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2315         GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2316
2317         max_segment = swiotlb_max_segment();
2318         if (!max_segment)
2319                 max_segment = rounddown(UINT_MAX, PAGE_SIZE);
2320
2321         st = kmalloc(sizeof(*st), GFP_KERNEL);
2322         if (st == NULL)
2323                 return ERR_PTR(-ENOMEM);
2324
2325 rebuild_st:
2326         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2327                 kfree(st);
2328                 return ERR_PTR(-ENOMEM);
2329         }
2330
2331         /* Get the list of pages out of our struct file.  They'll be pinned
2332          * at this point until we release them.
2333          *
2334          * Fail silently without starting the shrinker
2335          */
2336         mapping = obj->base.filp->f_mapping;
2337         noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
2338         noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
2339
2340         sg = st->sgl;
2341         st->nents = 0;
2342         for (i = 0; i < page_count; i++) {
2343                 const unsigned int shrink[] = {
2344                         I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
2345                         0,
2346                 }, *s = shrink;
2347                 gfp_t gfp = noreclaim;
2348
2349                 do {
2350                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2351                         if (likely(!IS_ERR(page)))
2352                                 break;
2353
2354                         if (!*s) {
2355                                 ret = PTR_ERR(page);
2356                                 goto err_sg;
2357                         }
2358
2359                         i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
2360                         cond_resched();
2361
2362                         /* We've tried hard to allocate the memory by reaping
2363                          * our own buffer, now let the real VM do its job and
2364                          * go down in flames if truly OOM.
2365                          *
2366                          * However, since graphics tend to be disposable,
2367                          * defer the oom here by reporting the ENOMEM back
2368                          * to userspace.
2369                          */
2370                         if (!*s) {
2371                                 /* reclaim and warn, but no oom */
2372                                 gfp = mapping_gfp_mask(mapping);
2373
2374                                 /* Our bo are always dirty and so we require
2375                                  * kswapd to reclaim our pages (direct reclaim
2376                                  * does not effectively begin pageout of our
2377                                  * buffers on its own). However, direct reclaim
2378                                  * only waits for kswapd when under allocation
2379                                  * congestion. So as a result __GFP_RECLAIM is
2380                                  * unreliable and fails to actually reclaim our
2381                                  * dirty pages -- unless you try over and over
2382                                  * again with !__GFP_NORETRY. However, we still
2383                                  * want to fail this allocation rather than
2384                                  * trigger the out-of-memory killer and for
2385                                  * this we want __GFP_RETRY_MAYFAIL.
2386                                  */
2387                                 gfp |= __GFP_RETRY_MAYFAIL;
2388                         }
2389                 } while (1);
2390
2391                 if (!i ||
2392                     sg->length >= max_segment ||
2393                     page_to_pfn(page) != last_pfn + 1) {
2394                         if (i)
2395                                 sg = sg_next(sg);
2396                         st->nents++;
2397                         sg_set_page(sg, page, PAGE_SIZE, 0);
2398                 } else {
2399                         sg->length += PAGE_SIZE;
2400                 }
2401                 last_pfn = page_to_pfn(page);
2402
2403                 /* Check that the i965g/gm workaround works. */
2404                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2405         }
2406         if (sg) /* loop terminated early; short sg table */
2407                 sg_mark_end(sg);
2408
2409         /* Trim unused sg entries to avoid wasting memory. */
2410         i915_sg_trim(st);
2411
2412         ret = i915_gem_gtt_prepare_pages(obj, st);
2413         if (ret) {
2414                 /* DMA remapping failed? One possible cause is that
2415                  * it could not reserve enough large entries, asking
2416                  * for PAGE_SIZE chunks instead may be helpful.
2417                  */
2418                 if (max_segment > PAGE_SIZE) {
2419                         for_each_sgt_page(page, sgt_iter, st)
2420                                 put_page(page);
2421                         sg_free_table(st);
2422
2423                         max_segment = PAGE_SIZE;
2424                         goto rebuild_st;
2425                 } else {
2426                         dev_warn(&dev_priv->drm.pdev->dev,
2427                                  "Failed to DMA remap %lu pages\n",
2428                                  page_count);
2429                         goto err_pages;
2430                 }
2431         }
2432
2433         if (i915_gem_object_needs_bit17_swizzle(obj))
2434                 i915_gem_object_do_bit_17_swizzle(obj, st);
2435
2436         return st;
2437
2438 err_sg:
2439         sg_mark_end(sg);
2440 err_pages:
2441         for_each_sgt_page(page, sgt_iter, st)
2442                 put_page(page);
2443         sg_free_table(st);
2444         kfree(st);
2445
2446         /* shmemfs first checks if there is enough memory to allocate the page
2447          * and reports ENOSPC should there be insufficient, along with the usual
2448          * ENOMEM for a genuine allocation failure.
2449          *
2450          * We use ENOSPC in our driver to mean that we have run out of aperture
2451          * space and so want to translate the error from shmemfs back to our
2452          * usual understanding of ENOMEM.
2453          */
2454         if (ret == -ENOSPC)
2455                 ret = -ENOMEM;
2456
2457         return ERR_PTR(ret);
2458 }
2459
2460 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2461                                  struct sg_table *pages)
2462 {
2463         lockdep_assert_held(&obj->mm.lock);
2464
2465         obj->mm.get_page.sg_pos = pages->sgl;
2466         obj->mm.get_page.sg_idx = 0;
2467
2468         obj->mm.pages = pages;
2469
2470         if (i915_gem_object_is_tiled(obj) &&
2471             to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
2472                 GEM_BUG_ON(obj->mm.quirked);
2473                 __i915_gem_object_pin_pages(obj);
2474                 obj->mm.quirked = true;
2475         }
2476 }
2477
2478 static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2479 {
2480         struct sg_table *pages;
2481
2482         if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2483                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2484                 return -EFAULT;
2485         }
2486
2487         pages = obj->ops->get_pages(obj);
2488         if (unlikely(IS_ERR(pages)))
2489                 return PTR_ERR(pages);
2490
2491         __i915_gem_object_set_pages(obj, pages);
2492         return 0;
2493 }
2494
2495 /* Ensure that the associated pages are gathered from the backing storage
2496  * and pinned into our object. i915_gem_object_pin_pages() may be called
2497  * multiple times before they are released by a single call to
2498  * i915_gem_object_unpin_pages() - once the pages are no longer referenced
2499  * either as a result of memory pressure (reaping pages under the shrinker)
2500  * or as the object is itself released.
2501  */
2502 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2503 {
2504         int err;
2505
2506         err = mutex_lock_interruptible(&obj->mm.lock);
2507         if (err)
2508                 return err;
2509
2510         if (unlikely(IS_ERR_OR_NULL(obj->mm.pages))) {
2511                 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2512
2513                 err = ____i915_gem_object_get_pages(obj);
2514                 if (err)
2515                         goto unlock;
2516
2517                 smp_mb__before_atomic();
2518         }
2519         atomic_inc(&obj->mm.pages_pin_count);
2520
2521 unlock:
2522         mutex_unlock(&obj->mm.lock);
2523         return err;
2524 }
2525
2526 /* The 'mapping' part of i915_gem_object_pin_map() below */
2527 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2528                                  enum i915_map_type type)
2529 {
2530         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2531         struct sg_table *sgt = obj->mm.pages;
2532         struct sgt_iter sgt_iter;
2533         struct page *page;
2534         struct page *stack_pages[32];
2535         struct page **pages = stack_pages;
2536         unsigned long i = 0;
2537         pgprot_t pgprot;
2538         void *addr;
2539
2540         /* A single page can always be kmapped */
2541         if (n_pages == 1 && type == I915_MAP_WB)
2542                 return kmap(sg_page(sgt->sgl));
2543
2544         if (n_pages > ARRAY_SIZE(stack_pages)) {
2545                 /* Too big for stack -- allocate temporary array instead */
2546                 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
2547                 if (!pages)
2548                         return NULL;
2549         }
2550
2551         for_each_sgt_page(page, sgt_iter, sgt)
2552                 pages[i++] = page;
2553
2554         /* Check that we have the expected number of pages */
2555         GEM_BUG_ON(i != n_pages);
2556
2557         switch (type) {
2558         default:
2559                 MISSING_CASE(type);
2560                 /* fallthrough to use PAGE_KERNEL anyway */
2561         case I915_MAP_WB:
2562                 pgprot = PAGE_KERNEL;
2563                 break;
2564         case I915_MAP_WC:
2565                 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2566                 break;
2567         }
2568         addr = vmap(pages, n_pages, 0, pgprot);
2569
2570         if (pages != stack_pages)
2571                 kvfree(pages);
2572
2573         return addr;
2574 }
2575
2576 /* get, pin, and map the pages of the object into kernel space */
2577 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2578                               enum i915_map_type type)
2579 {
2580         enum i915_map_type has_type;
2581         bool pinned;
2582         void *ptr;
2583         int ret;
2584
2585         GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
2586
2587         ret = mutex_lock_interruptible(&obj->mm.lock);
2588         if (ret)
2589                 return ERR_PTR(ret);
2590
2591         pinned = !(type & I915_MAP_OVERRIDE);
2592         type &= ~I915_MAP_OVERRIDE;
2593
2594         if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2595                 if (unlikely(IS_ERR_OR_NULL(obj->mm.pages))) {
2596                         GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2597
2598                         ret = ____i915_gem_object_get_pages(obj);
2599                         if (ret)
2600                                 goto err_unlock;
2601
2602                         smp_mb__before_atomic();
2603                 }
2604                 atomic_inc(&obj->mm.pages_pin_count);
2605                 pinned = false;
2606         }
2607         GEM_BUG_ON(!obj->mm.pages);
2608
2609         ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2610         if (ptr && has_type != type) {
2611                 if (pinned) {
2612                         ret = -EBUSY;
2613                         goto err_unpin;
2614                 }
2615
2616                 if (is_vmalloc_addr(ptr))
2617                         vunmap(ptr);
2618                 else
2619                         kunmap(kmap_to_page(ptr));
2620
2621                 ptr = obj->mm.mapping = NULL;
2622         }
2623
2624         if (!ptr) {
2625                 ptr = i915_gem_object_map(obj, type);
2626                 if (!ptr) {
2627                         ret = -ENOMEM;
2628                         goto err_unpin;
2629                 }
2630
2631                 obj->mm.mapping = page_pack_bits(ptr, type);
2632         }
2633
2634 out_unlock:
2635         mutex_unlock(&obj->mm.lock);
2636         return ptr;
2637
2638 err_unpin:
2639         atomic_dec(&obj->mm.pages_pin_count);
2640 err_unlock:
2641         ptr = ERR_PTR(ret);
2642         goto out_unlock;
2643 }
2644
2645 static int
2646 i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
2647                            const struct drm_i915_gem_pwrite *arg)
2648 {
2649         struct address_space *mapping = obj->base.filp->f_mapping;
2650         char __user *user_data = u64_to_user_ptr(arg->data_ptr);
2651         u64 remain, offset;
2652         unsigned int pg;
2653
2654         /* Before we instantiate/pin the backing store for our use, we
2655          * can prepopulate the shmemfs filp efficiently using a write into
2656          * the pagecache. We avoid the penalty of instantiating all the
2657          * pages, important if the user is just writing to a few and never
2658          * uses the object on the GPU, and using a direct write into shmemfs
2659          * allows it to avoid the cost of retrieving a page (either swapin
2660          * or clearing-before-use) before it is overwritten.
2661          */
2662         if (READ_ONCE(obj->mm.pages))
2663                 return -ENODEV;
2664
2665         /* Before the pages are instantiated the object is treated as being
2666          * in the CPU domain. The pages will be clflushed as required before
2667          * use, and we can freely write into the pages directly. If userspace
2668          * races pwrite with any other operation; corruption will ensue -
2669          * that is userspace's prerogative!
2670          */
2671
2672         remain = arg->size;
2673         offset = arg->offset;
2674         pg = offset_in_page(offset);
2675
2676         do {
2677                 unsigned int len, unwritten;
2678                 struct page *page;
2679                 void *data, *vaddr;
2680                 int err;
2681
2682                 len = PAGE_SIZE - pg;
2683                 if (len > remain)
2684                         len = remain;
2685
2686                 err = pagecache_write_begin(obj->base.filp, mapping,
2687                                             offset, len, 0,
2688                                             &page, &data);
2689                 if (err < 0)
2690                         return err;
2691
2692                 vaddr = kmap(page);
2693                 unwritten = copy_from_user(vaddr + pg, user_data, len);
2694                 kunmap(page);
2695
2696                 err = pagecache_write_end(obj->base.filp, mapping,
2697                                           offset, len, len - unwritten,
2698                                           page, data);
2699                 if (err < 0)
2700                         return err;
2701
2702                 if (unwritten)
2703                         return -EFAULT;
2704
2705                 remain -= len;
2706                 user_data += len;
2707                 offset += len;
2708                 pg = 0;
2709         } while (remain);
2710
2711         return 0;
2712 }
2713
2714 static bool ban_context(const struct i915_gem_context *ctx,
2715                         unsigned int score)
2716 {
2717         return (i915_gem_context_is_bannable(ctx) &&
2718                 score >= CONTEXT_SCORE_BAN_THRESHOLD);
2719 }
2720
2721 static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
2722 {
2723         unsigned int score;
2724         bool banned;
2725
2726         atomic_inc(&ctx->guilty_count);
2727
2728         score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
2729         banned = ban_context(ctx, score);
2730         DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n",
2731                          ctx->name, score, yesno(banned));
2732         if (!banned)
2733                 return;
2734
2735         i915_gem_context_set_banned(ctx);
2736         if (!IS_ERR_OR_NULL(ctx->file_priv)) {
2737                 atomic_inc(&ctx->file_priv->context_bans);
2738                 DRM_DEBUG_DRIVER("client %s has had %d context banned\n",
2739                                  ctx->name, atomic_read(&ctx->file_priv->context_bans));
2740         }
2741 }
2742
2743 static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
2744 {
2745         atomic_inc(&ctx->active_count);
2746 }
2747
2748 struct drm_i915_gem_request *
2749 i915_gem_find_active_request(struct intel_engine_cs *engine)
2750 {
2751         struct drm_i915_gem_request *request, *active = NULL;
2752         unsigned long flags;
2753
2754         /* We are called by the error capture and reset at a random
2755          * point in time. In particular, note that neither is crucially
2756          * ordered with an interrupt. After a hang, the GPU is dead and we
2757          * assume that no more writes can happen (we waited long enough for
2758          * all writes that were in transaction to be flushed) - adding an
2759          * extra delay for a recent interrupt is pointless. Hence, we do
2760          * not need an engine->irq_seqno_barrier() before the seqno reads.
2761          */
2762         spin_lock_irqsave(&engine->timeline->lock, flags);
2763         list_for_each_entry(request, &engine->timeline->requests, link) {
2764                 if (__i915_gem_request_completed(request,
2765                                                  request->global_seqno))
2766                         continue;
2767
2768                 GEM_BUG_ON(request->engine != engine);
2769                 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
2770                                     &request->fence.flags));
2771
2772                 active = request;
2773                 break;
2774         }
2775         spin_unlock_irqrestore(&engine->timeline->lock, flags);
2776
2777         return active;
2778 }
2779
2780 static bool engine_stalled(struct intel_engine_cs *engine)
2781 {
2782         if (!engine->hangcheck.stalled)
2783                 return false;
2784
2785         /* Check for possible seqno movement after hang declaration */
2786         if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) {
2787                 DRM_DEBUG_DRIVER("%s pardoned\n", engine->name);
2788                 return false;
2789         }
2790
2791         return true;
2792 }
2793
2794 /*
2795  * Ensure irq handler finishes, and not run again.
2796  * Also return the active request so that we only search for it once.
2797  */
2798 struct drm_i915_gem_request *
2799 i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
2800 {
2801         struct drm_i915_gem_request *request = NULL;
2802
2803         /* Prevent the signaler thread from updating the request
2804          * state (by calling dma_fence_signal) as we are processing
2805          * the reset. The write from the GPU of the seqno is
2806          * asynchronous and the signaler thread may see a different
2807          * value to us and declare the request complete, even though
2808          * the reset routine have picked that request as the active
2809          * (incomplete) request. This conflict is not handled
2810          * gracefully!
2811          */
2812         kthread_park(engine->breadcrumbs.signaler);
2813
2814         /* Prevent request submission to the hardware until we have
2815          * completed the reset in i915_gem_reset_finish(). If a request
2816          * is completed by one engine, it may then queue a request
2817          * to a second via its engine->irq_tasklet *just* as we are
2818          * calling engine->init_hw() and also writing the ELSP.
2819          * Turning off the engine->irq_tasklet until the reset is over
2820          * prevents the race.
2821          */
2822         tasklet_kill(&engine->irq_tasklet);
2823         tasklet_disable(&engine->irq_tasklet);
2824
2825         if (engine->irq_seqno_barrier)
2826                 engine->irq_seqno_barrier(engine);
2827
2828         request = i915_gem_find_active_request(engine);
2829         if (request && request->fence.error == -EIO)
2830                 request = ERR_PTR(-EIO); /* Previous reset failed! */
2831
2832         return request;
2833 }
2834
2835 int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
2836 {
2837         struct intel_engine_cs *engine;
2838         struct drm_i915_gem_request *request;
2839         enum intel_engine_id id;
2840         int err = 0;
2841
2842         for_each_engine(engine, dev_priv, id) {
2843                 request = i915_gem_reset_prepare_engine(engine);
2844                 if (IS_ERR(request)) {
2845                         err = PTR_ERR(request);
2846                         continue;
2847                 }
2848
2849                 engine->hangcheck.active_request = request;
2850         }
2851
2852         i915_gem_revoke_fences(dev_priv);
2853
2854         return err;
2855 }
2856
2857 static void skip_request(struct drm_i915_gem_request *request)
2858 {
2859         void *vaddr = request->ring->vaddr;
2860         u32 head;
2861
2862         /* As this request likely depends on state from the lost
2863          * context, clear out all the user operations leaving the
2864          * breadcrumb at the end (so we get the fence notifications).
2865          */
2866         head = request->head;
2867         if (request->postfix < head) {
2868                 memset(vaddr + head, 0, request->ring->size - head);
2869                 head = 0;
2870         }
2871         memset(vaddr + head, 0, request->postfix - head);
2872
2873         dma_fence_set_error(&request->fence, -EIO);
2874 }
2875
2876 static void engine_skip_context(struct drm_i915_gem_request *request)
2877 {
2878         struct intel_engine_cs *engine = request->engine;
2879         struct i915_gem_context *hung_ctx = request->ctx;
2880         struct intel_timeline *timeline;
2881         unsigned long flags;
2882
2883         timeline = i915_gem_context_lookup_timeline(hung_ctx, engine);
2884
2885         spin_lock_irqsave(&engine->timeline->lock, flags);
2886         spin_lock(&timeline->lock);
2887
2888         list_for_each_entry_continue(request, &engine->timeline->requests, link)
2889                 if (request->ctx == hung_ctx)
2890                         skip_request(request);
2891
2892         list_for_each_entry(request, &timeline->requests, link)
2893                 skip_request(request);
2894
2895         spin_unlock(&timeline->lock);
2896         spin_unlock_irqrestore(&engine->timeline->lock, flags);
2897 }
2898
2899 /* Returns the request if it was guilty of the hang */
2900 static struct drm_i915_gem_request *
2901 i915_gem_reset_request(struct intel_engine_cs *engine,
2902                        struct drm_i915_gem_request *request)
2903 {
2904         /* The guilty request will get skipped on a hung engine.
2905          *
2906          * Users of client default contexts do not rely on logical
2907          * state preserved between batches so it is safe to execute
2908          * queued requests following the hang. Non default contexts
2909          * rely on preserved state, so skipping a batch loses the
2910          * evolution of the state and it needs to be considered corrupted.
2911          * Executing more queued batches on top of corrupted state is
2912          * risky. But we take the risk by trying to advance through
2913          * the queued requests in order to make the client behaviour
2914          * more predictable around resets, by not throwing away random
2915          * amount of batches it has prepared for execution. Sophisticated
2916          * clients can use gem_reset_stats_ioctl and dma fence status
2917          * (exported via sync_file info ioctl on explicit fences) to observe
2918          * when it loses the context state and should rebuild accordingly.
2919          *
2920          * The context ban, and ultimately the client ban, mechanism are safety
2921          * valves if client submission ends up resulting in nothing more than
2922          * subsequent hangs.
2923          */
2924
2925         if (engine_stalled(engine)) {
2926                 i915_gem_context_mark_guilty(request->ctx);
2927                 skip_request(request);
2928
2929                 /* If this context is now banned, skip all pending requests. */
2930                 if (i915_gem_context_is_banned(request->ctx))
2931                         engine_skip_context(request);
2932         } else {
2933                 /*
2934                  * Since this is not the hung engine, it may have advanced
2935                  * since the hang declaration. Double check by refinding
2936                  * the active request at the time of the reset.
2937                  */
2938                 request = i915_gem_find_active_request(engine);
2939                 if (request) {
2940                         i915_gem_context_mark_innocent(request->ctx);
2941                         dma_fence_set_error(&request->fence, -EAGAIN);
2942
2943                         /* Rewind the engine to replay the incomplete rq */
2944                         spin_lock_irq(&engine->timeline->lock);
2945                         request = list_prev_entry(request, link);
2946                         if (&request->link == &engine->timeline->requests)
2947                                 request = NULL;
2948                         spin_unlock_irq(&engine->timeline->lock);
2949                 }
2950         }
2951
2952         return request;
2953 }
2954
2955 void i915_gem_reset_engine(struct intel_engine_cs *engine,
2956                            struct drm_i915_gem_request *request)
2957 {
2958         engine->irq_posted = 0;
2959
2960         if (request)
2961                 request = i915_gem_reset_request(engine, request);
2962
2963         if (request) {
2964                 DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
2965                                  engine->name, request->global_seqno);
2966         }
2967
2968         /* Setup the CS to resume from the breadcrumb of the hung request */
2969         engine->reset_hw(engine, request);
2970 }
2971
2972 void i915_gem_reset(struct drm_i915_private *dev_priv)
2973 {
2974         struct intel_engine_cs *engine;
2975         enum intel_engine_id id;
2976
2977         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2978
2979         i915_gem_retire_requests(dev_priv);
2980
2981         for_each_engine(engine, dev_priv, id) {
2982                 struct i915_gem_context *ctx;
2983
2984                 i915_gem_reset_engine(engine, engine->hangcheck.active_request);
2985                 ctx = fetch_and_zero(&engine->last_retired_context);
2986                 if (ctx)
2987                         engine->context_unpin(engine, ctx);
2988         }
2989
2990         i915_gem_restore_fences(dev_priv);
2991
2992         if (dev_priv->gt.awake) {
2993                 intel_sanitize_gt_powersave(dev_priv);
2994                 intel_enable_gt_powersave(dev_priv);
2995                 if (INTEL_GEN(dev_priv) >= 6)
2996                         gen6_rps_busy(dev_priv);
2997         }
2998 }
2999
3000 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
3001 {
3002         tasklet_enable(&engine->irq_tasklet);
3003         kthread_unpark(engine->breadcrumbs.signaler);
3004 }
3005
3006 void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
3007 {
3008         struct intel_engine_cs *engine;
3009         enum intel_engine_id id;
3010
3011         lockdep_assert_held(&dev_priv->drm.struct_mutex);
3012
3013         for_each_engine(engine, dev_priv, id) {
3014                 engine->hangcheck.active_request = NULL;
3015                 i915_gem_reset_finish_engine(engine);
3016         }
3017 }
3018
3019 static void nop_submit_request(struct drm_i915_gem_request *request)
3020 {
3021         GEM_BUG_ON(!i915_terminally_wedged(&request->i915->gpu_error));
3022         dma_fence_set_error(&request->fence, -EIO);
3023         i915_gem_request_submit(request);
3024         intel_engine_init_global_seqno(request->engine, request->global_seqno);
3025 }
3026
3027 static void engine_set_wedged(struct intel_engine_cs *engine)
3028 {
3029         struct drm_i915_gem_request *request;
3030         unsigned long flags;
3031
3032         /* We need to be sure that no thread is running the old callback as
3033          * we install the nop handler (otherwise we would submit a request
3034          * to hardware that will never complete). In order to prevent this
3035          * race, we wait until the machine is idle before making the swap
3036          * (using stop_machine()).
3037          */
3038         engine->submit_request = nop_submit_request;
3039
3040         /* Mark all executing requests as skipped */
3041         spin_lock_irqsave(&engine->timeline->lock, flags);
3042         list_for_each_entry(request, &engine->timeline->requests, link)
3043                 if (!i915_gem_request_completed(request))
3044                         dma_fence_set_error(&request->fence, -EIO);
3045         spin_unlock_irqrestore(&engine->timeline->lock, flags);
3046
3047         /*
3048          * Clear the execlists queue up before freeing the requests, as those
3049          * are the ones that keep the context and ringbuffer backing objects
3050          * pinned in place.
3051          */
3052
3053         if (i915.enable_execlists) {
3054                 struct execlist_port *port = engine->execlist_port;
3055                 unsigned long flags;
3056                 unsigned int n;
3057
3058                 spin_lock_irqsave(&engine->timeline->lock, flags);
3059
3060                 for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
3061                         i915_gem_request_put(port_request(&port[n]));
3062                 memset(engine->execlist_port, 0, sizeof(engine->execlist_port));
3063                 engine->execlist_queue = RB_ROOT;
3064                 engine->execlist_first = NULL;
3065
3066                 spin_unlock_irqrestore(&engine->timeline->lock, flags);
3067
3068                 /* The port is checked prior to scheduling a tasklet, but
3069                  * just in case we have suspended the tasklet to do the
3070                  * wedging make sure that when it wakes, it decides there
3071                  * is no work to do by clearing the irq_posted bit.
3072                  */
3073                 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
3074         }
3075
3076         /* Mark all pending requests as complete so that any concurrent
3077          * (lockless) lookup doesn't try and wait upon the request as we
3078          * reset it.
3079          */
3080         intel_engine_init_global_seqno(engine,
3081                                        intel_engine_last_submit(engine));
3082 }
3083
3084 static int __i915_gem_set_wedged_BKL(void *data)
3085 {
3086         struct drm_i915_private *i915 = data;
3087         struct intel_engine_cs *engine;
3088         enum intel_engine_id id;
3089
3090         for_each_engine(engine, i915, id)
3091                 engine_set_wedged(engine);
3092
3093         set_bit(I915_WEDGED, &i915->gpu_error.flags);
3094         wake_up_all(&i915->gpu_error.reset_queue);
3095
3096         return 0;
3097 }
3098
3099 void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
3100 {
3101         stop_machine(__i915_gem_set_wedged_BKL, dev_priv, NULL);
3102 }
3103
3104 bool i915_gem_unset_wedged(struct drm_i915_private *i915)
3105 {
3106         struct i915_gem_timeline *tl;
3107         int i;
3108
3109         lockdep_assert_held(&i915->drm.struct_mutex);
3110         if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
3111                 return true;
3112
3113         /* Before unwedging, make sure that all pending operations
3114          * are flushed and errored out - we may have requests waiting upon
3115          * third party fences. We marked all inflight requests as EIO, and
3116          * every execbuf since returned EIO, for consistency we want all
3117          * the currently pending requests to also be marked as EIO, which
3118          * is done inside our nop_submit_request - and so we must wait.
3119          *
3120          * No more can be submitted until we reset the wedged bit.
3121          */
3122         list_for_each_entry(tl, &i915->gt.timelines, link) {
3123                 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
3124                         struct drm_i915_gem_request *rq;
3125
3126                         rq = i915_gem_active_peek(&tl->engine[i].last_request,
3127                                                   &i915->drm.struct_mutex);
3128                         if (!rq)
3129                                 continue;
3130
3131                         /* We can't use our normal waiter as we want to
3132                          * avoid recursively trying to handle the current
3133                          * reset. The basic dma_fence_default_wait() installs
3134                          * a callback for dma_fence_signal(), which is
3135                          * triggered by our nop handler (indirectly, the
3136                          * callback enables the signaler thread which is
3137                          * woken by the nop_submit_request() advancing the seqno
3138                          * and when the seqno passes the fence, the signaler
3139                          * then signals the fence waking us up).
3140                          */
3141                         if (dma_fence_default_wait(&rq->fence, true,
3142                                                    MAX_SCHEDULE_TIMEOUT) < 0)
3143                                 return false;
3144                 }
3145         }
3146
3147         /* Undo nop_submit_request. We prevent all new i915 requests from
3148          * being queued (by disallowing execbuf whilst wedged) so having
3149          * waited for all active requests above, we know the system is idle
3150          * and do not have to worry about a thread being inside
3151          * engine->submit_request() as we swap over. So unlike installing
3152          * the nop_submit_request on reset, we can do this from normal
3153          * context and do not require stop_machine().
3154          */
3155         intel_engines_reset_default_submission(i915);
3156         i915_gem_contexts_lost(i915);
3157
3158         smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
3159         clear_bit(I915_WEDGED, &i915->gpu_error.flags);
3160
3161         return true;
3162 }
3163
3164 static void
3165 i915_gem_retire_work_handler(struct work_struct *work)
3166 {
3167         struct drm_i915_private *dev_priv =
3168                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
3169         struct drm_device *dev = &dev_priv->drm;
3170
3171         /* Come back later if the device is busy... */
3172         if (mutex_trylock(&dev->struct_mutex)) {
3173                 i915_gem_retire_requests(dev_priv);
3174                 mutex_unlock(&dev->struct_mutex);
3175         }
3176
3177         /* Keep the retire handler running until we are finally idle.
3178          * We do not need to do this test under locking as in the worst-case
3179          * we queue the retire worker once too often.
3180          */
3181         if (READ_ONCE(dev_priv->gt.awake)) {
3182                 i915_queue_hangcheck(dev_priv);
3183                 queue_delayed_work(dev_priv->wq,
3184                                    &dev_priv->gt.retire_work,
3185                                    round_jiffies_up_relative(HZ));
3186         }
3187 }
3188
3189 static void
3190 i915_gem_idle_work_handler(struct work_struct *work)
3191 {
3192         struct drm_i915_private *dev_priv =
3193                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
3194         struct drm_device *dev = &dev_priv->drm;
3195         bool rearm_hangcheck;
3196
3197         if (!READ_ONCE(dev_priv->gt.awake))
3198                 return;
3199
3200         /*
3201          * Wait for last execlists context complete, but bail out in case a
3202          * new request is submitted.
3203          */
3204         wait_for(intel_engines_are_idle(dev_priv), 10);
3205         if (READ_ONCE(dev_priv->gt.active_requests))
3206                 return;
3207
3208         rearm_hangcheck =
3209                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3210
3211         if (!mutex_trylock(&dev->struct_mutex)) {
3212                 /* Currently busy, come back later */
3213                 mod_delayed_work(dev_priv->wq,
3214                                  &dev_priv->gt.idle_work,
3215                                  msecs_to_jiffies(50));
3216                 goto out_rearm;
3217         }
3218
3219         /*
3220          * New request retired after this work handler started, extend active
3221          * period until next instance of the work.
3222          */
3223         if (work_pending(work))
3224                 goto out_unlock;
3225
3226         if (dev_priv->gt.active_requests)
3227                 goto out_unlock;
3228
3229         if (wait_for(intel_engines_are_idle(dev_priv), 10))
3230                 DRM_ERROR("Timeout waiting for engines to idle\n");
3231
3232         intel_engines_mark_idle(dev_priv);
3233         i915_gem_timelines_mark_idle(dev_priv);
3234
3235         GEM_BUG_ON(!dev_priv->gt.awake);
3236         dev_priv->gt.awake = false;
3237         rearm_hangcheck = false;
3238
3239         if (INTEL_GEN(dev_priv) >= 6)
3240                 gen6_rps_idle(dev_priv);
3241         intel_runtime_pm_put(dev_priv);
3242 out_unlock:
3243         mutex_unlock(&dev->struct_mutex);
3244
3245 out_rearm:
3246         if (rearm_hangcheck) {
3247                 GEM_BUG_ON(!dev_priv->gt.awake);
3248                 i915_queue_hangcheck(dev_priv);
3249         }
3250 }
3251
3252 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
3253 {
3254         struct drm_i915_private *i915 = to_i915(gem->dev);
3255         struct drm_i915_gem_object *obj = to_intel_bo(gem);
3256         struct drm_i915_file_private *fpriv = file->driver_priv;
3257         struct i915_lut_handle *lut, *ln;
3258
3259         mutex_lock(&i915->drm.struct_mutex);
3260
3261         list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
3262                 struct i915_gem_context *ctx = lut->ctx;
3263                 struct i915_vma *vma;
3264
3265                 GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
3266                 if (ctx->file_priv != fpriv)
3267                         continue;
3268
3269                 vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
3270                 GEM_BUG_ON(vma->obj != obj);
3271
3272                 /* We allow the process to have multiple handles to the same
3273                  * vma, in the same fd namespace, by virtue of flink/open.
3274                  */
3275                 GEM_BUG_ON(!vma->open_count);
3276                 if (!--vma->open_count && !i915_vma_is_ggtt(vma))
3277                         i915_vma_close(vma);
3278
3279                 list_del(&lut->obj_link);
3280                 list_del(&lut->ctx_link);
3281
3282                 kmem_cache_free(i915->luts, lut);
3283                 __i915_gem_object_release_unless_active(obj);
3284         }
3285
3286         mutex_unlock(&i915->drm.struct_mutex);
3287 }
3288
3289 static unsigned long to_wait_timeout(s64 timeout_ns)
3290 {
3291         if (timeout_ns < 0)
3292                 return MAX_SCHEDULE_TIMEOUT;
3293
3294         if (timeout_ns == 0)
3295                 return 0;
3296
3297         return nsecs_to_jiffies_timeout(timeout_ns);
3298 }
3299
3300 /**
3301  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3302  * @dev: drm device pointer
3303  * @data: ioctl data blob
3304  * @file: drm file pointer
3305  *
3306  * Returns 0 if successful, else an error is returned with the remaining time in
3307  * the timeout parameter.
3308  *  -ETIME: object is still busy after timeout
3309  *  -ERESTARTSYS: signal interrupted the wait
3310  *  -ENONENT: object doesn't exist
3311  * Also possible, but rare:
3312  *  -EAGAIN: incomplete, restart syscall
3313  *  -ENOMEM: damn
3314  *  -ENODEV: Internal IRQ fail
3315  *  -E?: The add request failed
3316  *
3317  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3318  * non-zero timeout parameter the wait ioctl will wait for the given number of
3319  * nanoseconds on an object becoming unbusy. Since the wait itself does so
3320  * without holding struct_mutex the object may become re-busied before this
3321  * function completes. A similar but shorter * race condition exists in the busy
3322  * ioctl
3323  */
3324 int
3325 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3326 {
3327</