Merge tag 'docs-4.15-2' of git://git.lwn.net/linux
[muen/linux.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_gem.c
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/ktime.h>
29 #include <linux/pagemap.h>
30 #include <drm/drmP.h>
31 #include <drm/amdgpu_drm.h>
32 #include "amdgpu.h"
33
34 void amdgpu_gem_object_free(struct drm_gem_object *gobj)
35 {
36         struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj);
37
38         if (robj) {
39                 if (robj->gem_base.import_attach)
40                         drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg);
41                 amdgpu_mn_unregister(robj);
42                 amdgpu_bo_unref(&robj);
43         }
44 }
45
46 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
47                              int alignment, u32 initial_domain,
48                              u64 flags, bool kernel,
49                              struct reservation_object *resv,
50                              struct drm_gem_object **obj)
51 {
52         struct amdgpu_bo *bo;
53         int r;
54
55         *obj = NULL;
56         /* At least align on page size */
57         if (alignment < PAGE_SIZE) {
58                 alignment = PAGE_SIZE;
59         }
60
61 retry:
62         r = amdgpu_bo_create(adev, size, alignment, kernel, initial_domain,
63                              flags, NULL, resv, 0, &bo);
64         if (r) {
65                 if (r != -ERESTARTSYS) {
66                         if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
67                                 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
68                                 goto retry;
69                         }
70                         DRM_ERROR("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
71                                   size, initial_domain, alignment, r);
72                 }
73                 return r;
74         }
75         *obj = &bo->gem_base;
76
77         return 0;
78 }
79
80 void amdgpu_gem_force_release(struct amdgpu_device *adev)
81 {
82         struct drm_device *ddev = adev->ddev;
83         struct drm_file *file;
84
85         mutex_lock(&ddev->filelist_mutex);
86
87         list_for_each_entry(file, &ddev->filelist, lhead) {
88                 struct drm_gem_object *gobj;
89                 int handle;
90
91                 WARN_ONCE(1, "Still active user space clients!\n");
92                 spin_lock(&file->table_lock);
93                 idr_for_each_entry(&file->object_idr, gobj, handle) {
94                         WARN_ONCE(1, "And also active allocations!\n");
95                         drm_gem_object_put_unlocked(gobj);
96                 }
97                 idr_destroy(&file->object_idr);
98                 spin_unlock(&file->table_lock);
99         }
100
101         mutex_unlock(&ddev->filelist_mutex);
102 }
103
104 /*
105  * Call from drm_gem_handle_create which appear in both new and open ioctl
106  * case.
107  */
108 int amdgpu_gem_object_open(struct drm_gem_object *obj,
109                            struct drm_file *file_priv)
110 {
111         struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
112         struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
113         struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
114         struct amdgpu_vm *vm = &fpriv->vm;
115         struct amdgpu_bo_va *bo_va;
116         struct mm_struct *mm;
117         int r;
118
119         mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
120         if (mm && mm != current->mm)
121                 return -EPERM;
122
123         if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
124             abo->tbo.resv != vm->root.base.bo->tbo.resv)
125                 return -EPERM;
126
127         r = amdgpu_bo_reserve(abo, false);
128         if (r)
129                 return r;
130
131         bo_va = amdgpu_vm_bo_find(vm, abo);
132         if (!bo_va) {
133                 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
134         } else {
135                 ++bo_va->ref_count;
136         }
137         amdgpu_bo_unreserve(abo);
138         return 0;
139 }
140
141 void amdgpu_gem_object_close(struct drm_gem_object *obj,
142                              struct drm_file *file_priv)
143 {
144         struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
145         struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
146         struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
147         struct amdgpu_vm *vm = &fpriv->vm;
148
149         struct amdgpu_bo_list_entry vm_pd;
150         struct list_head list, duplicates;
151         struct ttm_validate_buffer tv;
152         struct ww_acquire_ctx ticket;
153         struct amdgpu_bo_va *bo_va;
154         int r;
155
156         INIT_LIST_HEAD(&list);
157         INIT_LIST_HEAD(&duplicates);
158
159         tv.bo = &bo->tbo;
160         tv.shared = true;
161         list_add(&tv.head, &list);
162
163         amdgpu_vm_get_pd_bo(vm, &list, &vm_pd);
164
165         r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates);
166         if (r) {
167                 dev_err(adev->dev, "leaking bo va because "
168                         "we fail to reserve bo (%d)\n", r);
169                 return;
170         }
171         bo_va = amdgpu_vm_bo_find(vm, bo);
172         if (bo_va && --bo_va->ref_count == 0) {
173                 amdgpu_vm_bo_rmv(adev, bo_va);
174
175                 if (amdgpu_vm_ready(vm)) {
176                         struct dma_fence *fence = NULL;
177
178                         r = amdgpu_vm_clear_freed(adev, vm, &fence);
179                         if (unlikely(r)) {
180                                 dev_err(adev->dev, "failed to clear page "
181                                         "tables on GEM object close (%d)\n", r);
182                         }
183
184                         if (fence) {
185                                 amdgpu_bo_fence(bo, fence, true);
186                                 dma_fence_put(fence);
187                         }
188                 }
189         }
190         ttm_eu_backoff_reservation(&ticket, &list);
191 }
192
193 /*
194  * GEM ioctls.
195  */
196 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
197                             struct drm_file *filp)
198 {
199         struct amdgpu_device *adev = dev->dev_private;
200         struct amdgpu_fpriv *fpriv = filp->driver_priv;
201         struct amdgpu_vm *vm = &fpriv->vm;
202         union drm_amdgpu_gem_create *args = data;
203         uint64_t flags = args->in.domain_flags;
204         uint64_t size = args->in.bo_size;
205         struct reservation_object *resv = NULL;
206         struct drm_gem_object *gobj;
207         uint32_t handle;
208         int r;
209
210         /* reject invalid gem flags */
211         if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
212                       AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
213                       AMDGPU_GEM_CREATE_CPU_GTT_USWC |
214                       AMDGPU_GEM_CREATE_VRAM_CLEARED |
215                       AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
216                       AMDGPU_GEM_CREATE_EXPLICIT_SYNC))
217
218                 return -EINVAL;
219
220         /* reject invalid gem domains */
221         if (args->in.domains & ~(AMDGPU_GEM_DOMAIN_CPU |
222                                  AMDGPU_GEM_DOMAIN_GTT |
223                                  AMDGPU_GEM_DOMAIN_VRAM |
224                                  AMDGPU_GEM_DOMAIN_GDS |
225                                  AMDGPU_GEM_DOMAIN_GWS |
226                                  AMDGPU_GEM_DOMAIN_OA))
227                 return -EINVAL;
228
229         /* create a gem object to contain this object in */
230         if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
231             AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
232                 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
233                 if (args->in.domains == AMDGPU_GEM_DOMAIN_GDS)
234                         size = size << AMDGPU_GDS_SHIFT;
235                 else if (args->in.domains == AMDGPU_GEM_DOMAIN_GWS)
236                         size = size << AMDGPU_GWS_SHIFT;
237                 else if (args->in.domains == AMDGPU_GEM_DOMAIN_OA)
238                         size = size << AMDGPU_OA_SHIFT;
239                 else
240                         return -EINVAL;
241         }
242         size = roundup(size, PAGE_SIZE);
243
244         if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
245                 r = amdgpu_bo_reserve(vm->root.base.bo, false);
246                 if (r)
247                         return r;
248
249                 resv = vm->root.base.bo->tbo.resv;
250         }
251
252         r = amdgpu_gem_object_create(adev, size, args->in.alignment,
253                                      (u32)(0xffffffff & args->in.domains),
254                                      flags, false, resv, &gobj);
255         if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
256                 if (!r) {
257                         struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
258
259                         abo->parent = amdgpu_bo_ref(vm->root.base.bo);
260                 }
261                 amdgpu_bo_unreserve(vm->root.base.bo);
262         }
263         if (r)
264                 return r;
265
266         r = drm_gem_handle_create(filp, gobj, &handle);
267         /* drop reference from allocate - handle holds it now */
268         drm_gem_object_put_unlocked(gobj);
269         if (r)
270                 return r;
271
272         memset(args, 0, sizeof(*args));
273         args->out.handle = handle;
274         return 0;
275 }
276
277 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
278                              struct drm_file *filp)
279 {
280         struct amdgpu_device *adev = dev->dev_private;
281         struct drm_amdgpu_gem_userptr *args = data;
282         struct drm_gem_object *gobj;
283         struct amdgpu_bo *bo;
284         uint32_t handle;
285         int r;
286
287         if (offset_in_page(args->addr | args->size))
288                 return -EINVAL;
289
290         /* reject unknown flag values */
291         if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
292             AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
293             AMDGPU_GEM_USERPTR_REGISTER))
294                 return -EINVAL;
295
296         if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
297              !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
298
299                 /* if we want to write to it we must install a MMU notifier */
300                 return -EACCES;
301         }
302
303         /* create a gem object to contain this object in */
304         r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
305                                      0, 0, NULL, &gobj);
306         if (r)
307                 return r;
308
309         bo = gem_to_amdgpu_bo(gobj);
310         bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
311         bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
312         r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags);
313         if (r)
314                 goto release_object;
315
316         if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) {
317                 r = amdgpu_mn_register(bo, args->addr);
318                 if (r)
319                         goto release_object;
320         }
321
322         if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
323                 r = amdgpu_ttm_tt_get_user_pages(bo->tbo.ttm,
324                                                  bo->tbo.ttm->pages);
325                 if (r)
326                         goto unlock_mmap_sem;
327
328                 r = amdgpu_bo_reserve(bo, true);
329                 if (r)
330                         goto free_pages;
331
332                 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
333                 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
334                 amdgpu_bo_unreserve(bo);
335                 if (r)
336                         goto free_pages;
337         }
338
339         r = drm_gem_handle_create(filp, gobj, &handle);
340         /* drop reference from allocate - handle holds it now */
341         drm_gem_object_put_unlocked(gobj);
342         if (r)
343                 return r;
344
345         args->handle = handle;
346         return 0;
347
348 free_pages:
349         release_pages(bo->tbo.ttm->pages, bo->tbo.ttm->num_pages);
350
351 unlock_mmap_sem:
352         up_read(&current->mm->mmap_sem);
353
354 release_object:
355         drm_gem_object_put_unlocked(gobj);
356
357         return r;
358 }
359
360 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
361                           struct drm_device *dev,
362                           uint32_t handle, uint64_t *offset_p)
363 {
364         struct drm_gem_object *gobj;
365         struct amdgpu_bo *robj;
366
367         gobj = drm_gem_object_lookup(filp, handle);
368         if (gobj == NULL) {
369                 return -ENOENT;
370         }
371         robj = gem_to_amdgpu_bo(gobj);
372         if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
373             (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
374                 drm_gem_object_put_unlocked(gobj);
375                 return -EPERM;
376         }
377         *offset_p = amdgpu_bo_mmap_offset(robj);
378         drm_gem_object_put_unlocked(gobj);
379         return 0;
380 }
381
382 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
383                           struct drm_file *filp)
384 {
385         union drm_amdgpu_gem_mmap *args = data;
386         uint32_t handle = args->in.handle;
387         memset(args, 0, sizeof(*args));
388         return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
389 }
390
391 /**
392  * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
393  *
394  * @timeout_ns: timeout in ns
395  *
396  * Calculate the timeout in jiffies from an absolute timeout in ns.
397  */
398 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
399 {
400         unsigned long timeout_jiffies;
401         ktime_t timeout;
402
403         /* clamp timeout if it's to large */
404         if (((int64_t)timeout_ns) < 0)
405                 return MAX_SCHEDULE_TIMEOUT;
406
407         timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
408         if (ktime_to_ns(timeout) < 0)
409                 return 0;
410
411         timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
412         /*  clamp timeout to avoid unsigned-> signed overflow */
413         if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT )
414                 return MAX_SCHEDULE_TIMEOUT - 1;
415
416         return timeout_jiffies;
417 }
418
419 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
420                               struct drm_file *filp)
421 {
422         union drm_amdgpu_gem_wait_idle *args = data;
423         struct drm_gem_object *gobj;
424         struct amdgpu_bo *robj;
425         uint32_t handle = args->in.handle;
426         unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
427         int r = 0;
428         long ret;
429
430         gobj = drm_gem_object_lookup(filp, handle);
431         if (gobj == NULL) {
432                 return -ENOENT;
433         }
434         robj = gem_to_amdgpu_bo(gobj);
435         ret = reservation_object_wait_timeout_rcu(robj->tbo.resv, true, true,
436                                                   timeout);
437
438         /* ret == 0 means not signaled,
439          * ret > 0 means signaled
440          * ret < 0 means interrupted before timeout
441          */
442         if (ret >= 0) {
443                 memset(args, 0, sizeof(*args));
444                 args->out.status = (ret == 0);
445         } else
446                 r = ret;
447
448         drm_gem_object_put_unlocked(gobj);
449         return r;
450 }
451
452 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
453                                 struct drm_file *filp)
454 {
455         struct drm_amdgpu_gem_metadata *args = data;
456         struct drm_gem_object *gobj;
457         struct amdgpu_bo *robj;
458         int r = -1;
459
460         DRM_DEBUG("%d \n", args->handle);
461         gobj = drm_gem_object_lookup(filp, args->handle);
462         if (gobj == NULL)
463                 return -ENOENT;
464         robj = gem_to_amdgpu_bo(gobj);
465
466         r = amdgpu_bo_reserve(robj, false);
467         if (unlikely(r != 0))
468                 goto out;
469
470         if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
471                 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
472                 r = amdgpu_bo_get_metadata(robj, args->data.data,
473                                            sizeof(args->data.data),
474                                            &args->data.data_size_bytes,
475                                            &args->data.flags);
476         } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
477                 if (args->data.data_size_bytes > sizeof(args->data.data)) {
478                         r = -EINVAL;
479                         goto unreserve;
480                 }
481                 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
482                 if (!r)
483                         r = amdgpu_bo_set_metadata(robj, args->data.data,
484                                                    args->data.data_size_bytes,
485                                                    args->data.flags);
486         }
487
488 unreserve:
489         amdgpu_bo_unreserve(robj);
490 out:
491         drm_gem_object_put_unlocked(gobj);
492         return r;
493 }
494
495 /**
496  * amdgpu_gem_va_update_vm -update the bo_va in its VM
497  *
498  * @adev: amdgpu_device pointer
499  * @vm: vm to update
500  * @bo_va: bo_va to update
501  * @list: validation list
502  * @operation: map, unmap or clear
503  *
504  * Update the bo_va directly after setting its address. Errors are not
505  * vital here, so they are not reported back to userspace.
506  */
507 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
508                                     struct amdgpu_vm *vm,
509                                     struct amdgpu_bo_va *bo_va,
510                                     struct list_head *list,
511                                     uint32_t operation)
512 {
513         int r;
514
515         if (!amdgpu_vm_ready(vm))
516                 return;
517
518         r = amdgpu_vm_update_directories(adev, vm);
519         if (r)
520                 goto error;
521
522         r = amdgpu_vm_clear_freed(adev, vm, NULL);
523         if (r)
524                 goto error;
525
526         if (operation == AMDGPU_VA_OP_MAP ||
527             operation == AMDGPU_VA_OP_REPLACE)
528                 r = amdgpu_vm_bo_update(adev, bo_va, false);
529
530 error:
531         if (r && r != -ERESTARTSYS)
532                 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
533 }
534
535 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
536                           struct drm_file *filp)
537 {
538         const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
539                 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
540                 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK;
541         const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
542                 AMDGPU_VM_PAGE_PRT;
543
544         struct drm_amdgpu_gem_va *args = data;
545         struct drm_gem_object *gobj;
546         struct amdgpu_device *adev = dev->dev_private;
547         struct amdgpu_fpriv *fpriv = filp->driver_priv;
548         struct amdgpu_bo *abo;
549         struct amdgpu_bo_va *bo_va;
550         struct amdgpu_bo_list_entry vm_pd;
551         struct ttm_validate_buffer tv;
552         struct ww_acquire_ctx ticket;
553         struct list_head list, duplicates;
554         uint64_t va_flags;
555         int r = 0;
556
557         if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
558                 dev_err(&dev->pdev->dev,
559                         "va_address 0x%lX is in reserved area 0x%X\n",
560                         (unsigned long)args->va_address,
561                         AMDGPU_VA_RESERVED_SIZE);
562                 return -EINVAL;
563         }
564
565         if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
566                 dev_err(&dev->pdev->dev, "invalid flags combination 0x%08X\n",
567                         args->flags);
568                 return -EINVAL;
569         }
570
571         switch (args->operation) {
572         case AMDGPU_VA_OP_MAP:
573         case AMDGPU_VA_OP_UNMAP:
574         case AMDGPU_VA_OP_CLEAR:
575         case AMDGPU_VA_OP_REPLACE:
576                 break;
577         default:
578                 dev_err(&dev->pdev->dev, "unsupported operation %d\n",
579                         args->operation);
580                 return -EINVAL;
581         }
582
583         INIT_LIST_HEAD(&list);
584         INIT_LIST_HEAD(&duplicates);
585         if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
586             !(args->flags & AMDGPU_VM_PAGE_PRT)) {
587                 gobj = drm_gem_object_lookup(filp, args->handle);
588                 if (gobj == NULL)
589                         return -ENOENT;
590                 abo = gem_to_amdgpu_bo(gobj);
591                 tv.bo = &abo->tbo;
592                 tv.shared = false;
593                 list_add(&tv.head, &list);
594         } else {
595                 gobj = NULL;
596                 abo = NULL;
597         }
598
599         amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd);
600
601         r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates);
602         if (r)
603                 goto error_unref;
604
605         if (abo) {
606                 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
607                 if (!bo_va) {
608                         r = -ENOENT;
609                         goto error_backoff;
610                 }
611         } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
612                 bo_va = fpriv->prt_va;
613         } else {
614                 bo_va = NULL;
615         }
616
617         switch (args->operation) {
618         case AMDGPU_VA_OP_MAP:
619                 r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
620                                         args->map_size);
621                 if (r)
622                         goto error_backoff;
623
624                 va_flags = amdgpu_vm_get_pte_flags(adev, args->flags);
625                 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
626                                      args->offset_in_bo, args->map_size,
627                                      va_flags);
628                 break;
629         case AMDGPU_VA_OP_UNMAP:
630                 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
631                 break;
632
633         case AMDGPU_VA_OP_CLEAR:
634                 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
635                                                 args->va_address,
636                                                 args->map_size);
637                 break;
638         case AMDGPU_VA_OP_REPLACE:
639                 r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
640                                         args->map_size);
641                 if (r)
642                         goto error_backoff;
643
644                 va_flags = amdgpu_vm_get_pte_flags(adev, args->flags);
645                 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
646                                              args->offset_in_bo, args->map_size,
647                                              va_flags);
648                 break;
649         default:
650                 break;
651         }
652         if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !amdgpu_vm_debug)
653                 amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, &list,
654                                         args->operation);
655
656 error_backoff:
657         ttm_eu_backoff_reservation(&ticket, &list);
658
659 error_unref:
660         drm_gem_object_put_unlocked(gobj);
661         return r;
662 }
663
664 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
665                         struct drm_file *filp)
666 {
667         struct amdgpu_device *adev = dev->dev_private;
668         struct drm_amdgpu_gem_op *args = data;
669         struct drm_gem_object *gobj;
670         struct amdgpu_bo *robj;
671         int r;
672
673         gobj = drm_gem_object_lookup(filp, args->handle);
674         if (gobj == NULL) {
675                 return -ENOENT;
676         }
677         robj = gem_to_amdgpu_bo(gobj);
678
679         r = amdgpu_bo_reserve(robj, false);
680         if (unlikely(r))
681                 goto out;
682
683         switch (args->op) {
684         case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
685                 struct drm_amdgpu_gem_create_in info;
686                 void __user *out = u64_to_user_ptr(args->value);
687
688                 info.bo_size = robj->gem_base.size;
689                 info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT;
690                 info.domains = robj->preferred_domains;
691                 info.domain_flags = robj->flags;
692                 amdgpu_bo_unreserve(robj);
693                 if (copy_to_user(out, &info, sizeof(info)))
694                         r = -EFAULT;
695                 break;
696         }
697         case AMDGPU_GEM_OP_SET_PLACEMENT:
698                 if (robj->prime_shared_count && (args->value & AMDGPU_GEM_DOMAIN_VRAM)) {
699                         r = -EINVAL;
700                         amdgpu_bo_unreserve(robj);
701                         break;
702                 }
703                 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
704                         r = -EPERM;
705                         amdgpu_bo_unreserve(robj);
706                         break;
707                 }
708                 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
709                                                         AMDGPU_GEM_DOMAIN_GTT |
710                                                         AMDGPU_GEM_DOMAIN_CPU);
711                 robj->allowed_domains = robj->preferred_domains;
712                 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
713                         robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
714
715                 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
716                         amdgpu_vm_bo_invalidate(adev, robj, true);
717
718                 amdgpu_bo_unreserve(robj);
719                 break;
720         default:
721                 amdgpu_bo_unreserve(robj);
722                 r = -EINVAL;
723         }
724
725 out:
726         drm_gem_object_put_unlocked(gobj);
727         return r;
728 }
729
730 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
731                             struct drm_device *dev,
732                             struct drm_mode_create_dumb *args)
733 {
734         struct amdgpu_device *adev = dev->dev_private;
735         struct drm_gem_object *gobj;
736         uint32_t handle;
737         int r;
738
739         args->pitch = amdgpu_align_pitch(adev, args->width,
740                                          DIV_ROUND_UP(args->bpp, 8), 0);
741         args->size = (u64)args->pitch * args->height;
742         args->size = ALIGN(args->size, PAGE_SIZE);
743
744         r = amdgpu_gem_object_create(adev, args->size, 0,
745                                      AMDGPU_GEM_DOMAIN_VRAM,
746                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
747                                      false, NULL, &gobj);
748         if (r)
749                 return -ENOMEM;
750
751         r = drm_gem_handle_create(file_priv, gobj, &handle);
752         /* drop reference from allocate - handle holds it now */
753         drm_gem_object_put_unlocked(gobj);
754         if (r) {
755                 return r;
756         }
757         args->handle = handle;
758         return 0;
759 }
760
761 #if defined(CONFIG_DEBUG_FS)
762 static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
763 {
764         struct drm_gem_object *gobj = ptr;
765         struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
766         struct seq_file *m = data;
767
768         unsigned domain;
769         const char *placement;
770         unsigned pin_count;
771         uint64_t offset;
772
773         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
774         switch (domain) {
775         case AMDGPU_GEM_DOMAIN_VRAM:
776                 placement = "VRAM";
777                 break;
778         case AMDGPU_GEM_DOMAIN_GTT:
779                 placement = " GTT";
780                 break;
781         case AMDGPU_GEM_DOMAIN_CPU:
782         default:
783                 placement = " CPU";
784                 break;
785         }
786         seq_printf(m, "\t0x%08x: %12ld byte %s",
787                    id, amdgpu_bo_size(bo), placement);
788
789         offset = READ_ONCE(bo->tbo.mem.start);
790         if (offset != AMDGPU_BO_INVALID_OFFSET)
791                 seq_printf(m, " @ 0x%010Lx", offset);
792
793         pin_count = READ_ONCE(bo->pin_count);
794         if (pin_count)
795                 seq_printf(m, " pin count %d", pin_count);
796         seq_printf(m, "\n");
797
798         return 0;
799 }
800
801 static int amdgpu_debugfs_gem_info(struct seq_file *m, void *data)
802 {
803         struct drm_info_node *node = (struct drm_info_node *)m->private;
804         struct drm_device *dev = node->minor->dev;
805         struct drm_file *file;
806         int r;
807
808         r = mutex_lock_interruptible(&dev->filelist_mutex);
809         if (r)
810                 return r;
811
812         list_for_each_entry(file, &dev->filelist, lhead) {
813                 struct task_struct *task;
814
815                 /*
816                  * Although we have a valid reference on file->pid, that does
817                  * not guarantee that the task_struct who called get_pid() is
818                  * still alive (e.g. get_pid(current) => fork() => exit()).
819                  * Therefore, we need to protect this ->comm access using RCU.
820                  */
821                 rcu_read_lock();
822                 task = pid_task(file->pid, PIDTYPE_PID);
823                 seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid),
824                            task ? task->comm : "<unknown>");
825                 rcu_read_unlock();
826
827                 spin_lock(&file->table_lock);
828                 idr_for_each(&file->object_idr, amdgpu_debugfs_gem_bo_info, m);
829                 spin_unlock(&file->table_lock);
830         }
831
832         mutex_unlock(&dev->filelist_mutex);
833         return 0;
834 }
835
836 static const struct drm_info_list amdgpu_debugfs_gem_list[] = {
837         {"amdgpu_gem_info", &amdgpu_debugfs_gem_info, 0, NULL},
838 };
839 #endif
840
841 int amdgpu_gem_debugfs_init(struct amdgpu_device *adev)
842 {
843 #if defined(CONFIG_DEBUG_FS)
844         return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1);
845 #endif
846         return 0;
847 }