Merge tag 'drm-intel-next-fixes-2018-03-27' of git://anongit.freedesktop.org/drm...
[muen/linux.git] / drivers / gpu / drm / amd / amdkfd / kfd_process.c
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <linux/slab.h>
29 #include <linux/amd-iommu.h>
30 #include <linux/notifier.h>
31 #include <linux/compat.h>
32 #include <linux/mman.h>
33
34 struct mm_struct;
35
36 #include "kfd_priv.h"
37 #include "kfd_device_queue_manager.h"
38 #include "kfd_dbgmgr.h"
39 #include "kfd_iommu.h"
40
41 /*
42  * List of struct kfd_process (field kfd_process).
43  * Unique/indexed by mm_struct*
44  */
45 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
46 static DEFINE_MUTEX(kfd_processes_mutex);
47
48 DEFINE_SRCU(kfd_processes_srcu);
49
50 static struct workqueue_struct *kfd_process_wq;
51
52 static struct kfd_process *find_process(const struct task_struct *thread);
53 static void kfd_process_ref_release(struct kref *ref);
54 static struct kfd_process *create_process(const struct task_struct *thread,
55                                         struct file *filep);
56 static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep);
57
58 static void evict_process_worker(struct work_struct *work);
59 static void restore_process_worker(struct work_struct *work);
60
61
62 void kfd_process_create_wq(void)
63 {
64         if (!kfd_process_wq)
65                 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
66 }
67
68 void kfd_process_destroy_wq(void)
69 {
70         if (kfd_process_wq) {
71                 destroy_workqueue(kfd_process_wq);
72                 kfd_process_wq = NULL;
73         }
74 }
75
76 struct kfd_process *kfd_create_process(struct file *filep)
77 {
78         struct kfd_process *process;
79         struct task_struct *thread = current;
80
81         if (!thread->mm)
82                 return ERR_PTR(-EINVAL);
83
84         /* Only the pthreads threading model is supported. */
85         if (thread->group_leader->mm != thread->mm)
86                 return ERR_PTR(-EINVAL);
87
88         /*
89          * take kfd processes mutex before starting of process creation
90          * so there won't be a case where two threads of the same process
91          * create two kfd_process structures
92          */
93         mutex_lock(&kfd_processes_mutex);
94
95         /* A prior open of /dev/kfd could have already created the process. */
96         process = find_process(thread);
97         if (process)
98                 pr_debug("Process already found\n");
99         else
100                 process = create_process(thread, filep);
101
102         mutex_unlock(&kfd_processes_mutex);
103
104         return process;
105 }
106
107 struct kfd_process *kfd_get_process(const struct task_struct *thread)
108 {
109         struct kfd_process *process;
110
111         if (!thread->mm)
112                 return ERR_PTR(-EINVAL);
113
114         /* Only the pthreads threading model is supported. */
115         if (thread->group_leader->mm != thread->mm)
116                 return ERR_PTR(-EINVAL);
117
118         process = find_process(thread);
119
120         return process;
121 }
122
123 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
124 {
125         struct kfd_process *process;
126
127         hash_for_each_possible_rcu(kfd_processes_table, process,
128                                         kfd_processes, (uintptr_t)mm)
129                 if (process->mm == mm)
130                         return process;
131
132         return NULL;
133 }
134
135 static struct kfd_process *find_process(const struct task_struct *thread)
136 {
137         struct kfd_process *p;
138         int idx;
139
140         idx = srcu_read_lock(&kfd_processes_srcu);
141         p = find_process_by_mm(thread->mm);
142         srcu_read_unlock(&kfd_processes_srcu, idx);
143
144         return p;
145 }
146
147 void kfd_unref_process(struct kfd_process *p)
148 {
149         kref_put(&p->ref, kfd_process_ref_release);
150 }
151
152 static void kfd_process_destroy_pdds(struct kfd_process *p)
153 {
154         struct kfd_process_device *pdd, *temp;
155
156         list_for_each_entry_safe(pdd, temp, &p->per_device_data,
157                                  per_device_list) {
158                 pr_debug("Releasing pdd (topology id %d) for process (pasid %d)\n",
159                                 pdd->dev->id, p->pasid);
160
161                 if (pdd->vm)
162                         pdd->dev->kfd2kgd->destroy_process_vm(
163                                 pdd->dev->kgd, pdd->vm);
164
165                 list_del(&pdd->per_device_list);
166
167                 if (pdd->qpd.cwsr_kaddr)
168                         free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
169                                 get_order(KFD_CWSR_TBA_TMA_SIZE));
170
171                 kfree(pdd);
172         }
173 }
174
175 /* No process locking is needed in this function, because the process
176  * is not findable any more. We must assume that no other thread is
177  * using it any more, otherwise we couldn't safely free the process
178  * structure in the end.
179  */
180 static void kfd_process_wq_release(struct work_struct *work)
181 {
182         struct kfd_process *p = container_of(work, struct kfd_process,
183                                              release_work);
184
185         kfd_iommu_unbind_process(p);
186
187         kfd_process_destroy_pdds(p);
188         dma_fence_put(p->ef);
189
190         kfd_event_free_process(p);
191
192         kfd_pasid_free(p->pasid);
193         kfd_free_process_doorbells(p);
194
195         mutex_destroy(&p->mutex);
196
197         put_task_struct(p->lead_thread);
198
199         kfree(p);
200 }
201
202 static void kfd_process_ref_release(struct kref *ref)
203 {
204         struct kfd_process *p = container_of(ref, struct kfd_process, ref);
205
206         INIT_WORK(&p->release_work, kfd_process_wq_release);
207         queue_work(kfd_process_wq, &p->release_work);
208 }
209
210 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
211 {
212         struct kfd_process *p = container_of(rcu, struct kfd_process, rcu);
213
214         kfd_unref_process(p);
215 }
216
217 static void kfd_process_notifier_release(struct mmu_notifier *mn,
218                                         struct mm_struct *mm)
219 {
220         struct kfd_process *p;
221         struct kfd_process_device *pdd = NULL;
222
223         /*
224          * The kfd_process structure can not be free because the
225          * mmu_notifier srcu is read locked
226          */
227         p = container_of(mn, struct kfd_process, mmu_notifier);
228         if (WARN_ON(p->mm != mm))
229                 return;
230
231         mutex_lock(&kfd_processes_mutex);
232         hash_del_rcu(&p->kfd_processes);
233         mutex_unlock(&kfd_processes_mutex);
234         synchronize_srcu(&kfd_processes_srcu);
235
236         cancel_delayed_work_sync(&p->eviction_work);
237         cancel_delayed_work_sync(&p->restore_work);
238
239         mutex_lock(&p->mutex);
240
241         /* Iterate over all process device data structures and if the
242          * pdd is in debug mode, we should first force unregistration,
243          * then we will be able to destroy the queues
244          */
245         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
246                 struct kfd_dev *dev = pdd->dev;
247
248                 mutex_lock(kfd_get_dbgmgr_mutex());
249                 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
250                         if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
251                                 kfd_dbgmgr_destroy(dev->dbgmgr);
252                                 dev->dbgmgr = NULL;
253                         }
254                 }
255                 mutex_unlock(kfd_get_dbgmgr_mutex());
256         }
257
258         kfd_process_dequeue_from_all_devices(p);
259         pqm_uninit(&p->pqm);
260
261         /* Indicate to other users that MM is no longer valid */
262         p->mm = NULL;
263
264         mutex_unlock(&p->mutex);
265
266         mmu_notifier_unregister_no_release(&p->mmu_notifier, mm);
267         mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
268 }
269
270 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
271         .release = kfd_process_notifier_release,
272 };
273
274 static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep)
275 {
276         unsigned long  offset;
277         struct kfd_process_device *pdd = NULL;
278         struct kfd_dev *dev = NULL;
279         struct qcm_process_device *qpd = NULL;
280
281         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
282                 dev = pdd->dev;
283                 qpd = &pdd->qpd;
284                 if (!dev->cwsr_enabled || qpd->cwsr_kaddr)
285                         continue;
286                 offset = (dev->id | KFD_MMAP_RESERVED_MEM_MASK) << PAGE_SHIFT;
287                 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
288                         KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
289                         MAP_SHARED, offset);
290
291                 if (IS_ERR_VALUE(qpd->tba_addr)) {
292                         int err = qpd->tba_addr;
293
294                         pr_err("Failure to set tba address. error %d.\n", err);
295                         qpd->tba_addr = 0;
296                         qpd->cwsr_kaddr = NULL;
297                         return err;
298                 }
299
300                 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
301
302                 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
303                 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
304                         qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
305         }
306
307         return 0;
308 }
309
310 static struct kfd_process *create_process(const struct task_struct *thread,
311                                         struct file *filep)
312 {
313         struct kfd_process *process;
314         int err = -ENOMEM;
315
316         process = kzalloc(sizeof(*process), GFP_KERNEL);
317
318         if (!process)
319                 goto err_alloc_process;
320
321         process->pasid = kfd_pasid_alloc();
322         if (process->pasid == 0)
323                 goto err_alloc_pasid;
324
325         if (kfd_alloc_process_doorbells(process) < 0)
326                 goto err_alloc_doorbells;
327
328         kref_init(&process->ref);
329
330         mutex_init(&process->mutex);
331
332         process->mm = thread->mm;
333
334         /* register notifier */
335         process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
336         err = mmu_notifier_register(&process->mmu_notifier, process->mm);
337         if (err)
338                 goto err_mmu_notifier;
339
340         hash_add_rcu(kfd_processes_table, &process->kfd_processes,
341                         (uintptr_t)process->mm);
342
343         process->lead_thread = thread->group_leader;
344         get_task_struct(process->lead_thread);
345
346         INIT_LIST_HEAD(&process->per_device_data);
347
348         kfd_event_init_process(process);
349
350         err = pqm_init(&process->pqm, process);
351         if (err != 0)
352                 goto err_process_pqm_init;
353
354         /* init process apertures*/
355         process->is_32bit_user_mode = in_compat_syscall();
356         err = kfd_init_apertures(process);
357         if (err != 0)
358                 goto err_init_apertures;
359
360         INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
361         INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
362         process->last_restore_timestamp = get_jiffies_64();
363
364         err = kfd_process_init_cwsr(process, filep);
365         if (err)
366                 goto err_init_cwsr;
367
368         return process;
369
370 err_init_cwsr:
371         kfd_process_destroy_pdds(process);
372 err_init_apertures:
373         pqm_uninit(&process->pqm);
374 err_process_pqm_init:
375         hash_del_rcu(&process->kfd_processes);
376         synchronize_rcu();
377         mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
378 err_mmu_notifier:
379         mutex_destroy(&process->mutex);
380         kfd_free_process_doorbells(process);
381 err_alloc_doorbells:
382         kfd_pasid_free(process->pasid);
383 err_alloc_pasid:
384         kfree(process);
385 err_alloc_process:
386         return ERR_PTR(err);
387 }
388
389 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
390                                                         struct kfd_process *p)
391 {
392         struct kfd_process_device *pdd = NULL;
393
394         list_for_each_entry(pdd, &p->per_device_data, per_device_list)
395                 if (pdd->dev == dev)
396                         return pdd;
397
398         return NULL;
399 }
400
401 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
402                                                         struct kfd_process *p)
403 {
404         struct kfd_process_device *pdd = NULL;
405
406         pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
407         if (!pdd)
408                 return NULL;
409
410         pdd->dev = dev;
411         INIT_LIST_HEAD(&pdd->qpd.queues_list);
412         INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
413         pdd->qpd.dqm = dev->dqm;
414         pdd->qpd.pqm = &p->pqm;
415         pdd->qpd.evicted = 0;
416         pdd->process = p;
417         pdd->bound = PDD_UNBOUND;
418         pdd->already_dequeued = false;
419         list_add(&pdd->per_device_list, &p->per_device_data);
420
421         /* Create the GPUVM context for this specific device */
422         if (dev->kfd2kgd->create_process_vm(dev->kgd, &pdd->vm,
423                                             &p->kgd_process_info, &p->ef)) {
424                 pr_err("Failed to create process VM object\n");
425                 goto err_create_pdd;
426         }
427         return pdd;
428
429 err_create_pdd:
430         list_del(&pdd->per_device_list);
431         kfree(pdd);
432         return NULL;
433 }
434
435 /*
436  * Direct the IOMMU to bind the process (specifically the pasid->mm)
437  * to the device.
438  * Unbinding occurs when the process dies or the device is removed.
439  *
440  * Assumes that the process lock is held.
441  */
442 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
443                                                         struct kfd_process *p)
444 {
445         struct kfd_process_device *pdd;
446         int err;
447
448         pdd = kfd_get_process_device_data(dev, p);
449         if (!pdd) {
450                 pr_err("Process device data doesn't exist\n");
451                 return ERR_PTR(-ENOMEM);
452         }
453
454         err = kfd_iommu_bind_process_to_device(pdd);
455         if (err)
456                 return ERR_PTR(err);
457
458         return pdd;
459 }
460
461 struct kfd_process_device *kfd_get_first_process_device_data(
462                                                 struct kfd_process *p)
463 {
464         return list_first_entry(&p->per_device_data,
465                                 struct kfd_process_device,
466                                 per_device_list);
467 }
468
469 struct kfd_process_device *kfd_get_next_process_device_data(
470                                                 struct kfd_process *p,
471                                                 struct kfd_process_device *pdd)
472 {
473         if (list_is_last(&pdd->per_device_list, &p->per_device_data))
474                 return NULL;
475         return list_next_entry(pdd, per_device_list);
476 }
477
478 bool kfd_has_process_device_data(struct kfd_process *p)
479 {
480         return !(list_empty(&p->per_device_data));
481 }
482
483 /* This increments the process->ref counter. */
484 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
485 {
486         struct kfd_process *p, *ret_p = NULL;
487         unsigned int temp;
488
489         int idx = srcu_read_lock(&kfd_processes_srcu);
490
491         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
492                 if (p->pasid == pasid) {
493                         kref_get(&p->ref);
494                         ret_p = p;
495                         break;
496                 }
497         }
498
499         srcu_read_unlock(&kfd_processes_srcu, idx);
500
501         return ret_p;
502 }
503
504 /* This increments the process->ref counter. */
505 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
506 {
507         struct kfd_process *p;
508
509         int idx = srcu_read_lock(&kfd_processes_srcu);
510
511         p = find_process_by_mm(mm);
512         if (p)
513                 kref_get(&p->ref);
514
515         srcu_read_unlock(&kfd_processes_srcu, idx);
516
517         return p;
518 }
519
520 /* process_evict_queues - Evict all user queues of a process
521  *
522  * Eviction is reference-counted per process-device. This means multiple
523  * evictions from different sources can be nested safely.
524  */
525 static int process_evict_queues(struct kfd_process *p)
526 {
527         struct kfd_process_device *pdd;
528         int r = 0;
529         unsigned int n_evicted = 0;
530
531         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
532                 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
533                                                             &pdd->qpd);
534                 if (r) {
535                         pr_err("Failed to evict process queues\n");
536                         goto fail;
537                 }
538                 n_evicted++;
539         }
540
541         return r;
542
543 fail:
544         /* To keep state consistent, roll back partial eviction by
545          * restoring queues
546          */
547         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
548                 if (n_evicted == 0)
549                         break;
550                 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
551                                                               &pdd->qpd))
552                         pr_err("Failed to restore queues\n");
553
554                 n_evicted--;
555         }
556
557         return r;
558 }
559
560 /* process_restore_queues - Restore all user queues of a process */
561 static  int process_restore_queues(struct kfd_process *p)
562 {
563         struct kfd_process_device *pdd;
564         int r, ret = 0;
565
566         list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
567                 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
568                                                               &pdd->qpd);
569                 if (r) {
570                         pr_err("Failed to restore process queues\n");
571                         if (!ret)
572                                 ret = r;
573                 }
574         }
575
576         return ret;
577 }
578
579 static void evict_process_worker(struct work_struct *work)
580 {
581         int ret;
582         struct kfd_process *p;
583         struct delayed_work *dwork;
584
585         dwork = to_delayed_work(work);
586
587         /* Process termination destroys this worker thread. So during the
588          * lifetime of this thread, kfd_process p will be valid
589          */
590         p = container_of(dwork, struct kfd_process, eviction_work);
591         WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
592                   "Eviction fence mismatch\n");
593
594         /* Narrow window of overlap between restore and evict work
595          * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
596          * unreserves KFD BOs, it is possible to evicted again. But
597          * restore has few more steps of finish. So lets wait for any
598          * previous restore work to complete
599          */
600         flush_delayed_work(&p->restore_work);
601
602         pr_debug("Started evicting pasid %d\n", p->pasid);
603         ret = process_evict_queues(p);
604         if (!ret) {
605                 dma_fence_signal(p->ef);
606                 dma_fence_put(p->ef);
607                 p->ef = NULL;
608                 schedule_delayed_work(&p->restore_work,
609                                 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
610
611                 pr_debug("Finished evicting pasid %d\n", p->pasid);
612         } else
613                 pr_err("Failed to evict queues of pasid %d\n", p->pasid);
614 }
615
616 static void restore_process_worker(struct work_struct *work)
617 {
618         struct delayed_work *dwork;
619         struct kfd_process *p;
620         struct kfd_process_device *pdd;
621         int ret = 0;
622
623         dwork = to_delayed_work(work);
624
625         /* Process termination destroys this worker thread. So during the
626          * lifetime of this thread, kfd_process p will be valid
627          */
628         p = container_of(dwork, struct kfd_process, restore_work);
629
630         /* Call restore_process_bos on the first KGD device. This function
631          * takes care of restoring the whole process including other devices.
632          * Restore can fail if enough memory is not available. If so,
633          * reschedule again.
634          */
635         pdd = list_first_entry(&p->per_device_data,
636                                struct kfd_process_device,
637                                per_device_list);
638
639         pr_debug("Started restoring pasid %d\n", p->pasid);
640
641         /* Setting last_restore_timestamp before successful restoration.
642          * Otherwise this would have to be set by KGD (restore_process_bos)
643          * before KFD BOs are unreserved. If not, the process can be evicted
644          * again before the timestamp is set.
645          * If restore fails, the timestamp will be set again in the next
646          * attempt. This would mean that the minimum GPU quanta would be
647          * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
648          * functions)
649          */
650
651         p->last_restore_timestamp = get_jiffies_64();
652         ret = pdd->dev->kfd2kgd->restore_process_bos(p->kgd_process_info,
653                                                      &p->ef);
654         if (ret) {
655                 pr_debug("Failed to restore BOs of pasid %d, retry after %d ms\n",
656                          p->pasid, PROCESS_BACK_OFF_TIME_MS);
657                 ret = schedule_delayed_work(&p->restore_work,
658                                 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
659                 WARN(!ret, "reschedule restore work failed\n");
660                 return;
661         }
662
663         ret = process_restore_queues(p);
664         if (!ret)
665                 pr_debug("Finished restoring pasid %d\n", p->pasid);
666         else
667                 pr_err("Failed to restore queues of pasid %d\n", p->pasid);
668 }
669
670 void kfd_suspend_all_processes(void)
671 {
672         struct kfd_process *p;
673         unsigned int temp;
674         int idx = srcu_read_lock(&kfd_processes_srcu);
675
676         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
677                 cancel_delayed_work_sync(&p->eviction_work);
678                 cancel_delayed_work_sync(&p->restore_work);
679
680                 if (process_evict_queues(p))
681                         pr_err("Failed to suspend process %d\n", p->pasid);
682                 dma_fence_signal(p->ef);
683                 dma_fence_put(p->ef);
684                 p->ef = NULL;
685         }
686         srcu_read_unlock(&kfd_processes_srcu, idx);
687 }
688
689 int kfd_resume_all_processes(void)
690 {
691         struct kfd_process *p;
692         unsigned int temp;
693         int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
694
695         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
696                 if (!schedule_delayed_work(&p->restore_work, 0)) {
697                         pr_err("Restore process %d failed during resume\n",
698                                p->pasid);
699                         ret = -EFAULT;
700                 }
701         }
702         srcu_read_unlock(&kfd_processes_srcu, idx);
703         return ret;
704 }
705
706 int kfd_reserved_mem_mmap(struct kfd_process *process,
707                           struct vm_area_struct *vma)
708 {
709         struct kfd_dev *dev = kfd_device_by_id(vma->vm_pgoff);
710         struct kfd_process_device *pdd;
711         struct qcm_process_device *qpd;
712
713         if (!dev)
714                 return -EINVAL;
715         if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
716                 pr_err("Incorrect CWSR mapping size.\n");
717                 return -EINVAL;
718         }
719
720         pdd = kfd_get_process_device_data(dev, process);
721         if (!pdd)
722                 return -EINVAL;
723         qpd = &pdd->qpd;
724
725         qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
726                                         get_order(KFD_CWSR_TBA_TMA_SIZE));
727         if (!qpd->cwsr_kaddr) {
728                 pr_err("Error allocating per process CWSR buffer.\n");
729                 return -ENOMEM;
730         }
731
732         vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
733                 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
734         /* Mapping pages to user process */
735         return remap_pfn_range(vma, vma->vm_start,
736                                PFN_DOWN(__pa(qpd->cwsr_kaddr)),
737                                KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
738 }
739
740 void kfd_flush_tlb(struct kfd_process_device *pdd)
741 {
742         struct kfd_dev *dev = pdd->dev;
743         const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
744
745         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
746                 /* Nothing to flush until a VMID is assigned, which
747                  * only happens when the first queue is created.
748                  */
749                 if (pdd->qpd.vmid)
750                         f2g->invalidate_tlbs_vmid(dev->kgd, pdd->qpd.vmid);
751         } else {
752                 f2g->invalidate_tlbs(dev->kgd, pdd->process->pasid);
753         }
754 }
755
756 #if defined(CONFIG_DEBUG_FS)
757
758 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
759 {
760         struct kfd_process *p;
761         unsigned int temp;
762         int r = 0;
763
764         int idx = srcu_read_lock(&kfd_processes_srcu);
765
766         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
767                 seq_printf(m, "Process %d PASID %d:\n",
768                            p->lead_thread->tgid, p->pasid);
769
770                 mutex_lock(&p->mutex);
771                 r = pqm_debugfs_mqds(m, &p->pqm);
772                 mutex_unlock(&p->mutex);
773
774                 if (r)
775                         break;
776         }
777
778         srcu_read_unlock(&kfd_processes_srcu, idx);
779
780         return r;
781 }
782
783 #endif