kernel/sysctl.c: define minmax conv functions in terms of non-minmax versions
[muen/linux.git] / drivers / gpu / drm / nouveau / nouveau_dmem.c
1 /*
2  * Copyright 2018 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #include "nouveau_dmem.h"
23 #include "nouveau_drv.h"
24 #include "nouveau_chan.h"
25 #include "nouveau_dma.h"
26 #include "nouveau_mem.h"
27 #include "nouveau_bo.h"
28
29 #include <nvif/class.h>
30 #include <nvif/object.h>
31 #include <nvif/if500b.h>
32 #include <nvif/if900b.h>
33
34 #include <linux/sched/mm.h>
35 #include <linux/hmm.h>
36
37 /*
38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
39  * it in vram while in use. We likely want to overhaul memory management for
40  * nouveau to be more page like (not necessarily with system page size but a
41  * bigger page size) at lowest level and have some shim layer on top that would
42  * provide the same functionality as TTM.
43  */
44 #define DMEM_CHUNK_SIZE (2UL << 20)
45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
46
47 struct nouveau_migrate;
48
49 enum nouveau_aper {
50         NOUVEAU_APER_VIRT,
51         NOUVEAU_APER_VRAM,
52         NOUVEAU_APER_HOST,
53 };
54
55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
56                                       enum nouveau_aper, u64 dst_addr,
57                                       enum nouveau_aper, u64 src_addr);
58
59 struct nouveau_dmem_chunk {
60         struct list_head list;
61         struct nouveau_bo *bo;
62         struct nouveau_drm *drm;
63         unsigned long pfn_first;
64         unsigned long callocated;
65         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
66         spinlock_t lock;
67 };
68
69 struct nouveau_dmem_migrate {
70         nouveau_migrate_copy_t copy_func;
71         struct nouveau_channel *chan;
72 };
73
74 struct nouveau_dmem {
75         struct hmm_devmem *devmem;
76         struct nouveau_dmem_migrate migrate;
77         struct list_head chunk_free;
78         struct list_head chunk_full;
79         struct list_head chunk_empty;
80         struct mutex mutex;
81 };
82
83 struct nouveau_dmem_fault {
84         struct nouveau_drm *drm;
85         struct nouveau_fence *fence;
86         dma_addr_t *dma;
87         unsigned long npages;
88 };
89
90 struct nouveau_migrate {
91         struct vm_area_struct *vma;
92         struct nouveau_drm *drm;
93         struct nouveau_fence *fence;
94         unsigned long npages;
95         dma_addr_t *dma;
96         unsigned long dma_nr;
97 };
98
99 static void
100 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
101 {
102         struct nouveau_dmem_chunk *chunk;
103         struct nouveau_drm *drm;
104         unsigned long idx;
105
106         chunk = (void *)hmm_devmem_page_get_drvdata(page);
107         idx = page_to_pfn(page) - chunk->pfn_first;
108         drm = chunk->drm;
109
110         /*
111          * FIXME:
112          *
113          * This is really a bad example, we need to overhaul nouveau memory
114          * management to be more page focus and allow lighter locking scheme
115          * to be use in the process.
116          */
117         spin_lock(&chunk->lock);
118         clear_bit(idx, chunk->bitmap);
119         WARN_ON(!chunk->callocated);
120         chunk->callocated--;
121         /*
122          * FIXME when chunk->callocated reach 0 we should add the chunk to
123          * a reclaim list so that it can be freed in case of memory pressure.
124          */
125         spin_unlock(&chunk->lock);
126 }
127
128 static void
129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
130                                   const unsigned long *src_pfns,
131                                   unsigned long *dst_pfns,
132                                   unsigned long start,
133                                   unsigned long end,
134                                   void *private)
135 {
136         struct nouveau_dmem_fault *fault = private;
137         struct nouveau_drm *drm = fault->drm;
138         struct device *dev = drm->dev->dev;
139         unsigned long addr, i, npages = 0;
140         nouveau_migrate_copy_t copy;
141         int ret;
142
143
144         /* First allocate new memory */
145         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
146                 struct page *dpage, *spage;
147
148                 dst_pfns[i] = 0;
149                 spage = migrate_pfn_to_page(src_pfns[i]);
150                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
151                         continue;
152
153                 dpage = hmm_vma_alloc_locked_page(vma, addr);
154                 if (!dpage) {
155                         dst_pfns[i] = MIGRATE_PFN_ERROR;
156                         continue;
157                 }
158
159                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
160                               MIGRATE_PFN_LOCKED;
161                 npages++;
162         }
163
164         /* Allocate storage for DMA addresses, so we can unmap later. */
165         fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
166         if (!fault->dma)
167                 goto error;
168
169         /* Copy things over */
170         copy = drm->dmem->migrate.copy_func;
171         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
172                 struct nouveau_dmem_chunk *chunk;
173                 struct page *spage, *dpage;
174                 u64 src_addr, dst_addr;
175
176                 dpage = migrate_pfn_to_page(dst_pfns[i]);
177                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
178                         continue;
179
180                 spage = migrate_pfn_to_page(src_pfns[i]);
181                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
182                         dst_pfns[i] = MIGRATE_PFN_ERROR;
183                         __free_page(dpage);
184                         continue;
185                 }
186
187                 fault->dma[fault->npages] =
188                         dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
189                                            PCI_DMA_BIDIRECTIONAL,
190                                            DMA_ATTR_SKIP_CPU_SYNC);
191                 if (dma_mapping_error(dev, fault->dma[fault->npages])) {
192                         dst_pfns[i] = MIGRATE_PFN_ERROR;
193                         __free_page(dpage);
194                         continue;
195                 }
196
197                 dst_addr = fault->dma[fault->npages++];
198
199                 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
200                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
201                 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
202
203                 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
204                                    NOUVEAU_APER_VRAM, src_addr);
205                 if (ret) {
206                         dst_pfns[i] = MIGRATE_PFN_ERROR;
207                         __free_page(dpage);
208                         continue;
209                 }
210         }
211
212         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
213
214         return;
215
216 error:
217         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
218                 struct page *page;
219
220                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
221                         continue;
222
223                 page = migrate_pfn_to_page(dst_pfns[i]);
224                 dst_pfns[i] = MIGRATE_PFN_ERROR;
225                 if (page == NULL)
226                         continue;
227
228                 __free_page(page);
229         }
230 }
231
232 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
233                                          const unsigned long *src_pfns,
234                                          const unsigned long *dst_pfns,
235                                          unsigned long start,
236                                          unsigned long end,
237                                          void *private)
238 {
239         struct nouveau_dmem_fault *fault = private;
240         struct nouveau_drm *drm = fault->drm;
241
242         if (fault->fence) {
243                 nouveau_fence_wait(fault->fence, true, false);
244                 nouveau_fence_unref(&fault->fence);
245         } else {
246                 /*
247                  * FIXME wait for channel to be IDLE before calling finalizing
248                  * the hmem object below (nouveau_migrate_hmem_fini()).
249                  */
250         }
251
252         while (fault->npages--) {
253                 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
254                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
255         }
256         kfree(fault->dma);
257 }
258
259 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
260         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
261         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
262 };
263
264 static int
265 nouveau_dmem_fault(struct hmm_devmem *devmem,
266                    struct vm_area_struct *vma,
267                    unsigned long addr,
268                    const struct page *page,
269                    unsigned int flags,
270                    pmd_t *pmdp)
271 {
272         struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
273         unsigned long src[1] = {0}, dst[1] = {0};
274         struct nouveau_dmem_fault fault = {0};
275         int ret;
276
277
278
279         /*
280          * FIXME what we really want is to find some heuristic to migrate more
281          * than just one page on CPU fault. When such fault happens it is very
282          * likely that more surrounding page will CPU fault too.
283          */
284         fault.drm = nouveau_drm(drm_dev);
285         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
286                           addr + PAGE_SIZE, src, dst, &fault);
287         if (ret)
288                 return VM_FAULT_SIGBUS;
289
290         if (dst[0] == MIGRATE_PFN_ERROR)
291                 return VM_FAULT_SIGBUS;
292
293         return 0;
294 }
295
296 static const struct hmm_devmem_ops
297 nouveau_dmem_devmem_ops = {
298         .free = nouveau_dmem_free,
299         .fault = nouveau_dmem_fault,
300 };
301
302 static int
303 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
304 {
305         struct nouveau_dmem_chunk *chunk;
306         int ret;
307
308         if (drm->dmem == NULL)
309                 return -EINVAL;
310
311         mutex_lock(&drm->dmem->mutex);
312         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
313                                          struct nouveau_dmem_chunk,
314                                          list);
315         if (chunk == NULL) {
316                 mutex_unlock(&drm->dmem->mutex);
317                 return -ENOMEM;
318         }
319
320         list_del(&chunk->list);
321         mutex_unlock(&drm->dmem->mutex);
322
323         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
324                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
325                              &chunk->bo);
326         if (ret)
327                 goto out;
328
329         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
330         if (ret) {
331                 nouveau_bo_ref(NULL, &chunk->bo);
332                 goto out;
333         }
334
335         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
336         spin_lock_init(&chunk->lock);
337
338 out:
339         mutex_lock(&drm->dmem->mutex);
340         if (chunk->bo)
341                 list_add(&chunk->list, &drm->dmem->chunk_empty);
342         else
343                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
344         mutex_unlock(&drm->dmem->mutex);
345
346         return ret;
347 }
348
349 static struct nouveau_dmem_chunk *
350 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
351 {
352         struct nouveau_dmem_chunk *chunk;
353
354         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
355                                          struct nouveau_dmem_chunk,
356                                          list);
357         if (chunk)
358                 return chunk;
359
360         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
361                                          struct nouveau_dmem_chunk,
362                                          list);
363         if (chunk->bo)
364                 return chunk;
365
366         return NULL;
367 }
368
369 static int
370 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
371                          unsigned long npages,
372                          unsigned long *pages)
373 {
374         struct nouveau_dmem_chunk *chunk;
375         unsigned long c;
376         int ret;
377
378         memset(pages, 0xff, npages * sizeof(*pages));
379
380         mutex_lock(&drm->dmem->mutex);
381         for (c = 0; c < npages;) {
382                 unsigned long i;
383
384                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
385                 if (chunk == NULL) {
386                         mutex_unlock(&drm->dmem->mutex);
387                         ret = nouveau_dmem_chunk_alloc(drm);
388                         if (ret) {
389                                 if (c)
390                                         break;
391                                 return ret;
392                         }
393                         continue;
394                 }
395
396                 spin_lock(&chunk->lock);
397                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
398                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
399                         pages[c] = chunk->pfn_first + i;
400                         set_bit(i, chunk->bitmap);
401                         chunk->callocated++;
402                         c++;
403
404                         i = find_next_zero_bit(chunk->bitmap,
405                                         DMEM_CHUNK_NPAGES, i);
406                 }
407                 spin_unlock(&chunk->lock);
408         }
409         mutex_unlock(&drm->dmem->mutex);
410
411         return 0;
412 }
413
414 static struct page *
415 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
416 {
417         unsigned long pfns[1];
418         struct page *page;
419         int ret;
420
421         /* FIXME stop all the miss-match API ... */
422         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
423         if (ret)
424                 return NULL;
425
426         page = pfn_to_page(pfns[0]);
427         get_page(page);
428         lock_page(page);
429         return page;
430 }
431
432 static void
433 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
434 {
435         unlock_page(page);
436         put_page(page);
437 }
438
439 void
440 nouveau_dmem_resume(struct nouveau_drm *drm)
441 {
442         struct nouveau_dmem_chunk *chunk;
443         int ret;
444
445         if (drm->dmem == NULL)
446                 return;
447
448         mutex_lock(&drm->dmem->mutex);
449         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
450                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
451                 /* FIXME handle pin failure */
452                 WARN_ON(ret);
453         }
454         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
455                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
456                 /* FIXME handle pin failure */
457                 WARN_ON(ret);
458         }
459         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
460                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
461                 /* FIXME handle pin failure */
462                 WARN_ON(ret);
463         }
464         mutex_unlock(&drm->dmem->mutex);
465 }
466
467 void
468 nouveau_dmem_suspend(struct nouveau_drm *drm)
469 {
470         struct nouveau_dmem_chunk *chunk;
471
472         if (drm->dmem == NULL)
473                 return;
474
475         mutex_lock(&drm->dmem->mutex);
476         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
477                 nouveau_bo_unpin(chunk->bo);
478         }
479         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
480                 nouveau_bo_unpin(chunk->bo);
481         }
482         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
483                 nouveau_bo_unpin(chunk->bo);
484         }
485         mutex_unlock(&drm->dmem->mutex);
486 }
487
488 void
489 nouveau_dmem_fini(struct nouveau_drm *drm)
490 {
491         struct nouveau_dmem_chunk *chunk, *tmp;
492
493         if (drm->dmem == NULL)
494                 return;
495
496         mutex_lock(&drm->dmem->mutex);
497
498         WARN_ON(!list_empty(&drm->dmem->chunk_free));
499         WARN_ON(!list_empty(&drm->dmem->chunk_full));
500
501         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
502                 if (chunk->bo) {
503                         nouveau_bo_unpin(chunk->bo);
504                         nouveau_bo_ref(NULL, &chunk->bo);
505                 }
506                 list_del(&chunk->list);
507                 kfree(chunk);
508         }
509
510         mutex_unlock(&drm->dmem->mutex);
511 }
512
513 static int
514 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
515                     enum nouveau_aper dst_aper, u64 dst_addr,
516                     enum nouveau_aper src_aper, u64 src_addr)
517 {
518         struct nouveau_channel *chan = drm->dmem->migrate.chan;
519         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
520                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
521                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
522                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
523                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
524         int ret;
525
526         ret = RING_SPACE(chan, 13);
527         if (ret)
528                 return ret;
529
530         if (src_aper != NOUVEAU_APER_VIRT) {
531                 switch (src_aper) {
532                 case NOUVEAU_APER_VRAM:
533                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
534                         break;
535                 case NOUVEAU_APER_HOST:
536                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
537                         break;
538                 default:
539                         return -EINVAL;
540                 }
541                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
542         }
543
544         if (dst_aper != NOUVEAU_APER_VIRT) {
545                 switch (dst_aper) {
546                 case NOUVEAU_APER_VRAM:
547                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
548                         break;
549                 case NOUVEAU_APER_HOST:
550                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
551                         break;
552                 default:
553                         return -EINVAL;
554                 }
555                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
556         }
557
558         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
559         OUT_RING  (chan, upper_32_bits(src_addr));
560         OUT_RING  (chan, lower_32_bits(src_addr));
561         OUT_RING  (chan, upper_32_bits(dst_addr));
562         OUT_RING  (chan, lower_32_bits(dst_addr));
563         OUT_RING  (chan, PAGE_SIZE);
564         OUT_RING  (chan, PAGE_SIZE);
565         OUT_RING  (chan, PAGE_SIZE);
566         OUT_RING  (chan, npages);
567         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
568         OUT_RING  (chan, launch_dma);
569         return 0;
570 }
571
572 static int
573 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
574 {
575         switch (drm->ttm.copy.oclass) {
576         case PASCAL_DMA_COPY_A:
577         case PASCAL_DMA_COPY_B:
578         case  VOLTA_DMA_COPY_A:
579         case TURING_DMA_COPY_A:
580                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
581                 drm->dmem->migrate.chan = drm->ttm.chan;
582                 return 0;
583         default:
584                 break;
585         }
586         return -ENODEV;
587 }
588
589 void
590 nouveau_dmem_init(struct nouveau_drm *drm)
591 {
592         struct device *device = drm->dev->dev;
593         unsigned long i, size;
594         int ret;
595
596         /* This only make sense on PASCAL or newer */
597         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
598                 return;
599
600         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
601                 return;
602
603         mutex_init(&drm->dmem->mutex);
604         INIT_LIST_HEAD(&drm->dmem->chunk_free);
605         INIT_LIST_HEAD(&drm->dmem->chunk_full);
606         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
607
608         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
609
610         /* Initialize migration dma helpers before registering memory */
611         ret = nouveau_dmem_migrate_init(drm);
612         if (ret) {
613                 kfree(drm->dmem);
614                 drm->dmem = NULL;
615                 return;
616         }
617
618         /*
619          * FIXME we need some kind of policy to decide how much VRAM we
620          * want to register with HMM. For now just register everything
621          * and latter if we want to do thing like over commit then we
622          * could revisit this.
623          */
624         drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
625                                            device, size);
626         if (drm->dmem->devmem == NULL) {
627                 kfree(drm->dmem);
628                 drm->dmem = NULL;
629                 return;
630         }
631
632         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
633                 struct nouveau_dmem_chunk *chunk;
634                 struct page *page;
635                 unsigned long j;
636
637                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
638                 if (chunk == NULL) {
639                         nouveau_dmem_fini(drm);
640                         return;
641                 }
642
643                 chunk->drm = drm;
644                 chunk->pfn_first = drm->dmem->devmem->pfn_first;
645                 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
646                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
647
648                 page = pfn_to_page(chunk->pfn_first);
649                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
650                         hmm_devmem_page_set_drvdata(page, (long)chunk);
651                 }
652         }
653
654         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
655 }
656
657 static void
658 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
659                                     const unsigned long *src_pfns,
660                                     unsigned long *dst_pfns,
661                                     unsigned long start,
662                                     unsigned long end,
663                                     void *private)
664 {
665         struct nouveau_migrate *migrate = private;
666         struct nouveau_drm *drm = migrate->drm;
667         struct device *dev = drm->dev->dev;
668         unsigned long addr, i, npages = 0;
669         nouveau_migrate_copy_t copy;
670         int ret;
671
672         /* First allocate new memory */
673         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
674                 struct page *dpage, *spage;
675
676                 dst_pfns[i] = 0;
677                 spage = migrate_pfn_to_page(src_pfns[i]);
678                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
679                         continue;
680
681                 dpage = nouveau_dmem_page_alloc_locked(drm);
682                 if (!dpage)
683                         continue;
684
685                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
686                               MIGRATE_PFN_LOCKED |
687                               MIGRATE_PFN_DEVICE;
688                 npages++;
689         }
690
691         if (!npages)
692                 return;
693
694         /* Allocate storage for DMA addresses, so we can unmap later. */
695         migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
696         if (!migrate->dma)
697                 goto error;
698
699         /* Copy things over */
700         copy = drm->dmem->migrate.copy_func;
701         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
702                 struct nouveau_dmem_chunk *chunk;
703                 struct page *spage, *dpage;
704                 u64 src_addr, dst_addr;
705
706                 dpage = migrate_pfn_to_page(dst_pfns[i]);
707                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
708                         continue;
709
710                 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
711                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
712                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
713
714                 spage = migrate_pfn_to_page(src_pfns[i]);
715                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
716                         nouveau_dmem_page_free_locked(drm, dpage);
717                         dst_pfns[i] = 0;
718                         continue;
719                 }
720
721                 migrate->dma[migrate->dma_nr] =
722                         dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
723                                            PCI_DMA_BIDIRECTIONAL,
724                                            DMA_ATTR_SKIP_CPU_SYNC);
725                 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
726                         nouveau_dmem_page_free_locked(drm, dpage);
727                         dst_pfns[i] = 0;
728                         continue;
729                 }
730
731                 src_addr = migrate->dma[migrate->dma_nr++];
732
733                 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
734                                    NOUVEAU_APER_HOST, src_addr);
735                 if (ret) {
736                         nouveau_dmem_page_free_locked(drm, dpage);
737                         dst_pfns[i] = 0;
738                         continue;
739                 }
740         }
741
742         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
743
744         return;
745
746 error:
747         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
748                 struct page *page;
749
750                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
751                         continue;
752
753                 page = migrate_pfn_to_page(dst_pfns[i]);
754                 dst_pfns[i] = MIGRATE_PFN_ERROR;
755                 if (page == NULL)
756                         continue;
757
758                 __free_page(page);
759         }
760 }
761
762 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
763                                            const unsigned long *src_pfns,
764                                            const unsigned long *dst_pfns,
765                                            unsigned long start,
766                                            unsigned long end,
767                                            void *private)
768 {
769         struct nouveau_migrate *migrate = private;
770         struct nouveau_drm *drm = migrate->drm;
771
772         if (migrate->fence) {
773                 nouveau_fence_wait(migrate->fence, true, false);
774                 nouveau_fence_unref(&migrate->fence);
775         } else {
776                 /*
777                  * FIXME wait for channel to be IDLE before finalizing
778                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
779                  */
780         }
781
782         while (migrate->dma_nr--) {
783                 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
784                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
785         }
786         kfree(migrate->dma);
787
788         /*
789          * FIXME optimization: update GPU page table to point to newly
790          * migrated memory.
791          */
792 }
793
794 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
795         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
796         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
797 };
798
799 int
800 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
801                          struct vm_area_struct *vma,
802                          unsigned long start,
803                          unsigned long end)
804 {
805         unsigned long *src_pfns, *dst_pfns, npages;
806         struct nouveau_migrate migrate = {0};
807         unsigned long i, c, max;
808         int ret = 0;
809
810         npages = (end - start) >> PAGE_SHIFT;
811         max = min(SG_MAX_SINGLE_ALLOC, npages);
812         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
813         if (src_pfns == NULL)
814                 return -ENOMEM;
815         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
816         if (dst_pfns == NULL) {
817                 kfree(src_pfns);
818                 return -ENOMEM;
819         }
820
821         migrate.drm = drm;
822         migrate.vma = vma;
823         migrate.npages = npages;
824         for (i = 0; i < npages; i += c) {
825                 unsigned long next;
826
827                 c = min(SG_MAX_SINGLE_ALLOC, npages);
828                 next = start + (c << PAGE_SHIFT);
829                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
830                                   next, src_pfns, dst_pfns, &migrate);
831                 if (ret)
832                         goto out;
833                 start = next;
834         }
835
836 out:
837         kfree(dst_pfns);
838         kfree(src_pfns);
839         return ret;
840 }
841
842 static inline bool
843 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
844 {
845         if (!is_device_private_page(page))
846                 return false;
847
848         if (drm->dmem->devmem != page->pgmap->data)
849                 return false;
850
851         return true;
852 }
853
854 void
855 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
856                          struct hmm_range *range)
857 {
858         unsigned long i, npages;
859
860         npages = (range->end - range->start) >> PAGE_SHIFT;
861         for (i = 0; i < npages; ++i) {
862                 struct nouveau_dmem_chunk *chunk;
863                 struct page *page;
864                 uint64_t addr;
865
866                 page = hmm_pfn_to_page(range, range->pfns[i]);
867                 if (page == NULL)
868                         continue;
869
870                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
871                         continue;
872                 }
873
874                 if (!nouveau_dmem_page(drm, page)) {
875                         WARN(1, "Some unknown device memory !\n");
876                         range->pfns[i] = 0;
877                         continue;
878                 }
879
880                 chunk = (void *)hmm_devmem_page_get_drvdata(page);
881                 addr = page_to_pfn(page) - chunk->pfn_first;
882                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
883
884                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
885                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
886         }
887 }