a22d539b9d4e721884e605d0d4106277db4c6bf8
[muen/linux.git] / drivers / gpu / drm / i915 / gvt / kvmgt.c
1 /*
2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
3  *
4  * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23  * SOFTWARE.
24  *
25  * Authors:
26  *    Kevin Tian <kevin.tian@intel.com>
27  *    Jike Song <jike.song@intel.com>
28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
29  */
30
31 #include <linux/init.h>
32 #include <linux/device.h>
33 #include <linux/mm.h>
34 #include <linux/mmu_context.h>
35 #include <linux/types.h>
36 #include <linux/list.h>
37 #include <linux/rbtree.h>
38 #include <linux/spinlock.h>
39 #include <linux/eventfd.h>
40 #include <linux/uuid.h>
41 #include <linux/kvm_host.h>
42 #include <linux/vfio.h>
43 #include <linux/mdev.h>
44 #include <linux/debugfs.h>
45
46 #include "i915_drv.h"
47 #include "gvt.h"
48
49 static const struct intel_gvt_ops *intel_gvt_ops;
50
51 /* helper macros copied from vfio-pci */
52 #define VFIO_PCI_OFFSET_SHIFT   40
53 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
54 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
55 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
56
57 #define OPREGION_SIGNATURE "IntelGraphicsMem"
58
59 struct vfio_region;
60 struct intel_vgpu_regops {
61         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
62                         size_t count, loff_t *ppos, bool iswrite);
63         void (*release)(struct intel_vgpu *vgpu,
64                         struct vfio_region *region);
65 };
66
67 struct vfio_region {
68         u32                             type;
69         u32                             subtype;
70         size_t                          size;
71         u32                             flags;
72         const struct intel_vgpu_regops  *ops;
73         void                            *data;
74 };
75
76 struct kvmgt_pgfn {
77         gfn_t gfn;
78         struct hlist_node hnode;
79 };
80
81 struct kvmgt_guest_info {
82         struct kvm *kvm;
83         struct intel_vgpu *vgpu;
84         struct kvm_page_track_notifier_node track_node;
85 #define NR_BKT (1 << 18)
86         struct hlist_head ptable[NR_BKT];
87 #undef NR_BKT
88         struct dentry *debugfs_cache_entries;
89 };
90
91 struct gvt_dma {
92         struct intel_vgpu *vgpu;
93         struct rb_node gfn_node;
94         struct rb_node dma_addr_node;
95         gfn_t gfn;
96         dma_addr_t dma_addr;
97         unsigned long size;
98         struct kref ref;
99 };
100
101 static inline bool handle_valid(unsigned long handle)
102 {
103         return !!(handle & ~0xff);
104 }
105
106 static int kvmgt_guest_init(struct mdev_device *mdev);
107 static void intel_vgpu_release_work(struct work_struct *work);
108 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
109
110 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
111                 unsigned long size)
112 {
113         int total_pages;
114         int npage;
115         int ret;
116
117         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
118
119         for (npage = 0; npage < total_pages; npage++) {
120                 unsigned long cur_gfn = gfn + npage;
121
122                 ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1);
123                 WARN_ON(ret != 1);
124         }
125 }
126
127 /* Pin a normal or compound guest page for dma. */
128 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
129                 unsigned long size, struct page **page)
130 {
131         unsigned long base_pfn = 0;
132         int total_pages;
133         int npage;
134         int ret;
135
136         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
137         /*
138          * We pin the pages one-by-one to avoid allocating a big arrary
139          * on stack to hold pfns.
140          */
141         for (npage = 0; npage < total_pages; npage++) {
142                 unsigned long cur_gfn = gfn + npage;
143                 unsigned long pfn;
144
145                 ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1,
146                                      IOMMU_READ | IOMMU_WRITE, &pfn);
147                 if (ret != 1) {
148                         gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
149                                      cur_gfn, ret);
150                         goto err;
151                 }
152
153                 if (!pfn_valid(pfn)) {
154                         gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
155                         npage++;
156                         ret = -EFAULT;
157                         goto err;
158                 }
159
160                 if (npage == 0)
161                         base_pfn = pfn;
162                 else if (base_pfn + npage != pfn) {
163                         gvt_vgpu_err("The pages are not continuous\n");
164                         ret = -EINVAL;
165                         npage++;
166                         goto err;
167                 }
168         }
169
170         *page = pfn_to_page(base_pfn);
171         return 0;
172 err:
173         gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
174         return ret;
175 }
176
177 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
178                 dma_addr_t *dma_addr, unsigned long size)
179 {
180         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
181         struct page *page = NULL;
182         int ret;
183
184         ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
185         if (ret)
186                 return ret;
187
188         /* Setup DMA mapping. */
189         *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
190         ret = dma_mapping_error(dev, *dma_addr);
191         if (ret) {
192                 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
193                              page_to_pfn(page), ret);
194                 gvt_unpin_guest_page(vgpu, gfn, size);
195         }
196
197         return ret;
198 }
199
200 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
201                 dma_addr_t dma_addr, unsigned long size)
202 {
203         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
204
205         dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
206         gvt_unpin_guest_page(vgpu, gfn, size);
207 }
208
209 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
210                 dma_addr_t dma_addr)
211 {
212         struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node;
213         struct gvt_dma *itr;
214
215         while (node) {
216                 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
217
218                 if (dma_addr < itr->dma_addr)
219                         node = node->rb_left;
220                 else if (dma_addr > itr->dma_addr)
221                         node = node->rb_right;
222                 else
223                         return itr;
224         }
225         return NULL;
226 }
227
228 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
229 {
230         struct rb_node *node = vgpu->vdev.gfn_cache.rb_node;
231         struct gvt_dma *itr;
232
233         while (node) {
234                 itr = rb_entry(node, struct gvt_dma, gfn_node);
235
236                 if (gfn < itr->gfn)
237                         node = node->rb_left;
238                 else if (gfn > itr->gfn)
239                         node = node->rb_right;
240                 else
241                         return itr;
242         }
243         return NULL;
244 }
245
246 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
247                 dma_addr_t dma_addr, unsigned long size)
248 {
249         struct gvt_dma *new, *itr;
250         struct rb_node **link, *parent = NULL;
251
252         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
253         if (!new)
254                 return -ENOMEM;
255
256         new->vgpu = vgpu;
257         new->gfn = gfn;
258         new->dma_addr = dma_addr;
259         new->size = size;
260         kref_init(&new->ref);
261
262         /* gfn_cache maps gfn to struct gvt_dma. */
263         link = &vgpu->vdev.gfn_cache.rb_node;
264         while (*link) {
265                 parent = *link;
266                 itr = rb_entry(parent, struct gvt_dma, gfn_node);
267
268                 if (gfn < itr->gfn)
269                         link = &parent->rb_left;
270                 else
271                         link = &parent->rb_right;
272         }
273         rb_link_node(&new->gfn_node, parent, link);
274         rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache);
275
276         /* dma_addr_cache maps dma addr to struct gvt_dma. */
277         parent = NULL;
278         link = &vgpu->vdev.dma_addr_cache.rb_node;
279         while (*link) {
280                 parent = *link;
281                 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
282
283                 if (dma_addr < itr->dma_addr)
284                         link = &parent->rb_left;
285                 else
286                         link = &parent->rb_right;
287         }
288         rb_link_node(&new->dma_addr_node, parent, link);
289         rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache);
290
291         vgpu->vdev.nr_cache_entries++;
292         return 0;
293 }
294
295 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
296                                 struct gvt_dma *entry)
297 {
298         rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache);
299         rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache);
300         kfree(entry);
301         vgpu->vdev.nr_cache_entries--;
302 }
303
304 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
305 {
306         struct gvt_dma *dma;
307         struct rb_node *node = NULL;
308
309         for (;;) {
310                 mutex_lock(&vgpu->vdev.cache_lock);
311                 node = rb_first(&vgpu->vdev.gfn_cache);
312                 if (!node) {
313                         mutex_unlock(&vgpu->vdev.cache_lock);
314                         break;
315                 }
316                 dma = rb_entry(node, struct gvt_dma, gfn_node);
317                 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
318                 __gvt_cache_remove_entry(vgpu, dma);
319                 mutex_unlock(&vgpu->vdev.cache_lock);
320         }
321 }
322
323 static void gvt_cache_init(struct intel_vgpu *vgpu)
324 {
325         vgpu->vdev.gfn_cache = RB_ROOT;
326         vgpu->vdev.dma_addr_cache = RB_ROOT;
327         vgpu->vdev.nr_cache_entries = 0;
328         mutex_init(&vgpu->vdev.cache_lock);
329 }
330
331 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
332 {
333         hash_init(info->ptable);
334 }
335
336 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
337 {
338         struct kvmgt_pgfn *p;
339         struct hlist_node *tmp;
340         int i;
341
342         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
343                 hash_del(&p->hnode);
344                 kfree(p);
345         }
346 }
347
348 static struct kvmgt_pgfn *
349 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
350 {
351         struct kvmgt_pgfn *p, *res = NULL;
352
353         hash_for_each_possible(info->ptable, p, hnode, gfn) {
354                 if (gfn == p->gfn) {
355                         res = p;
356                         break;
357                 }
358         }
359
360         return res;
361 }
362
363 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
364                                 gfn_t gfn)
365 {
366         struct kvmgt_pgfn *p;
367
368         p = __kvmgt_protect_table_find(info, gfn);
369         return !!p;
370 }
371
372 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
373 {
374         struct kvmgt_pgfn *p;
375
376         if (kvmgt_gfn_is_write_protected(info, gfn))
377                 return;
378
379         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
380         if (WARN(!p, "gfn: 0x%llx\n", gfn))
381                 return;
382
383         p->gfn = gfn;
384         hash_add(info->ptable, &p->hnode, gfn);
385 }
386
387 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
388                                 gfn_t gfn)
389 {
390         struct kvmgt_pgfn *p;
391
392         p = __kvmgt_protect_table_find(info, gfn);
393         if (p) {
394                 hash_del(&p->hnode);
395                 kfree(p);
396         }
397 }
398
399 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
400                 size_t count, loff_t *ppos, bool iswrite)
401 {
402         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
403                         VFIO_PCI_NUM_REGIONS;
404         void *base = vgpu->vdev.region[i].data;
405         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
406
407         if (pos >= vgpu->vdev.region[i].size || iswrite) {
408                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
409                 return -EINVAL;
410         }
411         count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
412         memcpy(buf, base + pos, count);
413
414         return count;
415 }
416
417 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
418                 struct vfio_region *region)
419 {
420 }
421
422 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
423         .rw = intel_vgpu_reg_rw_opregion,
424         .release = intel_vgpu_reg_release_opregion,
425 };
426
427 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
428                 unsigned int type, unsigned int subtype,
429                 const struct intel_vgpu_regops *ops,
430                 size_t size, u32 flags, void *data)
431 {
432         struct vfio_region *region;
433
434         region = krealloc(vgpu->vdev.region,
435                         (vgpu->vdev.num_regions + 1) * sizeof(*region),
436                         GFP_KERNEL);
437         if (!region)
438                 return -ENOMEM;
439
440         vgpu->vdev.region = region;
441         vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
442         vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
443         vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
444         vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
445         vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
446         vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
447         vgpu->vdev.num_regions++;
448         return 0;
449 }
450
451 static int kvmgt_get_vfio_device(void *p_vgpu)
452 {
453         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
454
455         vgpu->vdev.vfio_device = vfio_device_get_from_dev(
456                 mdev_dev(vgpu->vdev.mdev));
457         if (!vgpu->vdev.vfio_device) {
458                 gvt_vgpu_err("failed to get vfio device\n");
459                 return -ENODEV;
460         }
461         return 0;
462 }
463
464
465 static int kvmgt_set_opregion(void *p_vgpu)
466 {
467         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
468         void *base;
469         int ret;
470
471         /* Each vgpu has its own opregion, although VFIO would create another
472          * one later. This one is used to expose opregion to VFIO. And the
473          * other one created by VFIO later, is used by guest actually.
474          */
475         base = vgpu_opregion(vgpu)->va;
476         if (!base)
477                 return -ENOMEM;
478
479         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
480                 memunmap(base);
481                 return -EINVAL;
482         }
483
484         ret = intel_vgpu_register_reg(vgpu,
485                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
486                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
487                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
488                         VFIO_REGION_INFO_FLAG_READ, base);
489
490         return ret;
491 }
492
493 static void kvmgt_put_vfio_device(void *vgpu)
494 {
495         if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
496                 return;
497
498         vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
499 }
500
501 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
502 {
503         struct intel_vgpu *vgpu = NULL;
504         struct intel_vgpu_type *type;
505         struct device *pdev;
506         void *gvt;
507         int ret;
508
509         pdev = mdev_parent_dev(mdev);
510         gvt = kdev_to_i915(pdev)->gvt;
511
512         type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
513         if (!type) {
514                 gvt_vgpu_err("failed to find type %s to create\n",
515                                                 kobject_name(kobj));
516                 ret = -EINVAL;
517                 goto out;
518         }
519
520         vgpu = intel_gvt_ops->vgpu_create(gvt, type);
521         if (IS_ERR_OR_NULL(vgpu)) {
522                 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
523                 gvt_err("failed to create intel vgpu: %d\n", ret);
524                 goto out;
525         }
526
527         INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work);
528
529         vgpu->vdev.mdev = mdev;
530         mdev_set_drvdata(mdev, vgpu);
531
532         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
533                      dev_name(mdev_dev(mdev)));
534         ret = 0;
535
536 out:
537         return ret;
538 }
539
540 static int intel_vgpu_remove(struct mdev_device *mdev)
541 {
542         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
543
544         if (handle_valid(vgpu->handle))
545                 return -EBUSY;
546
547         intel_gvt_ops->vgpu_destroy(vgpu);
548         return 0;
549 }
550
551 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
552                                      unsigned long action, void *data)
553 {
554         struct intel_vgpu *vgpu = container_of(nb,
555                                         struct intel_vgpu,
556                                         vdev.iommu_notifier);
557
558         if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
559                 struct vfio_iommu_type1_dma_unmap *unmap = data;
560                 struct gvt_dma *entry;
561                 unsigned long iov_pfn, end_iov_pfn;
562
563                 iov_pfn = unmap->iova >> PAGE_SHIFT;
564                 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
565
566                 mutex_lock(&vgpu->vdev.cache_lock);
567                 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
568                         entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
569                         if (!entry)
570                                 continue;
571
572                         gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
573                                            entry->size);
574                         __gvt_cache_remove_entry(vgpu, entry);
575                 }
576                 mutex_unlock(&vgpu->vdev.cache_lock);
577         }
578
579         return NOTIFY_OK;
580 }
581
582 static int intel_vgpu_group_notifier(struct notifier_block *nb,
583                                      unsigned long action, void *data)
584 {
585         struct intel_vgpu *vgpu = container_of(nb,
586                                         struct intel_vgpu,
587                                         vdev.group_notifier);
588
589         /* the only action we care about */
590         if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
591                 vgpu->vdev.kvm = data;
592
593                 if (!data)
594                         schedule_work(&vgpu->vdev.release_work);
595         }
596
597         return NOTIFY_OK;
598 }
599
600 static int intel_vgpu_open(struct mdev_device *mdev)
601 {
602         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
603         unsigned long events;
604         int ret;
605
606         vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
607         vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier;
608
609         events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
610         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
611                                 &vgpu->vdev.iommu_notifier);
612         if (ret != 0) {
613                 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
614                         ret);
615                 goto out;
616         }
617
618         events = VFIO_GROUP_NOTIFY_SET_KVM;
619         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
620                                 &vgpu->vdev.group_notifier);
621         if (ret != 0) {
622                 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
623                         ret);
624                 goto undo_iommu;
625         }
626
627         ret = kvmgt_guest_init(mdev);
628         if (ret)
629                 goto undo_group;
630
631         intel_gvt_ops->vgpu_activate(vgpu);
632
633         atomic_set(&vgpu->vdev.released, 0);
634         return ret;
635
636 undo_group:
637         vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
638                                         &vgpu->vdev.group_notifier);
639
640 undo_iommu:
641         vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
642                                         &vgpu->vdev.iommu_notifier);
643 out:
644         return ret;
645 }
646
647 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
648 {
649         struct eventfd_ctx *trigger;
650
651         trigger = vgpu->vdev.msi_trigger;
652         if (trigger) {
653                 eventfd_ctx_put(trigger);
654                 vgpu->vdev.msi_trigger = NULL;
655         }
656 }
657
658 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
659 {
660         struct kvmgt_guest_info *info;
661         int ret;
662
663         if (!handle_valid(vgpu->handle))
664                 return;
665
666         if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
667                 return;
668
669         intel_gvt_ops->vgpu_deactivate(vgpu);
670
671         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
672                                         &vgpu->vdev.iommu_notifier);
673         WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret);
674
675         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY,
676                                         &vgpu->vdev.group_notifier);
677         WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret);
678
679         info = (struct kvmgt_guest_info *)vgpu->handle;
680         kvmgt_guest_exit(info);
681
682         intel_vgpu_release_msi_eventfd_ctx(vgpu);
683
684         vgpu->vdev.kvm = NULL;
685         vgpu->handle = 0;
686 }
687
688 static void intel_vgpu_release(struct mdev_device *mdev)
689 {
690         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
691
692         __intel_vgpu_release(vgpu);
693 }
694
695 static void intel_vgpu_release_work(struct work_struct *work)
696 {
697         struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu,
698                                         vdev.release_work);
699
700         __intel_vgpu_release(vgpu);
701 }
702
703 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
704 {
705         u32 start_lo, start_hi;
706         u32 mem_type;
707
708         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
709                         PCI_BASE_ADDRESS_MEM_MASK;
710         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
711                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
712
713         switch (mem_type) {
714         case PCI_BASE_ADDRESS_MEM_TYPE_64:
715                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
716                                                 + bar + 4));
717                 break;
718         case PCI_BASE_ADDRESS_MEM_TYPE_32:
719         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
720                 /* 1M mem BAR treated as 32-bit BAR */
721         default:
722                 /* mem unknown type treated as 32-bit BAR */
723                 start_hi = 0;
724                 break;
725         }
726
727         return ((u64)start_hi << 32) | start_lo;
728 }
729
730 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
731                              void *buf, unsigned int count, bool is_write)
732 {
733         uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
734         int ret;
735
736         if (is_write)
737                 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
738                                         bar_start + off, buf, count);
739         else
740                 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
741                                         bar_start + off, buf, count);
742         return ret;
743 }
744
745 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
746 {
747         return off >= vgpu_aperture_offset(vgpu) &&
748                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
749 }
750
751 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
752                 void *buf, unsigned long count, bool is_write)
753 {
754         void *aperture_va;
755
756         if (!intel_vgpu_in_aperture(vgpu, off) ||
757             !intel_vgpu_in_aperture(vgpu, off + count)) {
758                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
759                 return -EINVAL;
760         }
761
762         aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
763                                         ALIGN_DOWN(off, PAGE_SIZE),
764                                         count + offset_in_page(off));
765         if (!aperture_va)
766                 return -EIO;
767
768         if (is_write)
769                 memcpy(aperture_va + offset_in_page(off), buf, count);
770         else
771                 memcpy(buf, aperture_va + offset_in_page(off), count);
772
773         io_mapping_unmap(aperture_va);
774
775         return 0;
776 }
777
778 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
779                         size_t count, loff_t *ppos, bool is_write)
780 {
781         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
782         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
783         uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
784         int ret = -EINVAL;
785
786
787         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
788                 gvt_vgpu_err("invalid index: %u\n", index);
789                 return -EINVAL;
790         }
791
792         switch (index) {
793         case VFIO_PCI_CONFIG_REGION_INDEX:
794                 if (is_write)
795                         ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
796                                                 buf, count);
797                 else
798                         ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
799                                                 buf, count);
800                 break;
801         case VFIO_PCI_BAR0_REGION_INDEX:
802                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
803                                         buf, count, is_write);
804                 break;
805         case VFIO_PCI_BAR2_REGION_INDEX:
806                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
807                 break;
808         case VFIO_PCI_BAR1_REGION_INDEX:
809         case VFIO_PCI_BAR3_REGION_INDEX:
810         case VFIO_PCI_BAR4_REGION_INDEX:
811         case VFIO_PCI_BAR5_REGION_INDEX:
812         case VFIO_PCI_VGA_REGION_INDEX:
813         case VFIO_PCI_ROM_REGION_INDEX:
814                 break;
815         default:
816                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
817                         return -EINVAL;
818
819                 index -= VFIO_PCI_NUM_REGIONS;
820                 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
821                                 ppos, is_write);
822         }
823
824         return ret == 0 ? count : ret;
825 }
826
827 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
828 {
829         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
830         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
831         struct intel_gvt *gvt = vgpu->gvt;
832         int offset;
833
834         /* Only allow MMIO GGTT entry access */
835         if (index != PCI_BASE_ADDRESS_0)
836                 return false;
837
838         offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
839                 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
840
841         return (offset >= gvt->device_info.gtt_start_offset &&
842                 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
843                         true : false;
844 }
845
846 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
847                         size_t count, loff_t *ppos)
848 {
849         unsigned int done = 0;
850         int ret;
851
852         while (count) {
853                 size_t filled;
854
855                 /* Only support GGTT entry 8 bytes read */
856                 if (count >= 8 && !(*ppos % 8) &&
857                         gtt_entry(mdev, ppos)) {
858                         u64 val;
859
860                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
861                                         ppos, false);
862                         if (ret <= 0)
863                                 goto read_err;
864
865                         if (copy_to_user(buf, &val, sizeof(val)))
866                                 goto read_err;
867
868                         filled = 8;
869                 } else if (count >= 4 && !(*ppos % 4)) {
870                         u32 val;
871
872                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
873                                         ppos, false);
874                         if (ret <= 0)
875                                 goto read_err;
876
877                         if (copy_to_user(buf, &val, sizeof(val)))
878                                 goto read_err;
879
880                         filled = 4;
881                 } else if (count >= 2 && !(*ppos % 2)) {
882                         u16 val;
883
884                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
885                                         ppos, false);
886                         if (ret <= 0)
887                                 goto read_err;
888
889                         if (copy_to_user(buf, &val, sizeof(val)))
890                                 goto read_err;
891
892                         filled = 2;
893                 } else {
894                         u8 val;
895
896                         ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
897                                         false);
898                         if (ret <= 0)
899                                 goto read_err;
900
901                         if (copy_to_user(buf, &val, sizeof(val)))
902                                 goto read_err;
903
904                         filled = 1;
905                 }
906
907                 count -= filled;
908                 done += filled;
909                 *ppos += filled;
910                 buf += filled;
911         }
912
913         return done;
914
915 read_err:
916         return -EFAULT;
917 }
918
919 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
920                                 const char __user *buf,
921                                 size_t count, loff_t *ppos)
922 {
923         unsigned int done = 0;
924         int ret;
925
926         while (count) {
927                 size_t filled;
928
929                 /* Only support GGTT entry 8 bytes write */
930                 if (count >= 8 && !(*ppos % 8) &&
931                         gtt_entry(mdev, ppos)) {
932                         u64 val;
933
934                         if (copy_from_user(&val, buf, sizeof(val)))
935                                 goto write_err;
936
937                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
938                                         ppos, true);
939                         if (ret <= 0)
940                                 goto write_err;
941
942                         filled = 8;
943                 } else if (count >= 4 && !(*ppos % 4)) {
944                         u32 val;
945
946                         if (copy_from_user(&val, buf, sizeof(val)))
947                                 goto write_err;
948
949                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
950                                         ppos, true);
951                         if (ret <= 0)
952                                 goto write_err;
953
954                         filled = 4;
955                 } else if (count >= 2 && !(*ppos % 2)) {
956                         u16 val;
957
958                         if (copy_from_user(&val, buf, sizeof(val)))
959                                 goto write_err;
960
961                         ret = intel_vgpu_rw(mdev, (char *)&val,
962                                         sizeof(val), ppos, true);
963                         if (ret <= 0)
964                                 goto write_err;
965
966                         filled = 2;
967                 } else {
968                         u8 val;
969
970                         if (copy_from_user(&val, buf, sizeof(val)))
971                                 goto write_err;
972
973                         ret = intel_vgpu_rw(mdev, &val, sizeof(val),
974                                         ppos, true);
975                         if (ret <= 0)
976                                 goto write_err;
977
978                         filled = 1;
979                 }
980
981                 count -= filled;
982                 done += filled;
983                 *ppos += filled;
984                 buf += filled;
985         }
986
987         return done;
988 write_err:
989         return -EFAULT;
990 }
991
992 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
993 {
994         unsigned int index;
995         u64 virtaddr;
996         unsigned long req_size, pgoff = 0;
997         pgprot_t pg_prot;
998         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
999
1000         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1001         if (index >= VFIO_PCI_ROM_REGION_INDEX)
1002                 return -EINVAL;
1003
1004         if (vma->vm_end < vma->vm_start)
1005                 return -EINVAL;
1006         if ((vma->vm_flags & VM_SHARED) == 0)
1007                 return -EINVAL;
1008         if (index != VFIO_PCI_BAR2_REGION_INDEX)
1009                 return -EINVAL;
1010
1011         pg_prot = vma->vm_page_prot;
1012         virtaddr = vma->vm_start;
1013         req_size = vma->vm_end - vma->vm_start;
1014         pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
1015
1016         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1017 }
1018
1019 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1020 {
1021         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1022                 return 1;
1023
1024         return 0;
1025 }
1026
1027 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1028                         unsigned int index, unsigned int start,
1029                         unsigned int count, uint32_t flags,
1030                         void *data)
1031 {
1032         return 0;
1033 }
1034
1035 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1036                         unsigned int index, unsigned int start,
1037                         unsigned int count, uint32_t flags, void *data)
1038 {
1039         return 0;
1040 }
1041
1042 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1043                 unsigned int index, unsigned int start, unsigned int count,
1044                 uint32_t flags, void *data)
1045 {
1046         return 0;
1047 }
1048
1049 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1050                 unsigned int index, unsigned int start, unsigned int count,
1051                 uint32_t flags, void *data)
1052 {
1053         struct eventfd_ctx *trigger;
1054
1055         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1056                 int fd = *(int *)data;
1057
1058                 trigger = eventfd_ctx_fdget(fd);
1059                 if (IS_ERR(trigger)) {
1060                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1061                         return PTR_ERR(trigger);
1062                 }
1063                 vgpu->vdev.msi_trigger = trigger;
1064         } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1065                 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1066
1067         return 0;
1068 }
1069
1070 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags,
1071                 unsigned int index, unsigned int start, unsigned int count,
1072                 void *data)
1073 {
1074         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1075                         unsigned int start, unsigned int count, uint32_t flags,
1076                         void *data) = NULL;
1077
1078         switch (index) {
1079         case VFIO_PCI_INTX_IRQ_INDEX:
1080                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1081                 case VFIO_IRQ_SET_ACTION_MASK:
1082                         func = intel_vgpu_set_intx_mask;
1083                         break;
1084                 case VFIO_IRQ_SET_ACTION_UNMASK:
1085                         func = intel_vgpu_set_intx_unmask;
1086                         break;
1087                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1088                         func = intel_vgpu_set_intx_trigger;
1089                         break;
1090                 }
1091                 break;
1092         case VFIO_PCI_MSI_IRQ_INDEX:
1093                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1094                 case VFIO_IRQ_SET_ACTION_MASK:
1095                 case VFIO_IRQ_SET_ACTION_UNMASK:
1096                         /* XXX Need masking support exported */
1097                         break;
1098                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1099                         func = intel_vgpu_set_msi_trigger;
1100                         break;
1101                 }
1102                 break;
1103         }
1104
1105         if (!func)
1106                 return -ENOTTY;
1107
1108         return func(vgpu, index, start, count, flags, data);
1109 }
1110
1111 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
1112                              unsigned long arg)
1113 {
1114         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1115         unsigned long minsz;
1116
1117         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1118
1119         if (cmd == VFIO_DEVICE_GET_INFO) {
1120                 struct vfio_device_info info;
1121
1122                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1123
1124                 if (copy_from_user(&info, (void __user *)arg, minsz))
1125                         return -EFAULT;
1126
1127                 if (info.argsz < minsz)
1128                         return -EINVAL;
1129
1130                 info.flags = VFIO_DEVICE_FLAGS_PCI;
1131                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1132                 info.num_regions = VFIO_PCI_NUM_REGIONS +
1133                                 vgpu->vdev.num_regions;
1134                 info.num_irqs = VFIO_PCI_NUM_IRQS;
1135
1136                 return copy_to_user((void __user *)arg, &info, minsz) ?
1137                         -EFAULT : 0;
1138
1139         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1140                 struct vfio_region_info info;
1141                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1142                 int i, ret;
1143                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1144                 size_t size;
1145                 int nr_areas = 1;
1146                 int cap_type_id;
1147
1148                 minsz = offsetofend(struct vfio_region_info, offset);
1149
1150                 if (copy_from_user(&info, (void __user *)arg, minsz))
1151                         return -EFAULT;
1152
1153                 if (info.argsz < minsz)
1154                         return -EINVAL;
1155
1156                 switch (info.index) {
1157                 case VFIO_PCI_CONFIG_REGION_INDEX:
1158                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1159                         info.size = vgpu->gvt->device_info.cfg_space_size;
1160                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1161                                      VFIO_REGION_INFO_FLAG_WRITE;
1162                         break;
1163                 case VFIO_PCI_BAR0_REGION_INDEX:
1164                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1165                         info.size = vgpu->cfg_space.bar[info.index].size;
1166                         if (!info.size) {
1167                                 info.flags = 0;
1168                                 break;
1169                         }
1170
1171                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1172                                      VFIO_REGION_INFO_FLAG_WRITE;
1173                         break;
1174                 case VFIO_PCI_BAR1_REGION_INDEX:
1175                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1176                         info.size = 0;
1177                         info.flags = 0;
1178                         break;
1179                 case VFIO_PCI_BAR2_REGION_INDEX:
1180                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1181                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1182                                         VFIO_REGION_INFO_FLAG_MMAP |
1183                                         VFIO_REGION_INFO_FLAG_READ |
1184                                         VFIO_REGION_INFO_FLAG_WRITE;
1185                         info.size = gvt_aperture_sz(vgpu->gvt);
1186
1187                         size = sizeof(*sparse) +
1188                                         (nr_areas * sizeof(*sparse->areas));
1189                         sparse = kzalloc(size, GFP_KERNEL);
1190                         if (!sparse)
1191                                 return -ENOMEM;
1192
1193                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1194                         sparse->header.version = 1;
1195                         sparse->nr_areas = nr_areas;
1196                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1197                         sparse->areas[0].offset =
1198                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1199                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1200                         break;
1201
1202                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1203                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1204                         info.size = 0;
1205                         info.flags = 0;
1206
1207                         gvt_dbg_core("get region info bar:%d\n", info.index);
1208                         break;
1209
1210                 case VFIO_PCI_ROM_REGION_INDEX:
1211                 case VFIO_PCI_VGA_REGION_INDEX:
1212                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1213                         info.size = 0;
1214                         info.flags = 0;
1215
1216                         gvt_dbg_core("get region info index:%d\n", info.index);
1217                         break;
1218                 default:
1219                         {
1220                                 struct vfio_region_info_cap_type cap_type = {
1221                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1222                                         .header.version = 1 };
1223
1224                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1225                                                 vgpu->vdev.num_regions)
1226                                         return -EINVAL;
1227
1228                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1229
1230                                 info.offset =
1231                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1232                                 info.size = vgpu->vdev.region[i].size;
1233                                 info.flags = vgpu->vdev.region[i].flags;
1234
1235                                 cap_type.type = vgpu->vdev.region[i].type;
1236                                 cap_type.subtype = vgpu->vdev.region[i].subtype;
1237
1238                                 ret = vfio_info_add_capability(&caps,
1239                                                         &cap_type.header,
1240                                                         sizeof(cap_type));
1241                                 if (ret)
1242                                         return ret;
1243                         }
1244                 }
1245
1246                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1247                         switch (cap_type_id) {
1248                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1249                                 ret = vfio_info_add_capability(&caps,
1250                                         &sparse->header, sizeof(*sparse) +
1251                                         (sparse->nr_areas *
1252                                                 sizeof(*sparse->areas)));
1253                                 kfree(sparse);
1254                                 if (ret)
1255                                         return ret;
1256                                 break;
1257                         default:
1258                                 return -EINVAL;
1259                         }
1260                 }
1261
1262                 if (caps.size) {
1263                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1264                         if (info.argsz < sizeof(info) + caps.size) {
1265                                 info.argsz = sizeof(info) + caps.size;
1266                                 info.cap_offset = 0;
1267                         } else {
1268                                 vfio_info_cap_shift(&caps, sizeof(info));
1269                                 if (copy_to_user((void __user *)arg +
1270                                                   sizeof(info), caps.buf,
1271                                                   caps.size)) {
1272                                         kfree(caps.buf);
1273                                         return -EFAULT;
1274                                 }
1275                                 info.cap_offset = sizeof(info);
1276                         }
1277
1278                         kfree(caps.buf);
1279                 }
1280
1281                 return copy_to_user((void __user *)arg, &info, minsz) ?
1282                         -EFAULT : 0;
1283         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1284                 struct vfio_irq_info info;
1285
1286                 minsz = offsetofend(struct vfio_irq_info, count);
1287
1288                 if (copy_from_user(&info, (void __user *)arg, minsz))
1289                         return -EFAULT;
1290
1291                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1292                         return -EINVAL;
1293
1294                 switch (info.index) {
1295                 case VFIO_PCI_INTX_IRQ_INDEX:
1296                 case VFIO_PCI_MSI_IRQ_INDEX:
1297                         break;
1298                 default:
1299                         return -EINVAL;
1300                 }
1301
1302                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1303
1304                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1305
1306                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1307                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1308                                        VFIO_IRQ_INFO_AUTOMASKED);
1309                 else
1310                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1311
1312                 return copy_to_user((void __user *)arg, &info, minsz) ?
1313                         -EFAULT : 0;
1314         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1315                 struct vfio_irq_set hdr;
1316                 u8 *data = NULL;
1317                 int ret = 0;
1318                 size_t data_size = 0;
1319
1320                 minsz = offsetofend(struct vfio_irq_set, count);
1321
1322                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1323                         return -EFAULT;
1324
1325                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1326                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1327
1328                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1329                                                 VFIO_PCI_NUM_IRQS, &data_size);
1330                         if (ret) {
1331                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1332                                 return -EINVAL;
1333                         }
1334                         if (data_size) {
1335                                 data = memdup_user((void __user *)(arg + minsz),
1336                                                    data_size);
1337                                 if (IS_ERR(data))
1338                                         return PTR_ERR(data);
1339                         }
1340                 }
1341
1342                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1343                                         hdr.start, hdr.count, data);
1344                 kfree(data);
1345
1346                 return ret;
1347         } else if (cmd == VFIO_DEVICE_RESET) {
1348                 intel_gvt_ops->vgpu_reset(vgpu);
1349                 return 0;
1350         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1351                 struct vfio_device_gfx_plane_info dmabuf;
1352                 int ret = 0;
1353
1354                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1355                                     dmabuf_id);
1356                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1357                         return -EFAULT;
1358                 if (dmabuf.argsz < minsz)
1359                         return -EINVAL;
1360
1361                 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1362                 if (ret != 0)
1363                         return ret;
1364
1365                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1366                                                                 -EFAULT : 0;
1367         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1368                 __u32 dmabuf_id;
1369                 __s32 dmabuf_fd;
1370
1371                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1372                         return -EFAULT;
1373
1374                 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1375                 return dmabuf_fd;
1376
1377         }
1378
1379         return -ENOTTY;
1380 }
1381
1382 static ssize_t
1383 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1384              char *buf)
1385 {
1386         struct mdev_device *mdev = mdev_from_dev(dev);
1387
1388         if (mdev) {
1389                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1390                         mdev_get_drvdata(mdev);
1391                 return sprintf(buf, "%d\n", vgpu->id);
1392         }
1393         return sprintf(buf, "\n");
1394 }
1395
1396 static ssize_t
1397 hw_id_show(struct device *dev, struct device_attribute *attr,
1398            char *buf)
1399 {
1400         struct mdev_device *mdev = mdev_from_dev(dev);
1401
1402         if (mdev) {
1403                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1404                         mdev_get_drvdata(mdev);
1405                 return sprintf(buf, "%u\n",
1406                                vgpu->submission.shadow_ctx->hw_id);
1407         }
1408         return sprintf(buf, "\n");
1409 }
1410
1411 static DEVICE_ATTR_RO(vgpu_id);
1412 static DEVICE_ATTR_RO(hw_id);
1413
1414 static struct attribute *intel_vgpu_attrs[] = {
1415         &dev_attr_vgpu_id.attr,
1416         &dev_attr_hw_id.attr,
1417         NULL
1418 };
1419
1420 static const struct attribute_group intel_vgpu_group = {
1421         .name = "intel_vgpu",
1422         .attrs = intel_vgpu_attrs,
1423 };
1424
1425 static const struct attribute_group *intel_vgpu_groups[] = {
1426         &intel_vgpu_group,
1427         NULL,
1428 };
1429
1430 static struct mdev_parent_ops intel_vgpu_ops = {
1431         .mdev_attr_groups       = intel_vgpu_groups,
1432         .create                 = intel_vgpu_create,
1433         .remove                 = intel_vgpu_remove,
1434
1435         .open                   = intel_vgpu_open,
1436         .release                = intel_vgpu_release,
1437
1438         .read                   = intel_vgpu_read,
1439         .write                  = intel_vgpu_write,
1440         .mmap                   = intel_vgpu_mmap,
1441         .ioctl                  = intel_vgpu_ioctl,
1442 };
1443
1444 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1445 {
1446         struct attribute **kvm_type_attrs;
1447         struct attribute_group **kvm_vgpu_type_groups;
1448
1449         intel_gvt_ops = ops;
1450         if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
1451                         &kvm_vgpu_type_groups))
1452                 return -EFAULT;
1453         intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1454
1455         return mdev_register_device(dev, &intel_vgpu_ops);
1456 }
1457
1458 static void kvmgt_host_exit(struct device *dev, void *gvt)
1459 {
1460         mdev_unregister_device(dev);
1461 }
1462
1463 static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1464 {
1465         struct kvmgt_guest_info *info;
1466         struct kvm *kvm;
1467         struct kvm_memory_slot *slot;
1468         int idx;
1469
1470         if (!handle_valid(handle))
1471                 return -ESRCH;
1472
1473         info = (struct kvmgt_guest_info *)handle;
1474         kvm = info->kvm;
1475
1476         idx = srcu_read_lock(&kvm->srcu);
1477         slot = gfn_to_memslot(kvm, gfn);
1478         if (!slot) {
1479                 srcu_read_unlock(&kvm->srcu, idx);
1480                 return -EINVAL;
1481         }
1482
1483         spin_lock(&kvm->mmu_lock);
1484
1485         if (kvmgt_gfn_is_write_protected(info, gfn))
1486                 goto out;
1487
1488         kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1489         kvmgt_protect_table_add(info, gfn);
1490
1491 out:
1492         spin_unlock(&kvm->mmu_lock);
1493         srcu_read_unlock(&kvm->srcu, idx);
1494         return 0;
1495 }
1496
1497 static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1498 {
1499         struct kvmgt_guest_info *info;
1500         struct kvm *kvm;
1501         struct kvm_memory_slot *slot;
1502         int idx;
1503
1504         if (!handle_valid(handle))
1505                 return 0;
1506
1507         info = (struct kvmgt_guest_info *)handle;
1508         kvm = info->kvm;
1509
1510         idx = srcu_read_lock(&kvm->srcu);
1511         slot = gfn_to_memslot(kvm, gfn);
1512         if (!slot) {
1513                 srcu_read_unlock(&kvm->srcu, idx);
1514                 return -EINVAL;
1515         }
1516
1517         spin_lock(&kvm->mmu_lock);
1518
1519         if (!kvmgt_gfn_is_write_protected(info, gfn))
1520                 goto out;
1521
1522         kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1523         kvmgt_protect_table_del(info, gfn);
1524
1525 out:
1526         spin_unlock(&kvm->mmu_lock);
1527         srcu_read_unlock(&kvm->srcu, idx);
1528         return 0;
1529 }
1530
1531 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1532                 const u8 *val, int len,
1533                 struct kvm_page_track_notifier_node *node)
1534 {
1535         struct kvmgt_guest_info *info = container_of(node,
1536                                         struct kvmgt_guest_info, track_node);
1537
1538         if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1539                 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1540                                                      (void *)val, len);
1541 }
1542
1543 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1544                 struct kvm_memory_slot *slot,
1545                 struct kvm_page_track_notifier_node *node)
1546 {
1547         int i;
1548         gfn_t gfn;
1549         struct kvmgt_guest_info *info = container_of(node,
1550                                         struct kvmgt_guest_info, track_node);
1551
1552         spin_lock(&kvm->mmu_lock);
1553         for (i = 0; i < slot->npages; i++) {
1554                 gfn = slot->base_gfn + i;
1555                 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1556                         kvm_slot_page_track_remove_page(kvm, slot, gfn,
1557                                                 KVM_PAGE_TRACK_WRITE);
1558                         kvmgt_protect_table_del(info, gfn);
1559                 }
1560         }
1561         spin_unlock(&kvm->mmu_lock);
1562 }
1563
1564 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1565 {
1566         struct intel_vgpu *itr;
1567         struct kvmgt_guest_info *info;
1568         int id;
1569         bool ret = false;
1570
1571         mutex_lock(&vgpu->gvt->lock);
1572         for_each_active_vgpu(vgpu->gvt, itr, id) {
1573                 if (!handle_valid(itr->handle))
1574                         continue;
1575
1576                 info = (struct kvmgt_guest_info *)itr->handle;
1577                 if (kvm && kvm == info->kvm) {
1578                         ret = true;
1579                         goto out;
1580                 }
1581         }
1582 out:
1583         mutex_unlock(&vgpu->gvt->lock);
1584         return ret;
1585 }
1586
1587 static int kvmgt_guest_init(struct mdev_device *mdev)
1588 {
1589         struct kvmgt_guest_info *info;
1590         struct intel_vgpu *vgpu;
1591         struct kvm *kvm;
1592
1593         vgpu = mdev_get_drvdata(mdev);
1594         if (handle_valid(vgpu->handle))
1595                 return -EEXIST;
1596
1597         kvm = vgpu->vdev.kvm;
1598         if (!kvm || kvm->mm != current->mm) {
1599                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1600                 return -ESRCH;
1601         }
1602
1603         if (__kvmgt_vgpu_exist(vgpu, kvm))
1604                 return -EEXIST;
1605
1606         info = vzalloc(sizeof(struct kvmgt_guest_info));
1607         if (!info)
1608                 return -ENOMEM;
1609
1610         vgpu->handle = (unsigned long)info;
1611         info->vgpu = vgpu;
1612         info->kvm = kvm;
1613         kvm_get_kvm(info->kvm);
1614
1615         kvmgt_protect_table_init(info);
1616         gvt_cache_init(vgpu);
1617
1618         mutex_init(&vgpu->dmabuf_lock);
1619         init_completion(&vgpu->vblank_done);
1620
1621         info->track_node.track_write = kvmgt_page_track_write;
1622         info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1623         kvm_page_track_register_notifier(kvm, &info->track_node);
1624
1625         info->debugfs_cache_entries = debugfs_create_ulong(
1626                                                 "kvmgt_nr_cache_entries",
1627                                                 0444, vgpu->debugfs,
1628                                                 &vgpu->vdev.nr_cache_entries);
1629         if (!info->debugfs_cache_entries)
1630                 gvt_vgpu_err("Cannot create kvmgt debugfs entry\n");
1631
1632         return 0;
1633 }
1634
1635 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1636 {
1637         debugfs_remove(info->debugfs_cache_entries);
1638
1639         kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1640         kvm_put_kvm(info->kvm);
1641         kvmgt_protect_table_destroy(info);
1642         gvt_cache_destroy(info->vgpu);
1643         vfree(info);
1644
1645         return true;
1646 }
1647
1648 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
1649 {
1650         /* nothing to do here */
1651         return 0;
1652 }
1653
1654 static void kvmgt_detach_vgpu(unsigned long handle)
1655 {
1656         /* nothing to do here */
1657 }
1658
1659 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1660 {
1661         struct kvmgt_guest_info *info;
1662         struct intel_vgpu *vgpu;
1663
1664         if (!handle_valid(handle))
1665                 return -ESRCH;
1666
1667         info = (struct kvmgt_guest_info *)handle;
1668         vgpu = info->vgpu;
1669
1670         /*
1671          * When guest is poweroff, msi_trigger is set to NULL, but vgpu's
1672          * config and mmio register isn't restored to default during guest
1673          * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
1674          * may be enabled, then once this vgpu is active, it will get inject
1675          * vblank interrupt request. But msi_trigger is null until msi is
1676          * enabled by guest. so if msi_trigger is null, success is still
1677          * returned and don't inject interrupt into guest.
1678          */
1679         if (vgpu->vdev.msi_trigger == NULL)
1680                 return 0;
1681
1682         if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1)
1683                 return 0;
1684
1685         return -EFAULT;
1686 }
1687
1688 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1689 {
1690         struct kvmgt_guest_info *info;
1691         kvm_pfn_t pfn;
1692
1693         if (!handle_valid(handle))
1694                 return INTEL_GVT_INVALID_ADDR;
1695
1696         info = (struct kvmgt_guest_info *)handle;
1697
1698         pfn = gfn_to_pfn(info->kvm, gfn);
1699         if (is_error_noslot_pfn(pfn))
1700                 return INTEL_GVT_INVALID_ADDR;
1701
1702         return pfn;
1703 }
1704
1705 static int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1706                 unsigned long size, dma_addr_t *dma_addr)
1707 {
1708         struct kvmgt_guest_info *info;
1709         struct intel_vgpu *vgpu;
1710         struct gvt_dma *entry;
1711         int ret;
1712
1713         if (!handle_valid(handle))
1714                 return -EINVAL;
1715
1716         info = (struct kvmgt_guest_info *)handle;
1717         vgpu = info->vgpu;
1718
1719         mutex_lock(&info->vgpu->vdev.cache_lock);
1720
1721         entry = __gvt_cache_find_gfn(info->vgpu, gfn);
1722         if (!entry) {
1723                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1724                 if (ret)
1725                         goto err_unlock;
1726
1727                 ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size);
1728                 if (ret)
1729                         goto err_unmap;
1730         } else {
1731                 kref_get(&entry->ref);
1732                 *dma_addr = entry->dma_addr;
1733         }
1734
1735         mutex_unlock(&info->vgpu->vdev.cache_lock);
1736         return 0;
1737
1738 err_unmap:
1739         gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1740 err_unlock:
1741         mutex_unlock(&info->vgpu->vdev.cache_lock);
1742         return ret;
1743 }
1744
1745 static void __gvt_dma_release(struct kref *ref)
1746 {
1747         struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1748
1749         gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1750                            entry->size);
1751         __gvt_cache_remove_entry(entry->vgpu, entry);
1752 }
1753
1754 static void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
1755 {
1756         struct kvmgt_guest_info *info;
1757         struct gvt_dma *entry;
1758
1759         if (!handle_valid(handle))
1760                 return;
1761
1762         info = (struct kvmgt_guest_info *)handle;
1763
1764         mutex_lock(&info->vgpu->vdev.cache_lock);
1765         entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
1766         if (entry)
1767                 kref_put(&entry->ref, __gvt_dma_release);
1768         mutex_unlock(&info->vgpu->vdev.cache_lock);
1769 }
1770
1771 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
1772                         void *buf, unsigned long len, bool write)
1773 {
1774         struct kvmgt_guest_info *info;
1775         struct kvm *kvm;
1776         int idx, ret;
1777         bool kthread = current->mm == NULL;
1778
1779         if (!handle_valid(handle))
1780                 return -ESRCH;
1781
1782         info = (struct kvmgt_guest_info *)handle;
1783         kvm = info->kvm;
1784
1785         if (kthread)
1786                 use_mm(kvm->mm);
1787
1788         idx = srcu_read_lock(&kvm->srcu);
1789         ret = write ? kvm_write_guest(kvm, gpa, buf, len) :
1790                       kvm_read_guest(kvm, gpa, buf, len);
1791         srcu_read_unlock(&kvm->srcu, idx);
1792
1793         if (kthread)
1794                 unuse_mm(kvm->mm);
1795
1796         return ret;
1797 }
1798
1799 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
1800                         void *buf, unsigned long len)
1801 {
1802         return kvmgt_rw_gpa(handle, gpa, buf, len, false);
1803 }
1804
1805 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
1806                         void *buf, unsigned long len)
1807 {
1808         return kvmgt_rw_gpa(handle, gpa, buf, len, true);
1809 }
1810
1811 static unsigned long kvmgt_virt_to_pfn(void *addr)
1812 {
1813         return PFN_DOWN(__pa(addr));
1814 }
1815
1816 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
1817 {
1818         struct kvmgt_guest_info *info;
1819         struct kvm *kvm;
1820
1821         if (!handle_valid(handle))
1822                 return false;
1823
1824         info = (struct kvmgt_guest_info *)handle;
1825         kvm = info->kvm;
1826
1827         return kvm_is_visible_gfn(kvm, gfn);
1828
1829 }
1830
1831 struct intel_gvt_mpt kvmgt_mpt = {
1832         .host_init = kvmgt_host_init,
1833         .host_exit = kvmgt_host_exit,
1834         .attach_vgpu = kvmgt_attach_vgpu,
1835         .detach_vgpu = kvmgt_detach_vgpu,
1836         .inject_msi = kvmgt_inject_msi,
1837         .from_virt_to_mfn = kvmgt_virt_to_pfn,
1838         .enable_page_track = kvmgt_page_track_add,
1839         .disable_page_track = kvmgt_page_track_remove,
1840         .read_gpa = kvmgt_read_gpa,
1841         .write_gpa = kvmgt_write_gpa,
1842         .gfn_to_mfn = kvmgt_gfn_to_pfn,
1843         .dma_map_guest_page = kvmgt_dma_map_guest_page,
1844         .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
1845         .set_opregion = kvmgt_set_opregion,
1846         .get_vfio_device = kvmgt_get_vfio_device,
1847         .put_vfio_device = kvmgt_put_vfio_device,
1848         .is_valid_gfn = kvmgt_is_valid_gfn,
1849 };
1850 EXPORT_SYMBOL_GPL(kvmgt_mpt);
1851
1852 static int __init kvmgt_init(void)
1853 {
1854         return 0;
1855 }
1856
1857 static void __exit kvmgt_exit(void)
1858 {
1859 }
1860
1861 module_init(kvmgt_init);
1862 module_exit(kvmgt_exit);
1863
1864 MODULE_LICENSE("GPL and additional rights");
1865 MODULE_AUTHOR("Intel Corporation");