786ade1843a2b69e3567538587388df4ce854c9d
[muen/linux.git] / virt / kvm / kvm_main.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * This module enables machines with Intel VT-x extensions to run virtual
5  * machines without emulation or binary translation.
6  *
7  * Copyright (C) 2006 Qumranet, Inc.
8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
9  *
10  * Authors:
11  *   Avi Kivity   <avi@qumranet.com>
12  *   Yaniv Kamay  <yaniv@qumranet.com>
13  *
14  * This work is licensed under the terms of the GNU GPL, version 2.  See
15  * the COPYING file in the top-level directory.
16  *
17  */
18
19 #include <kvm/iodev.h>
20
21 #include <linux/kvm_host.h>
22 #include <linux/kvm.h>
23 #include <linux/module.h>
24 #include <linux/errno.h>
25 #include <linux/percpu.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/syscore_ops.h>
34 #include <linux/cpu.h>
35 #include <linux/sched/signal.h>
36 #include <linux/sched/mm.h>
37 #include <linux/sched/stat.h>
38 #include <linux/cpumask.h>
39 #include <linux/smp.h>
40 #include <linux/anon_inodes.h>
41 #include <linux/profile.h>
42 #include <linux/kvm_para.h>
43 #include <linux/pagemap.h>
44 #include <linux/mman.h>
45 #include <linux/swap.h>
46 #include <linux/bitops.h>
47 #include <linux/spinlock.h>
48 #include <linux/compat.h>
49 #include <linux/srcu.h>
50 #include <linux/hugetlb.h>
51 #include <linux/slab.h>
52 #include <linux/sort.h>
53 #include <linux/bsearch.h>
54
55 #include <asm/processor.h>
56 #include <asm/io.h>
57 #include <asm/ioctl.h>
58 #include <linux/uaccess.h>
59 #include <asm/pgtable.h>
60
61 #include "coalesced_mmio.h"
62 #include "async_pf.h"
63 #include "vfio.h"
64
65 #define CREATE_TRACE_POINTS
66 #include <trace/events/kvm.h>
67
68 /* Worst case buffer size needed for holding an integer. */
69 #define ITOA_MAX_LEN 12
70
71 MODULE_AUTHOR("Qumranet");
72 MODULE_LICENSE("GPL");
73
74 /* Architectures should define their poll value according to the halt latency */
75 unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
76 module_param(halt_poll_ns, uint, 0644);
77 EXPORT_SYMBOL_GPL(halt_poll_ns);
78
79 /* Default doubles per-vcpu halt_poll_ns. */
80 unsigned int halt_poll_ns_grow = 2;
81 module_param(halt_poll_ns_grow, uint, 0644);
82 EXPORT_SYMBOL_GPL(halt_poll_ns_grow);
83
84 /* Default resets per-vcpu halt_poll_ns . */
85 unsigned int halt_poll_ns_shrink;
86 module_param(halt_poll_ns_shrink, uint, 0644);
87 EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);
88
89 /*
90  * Ordering of locks:
91  *
92  *      kvm->lock --> kvm->slots_lock --> kvm->irq_lock
93  */
94
95 DEFINE_SPINLOCK(kvm_lock);
96 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
97 LIST_HEAD(vm_list);
98
99 static cpumask_var_t cpus_hardware_enabled;
100 static int kvm_usage_count;
101 static atomic_t hardware_enable_failed;
102
103 struct kmem_cache *kvm_vcpu_cache;
104 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
105
106 static __read_mostly struct preempt_ops kvm_preempt_ops;
107
108 struct dentry *kvm_debugfs_dir;
109 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
110
111 static int kvm_debugfs_num_entries;
112 static const struct file_operations *stat_fops_per_vm[];
113
114 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
115                            unsigned long arg);
116 #ifdef CONFIG_KVM_COMPAT
117 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
118                                   unsigned long arg);
119 #define KVM_COMPAT(c)   .compat_ioctl   = (c)
120 #else
121 static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
122                                 unsigned long arg) { return -EINVAL; }
123 #define KVM_COMPAT(c)   .compat_ioctl   = kvm_no_compat_ioctl
124 #endif
125 static int hardware_enable_all(void);
126 static void hardware_disable_all(void);
127
128 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
129
130 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
131
132 __visible bool kvm_rebooting;
133 EXPORT_SYMBOL_GPL(kvm_rebooting);
134
135 static bool largepages_enabled = true;
136
137 #define KVM_EVENT_CREATE_VM 0
138 #define KVM_EVENT_DESTROY_VM 1
139 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
140 static unsigned long long kvm_createvm_count;
141 static unsigned long long kvm_active_vms;
142
143 __weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
144                 unsigned long start, unsigned long end, bool blockable)
145 {
146         return 0;
147 }
148
149 bool kvm_is_reserved_pfn(kvm_pfn_t pfn)
150 {
151         if (pfn_valid(pfn))
152                 return PageReserved(pfn_to_page(pfn));
153
154         return true;
155 }
156
157 /*
158  * Switches to specified vcpu, until a matching vcpu_put()
159  */
160 void vcpu_load(struct kvm_vcpu *vcpu)
161 {
162         int cpu = get_cpu();
163         preempt_notifier_register(&vcpu->preempt_notifier);
164         kvm_arch_vcpu_load(vcpu, cpu);
165         put_cpu();
166 }
167 EXPORT_SYMBOL_GPL(vcpu_load);
168
169 void vcpu_put(struct kvm_vcpu *vcpu)
170 {
171         preempt_disable();
172         kvm_arch_vcpu_put(vcpu);
173         preempt_notifier_unregister(&vcpu->preempt_notifier);
174         preempt_enable();
175 }
176 EXPORT_SYMBOL_GPL(vcpu_put);
177
178 /* TODO: merge with kvm_arch_vcpu_should_kick */
179 static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
180 {
181         int mode = kvm_vcpu_exiting_guest_mode(vcpu);
182
183         /*
184          * We need to wait for the VCPU to reenable interrupts and get out of
185          * READING_SHADOW_PAGE_TABLES mode.
186          */
187         if (req & KVM_REQUEST_WAIT)
188                 return mode != OUTSIDE_GUEST_MODE;
189
190         /*
191          * Need to kick a running VCPU, but otherwise there is nothing to do.
192          */
193         return mode == IN_GUEST_MODE;
194 }
195
196 static void ack_flush(void *_completed)
197 {
198 }
199
200 static inline bool kvm_kick_many_cpus(const struct cpumask *cpus, bool wait)
201 {
202         if (unlikely(!cpus))
203                 cpus = cpu_online_mask;
204
205         if (cpumask_empty(cpus))
206                 return false;
207
208         smp_call_function_many(cpus, ack_flush, NULL, wait);
209         return true;
210 }
211
212 bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
213                                  unsigned long *vcpu_bitmap, cpumask_var_t tmp)
214 {
215         int i, cpu, me;
216         struct kvm_vcpu *vcpu;
217         bool called;
218
219         me = get_cpu();
220
221         kvm_for_each_vcpu(i, vcpu, kvm) {
222                 if (vcpu_bitmap && !test_bit(i, vcpu_bitmap))
223                         continue;
224
225                 kvm_make_request(req, vcpu);
226                 cpu = vcpu->cpu;
227
228                 if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
229                         continue;
230
231                 if (tmp != NULL && cpu != -1 && cpu != me &&
232                     kvm_request_needs_ipi(vcpu, req))
233                         __cpumask_set_cpu(cpu, tmp);
234         }
235
236         called = kvm_kick_many_cpus(tmp, !!(req & KVM_REQUEST_WAIT));
237         put_cpu();
238
239         return called;
240 }
241
242 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
243 {
244         cpumask_var_t cpus;
245         bool called;
246
247         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
248
249         called = kvm_make_vcpus_request_mask(kvm, req, NULL, cpus);
250
251         free_cpumask_var(cpus);
252         return called;
253 }
254
255 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
256 void kvm_flush_remote_tlbs(struct kvm *kvm)
257 {
258         /*
259          * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
260          * kvm_make_all_cpus_request.
261          */
262         long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
263
264         /*
265          * We want to publish modifications to the page tables before reading
266          * mode. Pairs with a memory barrier in arch-specific code.
267          * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
268          * and smp_mb in walk_shadow_page_lockless_begin/end.
269          * - powerpc: smp_mb in kvmppc_prepare_to_enter.
270          *
271          * There is already an smp_mb__after_atomic() before
272          * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
273          * barrier here.
274          */
275         if (!kvm_arch_flush_remote_tlb(kvm)
276             || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
277                 ++kvm->stat.remote_tlb_flush;
278         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
279 }
280 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
281 #endif
282
283 void kvm_reload_remote_mmus(struct kvm *kvm)
284 {
285         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
286 }
287
288 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
289 {
290         struct page *page;
291         int r;
292
293         mutex_init(&vcpu->mutex);
294         vcpu->cpu = -1;
295         vcpu->kvm = kvm;
296         vcpu->vcpu_id = id;
297         vcpu->pid = NULL;
298         init_swait_queue_head(&vcpu->wq);
299         kvm_async_pf_vcpu_init(vcpu);
300
301         vcpu->pre_pcpu = -1;
302         INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
303
304         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
305         if (!page) {
306                 r = -ENOMEM;
307                 goto fail;
308         }
309         vcpu->run = page_address(page);
310
311         kvm_vcpu_set_in_spin_loop(vcpu, false);
312         kvm_vcpu_set_dy_eligible(vcpu, false);
313         vcpu->preempted = false;
314
315         r = kvm_arch_vcpu_init(vcpu);
316         if (r < 0)
317                 goto fail_free_run;
318         return 0;
319
320 fail_free_run:
321         free_page((unsigned long)vcpu->run);
322 fail:
323         return r;
324 }
325 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
326
327 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
328 {
329         /*
330          * no need for rcu_read_lock as VCPU_RUN is the only place that
331          * will change the vcpu->pid pointer and on uninit all file
332          * descriptors are already gone.
333          */
334         put_pid(rcu_dereference_protected(vcpu->pid, 1));
335         kvm_arch_vcpu_uninit(vcpu);
336         free_page((unsigned long)vcpu->run);
337 }
338 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
339
340 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
341 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
342 {
343         return container_of(mn, struct kvm, mmu_notifier);
344 }
345
346 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
347                                         struct mm_struct *mm,
348                                         unsigned long address,
349                                         pte_t pte)
350 {
351         struct kvm *kvm = mmu_notifier_to_kvm(mn);
352         int idx;
353
354         idx = srcu_read_lock(&kvm->srcu);
355         spin_lock(&kvm->mmu_lock);
356         kvm->mmu_notifier_seq++;
357         kvm_set_spte_hva(kvm, address, pte);
358         spin_unlock(&kvm->mmu_lock);
359         srcu_read_unlock(&kvm->srcu, idx);
360 }
361
362 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
363                                                     struct mm_struct *mm,
364                                                     unsigned long start,
365                                                     unsigned long end,
366                                                     bool blockable)
367 {
368         struct kvm *kvm = mmu_notifier_to_kvm(mn);
369         int need_tlb_flush = 0, idx;
370         int ret;
371
372         idx = srcu_read_lock(&kvm->srcu);
373         spin_lock(&kvm->mmu_lock);
374         /*
375          * The count increase must become visible at unlock time as no
376          * spte can be established without taking the mmu_lock and
377          * count is also read inside the mmu_lock critical section.
378          */
379         kvm->mmu_notifier_count++;
380         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
381         need_tlb_flush |= kvm->tlbs_dirty;
382         /* we've to flush the tlb before the pages can be freed */
383         if (need_tlb_flush)
384                 kvm_flush_remote_tlbs(kvm);
385
386         spin_unlock(&kvm->mmu_lock);
387
388         ret = kvm_arch_mmu_notifier_invalidate_range(kvm, start, end, blockable);
389
390         srcu_read_unlock(&kvm->srcu, idx);
391
392         return ret;
393 }
394
395 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
396                                                   struct mm_struct *mm,
397                                                   unsigned long start,
398                                                   unsigned long end)
399 {
400         struct kvm *kvm = mmu_notifier_to_kvm(mn);
401
402         spin_lock(&kvm->mmu_lock);
403         /*
404          * This sequence increase will notify the kvm page fault that
405          * the page that is going to be mapped in the spte could have
406          * been freed.
407          */
408         kvm->mmu_notifier_seq++;
409         smp_wmb();
410         /*
411          * The above sequence increase must be visible before the
412          * below count decrease, which is ensured by the smp_wmb above
413          * in conjunction with the smp_rmb in mmu_notifier_retry().
414          */
415         kvm->mmu_notifier_count--;
416         spin_unlock(&kvm->mmu_lock);
417
418         BUG_ON(kvm->mmu_notifier_count < 0);
419 }
420
421 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
422                                               struct mm_struct *mm,
423                                               unsigned long start,
424                                               unsigned long end)
425 {
426         struct kvm *kvm = mmu_notifier_to_kvm(mn);
427         int young, idx;
428
429         idx = srcu_read_lock(&kvm->srcu);
430         spin_lock(&kvm->mmu_lock);
431
432         young = kvm_age_hva(kvm, start, end);
433         if (young)
434                 kvm_flush_remote_tlbs(kvm);
435
436         spin_unlock(&kvm->mmu_lock);
437         srcu_read_unlock(&kvm->srcu, idx);
438
439         return young;
440 }
441
442 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
443                                         struct mm_struct *mm,
444                                         unsigned long start,
445                                         unsigned long end)
446 {
447         struct kvm *kvm = mmu_notifier_to_kvm(mn);
448         int young, idx;
449
450         idx = srcu_read_lock(&kvm->srcu);
451         spin_lock(&kvm->mmu_lock);
452         /*
453          * Even though we do not flush TLB, this will still adversely
454          * affect performance on pre-Haswell Intel EPT, where there is
455          * no EPT Access Bit to clear so that we have to tear down EPT
456          * tables instead. If we find this unacceptable, we can always
457          * add a parameter to kvm_age_hva so that it effectively doesn't
458          * do anything on clear_young.
459          *
460          * Also note that currently we never issue secondary TLB flushes
461          * from clear_young, leaving this job up to the regular system
462          * cadence. If we find this inaccurate, we might come up with a
463          * more sophisticated heuristic later.
464          */
465         young = kvm_age_hva(kvm, start, end);
466         spin_unlock(&kvm->mmu_lock);
467         srcu_read_unlock(&kvm->srcu, idx);
468
469         return young;
470 }
471
472 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
473                                        struct mm_struct *mm,
474                                        unsigned long address)
475 {
476         struct kvm *kvm = mmu_notifier_to_kvm(mn);
477         int young, idx;
478
479         idx = srcu_read_lock(&kvm->srcu);
480         spin_lock(&kvm->mmu_lock);
481         young = kvm_test_age_hva(kvm, address);
482         spin_unlock(&kvm->mmu_lock);
483         srcu_read_unlock(&kvm->srcu, idx);
484
485         return young;
486 }
487
488 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
489                                      struct mm_struct *mm)
490 {
491         struct kvm *kvm = mmu_notifier_to_kvm(mn);
492         int idx;
493
494         idx = srcu_read_lock(&kvm->srcu);
495         kvm_arch_flush_shadow_all(kvm);
496         srcu_read_unlock(&kvm->srcu, idx);
497 }
498
499 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
500         .flags                  = MMU_INVALIDATE_DOES_NOT_BLOCK,
501         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
502         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
503         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
504         .clear_young            = kvm_mmu_notifier_clear_young,
505         .test_young             = kvm_mmu_notifier_test_young,
506         .change_pte             = kvm_mmu_notifier_change_pte,
507         .release                = kvm_mmu_notifier_release,
508 };
509
510 static int kvm_init_mmu_notifier(struct kvm *kvm)
511 {
512         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
513         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
514 }
515
516 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
517
518 static int kvm_init_mmu_notifier(struct kvm *kvm)
519 {
520         return 0;
521 }
522
523 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
524
525 static struct kvm_memslots *kvm_alloc_memslots(void)
526 {
527         int i;
528         struct kvm_memslots *slots;
529
530         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
531         if (!slots)
532                 return NULL;
533
534         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
535                 slots->id_to_index[i] = slots->memslots[i].id = i;
536
537         return slots;
538 }
539
540 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
541 {
542         if (!memslot->dirty_bitmap)
543                 return;
544
545         kvfree(memslot->dirty_bitmap);
546         memslot->dirty_bitmap = NULL;
547 }
548
549 /*
550  * Free any memory in @free but not in @dont.
551  */
552 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
553                               struct kvm_memory_slot *dont)
554 {
555         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
556                 kvm_destroy_dirty_bitmap(free);
557
558         kvm_arch_free_memslot(kvm, free, dont);
559
560         free->npages = 0;
561 }
562
563 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
564 {
565         struct kvm_memory_slot *memslot;
566
567         if (!slots)
568                 return;
569
570         kvm_for_each_memslot(memslot, slots)
571                 kvm_free_memslot(kvm, memslot, NULL);
572
573         kvfree(slots);
574 }
575
576 static void kvm_destroy_vm_debugfs(struct kvm *kvm)
577 {
578         int i;
579
580         if (!kvm->debugfs_dentry)
581                 return;
582
583         debugfs_remove_recursive(kvm->debugfs_dentry);
584
585         if (kvm->debugfs_stat_data) {
586                 for (i = 0; i < kvm_debugfs_num_entries; i++)
587                         kfree(kvm->debugfs_stat_data[i]);
588                 kfree(kvm->debugfs_stat_data);
589         }
590 }
591
592 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
593 {
594         char dir_name[ITOA_MAX_LEN * 2];
595         struct kvm_stat_data *stat_data;
596         struct kvm_stats_debugfs_item *p;
597
598         if (!debugfs_initialized())
599                 return 0;
600
601         snprintf(dir_name, sizeof(dir_name), "%d-%d", task_pid_nr(current), fd);
602         kvm->debugfs_dentry = debugfs_create_dir(dir_name, kvm_debugfs_dir);
603
604         kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
605                                          sizeof(*kvm->debugfs_stat_data),
606                                          GFP_KERNEL);
607         if (!kvm->debugfs_stat_data)
608                 return -ENOMEM;
609
610         for (p = debugfs_entries; p->name; p++) {
611                 stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL);
612                 if (!stat_data)
613                         return -ENOMEM;
614
615                 stat_data->kvm = kvm;
616                 stat_data->offset = p->offset;
617                 kvm->debugfs_stat_data[p - debugfs_entries] = stat_data;
618                 debugfs_create_file(p->name, 0644, kvm->debugfs_dentry,
619                                     stat_data, stat_fops_per_vm[p->kind]);
620         }
621         return 0;
622 }
623
624 static struct kvm *kvm_create_vm(unsigned long type)
625 {
626         int r, i;
627         struct kvm *kvm = kvm_arch_alloc_vm();
628
629         if (!kvm)
630                 return ERR_PTR(-ENOMEM);
631
632         spin_lock_init(&kvm->mmu_lock);
633         mmgrab(current->mm);
634         kvm->mm = current->mm;
635         kvm_eventfd_init(kvm);
636         mutex_init(&kvm->lock);
637         mutex_init(&kvm->irq_lock);
638         mutex_init(&kvm->slots_lock);
639         refcount_set(&kvm->users_count, 1);
640         INIT_LIST_HEAD(&kvm->devices);
641
642         r = kvm_arch_init_vm(kvm, type);
643         if (r)
644                 goto out_err_no_disable;
645
646         r = hardware_enable_all();
647         if (r)
648                 goto out_err_no_disable;
649
650 #ifdef CONFIG_HAVE_KVM_IRQFD
651         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
652 #endif
653
654         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
655
656         r = -ENOMEM;
657         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
658                 struct kvm_memslots *slots = kvm_alloc_memslots();
659                 if (!slots)
660                         goto out_err_no_srcu;
661                 /*
662                  * Generations must be different for each address space.
663                  * Init kvm generation close to the maximum to easily test the
664                  * code of handling generation number wrap-around.
665                  */
666                 slots->generation = i * 2 - 150;
667                 rcu_assign_pointer(kvm->memslots[i], slots);
668         }
669
670         if (init_srcu_struct(&kvm->srcu))
671                 goto out_err_no_srcu;
672         if (init_srcu_struct(&kvm->irq_srcu))
673                 goto out_err_no_irq_srcu;
674         for (i = 0; i < KVM_NR_BUSES; i++) {
675                 rcu_assign_pointer(kvm->buses[i],
676                         kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL));
677                 if (!kvm->buses[i])
678                         goto out_err;
679         }
680
681         r = kvm_init_mmu_notifier(kvm);
682         if (r)
683                 goto out_err;
684
685         spin_lock(&kvm_lock);
686         list_add(&kvm->vm_list, &vm_list);
687         spin_unlock(&kvm_lock);
688
689         preempt_notifier_inc();
690
691         return kvm;
692
693 out_err:
694         cleanup_srcu_struct(&kvm->irq_srcu);
695 out_err_no_irq_srcu:
696         cleanup_srcu_struct(&kvm->srcu);
697 out_err_no_srcu:
698         hardware_disable_all();
699 out_err_no_disable:
700         refcount_set(&kvm->users_count, 0);
701         for (i = 0; i < KVM_NR_BUSES; i++)
702                 kfree(kvm_get_bus(kvm, i));
703         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
704                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
705         kvm_arch_free_vm(kvm);
706         mmdrop(current->mm);
707         return ERR_PTR(r);
708 }
709
710 static void kvm_destroy_devices(struct kvm *kvm)
711 {
712         struct kvm_device *dev, *tmp;
713
714         /*
715          * We do not need to take the kvm->lock here, because nobody else
716          * has a reference to the struct kvm at this point and therefore
717          * cannot access the devices list anyhow.
718          */
719         list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
720                 list_del(&dev->vm_node);
721                 dev->ops->destroy(dev);
722         }
723 }
724
725 static void kvm_destroy_vm(struct kvm *kvm)
726 {
727         int i;
728         struct mm_struct *mm = kvm->mm;
729
730         kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
731         kvm_destroy_vm_debugfs(kvm);
732         kvm_arch_sync_events(kvm);
733         spin_lock(&kvm_lock);
734         list_del(&kvm->vm_list);
735         spin_unlock(&kvm_lock);
736         kvm_free_irq_routing(kvm);
737         for (i = 0; i < KVM_NR_BUSES; i++) {
738                 struct kvm_io_bus *bus = kvm_get_bus(kvm, i);
739
740                 if (bus)
741                         kvm_io_bus_destroy(bus);
742                 kvm->buses[i] = NULL;
743         }
744         kvm_coalesced_mmio_free(kvm);
745 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
746         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
747 #else
748         kvm_arch_flush_shadow_all(kvm);
749 #endif
750         kvm_arch_destroy_vm(kvm);
751         kvm_destroy_devices(kvm);
752         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
753                 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
754         cleanup_srcu_struct(&kvm->irq_srcu);
755         cleanup_srcu_struct(&kvm->srcu);
756         kvm_arch_free_vm(kvm);
757         preempt_notifier_dec();
758         hardware_disable_all();
759         mmdrop(mm);
760 }
761
762 void kvm_get_kvm(struct kvm *kvm)
763 {
764         refcount_inc(&kvm->users_count);
765 }
766 EXPORT_SYMBOL_GPL(kvm_get_kvm);
767
768 void kvm_put_kvm(struct kvm *kvm)
769 {
770         if (refcount_dec_and_test(&kvm->users_count))
771                 kvm_destroy_vm(kvm);
772 }
773 EXPORT_SYMBOL_GPL(kvm_put_kvm);
774
775
776 static int kvm_vm_release(struct inode *inode, struct file *filp)
777 {
778         struct kvm *kvm = filp->private_data;
779
780         kvm_irqfd_release(kvm);
781
782         kvm_put_kvm(kvm);
783         return 0;
784 }
785
786 /*
787  * Allocation size is twice as large as the actual dirty bitmap size.
788  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
789  */
790 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
791 {
792         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
793
794         memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL);
795         if (!memslot->dirty_bitmap)
796                 return -ENOMEM;
797
798         return 0;
799 }
800
801 /*
802  * Insert memslot and re-sort memslots based on their GFN,
803  * so binary search could be used to lookup GFN.
804  * Sorting algorithm takes advantage of having initially
805  * sorted array and known changed memslot position.
806  */
807 static void update_memslots(struct kvm_memslots *slots,
808                             struct kvm_memory_slot *new,
809                             enum kvm_mr_change change)
810 {
811         int id = new->id;
812         int i = slots->id_to_index[id];
813         struct kvm_memory_slot *mslots = slots->memslots;
814
815         WARN_ON(mslots[i].id != id);
816         switch (change) {
817         case KVM_MR_CREATE:
818                 slots->used_slots++;
819                 WARN_ON(mslots[i].npages || !new->npages);
820                 break;
821         case KVM_MR_DELETE:
822                 slots->used_slots--;
823                 WARN_ON(new->npages || !mslots[i].npages);
824                 break;
825         default:
826                 break;
827         }
828
829         while (i < KVM_MEM_SLOTS_NUM - 1 &&
830                new->base_gfn <= mslots[i + 1].base_gfn) {
831                 if (!mslots[i + 1].npages)
832                         break;
833                 mslots[i] = mslots[i + 1];
834                 slots->id_to_index[mslots[i].id] = i;
835                 i++;
836         }
837
838         /*
839          * The ">=" is needed when creating a slot with base_gfn == 0,
840          * so that it moves before all those with base_gfn == npages == 0.
841          *
842          * On the other hand, if new->npages is zero, the above loop has
843          * already left i pointing to the beginning of the empty part of
844          * mslots, and the ">=" would move the hole backwards in this
845          * case---which is wrong.  So skip the loop when deleting a slot.
846          */
847         if (new->npages) {
848                 while (i > 0 &&
849                        new->base_gfn >= mslots[i - 1].base_gfn) {
850                         mslots[i] = mslots[i - 1];
851                         slots->id_to_index[mslots[i].id] = i;
852                         i--;
853                 }
854         } else
855                 WARN_ON_ONCE(i != slots->used_slots);
856
857         mslots[i] = *new;
858         slots->id_to_index[mslots[i].id] = i;
859 }
860
861 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
862 {
863         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
864
865 #ifdef __KVM_HAVE_READONLY_MEM
866         valid_flags |= KVM_MEM_READONLY;
867 #endif
868
869         if (mem->flags & ~valid_flags)
870                 return -EINVAL;
871
872         return 0;
873 }
874
875 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
876                 int as_id, struct kvm_memslots *slots)
877 {
878         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
879
880         /*
881          * Set the low bit in the generation, which disables SPTE caching
882          * until the end of synchronize_srcu_expedited.
883          */
884         WARN_ON(old_memslots->generation & 1);
885         slots->generation = old_memslots->generation + 1;
886
887         rcu_assign_pointer(kvm->memslots[as_id], slots);
888         synchronize_srcu_expedited(&kvm->srcu);
889
890         /*
891          * Increment the new memslot generation a second time. This prevents
892          * vm exits that race with memslot updates from caching a memslot
893          * generation that will (potentially) be valid forever.
894          *
895          * Generations must be unique even across address spaces.  We do not need
896          * a global counter for that, instead the generation space is evenly split
897          * across address spaces.  For example, with two address spaces, address
898          * space 0 will use generations 0, 4, 8, ... while * address space 1 will
899          * use generations 2, 6, 10, 14, ...
900          */
901         slots->generation += KVM_ADDRESS_SPACE_NUM * 2 - 1;
902
903         kvm_arch_memslots_updated(kvm, slots);
904
905         return old_memslots;
906 }
907
908 /*
909  * Allocate some memory and give it an address in the guest physical address
910  * space.
911  *
912  * Discontiguous memory is allowed, mostly for framebuffers.
913  *
914  * Must be called holding kvm->slots_lock for write.
915  */
916 int __kvm_set_memory_region(struct kvm *kvm,
917                             const struct kvm_userspace_memory_region *mem)
918 {
919         int r;
920         gfn_t base_gfn;
921         unsigned long npages;
922         struct kvm_memory_slot *slot;
923         struct kvm_memory_slot old, new;
924         struct kvm_memslots *slots = NULL, *old_memslots;
925         int as_id, id;
926         enum kvm_mr_change change;
927
928         r = check_memory_region_flags(mem);
929         if (r)
930                 goto out;
931
932         r = -EINVAL;
933         as_id = mem->slot >> 16;
934         id = (u16)mem->slot;
935
936         /* General sanity checks */
937         if (mem->memory_size & (PAGE_SIZE - 1))
938                 goto out;
939         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
940                 goto out;
941         /* We can read the guest memory with __xxx_user() later on. */
942         if ((id < KVM_USER_MEM_SLOTS) &&
943             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
944              !access_ok(VERIFY_WRITE,
945                         (void __user *)(unsigned long)mem->userspace_addr,
946                         mem->memory_size)))
947                 goto out;
948         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
949                 goto out;
950         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
951                 goto out;
952
953         slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
954         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
955         npages = mem->memory_size >> PAGE_SHIFT;
956
957         if (npages > KVM_MEM_MAX_NR_PAGES)
958                 goto out;
959
960         new = old = *slot;
961
962         new.id = id;
963         new.base_gfn = base_gfn;
964         new.npages = npages;
965         new.flags = mem->flags;
966
967         if (npages) {
968                 if (!old.npages)
969                         change = KVM_MR_CREATE;
970                 else { /* Modify an existing slot. */
971                         if ((mem->userspace_addr != old.userspace_addr) ||
972                             (npages != old.npages) ||
973                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
974                                 goto out;
975
976                         if (base_gfn != old.base_gfn)
977                                 change = KVM_MR_MOVE;
978                         else if (new.flags != old.flags)
979                                 change = KVM_MR_FLAGS_ONLY;
980                         else { /* Nothing to change. */
981                                 r = 0;
982                                 goto out;
983                         }
984                 }
985         } else {
986                 if (!old.npages)
987                         goto out;
988
989                 change = KVM_MR_DELETE;
990                 new.base_gfn = 0;
991                 new.flags = 0;
992         }
993
994         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
995                 /* Check for overlaps */
996                 r = -EEXIST;
997                 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
998                         if (slot->id == id)
999                                 continue;
1000                         if (!((base_gfn + npages <= slot->base_gfn) ||
1001                               (base_gfn >= slot->base_gfn + slot->npages)))
1002                                 goto out;
1003                 }
1004         }
1005
1006         /* Free page dirty bitmap if unneeded */
1007         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
1008                 new.dirty_bitmap = NULL;
1009
1010         r = -ENOMEM;
1011         if (change == KVM_MR_CREATE) {
1012                 new.userspace_addr = mem->userspace_addr;
1013
1014                 if (kvm_arch_create_memslot(kvm, &new, npages))
1015                         goto out_free;
1016         }
1017
1018         /* Allocate page dirty bitmap if needed */
1019         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
1020                 if (kvm_create_dirty_bitmap(&new) < 0)
1021                         goto out_free;
1022         }
1023
1024         slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
1025         if (!slots)
1026                 goto out_free;
1027         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
1028
1029         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
1030                 slot = id_to_memslot(slots, id);
1031                 slot->flags |= KVM_MEMSLOT_INVALID;
1032
1033                 old_memslots = install_new_memslots(kvm, as_id, slots);
1034
1035                 /* From this point no new shadow pages pointing to a deleted,
1036                  * or moved, memslot will be created.
1037                  *
1038                  * validation of sp->gfn happens in:
1039                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1040                  *      - kvm_is_visible_gfn (mmu_check_roots)
1041                  */
1042                 kvm_arch_flush_shadow_memslot(kvm, slot);
1043
1044                 /*
1045                  * We can re-use the old_memslots from above, the only difference
1046                  * from the currently installed memslots is the invalid flag.  This
1047                  * will get overwritten by update_memslots anyway.
1048                  */
1049                 slots = old_memslots;
1050         }
1051
1052         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
1053         if (r)
1054                 goto out_slots;
1055
1056         /* actual memory is freed via old in kvm_free_memslot below */
1057         if (change == KVM_MR_DELETE) {
1058                 new.dirty_bitmap = NULL;
1059                 memset(&new.arch, 0, sizeof(new.arch));
1060         }
1061
1062         update_memslots(slots, &new, change);
1063         old_memslots = install_new_memslots(kvm, as_id, slots);
1064
1065         kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
1066
1067         kvm_free_memslot(kvm, &old, &new);
1068         kvfree(old_memslots);
1069         return 0;
1070
1071 out_slots:
1072         kvfree(slots);
1073 out_free:
1074         kvm_free_memslot(kvm, &new, &old);
1075 out:
1076         return r;
1077 }
1078 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1079
1080 int kvm_set_memory_region(struct kvm *kvm,
1081                           const struct kvm_userspace_memory_region *mem)
1082 {
1083         int r;
1084
1085         mutex_lock(&kvm->slots_lock);
1086         r = __kvm_set_memory_region(kvm, mem);
1087         mutex_unlock(&kvm->slots_lock);
1088         return r;
1089 }
1090 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1091
1092 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1093                                           struct kvm_userspace_memory_region *mem)
1094 {
1095         if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1096                 return -EINVAL;
1097
1098         return kvm_set_memory_region(kvm, mem);
1099 }
1100
1101 int kvm_get_dirty_log(struct kvm *kvm,
1102                         struct kvm_dirty_log *log, int *is_dirty)
1103 {
1104         struct kvm_memslots *slots;
1105         struct kvm_memory_slot *memslot;
1106         int i, as_id, id;
1107         unsigned long n;
1108         unsigned long any = 0;
1109
1110         as_id = log->slot >> 16;
1111         id = (u16)log->slot;
1112         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1113                 return -EINVAL;
1114
1115         slots = __kvm_memslots(kvm, as_id);
1116         memslot = id_to_memslot(slots, id);
1117         if (!memslot->dirty_bitmap)
1118                 return -ENOENT;
1119
1120         n = kvm_dirty_bitmap_bytes(memslot);
1121
1122         for (i = 0; !any && i < n/sizeof(long); ++i)
1123                 any = memslot->dirty_bitmap[i];
1124
1125         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1126                 return -EFAULT;
1127
1128         if (any)
1129                 *is_dirty = 1;
1130         return 0;
1131 }
1132 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1133
1134 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1135 /**
1136  * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1137  *      are dirty write protect them for next write.
1138  * @kvm:        pointer to kvm instance
1139  * @log:        slot id and address to which we copy the log
1140  * @is_dirty:   flag set if any page is dirty
1141  *
1142  * We need to keep it in mind that VCPU threads can write to the bitmap
1143  * concurrently. So, to avoid losing track of dirty pages we keep the
1144  * following order:
1145  *
1146  *    1. Take a snapshot of the bit and clear it if needed.
1147  *    2. Write protect the corresponding page.
1148  *    3. Copy the snapshot to the userspace.
1149  *    4. Upon return caller flushes TLB's if needed.
1150  *
1151  * Between 2 and 4, the guest may write to the page using the remaining TLB
1152  * entry.  This is not a problem because the page is reported dirty using
1153  * the snapshot taken before and step 4 ensures that writes done after
1154  * exiting to userspace will be logged for the next call.
1155  *
1156  */
1157 int kvm_get_dirty_log_protect(struct kvm *kvm,
1158                         struct kvm_dirty_log *log, bool *is_dirty)
1159 {
1160         struct kvm_memslots *slots;
1161         struct kvm_memory_slot *memslot;
1162         int i, as_id, id;
1163         unsigned long n;
1164         unsigned long *dirty_bitmap;
1165         unsigned long *dirty_bitmap_buffer;
1166
1167         as_id = log->slot >> 16;
1168         id = (u16)log->slot;
1169         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1170                 return -EINVAL;
1171
1172         slots = __kvm_memslots(kvm, as_id);
1173         memslot = id_to_memslot(slots, id);
1174
1175         dirty_bitmap = memslot->dirty_bitmap;
1176         if (!dirty_bitmap)
1177                 return -ENOENT;
1178
1179         n = kvm_dirty_bitmap_bytes(memslot);
1180
1181         dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1182         memset(dirty_bitmap_buffer, 0, n);
1183
1184         spin_lock(&kvm->mmu_lock);
1185         *is_dirty = false;
1186         for (i = 0; i < n / sizeof(long); i++) {
1187                 unsigned long mask;
1188                 gfn_t offset;
1189
1190                 if (!dirty_bitmap[i])
1191                         continue;
1192
1193                 *is_dirty = true;
1194
1195                 mask = xchg(&dirty_bitmap[i], 0);
1196                 dirty_bitmap_buffer[i] = mask;
1197
1198                 if (mask) {
1199                         offset = i * BITS_PER_LONG;
1200                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1201                                                                 offset, mask);
1202                 }
1203         }
1204
1205         spin_unlock(&kvm->mmu_lock);
1206         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1207                 return -EFAULT;
1208         return 0;
1209 }
1210 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1211 #endif
1212
1213 bool kvm_largepages_enabled(void)
1214 {
1215         return largepages_enabled;
1216 }
1217
1218 void kvm_disable_largepages(void)
1219 {
1220         largepages_enabled = false;
1221 }
1222 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1223
1224 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1225 {
1226         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1227 }
1228 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1229
1230 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1231 {
1232         return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1233 }
1234
1235 bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1236 {
1237         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1238
1239         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1240               memslot->flags & KVM_MEMSLOT_INVALID)
1241                 return false;
1242
1243         return true;
1244 }
1245 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1246
1247 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1248 {
1249         struct vm_area_struct *vma;
1250         unsigned long addr, size;
1251
1252         size = PAGE_SIZE;
1253
1254         addr = gfn_to_hva(kvm, gfn);
1255         if (kvm_is_error_hva(addr))
1256                 return PAGE_SIZE;
1257
1258         down_read(&current->mm->mmap_sem);
1259         vma = find_vma(current->mm, addr);
1260         if (!vma)
1261                 goto out;
1262
1263         size = vma_kernel_pagesize(vma);
1264
1265 out:
1266         up_read(&current->mm->mmap_sem);
1267
1268         return size;
1269 }
1270
1271 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1272 {
1273         return slot->flags & KVM_MEM_READONLY;
1274 }
1275
1276 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1277                                        gfn_t *nr_pages, bool write)
1278 {
1279         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1280                 return KVM_HVA_ERR_BAD;
1281
1282         if (memslot_is_readonly(slot) && write)
1283                 return KVM_HVA_ERR_RO_BAD;
1284
1285         if (nr_pages)
1286                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1287
1288         return __gfn_to_hva_memslot(slot, gfn);
1289 }
1290
1291 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1292                                      gfn_t *nr_pages)
1293 {
1294         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1295 }
1296
1297 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1298                                         gfn_t gfn)
1299 {
1300         return gfn_to_hva_many(slot, gfn, NULL);
1301 }
1302 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1303
1304 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1305 {
1306         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1307 }
1308 EXPORT_SYMBOL_GPL(gfn_to_hva);
1309
1310 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1311 {
1312         return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1313 }
1314 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1315
1316 /*
1317  * Return the hva of a @gfn and the R/W attribute if possible.
1318  *
1319  * @slot: the kvm_memory_slot which contains @gfn
1320  * @gfn: the gfn to be translated
1321  * @writable: used to return the read/write attribute of the @slot if the hva
1322  * is valid and @writable is not NULL
1323  */
1324 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1325                                       gfn_t gfn, bool *writable)
1326 {
1327         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1328
1329         if (!kvm_is_error_hva(hva) && writable)
1330                 *writable = !memslot_is_readonly(slot);
1331
1332         return hva;
1333 }
1334
1335 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1336 {
1337         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1338
1339         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1340 }
1341
1342 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1343 {
1344         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1345
1346         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1347 }
1348
1349 static inline int check_user_page_hwpoison(unsigned long addr)
1350 {
1351         int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
1352
1353         rc = get_user_pages(addr, 1, flags, NULL, NULL);
1354         return rc == -EHWPOISON;
1355 }
1356
1357 /*
1358  * The fast path to get the writable pfn which will be stored in @pfn,
1359  * true indicates success, otherwise false is returned.  It's also the
1360  * only part that runs if we can are in atomic context.
1361  */
1362 static bool hva_to_pfn_fast(unsigned long addr, bool write_fault,
1363                             bool *writable, kvm_pfn_t *pfn)
1364 {
1365         struct page *page[1];
1366         int npages;
1367
1368         /*
1369          * Fast pin a writable pfn only if it is a write fault request
1370          * or the caller allows to map a writable pfn for a read fault
1371          * request.
1372          */
1373         if (!(write_fault || writable))
1374                 return false;
1375
1376         npages = __get_user_pages_fast(addr, 1, 1, page);
1377         if (npages == 1) {
1378                 *pfn = page_to_pfn(page[0]);
1379
1380                 if (writable)
1381                         *writable = true;
1382                 return true;
1383         }
1384
1385         return false;
1386 }
1387
1388 /*
1389  * The slow path to get the pfn of the specified host virtual address,
1390  * 1 indicates success, -errno is returned if error is detected.
1391  */
1392 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1393                            bool *writable, kvm_pfn_t *pfn)
1394 {
1395         unsigned int flags = FOLL_HWPOISON;
1396         struct page *page;
1397         int npages = 0;
1398
1399         might_sleep();
1400
1401         if (writable)
1402                 *writable = write_fault;
1403
1404         if (write_fault)
1405                 flags |= FOLL_WRITE;
1406         if (async)
1407                 flags |= FOLL_NOWAIT;
1408
1409         npages = get_user_pages_unlocked(addr, 1, &page, flags);
1410         if (npages != 1)
1411                 return npages;
1412
1413         /* map read fault as writable if possible */
1414         if (unlikely(!write_fault) && writable) {
1415                 struct page *wpage;
1416
1417                 if (__get_user_pages_fast(addr, 1, 1, &wpage) == 1) {
1418                         *writable = true;
1419                         put_page(page);
1420                         page = wpage;
1421                 }
1422         }
1423         *pfn = page_to_pfn(page);
1424         return npages;
1425 }
1426
1427 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1428 {
1429         if (unlikely(!(vma->vm_flags & VM_READ)))
1430                 return false;
1431
1432         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1433                 return false;
1434
1435         return true;
1436 }
1437
1438 static int hva_to_pfn_remapped(struct vm_area_struct *vma,
1439                                unsigned long addr, bool *async,
1440                                bool write_fault, bool *writable,
1441                                kvm_pfn_t *p_pfn)
1442 {
1443         unsigned long pfn;
1444         int r;
1445
1446         r = follow_pfn(vma, addr, &pfn);
1447         if (r) {
1448                 /*
1449                  * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
1450                  * not call the fault handler, so do it here.
1451                  */
1452                 bool unlocked = false;
1453                 r = fixup_user_fault(current, current->mm, addr,
1454                                      (write_fault ? FAULT_FLAG_WRITE : 0),
1455                                      &unlocked);
1456                 if (unlocked)
1457                         return -EAGAIN;
1458                 if (r)
1459                         return r;
1460
1461                 r = follow_pfn(vma, addr, &pfn);
1462                 if (r)
1463                         return r;
1464
1465         }
1466
1467         if (writable)
1468                 *writable = true;
1469
1470         /*
1471          * Get a reference here because callers of *hva_to_pfn* and
1472          * *gfn_to_pfn* ultimately call kvm_release_pfn_clean on the
1473          * returned pfn.  This is only needed if the VMA has VM_MIXEDMAP
1474          * set, but the kvm_get_pfn/kvm_release_pfn_clean pair will
1475          * simply do nothing for reserved pfns.
1476          *
1477          * Whoever called remap_pfn_range is also going to call e.g.
1478          * unmap_mapping_range before the underlying pages are freed,
1479          * causing a call to our MMU notifier.
1480          */ 
1481         kvm_get_pfn(pfn);
1482
1483         *p_pfn = pfn;
1484         return 0;
1485 }
1486
1487 /*
1488  * Pin guest page in memory and return its pfn.
1489  * @addr: host virtual address which maps memory to the guest
1490  * @atomic: whether this function can sleep
1491  * @async: whether this function need to wait IO complete if the
1492  *         host page is not in the memory
1493  * @write_fault: whether we should get a writable host page
1494  * @writable: whether it allows to map a writable host page for !@write_fault
1495  *
1496  * The function will map a writable host page for these two cases:
1497  * 1): @write_fault = true
1498  * 2): @write_fault = false && @writable, @writable will tell the caller
1499  *     whether the mapping is writable.
1500  */
1501 static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1502                         bool write_fault, bool *writable)
1503 {
1504         struct vm_area_struct *vma;
1505         kvm_pfn_t pfn = 0;
1506         int npages, r;
1507
1508         /* we can do it either atomically or asynchronously, not both */
1509         BUG_ON(atomic && async);
1510
1511         if (hva_to_pfn_fast(addr, write_fault, writable, &pfn))
1512                 return pfn;
1513
1514         if (atomic)
1515                 return KVM_PFN_ERR_FAULT;
1516
1517         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1518         if (npages == 1)
1519                 return pfn;
1520
1521         down_read(&current->mm->mmap_sem);
1522         if (npages == -EHWPOISON ||
1523               (!async && check_user_page_hwpoison(addr))) {
1524                 pfn = KVM_PFN_ERR_HWPOISON;
1525                 goto exit;
1526         }
1527
1528 retry:
1529         vma = find_vma_intersection(current->mm, addr, addr + 1);
1530
1531         if (vma == NULL)
1532                 pfn = KVM_PFN_ERR_FAULT;
1533         else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1534                 r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
1535                 if (r == -EAGAIN)
1536                         goto retry;
1537                 if (r < 0)
1538                         pfn = KVM_PFN_ERR_FAULT;
1539         } else {
1540                 if (async && vma_is_valid(vma, write_fault))
1541                         *async = true;
1542                 pfn = KVM_PFN_ERR_FAULT;
1543         }
1544 exit:
1545         up_read(&current->mm->mmap_sem);
1546         return pfn;
1547 }
1548
1549 kvm_pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn,
1550                                bool atomic, bool *async, bool write_fault,
1551                                bool *writable)
1552 {
1553         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1554
1555         if (addr == KVM_HVA_ERR_RO_BAD) {
1556                 if (writable)
1557                         *writable = false;
1558                 return KVM_PFN_ERR_RO_FAULT;
1559         }
1560
1561         if (kvm_is_error_hva(addr)) {
1562                 if (writable)
1563                         *writable = false;
1564                 return KVM_PFN_NOSLOT;
1565         }
1566
1567         /* Do not map writable pfn in the readonly memslot. */
1568         if (writable && memslot_is_readonly(slot)) {
1569                 *writable = false;
1570                 writable = NULL;
1571         }
1572
1573         return hva_to_pfn(addr, atomic, async, write_fault,
1574                           writable);
1575 }
1576 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1577
1578 kvm_pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1579                       bool *writable)
1580 {
1581         return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1582                                     write_fault, writable);
1583 }
1584 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1585
1586 kvm_pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1587 {
1588         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1589 }
1590 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1591
1592 kvm_pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1593 {
1594         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1595 }
1596 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1597
1598 kvm_pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1599 {
1600         return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1601 }
1602 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1603
1604 kvm_pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1605 {
1606         return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1607 }
1608 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1609
1610 kvm_pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1611 {
1612         return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1613 }
1614 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1615
1616 kvm_pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1617 {
1618         return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1619 }
1620 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1621
1622 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1623                             struct page **pages, int nr_pages)
1624 {
1625         unsigned long addr;
1626         gfn_t entry = 0;
1627
1628         addr = gfn_to_hva_many(slot, gfn, &entry);
1629         if (kvm_is_error_hva(addr))
1630                 return -1;
1631
1632         if (entry < nr_pages)
1633                 return 0;
1634
1635         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1636 }
1637 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1638
1639 static struct page *kvm_pfn_to_page(kvm_pfn_t pfn)
1640 {
1641         if (is_error_noslot_pfn(pfn))
1642                 return KVM_ERR_PTR_BAD_PAGE;
1643
1644         if (kvm_is_reserved_pfn(pfn)) {
1645                 WARN_ON(1);
1646                 return KVM_ERR_PTR_BAD_PAGE;
1647         }
1648
1649         return pfn_to_page(pfn);
1650 }
1651
1652 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1653 {
1654         kvm_pfn_t pfn;
1655
1656         pfn = gfn_to_pfn(kvm, gfn);
1657
1658         return kvm_pfn_to_page(pfn);
1659 }
1660 EXPORT_SYMBOL_GPL(gfn_to_page);
1661
1662 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1663 {
1664         kvm_pfn_t pfn;
1665
1666         pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1667
1668         return kvm_pfn_to_page(pfn);
1669 }
1670 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1671
1672 void kvm_release_page_clean(struct page *page)
1673 {
1674         WARN_ON(is_error_page(page));
1675
1676         kvm_release_pfn_clean(page_to_pfn(page));
1677 }
1678 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1679
1680 void kvm_release_pfn_clean(kvm_pfn_t pfn)
1681 {
1682         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1683                 put_page(pfn_to_page(pfn));
1684 }
1685 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1686
1687 void kvm_release_page_dirty(struct page *page)
1688 {
1689         WARN_ON(is_error_page(page));
1690
1691         kvm_release_pfn_dirty(page_to_pfn(page));
1692 }
1693 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1694
1695 void kvm_release_pfn_dirty(kvm_pfn_t pfn)
1696 {
1697         kvm_set_pfn_dirty(pfn);
1698         kvm_release_pfn_clean(pfn);
1699 }
1700 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
1701
1702 void kvm_set_pfn_dirty(kvm_pfn_t pfn)
1703 {
1704         if (!kvm_is_reserved_pfn(pfn)) {
1705                 struct page *page = pfn_to_page(pfn);
1706
1707                 if (!PageReserved(page))
1708                         SetPageDirty(page);
1709         }
1710 }
1711 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1712
1713 void kvm_set_pfn_accessed(kvm_pfn_t pfn)
1714 {
1715         if (!kvm_is_reserved_pfn(pfn))
1716                 mark_page_accessed(pfn_to_page(pfn));
1717 }
1718 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1719
1720 void kvm_get_pfn(kvm_pfn_t pfn)
1721 {
1722         if (!kvm_is_reserved_pfn(pfn))
1723                 get_page(pfn_to_page(pfn));
1724 }
1725 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1726
1727 static int next_segment(unsigned long len, int offset)
1728 {
1729         if (len > PAGE_SIZE - offset)
1730                 return PAGE_SIZE - offset;
1731         else
1732                 return len;
1733 }
1734
1735 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1736                                  void *data, int offset, int len)
1737 {
1738         int r;
1739         unsigned long addr;
1740
1741         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1742         if (kvm_is_error_hva(addr))
1743                 return -EFAULT;
1744         r = __copy_from_user(data, (void __user *)addr + offset, len);
1745         if (r)
1746                 return -EFAULT;
1747         return 0;
1748 }
1749
1750 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1751                         int len)
1752 {
1753         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1754
1755         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1756 }
1757 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1758
1759 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1760                              int offset, int len)
1761 {
1762         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1763
1764         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1765 }
1766 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1767
1768 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1769 {
1770         gfn_t gfn = gpa >> PAGE_SHIFT;
1771         int seg;
1772         int offset = offset_in_page(gpa);
1773         int ret;
1774
1775         while ((seg = next_segment(len, offset)) != 0) {
1776                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1777                 if (ret < 0)
1778                         return ret;
1779                 offset = 0;
1780                 len -= seg;
1781                 data += seg;
1782                 ++gfn;
1783         }
1784         return 0;
1785 }
1786 EXPORT_SYMBOL_GPL(kvm_read_guest);
1787
1788 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1789 {
1790         gfn_t gfn = gpa >> PAGE_SHIFT;
1791         int seg;
1792         int offset = offset_in_page(gpa);
1793         int ret;
1794
1795         while ((seg = next_segment(len, offset)) != 0) {
1796                 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1797                 if (ret < 0)
1798                         return ret;
1799                 offset = 0;
1800                 len -= seg;
1801                 data += seg;
1802                 ++gfn;
1803         }
1804         return 0;
1805 }
1806 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1807
1808 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1809                                    void *data, int offset, unsigned long len)
1810 {
1811         int r;
1812         unsigned long addr;
1813
1814         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1815         if (kvm_is_error_hva(addr))
1816                 return -EFAULT;
1817         pagefault_disable();
1818         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1819         pagefault_enable();
1820         if (r)
1821                 return -EFAULT;
1822         return 0;
1823 }
1824
1825 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1826                           unsigned long len)
1827 {
1828         gfn_t gfn = gpa >> PAGE_SHIFT;
1829         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1830         int offset = offset_in_page(gpa);
1831
1832         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1833 }
1834 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1835
1836 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1837                                void *data, unsigned long len)
1838 {
1839         gfn_t gfn = gpa >> PAGE_SHIFT;
1840         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1841         int offset = offset_in_page(gpa);
1842
1843         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1844 }
1845 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1846
1847 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1848                                   const void *data, int offset, int len)
1849 {
1850         int r;
1851         unsigned long addr;
1852
1853         addr = gfn_to_hva_memslot(memslot, gfn);
1854         if (kvm_is_error_hva(addr))
1855                 return -EFAULT;
1856         r = __copy_to_user((void __user *)addr + offset, data, len);
1857         if (r)
1858                 return -EFAULT;
1859         mark_page_dirty_in_slot(memslot, gfn);
1860         return 0;
1861 }
1862
1863 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1864                          const void *data, int offset, int len)
1865 {
1866         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1867
1868         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1869 }
1870 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1871
1872 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1873                               const void *data, int offset, int len)
1874 {
1875         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1876
1877         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1878 }
1879 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1880
1881 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1882                     unsigned long len)
1883 {
1884         gfn_t gfn = gpa >> PAGE_SHIFT;
1885         int seg;
1886         int offset = offset_in_page(gpa);
1887         int ret;
1888
1889         while ((seg = next_segment(len, offset)) != 0) {
1890                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1891                 if (ret < 0)
1892                         return ret;
1893                 offset = 0;
1894                 len -= seg;
1895                 data += seg;
1896                 ++gfn;
1897         }
1898         return 0;
1899 }
1900 EXPORT_SYMBOL_GPL(kvm_write_guest);
1901
1902 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1903                          unsigned long len)
1904 {
1905         gfn_t gfn = gpa >> PAGE_SHIFT;
1906         int seg;
1907         int offset = offset_in_page(gpa);
1908         int ret;
1909
1910         while ((seg = next_segment(len, offset)) != 0) {
1911                 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1912                 if (ret < 0)
1913                         return ret;
1914                 offset = 0;
1915                 len -= seg;
1916                 data += seg;
1917                 ++gfn;
1918         }
1919         return 0;
1920 }
1921 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1922
1923 static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
1924                                        struct gfn_to_hva_cache *ghc,
1925                                        gpa_t gpa, unsigned long len)
1926 {
1927         int offset = offset_in_page(gpa);
1928         gfn_t start_gfn = gpa >> PAGE_SHIFT;
1929         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1930         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1931         gfn_t nr_pages_avail;
1932
1933         ghc->gpa = gpa;
1934         ghc->generation = slots->generation;
1935         ghc->len = len;
1936         ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1937         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1938         if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1939                 ghc->hva += offset;
1940         } else {
1941                 /*
1942                  * If the requested region crosses two memslots, we still
1943                  * verify that the entire region is valid here.
1944                  */
1945                 while (start_gfn <= end_gfn) {
1946                         nr_pages_avail = 0;
1947                         ghc->memslot = __gfn_to_memslot(slots, start_gfn);
1948                         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1949                                                    &nr_pages_avail);
1950                         if (kvm_is_error_hva(ghc->hva))
1951                                 return -EFAULT;
1952                         start_gfn += nr_pages_avail;
1953                 }
1954                 /* Use the slow path for cross page reads and writes. */
1955                 ghc->memslot = NULL;
1956         }
1957         return 0;
1958 }
1959
1960 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1961                               gpa_t gpa, unsigned long len)
1962 {
1963         struct kvm_memslots *slots = kvm_memslots(kvm);
1964         return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
1965 }
1966 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1967
1968 int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1969                            void *data, int offset, unsigned long len)
1970 {
1971         struct kvm_memslots *slots = kvm_memslots(kvm);
1972         int r;
1973         gpa_t gpa = ghc->gpa + offset;
1974
1975         BUG_ON(len + offset > ghc->len);
1976
1977         if (slots->generation != ghc->generation)
1978                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
1979
1980         if (unlikely(!ghc->memslot))
1981                 return kvm_write_guest(kvm, gpa, data, len);
1982
1983         if (kvm_is_error_hva(ghc->hva))
1984                 return -EFAULT;
1985
1986         r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
1987         if (r)
1988                 return -EFAULT;
1989         mark_page_dirty_in_slot(ghc->memslot, gpa >> PAGE_SHIFT);
1990
1991         return 0;
1992 }
1993 EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);
1994
1995 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1996                            void *data, unsigned long len)
1997 {
1998         return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
1999 }
2000 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
2001
2002 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
2003                            void *data, unsigned long len)
2004 {
2005         struct kvm_memslots *slots = kvm_memslots(kvm);
2006         int r;
2007
2008         BUG_ON(len > ghc->len);
2009
2010         if (slots->generation != ghc->generation)
2011                 __kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len);
2012
2013         if (unlikely(!ghc->memslot))
2014                 return kvm_read_guest(kvm, ghc->gpa, data, len);
2015
2016         if (kvm_is_error_hva(ghc->hva))
2017                 return -EFAULT;
2018
2019         r = __copy_from_user(data, (void __user *)ghc->hva, len);
2020         if (r)
2021                 return -EFAULT;
2022
2023         return 0;
2024 }
2025 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
2026
2027 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
2028 {
2029         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
2030
2031         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
2032 }
2033 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
2034
2035 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
2036 {
2037         gfn_t gfn = gpa >> PAGE_SHIFT;
2038         int seg;
2039         int offset = offset_in_page(gpa);
2040         int ret;
2041
2042         while ((seg = next_segment(len, offset)) != 0) {
2043                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
2044                 if (ret < 0)
2045                         return ret;
2046                 offset = 0;
2047                 len -= seg;
2048                 ++gfn;
2049         }
2050         return 0;
2051 }
2052 EXPORT_SYMBOL_GPL(kvm_clear_guest);
2053
2054 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
2055                                     gfn_t gfn)
2056 {
2057         if (memslot && memslot->dirty_bitmap) {
2058                 unsigned long rel_gfn = gfn - memslot->base_gfn;
2059
2060                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
2061         }
2062 }
2063
2064 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
2065 {
2066         struct kvm_memory_slot *memslot;
2067
2068         memslot = gfn_to_memslot(kvm, gfn);
2069         mark_page_dirty_in_slot(memslot, gfn);
2070 }
2071 EXPORT_SYMBOL_GPL(mark_page_dirty);
2072
2073 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
2074 {
2075         struct kvm_memory_slot *memslot;
2076
2077         memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
2078         mark_page_dirty_in_slot(memslot, gfn);
2079 }
2080 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
2081
2082 void kvm_sigset_activate(struct kvm_vcpu *vcpu)
2083 {
2084         if (!vcpu->sigset_active)
2085                 return;
2086
2087         /*
2088          * This does a lockless modification of ->real_blocked, which is fine
2089          * because, only current can change ->real_blocked and all readers of
2090          * ->real_blocked don't care as long ->real_blocked is always a subset
2091          * of ->blocked.
2092          */
2093         sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
2094 }
2095
2096 void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
2097 {
2098         if (!vcpu->sigset_active)
2099                 return;
2100
2101         sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
2102         sigemptyset(&current->real_blocked);
2103 }
2104
2105 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
2106 {
2107         unsigned int old, val, grow;
2108
2109         old = val = vcpu->halt_poll_ns;
2110         grow = READ_ONCE(halt_poll_ns_grow);
2111         /* 10us base */
2112         if (val == 0 && grow)
2113                 val = 10000;
2114         else
2115                 val *= grow;
2116
2117         if (val > halt_poll_ns)
2118                 val = halt_poll_ns;
2119
2120         vcpu->halt_poll_ns = val;
2121         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
2122 }
2123
2124 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
2125 {
2126         unsigned int old, val, shrink;
2127
2128         old = val = vcpu->halt_poll_ns;
2129         shrink = READ_ONCE(halt_poll_ns_shrink);
2130         if (shrink == 0)
2131                 val = 0;
2132         else
2133                 val /= shrink;
2134
2135         vcpu->halt_poll_ns = val;
2136         trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
2137 }
2138
2139 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
2140 {
2141         int ret = -EINTR;
2142         int idx = srcu_read_lock(&vcpu->kvm->srcu);
2143
2144         if (kvm_arch_vcpu_runnable(vcpu)) {
2145                 kvm_make_request(KVM_REQ_UNHALT, vcpu);
2146                 goto out;
2147         }
2148         if (kvm_cpu_has_pending_timer(vcpu))
2149                 goto out;
2150         if (signal_pending(current))
2151                 goto out;
2152
2153         ret = 0;
2154 out:
2155         srcu_read_unlock(&vcpu->kvm->srcu, idx);
2156         return ret;
2157 }
2158
2159 /*
2160  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2161  */
2162 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2163 {
2164         ktime_t start, cur;
2165         DECLARE_SWAITQUEUE(wait);
2166         bool waited = false;
2167         u64 block_ns;
2168
2169         start = cur = ktime_get();
2170         if (vcpu->halt_poll_ns) {
2171                 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2172
2173                 ++vcpu->stat.halt_attempted_poll;
2174                 do {
2175                         /*
2176                          * This sets KVM_REQ_UNHALT if an interrupt
2177                          * arrives.
2178                          */
2179                         if (kvm_vcpu_check_block(vcpu) < 0) {
2180                                 ++vcpu->stat.halt_successful_poll;
2181                                 if (!vcpu_valid_wakeup(vcpu))
2182                                         ++vcpu->stat.halt_poll_invalid;
2183                                 goto out;
2184                         }
2185                         cur = ktime_get();
2186                 } while (single_task_running() && ktime_before(cur, stop));
2187         }
2188
2189         kvm_arch_vcpu_blocking(vcpu);
2190
2191         for (;;) {
2192                 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2193
2194                 if (kvm_vcpu_check_block(vcpu) < 0)
2195                         break;
2196
2197                 waited = true;
2198                 schedule();
2199         }
2200
2201         finish_swait(&vcpu->wq, &wait);
2202         cur = ktime_get();
2203
2204         kvm_arch_vcpu_unblocking(vcpu);
2205 out:
2206         block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2207
2208         if (!vcpu_valid_wakeup(vcpu))
2209                 shrink_halt_poll_ns(vcpu);
2210         else if (halt_poll_ns) {
2211                 if (block_ns <= vcpu->halt_poll_ns)
2212                         ;
2213                 /* we had a long block, shrink polling */
2214                 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2215                         shrink_halt_poll_ns(vcpu);
2216                 /* we had a short halt and our poll time is too small */
2217                 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2218                         block_ns < halt_poll_ns)
2219                         grow_halt_poll_ns(vcpu);
2220         } else
2221                 vcpu->halt_poll_ns = 0;
2222
2223         trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
2224         kvm_arch_vcpu_block_finish(vcpu);
2225 }
2226 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2227
2228 bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2229 {
2230         struct swait_queue_head *wqp;
2231
2232         wqp = kvm_arch_vcpu_wq(vcpu);
2233         if (swq_has_sleeper(wqp)) {
2234                 swake_up_one(wqp);
2235                 ++vcpu->stat.halt_wakeup;
2236                 return true;
2237         }
2238
2239         return false;
2240 }
2241 EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);
2242
2243 #ifndef CONFIG_S390
2244 /*
2245  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2246  */
2247 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2248 {
2249         int me;
2250         int cpu = vcpu->cpu;
2251
2252         if (kvm_vcpu_wake_up(vcpu))
2253                 return;
2254
2255         me = get_cpu();
2256         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2257                 if (kvm_arch_vcpu_should_kick(vcpu))
2258                         smp_send_reschedule(cpu);
2259         put_cpu();
2260 }
2261 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2262 #endif /* !CONFIG_S390 */
2263
2264 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2265 {
2266         struct pid *pid;
2267         struct task_struct *task = NULL;
2268         int ret = 0;
2269
2270         rcu_read_lock();
2271         pid = rcu_dereference(target->pid);
2272         if (pid)
2273                 task = get_pid_task(pid, PIDTYPE_PID);
2274         rcu_read_unlock();
2275         if (!task)
2276                 return ret;
2277         ret = yield_to(task, 1);
2278         put_task_struct(task);
2279
2280         return ret;
2281 }
2282 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2283
2284 /*
2285  * Helper that checks whether a VCPU is eligible for directed yield.
2286  * Most eligible candidate to yield is decided by following heuristics:
2287  *
2288  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2289  *  (preempted lock holder), indicated by @in_spin_loop.
2290  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2291  *
2292  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2293  *  chance last time (mostly it has become eligible now since we have probably
2294  *  yielded to lockholder in last iteration. This is done by toggling
2295  *  @dy_eligible each time a VCPU checked for eligibility.)
2296  *
2297  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2298  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2299  *  burning. Giving priority for a potential lock-holder increases lock
2300  *  progress.
2301  *
2302  *  Since algorithm is based on heuristics, accessing another VCPU data without
2303  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2304  *  and continue with next VCPU and so on.
2305  */
2306 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2307 {
2308 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2309         bool eligible;
2310
2311         eligible = !vcpu->spin_loop.in_spin_loop ||
2312                     vcpu->spin_loop.dy_eligible;
2313
2314         if (vcpu->spin_loop.in_spin_loop)
2315                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2316
2317         return eligible;
2318 #else
2319         return true;
2320 #endif
2321 }
2322
2323 void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
2324 {
2325         struct kvm *kvm = me->kvm;
2326         struct kvm_vcpu *vcpu;
2327         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2328         int yielded = 0;
2329         int try = 3;
2330         int pass;
2331         int i;
2332
2333         kvm_vcpu_set_in_spin_loop(me, true);
2334         /*
2335          * We boost the priority of a VCPU that is runnable but not
2336          * currently running, because it got preempted by something
2337          * else and called schedule in __vcpu_run.  Hopefully that
2338          * VCPU is holding the lock that we need and will release it.
2339          * We approximate round-robin by starting at the last boosted VCPU.
2340          */
2341         for (pass = 0; pass < 2 && !yielded && try; pass++) {
2342                 kvm_for_each_vcpu(i, vcpu, kvm) {
2343                         if (!pass && i <= last_boosted_vcpu) {
2344                                 i = last_boosted_vcpu;
2345                                 continue;
2346                         } else if (pass && i > last_boosted_vcpu)
2347                                 break;
2348                         if (!READ_ONCE(vcpu->preempted))
2349                                 continue;
2350                         if (vcpu == me)
2351                                 continue;
2352                         if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2353                                 continue;
2354                         if (yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu))
2355                                 continue;
2356                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2357                                 continue;
2358
2359                         yielded = kvm_vcpu_yield_to(vcpu);
2360                         if (yielded > 0) {
2361                                 kvm->last_boosted_vcpu = i;
2362                                 break;
2363                         } else if (yielded < 0) {
2364                                 try--;
2365                                 if (!try)
2366                                         break;
2367                         }
2368                 }
2369         }
2370         kvm_vcpu_set_in_spin_loop(me, false);
2371
2372         /* Ensure vcpu is not eligible during next spinloop */
2373         kvm_vcpu_set_dy_eligible(me, false);
2374 }
2375 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2376
2377 static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
2378 {
2379         struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
2380         struct page *page;
2381
2382         if (vmf->pgoff == 0)
2383                 page = virt_to_page(vcpu->run);
2384 #ifdef CONFIG_X86
2385         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2386                 page = virt_to_page(vcpu->arch.pio_data);
2387 #endif
2388 #ifdef CONFIG_KVM_MMIO
2389         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2390                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2391 #endif
2392         else
2393                 return kvm_arch_vcpu_fault(vcpu, vmf);
2394         get_page(page);
2395         vmf->page = page;
2396         return 0;
2397 }
2398
2399 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2400         .fault = kvm_vcpu_fault,
2401 };
2402
2403 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2404 {
2405         vma->vm_ops = &kvm_vcpu_vm_ops;
2406         return 0;
2407 }
2408
2409 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2410 {
2411         struct kvm_vcpu *vcpu = filp->private_data;
2412
2413         debugfs_remove_recursive(vcpu->debugfs_dentry);
2414         kvm_put_kvm(vcpu->kvm);
2415         return 0;
2416 }
2417
2418 static struct file_operations kvm_vcpu_fops = {
2419         .release        = kvm_vcpu_release,
2420         .unlocked_ioctl = kvm_vcpu_ioctl,
2421         .mmap           = kvm_vcpu_mmap,
2422         .llseek         = noop_llseek,
2423         KVM_COMPAT(kvm_vcpu_compat_ioctl),
2424 };
2425
2426 /*
2427  * Allocates an inode for the vcpu.
2428  */
2429 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2430 {
2431         char name[8 + 1 + ITOA_MAX_LEN + 1];
2432
2433         snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
2434         return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2435 }
2436
2437 static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2438 {
2439         char dir_name[ITOA_MAX_LEN * 2];
2440         int ret;
2441
2442         if (!kvm_arch_has_vcpu_debugfs())
2443                 return 0;
2444
2445         if (!debugfs_initialized())
2446                 return 0;
2447
2448         snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
2449         vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
2450                                                                 vcpu->kvm->debugfs_dentry);
2451         if (!vcpu->debugfs_dentry)
2452                 return -ENOMEM;
2453
2454         ret = kvm_arch_create_vcpu_debugfs(vcpu);
2455         if (ret < 0) {
2456                 debugfs_remove_recursive(vcpu->debugfs_dentry);
2457                 return ret;
2458         }
2459
2460         return 0;
2461 }
2462
2463 /*
2464  * Creates some virtual cpus.  Good luck creating more than one.
2465  */
2466 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2467 {
2468         int r;
2469         struct kvm_vcpu *vcpu;
2470
2471         if (id >= KVM_MAX_VCPU_ID)
2472                 return -EINVAL;
2473
2474         mutex_lock(&kvm->lock);
2475         if (kvm->created_vcpus == KVM_MAX_VCPUS) {
2476                 mutex_unlock(&kvm->lock);
2477                 return -EINVAL;
2478         }
2479
2480         kvm->created_vcpus++;
2481         mutex_unlock(&kvm->lock);
2482
2483         vcpu = kvm_arch_vcpu_create(kvm, id);
2484         if (IS_ERR(vcpu)) {
2485                 r = PTR_ERR(vcpu);
2486                 goto vcpu_decrement;
2487         }
2488
2489         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2490
2491         r = kvm_arch_vcpu_setup(vcpu);
2492         if (r)
2493                 goto vcpu_destroy;
2494
2495         r = kvm_create_vcpu_debugfs(vcpu);
2496         if (r)
2497                 goto vcpu_destroy;
2498
2499         mutex_lock(&kvm->lock);
2500         if (kvm_get_vcpu_by_id(kvm, id)) {
2501                 r = -EEXIST;
2502                 goto unlock_vcpu_destroy;
2503         }
2504
2505         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2506
2507         /* Now it's all set up, let userspace reach it */
2508         kvm_get_kvm(kvm);
2509         r = create_vcpu_fd(vcpu);
2510         if (r < 0) {
2511                 kvm_put_kvm(kvm);
2512                 goto unlock_vcpu_destroy;
2513         }
2514
2515         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2516
2517         /*
2518          * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2519          * before kvm->online_vcpu's incremented value.
2520          */
2521         smp_wmb();
2522         atomic_inc(&kvm->online_vcpus);
2523
2524         mutex_unlock(&kvm->lock);
2525         kvm_arch_vcpu_postcreate(vcpu);
2526         return r;
2527
2528 unlock_vcpu_destroy:
2529         mutex_unlock(&kvm->lock);
2530         debugfs_remove_recursive(vcpu->debugfs_dentry);
2531 vcpu_destroy:
2532         kvm_arch_vcpu_destroy(vcpu);
2533 vcpu_decrement:
2534         mutex_lock(&kvm->lock);
2535         kvm->created_vcpus--;
2536         mutex_unlock(&kvm->lock);
2537         return r;
2538 }
2539
2540 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2541 {
2542         if (sigset) {
2543                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2544                 vcpu->sigset_active = 1;
2545                 vcpu->sigset = *sigset;
2546         } else
2547                 vcpu->sigset_active = 0;
2548         return 0;
2549 }
2550
2551 static long kvm_vcpu_ioctl(struct file *filp,
2552                            unsigned int ioctl, unsigned long arg)
2553 {
2554         struct kvm_vcpu *vcpu = filp->private_data;
2555         void __user *argp = (void __user *)arg;
2556         int r;
2557         struct kvm_fpu *fpu = NULL;
2558         struct kvm_sregs *kvm_sregs = NULL;
2559
2560         if (vcpu->kvm->mm != current->mm)
2561                 return -EIO;
2562
2563         if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2564                 return -EINVAL;
2565
2566         /*
2567          * Some architectures have vcpu ioctls that are asynchronous to vcpu
2568          * execution; mutex_lock() would break them.
2569          */
2570         r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
2571         if (r != -ENOIOCTLCMD)
2572                 return r;
2573
2574         if (mutex_lock_killable(&vcpu->mutex))
2575                 return -EINTR;
2576         switch (ioctl) {
2577         case KVM_RUN: {
2578                 struct pid *oldpid;
2579                 r = -EINVAL;
2580                 if (arg)
2581                         goto out;
2582                 oldpid = rcu_access_pointer(vcpu->pid);
2583                 if (unlikely(oldpid != task_pid(current))) {
2584                         /* The thread running this VCPU changed. */
2585                         struct pid *newpid;
2586
2587                         r = kvm_arch_vcpu_run_pid_change(vcpu);
2588                         if (r)
2589                                 break;
2590
2591                         newpid = get_task_pid(current, PIDTYPE_PID);
2592                         rcu_assign_pointer(vcpu->pid, newpid);
2593                         if (oldpid)
2594                                 synchronize_rcu();
2595                         put_pid(oldpid);
2596                 }
2597                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2598                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2599                 break;
2600         }
2601         case KVM_GET_REGS: {
2602                 struct kvm_regs *kvm_regs;
2603
2604                 r = -ENOMEM;
2605                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2606                 if (!kvm_regs)
2607                         goto out;
2608                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2609                 if (r)
2610                         goto out_free1;
2611                 r = -EFAULT;
2612                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2613                         goto out_free1;
2614                 r = 0;
2615 out_free1:
2616                 kfree(kvm_regs);
2617                 break;
2618         }
2619         case KVM_SET_REGS: {
2620                 struct kvm_regs *kvm_regs;
2621
2622                 r = -ENOMEM;
2623                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2624                 if (IS_ERR(kvm_regs)) {
2625                         r = PTR_ERR(kvm_regs);
2626                         goto out;
2627                 }
2628                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2629                 kfree(kvm_regs);
2630                 break;
2631         }
2632         case KVM_GET_SREGS: {
2633                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2634                 r = -ENOMEM;
2635                 if (!kvm_sregs)
2636                         goto out;
2637                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2638                 if (r)
2639                         goto out;
2640                 r = -EFAULT;
2641                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2642                         goto out;
2643                 r = 0;
2644                 break;
2645         }
2646         case KVM_SET_SREGS: {
2647                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2648                 if (IS_ERR(kvm_sregs)) {
2649                         r = PTR_ERR(kvm_sregs);
2650                         kvm_sregs = NULL;
2651                         goto out;
2652                 }
2653                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2654                 break;
2655         }
2656         case KVM_GET_MP_STATE: {
2657                 struct kvm_mp_state mp_state;
2658
2659                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2660                 if (r)
2661                         goto out;
2662                 r = -EFAULT;
2663                 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2664                         goto out;
2665                 r = 0;
2666                 break;
2667         }
2668         case KVM_SET_MP_STATE: {
2669                 struct kvm_mp_state mp_state;
2670
2671                 r = -EFAULT;
2672                 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2673                         goto out;
2674                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2675                 break;
2676         }
2677         case KVM_TRANSLATE: {
2678                 struct kvm_translation tr;
2679
2680                 r = -EFAULT;
2681                 if (copy_from_user(&tr, argp, sizeof(tr)))
2682                         goto out;
2683                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2684                 if (r)
2685                         goto out;
2686                 r = -EFAULT;
2687                 if (copy_to_user(argp, &tr, sizeof(tr)))
2688                         goto out;
2689                 r = 0;
2690                 break;
2691         }
2692         case KVM_SET_GUEST_DEBUG: {
2693                 struct kvm_guest_debug dbg;
2694
2695                 r = -EFAULT;
2696                 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2697                         goto out;
2698                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2699                 break;
2700         }
2701         case KVM_SET_SIGNAL_MASK: {
2702                 struct kvm_signal_mask __user *sigmask_arg = argp;
2703                 struct kvm_signal_mask kvm_sigmask;
2704                 sigset_t sigset, *p;
2705
2706                 p = NULL;
2707                 if (argp) {
2708                         r = -EFAULT;
2709                         if (copy_from_user(&kvm_sigmask, argp,
2710                                            sizeof(kvm_sigmask)))
2711                                 goto out;
2712                         r = -EINVAL;
2713                         if (kvm_sigmask.len != sizeof(sigset))
2714                                 goto out;
2715                         r = -EFAULT;
2716                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2717                                            sizeof(sigset)))
2718                                 goto out;
2719                         p = &sigset;
2720                 }
2721                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2722                 break;
2723         }
2724         case KVM_GET_FPU: {
2725                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2726                 r = -ENOMEM;
2727                 if (!fpu)
2728                         goto out;
2729                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2730                 if (r)
2731                         goto out;
2732                 r = -EFAULT;
2733                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2734                         goto out;
2735                 r = 0;
2736                 break;
2737         }
2738         case KVM_SET_FPU: {
2739                 fpu = memdup_user(argp, sizeof(*fpu));
2740                 if (IS_ERR(fpu)) {
2741                         r = PTR_ERR(fpu);
2742                         fpu = NULL;
2743                         goto out;
2744                 }
2745                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2746                 break;
2747         }
2748         default:
2749                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2750         }
2751 out:
2752         mutex_unlock(&vcpu->mutex);
2753         kfree(fpu);
2754         kfree(kvm_sregs);
2755         return r;
2756 }
2757
2758 #ifdef CONFIG_KVM_COMPAT
2759 static long kvm_vcpu_compat_ioctl(struct file *filp,
2760                                   unsigned int ioctl, unsigned long arg)
2761 {
2762         struct kvm_vcpu *vcpu = filp->private_data;
2763         void __user *argp = compat_ptr(arg);
2764         int r;
2765
2766         if (vcpu->kvm->mm != current->mm)
2767                 return -EIO;
2768
2769         switch (ioctl) {
2770         case KVM_SET_SIGNAL_MASK: {
2771                 struct kvm_signal_mask __user *sigmask_arg = argp;
2772                 struct kvm_signal_mask kvm_sigmask;
2773                 sigset_t sigset;
2774
2775                 if (argp) {
2776                         r = -EFAULT;
2777                         if (copy_from_user(&kvm_sigmask, argp,
2778                                            sizeof(kvm_sigmask)))
2779                                 goto out;
2780                         r = -EINVAL;
2781                         if (kvm_sigmask.len != sizeof(compat_sigset_t))
2782                                 goto out;
2783                         r = -EFAULT;
2784                         if (get_compat_sigset(&sigset, (void *)sigmask_arg->sigset))
2785                                 goto out;
2786                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2787                 } else
2788                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2789                 break;
2790         }
2791         default:
2792                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2793         }
2794
2795 out:
2796         return r;
2797 }
2798 #endif
2799
2800 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2801                                  int (*accessor)(struct kvm_device *dev,
2802                                                  struct kvm_device_attr *attr),
2803                                  unsigned long arg)
2804 {
2805         struct kvm_device_attr attr;
2806
2807         if (!accessor)
2808                 return -EPERM;
2809
2810         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2811                 return -EFAULT;
2812
2813         return accessor(dev, &attr);
2814 }
2815
2816 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2817                              unsigned long arg)
2818 {
2819         struct kvm_device *dev = filp->private_data;
2820
2821         switch (ioctl) {
2822         case KVM_SET_DEVICE_ATTR:
2823                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2824         case KVM_GET_DEVICE_ATTR:
2825                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2826         case KVM_HAS_DEVICE_ATTR:
2827                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2828         default:
2829                 if (dev->ops->ioctl)
2830                         return dev->ops->ioctl(dev, ioctl, arg);
2831
2832                 return -ENOTTY;
2833         }
2834 }
2835
2836 static int kvm_device_release(struct inode *inode, struct file *filp)
2837 {
2838         struct kvm_device *dev = filp->private_data;
2839         struct kvm *kvm = dev->kvm;
2840
2841         kvm_put_kvm(kvm);
2842         return 0;
2843 }
2844
2845 static const struct file_operations kvm_device_fops = {
2846         .unlocked_ioctl = kvm_device_ioctl,
2847         .release = kvm_device_release,
2848         KVM_COMPAT(kvm_device_ioctl),
2849 };
2850
2851 struct kvm_device *kvm_device_from_filp(struct file *filp)
2852 {
2853         if (filp->f_op != &kvm_device_fops)
2854                 return NULL;
2855
2856         return filp->private_data;
2857 }
2858
2859 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2860 #ifdef CONFIG_KVM_MPIC
2861         [KVM_DEV_TYPE_FSL_MPIC_20]      = &kvm_mpic_ops,
2862         [KVM_DEV_TYPE_FSL_MPIC_42]      = &kvm_mpic_ops,
2863 #endif
2864 };
2865
2866 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2867 {
2868         if (type >= ARRAY_SIZE(kvm_device_ops_table))
2869                 return -ENOSPC;
2870
2871         if (kvm_device_ops_table[type] != NULL)
2872                 return -EEXIST;
2873
2874         kvm_device_ops_table[type] = ops;
2875         return 0;
2876 }
2877
2878 void kvm_unregister_device_ops(u32 type)
2879 {
2880         if (kvm_device_ops_table[type] != NULL)
2881                 kvm_device_ops_table[type] = NULL;
2882 }
2883
2884 static int kvm_ioctl_create_device(struct kvm *kvm,
2885                                    struct kvm_create_device *cd)
2886 {
2887         struct kvm_device_ops *ops = NULL;
2888         struct kvm_device *dev;
2889         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2890         int ret;
2891
2892         if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2893                 return -ENODEV;
2894
2895         ops = kvm_device_ops_table[cd->type];
2896         if (ops == NULL)
2897                 return -ENODEV;
2898
2899         if (test)
2900                 return 0;
2901
2902         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2903         if (!dev)
2904                 return -ENOMEM;
2905
2906         dev->ops = ops;
2907         dev->kvm = kvm;
2908
2909         mutex_lock(&kvm->lock);
2910         ret = ops->create(dev, cd->type);
2911         if (ret < 0) {
2912                 mutex_unlock(&kvm->lock);
2913                 kfree(dev);
2914                 return ret;
2915         }
2916         list_add(&dev->vm_node, &kvm->devices);
2917         mutex_unlock(&kvm->lock);
2918
2919         if (ops->init)
2920                 ops->init(dev);
2921
2922         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2923         if (ret < 0) {
2924                 mutex_lock(&kvm->lock);
2925                 list_del(&dev->vm_node);
2926                 mutex_unlock(&kvm->lock);
2927                 ops->destroy(dev);
2928                 return ret;
2929         }
2930
2931         kvm_get_kvm(kvm);
2932         cd->fd = ret;
2933         return 0;
2934 }
2935
2936 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2937 {
2938         switch (arg) {
2939         case KVM_CAP_USER_MEMORY:
2940         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2941         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2942         case KVM_CAP_INTERNAL_ERROR_DATA:
2943 #ifdef CONFIG_HAVE_KVM_MSI
2944         case KVM_CAP_SIGNAL_MSI:
2945 #endif
2946 #ifdef CONFIG_HAVE_KVM_IRQFD
2947         case KVM_CAP_IRQFD:
2948         case KVM_CAP_IRQFD_RESAMPLE:
2949 #endif
2950         case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2951         case KVM_CAP_CHECK_EXTENSION_VM:
2952                 return 1;
2953 #ifdef CONFIG_KVM_MMIO
2954         case KVM_CAP_COALESCED_MMIO:
2955                 return KVM_COALESCED_MMIO_PAGE_OFFSET;
2956         case KVM_CAP_COALESCED_PIO:
2957                 return 1;
2958 #endif
2959 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2960         case KVM_CAP_IRQ_ROUTING:
2961                 return KVM_MAX_IRQ_ROUTES;
2962 #endif
2963 #if KVM_ADDRESS_SPACE_NUM > 1
2964         case KVM_CAP_MULTI_ADDRESS_SPACE:
2965                 return KVM_ADDRESS_SPACE_NUM;
2966 #endif
2967         case KVM_CAP_MAX_VCPU_ID:
2968                 return KVM_MAX_VCPU_ID;
2969         default:
2970                 break;
2971         }
2972         return kvm_vm_ioctl_check_extension(kvm, arg);
2973 }
2974
2975 static long kvm_vm_ioctl(struct file *filp,
2976                            unsigned int ioctl, unsigned long arg)
2977 {
2978         struct kvm *kvm = filp->private_data;
2979         void __user *argp = (void __user *)arg;
2980         int r;
2981
2982         if (kvm->mm != current->mm)
2983                 return -EIO;
2984         switch (ioctl) {
2985         case KVM_CREATE_VCPU:
2986                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2987                 break;
2988         case KVM_SET_USER_MEMORY_REGION: {
2989                 struct kvm_userspace_memory_region kvm_userspace_mem;
2990
2991                 r = -EFAULT;
2992                 if (copy_from_user(&kvm_userspace_mem, argp,
2993                                                 sizeof(kvm_userspace_mem)))
2994                         goto out;
2995
2996                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2997                 break;
2998         }
2999         case KVM_GET_DIRTY_LOG: {
3000                 struct kvm_dirty_log log;
3001
3002                 r = -EFAULT;
3003                 if (copy_from_user(&log, argp, sizeof(log)))
3004                         goto out;
3005                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3006                 break;
3007         }
3008 #ifdef CONFIG_KVM_MMIO
3009         case KVM_REGISTER_COALESCED_MMIO: {
3010                 struct kvm_coalesced_mmio_zone zone;
3011
3012                 r = -EFAULT;
3013                 if (copy_from_user(&zone, argp, sizeof(zone)))
3014                         goto out;
3015                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
3016                 break;
3017         }
3018         case KVM_UNREGISTER_COALESCED_MMIO: {
3019                 struct kvm_coalesced_mmio_zone zone;
3020
3021                 r = -EFAULT;
3022                 if (copy_from_user(&zone, argp, sizeof(zone)))
3023                         goto out;
3024                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
3025                 break;
3026         }
3027 #endif
3028         case KVM_IRQFD: {
3029                 struct kvm_irqfd data;
3030
3031                 r = -EFAULT;
3032                 if (copy_from_user(&data, argp, sizeof(data)))
3033                         goto out;
3034                 r = kvm_irqfd(kvm, &data);
3035                 break;
3036         }
3037         case KVM_IOEVENTFD: {
3038                 struct kvm_ioeventfd data;
3039
3040                 r = -EFAULT;
3041                 if (copy_from_user(&data, argp, sizeof(data)))
3042                         goto out;
3043                 r = kvm_ioeventfd(kvm, &data);
3044                 break;
3045         }
3046 #ifdef CONFIG_HAVE_KVM_MSI
3047         case KVM_SIGNAL_MSI: {
3048                 struct kvm_msi msi;
3049
3050                 r = -EFAULT;
3051                 if (copy_from_user(&msi, argp, sizeof(msi)))
3052                         goto out;
3053                 r = kvm_send_userspace_msi(kvm, &msi);
3054                 break;
3055         }
3056 #endif
3057 #ifdef __KVM_HAVE_IRQ_LINE
3058         case KVM_IRQ_LINE_STATUS:
3059         case KVM_IRQ_LINE: {
3060                 struct kvm_irq_level irq_event;
3061
3062                 r = -EFAULT;
3063                 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
3064                         goto out;
3065
3066                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
3067                                         ioctl == KVM_IRQ_LINE_STATUS);
3068                 if (r)
3069                         goto out;
3070
3071                 r = -EFAULT;
3072                 if (ioctl == KVM_IRQ_LINE_STATUS) {
3073                         if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
3074                                 goto out;
3075                 }
3076
3077                 r = 0;
3078                 break;
3079         }
3080 #endif
3081 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
3082         case KVM_SET_GSI_ROUTING: {
3083                 struct kvm_irq_routing routing;
3084                 struct kvm_irq_routing __user *urouting;
3085                 struct kvm_irq_routing_entry *entries = NULL;
3086
3087                 r = -EFAULT;
3088                 if (copy_from_user(&routing, argp, sizeof(routing)))
3089                         goto out;
3090                 r = -EINVAL;
3091                 if (!kvm_arch_can_set_irq_routing(kvm))
3092                         goto out;
3093                 if (routing.nr > KVM_MAX_IRQ_ROUTES)
3094                         goto out;
3095                 if (routing.flags)
3096                         goto out;
3097                 if (routing.nr) {
3098                         r = -ENOMEM;
3099                         entries = vmalloc(array_size(sizeof(*entries),
3100                                                      routing.nr));
3101                         if (!entries)
3102                                 goto out;
3103                         r = -EFAULT;
3104                         urouting = argp;
3105                         if (copy_from_user(entries, urouting->entries,
3106                                            routing.nr * sizeof(*entries)))
3107                                 goto out_free_irq_routing;
3108                 }
3109                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
3110                                         routing.flags);
3111 out_free_irq_routing:
3112                 vfree(entries);
3113                 break;
3114         }
3115 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
3116         case KVM_CREATE_DEVICE: {
3117                 struct kvm_create_device cd;
3118
3119                 r = -EFAULT;
3120                 if (copy_from_user(&cd, argp, sizeof(cd)))
3121                         goto out;
3122
3123                 r = kvm_ioctl_create_device(kvm, &cd);
3124                 if (r)
3125                         goto out;
3126
3127                 r = -EFAULT;
3128                 if (copy_to_user(argp, &cd, sizeof(cd)))
3129                         goto out;
3130
3131                 r = 0;
3132                 break;
3133         }
3134         case KVM_CHECK_EXTENSION:
3135                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
3136                 break;
3137         default:
3138                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
3139         }
3140 out:
3141         return r;
3142 }
3143
3144 #ifdef CONFIG_KVM_COMPAT
3145 struct compat_kvm_dirty_log {
3146         __u32 slot;
3147         __u32 padding1;
3148         union {
3149                 compat_uptr_t dirty_bitmap; /* one bit per page */
3150                 __u64 padding2;
3151         };
3152 };
3153
3154 static long kvm_vm_compat_ioctl(struct file *filp,
3155                            unsigned int ioctl, unsigned long arg)
3156 {
3157         struct kvm *kvm = filp->private_data;
3158         int r;
3159
3160         if (kvm->mm != current->mm)
3161                 return -EIO;
3162         switch (ioctl) {
3163         case KVM_GET_DIRTY_LOG: {
3164                 struct compat_kvm_dirty_log compat_log;
3165                 struct kvm_dirty_log log;
3166
3167                 if (copy_from_user(&compat_log, (void __user *)arg,
3168                                    sizeof(compat_log)))
3169                         return -EFAULT;
3170                 log.slot         = compat_log.slot;
3171                 log.padding1     = compat_log.padding1;
3172                 log.padding2     = compat_log.padding2;
3173                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
3174
3175                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
3176                 break;
3177         }
3178         default:
3179                 r = kvm_vm_ioctl(filp, ioctl, arg);
3180         }
3181         return r;
3182 }
3183 #endif
3184
3185 static struct file_operations kvm_vm_fops = {
3186         .release        = kvm_vm_release,
3187         .unlocked_ioctl = kvm_vm_ioctl,
3188         .llseek         = noop_llseek,
3189         KVM_COMPAT(kvm_vm_compat_ioctl),
3190 };
3191
3192 static int kvm_dev_ioctl_create_vm(unsigned long type)
3193 {
3194         int r;
3195         struct kvm *kvm;
3196         struct file *file;
3197
3198         kvm = kvm_create_vm(type);
3199         if (IS_ERR(kvm))
3200                 return PTR_ERR(kvm);
3201 #ifdef CONFIG_KVM_MMIO
3202         r = kvm_coalesced_mmio_init(kvm);
3203         if (r < 0)
3204                 goto put_kvm;
3205 #endif
3206         r = get_unused_fd_flags(O_CLOEXEC);
3207         if (r < 0)
3208                 goto put_kvm;
3209
3210         file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
3211         if (IS_ERR(file)) {
3212                 put_unused_fd(r);
3213                 r = PTR_ERR(file);
3214                 goto put_kvm;
3215         }
3216
3217         /*
3218          * Don't call kvm_put_kvm anymore at this point; file->f_op is
3219          * already set, with ->release() being kvm_vm_release().  In error
3220          * cases it will be called by the final fput(file) and will take
3221          * care of doing kvm_put_kvm(kvm).
3222          */
3223         if (kvm_create_vm_debugfs(kvm, r) < 0) {
3224                 put_unused_fd(r);
3225                 fput(file);
3226                 return -ENOMEM;
3227         }
3228         kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);
3229
3230         fd_install(r, file);
3231         return r;
3232
3233 put_kvm:
3234         kvm_put_kvm(kvm);
3235         return r;
3236 }
3237
3238 static long kvm_dev_ioctl(struct file *filp,
3239                           unsigned int ioctl, unsigned long arg)
3240 {
3241         long r = -EINVAL;
3242
3243         switch (ioctl) {
3244         case KVM_GET_API_VERSION:
3245                 if (arg)
3246                         goto out;
3247                 r = KVM_API_VERSION;
3248                 break;
3249         case KVM_CREATE_VM:
3250                 r = kvm_dev_ioctl_create_vm(arg);
3251                 break;
3252         case KVM_CHECK_EXTENSION:
3253                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3254                 break;
3255         case KVM_GET_VCPU_MMAP_SIZE:
3256                 if (arg)
3257                         goto out;
3258                 r = PAGE_SIZE;     /* struct kvm_run */
3259 #ifdef CONFIG_X86
3260                 r += PAGE_SIZE;    /* pio data page */
3261 #endif
3262 #ifdef CONFIG_KVM_MMIO
3263                 r += PAGE_SIZE;    /* coalesced mmio ring page */
3264 #endif
3265                 break;
3266         case KVM_TRACE_ENABLE:
3267         case KVM_TRACE_PAUSE:
3268         case KVM_TRACE_DISABLE:
3269                 r = -EOPNOTSUPP;
3270                 break;
3271         default:
3272                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3273         }
3274 out:
3275         return r;
3276 }
3277
3278 static struct file_operations kvm_chardev_ops = {
3279         .unlocked_ioctl = kvm_dev_ioctl,
3280         .llseek         = noop_llseek,
3281         KVM_COMPAT(kvm_dev_ioctl),
3282 };
3283
3284 static struct miscdevice kvm_dev = {
3285         KVM_MINOR,
3286         "kvm",
3287         &kvm_chardev_ops,
3288 };
3289
3290 static void hardware_enable_nolock(void *junk)
3291 {
3292         int cpu = raw_smp_processor_id();
3293         int r;
3294
3295         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3296                 return;
3297
3298         cpumask_set_cpu(cpu, cpus_hardware_enabled);
3299
3300         r = kvm_arch_hardware_enable();
3301
3302         if (r) {
3303                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3304                 atomic_inc(&hardware_enable_failed);
3305                 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3306         }
3307 }
3308
3309 static int kvm_starting_cpu(unsigned int cpu)
3310 {
3311         raw_spin_lock(&kvm_count_lock);
3312         if (kvm_usage_count)
3313                 hardware_enable_nolock(NULL);
3314         raw_spin_unlock(&kvm_count_lock);
3315         return 0;
3316 }
3317
3318 static void hardware_disable_nolock(void *junk)
3319 {
3320         int cpu = raw_smp_processor_id();
3321
3322         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3323                 return;
3324         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3325         kvm_arch_hardware_disable();
3326 }
3327
3328 static int kvm_dying_cpu(unsigned int cpu)
3329 {
3330         raw_spin_lock(&kvm_count_lock);
3331         if (kvm_usage_count)
3332                 hardware_disable_nolock(NULL);
3333         raw_spin_unlock(&kvm_count_lock);
3334         return 0;
3335 }
3336
3337 static void hardware_disable_all_nolock(void)
3338 {
3339         BUG_ON(!kvm_usage_count);
3340
3341         kvm_usage_count--;
3342         if (!kvm_usage_count)
3343                 on_each_cpu(hardware_disable_nolock, NULL, 1);
3344 }
3345
3346 static void hardware_disable_all(void)
3347 {
3348         raw_spin_lock(&kvm_count_lock);
3349         hardware_disable_all_nolock();
3350         raw_spin_unlock(&kvm_count_lock);
3351 }
3352
3353 static int hardware_enable_all(void)
3354 {
3355         int r = 0;
3356
3357         raw_spin_lock(&kvm_count_lock);
3358
3359         kvm_usage_count++;
3360         if (kvm_usage_count == 1) {
3361                 atomic_set(&hardware_enable_failed, 0);
3362                 on_each_cpu(hardware_enable_nolock, NULL, 1);
3363
3364                 if (atomic_read(&hardware_enable_failed)) {
3365                         hardware_disable_all_nolock();
3366                         r = -EBUSY;
3367                 }
3368         }
3369
3370         raw_spin_unlock(&kvm_count_lock);
3371
3372         return r;
3373 }
3374
3375 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3376                       void *v)
3377 {
3378         /*
3379          * Some (well, at least mine) BIOSes hang on reboot if
3380          * in vmx root mode.
3381          *
3382          * And Intel TXT required VMX off for all cpu when system shutdown.
3383          */
3384         pr_info("kvm: exiting hardware virtualization\n");
3385         kvm_rebooting = true;
3386         on_each_cpu(hardware_disable_nolock, NULL, 1);
3387         return NOTIFY_OK;
3388 }
3389
3390 static struct notifier_block kvm_reboot_notifier = {
3391         .notifier_call = kvm_reboot,
3392         .priority = 0,
3393 };
3394
3395 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3396 {
3397         int i;
3398
3399         for (i = 0; i < bus->dev_count; i++) {
3400                 struct kvm_io_device *pos = bus->range[i].dev;
3401
3402                 kvm_iodevice_destructor(pos);
3403         }
3404         kfree(bus);
3405 }
3406
3407 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3408                                  const struct kvm_io_range *r2)
3409 {
3410         gpa_t addr1 = r1->addr;
3411         gpa_t addr2 = r2->addr;
3412
3413         if (addr1 < addr2)
3414                 return -1;
3415
3416         /* If r2->len == 0, match the exact address.  If r2->len != 0,
3417          * accept any overlapping write.  Any order is acceptable for
3418          * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3419          * we process all of them.
3420          */
3421         if (r2->len) {
3422                 addr1 += r1->len;
3423                 addr2 += r2->len;
3424         }
3425
3426         if (addr1 > addr2)
3427                 return 1;
3428
3429         return 0;
3430 }
3431
3432 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3433 {
3434         return kvm_io_bus_cmp(p1, p2);
3435 }
3436
3437 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3438                              gpa_t addr, int len)
3439 {
3440         struct kvm_io_range *range, key;
3441         int off;
3442
3443         key = (struct kvm_io_range) {
3444                 .addr = addr,
3445                 .len = len,
3446         };
3447
3448         range = bsearch(&key, bus->range, bus->dev_count,
3449                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3450         if (range == NULL)
3451                 return -ENOENT;
3452
3453         off = range - bus->range;
3454
3455         while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3456                 off--;
3457
3458         return off;
3459 }
3460
3461 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3462                               struct kvm_io_range *range, const void *val)
3463 {
3464         int idx;
3465
3466         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3467         if (idx < 0)
3468                 return -EOPNOTSUPP;
3469
3470         while (idx < bus->dev_count &&
3471                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3472                 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3473                                         range->len, val))
3474                         return idx;
3475                 idx++;
3476         }
3477
3478         return -EOPNOTSUPP;
3479 }
3480
3481 /* kvm_io_bus_write - called under kvm->slots_lock */
3482 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3483                      int len, const void *val)
3484 {
3485         struct kvm_io_bus *bus;
3486         struct kvm_io_range range;
3487         int r;
3488
3489         range = (struct kvm_io_range) {
3490                 .addr = addr,
3491                 .len = len,
3492         };
3493
3494         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3495         if (!bus)
3496                 return -ENOMEM;
3497         r = __kvm_io_bus_write(vcpu, bus, &range, val);
3498         return r < 0 ? r : 0;
3499 }
3500
3501 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3502 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3503                             gpa_t addr, int len, const void *val, long cookie)
3504 {
3505         struct kvm_io_bus *bus;
3506         struct kvm_io_range range;
3507
3508         range = (struct kvm_io_range) {
3509                 .addr = addr,
3510                 .len = len,
3511         };
3512
3513         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3514         if (!bus)
3515                 return -ENOMEM;
3516
3517         /* First try the device referenced by cookie. */
3518         if ((cookie >= 0) && (cookie < bus->dev_count) &&
3519             (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3520                 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3521                                         val))
3522                         return cookie;
3523
3524         /*
3525          * cookie contained garbage; fall back to search and return the
3526          * correct cookie value.
3527          */
3528         return __kvm_io_bus_write(vcpu, bus, &range, val);
3529 }
3530
3531 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3532                              struct kvm_io_range *range, void *val)
3533 {
3534         int idx;
3535
3536         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3537         if (idx < 0)
3538                 return -EOPNOTSUPP;
3539
3540         while (idx < bus->dev_count &&
3541                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3542                 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3543                                        range->len, val))
3544                         return idx;
3545                 idx++;
3546         }
3547
3548         return -EOPNOTSUPP;
3549 }
3550 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3551
3552 /* kvm_io_bus_read - called under kvm->slots_lock */
3553 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3554                     int len, void *val)
3555 {
3556         struct kvm_io_bus *bus;
3557         struct kvm_io_range range;
3558         int r;
3559
3560         range = (struct kvm_io_range) {
3561                 .addr = addr,
3562                 .len = len,
3563         };
3564
3565         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3566         if (!bus)
3567                 return -ENOMEM;
3568         r = __kvm_io_bus_read(vcpu, bus, &range, val);
3569         return r < 0 ? r : 0;
3570 }
3571
3572
3573 /* Caller must hold slots_lock. */
3574 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3575                             int len, struct kvm_io_device *dev)
3576 {
3577         int i;
3578         struct kvm_io_bus *new_bus, *bus;
3579         struct kvm_io_range range;
3580
3581         bus = kvm_get_bus(kvm, bus_idx);
3582         if (!bus)
3583                 return -ENOMEM;
3584
3585         /* exclude ioeventfd which is limited by maximum fd */
3586         if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3587                 return -ENOSPC;
3588
3589         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3590                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3591         if (!new_bus)
3592                 return -ENOMEM;
3593
3594         range = (struct kvm_io_range) {
3595                 .addr = addr,
3596                 .len = len,
3597                 .dev = dev,
3598         };
3599
3600         for (i = 0; i < bus->dev_count; i++)
3601                 if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
3602                         break;
3603
3604         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3605         new_bus->dev_count++;
3606         new_bus->range[i] = range;
3607         memcpy(new_bus->range + i + 1, bus->range + i,
3608                 (bus->dev_count - i) * sizeof(struct kvm_io_range));
3609         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3610         synchronize_srcu_expedited(&kvm->srcu);
3611         kfree(bus);
3612
3613         return 0;
3614 }
3615
3616 /* Caller must hold slots_lock. */
3617 void kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3618                                struct kvm_io_device *dev)
3619 {
3620         int i;
3621         struct kvm_io_bus *new_bus, *bus;
3622
3623         bus = kvm_get_bus(kvm, bus_idx);
3624         if (!bus)
3625                 return;
3626
3627         for (i = 0; i < bus->dev_count; i++)
3628                 if (bus->range[i].dev == dev) {
3629                         break;
3630                 }
3631
3632         if (i == bus->dev_count)
3633                 return;
3634
3635         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3636                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3637         if (!new_bus)  {
3638                 pr_err("kvm: failed to shrink bus, removing it completely\n");
3639                 goto broken;
3640         }
3641
3642         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3643         new_bus->dev_count--;
3644         memcpy(new_bus->range + i, bus->range + i + 1,
3645                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3646
3647 broken:
3648         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3649         synchronize_srcu_expedited(&kvm->srcu);
3650         kfree(bus);
3651         return;
3652 }
3653
3654 struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3655                                          gpa_t addr)
3656 {
3657         struct kvm_io_bus *bus;
3658         int dev_idx, srcu_idx;
3659         struct kvm_io_device *iodev = NULL;
3660
3661         srcu_idx = srcu_read_lock(&kvm->srcu);
3662
3663         bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
3664         if (!bus)
3665                 goto out_unlock;
3666
3667         dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
3668         if (dev_idx < 0)
3669                 goto out_unlock;
3670
3671         iodev = bus->range[dev_idx].dev;
3672
3673 out_unlock:
3674         srcu_read_unlock(&kvm->srcu, srcu_idx);
3675
3676         return iodev;
3677 }
3678 EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);
3679
3680 static int kvm_debugfs_open(struct inode *inode, struct file *file,
3681                            int (*get)(void *, u64 *), int (*set)(void *, u64),
3682                            const char *fmt)
3683 {
3684         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3685                                           inode->i_private;
3686
3687         /* The debugfs files are a reference to the kvm struct which
3688          * is still valid when kvm_destroy_vm is called.
3689          * To avoid the race between open and the removal of the debugfs
3690          * directory we test against the users count.
3691          */
3692         if (!refcount_inc_not_zero(&stat_data->kvm->users_count))
3693                 return -ENOENT;
3694
3695         if (simple_attr_open(inode, file, get, set, fmt)) {
3696                 kvm_put_kvm(stat_data->kvm);
3697                 return -ENOMEM;
3698         }
3699
3700         return 0;
3701 }
3702
3703 static int kvm_debugfs_release(struct inode *inode, struct file *file)
3704 {
3705         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)
3706                                           inode->i_private;
3707
3708         simple_attr_release(inode, file);
3709         kvm_put_kvm(stat_data->kvm);
3710
3711         return 0;
3712 }
3713
3714 static int vm_stat_get_per_vm(void *data, u64 *val)
3715 {
3716         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3717
3718         *val = *(ulong *)((void *)stat_data->kvm + stat_data->offset);
3719
3720         return 0;
3721 }
3722
3723 static int vm_stat_clear_per_vm(void *data, u64 val)
3724 {
3725         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3726
3727         if (val)
3728                 return -EINVAL;
3729
3730         *(ulong *)((void *)stat_data->kvm + stat_data->offset) = 0;
3731
3732         return 0;
3733 }
3734
3735 static int vm_stat_get_per_vm_open(struct inode *inode, struct file *file)
3736 {
3737         __simple_attr_check_format("%llu\n", 0ull);
3738         return kvm_debugfs_open(inode, file, vm_stat_get_per_vm,
3739                                 vm_stat_clear_per_vm, "%llu\n");
3740 }
3741
3742 static const struct file_operations vm_stat_get_per_vm_fops = {
3743         .owner   = THIS_MODULE,
3744         .open    = vm_stat_get_per_vm_open,
3745         .release = kvm_debugfs_release,
3746         .read    = simple_attr_read,
3747         .write   = simple_attr_write,
3748         .llseek  = no_llseek,
3749 };
3750
3751 static int vcpu_stat_get_per_vm(void *data, u64 *val)
3752 {
3753         int i;
3754         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3755         struct kvm_vcpu *vcpu;
3756
3757         *val = 0;
3758
3759         kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3760                 *val += *(u64 *)((void *)vcpu + stat_data->offset);
3761
3762         return 0;
3763 }
3764
3765 static int vcpu_stat_clear_per_vm(void *data, u64 val)
3766 {
3767         int i;
3768         struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data;
3769         struct kvm_vcpu *vcpu;
3770
3771         if (val)
3772                 return -EINVAL;
3773
3774         kvm_for_each_vcpu(i, vcpu, stat_data->kvm)
3775                 *(u64 *)((void *)vcpu + stat_data->offset) = 0;
3776
3777         return 0;
3778 }
3779
3780 static int vcpu_stat_get_per_vm_open(struct inode *inode, struct file *file)
3781 {
3782         __simple_attr_check_format("%llu\n", 0ull);
3783         return kvm_debugfs_open(inode, file, vcpu_stat_get_per_vm,
3784                                  vcpu_stat_clear_per_vm, "%llu\n");
3785 }
3786
3787 static const struct file_operations vcpu_stat_get_per_vm_fops = {
3788         .owner   = THIS_MODULE,
3789         .open    = vcpu_stat_get_per_vm_open,
3790         .release = kvm_debugfs_release,
3791         .read    = simple_attr_read,
3792         .write   = simple_attr_write,
3793         .llseek  = no_llseek,
3794 };
3795
3796 static const struct file_operations *stat_fops_per_vm[] = {
3797         [KVM_STAT_VCPU] = &vcpu_stat_get_per_vm_fops,
3798         [KVM_STAT_VM]   = &vm_stat_get_per_vm_fops,
3799 };
3800
3801 static int vm_stat_get(void *_offset, u64 *val)
3802 {
3803         unsigned offset = (long)_offset;
3804         struct kvm *kvm;
3805         struct kvm_stat_data stat_tmp = {.offset = offset};
3806         u64 tmp_val;
3807
3808         *val = 0;
3809         spin_lock(&kvm_lock);
3810         list_for_each_entry(kvm, &vm_list, vm_list) {
3811                 stat_tmp.kvm = kvm;
3812                 vm_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3813                 *val += tmp_val;
3814         }
3815         spin_unlock(&kvm_lock);
3816         return 0;
3817 }
3818
3819 static int vm_stat_clear(void *_offset, u64 val)
3820 {
3821         unsigned offset = (long)_offset;
3822         struct kvm *kvm;
3823         struct kvm_stat_data stat_tmp = {.offset = offset};
3824
3825         if (val)
3826                 return -EINVAL;
3827
3828         spin_lock(&kvm_lock);
3829         list_for_each_entry(kvm, &vm_list, vm_list) {
3830                 stat_tmp.kvm = kvm;
3831                 vm_stat_clear_per_vm((void *)&stat_tmp, 0);
3832         }
3833         spin_unlock(&kvm_lock);
3834
3835         return 0;
3836 }
3837
3838 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
3839
3840 static int vcpu_stat_get(void *_offset, u64 *val)
3841 {
3842         unsigned offset = (long)_offset;
3843         struct kvm *kvm;
3844         struct kvm_stat_data stat_tmp = {.offset = offset};
3845         u64 tmp_val;
3846
3847         *val = 0;
3848         spin_lock(&kvm_lock);
3849         list_for_each_entry(kvm, &vm_list, vm_list) {
3850                 stat_tmp.kvm = kvm;
3851                 vcpu_stat_get_per_vm((void *)&stat_tmp, &tmp_val);
3852                 *val += tmp_val;
3853         }
3854         spin_unlock(&kvm_lock);
3855         return 0;
3856 }
3857
3858 static int vcpu_stat_clear(void *_offset, u64 val)
3859 {
3860         unsigned offset = (long)_offset;
3861         struct kvm *kvm;
3862         struct kvm_stat_data stat_tmp = {.offset = offset};
3863
3864         if (val)
3865                 return -EINVAL;
3866
3867         spin_lock(&kvm_lock);
3868         list_for_each_entry(kvm, &vm_list, vm_list) {
3869                 stat_tmp.kvm = kvm;
3870                 vcpu_stat_clear_per_vm((void *)&stat_tmp, 0);
3871         }
3872         spin_unlock(&kvm_lock);
3873
3874         return 0;
3875 }
3876
3877 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
3878                         "%llu\n");
3879
3880 static const struct file_operations *stat_fops[] = {
3881         [KVM_STAT_VCPU] = &vcpu_stat_fops,
3882         [KVM_STAT_VM]   = &vm_stat_fops,
3883 };
3884
3885 static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
3886 {
3887         struct kobj_uevent_env *env;
3888         unsigned long long created, active;
3889
3890         if (!kvm_dev.this_device || !kvm)
3891                 return;
3892
3893         spin_lock(&kvm_lock);
3894         if (type == KVM_EVENT_CREATE_VM) {
3895                 kvm_createvm_count++;
3896                 kvm_active_vms++;
3897         } else if (type == KVM_EVENT_DESTROY_VM) {
3898                 kvm_active_vms--;
3899         }
3900         created = kvm_createvm_count;
3901         active = kvm_active_vms;
3902         spin_unlock(&kvm_lock);
3903
3904         env = kzalloc(sizeof(*env), GFP_KERNEL);
3905         if (!env)
3906                 return;
3907
3908         add_uevent_var(env, "CREATED=%llu", created);
3909         add_uevent_var(env, "COUNT=%llu", active);
3910
3911         if (type == KVM_EVENT_CREATE_VM) {
3912                 add_uevent_var(env, "EVENT=create");
3913                 kvm->userspace_pid = task_pid_nr(current);
3914         } else if (type == KVM_EVENT_DESTROY_VM) {
3915                 add_uevent_var(env, "EVENT=destroy");
3916         }
3917         add_uevent_var(env, "PID=%d", kvm->userspace_pid);
3918
3919         if (kvm->debugfs_dentry) {
3920                 char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);
3921
3922                 if (p) {
3923                         tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
3924                         if (!IS_ERR(tmp))
3925                                 add_uevent_var(env, "STATS_PATH=%s", tmp);
3926                         kfree(p);
3927                 }
3928         }
3929         /* no need for checks, since we are adding at most only 5 keys */
3930         env->envp[env->envp_idx++] = NULL;
3931         kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
3932         kfree(env);
3933 }
3934
3935 static void kvm_init_debug(void)
3936 {
3937         struct kvm_stats_debugfs_item *p;
3938
3939         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3940
3941         kvm_debugfs_num_entries = 0;
3942         for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) {
3943                 debugfs_create_file(p->name, 0644, kvm_debugfs_dir,
3944                                     (void *)(long)p->offset,
3945                                     stat_fops[p->kind]);
3946         }
3947 }
3948
3949 static int kvm_suspend(void)
3950 {
3951         if (kvm_usage_count)
3952                 hardware_disable_nolock(NULL);
3953         return 0;
3954 }
3955
3956 static void kvm_resume(void)
3957 {
3958         if (kvm_usage_count) {
3959                 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3960                 hardware_enable_nolock(NULL);
3961         }
3962 }
3963
3964 static struct syscore_ops kvm_syscore_ops = {
3965         .suspend = kvm_suspend,
3966         .resume = kvm_resume,
3967 };
3968
3969 static inline
3970 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3971 {
3972         return container_of(pn, struct kvm_vcpu, preempt_notifier);
3973 }
3974
3975 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3976 {
3977         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3978
3979         if (vcpu->preempted)
3980                 vcpu->preempted = false;
3981
3982         kvm_arch_sched_in(vcpu, cpu);
3983
3984         kvm_arch_vcpu_load(vcpu, cpu);
3985 }
3986
3987 static void kvm_sched_out(struct preempt_notifier *pn,
3988                           struct task_struct *next)
3989 {
3990         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3991
3992         if (current->state == TASK_RUNNING)
3993                 vcpu->preempted = true;
3994         kvm_arch_vcpu_put(vcpu);
3995 }
3996
3997 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3998                   struct module *module)
3999 {
4000         int r;
4001         int cpu;
4002
4003         r = kvm_arch_init(opaque);
4004         if (r)
4005                 goto out_fail;
4006
4007         /*
4008          * kvm_arch_init makes sure there's at most one caller
4009          * for architectures that support multiple implementations,
4010          * like intel and amd on x86.
4011          * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
4012          * conflicts in case kvm is already setup for another implementation.
4013          */
4014         r = kvm_irqfd_init();
4015         if (r)
4016                 goto out_irqfd;
4017
4018         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
4019                 r = -ENOMEM;
4020                 goto out_free_0;
4021         }
4022
4023         r = kvm_arch_hardware_setup();
4024         if (r < 0)
4025                 goto out_free_0a;
4026
4027         for_each_online_cpu(cpu) {
4028                 smp_call_function_single(cpu,
4029                                 kvm_arch_check_processor_compat,
4030                                 &r, 1);
4031                 if (r < 0)
4032                         goto out_free_1;
4033         }
4034
4035         r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "kvm/cpu:starting",
4036                                       kvm_starting_cpu, kvm_dying_cpu);
4037         if (r)
4038                 goto out_free_2;
4039         register_reboot_notifier(&kvm_reboot_notifier);
4040
4041         /* A kmem cache lets us meet the alignment requirements of fx_save. */
4042         if (!vcpu_align)
4043                 vcpu_align = __alignof__(struct kvm_vcpu);
4044         kvm_vcpu_cache =
4045                 kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
4046                                            SLAB_ACCOUNT,
4047                                            offsetof(struct kvm_vcpu, arch),
4048                                            sizeof_field(struct kvm_vcpu, arch),
4049                                            NULL);
4050         if (!kvm_vcpu_cache) {
4051                 r = -ENOMEM;
4052                 goto out_free_3;
4053         }
4054
4055         r = kvm_async_pf_init();
4056         if (r)
4057                 goto out_free;
4058
4059         kvm_chardev_ops.owner = module;
4060         kvm_vm_fops.owner = module;
4061         kvm_vcpu_fops.owner = module;
4062
4063         r = misc_register(&kvm_dev);
4064         if (r) {
4065                 pr_err("kvm: misc device register failed\n");
4066                 goto out_unreg;
4067         }
4068
4069         register_syscore_ops(&kvm_syscore_ops);
4070
4071         kvm_preempt_ops.sched_in = kvm_sched_in;
4072         kvm_preempt_ops.sched_out = kvm_sched_out;
4073
4074         kvm_init_debug();
4075
4076         r = kvm_vfio_ops_init();
4077         WARN_ON(r);
4078
4079         return 0;
4080
4081 out_unreg:
4082         kvm_async_pf_deinit();
4083 out_free:
4084         kmem_cache_destroy(kvm_vcpu_cache);
4085 out_free_3:
4086         unregister_reboot_notifier(&kvm_reboot_notifier);
4087         cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4088 out_free_2:
4089 out_free_1:
4090         kvm_arch_hardware_unsetup();
4091 out_free_0a:
4092         free_cpumask_var(cpus_hardware_enabled);
4093 out_free_0:
4094         kvm_irqfd_exit();
4095 out_irqfd:
4096         kvm_arch_exit();
4097 out_fail:
4098         return r;
4099 }
4100 EXPORT_SYMBOL_GPL(kvm_init);
4101
4102 void kvm_exit(void)
4103 {
4104         debugfs_remove_recursive(kvm_debugfs_dir);
4105         misc_deregister(&kvm_dev);
4106         kmem_cache_destroy(kvm_vcpu_cache);
4107         kvm_async_pf_deinit();
4108         unregister_syscore_ops(&kvm_syscore_ops);
4109         unregister_reboot_notifier(&kvm_reboot_notifier);
4110         cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
4111         on_each_cpu(hardware_disable_nolock, NULL, 1);
4112         kvm_arch_hardware_unsetup();
4113         kvm_arch_exit();
4114         kvm_irqfd_exit();
4115         free_cpumask_var(cpus_hardware_enabled);
4116         kvm_vfio_ops_exit();
4117 }
4118 EXPORT_SYMBOL_GPL(kvm_exit);