Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
[muen/linux.git] / arch / x86 / kvm / svm.c
1 /*
2  * Kernel-based Virtual Machine driver for Linux
3  *
4  * AMD SVM support
5  *
6  * Copyright (C) 2006 Qumranet, Inc.
7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
8  *
9  * Authors:
10  *   Yaniv Kamay  <yaniv@qumranet.com>
11  *   Avi Kivity   <avi@qumranet.com>
12  *
13  * This work is licensed under the terms of the GNU GPL, version 2.  See
14  * the COPYING file in the top-level directory.
15  *
16  */
17
18 #define pr_fmt(fmt) "SVM: " fmt
19
20 #include <linux/kvm_host.h>
21
22 #include "irq.h"
23 #include "mmu.h"
24 #include "kvm_cache_regs.h"
25 #include "x86.h"
26 #include "cpuid.h"
27 #include "pmu.h"
28
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/kernel.h>
32 #include <linux/vmalloc.h>
33 #include <linux/highmem.h>
34 #include <linux/sched.h>
35 #include <linux/trace_events.h>
36 #include <linux/slab.h>
37 #include <linux/amd-iommu.h>
38 #include <linux/hashtable.h>
39 #include <linux/frame.h>
40 #include <linux/psp-sev.h>
41 #include <linux/file.h>
42 #include <linux/pagemap.h>
43 #include <linux/swap.h>
44
45 #include <asm/apic.h>
46 #include <asm/perf_event.h>
47 #include <asm/tlbflush.h>
48 #include <asm/desc.h>
49 #include <asm/debugreg.h>
50 #include <asm/kvm_para.h>
51 #include <asm/irq_remapping.h>
52 #include <asm/nospec-branch.h>
53
54 #include <asm/virtext.h>
55 #include "trace.h"
56
57 #define __ex(x) __kvm_handle_fault_on_reboot(x)
58
59 MODULE_AUTHOR("Qumranet");
60 MODULE_LICENSE("GPL");
61
62 static const struct x86_cpu_id svm_cpu_id[] = {
63         X86_FEATURE_MATCH(X86_FEATURE_SVM),
64         {}
65 };
66 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
67
68 #define IOPM_ALLOC_ORDER 2
69 #define MSRPM_ALLOC_ORDER 1
70
71 #define SEG_TYPE_LDT 2
72 #define SEG_TYPE_BUSY_TSS16 3
73
74 #define SVM_FEATURE_NPT            (1 <<  0)
75 #define SVM_FEATURE_LBRV           (1 <<  1)
76 #define SVM_FEATURE_SVML           (1 <<  2)
77 #define SVM_FEATURE_NRIP           (1 <<  3)
78 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
79 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
80 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
81 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
82 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
83
84 #define SVM_AVIC_DOORBELL       0xc001011b
85
86 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
87 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
88 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
89
90 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
91
92 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
93 #define TSC_RATIO_MIN           0x0000000000000001ULL
94 #define TSC_RATIO_MAX           0x000000ffffffffffULL
95
96 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
97
98 /*
99  * 0xff is broadcast, so the max index allowed for physical APIC ID
100  * table is 0xfe.  APIC IDs above 0xff are reserved.
101  */
102 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
103
104 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
105 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
106 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
107
108 /* AVIC GATAG is encoded using VM and VCPU IDs */
109 #define AVIC_VCPU_ID_BITS               8
110 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
111
112 #define AVIC_VM_ID_BITS                 24
113 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
114 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
115
116 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
117                                                 (y & AVIC_VCPU_ID_MASK))
118 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
119 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
120
121 static bool erratum_383_found __read_mostly;
122
123 static const u32 host_save_user_msrs[] = {
124 #ifdef CONFIG_X86_64
125         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
126         MSR_FS_BASE,
127 #endif
128         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
129         MSR_TSC_AUX,
130 };
131
132 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
133
134 struct kvm_sev_info {
135         bool active;            /* SEV enabled guest */
136         unsigned int asid;      /* ASID used for this guest */
137         unsigned int handle;    /* SEV firmware handle */
138         int fd;                 /* SEV device fd */
139         unsigned long pages_locked; /* Number of pages locked */
140         struct list_head regions_list;  /* List of registered regions */
141 };
142
143 struct kvm_svm {
144         struct kvm kvm;
145
146         /* Struct members for AVIC */
147         u32 avic_vm_id;
148         u32 ldr_mode;
149         struct page *avic_logical_id_table_page;
150         struct page *avic_physical_id_table_page;
151         struct hlist_node hnode;
152
153         struct kvm_sev_info sev_info;
154 };
155
156 struct kvm_vcpu;
157
158 struct nested_state {
159         struct vmcb *hsave;
160         u64 hsave_msr;
161         u64 vm_cr_msr;
162         u64 vmcb;
163
164         /* These are the merged vectors */
165         u32 *msrpm;
166
167         /* gpa pointers to the real vectors */
168         u64 vmcb_msrpm;
169         u64 vmcb_iopm;
170
171         /* A VMEXIT is required but not yet emulated */
172         bool exit_required;
173
174         /* cache for intercepts of the guest */
175         u32 intercept_cr;
176         u32 intercept_dr;
177         u32 intercept_exceptions;
178         u64 intercept;
179
180         /* Nested Paging related state */
181         u64 nested_cr3;
182 };
183
184 #define MSRPM_OFFSETS   16
185 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
186
187 /*
188  * Set osvw_len to higher value when updated Revision Guides
189  * are published and we know what the new status bits are
190  */
191 static uint64_t osvw_len = 4, osvw_status;
192
193 struct vcpu_svm {
194         struct kvm_vcpu vcpu;
195         struct vmcb *vmcb;
196         unsigned long vmcb_pa;
197         struct svm_cpu_data *svm_data;
198         uint64_t asid_generation;
199         uint64_t sysenter_esp;
200         uint64_t sysenter_eip;
201         uint64_t tsc_aux;
202
203         u64 msr_decfg;
204
205         u64 next_rip;
206
207         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
208         struct {
209                 u16 fs;
210                 u16 gs;
211                 u16 ldt;
212                 u64 gs_base;
213         } host;
214
215         u64 spec_ctrl;
216
217         u32 *msrpm;
218
219         ulong nmi_iret_rip;
220
221         struct nested_state nested;
222
223         bool nmi_singlestep;
224         u64 nmi_singlestep_guest_rflags;
225
226         unsigned int3_injected;
227         unsigned long int3_rip;
228
229         /* cached guest cpuid flags for faster access */
230         bool nrips_enabled      : 1;
231
232         u32 ldr_reg;
233         struct page *avic_backing_page;
234         u64 *avic_physical_id_cache;
235         bool avic_is_running;
236
237         /*
238          * Per-vcpu list of struct amd_svm_iommu_ir:
239          * This is used mainly to store interrupt remapping information used
240          * when update the vcpu affinity. This avoids the need to scan for
241          * IRTE and try to match ga_tag in the IOMMU driver.
242          */
243         struct list_head ir_list;
244         spinlock_t ir_list_lock;
245
246         /* which host CPU was used for running this vcpu */
247         unsigned int last_cpu;
248 };
249
250 /*
251  * This is a wrapper of struct amd_iommu_ir_data.
252  */
253 struct amd_svm_iommu_ir {
254         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
255         void *data;             /* Storing pointer to struct amd_ir_data */
256 };
257
258 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
259 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
260
261 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
262 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
263 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
264 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
265
266 static DEFINE_PER_CPU(u64, current_tsc_ratio);
267 #define TSC_RATIO_DEFAULT       0x0100000000ULL
268
269 #define MSR_INVALID                     0xffffffffU
270
271 static const struct svm_direct_access_msrs {
272         u32 index;   /* Index of the MSR */
273         bool always; /* True if intercept is always on */
274 } direct_access_msrs[] = {
275         { .index = MSR_STAR,                            .always = true  },
276         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
277 #ifdef CONFIG_X86_64
278         { .index = MSR_GS_BASE,                         .always = true  },
279         { .index = MSR_FS_BASE,                         .always = true  },
280         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
281         { .index = MSR_LSTAR,                           .always = true  },
282         { .index = MSR_CSTAR,                           .always = true  },
283         { .index = MSR_SYSCALL_MASK,                    .always = true  },
284 #endif
285         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
286         { .index = MSR_IA32_PRED_CMD,                   .always = false },
287         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
288         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
289         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
290         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
291         { .index = MSR_INVALID,                         .always = false },
292 };
293
294 /* enable NPT for AMD64 and X86 with PAE */
295 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
296 static bool npt_enabled = true;
297 #else
298 static bool npt_enabled;
299 #endif
300
301 /*
302  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
303  * pause_filter_count: On processors that support Pause filtering(indicated
304  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
305  *      count value. On VMRUN this value is loaded into an internal counter.
306  *      Each time a pause instruction is executed, this counter is decremented
307  *      until it reaches zero at which time a #VMEXIT is generated if pause
308  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
309  *      Intercept Filtering for more details.
310  *      This also indicate if ple logic enabled.
311  *
312  * pause_filter_thresh: In addition, some processor families support advanced
313  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
314  *      the amount of time a guest is allowed to execute in a pause loop.
315  *      In this mode, a 16-bit pause filter threshold field is added in the
316  *      VMCB. The threshold value is a cycle count that is used to reset the
317  *      pause counter. As with simple pause filtering, VMRUN loads the pause
318  *      count value from VMCB into an internal counter. Then, on each pause
319  *      instruction the hardware checks the elapsed number of cycles since
320  *      the most recent pause instruction against the pause filter threshold.
321  *      If the elapsed cycle count is greater than the pause filter threshold,
322  *      then the internal pause count is reloaded from the VMCB and execution
323  *      continues. If the elapsed cycle count is less than the pause filter
324  *      threshold, then the internal pause count is decremented. If the count
325  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
326  *      triggered. If advanced pause filtering is supported and pause filter
327  *      threshold field is set to zero, the filter will operate in the simpler,
328  *      count only mode.
329  */
330
331 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
332 module_param(pause_filter_thresh, ushort, 0444);
333
334 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
335 module_param(pause_filter_count, ushort, 0444);
336
337 /* Default doubles per-vcpu window every exit. */
338 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
339 module_param(pause_filter_count_grow, ushort, 0444);
340
341 /* Default resets per-vcpu window every exit to pause_filter_count. */
342 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
343 module_param(pause_filter_count_shrink, ushort, 0444);
344
345 /* Default is to compute the maximum so we can never overflow. */
346 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
347 module_param(pause_filter_count_max, ushort, 0444);
348
349 /* allow nested paging (virtualized MMU) for all guests */
350 static int npt = true;
351 module_param(npt, int, S_IRUGO);
352
353 /* allow nested virtualization in KVM/SVM */
354 static int nested = true;
355 module_param(nested, int, S_IRUGO);
356
357 /* enable / disable AVIC */
358 static int avic;
359 #ifdef CONFIG_X86_LOCAL_APIC
360 module_param(avic, int, S_IRUGO);
361 #endif
362
363 /* enable/disable Virtual VMLOAD VMSAVE */
364 static int vls = true;
365 module_param(vls, int, 0444);
366
367 /* enable/disable Virtual GIF */
368 static int vgif = true;
369 module_param(vgif, int, 0444);
370
371 /* enable/disable SEV support */
372 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
373 module_param(sev, int, 0444);
374
375 static u8 rsm_ins_bytes[] = "\x0f\xaa";
376
377 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
378 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
379 static void svm_complete_interrupts(struct vcpu_svm *svm);
380
381 static int nested_svm_exit_handled(struct vcpu_svm *svm);
382 static int nested_svm_intercept(struct vcpu_svm *svm);
383 static int nested_svm_vmexit(struct vcpu_svm *svm);
384 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
385                                       bool has_error_code, u32 error_code);
386
387 enum {
388         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
389                             pause filter count */
390         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
391         VMCB_ASID,       /* ASID */
392         VMCB_INTR,       /* int_ctl, int_vector */
393         VMCB_NPT,        /* npt_en, nCR3, gPAT */
394         VMCB_CR,         /* CR0, CR3, CR4, EFER */
395         VMCB_DR,         /* DR6, DR7 */
396         VMCB_DT,         /* GDT, IDT */
397         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
398         VMCB_CR2,        /* CR2 only */
399         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
400         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
401                           * AVIC PHYSICAL_TABLE pointer,
402                           * AVIC LOGICAL_TABLE pointer
403                           */
404         VMCB_DIRTY_MAX,
405 };
406
407 /* TPR and CR2 are always written before VMRUN */
408 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
409
410 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
411
412 static unsigned int max_sev_asid;
413 static unsigned int min_sev_asid;
414 static unsigned long *sev_asid_bitmap;
415 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
416
417 struct enc_region {
418         struct list_head list;
419         unsigned long npages;
420         struct page **pages;
421         unsigned long uaddr;
422         unsigned long size;
423 };
424
425
426 static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
427 {
428         return container_of(kvm, struct kvm_svm, kvm);
429 }
430
431 static inline bool svm_sev_enabled(void)
432 {
433         return max_sev_asid;
434 }
435
436 static inline bool sev_guest(struct kvm *kvm)
437 {
438         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
439
440         return sev->active;
441 }
442
443 static inline int sev_get_asid(struct kvm *kvm)
444 {
445         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
446
447         return sev->asid;
448 }
449
450 static inline void mark_all_dirty(struct vmcb *vmcb)
451 {
452         vmcb->control.clean = 0;
453 }
454
455 static inline void mark_all_clean(struct vmcb *vmcb)
456 {
457         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
458                                & ~VMCB_ALWAYS_DIRTY_MASK;
459 }
460
461 static inline void mark_dirty(struct vmcb *vmcb, int bit)
462 {
463         vmcb->control.clean &= ~(1 << bit);
464 }
465
466 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
467 {
468         return container_of(vcpu, struct vcpu_svm, vcpu);
469 }
470
471 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
472 {
473         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
474         mark_dirty(svm->vmcb, VMCB_AVIC);
475 }
476
477 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
478 {
479         struct vcpu_svm *svm = to_svm(vcpu);
480         u64 *entry = svm->avic_physical_id_cache;
481
482         if (!entry)
483                 return false;
484
485         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
486 }
487
488 static void recalc_intercepts(struct vcpu_svm *svm)
489 {
490         struct vmcb_control_area *c, *h;
491         struct nested_state *g;
492
493         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
494
495         if (!is_guest_mode(&svm->vcpu))
496                 return;
497
498         c = &svm->vmcb->control;
499         h = &svm->nested.hsave->control;
500         g = &svm->nested;
501
502         c->intercept_cr = h->intercept_cr | g->intercept_cr;
503         c->intercept_dr = h->intercept_dr | g->intercept_dr;
504         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
505         c->intercept = h->intercept | g->intercept;
506 }
507
508 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
509 {
510         if (is_guest_mode(&svm->vcpu))
511                 return svm->nested.hsave;
512         else
513                 return svm->vmcb;
514 }
515
516 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
517 {
518         struct vmcb *vmcb = get_host_vmcb(svm);
519
520         vmcb->control.intercept_cr |= (1U << bit);
521
522         recalc_intercepts(svm);
523 }
524
525 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
526 {
527         struct vmcb *vmcb = get_host_vmcb(svm);
528
529         vmcb->control.intercept_cr &= ~(1U << bit);
530
531         recalc_intercepts(svm);
532 }
533
534 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
535 {
536         struct vmcb *vmcb = get_host_vmcb(svm);
537
538         return vmcb->control.intercept_cr & (1U << bit);
539 }
540
541 static inline void set_dr_intercepts(struct vcpu_svm *svm)
542 {
543         struct vmcb *vmcb = get_host_vmcb(svm);
544
545         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
546                 | (1 << INTERCEPT_DR1_READ)
547                 | (1 << INTERCEPT_DR2_READ)
548                 | (1 << INTERCEPT_DR3_READ)
549                 | (1 << INTERCEPT_DR4_READ)
550                 | (1 << INTERCEPT_DR5_READ)
551                 | (1 << INTERCEPT_DR6_READ)
552                 | (1 << INTERCEPT_DR7_READ)
553                 | (1 << INTERCEPT_DR0_WRITE)
554                 | (1 << INTERCEPT_DR1_WRITE)
555                 | (1 << INTERCEPT_DR2_WRITE)
556                 | (1 << INTERCEPT_DR3_WRITE)
557                 | (1 << INTERCEPT_DR4_WRITE)
558                 | (1 << INTERCEPT_DR5_WRITE)
559                 | (1 << INTERCEPT_DR6_WRITE)
560                 | (1 << INTERCEPT_DR7_WRITE);
561
562         recalc_intercepts(svm);
563 }
564
565 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
566 {
567         struct vmcb *vmcb = get_host_vmcb(svm);
568
569         vmcb->control.intercept_dr = 0;
570
571         recalc_intercepts(svm);
572 }
573
574 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
575 {
576         struct vmcb *vmcb = get_host_vmcb(svm);
577
578         vmcb->control.intercept_exceptions |= (1U << bit);
579
580         recalc_intercepts(svm);
581 }
582
583 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
584 {
585         struct vmcb *vmcb = get_host_vmcb(svm);
586
587         vmcb->control.intercept_exceptions &= ~(1U << bit);
588
589         recalc_intercepts(svm);
590 }
591
592 static inline void set_intercept(struct vcpu_svm *svm, int bit)
593 {
594         struct vmcb *vmcb = get_host_vmcb(svm);
595
596         vmcb->control.intercept |= (1ULL << bit);
597
598         recalc_intercepts(svm);
599 }
600
601 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
602 {
603         struct vmcb *vmcb = get_host_vmcb(svm);
604
605         vmcb->control.intercept &= ~(1ULL << bit);
606
607         recalc_intercepts(svm);
608 }
609
610 static inline bool vgif_enabled(struct vcpu_svm *svm)
611 {
612         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
613 }
614
615 static inline void enable_gif(struct vcpu_svm *svm)
616 {
617         if (vgif_enabled(svm))
618                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
619         else
620                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
621 }
622
623 static inline void disable_gif(struct vcpu_svm *svm)
624 {
625         if (vgif_enabled(svm))
626                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
627         else
628                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
629 }
630
631 static inline bool gif_set(struct vcpu_svm *svm)
632 {
633         if (vgif_enabled(svm))
634                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
635         else
636                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
637 }
638
639 static unsigned long iopm_base;
640
641 struct kvm_ldttss_desc {
642         u16 limit0;
643         u16 base0;
644         unsigned base1:8, type:5, dpl:2, p:1;
645         unsigned limit1:4, zero0:3, g:1, base2:8;
646         u32 base3;
647         u32 zero1;
648 } __attribute__((packed));
649
650 struct svm_cpu_data {
651         int cpu;
652
653         u64 asid_generation;
654         u32 max_asid;
655         u32 next_asid;
656         u32 min_asid;
657         struct kvm_ldttss_desc *tss_desc;
658
659         struct page *save_area;
660         struct vmcb *current_vmcb;
661
662         /* index = sev_asid, value = vmcb pointer */
663         struct vmcb **sev_vmcbs;
664 };
665
666 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
667
668 struct svm_init_data {
669         int cpu;
670         int r;
671 };
672
673 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
674
675 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
676 #define MSRS_RANGE_SIZE 2048
677 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
678
679 static u32 svm_msrpm_offset(u32 msr)
680 {
681         u32 offset;
682         int i;
683
684         for (i = 0; i < NUM_MSR_MAPS; i++) {
685                 if (msr < msrpm_ranges[i] ||
686                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
687                         continue;
688
689                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
690                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
691
692                 /* Now we have the u8 offset - but need the u32 offset */
693                 return offset / 4;
694         }
695
696         /* MSR not in any range */
697         return MSR_INVALID;
698 }
699
700 #define MAX_INST_SIZE 15
701
702 static inline void clgi(void)
703 {
704         asm volatile (__ex(SVM_CLGI));
705 }
706
707 static inline void stgi(void)
708 {
709         asm volatile (__ex(SVM_STGI));
710 }
711
712 static inline void invlpga(unsigned long addr, u32 asid)
713 {
714         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
715 }
716
717 static int get_npt_level(struct kvm_vcpu *vcpu)
718 {
719 #ifdef CONFIG_X86_64
720         return PT64_ROOT_4LEVEL;
721 #else
722         return PT32E_ROOT_LEVEL;
723 #endif
724 }
725
726 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
727 {
728         vcpu->arch.efer = efer;
729         if (!npt_enabled && !(efer & EFER_LMA))
730                 efer &= ~EFER_LME;
731
732         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
733         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
734 }
735
736 static int is_external_interrupt(u32 info)
737 {
738         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
739         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
740 }
741
742 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
743 {
744         struct vcpu_svm *svm = to_svm(vcpu);
745         u32 ret = 0;
746
747         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
748                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
749         return ret;
750 }
751
752 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
753 {
754         struct vcpu_svm *svm = to_svm(vcpu);
755
756         if (mask == 0)
757                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
758         else
759                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
760
761 }
762
763 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
764 {
765         struct vcpu_svm *svm = to_svm(vcpu);
766
767         if (svm->vmcb->control.next_rip != 0) {
768                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
769                 svm->next_rip = svm->vmcb->control.next_rip;
770         }
771
772         if (!svm->next_rip) {
773                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
774                                 EMULATE_DONE)
775                         printk(KERN_DEBUG "%s: NOP\n", __func__);
776                 return;
777         }
778         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
779                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
780                        __func__, kvm_rip_read(vcpu), svm->next_rip);
781
782         kvm_rip_write(vcpu, svm->next_rip);
783         svm_set_interrupt_shadow(vcpu, 0);
784 }
785
786 static void svm_queue_exception(struct kvm_vcpu *vcpu)
787 {
788         struct vcpu_svm *svm = to_svm(vcpu);
789         unsigned nr = vcpu->arch.exception.nr;
790         bool has_error_code = vcpu->arch.exception.has_error_code;
791         bool reinject = vcpu->arch.exception.injected;
792         u32 error_code = vcpu->arch.exception.error_code;
793
794         /*
795          * If we are within a nested VM we'd better #VMEXIT and let the guest
796          * handle the exception
797          */
798         if (!reinject &&
799             nested_svm_check_exception(svm, nr, has_error_code, error_code))
800                 return;
801
802         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
803                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
804
805                 /*
806                  * For guest debugging where we have to reinject #BP if some
807                  * INT3 is guest-owned:
808                  * Emulate nRIP by moving RIP forward. Will fail if injection
809                  * raises a fault that is not intercepted. Still better than
810                  * failing in all cases.
811                  */
812                 skip_emulated_instruction(&svm->vcpu);
813                 rip = kvm_rip_read(&svm->vcpu);
814                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
815                 svm->int3_injected = rip - old_rip;
816         }
817
818         svm->vmcb->control.event_inj = nr
819                 | SVM_EVTINJ_VALID
820                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
821                 | SVM_EVTINJ_TYPE_EXEPT;
822         svm->vmcb->control.event_inj_err = error_code;
823 }
824
825 static void svm_init_erratum_383(void)
826 {
827         u32 low, high;
828         int err;
829         u64 val;
830
831         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
832                 return;
833
834         /* Use _safe variants to not break nested virtualization */
835         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
836         if (err)
837                 return;
838
839         val |= (1ULL << 47);
840
841         low  = lower_32_bits(val);
842         high = upper_32_bits(val);
843
844         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
845
846         erratum_383_found = true;
847 }
848
849 static void svm_init_osvw(struct kvm_vcpu *vcpu)
850 {
851         /*
852          * Guests should see errata 400 and 415 as fixed (assuming that
853          * HLT and IO instructions are intercepted).
854          */
855         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
856         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
857
858         /*
859          * By increasing VCPU's osvw.length to 3 we are telling the guest that
860          * all osvw.status bits inside that length, including bit 0 (which is
861          * reserved for erratum 298), are valid. However, if host processor's
862          * osvw_len is 0 then osvw_status[0] carries no information. We need to
863          * be conservative here and therefore we tell the guest that erratum 298
864          * is present (because we really don't know).
865          */
866         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
867                 vcpu->arch.osvw.status |= 1;
868 }
869
870 static int has_svm(void)
871 {
872         const char *msg;
873
874         if (!cpu_has_svm(&msg)) {
875                 printk(KERN_INFO "has_svm: %s\n", msg);
876                 return 0;
877         }
878
879         return 1;
880 }
881
882 static void svm_hardware_disable(void)
883 {
884         /* Make sure we clean up behind us */
885         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
886                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
887
888         cpu_svm_disable();
889
890         amd_pmu_disable_virt();
891 }
892
893 static int svm_hardware_enable(void)
894 {
895
896         struct svm_cpu_data *sd;
897         uint64_t efer;
898         struct desc_struct *gdt;
899         int me = raw_smp_processor_id();
900
901         rdmsrl(MSR_EFER, efer);
902         if (efer & EFER_SVME)
903                 return -EBUSY;
904
905         if (!has_svm()) {
906                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
907                 return -EINVAL;
908         }
909         sd = per_cpu(svm_data, me);
910         if (!sd) {
911                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
912                 return -EINVAL;
913         }
914
915         sd->asid_generation = 1;
916         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
917         sd->next_asid = sd->max_asid + 1;
918         sd->min_asid = max_sev_asid + 1;
919
920         gdt = get_current_gdt_rw();
921         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
922
923         wrmsrl(MSR_EFER, efer | EFER_SVME);
924
925         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
926
927         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
928                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
929                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
930         }
931
932
933         /*
934          * Get OSVW bits.
935          *
936          * Note that it is possible to have a system with mixed processor
937          * revisions and therefore different OSVW bits. If bits are not the same
938          * on different processors then choose the worst case (i.e. if erratum
939          * is present on one processor and not on another then assume that the
940          * erratum is present everywhere).
941          */
942         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
943                 uint64_t len, status = 0;
944                 int err;
945
946                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
947                 if (!err)
948                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
949                                                       &err);
950
951                 if (err)
952                         osvw_status = osvw_len = 0;
953                 else {
954                         if (len < osvw_len)
955                                 osvw_len = len;
956                         osvw_status |= status;
957                         osvw_status &= (1ULL << osvw_len) - 1;
958                 }
959         } else
960                 osvw_status = osvw_len = 0;
961
962         svm_init_erratum_383();
963
964         amd_pmu_enable_virt();
965
966         return 0;
967 }
968
969 static void svm_cpu_uninit(int cpu)
970 {
971         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
972
973         if (!sd)
974                 return;
975
976         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
977         kfree(sd->sev_vmcbs);
978         __free_page(sd->save_area);
979         kfree(sd);
980 }
981
982 static int svm_cpu_init(int cpu)
983 {
984         struct svm_cpu_data *sd;
985         int r;
986
987         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
988         if (!sd)
989                 return -ENOMEM;
990         sd->cpu = cpu;
991         r = -ENOMEM;
992         sd->save_area = alloc_page(GFP_KERNEL);
993         if (!sd->save_area)
994                 goto err_1;
995
996         if (svm_sev_enabled()) {
997                 r = -ENOMEM;
998                 sd->sev_vmcbs = kmalloc((max_sev_asid + 1) * sizeof(void *), GFP_KERNEL);
999                 if (!sd->sev_vmcbs)
1000                         goto err_1;
1001         }
1002
1003         per_cpu(svm_data, cpu) = sd;
1004
1005         return 0;
1006
1007 err_1:
1008         kfree(sd);
1009         return r;
1010
1011 }
1012
1013 static bool valid_msr_intercept(u32 index)
1014 {
1015         int i;
1016
1017         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1018                 if (direct_access_msrs[i].index == index)
1019                         return true;
1020
1021         return false;
1022 }
1023
1024 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1025 {
1026         u8 bit_write;
1027         unsigned long tmp;
1028         u32 offset;
1029         u32 *msrpm;
1030
1031         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1032                                       to_svm(vcpu)->msrpm;
1033
1034         offset    = svm_msrpm_offset(msr);
1035         bit_write = 2 * (msr & 0x0f) + 1;
1036         tmp       = msrpm[offset];
1037
1038         BUG_ON(offset == MSR_INVALID);
1039
1040         return !!test_bit(bit_write,  &tmp);
1041 }
1042
1043 static void set_msr_interception(u32 *msrpm, unsigned msr,
1044                                  int read, int write)
1045 {
1046         u8 bit_read, bit_write;
1047         unsigned long tmp;
1048         u32 offset;
1049
1050         /*
1051          * If this warning triggers extend the direct_access_msrs list at the
1052          * beginning of the file
1053          */
1054         WARN_ON(!valid_msr_intercept(msr));
1055
1056         offset    = svm_msrpm_offset(msr);
1057         bit_read  = 2 * (msr & 0x0f);
1058         bit_write = 2 * (msr & 0x0f) + 1;
1059         tmp       = msrpm[offset];
1060
1061         BUG_ON(offset == MSR_INVALID);
1062
1063         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
1064         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1065
1066         msrpm[offset] = tmp;
1067 }
1068
1069 static void svm_vcpu_init_msrpm(u32 *msrpm)
1070 {
1071         int i;
1072
1073         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1074
1075         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1076                 if (!direct_access_msrs[i].always)
1077                         continue;
1078
1079                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1080         }
1081 }
1082
1083 static void add_msr_offset(u32 offset)
1084 {
1085         int i;
1086
1087         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1088
1089                 /* Offset already in list? */
1090                 if (msrpm_offsets[i] == offset)
1091                         return;
1092
1093                 /* Slot used by another offset? */
1094                 if (msrpm_offsets[i] != MSR_INVALID)
1095                         continue;
1096
1097                 /* Add offset to list */
1098                 msrpm_offsets[i] = offset;
1099
1100                 return;
1101         }
1102
1103         /*
1104          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1105          * increase MSRPM_OFFSETS in this case.
1106          */
1107         BUG();
1108 }
1109
1110 static void init_msrpm_offsets(void)
1111 {
1112         int i;
1113
1114         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1115
1116         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1117                 u32 offset;
1118
1119                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1120                 BUG_ON(offset == MSR_INVALID);
1121
1122                 add_msr_offset(offset);
1123         }
1124 }
1125
1126 static void svm_enable_lbrv(struct vcpu_svm *svm)
1127 {
1128         u32 *msrpm = svm->msrpm;
1129
1130         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1131         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1132         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1133         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1134         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1135 }
1136
1137 static void svm_disable_lbrv(struct vcpu_svm *svm)
1138 {
1139         u32 *msrpm = svm->msrpm;
1140
1141         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1142         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1143         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1144         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1145         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1146 }
1147
1148 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1149 {
1150         svm->nmi_singlestep = false;
1151
1152         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1153                 /* Clear our flags if they were not set by the guest */
1154                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1155                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1156                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1157                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1158         }
1159 }
1160
1161 /* Note:
1162  * This hash table is used to map VM_ID to a struct kvm_svm,
1163  * when handling AMD IOMMU GALOG notification to schedule in
1164  * a particular vCPU.
1165  */
1166 #define SVM_VM_DATA_HASH_BITS   8
1167 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1168 static u32 next_vm_id = 0;
1169 static bool next_vm_id_wrapped = 0;
1170 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1171
1172 /* Note:
1173  * This function is called from IOMMU driver to notify
1174  * SVM to schedule in a particular vCPU of a particular VM.
1175  */
1176 static int avic_ga_log_notifier(u32 ga_tag)
1177 {
1178         unsigned long flags;
1179         struct kvm_svm *kvm_svm;
1180         struct kvm_vcpu *vcpu = NULL;
1181         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1182         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1183
1184         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1185
1186         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1187         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1188                 if (kvm_svm->avic_vm_id != vm_id)
1189                         continue;
1190                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
1191                 break;
1192         }
1193         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1194
1195         /* Note:
1196          * At this point, the IOMMU should have already set the pending
1197          * bit in the vAPIC backing page. So, we just need to schedule
1198          * in the vcpu.
1199          */
1200         if (vcpu)
1201                 kvm_vcpu_wake_up(vcpu);
1202
1203         return 0;
1204 }
1205
1206 static __init int sev_hardware_setup(void)
1207 {
1208         struct sev_user_data_status *status;
1209         int rc;
1210
1211         /* Maximum number of encrypted guests supported simultaneously */
1212         max_sev_asid = cpuid_ecx(0x8000001F);
1213
1214         if (!max_sev_asid)
1215                 return 1;
1216
1217         /* Minimum ASID value that should be used for SEV guest */
1218         min_sev_asid = cpuid_edx(0x8000001F);
1219
1220         /* Initialize SEV ASID bitmap */
1221         sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid),
1222                                 sizeof(unsigned long), GFP_KERNEL);
1223         if (!sev_asid_bitmap)
1224                 return 1;
1225
1226         status = kmalloc(sizeof(*status), GFP_KERNEL);
1227         if (!status)
1228                 return 1;
1229
1230         /*
1231          * Check SEV platform status.
1232          *
1233          * PLATFORM_STATUS can be called in any state, if we failed to query
1234          * the PLATFORM status then either PSP firmware does not support SEV
1235          * feature or SEV firmware is dead.
1236          */
1237         rc = sev_platform_status(status, NULL);
1238         if (rc)
1239                 goto err;
1240
1241         pr_info("SEV supported\n");
1242
1243 err:
1244         kfree(status);
1245         return rc;
1246 }
1247
1248 static void grow_ple_window(struct kvm_vcpu *vcpu)
1249 {
1250         struct vcpu_svm *svm = to_svm(vcpu);
1251         struct vmcb_control_area *control = &svm->vmcb->control;
1252         int old = control->pause_filter_count;
1253
1254         control->pause_filter_count = __grow_ple_window(old,
1255                                                         pause_filter_count,
1256                                                         pause_filter_count_grow,
1257                                                         pause_filter_count_max);
1258
1259         if (control->pause_filter_count != old)
1260                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1261
1262         trace_kvm_ple_window_grow(vcpu->vcpu_id,
1263                                   control->pause_filter_count, old);
1264 }
1265
1266 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1267 {
1268         struct vcpu_svm *svm = to_svm(vcpu);
1269         struct vmcb_control_area *control = &svm->vmcb->control;
1270         int old = control->pause_filter_count;
1271
1272         control->pause_filter_count =
1273                                 __shrink_ple_window(old,
1274                                                     pause_filter_count,
1275                                                     pause_filter_count_shrink,
1276                                                     pause_filter_count);
1277         if (control->pause_filter_count != old)
1278                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1279
1280         trace_kvm_ple_window_shrink(vcpu->vcpu_id,
1281                                     control->pause_filter_count, old);
1282 }
1283
1284 static __init int svm_hardware_setup(void)
1285 {
1286         int cpu;
1287         struct page *iopm_pages;
1288         void *iopm_va;
1289         int r;
1290
1291         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1292
1293         if (!iopm_pages)
1294                 return -ENOMEM;
1295
1296         iopm_va = page_address(iopm_pages);
1297         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1298         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1299
1300         init_msrpm_offsets();
1301
1302         if (boot_cpu_has(X86_FEATURE_NX))
1303                 kvm_enable_efer_bits(EFER_NX);
1304
1305         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1306                 kvm_enable_efer_bits(EFER_FFXSR);
1307
1308         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1309                 kvm_has_tsc_control = true;
1310                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1311                 kvm_tsc_scaling_ratio_frac_bits = 32;
1312         }
1313
1314         /* Check for pause filtering support */
1315         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1316                 pause_filter_count = 0;
1317                 pause_filter_thresh = 0;
1318         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1319                 pause_filter_thresh = 0;
1320         }
1321
1322         if (nested) {
1323                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1324                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1325         }
1326
1327         if (sev) {
1328                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1329                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1330                         r = sev_hardware_setup();
1331                         if (r)
1332                                 sev = false;
1333                 } else {
1334                         sev = false;
1335                 }
1336         }
1337
1338         for_each_possible_cpu(cpu) {
1339                 r = svm_cpu_init(cpu);
1340                 if (r)
1341                         goto err;
1342         }
1343
1344         if (!boot_cpu_has(X86_FEATURE_NPT))
1345                 npt_enabled = false;
1346
1347         if (npt_enabled && !npt) {
1348                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1349                 npt_enabled = false;
1350         }
1351
1352         if (npt_enabled) {
1353                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1354                 kvm_enable_tdp();
1355         } else
1356                 kvm_disable_tdp();
1357
1358         if (avic) {
1359                 if (!npt_enabled ||
1360                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1361                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1362                         avic = false;
1363                 } else {
1364                         pr_info("AVIC enabled\n");
1365
1366                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1367                 }
1368         }
1369
1370         if (vls) {
1371                 if (!npt_enabled ||
1372                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1373                     !IS_ENABLED(CONFIG_X86_64)) {
1374                         vls = false;
1375                 } else {
1376                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1377                 }
1378         }
1379
1380         if (vgif) {
1381                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1382                         vgif = false;
1383                 else
1384                         pr_info("Virtual GIF supported\n");
1385         }
1386
1387         return 0;
1388
1389 err:
1390         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1391         iopm_base = 0;
1392         return r;
1393 }
1394
1395 static __exit void svm_hardware_unsetup(void)
1396 {
1397         int cpu;
1398
1399         if (svm_sev_enabled())
1400                 kfree(sev_asid_bitmap);
1401
1402         for_each_possible_cpu(cpu)
1403                 svm_cpu_uninit(cpu);
1404
1405         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1406         iopm_base = 0;
1407 }
1408
1409 static void init_seg(struct vmcb_seg *seg)
1410 {
1411         seg->selector = 0;
1412         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1413                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1414         seg->limit = 0xffff;
1415         seg->base = 0;
1416 }
1417
1418 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1419 {
1420         seg->selector = 0;
1421         seg->attrib = SVM_SELECTOR_P_MASK | type;
1422         seg->limit = 0xffff;
1423         seg->base = 0;
1424 }
1425
1426 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1427 {
1428         struct vcpu_svm *svm = to_svm(vcpu);
1429         u64 g_tsc_offset = 0;
1430
1431         if (is_guest_mode(vcpu)) {
1432                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1433                                svm->nested.hsave->control.tsc_offset;
1434                 svm->nested.hsave->control.tsc_offset = offset;
1435         } else
1436                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1437                                            svm->vmcb->control.tsc_offset,
1438                                            offset);
1439
1440         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1441
1442         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1443 }
1444
1445 static void avic_init_vmcb(struct vcpu_svm *svm)
1446 {
1447         struct vmcb *vmcb = svm->vmcb;
1448         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
1449         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1450         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1451         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
1452
1453         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1454         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1455         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1456         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1457         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1458 }
1459
1460 static void init_vmcb(struct vcpu_svm *svm)
1461 {
1462         struct vmcb_control_area *control = &svm->vmcb->control;
1463         struct vmcb_save_area *save = &svm->vmcb->save;
1464
1465         svm->vcpu.arch.hflags = 0;
1466
1467         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1468         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1469         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1470         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1471         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1472         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1473         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1474                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1475
1476         set_dr_intercepts(svm);
1477
1478         set_exception_intercept(svm, PF_VECTOR);
1479         set_exception_intercept(svm, UD_VECTOR);
1480         set_exception_intercept(svm, MC_VECTOR);
1481         set_exception_intercept(svm, AC_VECTOR);
1482         set_exception_intercept(svm, DB_VECTOR);
1483         /*
1484          * Guest access to VMware backdoor ports could legitimately
1485          * trigger #GP because of TSS I/O permission bitmap.
1486          * We intercept those #GP and allow access to them anyway
1487          * as VMware does.
1488          */
1489         if (enable_vmware_backdoor)
1490                 set_exception_intercept(svm, GP_VECTOR);
1491
1492         set_intercept(svm, INTERCEPT_INTR);
1493         set_intercept(svm, INTERCEPT_NMI);
1494         set_intercept(svm, INTERCEPT_SMI);
1495         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1496         set_intercept(svm, INTERCEPT_RDPMC);
1497         set_intercept(svm, INTERCEPT_CPUID);
1498         set_intercept(svm, INTERCEPT_INVD);
1499         set_intercept(svm, INTERCEPT_INVLPG);
1500         set_intercept(svm, INTERCEPT_INVLPGA);
1501         set_intercept(svm, INTERCEPT_IOIO_PROT);
1502         set_intercept(svm, INTERCEPT_MSR_PROT);
1503         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1504         set_intercept(svm, INTERCEPT_SHUTDOWN);
1505         set_intercept(svm, INTERCEPT_VMRUN);
1506         set_intercept(svm, INTERCEPT_VMMCALL);
1507         set_intercept(svm, INTERCEPT_VMLOAD);
1508         set_intercept(svm, INTERCEPT_VMSAVE);
1509         set_intercept(svm, INTERCEPT_STGI);
1510         set_intercept(svm, INTERCEPT_CLGI);
1511         set_intercept(svm, INTERCEPT_SKINIT);
1512         set_intercept(svm, INTERCEPT_WBINVD);
1513         set_intercept(svm, INTERCEPT_XSETBV);
1514         set_intercept(svm, INTERCEPT_RSM);
1515
1516         if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1517                 set_intercept(svm, INTERCEPT_MONITOR);
1518                 set_intercept(svm, INTERCEPT_MWAIT);
1519         }
1520
1521         if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1522                 set_intercept(svm, INTERCEPT_HLT);
1523
1524         control->iopm_base_pa = __sme_set(iopm_base);
1525         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1526         control->int_ctl = V_INTR_MASKING_MASK;
1527
1528         init_seg(&save->es);
1529         init_seg(&save->ss);
1530         init_seg(&save->ds);
1531         init_seg(&save->fs);
1532         init_seg(&save->gs);
1533
1534         save->cs.selector = 0xf000;
1535         save->cs.base = 0xffff0000;
1536         /* Executable/Readable Code Segment */
1537         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1538                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1539         save->cs.limit = 0xffff;
1540
1541         save->gdtr.limit = 0xffff;
1542         save->idtr.limit = 0xffff;
1543
1544         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1545         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1546
1547         svm_set_efer(&svm->vcpu, 0);
1548         save->dr6 = 0xffff0ff0;
1549         kvm_set_rflags(&svm->vcpu, 2);
1550         save->rip = 0x0000fff0;
1551         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1552
1553         /*
1554          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1555          * It also updates the guest-visible cr0 value.
1556          */
1557         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1558         kvm_mmu_reset_context(&svm->vcpu);
1559
1560         save->cr4 = X86_CR4_PAE;
1561         /* rdx = ?? */
1562
1563         if (npt_enabled) {
1564                 /* Setup VMCB for Nested Paging */
1565                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1566                 clr_intercept(svm, INTERCEPT_INVLPG);
1567                 clr_exception_intercept(svm, PF_VECTOR);
1568                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1569                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1570                 save->g_pat = svm->vcpu.arch.pat;
1571                 save->cr3 = 0;
1572                 save->cr4 = 0;
1573         }
1574         svm->asid_generation = 0;
1575
1576         svm->nested.vmcb = 0;
1577         svm->vcpu.arch.hflags = 0;
1578
1579         if (pause_filter_count) {
1580                 control->pause_filter_count = pause_filter_count;
1581                 if (pause_filter_thresh)
1582                         control->pause_filter_thresh = pause_filter_thresh;
1583                 set_intercept(svm, INTERCEPT_PAUSE);
1584         } else {
1585                 clr_intercept(svm, INTERCEPT_PAUSE);
1586         }
1587
1588         if (kvm_vcpu_apicv_active(&svm->vcpu))
1589                 avic_init_vmcb(svm);
1590
1591         /*
1592          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1593          * in VMCB and clear intercepts to avoid #VMEXIT.
1594          */
1595         if (vls) {
1596                 clr_intercept(svm, INTERCEPT_VMLOAD);
1597                 clr_intercept(svm, INTERCEPT_VMSAVE);
1598                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1599         }
1600
1601         if (vgif) {
1602                 clr_intercept(svm, INTERCEPT_STGI);
1603                 clr_intercept(svm, INTERCEPT_CLGI);
1604                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1605         }
1606
1607         if (sev_guest(svm->vcpu.kvm)) {
1608                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1609                 clr_exception_intercept(svm, UD_VECTOR);
1610         }
1611
1612         mark_all_dirty(svm->vmcb);
1613
1614         enable_gif(svm);
1615
1616 }
1617
1618 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1619                                        unsigned int index)
1620 {
1621         u64 *avic_physical_id_table;
1622         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1623
1624         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1625                 return NULL;
1626
1627         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
1628
1629         return &avic_physical_id_table[index];
1630 }
1631
1632 /**
1633  * Note:
1634  * AVIC hardware walks the nested page table to check permissions,
1635  * but does not use the SPA address specified in the leaf page
1636  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1637  * field of the VMCB. Therefore, we set up the
1638  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1639  */
1640 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1641 {
1642         struct kvm *kvm = vcpu->kvm;
1643         int ret;
1644
1645         if (kvm->arch.apic_access_page_done)
1646                 return 0;
1647
1648         ret = x86_set_memory_region(kvm,
1649                                     APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1650                                     APIC_DEFAULT_PHYS_BASE,
1651                                     PAGE_SIZE);
1652         if (ret)
1653                 return ret;
1654
1655         kvm->arch.apic_access_page_done = true;
1656         return 0;
1657 }
1658
1659 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1660 {
1661         int ret;
1662         u64 *entry, new_entry;
1663         int id = vcpu->vcpu_id;
1664         struct vcpu_svm *svm = to_svm(vcpu);
1665
1666         ret = avic_init_access_page(vcpu);
1667         if (ret)
1668                 return ret;
1669
1670         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1671                 return -EINVAL;
1672
1673         if (!svm->vcpu.arch.apic->regs)
1674                 return -EINVAL;
1675
1676         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1677
1678         /* Setting AVIC backing page address in the phy APIC ID table */
1679         entry = avic_get_physical_id_entry(vcpu, id);
1680         if (!entry)
1681                 return -EINVAL;
1682
1683         new_entry = READ_ONCE(*entry);
1684         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1685                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1686                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1687         WRITE_ONCE(*entry, new_entry);
1688
1689         svm->avic_physical_id_cache = entry;
1690
1691         return 0;
1692 }
1693
1694 static void __sev_asid_free(int asid)
1695 {
1696         struct svm_cpu_data *sd;
1697         int cpu, pos;
1698
1699         pos = asid - 1;
1700         clear_bit(pos, sev_asid_bitmap);
1701
1702         for_each_possible_cpu(cpu) {
1703                 sd = per_cpu(svm_data, cpu);
1704                 sd->sev_vmcbs[pos] = NULL;
1705         }
1706 }
1707
1708 static void sev_asid_free(struct kvm *kvm)
1709 {
1710         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1711
1712         __sev_asid_free(sev->asid);
1713 }
1714
1715 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1716 {
1717         struct sev_data_decommission *decommission;
1718         struct sev_data_deactivate *data;
1719
1720         if (!handle)
1721                 return;
1722
1723         data = kzalloc(sizeof(*data), GFP_KERNEL);
1724         if (!data)
1725                 return;
1726
1727         /* deactivate handle */
1728         data->handle = handle;
1729         sev_guest_deactivate(data, NULL);
1730
1731         wbinvd_on_all_cpus();
1732         sev_guest_df_flush(NULL);
1733         kfree(data);
1734
1735         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1736         if (!decommission)
1737                 return;
1738
1739         /* decommission handle */
1740         decommission->handle = handle;
1741         sev_guest_decommission(decommission, NULL);
1742
1743         kfree(decommission);
1744 }
1745
1746 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1747                                     unsigned long ulen, unsigned long *n,
1748                                     int write)
1749 {
1750         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1751         unsigned long npages, npinned, size;
1752         unsigned long locked, lock_limit;
1753         struct page **pages;
1754         int first, last;
1755
1756         /* Calculate number of pages. */
1757         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1758         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1759         npages = (last - first + 1);
1760
1761         locked = sev->pages_locked + npages;
1762         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1763         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1764                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1765                 return NULL;
1766         }
1767
1768         /* Avoid using vmalloc for smaller buffers. */
1769         size = npages * sizeof(struct page *);
1770         if (size > PAGE_SIZE)
1771                 pages = vmalloc(size);
1772         else
1773                 pages = kmalloc(size, GFP_KERNEL);
1774
1775         if (!pages)
1776                 return NULL;
1777
1778         /* Pin the user virtual address. */
1779         npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
1780         if (npinned != npages) {
1781                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1782                 goto err;
1783         }
1784
1785         *n = npages;
1786         sev->pages_locked = locked;
1787
1788         return pages;
1789
1790 err:
1791         if (npinned > 0)
1792                 release_pages(pages, npinned);
1793
1794         kvfree(pages);
1795         return NULL;
1796 }
1797
1798 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1799                              unsigned long npages)
1800 {
1801         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1802
1803         release_pages(pages, npages);
1804         kvfree(pages);
1805         sev->pages_locked -= npages;
1806 }
1807
1808 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1809 {
1810         uint8_t *page_virtual;
1811         unsigned long i;
1812
1813         if (npages == 0 || pages == NULL)
1814                 return;
1815
1816         for (i = 0; i < npages; i++) {
1817                 page_virtual = kmap_atomic(pages[i]);
1818                 clflush_cache_range(page_virtual, PAGE_SIZE);
1819                 kunmap_atomic(page_virtual);
1820         }
1821 }
1822
1823 static void __unregister_enc_region_locked(struct kvm *kvm,
1824                                            struct enc_region *region)
1825 {
1826         /*
1827          * The guest may change the memory encryption attribute from C=0 -> C=1
1828          * or vice versa for this memory range. Lets make sure caches are
1829          * flushed to ensure that guest data gets written into memory with
1830          * correct C-bit.
1831          */
1832         sev_clflush_pages(region->pages, region->npages);
1833
1834         sev_unpin_memory(kvm, region->pages, region->npages);
1835         list_del(&region->list);
1836         kfree(region);
1837 }
1838
1839 static struct kvm *svm_vm_alloc(void)
1840 {
1841         struct kvm_svm *kvm_svm = kzalloc(sizeof(struct kvm_svm), GFP_KERNEL);
1842         return &kvm_svm->kvm;
1843 }
1844
1845 static void svm_vm_free(struct kvm *kvm)
1846 {
1847         kfree(to_kvm_svm(kvm));
1848 }
1849
1850 static void sev_vm_destroy(struct kvm *kvm)
1851 {
1852         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1853         struct list_head *head = &sev->regions_list;
1854         struct list_head *pos, *q;
1855
1856         if (!sev_guest(kvm))
1857                 return;
1858
1859         mutex_lock(&kvm->lock);
1860
1861         /*
1862          * if userspace was terminated before unregistering the memory regions
1863          * then lets unpin all the registered memory.
1864          */
1865         if (!list_empty(head)) {
1866                 list_for_each_safe(pos, q, head) {
1867                         __unregister_enc_region_locked(kvm,
1868                                 list_entry(pos, struct enc_region, list));
1869                 }
1870         }
1871
1872         mutex_unlock(&kvm->lock);
1873
1874         sev_unbind_asid(kvm, sev->handle);
1875         sev_asid_free(kvm);
1876 }
1877
1878 static void avic_vm_destroy(struct kvm *kvm)
1879 {
1880         unsigned long flags;
1881         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1882
1883         if (!avic)
1884                 return;
1885
1886         if (kvm_svm->avic_logical_id_table_page)
1887                 __free_page(kvm_svm->avic_logical_id_table_page);
1888         if (kvm_svm->avic_physical_id_table_page)
1889                 __free_page(kvm_svm->avic_physical_id_table_page);
1890
1891         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1892         hash_del(&kvm_svm->hnode);
1893         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1894 }
1895
1896 static void svm_vm_destroy(struct kvm *kvm)
1897 {
1898         avic_vm_destroy(kvm);
1899         sev_vm_destroy(kvm);
1900 }
1901
1902 static int avic_vm_init(struct kvm *kvm)
1903 {
1904         unsigned long flags;
1905         int err = -ENOMEM;
1906         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1907         struct kvm_svm *k2;
1908         struct page *p_page;
1909         struct page *l_page;
1910         u32 vm_id;
1911
1912         if (!avic)
1913                 return 0;
1914
1915         /* Allocating physical APIC ID table (4KB) */
1916         p_page = alloc_page(GFP_KERNEL);
1917         if (!p_page)
1918                 goto free_avic;
1919
1920         kvm_svm->avic_physical_id_table_page = p_page;
1921         clear_page(page_address(p_page));
1922
1923         /* Allocating logical APIC ID table (4KB) */
1924         l_page = alloc_page(GFP_KERNEL);
1925         if (!l_page)
1926                 goto free_avic;
1927
1928         kvm_svm->avic_logical_id_table_page = l_page;
1929         clear_page(page_address(l_page));
1930
1931         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1932  again:
1933         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
1934         if (vm_id == 0) { /* id is 1-based, zero is not okay */
1935                 next_vm_id_wrapped = 1;
1936                 goto again;
1937         }
1938         /* Is it still in use? Only possible if wrapped at least once */
1939         if (next_vm_id_wrapped) {
1940                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
1941                         if (k2->avic_vm_id == vm_id)
1942                                 goto again;
1943                 }
1944         }
1945         kvm_svm->avic_vm_id = vm_id;
1946         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
1947         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1948
1949         return 0;
1950
1951 free_avic:
1952         avic_vm_destroy(kvm);
1953         return err;
1954 }
1955
1956 static inline int
1957 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1958 {
1959         int ret = 0;
1960         unsigned long flags;
1961         struct amd_svm_iommu_ir *ir;
1962         struct vcpu_svm *svm = to_svm(vcpu);
1963
1964         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1965                 return 0;
1966
1967         /*
1968          * Here, we go through the per-vcpu ir_list to update all existing
1969          * interrupt remapping table entry targeting this vcpu.
1970          */
1971         spin_lock_irqsave(&svm->ir_list_lock, flags);
1972
1973         if (list_empty(&svm->ir_list))
1974                 goto out;
1975
1976         list_for_each_entry(ir, &svm->ir_list, node) {
1977                 ret = amd_iommu_update_ga(cpu, r, ir->data);
1978                 if (ret)
1979                         break;
1980         }
1981 out:
1982         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
1983         return ret;
1984 }
1985
1986 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1987 {
1988         u64 entry;
1989         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
1990         int h_physical_id = kvm_cpu_get_apicid(cpu);
1991         struct vcpu_svm *svm = to_svm(vcpu);
1992
1993         if (!kvm_vcpu_apicv_active(vcpu))
1994                 return;
1995
1996         if (WARN_ON(h_physical_id >= AVIC_MAX_PHYSICAL_ID_COUNT))
1997                 return;
1998
1999         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2000         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2001
2002         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2003         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2004
2005         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2006         if (svm->avic_is_running)
2007                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2008
2009         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2010         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2011                                         svm->avic_is_running);
2012 }
2013
2014 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2015 {
2016         u64 entry;
2017         struct vcpu_svm *svm = to_svm(vcpu);
2018
2019         if (!kvm_vcpu_apicv_active(vcpu))
2020                 return;
2021
2022         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2023         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2024                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2025
2026         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2027         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2028 }
2029
2030 /**
2031  * This function is called during VCPU halt/unhalt.
2032  */
2033 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2034 {
2035         struct vcpu_svm *svm = to_svm(vcpu);
2036
2037         svm->avic_is_running = is_run;
2038         if (is_run)
2039                 avic_vcpu_load(vcpu, vcpu->cpu);
2040         else
2041                 avic_vcpu_put(vcpu);
2042 }
2043
2044 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
2045 {
2046         struct vcpu_svm *svm = to_svm(vcpu);
2047         u32 dummy;
2048         u32 eax = 1;
2049
2050         vcpu->arch.microcode_version = 0x01000065;
2051         svm->spec_ctrl = 0;
2052
2053         if (!init_event) {
2054                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2055                                            MSR_IA32_APICBASE_ENABLE;
2056                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2057                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2058         }
2059         init_vmcb(svm);
2060
2061         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
2062         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
2063
2064         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2065                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
2066 }
2067
2068 static int avic_init_vcpu(struct vcpu_svm *svm)
2069 {
2070         int ret;
2071
2072         if (!kvm_vcpu_apicv_active(&svm->vcpu))
2073                 return 0;
2074
2075         ret = avic_init_backing_page(&svm->vcpu);
2076         if (ret)
2077                 return ret;
2078
2079         INIT_LIST_HEAD(&svm->ir_list);
2080         spin_lock_init(&svm->ir_list_lock);
2081
2082         return ret;
2083 }
2084
2085 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
2086 {
2087         struct vcpu_svm *svm;
2088         struct page *page;
2089         struct page *msrpm_pages;
2090         struct page *hsave_page;
2091         struct page *nested_msrpm_pages;
2092         int err;
2093
2094         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2095         if (!svm) {
2096                 err = -ENOMEM;
2097                 goto out;
2098         }
2099
2100         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
2101         if (err)
2102                 goto free_svm;
2103
2104         err = -ENOMEM;
2105         page = alloc_page(GFP_KERNEL);
2106         if (!page)
2107                 goto uninit;
2108
2109         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
2110         if (!msrpm_pages)
2111                 goto free_page1;
2112
2113         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
2114         if (!nested_msrpm_pages)
2115                 goto free_page2;
2116
2117         hsave_page = alloc_page(GFP_KERNEL);
2118         if (!hsave_page)
2119                 goto free_page3;
2120
2121         err = avic_init_vcpu(svm);
2122         if (err)
2123                 goto free_page4;
2124
2125         /* We initialize this flag to true to make sure that the is_running
2126          * bit would be set the first time the vcpu is loaded.
2127          */
2128         svm->avic_is_running = true;
2129
2130         svm->nested.hsave = page_address(hsave_page);
2131
2132         svm->msrpm = page_address(msrpm_pages);
2133         svm_vcpu_init_msrpm(svm->msrpm);
2134
2135         svm->nested.msrpm = page_address(nested_msrpm_pages);
2136         svm_vcpu_init_msrpm(svm->nested.msrpm);
2137
2138         svm->vmcb = page_address(page);
2139         clear_page(svm->vmcb);
2140         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2141         svm->asid_generation = 0;
2142         init_vmcb(svm);
2143
2144         svm_init_osvw(&svm->vcpu);
2145
2146         return &svm->vcpu;
2147
2148 free_page4:
2149         __free_page(hsave_page);
2150 free_page3:
2151         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2152 free_page2:
2153         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2154 free_page1:
2155         __free_page(page);
2156 uninit:
2157         kvm_vcpu_uninit(&svm->vcpu);
2158 free_svm:
2159         kmem_cache_free(kvm_vcpu_cache, svm);
2160 out:
2161         return ERR_PTR(err);
2162 }
2163
2164 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2165 {
2166         struct vcpu_svm *svm = to_svm(vcpu);
2167
2168         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2169         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2170         __free_page(virt_to_page(svm->nested.hsave));
2171         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2172         kvm_vcpu_uninit(vcpu);
2173         kmem_cache_free(kvm_vcpu_cache, svm);
2174         /*
2175          * The vmcb page can be recycled, causing a false negative in
2176          * svm_vcpu_load(). So do a full IBPB now.
2177          */
2178         indirect_branch_prediction_barrier();
2179 }
2180
2181 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2182 {
2183         struct vcpu_svm *svm = to_svm(vcpu);
2184         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2185         int i;
2186
2187         if (unlikely(cpu != vcpu->cpu)) {
2188                 svm->asid_generation = 0;
2189                 mark_all_dirty(svm->vmcb);
2190         }
2191
2192 #ifdef CONFIG_X86_64
2193         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2194 #endif
2195         savesegment(fs, svm->host.fs);
2196         savesegment(gs, svm->host.gs);
2197         svm->host.ldt = kvm_read_ldt();
2198
2199         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2200                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2201
2202         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2203                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2204                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2205                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2206                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2207                 }
2208         }
2209         /* This assumes that the kernel never uses MSR_TSC_AUX */
2210         if (static_cpu_has(X86_FEATURE_RDTSCP))
2211                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2212
2213         if (sd->current_vmcb != svm->vmcb) {
2214                 sd->current_vmcb = svm->vmcb;
2215                 indirect_branch_prediction_barrier();
2216         }
2217         avic_vcpu_load(vcpu, cpu);
2218 }
2219
2220 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2221 {
2222         struct vcpu_svm *svm = to_svm(vcpu);
2223         int i;
2224
2225         avic_vcpu_put(vcpu);
2226
2227         ++vcpu->stat.host_state_reload;
2228         kvm_load_ldt(svm->host.ldt);
2229 #ifdef CONFIG_X86_64
2230         loadsegment(fs, svm->host.fs);
2231         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2232         load_gs_index(svm->host.gs);
2233 #else
2234 #ifdef CONFIG_X86_32_LAZY_GS
2235         loadsegment(gs, svm->host.gs);
2236 #endif
2237 #endif
2238         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2239                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2240 }
2241
2242 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2243 {
2244         avic_set_running(vcpu, false);
2245 }
2246
2247 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2248 {
2249         avic_set_running(vcpu, true);
2250 }
2251
2252 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2253 {
2254         struct vcpu_svm *svm = to_svm(vcpu);
2255         unsigned long rflags = svm->vmcb->save.rflags;
2256
2257         if (svm->nmi_singlestep) {
2258                 /* Hide our flags if they were not set by the guest */
2259                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2260                         rflags &= ~X86_EFLAGS_TF;
2261                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2262                         rflags &= ~X86_EFLAGS_RF;
2263         }
2264         return rflags;
2265 }
2266
2267 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2268 {
2269         if (to_svm(vcpu)->nmi_singlestep)
2270                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2271
2272        /*
2273         * Any change of EFLAGS.VM is accompanied by a reload of SS
2274         * (caused by either a task switch or an inter-privilege IRET),
2275         * so we do not need to update the CPL here.
2276         */
2277         to_svm(vcpu)->vmcb->save.rflags = rflags;
2278 }
2279
2280 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2281 {
2282         switch (reg) {
2283         case VCPU_EXREG_PDPTR:
2284                 BUG_ON(!npt_enabled);
2285                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2286                 break;
2287         default:
2288                 BUG();
2289         }
2290 }
2291
2292 static void svm_set_vintr(struct vcpu_svm *svm)
2293 {
2294         set_intercept(svm, INTERCEPT_VINTR);
2295 }
2296
2297 static void svm_clear_vintr(struct vcpu_svm *svm)
2298 {
2299         clr_intercept(svm, INTERCEPT_VINTR);
2300 }
2301
2302 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2303 {
2304         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2305
2306         switch (seg) {
2307         case VCPU_SREG_CS: return &save->cs;
2308         case VCPU_SREG_DS: return &save->ds;
2309         case VCPU_SREG_ES: return &save->es;
2310         case VCPU_SREG_FS: return &save->fs;
2311         case VCPU_SREG_GS: return &save->gs;
2312         case VCPU_SREG_SS: return &save->ss;
2313         case VCPU_SREG_TR: return &save->tr;
2314         case VCPU_SREG_LDTR: return &save->ldtr;
2315         }
2316         BUG();
2317         return NULL;
2318 }
2319
2320 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2321 {
2322         struct vmcb_seg *s = svm_seg(vcpu, seg);
2323
2324         return s->base;
2325 }
2326
2327 static void svm_get_segment(struct kvm_vcpu *vcpu,
2328                             struct kvm_segment *var, int seg)
2329 {
2330         struct vmcb_seg *s = svm_seg(vcpu, seg);
2331
2332         var->base = s->base;
2333         var->limit = s->limit;
2334         var->selector = s->selector;
2335         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2336         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2337         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2338         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2339         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2340         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2341         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2342
2343         /*
2344          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2345          * However, the SVM spec states that the G bit is not observed by the
2346          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2347          * So let's synthesize a legal G bit for all segments, this helps
2348          * running KVM nested. It also helps cross-vendor migration, because
2349          * Intel's vmentry has a check on the 'G' bit.
2350          */
2351         var->g = s->limit > 0xfffff;
2352
2353         /*
2354          * AMD's VMCB does not have an explicit unusable field, so emulate it
2355          * for cross vendor migration purposes by "not present"
2356          */
2357         var->unusable = !var->present;
2358
2359         switch (seg) {
2360         case VCPU_SREG_TR:
2361                 /*
2362                  * Work around a bug where the busy flag in the tr selector
2363                  * isn't exposed
2364                  */
2365                 var->type |= 0x2;
2366                 break;
2367         case VCPU_SREG_DS:
2368         case VCPU_SREG_ES:
2369         case VCPU_SREG_FS:
2370         case VCPU_SREG_GS:
2371                 /*
2372                  * The accessed bit must always be set in the segment
2373                  * descriptor cache, although it can be cleared in the
2374                  * descriptor, the cached bit always remains at 1. Since
2375                  * Intel has a check on this, set it here to support
2376                  * cross-vendor migration.
2377                  */
2378                 if (!var->unusable)
2379                         var->type |= 0x1;
2380                 break;
2381         case VCPU_SREG_SS:
2382                 /*
2383                  * On AMD CPUs sometimes the DB bit in the segment
2384                  * descriptor is left as 1, although the whole segment has
2385                  * been made unusable. Clear it here to pass an Intel VMX
2386                  * entry check when cross vendor migrating.
2387                  */
2388                 if (var->unusable)
2389                         var->db = 0;
2390                 /* This is symmetric with svm_set_segment() */
2391                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2392                 break;
2393         }
2394 }
2395
2396 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2397 {
2398         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2399
2400         return save->cpl;
2401 }
2402
2403 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2404 {
2405         struct vcpu_svm *svm = to_svm(vcpu);
2406
2407         dt->size = svm->vmcb->save.idtr.limit;
2408         dt->address = svm->vmcb->save.idtr.base;
2409 }
2410
2411 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2412 {
2413         struct vcpu_svm *svm = to_svm(vcpu);
2414
2415         svm->vmcb->save.idtr.limit = dt->size;
2416         svm->vmcb->save.idtr.base = dt->address ;
2417         mark_dirty(svm->vmcb, VMCB_DT);
2418 }
2419
2420 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2421 {
2422         struct vcpu_svm *svm = to_svm(vcpu);
2423
2424         dt->size = svm->vmcb->save.gdtr.limit;
2425         dt->address = svm->vmcb->save.gdtr.base;
2426 }
2427
2428 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2429 {
2430         struct vcpu_svm *svm = to_svm(vcpu);
2431
2432         svm->vmcb->save.gdtr.limit = dt->size;
2433         svm->vmcb->save.gdtr.base = dt->address ;
2434         mark_dirty(svm->vmcb, VMCB_DT);
2435 }
2436
2437 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2438 {
2439 }
2440
2441 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
2442 {
2443 }
2444
2445 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2446 {
2447 }
2448
2449 static void update_cr0_intercept(struct vcpu_svm *svm)
2450 {
2451         ulong gcr0 = svm->vcpu.arch.cr0;
2452         u64 *hcr0 = &svm->vmcb->save.cr0;
2453
2454         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2455                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2456
2457         mark_dirty(svm->vmcb, VMCB_CR);
2458
2459         if (gcr0 == *hcr0) {
2460                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2461                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2462         } else {
2463                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2464                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2465         }
2466 }
2467
2468 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2469 {
2470         struct vcpu_svm *svm = to_svm(vcpu);
2471
2472 #ifdef CONFIG_X86_64
2473         if (vcpu->arch.efer & EFER_LME) {
2474                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2475                         vcpu->arch.efer |= EFER_LMA;
2476                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2477                 }
2478
2479                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2480                         vcpu->arch.efer &= ~EFER_LMA;
2481                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2482                 }
2483         }
2484 #endif
2485         vcpu->arch.cr0 = cr0;
2486
2487         if (!npt_enabled)
2488                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2489
2490         /*
2491          * re-enable caching here because the QEMU bios
2492          * does not do it - this results in some delay at
2493          * reboot
2494          */
2495         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2496                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2497         svm->vmcb->save.cr0 = cr0;
2498         mark_dirty(svm->vmcb, VMCB_CR);
2499         update_cr0_intercept(svm);
2500 }
2501
2502 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2503 {
2504         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2505         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2506
2507         if (cr4 & X86_CR4_VMXE)
2508                 return 1;
2509
2510         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2511                 svm_flush_tlb(vcpu, true);
2512
2513         vcpu->arch.cr4 = cr4;
2514         if (!npt_enabled)
2515                 cr4 |= X86_CR4_PAE;
2516         cr4 |= host_cr4_mce;
2517         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2518         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2519         return 0;
2520 }
2521
2522 static void svm_set_segment(struct kvm_vcpu *vcpu,
2523                             struct kvm_segment *var, int seg)
2524 {
2525         struct vcpu_svm *svm = to_svm(vcpu);
2526         struct vmcb_seg *s = svm_seg(vcpu, seg);
2527
2528         s->base = var->base;
2529         s->limit = var->limit;
2530         s->selector = var->selector;
2531         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2532         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2533         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2534         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2535         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2536         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2537         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2538         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2539
2540         /*
2541          * This is always accurate, except if SYSRET returned to a segment
2542          * with SS.DPL != 3.  Intel does not have this quirk, and always
2543          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2544          * would entail passing the CPL to userspace and back.
2545          */
2546         if (seg == VCPU_SREG_SS)
2547                 /* This is symmetric with svm_get_segment() */
2548                 svm->vmcb->save.cpl = (var->dpl & 3);
2549
2550         mark_dirty(svm->vmcb, VMCB_SEG);
2551 }
2552
2553 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2554 {
2555         struct vcpu_svm *svm = to_svm(vcpu);
2556
2557         clr_exception_intercept(svm, BP_VECTOR);
2558
2559         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2560                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2561                         set_exception_intercept(svm, BP_VECTOR);
2562         } else
2563                 vcpu->guest_debug = 0;
2564 }
2565
2566 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2567 {
2568         if (sd->next_asid > sd->max_asid) {
2569                 ++sd->asid_generation;
2570                 sd->next_asid = sd->min_asid;
2571                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2572         }
2573
2574         svm->asid_generation = sd->asid_generation;
2575         svm->vmcb->control.asid = sd->next_asid++;
2576
2577         mark_dirty(svm->vmcb, VMCB_ASID);
2578 }
2579
2580 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2581 {
2582         return to_svm(vcpu)->vmcb->save.dr6;
2583 }
2584
2585 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2586 {
2587         struct vcpu_svm *svm = to_svm(vcpu);
2588
2589         svm->vmcb->save.dr6 = value;
2590         mark_dirty(svm->vmcb, VMCB_DR);
2591 }
2592
2593 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2594 {
2595         struct vcpu_svm *svm = to_svm(vcpu);
2596
2597         get_debugreg(vcpu->arch.db[0], 0);
2598         get_debugreg(vcpu->arch.db[1], 1);
2599         get_debugreg(vcpu->arch.db[2], 2);
2600         get_debugreg(vcpu->arch.db[3], 3);
2601         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2602         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2603
2604         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2605         set_dr_intercepts(svm);
2606 }
2607
2608 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2609 {
2610         struct vcpu_svm *svm = to_svm(vcpu);
2611
2612         svm->vmcb->save.dr7 = value;
2613         mark_dirty(svm->vmcb, VMCB_DR);
2614 }
2615
2616 static int pf_interception(struct vcpu_svm *svm)
2617 {
2618         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2619         u64 error_code = svm->vmcb->control.exit_info_1;
2620
2621         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2622                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2623                         svm->vmcb->control.insn_bytes : NULL,
2624                         svm->vmcb->control.insn_len);
2625 }
2626
2627 static int npf_interception(struct vcpu_svm *svm)
2628 {
2629         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2630         u64 error_code = svm->vmcb->control.exit_info_1;
2631
2632         trace_kvm_page_fault(fault_address, error_code);
2633         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2634                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2635                         svm->vmcb->control.insn_bytes : NULL,
2636                         svm->vmcb->control.insn_len);
2637 }
2638
2639 static int db_interception(struct vcpu_svm *svm)
2640 {
2641         struct kvm_run *kvm_run = svm->vcpu.run;
2642
2643         if (!(svm->vcpu.guest_debug &
2644               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2645                 !svm->nmi_singlestep) {
2646                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2647                 return 1;
2648         }
2649
2650         if (svm->nmi_singlestep) {
2651                 disable_nmi_singlestep(svm);
2652         }
2653
2654         if (svm->vcpu.guest_debug &
2655             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2656                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2657                 kvm_run->debug.arch.pc =
2658                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2659                 kvm_run->debug.arch.exception = DB_VECTOR;
2660                 return 0;
2661         }
2662
2663         return 1;
2664 }
2665
2666 static int bp_interception(struct vcpu_svm *svm)
2667 {
2668         struct kvm_run *kvm_run = svm->vcpu.run;
2669
2670         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2671         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2672         kvm_run->debug.arch.exception = BP_VECTOR;
2673         return 0;
2674 }
2675
2676 static int ud_interception(struct vcpu_svm *svm)
2677 {
2678         return handle_ud(&svm->vcpu);
2679 }
2680
2681 static int ac_interception(struct vcpu_svm *svm)
2682 {
2683         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2684         return 1;
2685 }
2686
2687 static int gp_interception(struct vcpu_svm *svm)
2688 {
2689         struct kvm_vcpu *vcpu = &svm->vcpu;
2690         u32 error_code = svm->vmcb->control.exit_info_1;
2691         int er;
2692
2693         WARN_ON_ONCE(!enable_vmware_backdoor);
2694
2695         er = emulate_instruction(vcpu,
2696                 EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
2697         if (er == EMULATE_USER_EXIT)
2698                 return 0;
2699         else if (er != EMULATE_DONE)
2700                 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2701         return 1;
2702 }
2703
2704 static bool is_erratum_383(void)
2705 {
2706         int err, i;
2707         u64 value;
2708
2709         if (!erratum_383_found)
2710                 return false;
2711
2712         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2713         if (err)
2714                 return false;
2715
2716         /* Bit 62 may or may not be set for this mce */
2717         value &= ~(1ULL << 62);
2718
2719         if (value != 0xb600000000010015ULL)
2720                 return false;
2721
2722         /* Clear MCi_STATUS registers */
2723         for (i = 0; i < 6; ++i)
2724                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2725
2726         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2727         if (!err) {
2728                 u32 low, high;
2729
2730                 value &= ~(1ULL << 2);
2731                 low    = lower_32_bits(value);
2732                 high   = upper_32_bits(value);
2733
2734                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2735         }
2736
2737         /* Flush tlb to evict multi-match entries */
2738         __flush_tlb_all();
2739
2740         return true;
2741 }
2742
2743 static void svm_handle_mce(struct vcpu_svm *svm)
2744 {
2745         if (is_erratum_383()) {
2746                 /*
2747                  * Erratum 383 triggered. Guest state is corrupt so kill the
2748                  * guest.
2749                  */
2750                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2751
2752                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2753
2754                 return;
2755         }
2756
2757         /*
2758          * On an #MC intercept the MCE handler is not called automatically in
2759          * the host. So do it by hand here.
2760          */
2761         asm volatile (
2762                 "int $0x12\n");
2763         /* not sure if we ever come back to this point */
2764
2765         return;
2766 }
2767
2768 static int mc_interception(struct vcpu_svm *svm)
2769 {
2770         return 1;
2771 }
2772
2773 static int shutdown_interception(struct vcpu_svm *svm)
2774 {
2775         struct kvm_run *kvm_run = svm->vcpu.run;
2776
2777         /*
2778          * VMCB is undefined after a SHUTDOWN intercept
2779          * so reinitialize it.
2780          */
2781         clear_page(svm->vmcb);
2782         init_vmcb(svm);
2783
2784         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2785         return 0;
2786 }
2787
2788 static int io_interception(struct vcpu_svm *svm)
2789 {
2790         struct kvm_vcpu *vcpu = &svm->vcpu;
2791         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2792         int size, in, string;
2793         unsigned port;
2794
2795         ++svm->vcpu.stat.io_exits;
2796         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2797         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2798         if (string)
2799                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2800
2801         port = io_info >> 16;
2802         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2803         svm->next_rip = svm->vmcb->control.exit_info_2;
2804
2805         return kvm_fast_pio(&svm->vcpu, size, port, in);
2806 }
2807
2808 static int nmi_interception(struct vcpu_svm *svm)
2809 {
2810         return 1;
2811 }
2812
2813 static int intr_interception(struct vcpu_svm *svm)
2814 {
2815         ++svm->vcpu.stat.irq_exits;
2816         return 1;
2817 }
2818
2819 static int nop_on_interception(struct vcpu_svm *svm)
2820 {
2821         return 1;
2822 }
2823
2824 static int halt_interception(struct vcpu_svm *svm)
2825 {
2826         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2827         return kvm_emulate_halt(&svm->vcpu);
2828 }
2829
2830 static int vmmcall_interception(struct vcpu_svm *svm)
2831 {
2832         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2833         return kvm_emulate_hypercall(&svm->vcpu);
2834 }
2835
2836 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2837 {
2838         struct vcpu_svm *svm = to_svm(vcpu);
2839
2840         return svm->nested.nested_cr3;
2841 }
2842
2843 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2844 {
2845         struct vcpu_svm *svm = to_svm(vcpu);
2846         u64 cr3 = svm->nested.nested_cr3;
2847         u64 pdpte;
2848         int ret;
2849
2850         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2851                                        offset_in_page(cr3) + index * 8, 8);
2852         if (ret)
2853                 return 0;
2854         return pdpte;
2855 }
2856
2857 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2858                                    unsigned long root)
2859 {
2860         struct vcpu_svm *svm = to_svm(vcpu);
2861
2862         svm->vmcb->control.nested_cr3 = __sme_set(root);
2863         mark_dirty(svm->vmcb, VMCB_NPT);
2864         svm_flush_tlb(vcpu, true);
2865 }
2866
2867 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2868                                        struct x86_exception *fault)
2869 {
2870         struct vcpu_svm *svm = to_svm(vcpu);
2871
2872         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2873                 /*
2874                  * TODO: track the cause of the nested page fault, and
2875                  * correctly fill in the high bits of exit_info_1.
2876                  */
2877                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2878                 svm->vmcb->control.exit_code_hi = 0;
2879                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2880                 svm->vmcb->control.exit_info_2 = fault->address;
2881         }
2882
2883         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2884         svm->vmcb->control.exit_info_1 |= fault->error_code;
2885
2886         /*
2887          * The present bit is always zero for page structure faults on real
2888          * hardware.
2889          */
2890         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2891                 svm->vmcb->control.exit_info_1 &= ~1;
2892
2893         nested_svm_vmexit(svm);
2894 }
2895
2896 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2897 {
2898         WARN_ON(mmu_is_nested(vcpu));
2899         kvm_init_shadow_mmu(vcpu);
2900         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
2901         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2902         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2903         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2904         vcpu->arch.mmu.shadow_root_level = get_npt_level(vcpu);
2905         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2906         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2907 }
2908
2909 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2910 {
2911         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2912 }
2913
2914 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2915 {
2916         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
2917             !is_paging(&svm->vcpu)) {
2918                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2919                 return 1;
2920         }
2921
2922         if (svm->vmcb->save.cpl) {
2923                 kvm_inject_gp(&svm->vcpu, 0);
2924                 return 1;
2925         }
2926
2927         return 0;
2928 }
2929
2930 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2931                                       bool has_error_code, u32 error_code)
2932 {
2933         int vmexit;
2934
2935         if (!is_guest_mode(&svm->vcpu))
2936                 return 0;
2937
2938         vmexit = nested_svm_intercept(svm);
2939         if (vmexit != NESTED_EXIT_DONE)
2940                 return 0;
2941
2942         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2943         svm->vmcb->control.exit_code_hi = 0;
2944         svm->vmcb->control.exit_info_1 = error_code;
2945
2946         /*
2947          * FIXME: we should not write CR2 when L1 intercepts an L2 #PF exception.
2948          * The fix is to add the ancillary datum (CR2 or DR6) to structs
2949          * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6 can be
2950          * written only when inject_pending_event runs (DR6 would written here
2951          * too).  This should be conditional on a new capability---if the
2952          * capability is disabled, kvm_multiple_exception would write the
2953          * ancillary information to CR2 or DR6, for backwards ABI-compatibility.
2954          */
2955         if (svm->vcpu.arch.exception.nested_apf)
2956                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
2957         else
2958                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2959
2960         svm->nested.exit_required = true;
2961         return vmexit;
2962 }
2963
2964 /* This function returns true if it is save to enable the irq window */
2965 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2966 {
2967         if (!is_guest_mode(&svm->vcpu))
2968                 return true;
2969
2970         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2971                 return true;
2972
2973         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2974                 return false;
2975
2976         /*
2977          * if vmexit was already requested (by intercepted exception
2978          * for instance) do not overwrite it with "external interrupt"
2979          * vmexit.
2980          */
2981         if (svm->nested.exit_required)
2982                 return false;
2983
2984         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2985         svm->vmcb->control.exit_info_1 = 0;
2986         svm->vmcb->control.exit_info_2 = 0;
2987
2988         if (svm->nested.intercept & 1ULL) {
2989                 /*
2990                  * The #vmexit can't be emulated here directly because this
2991                  * code path runs with irqs and preemption disabled. A
2992                  * #vmexit emulation might sleep. Only signal request for
2993                  * the #vmexit here.
2994                  */
2995                 svm->nested.exit_required = true;
2996                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2997                 return false;
2998         }
2999
3000         return true;
3001 }
3002
3003 /* This function returns true if it is save to enable the nmi window */
3004 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3005 {
3006         if (!is_guest_mode(&svm->vcpu))
3007                 return true;
3008
3009         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3010                 return true;
3011
3012         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3013         svm->nested.exit_required = true;
3014
3015         return false;
3016 }
3017
3018 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
3019 {
3020         struct page *page;
3021
3022         might_sleep();
3023
3024         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
3025         if (is_error_page(page))
3026                 goto error;
3027
3028         *_page = page;
3029
3030         return kmap(page);
3031
3032 error:
3033         kvm_inject_gp(&svm->vcpu, 0);
3034
3035         return NULL;
3036 }
3037
3038 static void nested_svm_unmap(struct page *page)
3039 {
3040         kunmap(page);
3041         kvm_release_page_dirty(page);
3042 }
3043
3044 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
3045 {
3046         unsigned port, size, iopm_len;
3047         u16 val, mask;
3048         u8 start_bit;
3049         u64 gpa;
3050
3051         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3052                 return NESTED_EXIT_HOST;
3053
3054         port = svm->vmcb->control.exit_info_1 >> 16;
3055         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3056                 SVM_IOIO_SIZE_SHIFT;
3057         gpa  = svm->nested.vmcb_iopm + (port / 8);
3058         start_bit = port % 8;
3059         iopm_len = (start_bit + size > 8) ? 2 : 1;
3060         mask = (0xf >> (4 - size)) << start_bit;
3061         val = 0;
3062
3063         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
3064                 return NESTED_EXIT_DONE;
3065
3066         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3067 }
3068
3069 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
3070 {
3071         u32 offset, msr, value;
3072         int write, mask;
3073
3074         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3075                 return NESTED_EXIT_HOST;
3076
3077         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3078         offset = svm_msrpm_offset(msr);
3079         write  = svm->vmcb->control.exit_info_1 & 1;
3080         mask   = 1 << ((2 * (msr & 0xf)) + write);
3081
3082         if (offset == MSR_INVALID)
3083                 return NESTED_EXIT_DONE;
3084
3085         /* Offset is in 32 bit units but need in 8 bit units */
3086         offset *= 4;
3087
3088         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
3089                 return NESTED_EXIT_DONE;
3090
3091         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3092 }
3093
3094 /* DB exceptions for our internal use must not cause vmexit */
3095 static int nested_svm_intercept_db(struct vcpu_svm *svm)
3096 {
3097         unsigned long dr6;
3098
3099         /* if we're not singlestepping, it's not ours */
3100         if (!svm->nmi_singlestep)
3101                 return NESTED_EXIT_DONE;
3102
3103         /* if it's not a singlestep exception, it's not ours */
3104         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3105                 return NESTED_EXIT_DONE;
3106         if (!(dr6 & DR6_BS))
3107                 return NESTED_EXIT_DONE;
3108
3109         /* if the guest is singlestepping, it should get the vmexit */
3110         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3111                 disable_nmi_singlestep(svm);
3112                 return NESTED_EXIT_DONE;
3113         }
3114
3115         /* it's ours, the nested hypervisor must not see this one */
3116         return NESTED_EXIT_HOST;
3117 }
3118
3119 static int nested_svm_exit_special(struct vcpu_svm *svm)
3120 {
3121         u32 exit_code = svm->vmcb->control.exit_code;
3122
3123         switch (exit_code) {
3124         case SVM_EXIT_INTR:
3125         case SVM_EXIT_NMI:
3126         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
3127                 return NESTED_EXIT_HOST;
3128         case SVM_EXIT_NPF:
3129                 /* For now we are always handling NPFs when using them */
3130                 if (npt_enabled)
3131                         return NESTED_EXIT_HOST;
3132                 break;
3133         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
3134                 /* When we're shadowing, trap PFs, but not async PF */
3135                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
3136                         return NESTED_EXIT_HOST;
3137                 break;
3138         default:
3139                 break;
3140         }
3141
3142         return NESTED_EXIT_CONTINUE;
3143 }
3144
3145 /*
3146  * If this function returns true, this #vmexit was already handled
3147  */
3148 static int nested_svm_intercept(struct vcpu_svm *svm)
3149 {
3150         u32 exit_code = svm->vmcb->control.exit_code;
3151         int vmexit = NESTED_EXIT_HOST;
3152
3153         switch (exit_code) {
3154         case SVM_EXIT_MSR:
3155                 vmexit = nested_svm_exit_handled_msr(svm);
3156                 break;
3157         case SVM_EXIT_IOIO:
3158                 vmexit = nested_svm_intercept_ioio(svm);
3159                 break;
3160         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3161                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3162                 if (svm->nested.intercept_cr & bit)
3163                         vmexit = NESTED_EXIT_DONE;
3164                 break;
3165         }
3166         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3167                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3168                 if (svm->nested.intercept_dr & bit)
3169                         vmexit = NESTED_EXIT_DONE;
3170                 break;
3171         }
3172         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3173                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3174                 if (svm->nested.intercept_exceptions & excp_bits) {
3175                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3176                                 vmexit = nested_svm_intercept_db(svm);
3177                         else
3178                                 vmexit = NESTED_EXIT_DONE;
3179                 }
3180                 /* async page fault always cause vmexit */
3181                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3182                          svm->vcpu.arch.exception.nested_apf != 0)
3183                         vmexit = NESTED_EXIT_DONE;
3184                 break;
3185         }
3186         case SVM_EXIT_ERR: {
3187                 vmexit = NESTED_EXIT_DONE;
3188                 break;
3189         }
3190         default: {
3191                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3192                 if (svm->nested.intercept & exit_bits)
3193                         vmexit = NESTED_EXIT_DONE;
3194         }
3195         }
3196
3197         return vmexit;
3198 }
3199
3200 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3201 {
3202         int vmexit;
3203
3204         vmexit = nested_svm_intercept(svm);
3205
3206         if (vmexit == NESTED_EXIT_DONE)
3207                 nested_svm_vmexit(svm);
3208
3209         return vmexit;
3210 }
3211
3212 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3213 {
3214         struct vmcb_control_area *dst  = &dst_vmcb->control;
3215         struct vmcb_control_area *from = &from_vmcb->control;
3216
3217         dst->intercept_cr         = from->intercept_cr;
3218         dst->intercept_dr         = from->intercept_dr;
3219         dst->intercept_exceptions = from->intercept_exceptions;
3220         dst->intercept            = from->intercept;
3221         dst->iopm_base_pa         = from->iopm_base_pa;
3222         dst->msrpm_base_pa        = from->msrpm_base_pa;
3223         dst->tsc_offset           = from->tsc_offset;
3224         dst->asid                 = from->asid;
3225         dst->tlb_ctl              = from->tlb_ctl;
3226         dst->int_ctl              = from->int_ctl;
3227         dst->int_vector           = from->int_vector;
3228         dst->int_state            = from->int_state;
3229         dst->exit_code            = from->exit_code;
3230         dst->exit_code_hi         = from->exit_code_hi;
3231         dst->exit_info_1          = from->exit_info_1;
3232         dst->exit_info_2          = from->exit_info_2;
3233         dst->exit_int_info        = from->exit_int_info;
3234         dst->exit_int_info_err    = from->exit_int_info_err;
3235         dst->nested_ctl           = from->nested_ctl;
3236         dst->event_inj            = from->event_inj;
3237         dst->event_inj_err        = from->event_inj_err;
3238         dst->nested_cr3           = from->nested_cr3;
3239         dst->virt_ext              = from->virt_ext;
3240 }
3241
3242 static int nested_svm_vmexit(struct vcpu_svm *svm)
3243 {
3244         struct vmcb *nested_vmcb;
3245         struct vmcb *hsave = svm->nested.hsave;
3246         struct vmcb *vmcb = svm->vmcb;
3247         struct page *page;
3248
3249         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3250                                        vmcb->control.exit_info_1,
3251                                        vmcb->control.exit_info_2,
3252                                        vmcb->control.exit_int_info,
3253                                        vmcb->control.exit_int_info_err,
3254                                        KVM_ISA_SVM);
3255
3256         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
3257         if (!nested_vmcb)
3258                 return 1;
3259
3260         /* Exit Guest-Mode */
3261         leave_guest_mode(&svm->vcpu);
3262         svm->nested.vmcb = 0;
3263
3264         /* Give the current vmcb to the guest */
3265         disable_gif(svm);
3266
3267         nested_vmcb->save.es     = vmcb->save.es;
3268         nested_vmcb->save.cs     = vmcb->save.cs;
3269         nested_vmcb->save.ss     = vmcb->save.ss;
3270         nested_vmcb->save.ds     = vmcb->save.ds;
3271         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3272         nested_vmcb->save.idtr   = vmcb->save.idtr;
3273         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3274         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3275         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3276         nested_vmcb->save.cr2    = vmcb->save.cr2;
3277         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3278         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3279         nested_vmcb->save.rip    = vmcb->save.rip;
3280         nested_vmcb->save.rsp    = vmcb->save.rsp;
3281         nested_vmcb->save.rax    = vmcb->save.rax;
3282         nested_vmcb->save.dr7    = vmcb->save.dr7;
3283         nested_vmcb->save.dr6    = vmcb->save.dr6;
3284         nested_vmcb->save.cpl    = vmcb->save.cpl;
3285
3286         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3287         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3288         nested_vmcb->control.int_state         = vmcb->control.int_state;
3289         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3290         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3291         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3292         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3293         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3294         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3295
3296         if (svm->nrips_enabled)
3297                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3298
3299         /*
3300          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3301          * to make sure that we do not lose injected events. So check event_inj
3302          * here and copy it to exit_int_info if it is valid.
3303          * Exit_int_info and event_inj can't be both valid because the case
3304          * below only happens on a VMRUN instruction intercept which has
3305          * no valid exit_int_info set.
3306          */
3307         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3308                 struct vmcb_control_area *nc = &nested_vmcb->control;
3309
3310                 nc->exit_int_info     = vmcb->control.event_inj;
3311                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3312         }
3313
3314         nested_vmcb->control.tlb_ctl           = 0;
3315         nested_vmcb->control.event_inj         = 0;
3316         nested_vmcb->control.event_inj_err     = 0;
3317
3318         /* We always set V_INTR_MASKING and remember the old value in hflags */
3319         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3320                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3321
3322         /* Restore the original control entries */
3323         copy_vmcb_control_area(vmcb, hsave);
3324
3325         kvm_clear_exception_queue(&svm->vcpu);
3326         kvm_clear_interrupt_queue(&svm->vcpu);
3327
3328         svm->nested.nested_cr3 = 0;
3329
3330         /* Restore selected save entries */
3331         svm->vmcb->save.es = hsave->save.es;
3332         svm->vmcb->save.cs = hsave->save.cs;
3333         svm->vmcb->save.ss = hsave->save.ss;
3334         svm->vmcb->save.ds = hsave->save.ds;
3335         svm->vmcb->save.gdtr = hsave->save.gdtr;
3336         svm->vmcb->save.idtr = hsave->save.idtr;
3337         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3338         svm_set_efer(&svm->vcpu, hsave->save.efer);
3339         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3340         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3341         if (npt_enabled) {
3342                 svm->vmcb->save.cr3 = hsave->save.cr3;
3343                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3344         } else {
3345                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3346         }
3347         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
3348         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
3349         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
3350         svm->vmcb->save.dr7 = 0;
3351         svm->vmcb->save.cpl = 0;
3352         svm->vmcb->control.exit_int_info = 0;
3353
3354         mark_all_dirty(svm->vmcb);
3355
3356         nested_svm_unmap(page);
3357
3358         nested_svm_uninit_mmu_context(&svm->vcpu);
3359         kvm_mmu_reset_context(&svm->vcpu);
3360         kvm_mmu_load(&svm->vcpu);
3361
3362         return 0;
3363 }
3364
3365 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3366 {
3367         /*
3368          * This function merges the msr permission bitmaps of kvm and the
3369          * nested vmcb. It is optimized in that it only merges the parts where
3370          * the kvm msr permission bitmap may contain zero bits
3371          */
3372         int i;
3373
3374         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3375                 return true;
3376
3377         for (i = 0; i < MSRPM_OFFSETS; i++) {
3378                 u32 value, p;
3379                 u64 offset;
3380
3381                 if (msrpm_offsets[i] == 0xffffffff)
3382                         break;
3383
3384                 p      = msrpm_offsets[i];
3385                 offset = svm->nested.vmcb_msrpm + (p * 4);
3386
3387                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3388                         return false;
3389
3390                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3391         }
3392
3393         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3394
3395         return true;
3396 }
3397
3398 static bool nested_vmcb_checks(struct vmcb *vmcb)
3399 {
3400         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3401                 return false;
3402
3403         if (vmcb->control.asid == 0)
3404                 return false;
3405
3406         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3407             !npt_enabled)
3408                 return false;
3409
3410         return true;
3411 }
3412
3413 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3414                                  struct vmcb *nested_vmcb, struct page *page)
3415 {
3416         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3417                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3418         else
3419                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3420
3421         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3422                 kvm_mmu_unload(&svm->vcpu);
3423                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3424                 nested_svm_init_mmu_context(&svm->vcpu);
3425         }
3426
3427         /* Load the nested guest state */
3428         svm->vmcb->save.es = nested_vmcb->save.es;
3429         svm->vmcb->save.cs = nested_vmcb->save.cs;
3430         svm->vmcb->save.ss = nested_vmcb->save.ss;
3431         svm->vmcb->save.ds = nested_vmcb->save.ds;
3432         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3433         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3434         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3435         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3436         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3437         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3438         if (npt_enabled) {
3439                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3440                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3441         } else
3442                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3443
3444         /* Guest paging mode is active - reset mmu */
3445         kvm_mmu_reset_context(&svm->vcpu);
3446
3447         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3448         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
3449         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
3450         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
3451
3452         /* In case we don't even reach vcpu_run, the fields are not updated */
3453         svm->vmcb->save.rax = nested_vmcb->save.rax;
3454         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3455         svm->vmcb->save.rip = nested_vmcb->save.rip;
3456         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3457         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3458         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3459
3460         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3461         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3462
3463         /* cache intercepts */
3464         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3465         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3466         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3467         svm->nested.intercept            = nested_vmcb->control.intercept;
3468
3469         svm_flush_tlb(&svm->vcpu, true);
3470         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3471         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3472                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3473         else
3474                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3475
3476         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3477                 /* We only want the cr8 intercept bits of the guest */
3478                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3479                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3480         }
3481
3482         /* We don't want to see VMMCALLs from a nested guest */
3483         clr_intercept(svm, INTERCEPT_VMMCALL);
3484
3485         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3486         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3487         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3488         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
3489         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3490         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3491
3492         nested_svm_unmap(page);
3493
3494         /* Enter Guest-Mode */
3495         enter_guest_mode(&svm->vcpu);
3496
3497         /*
3498          * Merge guest and host intercepts - must be called  with vcpu in
3499          * guest-mode to take affect here
3500          */
3501         recalc_intercepts(svm);
3502
3503         svm->nested.vmcb = vmcb_gpa;
3504
3505         enable_gif(svm);
3506
3507         mark_all_dirty(svm->vmcb);
3508 }
3509
3510 static bool nested_svm_vmrun(struct vcpu_svm *svm)
3511 {
3512         struct vmcb *nested_vmcb;
3513         struct vmcb *hsave = svm->nested.hsave;
3514         struct vmcb *vmcb = svm->vmcb;
3515         struct page *page;
3516         u64 vmcb_gpa;
3517
3518         vmcb_gpa = svm->vmcb->save.rax;
3519
3520         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3521         if (!nested_vmcb)
3522                 return false;
3523
3524         if (!nested_vmcb_checks(nested_vmcb)) {
3525                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3526                 nested_vmcb->control.exit_code_hi = 0;
3527                 nested_vmcb->control.exit_info_1  = 0;
3528                 nested_vmcb->control.exit_info_2  = 0;
3529
3530                 nested_svm_unmap(page);
3531
3532                 return false;
3533         }
3534
3535         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3536                                nested_vmcb->save.rip,
3537                                nested_vmcb->control.int_ctl,
3538                                nested_vmcb->control.event_inj,
3539                                nested_vmcb->control.nested_ctl);
3540
3541         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3542                                     nested_vmcb->control.intercept_cr >> 16,
3543                                     nested_vmcb->control.intercept_exceptions,
3544                                     nested_vmcb->control.intercept);
3545
3546         /* Clear internal status */
3547         kvm_clear_exception_queue(&svm->vcpu);
3548         kvm_clear_interrupt_queue(&svm->vcpu);
3549
3550         /*
3551          * Save the old vmcb, so we don't need to pick what we save, but can
3552          * restore everything when a VMEXIT occurs
3553          */
3554         hsave->save.es     = vmcb->save.es;
3555         hsave->save.cs     = vmcb->save.cs;
3556         hsave->save.ss     = vmcb->save.ss;
3557         hsave->save.ds     = vmcb->save.ds;
3558         hsave->save.gdtr   = vmcb->save.gdtr;
3559         hsave->save.idtr   = vmcb->save.idtr;
3560         hsave->save.efer   = svm->vcpu.arch.efer;
3561         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3562         hsave->save.cr4    = svm->vcpu.arch.cr4;
3563         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3564         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3565         hsave->save.rsp    = vmcb->save.rsp;
3566         hsave->save.rax    = vmcb->save.rax;
3567         if (npt_enabled)
3568                 hsave->save.cr3    = vmcb->save.cr3;
3569         else
3570                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3571
3572         copy_vmcb_control_area(hsave, vmcb);
3573
3574         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, page);
3575
3576         return true;
3577 }
3578
3579 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3580 {
3581         to_vmcb->save.fs = from_vmcb->save.fs;
3582         to_vmcb->save.gs = from_vmcb->save.gs;
3583         to_vmcb->save.tr = from_vmcb->save.tr;
3584         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3585         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3586         to_vmcb->save.star = from_vmcb->save.star;
3587         to_vmcb->save.lstar = from_vmcb->save.lstar;
3588         to_vmcb->save.cstar = from_vmcb->save.cstar;
3589         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3590         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3591         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3592         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3593 }
3594
3595 static int vmload_interception(struct vcpu_svm *svm)
3596 {
3597         struct vmcb *nested_vmcb;
3598         struct page *page;
3599         int ret;
3600
3601         if (nested_svm_check_permissions(svm))
3602                 return 1;
3603
3604         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3605         if (!nested_vmcb)
3606                 return 1;
3607
3608         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3609         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3610
3611         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3612         nested_svm_unmap(page);
3613
3614         return ret;
3615 }
3616
3617 static int vmsave_interception(struct vcpu_svm *svm)
3618 {
3619         struct vmcb *nested_vmcb;
3620         struct page *page;
3621         int ret;
3622
3623         if (nested_svm_check_permissions(svm))
3624                 return 1;
3625
3626         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
3627         if (!nested_vmcb)
3628                 return 1;
3629
3630         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
3631         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3632
3633         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3634         nested_svm_unmap(page);
3635
3636         return ret;
3637 }
3638
3639 static int vmrun_interception(struct vcpu_svm *svm)
3640 {
3641         if (nested_svm_check_permissions(svm))
3642                 return 1;
3643
3644         /* Save rip after vmrun instruction */
3645         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
3646
3647         if (!nested_svm_vmrun(svm))
3648                 return 1;
3649
3650         if (!nested_svm_vmrun_msrpm(svm))
3651                 goto failed;
3652
3653         return 1;
3654
3655 failed:
3656
3657         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3658         svm->vmcb->control.exit_code_hi = 0;