arch/x86/kvm/vmx/vmx.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * This module enables machines with Intel VT-x extensions to run virtual
   6  * machines without emulation or binary translation.
   7  *
   8  * Copyright (C) 2006 Qumranet, Inc.
   9  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Avi Kivity   <avi@qumranet.com>
  13  *   Yaniv Kamay  <yaniv@qumranet.com>
  14  */
  15
  16 #include <linux/frame.h>
  17 #include <linux/highmem.h>
  18 #include <linux/hrtimer.h>
  19 #include <linux/kernel.h>
  20 #include <linux/kvm_host.h>
  21 #include <linux/module.h>
  22 #include <linux/moduleparam.h>
  23 #include <linux/mod_devicetable.h>
  24 #include <linux/mm.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/smt.h>
  27 #include <linux/slab.h>
  28 #include <linux/tboot.h>
  29 #include <linux/trace_events.h>
  30 #include <linux/entry-kvm.h>
  31
  32 #include <asm/apic.h>
  33 #include <asm/asm.h>
  34 #include <asm/cpu.h>
  35 #include <asm/cpu_device_id.h>
  36 #include <asm/debugreg.h>
  37 #include <asm/desc.h>
  38 #include <asm/fpu/internal.h>
  39 #include <asm/io.h>
  40 #include <asm/irq_remapping.h>
  41 #include <asm/kexec.h>
  42 #include <asm/perf_event.h>
  43 #include <asm/mce.h>
  44 #include <asm/mmu_context.h>
  45 #include <asm/mshyperv.h>
  46 #include <asm/mwait.h>
  47 #include <asm/spec-ctrl.h>
  48 #include <asm/virtext.h>
  49 #include <asm/vmx.h>
  50
  51 #include "capabilities.h"
  52 #include "cpuid.h"
  53 #include "evmcs.h"
  54 #include "irq.h"
  55 #include "kvm_cache_regs.h"
  56 #include "lapic.h"
  57 #include "mmu.h"
  58 #include "nested.h"
  59 #include "ops.h"
  60 #include "pmu.h"
  61 #include "trace.h"
  62 #include "vmcs.h"
  63 #include "vmcs12.h"
  64 #include "vmx.h"
  65 #include "x86.h"
  66
  67 MODULE_AUTHOR("Qumranet");
  68 MODULE_LICENSE("GPL");
  69
  70 #ifdef MODULE
  71 static const struct x86_cpu_id vmx_cpu_id[] = {
  72         X86_MATCH_FEATURE(X86_FEATURE_VMX, NULL),
  73         {}
  74 };
  75 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  76 #endif
  77
  78 bool __read_mostly enable_vpid = 1;
  79 module_param_named(vpid, enable_vpid, bool, 0444);
  80
  81 static bool __read_mostly enable_vnmi = 1;
  82 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
  83
  84 bool __read_mostly flexpriority_enabled = 1;
  85 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  86
  87 bool __read_mostly enable_ept = 1;
  88 module_param_named(ept, enable_ept, bool, S_IRUGO);
  89
  90 bool __read_mostly enable_unrestricted_guest = 1;
  91 module_param_named(unrestricted_guest,
  92                         enable_unrestricted_guest, bool, S_IRUGO);
  93
  94 bool __read_mostly enable_ept_ad_bits = 1;
  95 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  96
  97 static bool __read_mostly emulate_invalid_guest_state = true;
  98 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  99
 100 static bool __read_mostly fasteoi = 1;
 101 module_param(fasteoi, bool, S_IRUGO);
 102
 103 bool __read_mostly enable_apicv = 1;
 104 module_param(enable_apicv, bool, S_IRUGO);
 105
 106 /*
 107  * If nested=1, nested virtualization is supported, i.e., guests may use
 108  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 109  * use VMX instructions.
 110  */
 111 static bool __read_mostly nested = 1;
 112 module_param(nested, bool, S_IRUGO);
 113
 114 bool __read_mostly enable_pml = 1;
 115 module_param_named(pml, enable_pml, bool, S_IRUGO);
 116
 117 static bool __read_mostly dump_invalid_vmcs = 0;
 118 module_param(dump_invalid_vmcs, bool, 0644);
 119
 120 #define MSR_BITMAP_MODE_X2APIC          1
 121 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 122
 123 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 124
 125 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 126 static int __read_mostly cpu_preemption_timer_multi;
 127 static bool __read_mostly enable_preemption_timer = 1;
 128 #ifdef CONFIG_X86_64
 129 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 130 #endif
 131
 132 extern bool __read_mostly allow_smaller_maxphyaddr;
 133 module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);
 134
 135 #define KVM_VM_CR0_ALWAYS_OFF (X86_CR0_NW | X86_CR0_CD)
 136 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
 137 #define KVM_VM_CR0_ALWAYS_ON                            \
 138         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST |      \
 139          X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
 140
 141 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
 142 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 143 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 144
 145 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 146
 147 #define MSR_IA32_RTIT_STATUS_MASK (~(RTIT_STATUS_FILTEREN | \
 148         RTIT_STATUS_CONTEXTEN | RTIT_STATUS_TRIGGEREN | \
 149         RTIT_STATUS_ERROR | RTIT_STATUS_STOPPED | \
 150         RTIT_STATUS_BYTECNT))
 151
 152 #define MSR_IA32_RTIT_OUTPUT_BASE_MASK \
 153         (~((1UL << cpuid_query_maxphyaddr(vcpu)) - 1) | 0x7f)
 154
 155 /*
 156  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 157  * ple_gap:    upper bound on the amount of time between two successive
 158  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 159  *             According to test, this time is usually smaller than 128 cycles.
 160  * ple_window: upper bound on the amount of time a guest is allowed to execute
 161  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 162  *             less than 2^12 cycles
 163  * Time is measured based on a counter that runs at the same rate as the TSC,
 164  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 165  */
 166 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
 167 module_param(ple_gap, uint, 0444);
 168
 169 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 170 module_param(ple_window, uint, 0444);
 171
 172 /* Default doubles per-vcpu window every exit. */
 173 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 174 module_param(ple_window_grow, uint, 0444);
 175
 176 /* Default resets per-vcpu window every exit to ple_window. */
 177 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 178 module_param(ple_window_shrink, uint, 0444);
 179
 180 /* Default is to compute the maximum so we can never overflow. */
 181 static unsigned int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 182 module_param(ple_window_max, uint, 0444);
 183
 184 /* Default is SYSTEM mode, 1 for host-guest mode */
 185 int __read_mostly pt_mode = PT_MODE_SYSTEM;
 186 module_param(pt_mode, int, S_IRUGO);
 187
 188 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 189 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 190 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 191
 192 /* Storage for pre module init parameter parsing */
 193 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
 194
 195 static const struct {
 196         const char *option;
 197         bool for_parse;
 198 } vmentry_l1d_param[] = {
 199         [VMENTER_L1D_FLUSH_AUTO]         = {"auto", true},
 200         [VMENTER_L1D_FLUSH_NEVER]        = {"never", true},
 201         [VMENTER_L1D_FLUSH_COND]         = {"cond", true},
 202         [VMENTER_L1D_FLUSH_ALWAYS]       = {"always", true},
 203         [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
 204         [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
 205 };
 206
 207 #define L1D_CACHE_ORDER 4
 208 static void *vmx_l1d_flush_pages;
 209
 210 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 211 {
 212         struct page *page;
 213         unsigned int i;
 214
 215         if (!boot_cpu_has_bug(X86_BUG_L1TF)) {
 216                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 217                 return 0;
 218         }
 219
 220         if (!enable_ept) {
 221                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
 222                 return 0;
 223         }
 224
 225         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
 226                 u64 msr;
 227
 228                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
 229                 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
 230                         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 231                         return 0;
 232                 }
 233         }
 234
 235         /* If set to auto use the default l1tf mitigation method */
 236         if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
 237                 switch (l1tf_mitigation) {
 238                 case L1TF_MITIGATION_OFF:
 239                         l1tf = VMENTER_L1D_FLUSH_NEVER;
 240                         break;
 241                 case L1TF_MITIGATION_FLUSH_NOWARN:
 242                 case L1TF_MITIGATION_FLUSH:
 243                 case L1TF_MITIGATION_FLUSH_NOSMT:
 244                         l1tf = VMENTER_L1D_FLUSH_COND;
 245                         break;
 246                 case L1TF_MITIGATION_FULL:
 247                 case L1TF_MITIGATION_FULL_FORCE:
 248                         l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 249                         break;
 250                 }
 251         } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
 252                 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 253         }
 254
 255         if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
 256             !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
 257                 /*
 258                  * This allocation for vmx_l1d_flush_pages is not tied to a VM
 259                  * lifetime and so should not be charged to a memcg.
 260                  */
 261                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
 262                 if (!page)
 263                         return -ENOMEM;
 264                 vmx_l1d_flush_pages = page_address(page);
 265
 266                 /*
 267                  * Initialize each page with a different pattern in
 268                  * order to protect against KSM in the nested
 269                  * virtualization case.
 270                  */
 271                 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
 272                         memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
 273                                PAGE_SIZE);
 274                 }
 275         }
 276
 277         l1tf_vmx_mitigation = l1tf;
 278
 279         if (l1tf != VMENTER_L1D_FLUSH_NEVER)
 280                 static_branch_enable(&vmx_l1d_should_flush);
 281         else
 282                 static_branch_disable(&vmx_l1d_should_flush);
 283
 284         if (l1tf == VMENTER_L1D_FLUSH_COND)
 285                 static_branch_enable(&vmx_l1d_flush_cond);
 286         else
 287                 static_branch_disable(&vmx_l1d_flush_cond);
 288         return 0;
 289 }
 290
 291 static int vmentry_l1d_flush_parse(const char *s)
 292 {
 293         unsigned int i;
 294
 295         if (s) {
 296                 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 297                         if (vmentry_l1d_param[i].for_parse &&
 298                             sysfs_streq(s, vmentry_l1d_param[i].option))
 299                                 return i;
 300                 }
 301         }
 302         return -EINVAL;
 303 }
 304
 305 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 306 {
 307         int l1tf, ret;
 308
 309         l1tf = vmentry_l1d_flush_parse(s);
 310         if (l1tf < 0)
 311                 return l1tf;
 312
 313         if (!boot_cpu_has(X86_BUG_L1TF))
 314                 return 0;
 315
 316         /*
 317          * Has vmx_init() run already? If not then this is the pre init
 318          * parameter parsing. In that case just store the value and let
 319          * vmx_init() do the proper setup after enable_ept has been
 320          * established.
 321          */
 322         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
 323                 vmentry_l1d_flush_param = l1tf;
 324                 return 0;
 325         }
 326
 327         mutex_lock(&vmx_l1d_flush_mutex);
 328         ret = vmx_setup_l1d_flush(l1tf);
 329         mutex_unlock(&vmx_l1d_flush_mutex);
 330         return ret;
 331 }
 332
 333 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 334 {
 335         if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
 336                 return sprintf(s, "???\n");
 337
 338         return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 339 }
 340
 341 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 342         .set = vmentry_l1d_flush_set,
 343         .get = vmentry_l1d_flush_get,
 344 };
 345 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 346
 347 static bool guest_state_valid(struct kvm_vcpu *vcpu);
 348 static u32 vmx_segment_access_rights(struct kvm_segment *var);
 349 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 350                                                           u32 msr, int type);
 351
 352 void vmx_vmexit(void);
 353
 354 #define vmx_insn_failed(fmt...)         \
 355 do {                                    \
 356         WARN_ONCE(1, fmt);              \
 357         pr_warn_ratelimited(fmt);       \
 358 } while (0)
 359
 360 asmlinkage void vmread_error(unsigned long field, bool fault)
 361 {
 362         if (fault)
 363                 kvm_spurious_fault();
 364         else
 365                 vmx_insn_failed("kvm: vmread failed: field=%lx\n", field);
 366 }
 367
 368 noinline void vmwrite_error(unsigned long field, unsigned long value)
 369 {
 370         vmx_insn_failed("kvm: vmwrite failed: field=%lx val=%lx err=%d\n",
 371                         field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
 372 }
 373
 374 noinline void vmclear_error(struct vmcs *vmcs, u64 phys_addr)
 375 {
 376         vmx_insn_failed("kvm: vmclear failed: %p/%llx\n", vmcs, phys_addr);
 377 }
 378
 379 noinline void vmptrld_error(struct vmcs *vmcs, u64 phys_addr)
 380 {
 381         vmx_insn_failed("kvm: vmptrld failed: %p/%llx\n", vmcs, phys_addr);
 382 }
 383
 384 noinline void invvpid_error(unsigned long ext, u16 vpid, gva_t gva)
 385 {
 386         vmx_insn_failed("kvm: invvpid failed: ext=0x%lx vpid=%u gva=0x%lx\n",
 387                         ext, vpid, gva);
 388 }
 389
 390 noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa)
 391 {
 392         vmx_insn_failed("kvm: invept failed: ext=0x%lx eptp=%llx gpa=0x%llx\n",
 393                         ext, eptp, gpa);
 394 }
 395
 396 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 397 DEFINE_PER_CPU(struct vmcs *, current_vmcs);
 398 /*
 399  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
 400  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
 401  */
 402 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
 403
 404 /*
 405  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
 406  * can find which vCPU should be waken up.
 407  */
 408 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
 409 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
 410
 411 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
 412 static DEFINE_SPINLOCK(vmx_vpid_lock);
 413
 414 struct vmcs_config vmcs_config;
 415 struct vmx_capability vmx_capability;
 416
 417 #define VMX_SEGMENT_FIELD(seg)                                  \
 418         [VCPU_SREG_##seg] = {                                   \
 419                 .selector = GUEST_##seg##_SELECTOR,             \
 420                 .base = GUEST_##seg##_BASE,                     \
 421                 .limit = GUEST_##seg##_LIMIT,                   \
 422                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
 423         }
 424
 425 static const struct kvm_vmx_segment_field {
 426         unsigned selector;
 427         unsigned base;
 428         unsigned limit;
 429         unsigned ar_bytes;
 430 } kvm_vmx_segment_fields[] = {
 431         VMX_SEGMENT_FIELD(CS),
 432         VMX_SEGMENT_FIELD(DS),
 433         VMX_SEGMENT_FIELD(ES),
 434         VMX_SEGMENT_FIELD(FS),
 435         VMX_SEGMENT_FIELD(GS),
 436         VMX_SEGMENT_FIELD(SS),
 437         VMX_SEGMENT_FIELD(TR),
 438         VMX_SEGMENT_FIELD(LDTR),
 439 };
 440
 441 static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
 442 {
 443         vmx->segment_cache.bitmask = 0;
 444 }
 445
 446 static unsigned long host_idt_base;
 447
 448 /*
 449  * Though SYSCALL is only supported in 64-bit mode on Intel CPUs, kvm
 450  * will emulate SYSCALL in legacy mode if the vendor string in guest
 451  * CPUID.0:{EBX,ECX,EDX} is "AuthenticAMD" or "AMDisbetter!" To
 452  * support this emulation, IA32_STAR must always be included in
 453  * vmx_msr_index[], even in i386 builds.
 454  */
 455 const u32 vmx_msr_index[] = {
 456 #ifdef CONFIG_X86_64
 457         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
 458 #endif
 459         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
 460         MSR_IA32_TSX_CTRL,
 461 };
 462
 463 #if IS_ENABLED(CONFIG_HYPERV)
 464 static bool __read_mostly enlightened_vmcs = true;
 465 module_param(enlightened_vmcs, bool, 0444);
 466
 467 /* check_ept_pointer() should be under protection of ept_pointer_lock. */
 468 static void check_ept_pointer_match(struct kvm *kvm)
 469 {
 470         struct kvm_vcpu *vcpu;
 471         u64 tmp_eptp = INVALID_PAGE;
 472         int i;
 473
 474         kvm_for_each_vcpu(i, vcpu, kvm) {
 475                 if (!VALID_PAGE(tmp_eptp)) {
 476                         tmp_eptp = to_vmx(vcpu)->ept_pointer;
 477                 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
 478                         to_kvm_vmx(kvm)->ept_pointers_match
 479                                 = EPT_POINTERS_MISMATCH;
 480                         return;
 481                 }
 482         }
 483
 484         to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
 485 }
 486
 487 static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush,
 488                 void *data)
 489 {
 490         struct kvm_tlb_range *range = data;
 491
 492         return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
 493                         range->pages);
 494 }
 495
 496 static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
 497                 struct kvm_vcpu *vcpu, struct kvm_tlb_range *range)
 498 {
 499         u64 ept_pointer = to_vmx(vcpu)->ept_pointer;
 500
 501         /*
 502          * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs address
 503          * of the base of EPT PML4 table, strip off EPT configuration
 504          * information.
 505          */
 506         if (range)
 507                 return hyperv_flush_guest_mapping_range(ept_pointer & PAGE_MASK,
 508                                 kvm_fill_hv_flush_list_func, (void *)range);
 509         else
 510                 return hyperv_flush_guest_mapping(ept_pointer & PAGE_MASK);
 511 }
 512
 513 static int hv_remote_flush_tlb_with_range(struct kvm *kvm,
 514                 struct kvm_tlb_range *range)
 515 {
 516         struct kvm_vcpu *vcpu;
 517         int ret = 0, i;
 518
 519         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
 520
 521         if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
 522                 check_ept_pointer_match(kvm);
 523
 524         if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
 525                 kvm_for_each_vcpu(i, vcpu, kvm) {
 526                         /* If ept_pointer is invalid pointer, bypass flush request. */
 527                         if (VALID_PAGE(to_vmx(vcpu)->ept_pointer))
 528                                 ret |= __hv_remote_flush_tlb_with_range(
 529                                         kvm, vcpu, range);
 530                 }
 531         } else {
 532                 ret = __hv_remote_flush_tlb_with_range(kvm,
 533                                 kvm_get_vcpu(kvm, 0), range);
 534         }
 535
 536         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
 537         return ret;
 538 }
 539 static int hv_remote_flush_tlb(struct kvm *kvm)
 540 {
 541         return hv_remote_flush_tlb_with_range(kvm, NULL);
 542 }
 543
 544 static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu)
 545 {
 546         struct hv_enlightened_vmcs *evmcs;
 547         struct hv_partition_assist_pg **p_hv_pa_pg =
 548                         &vcpu->kvm->arch.hyperv.hv_pa_pg;
 549         /*
 550          * Synthetic VM-Exit is not enabled in current code and so All
 551          * evmcs in singe VM shares same assist page.
 552          */
 553         if (!*p_hv_pa_pg)
 554                 *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL);
 555
 556         if (!*p_hv_pa_pg)
 557                 return -ENOMEM;
 558
 559         evmcs = (struct hv_enlightened_vmcs *)to_vmx(vcpu)->loaded_vmcs->vmcs;
 560
 561         evmcs->partition_assist_page =
 562                 __pa(*p_hv_pa_pg);
 563         evmcs->hv_vm_id = (unsigned long)vcpu->kvm;
 564         evmcs->hv_enlightenments_control.nested_flush_hypercall = 1;
 565
 566         return 0;
 567 }
 568
 569 #endif /* IS_ENABLED(CONFIG_HYPERV) */
 570
 571 /*
 572  * Comment's format: document - errata name - stepping - processor name.
 573  * Refer from
 574  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
 575  */
 576 static u32 vmx_preemption_cpu_tfms[] = {
 577 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
 578 0x000206E6,
 579 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
 580 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
 581 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
 582 0x00020652,
 583 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
 584 0x00020655,
 585 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
 586 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
 587 /*
 588  * 320767.pdf - AAP86  - B1 -
 589  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
 590  */
 591 0x000106E5,
 592 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
 593 0x000106A0,
 594 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
 595 0x000106A1,
 596 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
 597 0x000106A4,
 598  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
 599  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
 600  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
 601 0x000106A5,
 602  /* Xeon E3-1220 V2 */
 603 0x000306A8,
 604 };
 605
 606 static inline bool cpu_has_broken_vmx_preemption_timer(void)
 607 {
 608         u32 eax = cpuid_eax(0x00000001), i;
 609
 610         /* Clear the reserved bits */
 611         eax &= ~(0x3U << 14 | 0xfU << 28);
 612         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
 613                 if (eax == vmx_preemption_cpu_tfms[i])
 614                         return true;
 615
 616         return false;
 617 }
 618
 619 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
 620 {
 621         return flexpriority_enabled && lapic_in_kernel(vcpu);
 622 }
 623
 624 static inline bool report_flexpriority(void)
 625 {
 626         return flexpriority_enabled;
 627 }
 628
 629 static inline int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
 630 {
 631         int i;
 632
 633         for (i = 0; i < vmx->nmsrs; ++i)
 634                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
 635                         return i;
 636         return -1;
 637 }
 638
 639 struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
 640 {
 641         int i;
 642
 643         i = __find_msr_index(vmx, msr);
 644         if (i >= 0)
 645                 return &vmx->guest_msrs[i];
 646         return NULL;
 647 }
 648
 649 static int vmx_set_guest_msr(struct vcpu_vmx *vmx, struct shared_msr_entry *msr, u64 data)
 650 {
 651         int ret = 0;
 652
 653         u64 old_msr_data = msr->data;
 654         msr->data = data;
 655         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
 656                 preempt_disable();
 657                 ret = kvm_set_shared_msr(msr->index, msr->data,
 658                                          msr->mask);
 659                 preempt_enable();
 660                 if (ret)
 661                         msr->data = old_msr_data;
 662         }
 663         return ret;
 664 }
 665
 666 #ifdef CONFIG_KEXEC_CORE
 667 static void crash_vmclear_local_loaded_vmcss(void)
 668 {
 669         int cpu = raw_smp_processor_id();
 670         struct loaded_vmcs *v;
 671
 672         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
 673                             loaded_vmcss_on_cpu_link)
 674                 vmcs_clear(v->vmcs);
 675 }
 676 #endif /* CONFIG_KEXEC_CORE */
 677
 678 static void __loaded_vmcs_clear(void *arg)
 679 {
 680         struct loaded_vmcs *loaded_vmcs = arg;
 681         int cpu = raw_smp_processor_id();
 682
 683         if (loaded_vmcs->cpu != cpu)
 684                 return; /* vcpu migration can race with cpu offline */
 685         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
 686                 per_cpu(current_vmcs, cpu) = NULL;
 687
 688         vmcs_clear(loaded_vmcs->vmcs);
 689         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
 690                 vmcs_clear(loaded_vmcs->shadow_vmcs);
 691
 692         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
 693
 694         /*
 695          * Ensure all writes to loaded_vmcs, including deleting it from its
 696          * current percpu list, complete before setting loaded_vmcs->vcpu to
 697          * -1, otherwise a different cpu can see vcpu == -1 first and add
 698          * loaded_vmcs to its percpu list before it's deleted from this cpu's
 699          * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs().
 700          */
 701         smp_wmb();
 702
 703         loaded_vmcs->cpu = -1;
 704         loaded_vmcs->launched = 0;
 705 }
 706
 707 void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
 708 {
 709         int cpu = loaded_vmcs->cpu;
 710
 711         if (cpu != -1)
 712                 smp_call_function_single(cpu,
 713                          __loaded_vmcs_clear, loaded_vmcs, 1);
 714 }
 715
 716 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
 717                                        unsigned field)
 718 {
 719         bool ret;
 720         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
 721
 722         if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) {
 723                 kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS);
 724                 vmx->segment_cache.bitmask = 0;
 725         }
 726         ret = vmx->segment_cache.bitmask & mask;
 727         vmx->segment_cache.bitmask |= mask;
 728         return ret;
 729 }
 730
 731 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
 732 {
 733         u16 *p = &vmx->segment_cache.seg[seg].selector;
 734
 735         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
 736                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
 737         return *p;
 738 }
 739
 740 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
 741 {
 742         ulong *p = &vmx->segment_cache.seg[seg].base;
 743
 744         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
 745                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
 746         return *p;
 747 }
 748
 749 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
 750 {
 751         u32 *p = &vmx->segment_cache.seg[seg].limit;
 752
 753         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
 754                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
 755         return *p;
 756 }
 757
 758 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
 759 {
 760         u32 *p = &vmx->segment_cache.seg[seg].ar;
 761
 762         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
 763                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
 764         return *p;
 765 }
 766
 767 void update_exception_bitmap(struct kvm_vcpu *vcpu)
 768 {
 769         u32 eb;
 770
 771         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
 772              (1u << DB_VECTOR) | (1u << AC_VECTOR);
 773         /*
 774          * Guest access to VMware backdoor ports could legitimately
 775          * trigger #GP because of TSS I/O permission bitmap.
 776          * We intercept those #GP and allow access to them anyway
 777          * as VMware does.
 778          */
 779         if (enable_vmware_backdoor)
 780                 eb |= (1u << GP_VECTOR);
 781         if ((vcpu->guest_debug &
 782              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
 783             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
 784                 eb |= 1u << BP_VECTOR;
 785         if (to_vmx(vcpu)->rmode.vm86_active)
 786                 eb = ~0;
 787         if (!vmx_need_pf_intercept(vcpu))
 788                 eb &= ~(1u << PF_VECTOR);
 789
 790         /* When we are running a nested L2 guest and L1 specified for it a
 791          * certain exception bitmap, we must trap the same exceptions and pass
 792          * them to L1. When running L2, we will only handle the exceptions
 793          * specified above if L1 did not want them.
 794          */
 795         if (is_guest_mode(vcpu))
 796                 eb |= get_vmcs12(vcpu)->exception_bitmap;
 797         else {
 798                 /*
 799                  * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched
 800                  * between guest and host.  In that case we only care about present
 801                  * faults.  For vmcs02, however, PFEC_MASK and PFEC_MATCH are set in
 802                  * prepare_vmcs02_rare.
 803                  */
 804                 bool selective_pf_trap = enable_ept && (eb & (1u << PF_VECTOR));
 805                 int mask = selective_pf_trap ? PFERR_PRESENT_MASK : 0;
 806                 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask);
 807                 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, mask);
 808         }
 809
 810         vmcs_write32(EXCEPTION_BITMAP, eb);
 811 }
 812
 813 /*
 814  * Check if MSR is intercepted for currently loaded MSR bitmap.
 815  */
 816 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 817 {
 818         unsigned long *msr_bitmap;
 819         int f = sizeof(unsigned long);
 820
 821         if (!cpu_has_vmx_msr_bitmap())
 822                 return true;
 823
 824         msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
 825
 826         if (msr <= 0x1fff) {
 827                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
 828         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
 829                 msr &= 0x1fff;
 830                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
 831         }
 832
 833         return true;
 834 }
 835
 836 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 837                 unsigned long entry, unsigned long exit)
 838 {
 839         vm_entry_controls_clearbit(vmx, entry);
 840         vm_exit_controls_clearbit(vmx, exit);
 841 }
 842
 843 int vmx_find_msr_index(struct vmx_msrs *m, u32 msr)
 844 {
 845         unsigned int i;
 846
 847         for (i = 0; i < m->nr; ++i) {
 848                 if (m->val[i].index == msr)
 849                         return i;
 850         }
 851         return -ENOENT;
 852 }
 853
 854 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
 855 {
 856         int i;
 857         struct msr_autoload *m = &vmx->msr_autoload;
 858
 859         switch (msr) {
 860         case MSR_EFER:
 861                 if (cpu_has_load_ia32_efer()) {
 862                         clear_atomic_switch_msr_special(vmx,
 863                                         VM_ENTRY_LOAD_IA32_EFER,
 864                                         VM_EXIT_LOAD_IA32_EFER);
 865                         return;
 866                 }
 867                 break;
 868         case MSR_CORE_PERF_GLOBAL_CTRL:
 869                 if (cpu_has_load_perf_global_ctrl()) {
 870                         clear_atomic_switch_msr_special(vmx,
 871                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 872                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
 873                         return;
 874                 }
 875                 break;
 876         }
 877         i = vmx_find_msr_index(&m->guest, msr);
 878         if (i < 0)
 879                 goto skip_guest;
 880         --m->guest.nr;
 881         m->guest.val[i] = m->guest.val[m->guest.nr];
 882         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 883
 884 skip_guest:
 885         i = vmx_find_msr_index(&m->host, msr);
 886         if (i < 0)
 887                 return;
 888
 889         --m->host.nr;
 890         m->host.val[i] = m->host.val[m->host.nr];
 891         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 892 }
 893
 894 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 895                 unsigned long entry, unsigned long exit,
 896                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
 897                 u64 guest_val, u64 host_val)
 898 {
 899         vmcs_write64(guest_val_vmcs, guest_val);
 900         if (host_val_vmcs != HOST_IA32_EFER)
 901                 vmcs_write64(host_val_vmcs, host_val);
 902         vm_entry_controls_setbit(vmx, entry);
 903         vm_exit_controls_setbit(vmx, exit);
 904 }
 905
 906 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
 907                                   u64 guest_val, u64 host_val, bool entry_only)
 908 {
 909         int i, j = 0;
 910         struct msr_autoload *m = &vmx->msr_autoload;
 911
 912         switch (msr) {
 913         case MSR_EFER:
 914                 if (cpu_has_load_ia32_efer()) {
 915                         add_atomic_switch_msr_special(vmx,
 916                                         VM_ENTRY_LOAD_IA32_EFER,
 917                                         VM_EXIT_LOAD_IA32_EFER,
 918                                         GUEST_IA32_EFER,
 919                                         HOST_IA32_EFER,
 920                                         guest_val, host_val);
 921                         return;
 922                 }
 923                 break;
 924         case MSR_CORE_PERF_GLOBAL_CTRL:
 925                 if (cpu_has_load_perf_global_ctrl()) {
 926                         add_atomic_switch_msr_special(vmx,
 927                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
 928                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
 929                                         GUEST_IA32_PERF_GLOBAL_CTRL,
 930                                         HOST_IA32_PERF_GLOBAL_CTRL,
 931                                         guest_val, host_val);
 932                         return;
 933                 }
 934                 break;
 935         case MSR_IA32_PEBS_ENABLE:
 936                 /* PEBS needs a quiescent period after being disabled (to write
 937                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
 938                  * provide that period, so a CPU could write host's record into
 939                  * guest's memory.
 940                  */
 941                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
 942         }
 943
 944         i = vmx_find_msr_index(&m->guest, msr);
 945         if (!entry_only)
 946                 j = vmx_find_msr_index(&m->host, msr);
 947
 948         if ((i < 0 && m->guest.nr == NR_LOADSTORE_MSRS) ||
 949                 (j < 0 &&  m->host.nr == NR_LOADSTORE_MSRS)) {
 950                 printk_once(KERN_WARNING "Not enough msr switch entries. "
 951                                 "Can't add msr %x\n", msr);
 952                 return;
 953         }
 954         if (i < 0) {
 955                 i = m->guest.nr++;
 956                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
 957         }
 958         m->guest.val[i].index = msr;
 959         m->guest.val[i].value = guest_val;
 960
 961         if (entry_only)
 962                 return;
 963
 964         if (j < 0) {
 965                 j = m->host.nr++;
 966                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
 967         }
 968         m->host.val[j].index = msr;
 969         m->host.val[j].value = host_val;
 970 }
 971
 972 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 973 {
 974         u64 guest_efer = vmx->vcpu.arch.efer;
 975         u64 ignore_bits = 0;
 976
 977         /* Shadow paging assumes NX to be available.  */
 978         if (!enable_ept)
 979                 guest_efer |= EFER_NX;
 980
 981         /*
 982          * LMA and LME handled by hardware; SCE meaningless outside long mode.
 983          */
 984         ignore_bits |= EFER_SCE;
 985 #ifdef CONFIG_X86_64
 986         ignore_bits |= EFER_LMA | EFER_LME;
 987         /* SCE is meaningful only in long mode on Intel */
 988         if (guest_efer & EFER_LMA)
 989                 ignore_bits &= ~(u64)EFER_SCE;
 990 #endif
 991
 992         /*
 993          * On EPT, we can't emulate NX, so we must switch EFER atomically.
 994          * On CPUs that support "load IA32_EFER", always switch EFER
 995          * atomically, since it's faster than switching it manually.
 996          */
 997         if (cpu_has_load_ia32_efer() ||
 998             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
 999                 if (!(guest_efer & EFER_LMA))
1000                         guest_efer &= ~EFER_LME;
1001                 if (guest_efer != host_efer)
1002                         add_atomic_switch_msr(vmx, MSR_EFER,
1003                                               guest_efer, host_efer, false);
1004                 else
1005                         clear_atomic_switch_msr(vmx, MSR_EFER);
1006                 return false;
1007         } else {
1008                 clear_atomic_switch_msr(vmx, MSR_EFER);
1009
1010                 guest_efer &= ~ignore_bits;
1011                 guest_efer |= host_efer & ignore_bits;
1012
1013                 vmx->guest_msrs[efer_offset].data = guest_efer;
1014                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
1015
1016                 return true;
1017         }
1018 }
1019
1020 #ifdef CONFIG_X86_32
1021 /*
1022  * On 32-bit kernels, VM exits still load the FS and GS bases from the
1023  * VMCS rather than the segment table.  KVM uses this helper to figure
1024  * out the current bases to poke them into the VMCS before entry.
1025  */
1026 static unsigned long segment_base(u16 selector)
1027 {
1028         struct desc_struct *table;
1029         unsigned long v;
1030
1031         if (!(selector & ~SEGMENT_RPL_MASK))
1032                 return 0;
1033
1034         table = get_current_gdt_ro();
1035
1036         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
1037                 u16 ldt_selector = kvm_read_ldt();
1038
1039                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
1040                         return 0;
1041
1042                 table = (struct desc_struct *)segment_base(ldt_selector);
1043         }
1044         v = get_desc_base(&table[selector >> 3]);
1045         return v;
1046 }
1047 #endif
1048
1049 static inline bool pt_can_write_msr(struct vcpu_vmx *vmx)
1050 {
1051         return vmx_pt_mode_is_host_guest() &&
1052                !(vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN);
1053 }
1054
1055 static inline void pt_load_msr(struct pt_ctx *ctx, u32 addr_range)
1056 {
1057         u32 i;
1058
1059         wrmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1060         wrmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1061         wrmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1062         wrmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1063         for (i = 0; i < addr_range; i++) {
1064                 wrmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1065                 wrmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1066         }
1067 }
1068
1069 static inline void pt_save_msr(struct pt_ctx *ctx, u32 addr_range)
1070 {
1071         u32 i;
1072
1073         rdmsrl(MSR_IA32_RTIT_STATUS, ctx->status);
1074         rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, ctx->output_base);
1075         rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, ctx->output_mask);
1076         rdmsrl(MSR_IA32_RTIT_CR3_MATCH, ctx->cr3_match);
1077         for (i = 0; i < addr_range; i++) {
1078                 rdmsrl(MSR_IA32_RTIT_ADDR0_A + i * 2, ctx->addr_a[i]);
1079                 rdmsrl(MSR_IA32_RTIT_ADDR0_B + i * 2, ctx->addr_b[i]);
1080         }
1081 }
1082
1083 static void pt_guest_enter(struct vcpu_vmx *vmx)
1084 {
1085         if (vmx_pt_mode_is_system())
1086                 return;
1087
1088         /*
1089          * GUEST_IA32_RTIT_CTL is already set in the VMCS.
1090          * Save host state before VM entry.
1091          */
1092         rdmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1093         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1094                 wrmsrl(MSR_IA32_RTIT_CTL, 0);
1095                 pt_save_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1096                 pt_load_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1097         }
1098 }
1099
1100 static void pt_guest_exit(struct vcpu_vmx *vmx)
1101 {
1102         if (vmx_pt_mode_is_system())
1103                 return;
1104
1105         if (vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) {
1106                 pt_save_msr(&vmx->pt_desc.guest, vmx->pt_desc.addr_range);
1107                 pt_load_msr(&vmx->pt_desc.host, vmx->pt_desc.addr_range);
1108         }
1109
1110         /* Reload host state (IA32_RTIT_CTL will be cleared on VM exit). */
1111         wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
1112 }
1113
1114 void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
1115                         unsigned long fs_base, unsigned long gs_base)
1116 {
1117         if (unlikely(fs_sel != host->fs_sel)) {
1118                 if (!(fs_sel & 7))
1119                         vmcs_write16(HOST_FS_SELECTOR, fs_sel);
1120                 else
1121                         vmcs_write16(HOST_FS_SELECTOR, 0);
1122                 host->fs_sel = fs_sel;
1123         }
1124         if (unlikely(gs_sel != host->gs_sel)) {
1125                 if (!(gs_sel & 7))
1126                         vmcs_write16(HOST_GS_SELECTOR, gs_sel);
1127                 else
1128                         vmcs_write16(HOST_GS_SELECTOR, 0);
1129                 host->gs_sel = gs_sel;
1130         }
1131         if (unlikely(fs_base != host->fs_base)) {
1132                 vmcs_writel(HOST_FS_BASE, fs_base);
1133                 host->fs_base = fs_base;
1134         }
1135         if (unlikely(gs_base != host->gs_base)) {
1136                 vmcs_writel(HOST_GS_BASE, gs_base);
1137                 host->gs_base = gs_base;
1138         }
1139 }
1140
1141 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1142 {
1143         struct vcpu_vmx *vmx = to_vmx(vcpu);
1144         struct vmcs_host_state *host_state;
1145 #ifdef CONFIG_X86_64
1146         int cpu = raw_smp_processor_id();
1147 #endif
1148         unsigned long fs_base, gs_base;
1149         u16 fs_sel, gs_sel;
1150         int i;
1151
1152         vmx->req_immediate_exit = false;
1153
1154         /*
1155          * Note that guest MSRs to be saved/restored can also be changed
1156          * when guest state is loaded. This happens when guest transitions
1157          * to/from long-mode by setting MSR_EFER.LMA.
1158          */
1159         if (!vmx->guest_msrs_ready) {
1160                 vmx->guest_msrs_ready = true;
1161                 for (i = 0; i < vmx->save_nmsrs; ++i)
1162                         kvm_set_shared_msr(vmx->guest_msrs[i].index,
1163                                            vmx->guest_msrs[i].data,
1164                                            vmx->guest_msrs[i].mask);
1165
1166         }
1167
1168         if (vmx->nested.need_vmcs12_to_shadow_sync)
1169                 nested_sync_vmcs12_to_shadow(vcpu);
1170
1171         if (vmx->guest_state_loaded)
1172                 return;
1173
1174         host_state = &vmx->loaded_vmcs->host_state;
1175
1176         /*
1177          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
1178          * allow segment selectors with cpl > 0 or ti == 1.
1179          */
1180         host_state->ldt_sel = kvm_read_ldt();
1181
1182 #ifdef CONFIG_X86_64
1183         savesegment(ds, host_state->ds_sel);
1184         savesegment(es, host_state->es_sel);
1185
1186         gs_base = cpu_kernelmode_gs_base(cpu);
1187         if (likely(is_64bit_mm(current->mm))) {
1188                 current_save_fsgs();
1189                 fs_sel = current->thread.fsindex;
1190                 gs_sel = current->thread.gsindex;
1191                 fs_base = current->thread.fsbase;
1192                 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
1193         } else {
1194                 savesegment(fs, fs_sel);
1195                 savesegment(gs, gs_sel);
1196                 fs_base = read_msr(MSR_FS_BASE);
1197                 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
1198         }
1199
1200         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1201 #else
1202         savesegment(fs, fs_sel);
1203         savesegment(gs, gs_sel);
1204         fs_base = segment_base(fs_sel);
1205         gs_base = segment_base(gs_sel);
1206 #endif
1207
1208         vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
1209         vmx->guest_state_loaded = true;
1210 }
1211
1212 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
1213 {
1214         struct vmcs_host_state *host_state;
1215
1216         if (!vmx->guest_state_loaded)
1217                 return;
1218
1219         host_state = &vmx->loaded_vmcs->host_state;
1220
1221         ++vmx->vcpu.stat.host_state_reload;
1222
1223 #ifdef CONFIG_X86_64
1224         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1225 #endif
1226         if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
1227                 kvm_load_ldt(host_state->ldt_sel);
1228 #ifdef CONFIG_X86_64
1229                 load_gs_index(host_state->gs_sel);
1230 #else
1231                 loadsegment(gs, host_state->gs_sel);
1232 #endif
1233         }
1234         if (host_state->fs_sel & 7)
1235                 loadsegment(fs, host_state->fs_sel);
1236 #ifdef CONFIG_X86_64
1237         if (unlikely(host_state->ds_sel | host_state->es_sel)) {
1238                 loadsegment(ds, host_state->ds_sel);
1239                 loadsegment(es, host_state->es_sel);
1240         }
1241 #endif
1242         invalidate_tss_limit();
1243 #ifdef CONFIG_X86_64
1244         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1245 #endif
1246         load_fixmap_gdt(raw_smp_processor_id());
1247         vmx->guest_state_loaded = false;
1248         vmx->guest_msrs_ready = false;
1249 }
1250
1251 #ifdef CONFIG_X86_64
1252 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
1253 {
1254         preempt_disable();
1255         if (vmx->guest_state_loaded)
1256                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
1257         preempt_enable();
1258         return vmx->msr_guest_kernel_gs_base;
1259 }
1260
1261 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
1262 {
1263         preempt_disable();
1264         if (vmx->guest_state_loaded)
1265                 wrmsrl(MSR_KERNEL_GS_BASE, data);
1266         preempt_enable();
1267         vmx->msr_guest_kernel_gs_base = data;
1268 }
1269 #endif
1270
1271 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
1272 {
1273         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1274         struct pi_desc old, new;
1275         unsigned int dest;
1276
1277         /*
1278          * In case of hot-plug or hot-unplug, we may have to undo
1279          * vmx_vcpu_pi_put even if there is no assigned device.  And we
1280          * always keep PI.NDST up to date for simplicity: it makes the
1281          * code easier, and CPU migration is not a fast path.
1282          */
1283         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
1284                 return;
1285
1286         /*
1287          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
1288          * PI.NDST: pi_post_block is the one expected to change PID.NDST and the
1289          * wakeup handler expects the vCPU to be on the blocked_vcpu_list that
1290          * matches PI.NDST. Otherwise, a vcpu may not be able to be woken up
1291          * correctly.
1292          */
1293         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR || vcpu->cpu == cpu) {
1294                 pi_clear_sn(pi_desc);
1295                 goto after_clear_sn;
1296         }
1297
1298         /* The full case.  */
1299         do {
1300                 old.control = new.control = pi_desc->control;
1301
1302                 dest = cpu_physical_id(cpu);
1303
1304                 if (x2apic_enabled())
1305                         new.ndst = dest;
1306                 else
1307                         new.ndst = (dest << 8) & 0xFF00;
1308
1309                 new.sn = 0;
1310         } while (cmpxchg64(&pi_desc->control, old.control,
1311                            new.control) != old.control);
1312
1313 after_clear_sn:
1314
1315         /*
1316          * Clear SN before reading the bitmap.  The VT-d firmware
1317          * writes the bitmap and reads SN atomically (5.2.3 in the
1318          * spec), so it doesn't really have a memory barrier that
1319          * pairs with this, but we cannot do that and we need one.
1320          */
1321         smp_mb__after_atomic();
1322
1323         if (!pi_is_pir_empty(pi_desc))
1324                 pi_set_on(pi_desc);
1325 }
1326
1327 void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
1328                         struct loaded_vmcs *buddy)
1329 {
1330         struct vcpu_vmx *vmx = to_vmx(vcpu);
1331         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
1332         struct vmcs *prev;
1333
1334         if (!already_loaded) {
1335                 loaded_vmcs_clear(vmx->loaded_vmcs);
1336                 local_irq_disable();
1337
1338                 /*
1339                  * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to
1340                  * this cpu's percpu list, otherwise it may not yet be deleted
1341                  * from its previous cpu's percpu list.  Pairs with the
1342                  * smb_wmb() in __loaded_vmcs_clear().
1343                  */
1344                 smp_rmb();
1345
1346                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
1347                          &per_cpu(loaded_vmcss_on_cpu, cpu));
1348                 local_irq_enable();
1349         }
1350
1351         prev = per_cpu(current_vmcs, cpu);
1352         if (prev != vmx->loaded_vmcs->vmcs) {
1353                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
1354                 vmcs_load(vmx->loaded_vmcs->vmcs);
1355
1356                 /*
1357                  * No indirect branch prediction barrier needed when switching
1358                  * the active VMCS within a guest, e.g. on nested VM-Enter.
1359                  * The L1 VMM can protect itself with retpolines, IBPB or IBRS.
1360                  */
1361                 if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev))
1362                         indirect_branch_prediction_barrier();
1363         }
1364
1365         if (!already_loaded) {
1366                 void *gdt = get_current_gdt_ro();
1367                 unsigned long sysenter_esp;
1368
1369                 /*
1370                  * Flush all EPTP/VPID contexts, the new pCPU may have stale
1371                  * TLB entries from its previous association with the vCPU.
1372                  */
1373                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1374
1375                 /*
1376                  * Linux uses per-cpu TSS and GDT, so set these when switching
1377                  * processors.  See 22.2.4.
1378                  */
1379                 vmcs_writel(HOST_TR_BASE,
1380                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
1381                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
1382
1383                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
1384                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
1385
1386                 vmx->loaded_vmcs->cpu = cpu;
1387         }
1388
1389         /* Setup TSC multiplier */
1390         if (kvm_has_tsc_control &&
1391             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
1392                 decache_tsc_multiplier(vmx);
1393 }
1394
1395 /*
1396  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
1397  * vcpu mutex is already taken.
1398  */
1399 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1400 {
1401         struct vcpu_vmx *vmx = to_vmx(vcpu);
1402
1403         vmx_vcpu_load_vmcs(vcpu, cpu, NULL);
1404
1405         vmx_vcpu_pi_load(vcpu, cpu);
1406
1407         vmx->host_debugctlmsr = get_debugctlmsr();
1408 }
1409
1410 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
1411 {
1412         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
1413
1414         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
1415                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
1416                 !kvm_vcpu_apicv_active(vcpu))
1417                 return;
1418
1419         /* Set SN when the vCPU is preempted */
1420         if (vcpu->preempted)
1421                 pi_set_sn(pi_desc);
1422 }
1423
1424 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
1425 {
1426         vmx_vcpu_pi_put(vcpu);
1427
1428         vmx_prepare_switch_to_host(to_vmx(vcpu));
1429 }
1430
1431 static bool emulation_required(struct kvm_vcpu *vcpu)
1432 {
1433         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
1434 }
1435
1436 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
1437 {
1438         struct vcpu_vmx *vmx = to_vmx(vcpu);
1439         unsigned long rflags, save_rflags;
1440
1441         if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) {
1442                 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
1443                 rflags = vmcs_readl(GUEST_RFLAGS);
1444                 if (vmx->rmode.vm86_active) {
1445                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
1446                         save_rflags = vmx->rmode.save_rflags;
1447                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
1448                 }
1449                 vmx->rflags = rflags;
1450         }
1451         return vmx->rflags;
1452 }
1453
1454 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1455 {
1456         struct vcpu_vmx *vmx = to_vmx(vcpu);
1457         unsigned long old_rflags;
1458
1459         if (enable_unrestricted_guest) {
1460                 kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
1461                 vmx->rflags = rflags;
1462                 vmcs_writel(GUEST_RFLAGS, rflags);
1463                 return;
1464         }
1465
1466         old_rflags = vmx_get_rflags(vcpu);
1467         vmx->rflags = rflags;
1468         if (vmx->rmode.vm86_active) {
1469                 vmx->rmode.save_rflags = rflags;
1470                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
1471         }
1472         vmcs_writel(GUEST_RFLAGS, rflags);
1473
1474         if ((old_rflags ^ vmx->rflags) & X86_EFLAGS_VM)
1475                 vmx->emulation_required = emulation_required(vcpu);
1476 }
1477
1478 u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
1479 {
1480         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1481         int ret = 0;
1482
1483         if (interruptibility & GUEST_INTR_STATE_STI)
1484                 ret |= KVM_X86_SHADOW_INT_STI;
1485         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
1486                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
1487
1488         return ret;
1489 }
1490
1491 void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
1492 {
1493         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
1494         u32 interruptibility = interruptibility_old;
1495
1496         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
1497
1498         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
1499                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
1500         else if (mask & KVM_X86_SHADOW_INT_STI)
1501                 interruptibility |= GUEST_INTR_STATE_STI;
1502
1503         if ((interruptibility != interruptibility_old))
1504                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
1505 }
1506
1507 static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
1508 {
1509         struct vcpu_vmx *vmx = to_vmx(vcpu);
1510         unsigned long value;
1511
1512         /*
1513          * Any MSR write that attempts to change bits marked reserved will
1514          * case a #GP fault.
1515          */
1516         if (data & vmx->pt_desc.ctl_bitmask)
1517                 return 1;
1518
1519         /*
1520          * Any attempt to modify IA32_RTIT_CTL while TraceEn is set will
1521          * result in a #GP unless the same write also clears TraceEn.
1522          */
1523         if ((vmx->pt_desc.guest.ctl & RTIT_CTL_TRACEEN) &&
1524                 ((vmx->pt_desc.guest.ctl ^ data) & ~RTIT_CTL_TRACEEN))
1525                 return 1;
1526
1527         /*
1528          * WRMSR to IA32_RTIT_CTL that sets TraceEn but clears this bit
1529          * and FabricEn would cause #GP, if
1530          * CPUID.(EAX=14H, ECX=0):ECX.SNGLRGNOUT[bit 2] = 0
1531          */
1532         if ((data & RTIT_CTL_TRACEEN) && !(data & RTIT_CTL_TOPA) &&
1533                 !(data & RTIT_CTL_FABRIC_EN) &&
1534                 !intel_pt_validate_cap(vmx->pt_desc.caps,
1535                                         PT_CAP_single_range_output))
1536                 return 1;
1537
1538         /*
1539          * MTCFreq, CycThresh and PSBFreq encodings check, any MSR write that
1540          * utilize encodings marked reserved will casue a #GP fault.
1541          */
1542         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc_periods);
1543         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_mtc) &&
1544                         !test_bit((data & RTIT_CTL_MTC_RANGE) >>
1545                         RTIT_CTL_MTC_RANGE_OFFSET, &value))
1546                 return 1;
1547         value = intel_pt_validate_cap(vmx->pt_desc.caps,
1548                                                 PT_CAP_cycle_thresholds);
1549         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1550                         !test_bit((data & RTIT_CTL_CYC_THRESH) >>
1551                         RTIT_CTL_CYC_THRESH_OFFSET, &value))
1552                 return 1;
1553         value = intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_periods);
1554         if (intel_pt_validate_cap(vmx->pt_desc.caps, PT_CAP_psb_cyc) &&
1555                         !test_bit((data & RTIT_CTL_PSB_FREQ) >>
1556                         RTIT_CTL_PSB_FREQ_OFFSET, &value))
1557                 return 1;
1558
1559         /*
1560          * If ADDRx_CFG is reserved or the encodings is >2 will
1561          * cause a #GP fault.
1562          */
1563         value = (data & RTIT_CTL_ADDR0) >> RTIT_CTL_ADDR0_OFFSET;
1564         if ((value && (vmx->pt_desc.addr_range < 1)) || (value > 2))
1565                 return 1;
1566         value = (data & RTIT_CTL_ADDR1) >> RTIT_CTL_ADDR1_OFFSET;
1567         if ((value && (vmx->pt_desc.addr_range < 2)) || (value > 2))
1568                 return 1;
1569         value = (data & RTIT_CTL_ADDR2) >> RTIT_CTL_ADDR2_OFFSET;
1570         if ((value && (vmx->pt_desc.addr_range < 3)) || (value > 2))
1571                 return 1;
1572         value = (data & RTIT_CTL_ADDR3) >> RTIT_CTL_ADDR3_OFFSET;
1573         if ((value && (vmx->pt_desc.addr_range < 4)) || (value > 2))
1574                 return 1;
1575
1576         return 0;
1577 }
1578
1579 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
1580 {
1581         unsigned long rip, orig_rip;
1582
1583         /*
1584          * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on
1585          * undefined behavior: Intel's SDM doesn't mandate the VMCS field be
1586          * set when EPT misconfig occurs.  In practice, real hardware updates
1587          * VM_EXIT_INSTRUCTION_LEN on EPT misconfig, but other hypervisors
1588          * (namely Hyper-V) don't set it due to it being undefined behavior,
1589          * i.e. we end up advancing IP with some random value.
1590          */
1591         if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
1592             to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) {
1593                 orig_rip = kvm_rip_read(vcpu);
1594                 rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
1595 #ifdef CONFIG_X86_64
1596                 /*
1597                  * We need to mask out the high 32 bits of RIP if not in 64-bit
1598                  * mode, but just finding out that we are in 64-bit mode is
1599                  * quite expensive.  Only do it if there was a carry.
1600                  */
1601                 if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu))
1602                         rip = (u32)rip;
1603 #endif
1604                 kvm_rip_write(vcpu, rip);
1605         } else {
1606                 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
1607                         return 0;
1608         }
1609
1610         /* skipping an emulated instruction also counts */
1611         vmx_set_interrupt_shadow(vcpu, 0);
1612
1613         return 1;
1614 }
1615
1616 /*
1617  * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns
1618  * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value
1619  * indicates whether exit to userspace is needed.
1620  */
1621 int vmx_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
1622                               struct x86_exception *e)
1623 {
1624         if (r == X86EMUL_PROPAGATE_FAULT) {
1625                 kvm_inject_emulated_page_fault(vcpu, e);
1626                 return 1;
1627         }
1628
1629         /*
1630          * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
1631          * while handling a VMX instruction KVM could've handled the request
1632          * correctly by exiting to userspace and performing I/O but there
1633          * doesn't seem to be a real use-case behind such requests, just return
1634          * KVM_EXIT_INTERNAL_ERROR for now.
1635          */
1636         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1637         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1638         vcpu->run->internal.ndata = 0;
1639
1640         return 0;
1641 }
1642
1643 /*
1644  * Recognizes a pending MTF VM-exit and records the nested state for later
1645  * delivery.
1646  */
1647 static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
1648 {
1649         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1650         struct vcpu_vmx *vmx = to_vmx(vcpu);
1651
1652         if (!is_guest_mode(vcpu))
1653                 return;
1654
1655         /*
1656          * Per the SDM, MTF takes priority over debug-trap exceptions besides
1657          * T-bit traps. As instruction emulation is completed (i.e. at the
1658          * instruction boundary), any #DB exception pending delivery must be a
1659          * debug-trap. Record the pending MTF state to be delivered in
1660          * vmx_check_nested_events().
1661          */
1662         if (nested_cpu_has_mtf(vmcs12) &&
1663             (!vcpu->arch.exception.pending ||
1664              vcpu->arch.exception.nr == DB_VECTOR))
1665                 vmx->nested.mtf_pending = true;
1666         else
1667                 vmx->nested.mtf_pending = false;
1668 }
1669
1670 static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
1671 {
1672         vmx_update_emulated_instruction(vcpu);
1673         return skip_emulated_instruction(vcpu);
1674 }
1675
1676 static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
1677 {
1678         /*
1679          * Ensure that we clear the HLT state in the VMCS.  We don't need to
1680          * explicitly skip the instruction because if the HLT state is set,
1681          * then the instruction is already executing and RIP has already been
1682          * advanced.
1683          */
1684         if (kvm_hlt_in_guest(vcpu->kvm) &&
1685                         vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
1686                 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
1687 }
1688
1689 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
1690 {
1691         struct vcpu_vmx *vmx = to_vmx(vcpu);
1692         unsigned nr = vcpu->arch.exception.nr;
1693         bool has_error_code = vcpu->arch.exception.has_error_code;
1694         u32 error_code = vcpu->arch.exception.error_code;
1695         u32 intr_info = nr | INTR_INFO_VALID_MASK;
1696
1697         kvm_deliver_exception_payload(vcpu);
1698
1699         if (has_error_code) {
1700                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
1701                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
1702         }
1703
1704         if (vmx->rmode.vm86_active) {
1705                 int inc_eip = 0;
1706                 if (kvm_exception_is_soft(nr))
1707                         inc_eip = vcpu->arch.event_exit_inst_len;
1708                 kvm_inject_realmode_interrupt(vcpu, nr, inc_eip);
1709                 return;
1710         }
1711
1712         WARN_ON_ONCE(vmx->emulation_required);
1713
1714         if (kvm_exception_is_soft(nr)) {
1715                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
1716                              vmx->vcpu.arch.event_exit_inst_len);
1717                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
1718         } else
1719                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
1720
1721         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
1722
1723         vmx_clear_hlt(vcpu);
1724 }
1725
1726 /*
1727  * Swap MSR entry in host/guest MSR entry array.
1728  */
1729 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
1730 {
1731         struct shared_msr_entry tmp;
1732
1733         tmp = vmx->guest_msrs[to];
1734         vmx->guest_msrs[to] = vmx->guest_msrs[from];
1735         vmx->guest_msrs[from] = tmp;
1736 }
1737
1738 /*
1739  * Set up the vmcs to automatically save and restore system
1740  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
1741  * mode, as fiddling with msrs is very expensive.
1742  */
1743 static void setup_msrs(struct vcpu_vmx *vmx)
1744 {
1745         int save_nmsrs, index;
1746
1747         save_nmsrs = 0;
1748 #ifdef CONFIG_X86_64
1749         /*
1750          * The SYSCALL MSRs are only needed on long mode guests, and only
1751          * when EFER.SCE is set.
1752          */
1753         if (is_long_mode(&vmx->vcpu) && (vmx->vcpu.arch.efer & EFER_SCE)) {
1754                 index = __find_msr_index(vmx, MSR_STAR);
1755                 if (index >= 0)
1756                         move_msr_up(vmx, index, save_nmsrs++);
1757                 index = __find_msr_index(vmx, MSR_LSTAR);
1758                 if (index >= 0)
1759                         move_msr_up(vmx, index, save_nmsrs++);
1760                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
1761                 if (index >= 0)
1762                         move_msr_up(vmx, index, save_nmsrs++);
1763         }
1764 #endif
1765         index = __find_msr_index(vmx, MSR_EFER);
1766         if (index >= 0 && update_transition_efer(vmx, index))
1767                 move_msr_up(vmx, index, save_nmsrs++);
1768         index = __find_msr_index(vmx, MSR_TSC_AUX);
1769         if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
1770                 move_msr_up(vmx, index, save_nmsrs++);
1771         index = __find_msr_index(vmx, MSR_IA32_TSX_CTRL);
1772         if (index >= 0)
1773                 move_msr_up(vmx, index, save_nmsrs++);
1774
1775         vmx->save_nmsrs = save_nmsrs;
1776         vmx->guest_msrs_ready = false;
1777
1778         if (cpu_has_vmx_msr_bitmap())
1779                 vmx_update_msr_bitmap(&vmx->vcpu);
1780 }
1781
1782 static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1783 {
1784         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1785         u64 g_tsc_offset = 0;
1786
1787         /*
1788          * We're here if L1 chose not to trap WRMSR to TSC. According
1789          * to the spec, this should set L1's TSC; The offset that L1
1790          * set for L2 remains unchanged, and still needs to be added
1791          * to the newly set TSC to get L2's TSC.
1792          */
1793         if (is_guest_mode(vcpu) &&
1794             (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING))
1795                 g_tsc_offset = vmcs12->tsc_offset;
1796
1797         trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1798                                    vcpu->arch.tsc_offset - g_tsc_offset,
1799                                    offset);
1800         vmcs_write64(TSC_OFFSET, offset + g_tsc_offset);
1801         return offset + g_tsc_offset;
1802 }
1803
1804 /*
1805  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
1806  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
1807  * all guests if the "nested" module option is off, and can also be disabled
1808  * for a single guest by disabling its VMX cpuid bit.
1809  */
1810 bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
1811 {
1812         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
1813 }
1814
1815 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
1816                                                  uint64_t val)
1817 {
1818         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
1819
1820         return !(val & ~valid_bits);
1821 }
1822
1823 static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
1824 {
1825         switch (msr->index) {
1826         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1827                 if (!nested)
1828                         return 1;
1829                 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
1830         case MSR_IA32_PERF_CAPABILITIES:
1831                 msr->data = vmx_get_perf_capabilities();
1832                 return 0;
1833         default:
1834                 return KVM_MSR_RET_INVALID;
1835         }
1836 }
1837
1838 /*
1839  * Reads an msr value (of 'msr_index') into 'pdata'.
1840  * Returns 0 on success, non-0 otherwise.
1841  * Assumes vcpu_load() was already called.
1842  */
1843 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1844 {
1845         struct vcpu_vmx *vmx = to_vmx(vcpu);
1846         struct shared_msr_entry *msr;
1847         u32 index;
1848
1849         switch (msr_info->index) {
1850 #ifdef CONFIG_X86_64
1851         case MSR_FS_BASE:
1852                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
1853                 break;
1854         case MSR_GS_BASE:
1855                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
1856                 break;
1857         case MSR_KERNEL_GS_BASE:
1858                 msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
1859                 break;
1860 #endif
1861         case MSR_EFER:
1862                 return kvm_get_msr_common(vcpu, msr_info);
1863         case MSR_IA32_TSX_CTRL:
1864                 if (!msr_info->host_initiated &&
1865                     !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
1866                         return 1;
1867                 goto find_shared_msr;
1868         case MSR_IA32_UMWAIT_CONTROL:
1869                 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
1870                         return 1;
1871
1872                 msr_info->data = vmx->msr_ia32_umwait_control;
1873                 break;
1874         case MSR_IA32_SPEC_CTRL:
1875                 if (!msr_info->host_initiated &&
1876                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
1877                         return 1;
1878
1879                 msr_info->data = to_vmx(vcpu)->spec_ctrl;
1880                 break;
1881         case MSR_IA32_SYSENTER_CS:
1882                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
1883                 break;
1884         case MSR_IA32_SYSENTER_EIP:
1885                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
1886                 break;
1887         case MSR_IA32_SYSENTER_ESP:
1888                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
1889                 break;
1890         case MSR_IA32_BNDCFGS:
1891                 if (!kvm_mpx_supported() ||
1892                     (!msr_info->host_initiated &&
1893                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
1894                         return 1;
1895                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
1896                 break;
1897         case MSR_IA32_MCG_EXT_CTL:
1898                 if (!msr_info->host_initiated &&
1899                     !(vmx->msr_ia32_feature_control &
1900                       FEAT_CTL_LMCE_ENABLED))
1901                         return 1;
1902                 msr_info->data = vcpu->arch.mcg_ext_ctl;
1903                 break;
1904         case MSR_IA32_FEAT_CTL:
1905                 msr_info->data = vmx->msr_ia32_feature_control;
1906                 break;
1907         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
1908                 if (!nested_vmx_allowed(vcpu))
1909                         return 1;
1910                 if (vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
1911                                     &msr_info->data))
1912                         return 1;
1913                 /*
1914                  * Enlightened VMCS v1 doesn't have certain fields, but buggy
1915                  * Hyper-V versions are still trying to use corresponding
1916                  * features when they are exposed. Filter out the essential
1917                  * minimum.
1918                  */
1919                 if (!msr_info->host_initiated &&
1920                     vmx->nested.enlightened_vmcs_enabled)
1921                         nested_evmcs_filter_control_msr(msr_info->index,
1922                                                         &msr_info->data);
1923                 break;
1924         case MSR_IA32_RTIT_CTL:
1925                 if (!vmx_pt_mode_is_host_guest())
1926                         return 1;
1927                 msr_info->data = vmx->pt_desc.guest.ctl;
1928                 break;
1929         case MSR_IA32_RTIT_STATUS:
1930                 if (!vmx_pt_mode_is_host_guest())
1931                         return 1;
1932                 msr_info->data = vmx->pt_desc.guest.status;
1933                 break;
1934         case MSR_IA32_RTIT_CR3_MATCH:
1935                 if (!vmx_pt_mode_is_host_guest() ||
1936                         !intel_pt_validate_cap(vmx->pt_desc.caps,
1937                                                 PT_CAP_cr3_filtering))
1938                         return 1;
1939                 msr_info->data = vmx->pt_desc.guest.cr3_match;
1940                 break;
1941         case MSR_IA32_RTIT_OUTPUT_BASE:
1942                 if (!vmx_pt_mode_is_host_guest() ||
1943                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1944                                         PT_CAP_topa_output) &&
1945                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1946                                         PT_CAP_single_range_output)))
1947                         return 1;
1948                 msr_info->data = vmx->pt_desc.guest.output_base;
1949                 break;
1950         case MSR_IA32_RTIT_OUTPUT_MASK:
1951                 if (!vmx_pt_mode_is_host_guest() ||
1952                         (!intel_pt_validate_cap(vmx->pt_desc.caps,
1953                                         PT_CAP_topa_output) &&
1954                          !intel_pt_validate_cap(vmx->pt_desc.caps,
1955                                         PT_CAP_single_range_output)))
1956                         return 1;
1957                 msr_info->data = vmx->pt_desc.guest.output_mask;
1958                 break;
1959         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
1960                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
1961                 if (!vmx_pt_mode_is_host_guest() ||
1962                         (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
1963                                         PT_CAP_num_address_ranges)))
1964                         return 1;
1965                 if (index % 2)
1966                         msr_info->data = vmx->pt_desc.guest.addr_b[index / 2];
1967                 else
1968                         msr_info->data = vmx->pt_desc.guest.addr_a[index / 2];
1969                 break;
1970         case MSR_TSC_AUX:
1971                 if (!msr_info->host_initiated &&
1972                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
1973                         return 1;
1974                 goto find_shared_msr;
1975         default:
1976         find_shared_msr:
1977                 msr = find_msr_entry(vmx, msr_info->index);
1978                 if (msr) {
1979                         msr_info->data = msr->data;
1980                         break;
1981                 }
1982                 return kvm_get_msr_common(vcpu, msr_info);
1983         }
1984
1985         return 0;
1986 }
1987
1988 static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu,
1989                                                     u64 data)
1990 {
1991 #ifdef CONFIG_X86_64
1992         if (!guest_cpuid_has(vcpu, X86_FEATURE_LM))
1993                 return (u32)data;
1994 #endif
1995         return (unsigned long)data;
1996 }
1997
1998 /*
1999  * Writes msr value into the appropriate "register".
2000  * Returns 0 on success, non-0 otherwise.
2001  * Assumes vcpu_load() was already called.
2002  */
2003 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2004 {
2005         struct vcpu_vmx *vmx = to_vmx(vcpu);
2006         struct shared_msr_entry *msr;
2007         int ret = 0;
2008         u32 msr_index = msr_info->index;
2009         u64 data = msr_info->data;
2010         u32 index;
2011
2012         switch (msr_index) {
2013         case MSR_EFER:
2014                 ret = kvm_set_msr_common(vcpu, msr_info);
2015                 break;
2016 #ifdef CONFIG_X86_64
2017         case MSR_FS_BASE:
2018                 vmx_segment_cache_clear(vmx);
2019                 vmcs_writel(GUEST_FS_BASE, data);
2020                 break;
2021         case MSR_GS_BASE:
2022                 vmx_segment_cache_clear(vmx);
2023                 vmcs_writel(GUEST_GS_BASE, data);
2024                 break;
2025         case MSR_KERNEL_GS_BASE:
2026                 vmx_write_guest_kernel_gs_base(vmx, data);
2027                 break;
2028 #endif
2029         case MSR_IA32_SYSENTER_CS:
2030                 if (is_guest_mode(vcpu))
2031                         get_vmcs12(vcpu)->guest_sysenter_cs = data;
2032                 vmcs_write32(GUEST_SYSENTER_CS, data);
2033                 break;
2034         case MSR_IA32_SYSENTER_EIP:
2035                 if (is_guest_mode(vcpu)) {
2036                         data = nested_vmx_truncate_sysenter_addr(vcpu, data);
2037                         get_vmcs12(vcpu)->guest_sysenter_eip = data;
2038                 }
2039                 vmcs_writel(GUEST_SYSENTER_EIP, data);
2040                 break;
2041         case MSR_IA32_SYSENTER_ESP:
2042                 if (is_guest_mode(vcpu)) {
2043                         data = nested_vmx_truncate_sysenter_addr(vcpu, data);
2044                         get_vmcs12(vcpu)->guest_sysenter_esp = data;
2045                 }
2046                 vmcs_writel(GUEST_SYSENTER_ESP, data);
2047                 break;
2048         case MSR_IA32_DEBUGCTLMSR:
2049                 if (is_guest_mode(vcpu) && get_vmcs12(vcpu)->vm_exit_controls &
2050                                                 VM_EXIT_SAVE_DEBUG_CONTROLS)
2051                         get_vmcs12(vcpu)->guest_ia32_debugctl = data;
2052
2053                 ret = kvm_set_msr_common(vcpu, msr_info);
2054                 break;
2055
2056         case MSR_IA32_BNDCFGS:
2057                 if (!kvm_mpx_supported() ||
2058                     (!msr_info->host_initiated &&
2059                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
2060                         return 1;
2061                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
2062                     (data & MSR_IA32_BNDCFGS_RSVD))
2063                         return 1;
2064                 vmcs_write64(GUEST_BNDCFGS, data);
2065                 break;
2066         case MSR_IA32_UMWAIT_CONTROL:
2067                 if (!msr_info->host_initiated && !vmx_has_waitpkg(vmx))
2068                         return 1;
2069
2070                 /* The reserved bit 1 and non-32 bit [63:32] should be zero */
2071                 if (data & (BIT_ULL(1) | GENMASK_ULL(63, 32)))
2072                         return 1;
2073
2074                 vmx->msr_ia32_umwait_control = data;
2075                 break;
2076         case MSR_IA32_SPEC_CTRL:
2077                 if (!msr_info->host_initiated &&
2078                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
2079                         return 1;
2080
2081                 if (kvm_spec_ctrl_test_value(data))
2082                         return 1;
2083
2084                 vmx->spec_ctrl = data;
2085                 if (!data)
2086                         break;
2087
2088                 /*
2089                  * For non-nested:
2090                  * When it's written (to non-zero) for the first time, pass
2091                  * it through.
2092                  *
2093                  * For nested:
2094                  * The handling of the MSR bitmap for L2 guests is done in
2095                  * nested_vmx_prepare_msr_bitmap. We should not touch the
2096                  * vmcs02.msr_bitmap here since it gets completely overwritten
2097                  * in the merging. We update the vmcs01 here for L1 as well
2098                  * since it will end up touching the MSR anyway now.
2099                  */
2100                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
2101                                               MSR_IA32_SPEC_CTRL,
2102                                               MSR_TYPE_RW);
2103                 break;
2104         case MSR_IA32_TSX_CTRL:
2105                 if (!msr_info->host_initiated &&
2106                     !(vcpu->arch.arch_capabilities & ARCH_CAP_TSX_CTRL_MSR))
2107                         return 1;
2108                 if (data & ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR))
2109                         return 1;
2110                 goto find_shared_msr;
2111         case MSR_IA32_PRED_CMD:
2112                 if (!msr_info->host_initiated &&
2113                     !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
2114                         return 1;
2115
2116                 if (data & ~PRED_CMD_IBPB)
2117                         return 1;
2118                 if (!boot_cpu_has(X86_FEATURE_SPEC_CTRL))
2119                         return 1;
2120                 if (!data)
2121                         break;
2122
2123                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
2124
2125                 /*
2126                  * For non-nested:
2127                  * When it's written (to non-zero) for the first time, pass
2128                  * it through.
2129                  *
2130                  * For nested:
2131                  * The handling of the MSR bitmap for L2 guests is done in
2132                  * nested_vmx_prepare_msr_bitmap. We should not touch the
2133                  * vmcs02.msr_bitmap here since it gets completely overwritten
2134                  * in the merging.
2135                  */
2136                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
2137                                               MSR_TYPE_W);
2138                 break;
2139         case MSR_IA32_CR_PAT:
2140                 if (!kvm_pat_valid(data))
2141                         return 1;
2142
2143                 if (is_guest_mode(vcpu) &&
2144                     get_vmcs12(vcpu)->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
2145                         get_vmcs12(vcpu)->guest_ia32_pat = data;
2146
2147                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2148                         vmcs_write64(GUEST_IA32_PAT, data);
2149                         vcpu->arch.pat = data;
2150                         break;
2151                 }
2152                 ret = kvm_set_msr_common(vcpu, msr_info);
2153                 break;
2154         case MSR_IA32_TSC_ADJUST:
2155                 ret = kvm_set_msr_common(vcpu, msr_info);
2156                 break;
2157         case MSR_IA32_MCG_EXT_CTL:
2158                 if ((!msr_info->host_initiated &&
2159                      !(to_vmx(vcpu)->msr_ia32_feature_control &
2160                        FEAT_CTL_LMCE_ENABLED)) ||
2161                     (data & ~MCG_EXT_CTL_LMCE_EN))
2162                         return 1;
2163                 vcpu->arch.mcg_ext_ctl = data;
2164                 break;
2165         case MSR_IA32_FEAT_CTL:
2166                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
2167                     (to_vmx(vcpu)->msr_ia32_feature_control &
2168                      FEAT_CTL_LOCKED && !msr_info->host_initiated))
2169                         return 1;
2170                 vmx->msr_ia32_feature_control = data;
2171                 if (msr_info->host_initiated && data == 0)
2172                         vmx_leave_nested(vcpu);
2173                 break;
2174         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
2175                 if (!msr_info->host_initiated)
2176                         return 1; /* they are read-only */
2177                 if (!nested_vmx_allowed(vcpu))
2178                         return 1;
2179                 return vmx_set_vmx_msr(vcpu, msr_index, data);
2180         case MSR_IA32_RTIT_CTL:
2181                 if (!vmx_pt_mode_is_host_guest() ||
2182                         vmx_rtit_ctl_check(vcpu, data) ||
2183                         vmx->nested.vmxon)
2184                         return 1;
2185                 vmcs_write64(GUEST_IA32_RTIT_CTL, data);
2186                 vmx->pt_desc.guest.ctl = data;
2187                 pt_update_intercept_for_msr(vmx);
2188                 break;
2189         case MSR_IA32_RTIT_STATUS:
2190                 if (!pt_can_write_msr(vmx))
2191                         return 1;
2192                 if (data & MSR_IA32_RTIT_STATUS_MASK)
2193                         return 1;
2194                 vmx->pt_desc.guest.status = data;
2195                 break;
2196         case MSR_IA32_RTIT_CR3_MATCH:
2197                 if (!pt_can_write_msr(vmx))
2198                         return 1;
2199                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2200                                            PT_CAP_cr3_filtering))
2201                         return 1;
2202                 vmx->pt_desc.guest.cr3_match = data;
2203                 break;
2204         case MSR_IA32_RTIT_OUTPUT_BASE:
2205                 if (!pt_can_write_msr(vmx))
2206                         return 1;
2207                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2208                                            PT_CAP_topa_output) &&
2209                     !intel_pt_validate_cap(vmx->pt_desc.caps,
2210                                            PT_CAP_single_range_output))
2211                         return 1;
2212                 if (data & MSR_IA32_RTIT_OUTPUT_BASE_MASK)
2213                         return 1;
2214                 vmx->pt_desc.guest.output_base = data;
2215                 break;
2216         case MSR_IA32_RTIT_OUTPUT_MASK:
2217                 if (!pt_can_write_msr(vmx))
2218                         return 1;
2219                 if (!intel_pt_validate_cap(vmx->pt_desc.caps,
2220                                            PT_CAP_topa_output) &&
2221                     !intel_pt_validate_cap(vmx->pt_desc.caps,
2222                                            PT_CAP_single_range_output))
2223                         return 1;
2224                 vmx->pt_desc.guest.output_mask = data;
2225                 break;
2226         case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
2227                 if (!pt_can_write_msr(vmx))
2228                         return 1;
2229                 index = msr_info->index - MSR_IA32_RTIT_ADDR0_A;
2230                 if (index >= 2 * intel_pt_validate_cap(vmx->pt_desc.caps,
2231                                                        PT_CAP_num_address_ranges))
2232                         return 1;
2233                 if (is_noncanonical_address(data, vcpu))
2234                         return 1;
2235                 if (index % 2)
2236                         vmx->pt_desc.guest.addr_b[index / 2] = data;
2237                 else
2238                         vmx->pt_desc.guest.addr_a[index / 2] = data;
2239                 break;
2240         case MSR_TSC_AUX:
2241                 if (!msr_info->host_initiated &&
2242                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
2243                         return 1;
2244                 /* Check reserved bit, higher 32 bits should be zero */
2245                 if ((data >> 32) != 0)
2246                         return 1;
2247                 goto find_shared_msr;
2248
2249         default:
2250         find_shared_msr:
2251                 msr = find_msr_entry(vmx, msr_index);
2252                 if (msr)
2253                         ret = vmx_set_guest_msr(vmx, msr, data);
2254                 else
2255                         ret = kvm_set_msr_common(vcpu, msr_info);
2256         }
2257
2258         return ret;
2259 }
2260
2261 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2262 {
2263         unsigned long guest_owned_bits;
2264
2265         kvm_register_mark_available(vcpu, reg);
2266
2267         switch (reg) {
2268         case VCPU_REGS_RSP:
2269                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
2270                 break;
2271         case VCPU_REGS_RIP:
2272                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
2273                 break;
2274         case VCPU_EXREG_PDPTR:
2275                 if (enable_ept)
2276                         ept_save_pdptrs(vcpu);
2277                 break;
2278         case VCPU_EXREG_CR0:
2279                 guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
2280
2281                 vcpu->arch.cr0 &= ~guest_owned_bits;
2282                 vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits;
2283                 break;
2284         case VCPU_EXREG_CR3:
2285                 if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu)))
2286                         vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
2287                 break;
2288         case VCPU_EXREG_CR4:
2289                 guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
2290
2291                 vcpu->arch.cr4 &= ~guest_owned_bits;
2292                 vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits;
2293                 break;
2294         default:
2295                 WARN_ON_ONCE(1);
2296                 break;
2297         }
2298 }
2299
2300 static __init int cpu_has_kvm_support(void)
2301 {
2302         return cpu_has_vmx();
2303 }
2304
2305 static __init int vmx_disabled_by_bios(void)
2306 {
2307         return !boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2308                !boot_cpu_has(X86_FEATURE_VMX);
2309 }
2310
2311 static int kvm_cpu_vmxon(u64 vmxon_pointer)
2312 {
2313         u64 msr;
2314
2315         cr4_set_bits(X86_CR4_VMXE);
2316         intel_pt_handle_vmx(1);
2317
2318         asm_volatile_goto("1: vmxon %[vmxon_pointer]\n\t"
2319                           _ASM_EXTABLE(1b, %l[fault])
2320                           : : [vmxon_pointer] "m"(vmxon_pointer)
2321                           : : fault);
2322         return 0;
2323
2324 fault:
2325         WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
2326                   rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
2327         intel_pt_handle_vmx(0);
2328         cr4_clear_bits(X86_CR4_VMXE);
2329
2330         return -EFAULT;
2331 }
2332
2333 static int hardware_enable(void)
2334 {
2335         int cpu = raw_smp_processor_id();
2336         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
2337         int r;
2338
2339         if (cr4_read_shadow() & X86_CR4_VMXE)
2340                 return -EBUSY;
2341
2342         /*
2343          * This can happen if we hot-added a CPU but failed to allocate
2344          * VP assist page for it.
2345          */
2346         if (static_branch_unlikely(&enable_evmcs) &&
2347             !hv_get_vp_assist_page(cpu))
2348                 return -EFAULT;
2349
2350         r = kvm_cpu_vmxon(phys_addr);
2351         if (r)
2352                 return r;
2353
2354         if (enable_ept)
2355                 ept_sync_global();
2356
2357         return 0;
2358 }
2359
2360 static void vmclear_local_loaded_vmcss(void)
2361 {
2362         int cpu = raw_smp_processor_id();
2363         struct loaded_vmcs *v, *n;
2364
2365         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
2366                                  loaded_vmcss_on_cpu_link)
2367                 __loaded_vmcs_clear(v);
2368 }
2369
2370
2371 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
2372  * tricks.
2373  */
2374 static void kvm_cpu_vmxoff(void)
2375 {
2376         asm volatile (__ex("vmxoff"));
2377
2378         intel_pt_handle_vmx(0);
2379         cr4_clear_bits(X86_CR4_VMXE);
2380 }
2381
2382 static void hardware_disable(void)
2383 {
2384         vmclear_local_loaded_vmcss();
2385         kvm_cpu_vmxoff();
2386 }
2387
2388 /*
2389  * There is no X86_FEATURE for SGX yet, but anyway we need to query CPUID
2390  * directly instead of going through cpu_has(), to ensure KVM is trapping
2391  * ENCLS whenever it's supported in hardware.  It does not matter whether
2392  * the host OS supports or has enabled SGX.
2393  */
2394 static bool cpu_has_sgx(void)
2395 {
2396         return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0));
2397 }
2398
2399 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
2400                                       u32 msr, u32 *result)
2401 {
2402         u32 vmx_msr_low, vmx_msr_high;
2403         u32 ctl = ctl_min | ctl_opt;
2404
2405         rdmsr(msr, vmx_msr_low, vmx_msr_high);
2406
2407         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
2408         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
2409
2410         /* Ensure minimum (required) set of control bits are supported. */
2411         if (ctl_min & ~ctl)
2412                 return -EIO;
2413
2414         *result = ctl;
2415         return 0;
2416 }
2417
2418 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf,
2419                                     struct vmx_capability *vmx_cap)
2420 {
2421         u32 vmx_msr_low, vmx_msr_high;
2422         u32 min, opt, min2, opt2;
2423         u32 _pin_based_exec_control = 0;
2424         u32 _cpu_based_exec_control = 0;
2425         u32 _cpu_based_2nd_exec_control = 0;
2426         u32 _vmexit_control = 0;
2427         u32 _vmentry_control = 0;
2428
2429         memset(vmcs_conf, 0, sizeof(*vmcs_conf));
2430         min = CPU_BASED_HLT_EXITING |
2431 #ifdef CONFIG_X86_64
2432               CPU_BASED_CR8_LOAD_EXITING |
2433               CPU_BASED_CR8_STORE_EXITING |
2434 #endif
2435               CPU_BASED_CR3_LOAD_EXITING |
2436               CPU_BASED_CR3_STORE_EXITING |
2437               CPU_BASED_UNCOND_IO_EXITING |
2438               CPU_BASED_MOV_DR_EXITING |
2439               CPU_BASED_USE_TSC_OFFSETTING |
2440               CPU_BASED_MWAIT_EXITING |
2441               CPU_BASED_MONITOR_EXITING |
2442               CPU_BASED_INVLPG_EXITING |
2443               CPU_BASED_RDPMC_EXITING;
2444
2445         opt = CPU_BASED_TPR_SHADOW |
2446               CPU_BASED_USE_MSR_BITMAPS |
2447               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
2448         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
2449                                 &_cpu_based_exec_control) < 0)
2450                 return -EIO;
2451 #ifdef CONFIG_X86_64
2452         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2453                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
2454                                            ~CPU_BASED_CR8_STORE_EXITING;
2455 #endif
2456         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
2457                 min2 = 0;
2458                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2459                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2460                         SECONDARY_EXEC_WBINVD_EXITING |
2461                         SECONDARY_EXEC_ENABLE_VPID |
2462                         SECONDARY_EXEC_ENABLE_EPT |
2463                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
2464                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
2465                         SECONDARY_EXEC_DESC |
2466                         SECONDARY_EXEC_RDTSCP |
2467                         SECONDARY_EXEC_ENABLE_INVPCID |
2468                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
2469                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2470                         SECONDARY_EXEC_SHADOW_VMCS |
2471                         SECONDARY_EXEC_XSAVES |
2472                         SECONDARY_EXEC_RDSEED_EXITING |
2473                         SECONDARY_EXEC_RDRAND_EXITING |
2474                         SECONDARY_EXEC_ENABLE_PML |
2475                         SECONDARY_EXEC_TSC_SCALING |
2476                         SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2477                         SECONDARY_EXEC_PT_USE_GPA |
2478                         SECONDARY_EXEC_PT_CONCEAL_VMX |
2479                         SECONDARY_EXEC_ENABLE_VMFUNC;
2480                 if (cpu_has_sgx())
2481                         opt2 |= SECONDARY_EXEC_ENCLS_EXITING;
2482                 if (adjust_vmx_controls(min2, opt2,
2483                                         MSR_IA32_VMX_PROCBASED_CTLS2,
2484                                         &_cpu_based_2nd_exec_control) < 0)
2485                         return -EIO;
2486         }
2487 #ifndef CONFIG_X86_64
2488         if (!(_cpu_based_2nd_exec_control &
2489                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2490                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
2491 #endif
2492
2493         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
2494                 _cpu_based_2nd_exec_control &= ~(
2495                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
2496                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
2497                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2498
2499         rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
2500                 &vmx_cap->ept, &vmx_cap->vpid);
2501
2502         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
2503                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
2504                    enabled */
2505                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
2506                                              CPU_BASED_CR3_STORE_EXITING |
2507                                              CPU_BASED_INVLPG_EXITING);
2508         } else if (vmx_cap->ept) {
2509                 vmx_cap->ept = 0;
2510                 pr_warn_once("EPT CAP should not exist if not support "
2511                                 "1-setting enable EPT VM-execution control\n");
2512         }
2513         if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
2514                 vmx_cap->vpid) {
2515                 vmx_cap->vpid = 0;
2516                 pr_warn_once("VPID CAP should not exist if not support "
2517                                 "1-setting enable VPID VM-execution control\n");
2518         }
2519
2520         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
2521 #ifdef CONFIG_X86_64
2522         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
2523 #endif
2524         opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
2525               VM_EXIT_LOAD_IA32_PAT |
2526               VM_EXIT_LOAD_IA32_EFER |
2527               VM_EXIT_CLEAR_BNDCFGS |
2528               VM_EXIT_PT_CONCEAL_PIP |
2529               VM_EXIT_CLEAR_IA32_RTIT_CTL;
2530         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
2531                                 &_vmexit_control) < 0)
2532                 return -EIO;
2533
2534         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
2535         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
2536                  PIN_BASED_VMX_PREEMPTION_TIMER;
2537         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
2538                                 &_pin_based_exec_control) < 0)
2539                 return -EIO;
2540
2541         if (cpu_has_broken_vmx_preemption_timer())
2542                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
2543         if (!(_cpu_based_2nd_exec_control &
2544                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
2545                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
2546
2547         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
2548         opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
2549               VM_ENTRY_LOAD_IA32_PAT |
2550               VM_ENTRY_LOAD_IA32_EFER |
2551               VM_ENTRY_LOAD_BNDCFGS |
2552               VM_ENTRY_PT_CONCEAL_PIP |
2553               VM_ENTRY_LOAD_IA32_RTIT_CTL;
2554         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
2555                                 &_vmentry_control) < 0)
2556                 return -EIO;
2557
2558         /*
2559          * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they
2560          * can't be used due to an errata where VM Exit may incorrectly clear
2561          * IA32_PERF_GLOBAL_CTRL[34:32].  Workaround the errata by using the
2562          * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL.
2563          */
2564         if (boot_cpu_data.x86 == 0x6) {
2565                 switch (boot_cpu_data.x86_model) {
2566                 case 26: /* AAK155 */
2567                 case 30: /* AAP115 */
2568                 case 37: /* AAT100 */
2569                 case 44: /* BC86,AAY89,BD102 */
2570                 case 46: /* BA97 */
2571                         _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
2572                         _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
2573                         pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
2574                                         "does not work properly. Using workaround\n");
2575                         break;
2576                 default:
2577                         break;
2578                 }
2579         }
2580
2581
2582         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
2583
2584         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
2585         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
2586                 return -EIO;
2587
2588 #ifdef CONFIG_X86_64
2589         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
2590         if (vmx_msr_high & (1u<<16))
2591                 return -EIO;
2592 #endif
2593
2594         /* Require Write-Back (WB) memory type for VMCS accesses. */
2595         if (((vmx_msr_high >> 18) & 15) != 6)
2596                 return -EIO;
2597
2598         vmcs_conf->size = vmx_msr_high & 0x1fff;
2599         vmcs_conf->order = get_order(vmcs_conf->size);
2600         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
2601
2602         vmcs_conf->revision_id = vmx_msr_low;
2603
2604         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
2605         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
2606         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
2607         vmcs_conf->vmexit_ctrl         = _vmexit_control;
2608         vmcs_conf->vmentry_ctrl        = _vmentry_control;
2609
2610         if (static_branch_unlikely(&enable_evmcs))
2611                 evmcs_sanitize_exec_ctrls(vmcs_conf);
2612
2613         return 0;
2614 }
2615
2616 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)
2617 {
2618         int node = cpu_to_node(cpu);
2619         struct page *pages;
2620         struct vmcs *vmcs;
2621
2622         pages = __alloc_pages_node(node, flags, vmcs_config.order);
2623         if (!pages)
2624                 return NULL;
2625         vmcs = page_address(pages);
2626         memset(vmcs, 0, vmcs_config.size);
2627
2628         /* KVM supports Enlightened VMCS v1 only */
2629         if (static_branch_unlikely(&enable_evmcs))
2630                 vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
2631         else
2632                 vmcs->hdr.revision_id = vmcs_config.revision_id;
2633
2634         if (shadow)
2635                 vmcs->hdr.shadow_vmcs = 1;
2636         return vmcs;
2637 }
2638
2639 void free_vmcs(struct vmcs *vmcs)
2640 {
2641         free_pages((unsigned long)vmcs, vmcs_config.order);
2642 }
2643
2644 /*
2645  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
2646  */
2647 void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2648 {
2649         if (!loaded_vmcs->vmcs)
2650                 return;
2651         loaded_vmcs_clear(loaded_vmcs);
2652         free_vmcs(loaded_vmcs->vmcs);
2653         loaded_vmcs->vmcs = NULL;
2654         if (loaded_vmcs->msr_bitmap)
2655                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
2656         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
2657 }
2658
2659 int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
2660 {
2661         loaded_vmcs->vmcs = alloc_vmcs(false);
2662         if (!loaded_vmcs->vmcs)
2663                 return -ENOMEM;
2664
2665         vmcs_clear(loaded_vmcs->vmcs);
2666
2667         loaded_vmcs->shadow_vmcs = NULL;
2668         loaded_vmcs->hv_timer_soft_disabled = false;
2669         loaded_vmcs->cpu = -1;
2670         loaded_vmcs->launched = 0;
2671
2672         if (cpu_has_vmx_msr_bitmap()) {
2673                 loaded_vmcs->msr_bitmap = (unsigned long *)
2674                                 __get_free_page(GFP_KERNEL_ACCOUNT);
2675                 if (!loaded_vmcs->msr_bitmap)
2676                         goto out_vmcs;
2677                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
2678
2679                 if (IS_ENABLED(CONFIG_HYPERV) &&
2680                     static_branch_unlikely(&enable_evmcs) &&
2681                     (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
2682                         struct hv_enlightened_vmcs *evmcs =
2683                                 (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
2684
2685                         evmcs->hv_enlightenments_control.msr_bitmap = 1;
2686                 }
2687         }
2688
2689         memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
2690         memset(&loaded_vmcs->controls_shadow, 0,
2691                 sizeof(struct vmcs_controls_shadow));
2692
2693         return 0;
2694
2695 out_vmcs:
2696         free_loaded_vmcs(loaded_vmcs);
2697         return -ENOMEM;
2698 }
2699
2700 static void free_kvm_area(void)
2701 {
2702         int cpu;
2703
2704         for_each_possible_cpu(cpu) {
2705                 free_vmcs(per_cpu(vmxarea, cpu));
2706                 per_cpu(vmxarea, cpu) = NULL;
2707         }
2708 }
2709
2710 static __init int alloc_kvm_area(void)
2711 {
2712         int cpu;
2713
2714         for_each_possible_cpu(cpu) {
2715                 struct vmcs *vmcs;
2716
2717                 vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL);
2718                 if (!vmcs) {
2719                         free_kvm_area();
2720                         return -ENOMEM;
2721                 }
2722
2723                 /*
2724                  * When eVMCS is enabled, alloc_vmcs_cpu() sets
2725                  * vmcs->revision_id to KVM_EVMCS_VERSION instead of
2726                  * revision_id reported by MSR_IA32_VMX_BASIC.
2727                  *
2728                  * However, even though not explicitly documented by
2729                  * TLFS, VMXArea passed as VMXON argument should
2730                  * still be marked with revision_id reported by
2731                  * physical CPU.
2732                  */
2733                 if (static_branch_unlikely(&enable_evmcs))
2734                         vmcs->hdr.revision_id = vmcs_config.revision_id;
2735
2736                 per_cpu(vmxarea, cpu) = vmcs;
2737         }
2738         return 0;
2739 }
2740
2741 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
2742                 struct kvm_segment *save)
2743 {
2744         if (!emulate_invalid_guest_state) {
2745                 /*
2746                  * CS and SS RPL should be equal during guest entry according
2747                  * to VMX spec, but in reality it is not always so. Since vcpu
2748                  * is in the middle of the transition from real mode to
2749                  * protected mode it is safe to assume that RPL 0 is a good
2750                  * default value.
2751                  */
2752                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
2753                         save->selector &= ~SEGMENT_RPL_MASK;
2754                 save->dpl = save->selector & SEGMENT_RPL_MASK;
2755                 save->s = 1;
2756         }
2757         vmx_set_segment(vcpu, save, seg);
2758 }
2759
2760 static void enter_pmode(struct kvm_vcpu *vcpu)
2761 {
2762         unsigned long flags;
2763         struct vcpu_vmx *vmx = to_vmx(vcpu);
2764
2765         /*
2766          * Update real mode segment cache. It may be not up-to-date if sement
2767          * register was written while vcpu was in a guest mode.
2768          */
2769         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2770         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2771         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2772         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2773         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2774         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2775
2776         vmx->rmode.vm86_active = 0;
2777
2778         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2779
2780         flags = vmcs_readl(GUEST_RFLAGS);
2781         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
2782         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
2783         vmcs_writel(GUEST_RFLAGS, flags);
2784
2785         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
2786                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
2787
2788         update_exception_bitmap(vcpu);
2789
2790         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2791         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2792         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2793         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2794         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2795         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2796 }
2797
2798 static void fix_rmode_seg(int seg, struct kvm_segment *save)
2799 {
2800         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2801         struct kvm_segment var = *save;
2802
2803         var.dpl = 0x3;
2804         if (seg == VCPU_SREG_CS)
2805                 var.type = 0x3;
2806
2807         if (!emulate_invalid_guest_state) {
2808                 var.selector = var.base >> 4;
2809                 var.base = var.base & 0xffff0;
2810                 var.limit = 0xffff;
2811                 var.g = 0;
2812                 var.db = 0;
2813                 var.present = 1;
2814                 var.s = 1;
2815                 var.l = 0;
2816                 var.unusable = 0;
2817                 var.type = 0x3;
2818                 var.avl = 0;
2819                 if (save->base & 0xf)
2820                         printk_once(KERN_WARNING "kvm: segment base is not "
2821                                         "paragraph aligned when entering "
2822                                         "protected mode (seg=%d)", seg);
2823         }
2824
2825         vmcs_write16(sf->selector, var.selector);
2826         vmcs_writel(sf->base, var.base);
2827         vmcs_write32(sf->limit, var.limit);
2828         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
2829 }
2830
2831 static void enter_rmode(struct kvm_vcpu *vcpu)
2832 {
2833         unsigned long flags;
2834         struct vcpu_vmx *vmx = to_vmx(vcpu);
2835         struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
2836
2837         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2838         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2839         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2840         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2841         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2842         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
2843         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
2844
2845         vmx->rmode.vm86_active = 1;
2846
2847         /*
2848          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
2849          * vcpu. Warn the user that an update is overdue.
2850          */
2851         if (!kvm_vmx->tss_addr)
2852                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
2853                              "called before entering vcpu\n");
2854
2855         vmx_segment_cache_clear(vmx);
2856
2857         vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
2858         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
2859         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2860
2861         flags = vmcs_readl(GUEST_RFLAGS);
2862         vmx->rmode.save_rflags = flags;
2863
2864         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
2865
2866         vmcs_writel(GUEST_RFLAGS, flags);
2867         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
2868         update_exception_bitmap(vcpu);
2869
2870         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
2871         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
2872         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2873         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2874         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2875         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2876
2877         kvm_mmu_reset_context(vcpu);
2878 }
2879
2880 void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
2881 {
2882         struct vcpu_vmx *vmx = to_vmx(vcpu);
2883         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
2884
2885         if (!msr)
2886                 return;
2887
2888         vcpu->arch.efer = efer;
2889         if (efer & EFER_LMA) {
2890                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2891                 msr->data = efer;
2892         } else {
2893                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2894
2895                 msr->data = efer & ~EFER_LME;
2896         }
2897         setup_msrs(vmx);
2898 }
2899
2900 #ifdef CONFIG_X86_64
2901
2902 static void enter_lmode(struct kvm_vcpu *vcpu)
2903 {
2904         u32 guest_tr_ar;
2905
2906         vmx_segment_cache_clear(to_vmx(vcpu));
2907
2908         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
2909         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
2910                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
2911                                      __func__);
2912                 vmcs_write32(GUEST_TR_AR_BYTES,
2913                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
2914                              | VMX_AR_TYPE_BUSY_64_TSS);
2915         }
2916         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
2917 }
2918
2919 static void exit_lmode(struct kvm_vcpu *vcpu)
2920 {
2921         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
2922         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
2923 }
2924
2925 #endif
2926
2927 static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
2928 {
2929         struct vcpu_vmx *vmx = to_vmx(vcpu);
2930
2931         /*
2932          * INVEPT must be issued when EPT is enabled, irrespective of VPID, as
2933          * the CPU is not required to invalidate guest-physical mappings on
2934          * VM-Entry, even if VPID is disabled.  Guest-physical mappings are
2935          * associated with the root EPT structure and not any particular VPID
2936          * (INVVPID also isn't required to invalidate guest-physical mappings).
2937          */
2938         if (enable_ept) {
2939                 ept_sync_global();
2940         } else if (enable_vpid) {
2941                 if (cpu_has_vmx_invvpid_global()) {
2942                         vpid_sync_vcpu_global();
2943                 } else {
2944                         vpid_sync_vcpu_single(vmx->vpid);
2945                         vpid_sync_vcpu_single(vmx->nested.vpid02);
2946                 }
2947         }
2948 }
2949
2950 static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
2951 {
2952         struct kvm_mmu *mmu = vcpu->arch.mmu;
2953         u64 root_hpa = mmu->root_hpa;
2954
2955         /* No flush required if the current context is invalid. */
2956         if (!VALID_PAGE(root_hpa))
2957                 return;
2958
2959         if (enable_ept)
2960                 ept_sync_context(construct_eptp(vcpu, root_hpa,
2961                                                 mmu->shadow_root_level));
2962         else if (!is_guest_mode(vcpu))
2963                 vpid_sync_context(to_vmx(vcpu)->vpid);
2964         else
2965                 vpid_sync_context(nested_get_vpid02(vcpu));
2966 }
2967
2968 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
2969 {
2970         /*
2971          * vpid_sync_vcpu_addr() is a nop if vmx->vpid==0, see the comment in
2972          * vmx_flush_tlb_guest() for an explanation of why this is ok.
2973          */
2974         vpid_sync_vcpu_addr(to_vmx(vcpu)->vpid, addr);
2975 }
2976
2977 static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
2978 {
2979         /*
2980          * vpid_sync_context() is a nop if vmx->vpid==0, e.g. if enable_vpid==0
2981          * or a vpid couldn't be allocated for this vCPU.  VM-Enter and VM-Exit
2982          * are required to flush GVA->{G,H}PA mappings from the TLB if vpid is
2983          * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed),
2984          * i.e. no explicit INVVPID is necessary.
2985          */
2986         vpid_sync_context(to_vmx(vcpu)->vpid);
2987 }
2988
2989 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu)
2990 {
2991         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
2992
2993         if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR))
2994                 return;
2995
2996         if (is_pae_paging(vcpu)) {
2997                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
2998                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
2999                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
3000                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
3001         }
3002 }
3003
3004 void ept_save_pdptrs(struct kvm_vcpu *vcpu)
3005 {
3006         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
3007
3008         if (WARN_ON_ONCE(!is_pae_paging(vcpu)))
3009                 return;
3010
3011         mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
3012         mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
3013         mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
3014         mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
3015
3016         kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
3017 }
3018
3019 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
3020                                         unsigned long cr0,
3021                                         struct kvm_vcpu *vcpu)
3022 {
3023         struct vcpu_vmx *vmx = to_vmx(vcpu);
3024
3025         if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
3026                 vmx_cache_reg(vcpu, VCPU_EXREG_CR3);
3027         if (!(cr0 & X86_CR0_PG)) {
3028                 /* From paging/starting to nonpaging */
3029                 exec_controls_setbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
3030                                           CPU_BASED_CR3_STORE_EXITING);
3031                 vcpu->arch.cr0 = cr0;
3032                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3033         } else if (!is_paging(vcpu)) {
3034                 /* From nonpaging to paging */
3035                 exec_controls_clearbit(vmx, CPU_BASED_CR3_LOAD_EXITING |
3036                                             CPU_BASED_CR3_STORE_EXITING);
3037                 vcpu->arch.cr0 = cr0;
3038                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
3039         }
3040
3041         if (!(cr0 & X86_CR0_WP))
3042                 *hw_cr0 &= ~X86_CR0_WP;
3043 }
3044
3045 void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
3046 {
3047         struct vcpu_vmx *vmx = to_vmx(vcpu);
3048         unsigned long hw_cr0;
3049
3050         hw_cr0 = (cr0 & ~KVM_VM_CR0_ALWAYS_OFF);
3051         if (enable_unrestricted_guest)
3052                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
3053         else {
3054                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
3055
3056                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
3057                         enter_pmode(vcpu);
3058
3059                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
3060                         enter_rmode(vcpu);
3061         }
3062
3063 #ifdef CONFIG_X86_64
3064         if (vcpu->arch.efer & EFER_LME) {
3065                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
3066                         enter_lmode(vcpu);
3067                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
3068                         exit_lmode(vcpu);
3069         }
3070 #endif
3071
3072         if (enable_ept && !enable_unrestricted_guest)
3073                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
3074
3075         vmcs_writel(CR0_READ_SHADOW, cr0);
3076         vmcs_writel(GUEST_CR0, hw_cr0);
3077         vcpu->arch.cr0 = cr0;
3078         kvm_register_mark_available(vcpu, VCPU_EXREG_CR0);
3079
3080         /* depends on vcpu->arch.cr0 to be set to a new value */
3081         vmx->emulation_required = emulation_required(vcpu);
3082 }
3083
3084 static int vmx_get_max_tdp_level(void)
3085 {
3086         if (cpu_has_vmx_ept_5levels())
3087                 return 5;
3088         return 4;
3089 }
3090
3091 u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa,
3092                    int root_level)
3093 {
3094         u64 eptp = VMX_EPTP_MT_WB;
3095
3096         eptp |= (root_level == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
3097
3098         if (enable_ept_ad_bits &&
3099             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
3100                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
3101         eptp |= (root_hpa & PAGE_MASK);
3102
3103         return eptp;
3104 }
3105
3106 static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd,
3107                              int pgd_level)
3108 {
3109         struct kvm *kvm = vcpu->kvm;
3110         bool update_guest_cr3 = true;
3111         unsigned long guest_cr3;
3112         u64 eptp;
3113
3114         if (enable_ept) {
3115                 eptp = construct_eptp(vcpu, pgd, pgd_level);
3116                 vmcs_write64(EPT_POINTER, eptp);
3117
3118                 if (kvm_x86_ops.tlb_remote_flush) {
3119                         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
3120                         to_vmx(vcpu)->ept_pointer = eptp;
3121                         to_kvm_vmx(kvm)->ept_pointers_match
3122                                 = EPT_POINTERS_CHECK;
3123                         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
3124                 }
3125
3126                 if (!enable_unrestricted_guest && !is_paging(vcpu))
3127                         guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
3128                 else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
3129                         guest_cr3 = vcpu->arch.cr3;
3130                 else /* vmcs01.GUEST_CR3 is already up-to-date. */
3131                         update_guest_cr3 = false;
3132                 vmx_ept_load_pdptrs(vcpu);
3133         } else {
3134                 guest_cr3 = pgd;
3135         }
3136
3137         if (update_guest_cr3)
3138                 vmcs_writel(GUEST_CR3, guest_cr3);
3139 }
3140
3141 int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
3142 {
3143         struct vcpu_vmx *vmx = to_vmx(vcpu);
3144         /*
3145          * Pass through host's Machine Check Enable value to hw_cr4, which
3146          * is in force while we are in guest mode.  Do not let guests control
3147          * this bit, even if host CR4.MCE == 0.
3148          */
3149         unsigned long hw_cr4;
3150
3151         hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
3152         if (enable_unrestricted_guest)
3153                 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
3154         else if (vmx->rmode.vm86_active)
3155                 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
3156         else
3157                 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
3158
3159         if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
3160                 if (cr4 & X86_CR4_UMIP) {
3161                         secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_DESC);
3162                         hw_cr4 &= ~X86_CR4_UMIP;
3163                 } else if (!is_guest_mode(vcpu) ||
3164                         !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC)) {
3165                         secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_DESC);
3166                 }
3167         }
3168
3169         if (cr4 & X86_CR4_VMXE) {
3170                 /*
3171                  * To use VMXON (and later other VMX instructions), a guest
3172                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
3173                  * So basically the check on whether to allow nested VMX
3174                  * is here.  We operate under the default treatment of SMM,
3175                  * so VMX cannot be enabled under SMM.
3176                  */
3177                 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
3178                         return 1;
3179         }
3180
3181         if (vmx->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
3182                 return 1;
3183
3184         vcpu->arch.cr4 = cr4;
3185         kvm_register_mark_available(vcpu, VCPU_EXREG_CR4);
3186
3187         if (!enable_unrestricted_guest) {
3188                 if (enable_ept) {
3189                         if (!is_paging(vcpu)) {
3190                                 hw_cr4 &= ~X86_CR4_PAE;
3191                                 hw_cr4 |= X86_CR4_PSE;
3192                         } else if (!(cr4 & X86_CR4_PAE)) {
3193                                 hw_cr4 &= ~X86_CR4_PAE;
3194                         }
3195                 }
3196
3197                 /*
3198                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
3199                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
3200                  * to be manually disabled when guest switches to non-paging
3201                  * mode.
3202                  *
3203                  * If !enable_unrestricted_guest, the CPU is always running
3204                  * with CR0.PG=1 and CR4 needs to be modified.
3205                  * If enable_unrestricted_guest, the CPU automatically
3206                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
3207                  */
3208                 if (!is_paging(vcpu))
3209                         hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
3210         }
3211
3212         vmcs_writel(CR4_READ_SHADOW, cr4);
3213         vmcs_writel(GUEST_CR4, hw_cr4);
3214         return 0;
3215 }
3216
3217 void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3218 {
3219         struct vcpu_vmx *vmx = to_vmx(vcpu);
3220         u32 ar;
3221
3222         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3223                 *var = vmx->rmode.segs[seg];
3224                 if (seg == VCPU_SREG_TR
3225                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3226                         return;
3227                 var->base = vmx_read_guest_seg_base(vmx, seg);
3228                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3229                 return;
3230         }
3231         var->base = vmx_read_guest_seg_base(vmx, seg);
3232         var->limit = vmx_read_guest_seg_limit(vmx, seg);
3233         var->selector = vmx_read_guest_seg_selector(vmx, seg);
3234         ar = vmx_read_guest_seg_ar(vmx, seg);
3235         var->unusable = (ar >> 16) & 1;
3236         var->type = ar & 15;
3237         var->s = (ar >> 4) & 1;
3238         var->dpl = (ar >> 5) & 3;
3239         /*
3240          * Some userspaces do not preserve unusable property. Since usable
3241          * segment has to be present according to VMX spec we can use present
3242          * property to amend userspace bug by making unusable segment always
3243          * nonpresent. vmx_segment_access_rights() already marks nonpresent
3244          * segment as unusable.
3245          */
3246         var->present = !var->unusable;
3247         var->avl = (ar >> 12) & 1;
3248         var->l = (ar >> 13) & 1;
3249         var->db = (ar >> 14) & 1;
3250         var->g = (ar >> 15) & 1;
3251 }
3252
3253 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
3254 {
3255         struct kvm_segment s;
3256
3257         if (to_vmx(vcpu)->rmode.vm86_active) {
3258                 vmx_get_segment(vcpu, &s, seg);
3259                 return s.base;
3260         }
3261         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
3262 }
3263
3264 int vmx_get_cpl(struct kvm_vcpu *vcpu)
3265 {
3266         struct vcpu_vmx *vmx = to_vmx(vcpu);
3267
3268         if (unlikely(vmx->rmode.vm86_active))
3269                 return 0;
3270         else {
3271                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
3272                 return VMX_AR_DPL(ar);
3273         }
3274 }
3275
3276 static u32 vmx_segment_access_rights(struct kvm_segment *var)
3277 {
3278         u32 ar;
3279
3280         if (var->unusable || !var->present)
3281                 ar = 1 << 16;
3282         else {
3283                 ar = var->type & 15;
3284                 ar |= (var->s & 1) << 4;
3285                 ar |= (var->dpl & 3) << 5;
3286                 ar |= (var->present & 1) << 7;
3287                 ar |= (var->avl & 1) << 12;
3288                 ar |= (var->l & 1) << 13;
3289                 ar |= (var->db & 1) << 14;
3290                 ar |= (var->g & 1) << 15;
3291         }
3292
3293         return ar;
3294 }
3295
3296 void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
3297 {
3298         struct vcpu_vmx *vmx = to_vmx(vcpu);
3299         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3300
3301         vmx_segment_cache_clear(vmx);
3302
3303         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
3304                 vmx->rmode.segs[seg] = *var;
3305                 if (seg == VCPU_SREG_TR)
3306                         vmcs_write16(sf->selector, var->selector);
3307                 else if (var->s)
3308                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3309                 goto out;
3310         }
3311
3312         vmcs_writel(sf->base, var->base);
3313         vmcs_write32(sf->limit, var->limit);
3314         vmcs_write16(sf->selector, var->selector);
3315
3316         /*
3317          *   Fix the "Accessed" bit in AR field of segment registers for older
3318          * qemu binaries.
3319          *   IA32 arch specifies that at the time of processor reset the
3320          * "Accessed" bit in the AR field of segment registers is 1. And qemu
3321          * is setting it to 0 in the userland code. This causes invalid guest
3322          * state vmexit when "unrestricted guest" mode is turned on.
3323          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
3324          * tree. Newer qemu binaries with that qemu fix would not need this
3325          * kvm hack.
3326          */
3327         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
3328                 var->type |= 0x1; /* Accessed */
3329
3330         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
3331
3332 out:
3333         vmx->emulation_required = emulation_required(vcpu);
3334 }
3335
3336 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
3337 {
3338         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
3339
3340         *db = (ar >> 14) & 1;
3341         *l = (ar >> 13) & 1;
3342 }
3343
3344 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3345 {
3346         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
3347         dt->address = vmcs_readl(GUEST_IDTR_BASE);
3348 }
3349
3350 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3351 {
3352         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
3353         vmcs_writel(GUEST_IDTR_BASE, dt->address);
3354 }
3355
3356 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3357 {
3358         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
3359         dt->address = vmcs_readl(GUEST_GDTR_BASE);
3360 }
3361
3362 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
3363 {
3364         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
3365         vmcs_writel(GUEST_GDTR_BASE, dt->address);
3366 }
3367
3368 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3369 {
3370         struct kvm_segment var;
3371         u32 ar;
3372
3373         vmx_get_segment(vcpu, &var, seg);
3374         var.dpl = 0x3;
3375         if (seg == VCPU_SREG_CS)
3376                 var.type = 0x3;
3377         ar = vmx_segment_access_rights(&var);
3378
3379         if (var.base != (var.selector << 4))
3380                 return false;
3381         if (var.limit != 0xffff)
3382                 return false;
3383         if (ar != 0xf3)
3384                 return false;
3385
3386         return true;
3387 }
3388
3389 static bool code_segment_valid(struct kvm_vcpu *vcpu)
3390 {
3391         struct kvm_segment cs;
3392         unsigned int cs_rpl;
3393
3394         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
3395         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
3396
3397         if (cs.unusable)
3398                 return false;
3399         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
3400                 return false;
3401         if (!cs.s)
3402                 return false;
3403         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
3404                 if (cs.dpl > cs_rpl)
3405                         return false;
3406         } else {
3407                 if (cs.dpl != cs_rpl)
3408                         return false;
3409         }
3410         if (!cs.present)
3411                 return false;
3412
3413         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
3414         return true;
3415 }
3416
3417 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
3418 {
3419         struct kvm_segment ss;
3420         unsigned int ss_rpl;
3421
3422         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
3423         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
3424
3425         if (ss.unusable)
3426                 return true;
3427         if (ss.type != 3 && ss.type != 7)
3428                 return false;
3429         if (!ss.s)
3430             &nbs