arch/x86/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "cpuid.h"
  22 #include "lapic.h"
  23
  24 #include <linux/kvm_host.h>
  25 #include <linux/module.h>
  26 #include <linux/kernel.h>
  27 #include <linux/mm.h>
  28 #include <linux/highmem.h>
  29 #include <linux/sched.h>
  30 #include <linux/moduleparam.h>
  31 #include <linux/mod_devicetable.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/tboot.h>
  35 #include <linux/hrtimer.h>
  36 #include <linux/frame.h>
  37 #include <linux/nospec.h>
  38 #include "kvm_cache_regs.h"
  39 #include "x86.h"
  40
  41 #include <asm/asm.h>
  42 #include <asm/cpu.h>
  43 #include <asm/io.h>
  44 #include <asm/desc.h>
  45 #include <asm/vmx.h>
  46 #include <asm/virtext.h>
  47 #include <asm/mce.h>
  48 #include <asm/fpu/internal.h>
  49 #include <asm/perf_event.h>
  50 #include <asm/debugreg.h>
  51 #include <asm/kexec.h>
  52 #include <asm/apic.h>
  53 #include <asm/irq_remapping.h>
  54 #include <asm/mmu_context.h>
  55 #include <asm/spec-ctrl.h>
  56 #include <asm/mshyperv.h>
  57
  58 #include "trace.h"
  59 #include "pmu.h"
  60 #include "vmx_evmcs.h"
  61
  62 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  63 #define __ex_clear(x, reg) \
  64         ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
  65
  66 MODULE_AUTHOR("Qumranet");
  67 MODULE_LICENSE("GPL");
  68
  69 static const struct x86_cpu_id vmx_cpu_id[] = {
  70         X86_FEATURE_MATCH(X86_FEATURE_VMX),
  71         {}
  72 };
  73 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  74
  75 static bool __read_mostly enable_vpid = 1;
  76 module_param_named(vpid, enable_vpid, bool, 0444);
  77
  78 static bool __read_mostly enable_vnmi = 1;
  79 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
  80
  81 static bool __read_mostly flexpriority_enabled = 1;
  82 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  83
  84 static bool __read_mostly enable_ept = 1;
  85 module_param_named(ept, enable_ept, bool, S_IRUGO);
  86
  87 static bool __read_mostly enable_unrestricted_guest = 1;
  88 module_param_named(unrestricted_guest,
  89                         enable_unrestricted_guest, bool, S_IRUGO);
  90
  91 static bool __read_mostly enable_ept_ad_bits = 1;
  92 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  93
  94 static bool __read_mostly emulate_invalid_guest_state = true;
  95 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  96
  97 static bool __read_mostly fasteoi = 1;
  98 module_param(fasteoi, bool, S_IRUGO);
  99
 100 static bool __read_mostly enable_apicv = 1;
 101 module_param(enable_apicv, bool, S_IRUGO);
 102
 103 static bool __read_mostly enable_shadow_vmcs = 1;
 104 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 105 /*
 106  * If nested=1, nested virtualization is supported, i.e., guests may use
 107  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 108  * use VMX instructions.
 109  */
 110 static bool __read_mostly nested = 0;
 111 module_param(nested, bool, S_IRUGO);
 112
 113 static u64 __read_mostly host_xss;
 114
 115 static bool __read_mostly enable_pml = 1;
 116 module_param_named(pml, enable_pml, bool, S_IRUGO);
 117
 118 #define MSR_TYPE_R      1
 119 #define MSR_TYPE_W      2
 120 #define MSR_TYPE_RW     3
 121
 122 #define MSR_BITMAP_MODE_X2APIC          1
 123 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 124 #define MSR_BITMAP_MODE_LM              4
 125
 126 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 127
 128 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 129 static int __read_mostly cpu_preemption_timer_multi;
 130 static bool __read_mostly enable_preemption_timer = 1;
 131 #ifdef CONFIG_X86_64
 132 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 133 #endif
 134
 135 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
 136 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
 137 #define KVM_VM_CR0_ALWAYS_ON                            \
 138         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST |      \
 139          X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
 140 #define KVM_CR4_GUEST_OWNED_BITS                                      \
 141         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
 142          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
 143
 144 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
 145 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 146 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 147
 148 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 149
 150 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 151
 152 /*
 153  * Hyper-V requires all of these, so mark them as supported even though
 154  * they are just treated the same as all-context.
 155  */
 156 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
 157         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
 158         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
 159         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
 160         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
 161
 162 /*
 163  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 164  * ple_gap:    upper bound on the amount of time between two successive
 165  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 166  *             According to test, this time is usually smaller than 128 cycles.
 167  * ple_window: upper bound on the amount of time a guest is allowed to execute
 168  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 169  *             less than 2^12 cycles
 170  * Time is measured based on a counter that runs at the same rate as the TSC,
 171  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 172  */
 173 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
 174
 175 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 176 module_param(ple_window, uint, 0444);
 177
 178 /* Default doubles per-vcpu window every exit. */
 179 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 180 module_param(ple_window_grow, uint, 0444);
 181
 182 /* Default resets per-vcpu window every exit to ple_window. */
 183 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 184 module_param(ple_window_shrink, uint, 0444);
 185
 186 /* Default is to compute the maximum so we can never overflow. */
 187 static unsigned int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 188 module_param(ple_window_max, uint, 0444);
 189
 190 extern const ulong vmx_return;
 191
 192 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 193 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 194 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 195
 196 /* Storage for pre module init parameter parsing */
 197 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
 198
 199 static const struct {
 200         const char *option;
 201         enum vmx_l1d_flush_state cmd;
 202 } vmentry_l1d_param[] = {
 203         {"auto",        VMENTER_L1D_FLUSH_AUTO},
 204         {"never",       VMENTER_L1D_FLUSH_NEVER},
 205         {"cond",        VMENTER_L1D_FLUSH_COND},
 206         {"always",      VMENTER_L1D_FLUSH_ALWAYS},
 207 };
 208
 209 #define L1D_CACHE_ORDER 4
 210 static void *vmx_l1d_flush_pages;
 211
 212 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 213 {
 214         struct page *page;
 215         unsigned int i;
 216
 217         if (!enable_ept) {
 218                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
 219                 return 0;
 220         }
 221
 222        if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
 223                u64 msr;
 224
 225                rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
 226                if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
 227                        l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 228                        return 0;
 229                }
 230        }
 231
 232         /* If set to auto use the default l1tf mitigation method */
 233         if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
 234                 switch (l1tf_mitigation) {
 235                 case L1TF_MITIGATION_OFF:
 236                         l1tf = VMENTER_L1D_FLUSH_NEVER;
 237                         break;
 238                 case L1TF_MITIGATION_FLUSH_NOWARN:
 239                 case L1TF_MITIGATION_FLUSH:
 240                 case L1TF_MITIGATION_FLUSH_NOSMT:
 241                         l1tf = VMENTER_L1D_FLUSH_COND;
 242                         break;
 243                 case L1TF_MITIGATION_FULL:
 244                 case L1TF_MITIGATION_FULL_FORCE:
 245                         l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 246                         break;
 247                 }
 248         } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
 249                 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 250         }
 251
 252         if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
 253             !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
 254                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
 255                 if (!page)
 256                         return -ENOMEM;
 257                 vmx_l1d_flush_pages = page_address(page);
 258
 259                 /*
 260                  * Initialize each page with a different pattern in
 261                  * order to protect against KSM in the nested
 262                  * virtualization case.
 263                  */
 264                 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
 265                         memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
 266                                PAGE_SIZE);
 267                 }
 268         }
 269
 270         l1tf_vmx_mitigation = l1tf;
 271
 272         if (l1tf != VMENTER_L1D_FLUSH_NEVER)
 273                 static_branch_enable(&vmx_l1d_should_flush);
 274         else
 275                 static_branch_disable(&vmx_l1d_should_flush);
 276
 277         if (l1tf == VMENTER_L1D_FLUSH_COND)
 278                 static_branch_enable(&vmx_l1d_flush_cond);
 279         else
 280                 static_branch_disable(&vmx_l1d_flush_cond);
 281         return 0;
 282 }
 283
 284 static int vmentry_l1d_flush_parse(const char *s)
 285 {
 286         unsigned int i;
 287
 288         if (s) {
 289                 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 290                         if (sysfs_streq(s, vmentry_l1d_param[i].option))
 291                                 return vmentry_l1d_param[i].cmd;
 292                 }
 293         }
 294         return -EINVAL;
 295 }
 296
 297 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 298 {
 299         int l1tf, ret;
 300
 301         if (!boot_cpu_has(X86_BUG_L1TF))
 302                 return 0;
 303
 304         l1tf = vmentry_l1d_flush_parse(s);
 305         if (l1tf < 0)
 306                 return l1tf;
 307
 308         /*
 309          * Has vmx_init() run already? If not then this is the pre init
 310          * parameter parsing. In that case just store the value and let
 311          * vmx_init() do the proper setup after enable_ept has been
 312          * established.
 313          */
 314         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
 315                 vmentry_l1d_flush_param = l1tf;
 316                 return 0;
 317         }
 318
 319         mutex_lock(&vmx_l1d_flush_mutex);
 320         ret = vmx_setup_l1d_flush(l1tf);
 321         mutex_unlock(&vmx_l1d_flush_mutex);
 322         return ret;
 323 }
 324
 325 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 326 {
 327         return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 328 }
 329
 330 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 331         .set = vmentry_l1d_flush_set,
 332         .get = vmentry_l1d_flush_get,
 333 };
 334 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 335
 336 enum ept_pointers_status {
 337         EPT_POINTERS_CHECK = 0,
 338         EPT_POINTERS_MATCH = 1,
 339         EPT_POINTERS_MISMATCH = 2
 340 };
 341
 342 struct kvm_vmx {
 343         struct kvm kvm;
 344
 345         unsigned int tss_addr;
 346         bool ept_identity_pagetable_done;
 347         gpa_t ept_identity_map_addr;
 348
 349         enum ept_pointers_status ept_pointers_match;
 350         spinlock_t ept_pointer_lock;
 351 };
 352
 353 #define NR_AUTOLOAD_MSRS 8
 354
 355 struct vmcs_hdr {
 356         u32 revision_id:31;
 357         u32 shadow_vmcs:1;
 358 };
 359
 360 struct vmcs {
 361         struct vmcs_hdr hdr;
 362         u32 abort;
 363         char data[0];
 364 };
 365
 366 /*
 367  * vmcs_host_state tracks registers that are loaded from the VMCS on VMEXIT
 368  * and whose values change infrequently, but are not constant.  I.e. this is
 369  * used as a write-through cache of the corresponding VMCS fields.
 370  */
 371 struct vmcs_host_state {
 372         unsigned long cr3;      /* May not match real cr3 */
 373         unsigned long cr4;      /* May not match real cr4 */
 374         unsigned long gs_base;
 375         unsigned long fs_base;
 376
 377         u16           fs_sel, gs_sel, ldt_sel;
 378 #ifdef CONFIG_X86_64
 379         u16           ds_sel, es_sel;
 380 #endif
 381 };
 382
 383 /*
 384  * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 385  * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 386  * loaded on this CPU (so we can clear them if the CPU goes down).
 387  */
 388 struct loaded_vmcs {
 389         struct vmcs *vmcs;
 390         struct vmcs *shadow_vmcs;
 391         int cpu;
 392         bool launched;
 393         bool nmi_known_unmasked;
 394         /* Support for vnmi-less CPUs */
 395         int soft_vnmi_blocked;
 396         ktime_t entry_time;
 397         s64 vnmi_blocked_time;
 398         unsigned long *msr_bitmap;
 399         struct list_head loaded_vmcss_on_cpu_link;
 400         struct vmcs_host_state host_state;
 401 };
 402
 403 struct shared_msr_entry {
 404         unsigned index;
 405         u64 data;
 406         u64 mask;
 407 };
 408
 409 /*
 410  * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 411  * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 412  * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 413  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 414  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 415  * More than one of these structures may exist, if L1 runs multiple L2 guests.
 416  * nested_vmx_run() will use the data here to build the vmcs02: a VMCS for the
 417  * underlying hardware which will be used to run L2.
 418  * This structure is packed to ensure that its layout is identical across
 419  * machines (necessary for live migration).
 420  *
 421  * IMPORTANT: Changing the layout of existing fields in this structure
 422  * will break save/restore compatibility with older kvm releases. When
 423  * adding new fields, either use space in the reserved padding* arrays
 424  * or add the new fields to the end of the structure.
 425  */
 426 typedef u64 natural_width;
 427 struct __packed vmcs12 {
 428         /* According to the Intel spec, a VMCS region must start with the
 429          * following two fields. Then follow implementation-specific data.
 430          */
 431         struct vmcs_hdr hdr;
 432         u32 abort;
 433
 434         u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
 435         u32 padding[7]; /* room for future expansion */
 436
 437         u64 io_bitmap_a;
 438         u64 io_bitmap_b;
 439         u64 msr_bitmap;
 440         u64 vm_exit_msr_store_addr;
 441         u64 vm_exit_msr_load_addr;
 442         u64 vm_entry_msr_load_addr;
 443         u64 tsc_offset;
 444         u64 virtual_apic_page_addr;
 445         u64 apic_access_addr;
 446         u64 posted_intr_desc_addr;
 447         u64 ept_pointer;
 448         u64 eoi_exit_bitmap0;
 449         u64 eoi_exit_bitmap1;
 450         u64 eoi_exit_bitmap2;
 451         u64 eoi_exit_bitmap3;
 452         u64 xss_exit_bitmap;
 453         u64 guest_physical_address;
 454         u64 vmcs_link_pointer;
 455         u64 guest_ia32_debugctl;
 456         u64 guest_ia32_pat;
 457         u64 guest_ia32_efer;
 458         u64 guest_ia32_perf_global_ctrl;
 459         u64 guest_pdptr0;
 460         u64 guest_pdptr1;
 461         u64 guest_pdptr2;
 462         u64 guest_pdptr3;
 463         u64 guest_bndcfgs;
 464         u64 host_ia32_pat;
 465         u64 host_ia32_efer;
 466         u64 host_ia32_perf_global_ctrl;
 467         u64 vmread_bitmap;
 468         u64 vmwrite_bitmap;
 469         u64 vm_function_control;
 470         u64 eptp_list_address;
 471         u64 pml_address;
 472         u64 padding64[3]; /* room for future expansion */
 473         /*
 474          * To allow migration of L1 (complete with its L2 guests) between
 475          * machines of different natural widths (32 or 64 bit), we cannot have
 476          * unsigned long fields with no explict size. We use u64 (aliased
 477          * natural_width) instead. Luckily, x86 is little-endian.
 478          */
 479         natural_width cr0_guest_host_mask;
 480         natural_width cr4_guest_host_mask;
 481         natural_width cr0_read_shadow;
 482         natural_width cr4_read_shadow;
 483         natural_width cr3_target_value0;
 484         natural_width cr3_target_value1;
 485         natural_width cr3_target_value2;
 486         natural_width cr3_target_value3;
 487         natural_width exit_qualification;
 488         natural_width guest_linear_address;
 489         natural_width guest_cr0;
 490         natural_width guest_cr3;
 491         natural_width guest_cr4;
 492         natural_width guest_es_base;
 493         natural_width guest_cs_base;
 494         natural_width guest_ss_base;
 495         natural_width guest_ds_base;
 496         natural_width guest_fs_base;
 497         natural_width guest_gs_base;
 498         natural_width guest_ldtr_base;
 499         natural_width guest_tr_base;
 500         natural_width guest_gdtr_base;
 501         natural_width guest_idtr_base;
 502         natural_width guest_dr7;
 503         natural_width guest_rsp;
 504         natural_width guest_rip;
 505         natural_width guest_rflags;
 506         natural_width guest_pending_dbg_exceptions;
 507         natural_width guest_sysenter_esp;
 508         natural_width guest_sysenter_eip;
 509         natural_width host_cr0;
 510         natural_width host_cr3;
 511         natural_width host_cr4;
 512         natural_width host_fs_base;
 513         natural_width host_gs_base;
 514         natural_width host_tr_base;
 515         natural_width host_gdtr_base;
 516         natural_width host_idtr_base;
 517         natural_width host_ia32_sysenter_esp;
 518         natural_width host_ia32_sysenter_eip;
 519         natural_width host_rsp;
 520         natural_width host_rip;
 521         natural_width paddingl[8]; /* room for future expansion */
 522         u32 pin_based_vm_exec_control;
 523         u32 cpu_based_vm_exec_control;
 524         u32 exception_bitmap;
 525         u32 page_fault_error_code_mask;
 526         u32 page_fault_error_code_match;
 527         u32 cr3_target_count;
 528         u32 vm_exit_controls;
 529         u32 vm_exit_msr_store_count;
 530         u32 vm_exit_msr_load_count;
 531         u32 vm_entry_controls;
 532         u32 vm_entry_msr_load_count;
 533         u32 vm_entry_intr_info_field;
 534         u32 vm_entry_exception_error_code;
 535         u32 vm_entry_instruction_len;
 536         u32 tpr_threshold;
 537         u32 secondary_vm_exec_control;
 538         u32 vm_instruction_error;
 539         u32 vm_exit_reason;
 540         u32 vm_exit_intr_info;
 541         u32 vm_exit_intr_error_code;
 542         u32 idt_vectoring_info_field;
 543         u32 idt_vectoring_error_code;
 544         u32 vm_exit_instruction_len;
 545         u32 vmx_instruction_info;
 546         u32 guest_es_limit;
 547         u32 guest_cs_limit;
 548         u32 guest_ss_limit;
 549         u32 guest_ds_limit;
 550         u32 guest_fs_limit;
 551         u32 guest_gs_limit;
 552         u32 guest_ldtr_limit;
 553         u32 guest_tr_limit;
 554         u32 guest_gdtr_limit;
 555         u32 guest_idtr_limit;
 556         u32 guest_es_ar_bytes;
 557         u32 guest_cs_ar_bytes;
 558         u32 guest_ss_ar_bytes;
 559         u32 guest_ds_ar_bytes;
 560         u32 guest_fs_ar_bytes;
 561         u32 guest_gs_ar_bytes;
 562         u32 guest_ldtr_ar_bytes;
 563         u32 guest_tr_ar_bytes;
 564         u32 guest_interruptibility_info;
 565         u32 guest_activity_state;
 566         u32 guest_sysenter_cs;
 567         u32 host_ia32_sysenter_cs;
 568         u32 vmx_preemption_timer_value;
 569         u32 padding32[7]; /* room for future expansion */
 570         u16 virtual_processor_id;
 571         u16 posted_intr_nv;
 572         u16 guest_es_selector;
 573         u16 guest_cs_selector;
 574         u16 guest_ss_selector;
 575         u16 guest_ds_selector;
 576         u16 guest_fs_selector;
 577         u16 guest_gs_selector;
 578         u16 guest_ldtr_selector;
 579         u16 guest_tr_selector;
 580         u16 guest_intr_status;
 581         u16 host_es_selector;
 582         u16 host_cs_selector;
 583         u16 host_ss_selector;
 584         u16 host_ds_selector;
 585         u16 host_fs_selector;
 586         u16 host_gs_selector;
 587         u16 host_tr_selector;
 588         u16 guest_pml_index;
 589 };
 590
 591 /*
 592  * For save/restore compatibility, the vmcs12 field offsets must not change.
 593  */
 594 #define CHECK_OFFSET(field, loc)                                \
 595         BUILD_BUG_ON_MSG(offsetof(struct vmcs12, field) != (loc),       \
 596                 "Offset of " #field " in struct vmcs12 has changed.")
 597
 598 static inline void vmx_check_vmcs12_offsets(void) {
 599         CHECK_OFFSET(hdr, 0);
 600         CHECK_OFFSET(abort, 4);
 601         CHECK_OFFSET(launch_state, 8);
 602         CHECK_OFFSET(io_bitmap_a, 40);
 603         CHECK_OFFSET(io_bitmap_b, 48);
 604         CHECK_OFFSET(msr_bitmap, 56);
 605         CHECK_OFFSET(vm_exit_msr_store_addr, 64);
 606         CHECK_OFFSET(vm_exit_msr_load_addr, 72);
 607         CHECK_OFFSET(vm_entry_msr_load_addr, 80);
 608         CHECK_OFFSET(tsc_offset, 88);
 609         CHECK_OFFSET(virtual_apic_page_addr, 96);
 610         CHECK_OFFSET(apic_access_addr, 104);
 611         CHECK_OFFSET(posted_intr_desc_addr, 112);
 612         CHECK_OFFSET(ept_pointer, 120);
 613         CHECK_OFFSET(eoi_exit_bitmap0, 128);
 614         CHECK_OFFSET(eoi_exit_bitmap1, 136);
 615         CHECK_OFFSET(eoi_exit_bitmap2, 144);
 616         CHECK_OFFSET(eoi_exit_bitmap3, 152);
 617         CHECK_OFFSET(xss_exit_bitmap, 160);
 618         CHECK_OFFSET(guest_physical_address, 168);
 619         CHECK_OFFSET(vmcs_link_pointer, 176);
 620         CHECK_OFFSET(guest_ia32_debugctl, 184);
 621         CHECK_OFFSET(guest_ia32_pat, 192);
 622         CHECK_OFFSET(guest_ia32_efer, 200);
 623         CHECK_OFFSET(guest_ia32_perf_global_ctrl, 208);
 624         CHECK_OFFSET(guest_pdptr0, 216);
 625         CHECK_OFFSET(guest_pdptr1, 224);
 626         CHECK_OFFSET(guest_pdptr2, 232);
 627         CHECK_OFFSET(guest_pdptr3, 240);
 628         CHECK_OFFSET(guest_bndcfgs, 248);
 629         CHECK_OFFSET(host_ia32_pat, 256);
 630         CHECK_OFFSET(host_ia32_efer, 264);
 631         CHECK_OFFSET(host_ia32_perf_global_ctrl, 272);
 632         CHECK_OFFSET(vmread_bitmap, 280);
 633         CHECK_OFFSET(vmwrite_bitmap, 288);
 634         CHECK_OFFSET(vm_function_control, 296);
 635         CHECK_OFFSET(eptp_list_address, 304);
 636         CHECK_OFFSET(pml_address, 312);
 637         CHECK_OFFSET(cr0_guest_host_mask, 344);
 638         CHECK_OFFSET(cr4_guest_host_mask, 352);
 639         CHECK_OFFSET(cr0_read_shadow, 360);
 640         CHECK_OFFSET(cr4_read_shadow, 368);
 641         CHECK_OFFSET(cr3_target_value0, 376);
 642         CHECK_OFFSET(cr3_target_value1, 384);
 643         CHECK_OFFSET(cr3_target_value2, 392);
 644         CHECK_OFFSET(cr3_target_value3, 400);
 645         CHECK_OFFSET(exit_qualification, 408);
 646         CHECK_OFFSET(guest_linear_address, 416);
 647         CHECK_OFFSET(guest_cr0, 424);
 648         CHECK_OFFSET(guest_cr3, 432);
 649         CHECK_OFFSET(guest_cr4, 440);
 650         CHECK_OFFSET(guest_es_base, 448);
 651         CHECK_OFFSET(guest_cs_base, 456);
 652         CHECK_OFFSET(guest_ss_base, 464);
 653         CHECK_OFFSET(guest_ds_base, 472);
 654         CHECK_OFFSET(guest_fs_base, 480);
 655         CHECK_OFFSET(guest_gs_base, 488);
 656         CHECK_OFFSET(guest_ldtr_base, 496);
 657         CHECK_OFFSET(guest_tr_base, 504);
 658         CHECK_OFFSET(guest_gdtr_base, 512);
 659         CHECK_OFFSET(guest_idtr_base, 520);
 660         CHECK_OFFSET(guest_dr7, 528);
 661         CHECK_OFFSET(guest_rsp, 536);
 662         CHECK_OFFSET(guest_rip, 544);
 663         CHECK_OFFSET(guest_rflags, 552);
 664         CHECK_OFFSET(guest_pending_dbg_exceptions, 560);
 665         CHECK_OFFSET(guest_sysenter_esp, 568);
 666         CHECK_OFFSET(guest_sysenter_eip, 576);
 667         CHECK_OFFSET(host_cr0, 584);
 668         CHECK_OFFSET(host_cr3, 592);
 669         CHECK_OFFSET(host_cr4, 600);
 670         CHECK_OFFSET(host_fs_base, 608);
 671         CHECK_OFFSET(host_gs_base, 616);
 672         CHECK_OFFSET(host_tr_base, 624);
 673         CHECK_OFFSET(host_gdtr_base, 632);
 674         CHECK_OFFSET(host_idtr_base, 640);
 675         CHECK_OFFSET(host_ia32_sysenter_esp, 648);
 676         CHECK_OFFSET(host_ia32_sysenter_eip, 656);
 677         CHECK_OFFSET(host_rsp, 664);
 678         CHECK_OFFSET(host_rip, 672);
 679         CHECK_OFFSET(pin_based_vm_exec_control, 744);
 680         CHECK_OFFSET(cpu_based_vm_exec_control, 748);
 681         CHECK_OFFSET(exception_bitmap, 752);
 682         CHECK_OFFSET(page_fault_error_code_mask, 756);
 683         CHECK_OFFSET(page_fault_error_code_match, 760);
 684         CHECK_OFFSET(cr3_target_count, 764);
 685         CHECK_OFFSET(vm_exit_controls, 768);
 686         CHECK_OFFSET(vm_exit_msr_store_count, 772);
 687         CHECK_OFFSET(vm_exit_msr_load_count, 776);
 688         CHECK_OFFSET(vm_entry_controls, 780);
 689         CHECK_OFFSET(vm_entry_msr_load_count, 784);
 690         CHECK_OFFSET(vm_entry_intr_info_field, 788);
 691         CHECK_OFFSET(vm_entry_exception_error_code, 792);
 692         CHECK_OFFSET(vm_entry_instruction_len, 796);
 693         CHECK_OFFSET(tpr_threshold, 800);
 694         CHECK_OFFSET(secondary_vm_exec_control, 804);
 695         CHECK_OFFSET(vm_instruction_error, 808);
 696         CHECK_OFFSET(vm_exit_reason, 812);
 697         CHECK_OFFSET(vm_exit_intr_info, 816);
 698         CHECK_OFFSET(vm_exit_intr_error_code, 820);
 699         CHECK_OFFSET(idt_vectoring_info_field, 824);
 700         CHECK_OFFSET(idt_vectoring_error_code, 828);
 701         CHECK_OFFSET(vm_exit_instruction_len, 832);
 702         CHECK_OFFSET(vmx_instruction_info, 836);
 703         CHECK_OFFSET(guest_es_limit, 840);
 704         CHECK_OFFSET(guest_cs_limit, 844);
 705         CHECK_OFFSET(guest_ss_limit, 848);
 706         CHECK_OFFSET(guest_ds_limit, 852);
 707         CHECK_OFFSET(guest_fs_limit, 856);
 708         CHECK_OFFSET(guest_gs_limit, 860);
 709         CHECK_OFFSET(guest_ldtr_limit, 864);
 710         CHECK_OFFSET(guest_tr_limit, 868);
 711         CHECK_OFFSET(guest_gdtr_limit, 872);
 712         CHECK_OFFSET(guest_idtr_limit, 876);
 713         CHECK_OFFSET(guest_es_ar_bytes, 880);
 714         CHECK_OFFSET(guest_cs_ar_bytes, 884);
 715         CHECK_OFFSET(guest_ss_ar_bytes, 888);
 716         CHECK_OFFSET(guest_ds_ar_bytes, 892);
 717         CHECK_OFFSET(guest_fs_ar_bytes, 896);
 718         CHECK_OFFSET(guest_gs_ar_bytes, 900);
 719         CHECK_OFFSET(guest_ldtr_ar_bytes, 904);
 720         CHECK_OFFSET(guest_tr_ar_bytes, 908);
 721         CHECK_OFFSET(guest_interruptibility_info, 912);
 722         CHECK_OFFSET(guest_activity_state, 916);
 723         CHECK_OFFSET(guest_sysenter_cs, 920);
 724         CHECK_OFFSET(host_ia32_sysenter_cs, 924);
 725         CHECK_OFFSET(vmx_preemption_timer_value, 928);
 726         CHECK_OFFSET(virtual_processor_id, 960);
 727         CHECK_OFFSET(posted_intr_nv, 962);
 728         CHECK_OFFSET(guest_es_selector, 964);
 729         CHECK_OFFSET(guest_cs_selector, 966);
 730         CHECK_OFFSET(guest_ss_selector, 968);
 731         CHECK_OFFSET(guest_ds_selector, 970);
 732         CHECK_OFFSET(guest_fs_selector, 972);
 733         CHECK_OFFSET(guest_gs_selector, 974);
 734         CHECK_OFFSET(guest_ldtr_selector, 976);
 735         CHECK_OFFSET(guest_tr_selector, 978);
 736         CHECK_OFFSET(guest_intr_status, 980);
 737         CHECK_OFFSET(host_es_selector, 982);
 738         CHECK_OFFSET(host_cs_selector, 984);
 739         CHECK_OFFSET(host_ss_selector, 986);
 740         CHECK_OFFSET(host_ds_selector, 988);
 741         CHECK_OFFSET(host_fs_selector, 990);
 742         CHECK_OFFSET(host_gs_selector, 992);
 743         CHECK_OFFSET(host_tr_selector, 994);
 744         CHECK_OFFSET(guest_pml_index, 996);
 745 }
 746
 747 /*
 748  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 749  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 750  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 751  *
 752  * IMPORTANT: Changing this value will break save/restore compatibility with
 753  * older kvm releases.
 754  */
 755 #define VMCS12_REVISION 0x11e57ed0
 756
 757 /*
 758  * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 759  * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 760  * current implementation, 4K are reserved to avoid future complications.
 761  */
 762 #define VMCS12_SIZE 0x1000
 763
 764 /*
 765  * VMCS12_MAX_FIELD_INDEX is the highest index value used in any
 766  * supported VMCS12 field encoding.
 767  */
 768 #define VMCS12_MAX_FIELD_INDEX 0x17
 769
 770 struct nested_vmx_msrs {
 771         /*
 772          * We only store the "true" versions of the VMX capability MSRs. We
 773          * generate the "non-true" versions by setting the must-be-1 bits
 774          * according to the SDM.
 775          */
 776         u32 procbased_ctls_low;
 777         u32 procbased_ctls_high;
 778         u32 secondary_ctls_low;
 779         u32 secondary_ctls_high;
 780         u32 pinbased_ctls_low;
 781         u32 pinbased_ctls_high;
 782         u32 exit_ctls_low;
 783         u32 exit_ctls_high;
 784         u32 entry_ctls_low;
 785         u32 entry_ctls_high;
 786         u32 misc_low;
 787         u32 misc_high;
 788         u32 ept_caps;
 789         u32 vpid_caps;
 790         u64 basic;
 791         u64 cr0_fixed0;
 792         u64 cr0_fixed1;
 793         u64 cr4_fixed0;
 794         u64 cr4_fixed1;
 795         u64 vmcs_enum;
 796         u64 vmfunc_controls;
 797 };
 798
 799 /*
 800  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 801  * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 802  */
 803 struct nested_vmx {
 804         /* Has the level1 guest done vmxon? */
 805         bool vmxon;
 806         gpa_t vmxon_ptr;
 807         bool pml_full;
 808
 809         /* The guest-physical address of the current VMCS L1 keeps for L2 */
 810         gpa_t current_vmptr;
 811         /*
 812          * Cache of the guest's VMCS, existing outside of guest memory.
 813          * Loaded from guest memory during VMPTRLD. Flushed to guest
 814          * memory during VMCLEAR and VMPTRLD.
 815          */
 816         struct vmcs12 *cached_vmcs12;
 817         /*
 818          * Cache of the guest's shadow VMCS, existing outside of guest
 819          * memory. Loaded from guest memory during VM entry. Flushed
 820          * to guest memory during VM exit.
 821          */
 822         struct vmcs12 *cached_shadow_vmcs12;
 823         /*
 824          * Indicates if the shadow vmcs must be updated with the
 825          * data hold by vmcs12
 826          */
 827         bool sync_shadow_vmcs;
 828         bool dirty_vmcs12;
 829
 830         bool change_vmcs01_virtual_apic_mode;
 831
 832         /* L2 must run next, and mustn't decide to exit to L1. */
 833         bool nested_run_pending;
 834
 835         struct loaded_vmcs vmcs02;
 836
 837         /*
 838          * Guest pages referred to in the vmcs02 with host-physical
 839          * pointers, so we must keep them pinned while L2 runs.
 840          */
 841         struct page *apic_access_page;
 842         struct page *virtual_apic_page;
 843         struct page *pi_desc_page;
 844         struct pi_desc *pi_desc;
 845         bool pi_pending;
 846         u16 posted_intr_nv;
 847
 848         struct hrtimer preemption_timer;
 849         bool preemption_timer_expired;
 850
 851         /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
 852         u64 vmcs01_debugctl;
 853
 854         u16 vpid02;
 855         u16 last_vpid;
 856
 857         struct nested_vmx_msrs msrs;
 858
 859         /* SMM related state */
 860         struct {
 861                 /* in VMX operation on SMM entry? */
 862                 bool vmxon;
 863                 /* in guest mode on SMM entry? */
 864                 bool guest_mode;
 865         } smm;
 866 };
 867
 868 #define POSTED_INTR_ON  0
 869 #define POSTED_INTR_SN  1
 870
 871 /* Posted-Interrupt Descriptor */
 872 struct pi_desc {
 873         u32 pir[8];     /* Posted interrupt requested */
 874         union {
 875                 struct {
 876                                 /* bit 256 - Outstanding Notification */
 877                         u16     on      : 1,
 878                                 /* bit 257 - Suppress Notification */
 879                                 sn      : 1,
 880                                 /* bit 271:258 - Reserved */
 881                                 rsvd_1  : 14;
 882                                 /* bit 279:272 - Notification Vector */
 883                         u8      nv;
 884                                 /* bit 287:280 - Reserved */
 885                         u8      rsvd_2;
 886                                 /* bit 319:288 - Notification Destination */
 887                         u32     ndst;
 888                 };
 889                 u64 control;
 890         };
 891         u32 rsvd[6];
 892 } __aligned(64);
 893
 894 static bool pi_test_and_set_on(struct pi_desc *pi_desc)
 895 {
 896         return test_and_set_bit(POSTED_INTR_ON,
 897                         (unsigned long *)&pi_desc->control);
 898 }
 899
 900 static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
 901 {
 902         return test_and_clear_bit(POSTED_INTR_ON,
 903                         (unsigned long *)&pi_desc->control);
 904 }
 905
 906 static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
 907 {
 908         return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
 909 }
 910
 911 static inline void pi_clear_sn(struct pi_desc *pi_desc)
 912 {
 913         return clear_bit(POSTED_INTR_SN,
 914                         (unsigned long *)&pi_desc->control);
 915 }
 916
 917 static inline void pi_set_sn(struct pi_desc *pi_desc)
 918 {
 919         return set_bit(POSTED_INTR_SN,
 920                         (unsigned long *)&pi_desc->control);
 921 }
 922
 923 static inline void pi_clear_on(struct pi_desc *pi_desc)
 924 {
 925         clear_bit(POSTED_INTR_ON,
 926                   (unsigned long *)&pi_desc->control);
 927 }
 928
 929 static inline int pi_test_on(struct pi_desc *pi_desc)
 930 {
 931         return test_bit(POSTED_INTR_ON,
 932                         (unsigned long *)&pi_desc->control);
 933 }
 934
 935 static inline int pi_test_sn(struct pi_desc *pi_desc)
 936 {
 937         return test_bit(POSTED_INTR_SN,
 938                         (unsigned long *)&pi_desc->control);
 939 }
 940
 941 struct vmx_msrs {
 942         unsigned int            nr;
 943         struct vmx_msr_entry    val[NR_AUTOLOAD_MSRS];
 944 };
 945
 946 struct vcpu_vmx {
 947         struct kvm_vcpu       vcpu;
 948         unsigned long         host_rsp;
 949         u8                    fail;
 950         u8                    msr_bitmap_mode;
 951         u32                   exit_intr_info;
 952         u32                   idt_vectoring_info;
 953         ulong                 rflags;
 954         struct shared_msr_entry *guest_msrs;
 955         int                   nmsrs;
 956         int                   save_nmsrs;
 957         unsigned long         host_idt_base;
 958 #ifdef CONFIG_X86_64
 959         u64                   msr_host_kernel_gs_base;
 960         u64                   msr_guest_kernel_gs_base;
 961 #endif
 962
 963         u64                   arch_capabilities;
 964         u64                   spec_ctrl;
 965
 966         u32 vm_entry_controls_shadow;
 967         u32 vm_exit_controls_shadow;
 968         u32 secondary_exec_control;
 969
 970         /*
 971          * loaded_vmcs points to the VMCS currently used in this vcpu. For a
 972          * non-nested (L1) guest, it always points to vmcs01. For a nested
 973          * guest (L2), it points to a different VMCS.  loaded_cpu_state points
 974          * to the VMCS whose state is loaded into the CPU registers that only
 975          * need to be switched when transitioning to/from the kernel; a NULL
 976          * value indicates that host state is loaded.
 977          */
 978         struct loaded_vmcs    vmcs01;
 979         struct loaded_vmcs   *loaded_vmcs;
 980         struct loaded_vmcs   *loaded_cpu_state;
 981         bool                  __launched; /* temporary, used in vmx_vcpu_run */
 982         struct msr_autoload {
 983                 struct vmx_msrs guest;
 984                 struct vmx_msrs host;
 985         } msr_autoload;
 986
 987         struct {
 988                 int vm86_active;
 989                 ulong save_rflags;
 990                 struct kvm_segment segs[8];
 991         } rmode;
 992         struct {
 993                 u32 bitmask; /* 4 bits per segment (1 bit per field) */
 994                 struct kvm_save_segment {
 995                         u16 selector;
 996                         unsigned long base;
 997                         u32 limit;
 998                         u32 ar;
 999                 } seg[8];
1000         } segment_cache;
1001         int vpid;
1002         bool emulation_required;
1003
1004         u32 exit_reason;
1005
1006         /* Posted interrupt descriptor */
1007         struct pi_desc pi_desc;
1008
1009         /* Support for a guest hypervisor (nested VMX) */
1010         struct nested_vmx nested;
1011
1012         /* Dynamic PLE window. */
1013         int ple_window;
1014         bool ple_window_dirty;
1015
1016         /* Support for PML */
1017 #define PML_ENTITY_NUM          512
1018         struct page *pml_pg;
1019
1020         /* apic deadline value in host tsc */
1021         u64 hv_deadline_tsc;
1022
1023         u64 current_tsc_ratio;
1024
1025         u32 host_pkru;
1026
1027         unsigned long host_debugctlmsr;
1028
1029         /*
1030          * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
1031          * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
1032          * in msr_ia32_feature_control_valid_bits.
1033          */
1034         u64 msr_ia32_feature_control;
1035         u64 msr_ia32_feature_control_valid_bits;
1036         u64 ept_pointer;
1037 };
1038
1039 enum segment_cache_field {
1040         SEG_FIELD_SEL = 0,
1041         SEG_FIELD_BASE = 1,
1042         SEG_FIELD_LIMIT = 2,
1043         SEG_FIELD_AR = 3,
1044
1045         SEG_FIELD_NR = 4
1046 };
1047
1048 static inline struct kvm_vmx *to_kvm_vmx(struct kvm *kvm)
1049 {
1050         return container_of(kvm, struct kvm_vmx, kvm);
1051 }
1052
1053 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
1054 {
1055         return container_of(vcpu, struct vcpu_vmx, vcpu);
1056 }
1057
1058 static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
1059 {
1060         return &(to_vmx(vcpu)->pi_desc);
1061 }
1062
1063 #define ROL16(val, n) ((u16)(((u16)(val) << (n)) | ((u16)(val) >> (16 - (n)))))
1064 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
1065 #define FIELD(number, name)     [ROL16(number, 6)] = VMCS12_OFFSET(name)
1066 #define FIELD64(number, name)                                           \
1067         FIELD(number, name),                                            \
1068         [ROL16(number##_HIGH, 6)] = VMCS12_OFFSET(name) + sizeof(u32)
1069
1070
1071 static u16 shadow_read_only_fields[] = {
1072 #define SHADOW_FIELD_RO(x) x,
1073 #include "vmx_shadow_fields.h"
1074 };
1075 static int max_shadow_read_only_fields =
1076         ARRAY_SIZE(shadow_read_only_fields);
1077
1078 static u16 shadow_read_write_fields[] = {
1079 #define SHADOW_FIELD_RW(x) x,
1080 #include "vmx_shadow_fields.h"
1081 };
1082 static int max_shadow_read_write_fields =
1083         ARRAY_SIZE(shadow_read_write_fields);
1084
1085 static const unsigned short vmcs_field_to_offset_table[] = {
1086         FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
1087         FIELD(POSTED_INTR_NV, posted_intr_nv),
1088         FIELD(GUEST_ES_SELECTOR, guest_es_selector),
1089         FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
1090         FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
1091         FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
1092         FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
1093         FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
1094         FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
1095         FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
1096         FIELD(GUEST_INTR_STATUS, guest_intr_status),
1097         FIELD(GUEST_PML_INDEX, guest_pml_index),
1098         FIELD(HOST_ES_SELECTOR, host_es_selector),
1099         FIELD(HOST_CS_SELECTOR, host_cs_selector),
1100         FIELD(HOST_SS_SELECTOR, host_ss_selector),
1101         FIELD(HOST_DS_SELECTOR, host_ds_selector),
1102         FIELD(HOST_FS_SELECTOR, host_fs_selector),
1103         FIELD(HOST_GS_SELECTOR, host_gs_selector),
1104         FIELD(HOST_TR_SELECTOR, host_tr_selector),
1105         FIELD64(IO_BITMAP_A, io_bitmap_a),
1106         FIELD64(IO_BITMAP_B, io_bitmap_b),
1107         FIELD64(MSR_BITMAP, msr_bitmap),
1108         FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
1109         FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
1110         FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
1111         FIELD64(PML_ADDRESS, pml_address),
1112         FIELD64(TSC_OFFSET, tsc_offset),
1113         FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
1114         FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
1115         FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
1116         FIELD64(VM_FUNCTION_CONTROL, vm_function_control),
1117         FIELD64(EPT_POINTER, ept_pointer),
1118         FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
1119         FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
1120         FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
1121         FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
1122         FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
1123         FIELD64(VMREAD_BITMAP, vmread_bitmap),
1124         FIELD64(VMWRITE_BITMAP, vmwrite_bitmap),
1125         FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
1126         FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
1127         FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
1128         FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
1129         FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
1130         FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
1131         FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
1132         FIELD64(GUEST_PDPTR0, guest_pdptr0),
1133         FIELD64(GUEST_PDPTR1, guest_pdptr1),
1134         FIELD64(GUEST_PDPTR2, guest_pdptr2),
1135         FIELD64(GUEST_PDPTR3, guest_pdptr3),
1136         FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
1137         FIELD64(HOST_IA32_PAT, host_ia32_pat),
1138         FIELD64(HOST_IA32_EFER, host_ia32_efer),
1139         FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
1140         FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
1141         FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
1142         FIELD(EXCEPTION_BITMAP, exception_bitmap),
1143         FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
1144         FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
1145         FIELD(CR3_TARGET_COUNT, cr3_target_count),
1146         FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
1147         FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
1148         FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
1149         FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
1150         FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
1151         FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
1152         FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
1153         FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
1154         FIELD(TPR_THRESHOLD, tpr_threshold),
1155         FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
1156         FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
1157         FIELD(VM_EXIT_REASON, vm_exit_reason),
1158         FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
1159         FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
1160         FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
1161         FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
1162         FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
1163         FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
1164         FIELD(GUEST_ES_LIMIT, guest_es_limit),
1165         FIELD(GUEST_CS_LIMIT, guest_cs_limit),
1166         FIELD(GUEST_SS_LIMIT, guest_ss_limit),
1167         FIELD(GUEST_DS_LIMIT, guest_ds_limit),
1168         FIELD(GUEST_FS_LIMIT, guest_fs_limit),
1169         FIELD(GUEST_GS_LIMIT, guest_gs_limit),
1170         FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
1171         FIELD(GUEST_TR_LIMIT, guest_tr_limit),
1172         FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
1173         FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
1174         FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
1175         FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
1176         FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
1177         FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
1178         FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
1179         FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
1180         FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
1181         FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
1182         FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
1183         FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
1184         FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
1185         FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
1186         FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
1187         FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
1188         FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
1189         FIELD(CR0_READ_SHADOW, cr0_read_shadow),
1190         FIELD(CR4_READ_SHADOW, cr4_read_shadow),
1191         FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
1192         FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
1193         FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
1194         FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
1195         FIELD(EXIT_QUALIFICATION, exit_qualification),
1196         FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
1197         FIELD(GUEST_CR0, guest_cr0),
1198         FIELD(GUEST_CR3, guest_cr3),
1199         FIELD(GUEST_CR4, guest_cr4),
1200         FIELD(GUEST_ES_BASE, guest_es_base),
1201         FIELD(GUEST_CS_BASE, guest_cs_base),
1202         FIELD(GUEST_SS_BASE, guest_ss_base),
1203         FIELD(GUEST_DS_BASE, guest_ds_base),
1204         FIELD(GUEST_FS_BASE, guest_fs_base),
1205         FIELD(GUEST_GS_BASE, guest_gs_base),
1206         FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
1207         FIELD(GUEST_TR_BASE, guest_tr_base),
1208         FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
1209         FIELD(GUEST_IDTR_BASE, guest_idtr_base),
1210         FIELD(GUEST_DR7, guest_dr7),
1211         FIELD(GUEST_RSP, guest_rsp),
1212         FIELD(GUEST_RIP, guest_rip),
1213         FIELD(GUEST_RFLAGS, guest_rflags),
1214         FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
1215         FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
1216         FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
1217         FIELD(HOST_CR0, host_cr0),
1218         FIELD(HOST_CR3, host_cr3),
1219         FIELD(HOST_CR4, host_cr4),
1220         FIELD(HOST_FS_BASE, host_fs_base),
1221         FIELD(HOST_GS_BASE, host_gs_base),
1222         FIELD(HOST_TR_BASE, host_tr_base),
1223         FIELD(HOST_GDTR_BASE, host_gdtr_base),
1224         FIELD(HOST_IDTR_BASE, host_idtr_base),
1225         FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
1226         FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
1227         FIELD(HOST_RSP, host_rsp),
1228         FIELD(HOST_RIP, host_rip),
1229 };
1230
1231 static inline short vmcs_field_to_offset(unsigned long field)
1232 {
1233         const size_t size = ARRAY_SIZE(vmcs_field_to_offset_table);
1234         unsigned short offset;
1235         unsigned index;
1236
1237         if (field >> 15)
1238                 return -ENOENT;
1239
1240         index = ROL16(field, 6);
1241         if (index >= size)
1242                 return -ENOENT;
1243
1244         index = array_index_nospec(index, size);
1245         offset = vmcs_field_to_offset_table[index];
1246         if (offset == 0)
1247                 return -ENOENT;
1248         return offset;
1249 }
1250
1251 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
1252 {
1253         return to_vmx(vcpu)->nested.cached_vmcs12;
1254 }
1255
1256 static inline struct vmcs12 *get_shadow_vmcs12(struct kvm_vcpu *vcpu)
1257 {
1258         return to_vmx(vcpu)->nested.cached_shadow_vmcs12;
1259 }
1260
1261 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu);
1262 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
1263 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa);
1264 static bool vmx_xsaves_supported(void);
1265 static void vmx_set_segment(struct kvm_vcpu *vcpu,
1266                             struct kvm_segment *var, int seg);
1267 static void vmx_get_segment(struct kvm_vcpu *vcpu,
1268                             struct kvm_segment *var, int seg);
1269 static bool guest_state_valid(struct kvm_vcpu *vcpu);
1270 static u32 vmx_segment_access_rights(struct kvm_segment *var);
1271 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
1272 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
1273 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
1274 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
1275                                             u16 error_code);
1276 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
1277 static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
1278                                                           u32 msr, int type);
1279
1280 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
1281 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
1282 /*
1283  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
1284  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
1285  */
1286 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
1287
1288 /*
1289  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
1290  * can find which vCPU should be waken up.
1291  */
1292 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
1293 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
1294
1295 enum {
1296         VMX_VMREAD_BITMAP,
1297         VMX_VMWRITE_BITMAP,
1298         VMX_BITMAP_NR
1299 };
1300
1301 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
1302
1303 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
1304 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
1305
1306 static bool cpu_has_load_ia32_efer;
1307 static bool cpu_has_load_perf_global_ctrl;
1308
1309 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
1310 static DEFINE_SPINLOCK(vmx_vpid_lock);
1311
1312 static struct vmcs_config {
1313         int size;
1314         int order;
1315         u32 basic_cap;
1316         u32 revision_id;
1317         u32 pin_based_exec_ctrl;
1318         u32 cpu_based_exec_ctrl;
1319         u32 cpu_based_2nd_exec_ctrl;
1320         u32 vmexit_ctrl;
1321         u32 vmentry_ctrl;
1322         struct nested_vmx_msrs nested;
1323 } vmcs_config;
1324
1325 static struct vmx_capability {
1326         u32 ept;
1327         u32 vpid;
1328 } vmx_capability;
1329
1330 #define VMX_SEGMENT_FIELD(seg)                                  \
1331         [VCPU_SREG_##seg] = {                                   \
1332                 .selector = GUEST_##seg##_SELECTOR,             \
1333                 .base = GUEST_##seg##_BASE,                     \
1334                 .limit = GUEST_##seg##_LIMIT,                   \
1335                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
1336         }
1337
1338 static const struct kvm_vmx_segment_field {
1339         unsigned selector;
1340         unsigned base;
1341         unsigned limit;
1342         unsigned ar_bytes;
1343 } kvm_vmx_segment_fields[] = {
1344         VMX_SEGMENT_FIELD(CS),
1345         VMX_SEGMENT_FIELD(DS),
1346         VMX_SEGMENT_FIELD(ES),
1347         VMX_SEGMENT_FIELD(FS),
1348         VMX_SEGMENT_FIELD(GS),
1349         VMX_SEGMENT_FIELD(SS),
1350         VMX_SEGMENT_FIELD(TR),
1351         VMX_SEGMENT_FIELD(LDTR),
1352 };
1353
1354 static u64 host_efer;
1355
1356 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
1357
1358 /*
1359  * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
1360  * away by decrementing the array size.
1361  */
1362 static const u32 vmx_msr_index[] = {
1363 #ifdef CONFIG_X86_64
1364         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
1365 #endif
1366         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
1367 };
1368
1369 DEFINE_STATIC_KEY_FALSE(enable_evmcs);
1370
1371 #define current_evmcs ((struct hv_enlightened_vmcs *)this_cpu_read(current_vmcs))
1372
1373 #define KVM_EVMCS_VERSION 1
1374
1375 #if IS_ENABLED(CONFIG_HYPERV)
1376 static bool __read_mostly enlightened_vmcs = true;
1377 module_param(enlightened_vmcs, bool, 0444);
1378
1379 static inline void evmcs_write64(unsigned long field, u64 value)
1380 {
1381         u16 clean_field;
1382         int offset = get_evmcs_offset(field, &clean_field);
1383
1384         if (offset < 0)
1385                 return;
1386
1387         *(u64 *)((char *)current_evmcs + offset) = value;
1388
1389         current_evmcs->hv_clean_fields &= ~clean_field;
1390 }
1391
1392 static inline void evmcs_write32(unsigned long field, u32 value)
1393 {
1394         u16 clean_field;
1395         int offset = get_evmcs_offset(field, &clean_field);
1396
1397         if (offset < 0)
1398                 return;
1399
1400         *(u32 *)((char *)current_evmcs + offset) = value;
1401         current_evmcs->hv_clean_fields &= ~clean_field;
1402 }
1403
1404 static inline void evmcs_write16(unsigned long field, u16 value)
1405 {
1406         u16 clean_field;
1407         int offset = get_evmcs_offset(field, &clean_field);
1408
1409         if (offset < 0)
1410                 return;
1411
1412         *(u16 *)((char *)current_evmcs + offset) = value;
1413         current_evmcs->hv_clean_fields &= ~clean_field;
1414 }
1415
1416 static inline u64 evmcs_read64(unsigned long field)
1417 {
1418         int offset = get_evmcs_offset(field, NULL);
1419
1420         if (offset < 0)
1421                 return 0;
1422
1423         return *(u64 *)((char *)current_evmcs + offset);
1424 }
1425
1426 static inline u32 evmcs_read32(unsigned long field)
1427 {
1428         int offset = get_evmcs_offset(field, NULL);
1429
1430         if (offset < 0)
1431                 return 0;
1432
1433         return *(u32 *)((char *)current_evmcs + offset);
1434 }
1435
1436 static inline u16 evmcs_read16(unsigned long field)
1437 {
1438         int offset = get_evmcs_offset(field, NULL);
1439
1440         if (offset < 0)
1441                 return 0;
1442
1443         return *(u16 *)((char *)current_evmcs + offset);
1444 }
1445
1446 static inline void evmcs_touch_msr_bitmap(void)
1447 {
1448         if (unlikely(!current_evmcs))
1449                 return;
1450
1451         if (current_evmcs->hv_enlightenments_control.msr_bitmap)
1452                 current_evmcs->hv_clean_fields &=
1453                         ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
1454 }
1455
1456 static void evmcs_load(u64 phys_addr)
1457 {
1458         struct hv_vp_assist_page *vp_ap =
1459                 hv_get_vp_assist_page(smp_processor_id());
1460
1461         vp_ap->current_nested_vmcs = phys_addr;
1462         vp_ap->enlighten_vmentry = 1;
1463 }
1464
1465 static void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf)
1466 {
1467         /*
1468          * Enlightened VMCSv1 doesn't support these:
1469          *
1470          *      POSTED_INTR_NV                  = 0x00000002,
1471          *      GUEST_INTR_STATUS               = 0x00000810,
1472          *      APIC_ACCESS_ADDR                = 0x00002014,
1473          *      POSTED_INTR_DESC_ADDR           = 0x00002016,
1474          *      EOI_EXIT_BITMAP0                = 0x0000201c,
1475          *      EOI_EXIT_BITMAP1                = 0x0000201e,
1476          *      EOI_EXIT_BITMAP2                = 0x00002020,
1477          *      EOI_EXIT_BITMAP3                = 0x00002022,
1478          */
1479         vmcs_conf->pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
1480         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1481                 ~SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1482         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1483                 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1484         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1485                 ~SECONDARY_EXEC_APIC_REGISTER_VIRT;
1486
1487         /*
1488          *      GUEST_PML_INDEX                 = 0x00000812,
1489          *      PML_ADDRESS                     = 0x0000200e,
1490          */
1491         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_ENABLE_PML;
1492
1493         /*      VM_FUNCTION_CONTROL             = 0x00002018, */
1494         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_ENABLE_VMFUNC;
1495
1496         /*
1497          *      EPTP_LIST_ADDRESS               = 0x00002024,
1498          *      VMREAD_BITMAP                   = 0x00002026,
1499          *      VMWRITE_BITMAP                  = 0x00002028,
1500          */
1501         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_SHADOW_VMCS;
1502
1503         /*
1504          *      TSC_MULTIPLIER                  = 0x00002032,
1505          */
1506         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_TSC_SCALING;
1507
1508         /*
1509          *      PLE_GAP                         = 0x00004020,
1510          *      PLE_WINDOW                      = 0x00004022,
1511          */
1512         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1513
1514         /*
1515          *      VMX_PREEMPTION_TIMER_VALUE      = 0x0000482E,
1516          */
1517         vmcs_conf->pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
1518
1519         /*
1520          *      GUEST_IA32_PERF_GLOBAL_CTRL     = 0x00002808,
1521          *      HOST_IA32_PERF_GLOBAL_CTRL      = 0x00002c04,
1522          */
1523         vmcs_conf->vmexit_ctrl &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
1524         vmcs_conf->vmentry_ctrl &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
1525
1526         /*
1527          * Currently unsupported in KVM:
1528          *      GUEST_IA32_RTIT_CTL             = 0x00002814,
1529          */
1530 }
1531
1532 /* check_ept_pointer() should be under protection of ept_pointer_lock. */
1533 static void check_ept_pointer_match(struct kvm *kvm)
1534 {
1535         struct kvm_vcpu *vcpu;
1536         u64 tmp_eptp = INVALID_PAGE;
1537         int i;
1538
1539         kvm_for_each_vcpu(i, vcpu, kvm) {
1540                 if (!VALID_PAGE(tmp_eptp)) {
1541                         tmp_eptp = to_vmx(vcpu)->ept_pointer;
1542                 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
1543                         to_kvm_vmx(kvm)->ept_pointers_match
1544                                 = EPT_POINTERS_MISMATCH;
1545                         return;
1546                 }
1547         }
1548
1549         to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
1550 }
1551
1552 static int vmx_hv_remote_flush_tlb(struct kvm *kvm)
1553 {
1554         int ret;
1555
1556         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
1557
1558         if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
1559                 check_ept_pointer_match(kvm);
1560
1561         if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
1562                 ret = -ENOTSUPP;
1563                 goto out;
1564         }
1565
1566         ret = hyperv_flush_guest_mapping(
1567                         to_vmx(kvm_get_vcpu(kvm, 0))->ept_pointer);
1568
1569 out:
1570         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
1571         return ret;
1572 }
1573 #else /* !IS_ENABLED(CONFIG_HYPERV) */
1574 static inline void evmcs_write64(unsigned long field, u64 value) {}
1575 static inline void evmcs_write32(unsigned long field, u32 value) {}
1576 static inline void evmcs_write16(unsigned long field, u16 value) {}
1577 static inline u64 evmcs_read64(unsigned long field) { return 0; }
1578 static inline u32 evmcs_read32(unsigned long field) { return 0; }
1579 static inline u16 evmcs_read16(unsigned long field) { return 0; }
1580 static inline void evmcs_load(u64 phys_addr) {}
1581 static inline void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) {}
1582 static inline void evmcs_touch_msr_bitmap(void) {}
1583 #endif /* IS_ENABLED(CONFIG_HYPERV) */
1584
1585 static inline bool is_exception_n(u32 intr_info, u8 vector)
1586 {
1587         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1588                              INTR_INFO_VALID_MASK)) ==
1589                 (INTR_TYPE_HARD_EXCEPTION | vector | INTR_INFO_VALID_MASK);
1590 }
1591
1592 static inline bool is_debug(u32 intr_info)
1593 {
1594         return is_exception_n(intr_info, DB_VECTOR);
1595 }
1596
1597 static inline bool is_breakpoint(u32 intr_info)
1598 {
1599         return is_exception_n(intr_info, BP_VECTOR);
1600 }
1601
1602 static inline bool is_page_fault(u32 intr_info)
1603 {
1604         return is_exception_n(intr_info, PF_VECTOR);
1605 }
1606
1607 static inline bool is_no_device(u32 intr_info)
1608 {
1609         return is_exception_n(intr_info, NM_VECTOR);
1610 }
1611
1612 static inline bool is_invalid_opcode(u32 intr_info)
1613 {
1614         return is_exception_n(intr_info, UD_VECTOR);
1615 }
1616
1617 static inline bool is_gp_fault(u32 intr_info)
1618 {
1619         return is_exception_n(intr_info, GP_VECTOR);
1620 }
1621
1622 static inline bool is_external_interrupt(u32 intr_info)
1623 {
1624         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1625                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1626 }
1627
1628 static inline bool is_machine_check(u32 intr_info)
1629 {
1630         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1631                              INTR_INFO_VALID_MASK)) ==
1632                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
1633 }
1634
1635 /* Undocumented: icebp/int1 */
1636 static inline bool is_icebp(u32 intr_info)
1637 {
1638         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1639                 == (INTR_TYPE_PRIV_SW_EXCEPTION | INTR_INFO_VALID_MASK);
1640 }
1641
1642 static inline bool cpu_has_vmx_msr_bitmap(void)
1643 {
1644         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
1645 }
1646
1647 static inline bool cpu_has_vmx_tpr_shadow(void)
1648 {
1649         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
1650 }
1651
1652 static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
1653 {
1654         return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
1655 }
1656
1657 static inline bool cpu_has_secondary_exec_ctrls(void)
1658 {
1659         return vmcs_config.cpu_based_exec_ctrl &
1660                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1661 }
1662
1663 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
1664 {
1665         return vmcs_config.cpu_based_2nd_exec_ctrl &
1666                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1667 }
1668
1669 static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
1670 {
1671         return vmcs_config.cpu_based_2nd_exec_ctrl &
1672                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
1673 }
1674
1675 static inline bool cpu_has_vmx_apic_register_virt(void)
1676 {
1677         return vmcs_config.cpu_based_2nd_exec_ctrl &
1678                 SECONDARY_EXEC_APIC_REGISTER_VIRT;
1679 }
1680
1681 static inline bool cpu_has_vmx_virtual_intr_delivery(void)
1682 {
1683         return vmcs_config.cpu_based_2nd_exec_ctrl &
1684                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1685 }
1686
1687 /*
1688  * Comment's format: document - errata name - stepping - processor name.
1689  * Refer from
1690  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
1691  */
1692 static u32 vmx_preemption_cpu_tfms[] = {
1693 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
1694 0x000206E6,
1695 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
1696 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
1697 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
1698 0x00020652,
1699 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
1700 0x00020655,
1701 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
1702 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
1703 /*
1704  * 320767.pdf - AAP86  - B1 -
1705  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
1706  */
1707 0x000106E5,
1708 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
1709 0x000106A0,
1710 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
1711 0x000106A1,
1712 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
1713 0x000106A4,
1714  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
1715  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
1716  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
1717 0x000106A5,
1718 };
1719
1720 static inline bool cpu_has_broken_vmx_preemption_timer(void)
1721 {
1722         u32 eax = cpuid_eax(0x00000001), i;
1723
1724         /* Clear the reserved bits */
1725         eax &= ~(0x3U << 14 | 0xfU << 28);
1726         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
1727                 if (eax == vmx_preemption_cpu_tfms[i])
1728                         return true;
1729
1730         return false;
1731 }
1732
1733 static inline bool cpu_has_vmx_preemption_timer(void)
1734 {
1735         return vmcs_config.pin_based_exec_ctrl &
1736                 PIN_BASED_VMX_PREEMPTION_TIMER;
1737 }
1738
1739 static inline bool cpu_has_vmx_posted_intr(void)
1740 {
1741         return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
1742                 vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
1743 }
1744
1745 static inline bool cpu_has_vmx_apicv(void)
1746 {
1747         return cpu_has_vmx_apic_register_virt() &&
1748                 cpu_has_vmx_virtual_intr_delivery() &&
1749                 cpu_has_vmx_posted_intr();
1750 }
1751
1752 static inline bool cpu_has_vmx_flexpriority(void)
1753 {
1754         return cpu_has_vmx_tpr_shadow() &&
1755                 cpu_has_vmx_virtualize_apic_accesses();
1756 }
1757
1758 static inline bool cpu_has_vmx_ept_execute_only(void)
1759 {
1760         return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
1761 }
1762
1763 static inline bool cpu_has_vmx_ept_2m_page(void)
1764 {
1765         return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1766 }
1767
1768 static inline bool cpu_has_vmx_ept_1g_page(void)
1769 {
1770         return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1771 }
1772
1773 static inline bool cpu_has_vmx_ept_4levels(void)
1774 {
1775         return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
1776 }
1777
1778 static inline bool cpu_has_vmx_ept_mt_wb(void)
1779 {
1780         return vmx_capability.ept & VMX_EPTP_WB_BIT;
1781 }
1782
1783 static inline bool cpu_has_vmx_ept_5levels(void)
1784 {
1785         return vmx_capability.ept & VMX_EPT_PAGE_WALK_5_BIT;
1786 }
1787
1788 static inline bool cpu_has_vmx_ept_ad_bits(void)
1789 {
1790         return vmx_capability.ept & VMX_EPT_AD_BIT;
1791 }
1792
1793 static inline bool cpu_has_vmx_invept_context(void)
1794 {
1795         return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
1796 }
1797
1798 static inline bool cpu_has_vmx_invept_global(void)
1799 {
1800         return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
1801 }
1802
1803 static inline bool cpu_has_vmx_invvpid_individual_addr(void)
1804 {
1805         return vmx_capability.vpid & VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT;
1806 }
1807
1808 static inline bool cpu_has_vmx_invvpid_single(void)
1809 {
1810         return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
1811 }
1812
1813 static inline bool cpu_has_vmx_invvpid_global(void)
1814 {
1815         return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
1816 }
1817
1818 static inline bool cpu_has_vmx_invvpid(void)
1819 {
1820         return vmx_capability.vpid & VMX_VPID_INVVPID_BIT;
1821 }
1822
1823 static inline bool cpu_has_vmx_ept(void)
1824 {
1825         return vmcs_config.cpu_based_2nd_exec_ctrl &
1826                 SECONDARY_EXEC_ENABLE_EPT;
1827 }
1828
1829 static inline bool cpu_has_vmx_unrestricted_guest(void)
1830 {
1831         return vmcs_config.cpu_based_2nd_exec_ctrl &
1832                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
1833 }
1834
1835 static inline bool cpu_has_vmx_ple(void)
1836 {
1837         return vmcs_config.cpu_based_2nd_exec_ctrl &
1838                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1839 }
1840
1841 static inline bool cpu_has_vmx_basic_inout(void)
1842 {
1843         return  (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT);
1844 }
1845
1846 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
1847 {
1848         return flexpriority_enabled && lapic_in_kernel(vcpu);
1849 }
1850
1851 static inline bool cpu_has_vmx_vpid(void)
1852 {
1853         return vmcs_config.cpu_based_2nd_exec_ctrl &
1854                 SECONDARY_EXEC_ENABLE_VPID;
1855 }
1856
1857 static inline bool cpu_has_vmx_rdtscp(void)
1858 {
1859         return vmcs_config.cpu_based_2nd_exec_ctrl &
1860                 SECONDARY_EXEC_RDTSCP;
1861 }
1862
1863 static inline bool cpu_has_vmx_invpcid(void)
1864 {
1865         return vmcs_config.cpu_based_2nd_exec_ctrl &
1866                 SECONDARY_EXEC_ENABLE_INVPCID;
1867 }
1868
1869 static inline bool cpu_has_virtual_nmis(void)
1870 {
1871         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
1872 }
1873
1874 static inline bool cpu_has_vmx_wbinvd_exit(void)
1875 {
1876         return vmcs_config.cpu_based_2nd_exec_ctrl &
1877                 SECONDARY_EXEC_WBINVD_EXITING;
1878 }
1879
1880 static inline bool cpu_has_vmx_shadow_vmcs(void)
1881 {
1882         u64 vmx_msr;
1883         rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
1884         /* check if the cpu supports writing r/o exit information fields */
1885         if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
1886                 return false;
1887
1888         return vmcs_config.cpu_based_2nd_exec_ctrl &
1889                 SECONDARY_EXEC_SHADOW_VMCS;
1890 }
1891
1892 static inline bool cpu_has_vmx_pml(void)
1893 {
1894         return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
1895 }
1896
1897 static inline bool cpu_has_vmx_tsc_scaling(void)
1898 {
1899         return vmcs_config.cpu_based_2nd_exec_ctrl &
1900                 SECONDARY_EXEC_TSC_SCALING;
1901 }
1902
1903 static inline bool cpu_has_vmx_vmfunc(void)
1904 {
1905         return vmcs_config.cpu_based_2nd_exec_ctrl &
1906                 SECONDARY_EXEC_ENABLE_VMFUNC;
1907 }
1908
1909 static bool vmx_umip_emulated(void)
1910 {
1911         return vmcs_config.cpu_based_2nd_exec_ctrl &
1912                 SECONDARY_EXEC_DESC;
1913 }
1914
1915 static inline bool report_flexpriority(void)
1916 {
1917         return flexpriority_enabled;
1918 }
1919
1920 static inline unsigned nested_cpu_vmx_misc_cr3_count(struct kvm_vcpu *vcpu)
1921 {
1922         return vmx_misc_cr3_count(to_vmx(vcpu)->nested.msrs.misc_low);
1923 }
1924
1925 /*
1926  * Do the virtual VMX capability MSRs specify that L1 can use VMWRITE
1927  * to modify any valid field of the VMCS, or are the VM-exit
1928  * information fields read-only?
1929  */
1930 static inline bool nested_cpu_has_vmwrite_any_field(struct kvm_vcpu *vcpu)
1931 {
1932         return to_vmx(vcpu)->nested.msrs.misc_low &
1933                 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS;
1934 }
1935
1936 static inline bool nested_cpu_has_zero_length_injection(struct kvm_vcpu *vcpu)
1937 {
1938         return to_vmx(vcpu)->nested.msrs.misc_low & VMX_MISC_ZERO_LEN_INS;
1939 }
1940
1941 static inline bool nested_cpu_supports_monitor_trap_flag(struct kvm_vcpu *vcpu)
1942 {
1943         return to_vmx(vcpu)->nested.msrs.procbased_ctls_high &
1944                         CPU_BASED_MONITOR_TRAP_FLAG;
1945 }
1946
1947 static inline bool nested_cpu_has_vmx_shadow_vmcs(struct kvm_vcpu *vcpu)
1948 {
1949         return to_vmx(vcpu)->nested.msrs.secondary_ctls_high &
1950                 SECONDARY_EXEC_SHADOW_VMCS;
1951 }
1952
1953 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
1954 {
1955         return vmcs12->cpu_based_vm_exec_control & bit;
1956 }
1957
1958 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
1959 {
1960         return (vmcs12->cpu_based_vm_exec_control &
1961                         CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
1962                 (vmcs12->secondary_vm_exec_control & bit);
1963 }
1964
1965 static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
1966 {
1967         return vmcs12->pin_based_vm_exec_control &
1968                 PIN_BASED_VMX_PREEMPTION_TIMER;
1969 }
1970
1971 static inline bool nested_cpu_has_nmi_exiting(struct vmcs12 *vmcs12)
1972 {
1973         return vmcs12->pin_based_vm_exec_control & PIN_BASED_NMI_EXITING;
1974 }
1975
1976 static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
1977 {
1978         return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
1979 }
1980
1981 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
1982 {
1983         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
1984 }
1985
1986 static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
1987 {
1988         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
1989 }
1990
1991 static inline bool nested_cpu_has_pml(struct vmcs12 *vmcs12)
1992 {
1993         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML);
1994 }
1995
1996 static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
1997 {
1998         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
1999 }
2000
2001 static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
2002 {
2003         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
2004 }
2005
2006 static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
2007 {
2008         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
2009 }
2010
2011 static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
2012 {
2013         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2014 }
2015
2016 static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
2017 {
2018         return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
2019 }
2020
2021 static inline bool nested_cpu_has_vmfunc(struct vmcs12 *vmcs12)
2022 {
2023         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VMFUNC);
2024 }
2025
2026 static inline bool nested_cpu_has_eptp_switching(struct vmcs12 *vmcs12)
2027 {
2028         return nested_cpu_has_vmfunc(vmcs12) &&
2029                 (vmcs12->vm_function_control &
2030                  VMX_VMFUNC_EPTP_SWITCHING);
2031 }
2032
2033 static inline bool nested_cpu_has_shadow_vmcs(struct vmcs12 *vmcs12)
2034 {
2035         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_SHADOW_VMCS);
2036 }
2037
2038 static inline bool is_nmi(u32 intr_info)
2039 {
2040         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
2041                 == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
2042 }
2043
2044 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
2045                               u32 exit_intr_info,
2046                               unsigned long exit_qualification);
2047 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
2048                         struct vmcs12 *vmcs12,
2049                         u32 reason, unsigned long qualification);
2050
2051 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
2052 {
2053         int i;
2054
2055         for (i = 0; i < vmx->nmsrs; ++i)
2056                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
2057                         return i;
2058         return -1;
2059 }
2060
2061 static inline void __invvpid(int ext, u16 vpid, gva_t gva)
2062 {
2063     struct {
2064         u64 vpid : 16;
2065         u64 rsvd : 48;
2066         u64 gva;
2067     } operand = { vpid, 0, gva };
2068     bool error;
2069
2070     asm volatile (__ex(ASM_VMX_INVVPID) CC_SET(na)
2071                   : CC_OUT(na) (error) : "a"(&operand), "c"(ext)
2072                   : "memory");
2073     BUG_ON(error);
2074 }
2075
2076 static inline void __invept(int ext, u64 eptp, gpa_t gpa)
2077 {
2078         struct {
2079                 u64 eptp, gpa;
2080         } operand = {eptp, gpa};
2081         bool error;
2082
2083         asm volatile (__ex(ASM_VMX_INVEPT) CC_SET(na)
2084                       : CC_OUT(na) (error) : "a" (&operand), "c" (ext)
2085                       : "memory");
2086         BUG_ON(error);
2087 }
2088
2089 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
2090 {
2091         int i;
2092
2093         i = __find_msr_index(vmx, msr);
2094         if (i >= 0)
2095                 return &vmx->guest_msrs[i];
2096         return NULL;
2097 }
2098
2099 static void vmcs_clear(struct vmcs *vmcs)
2100 {
2101         u64 phys_addr = __pa(vmcs);
2102         bool error;
2103
2104         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) CC_SET(na)
2105                       : CC_OUT(na) (error) : "a"(&phys_addr), "m"(phys_addr)
2106                       : "memory");
2107         if (unlikely(error))
2108                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
2109                        vmcs, phys_addr);
2110 }
2111
2112 static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
2113 {
2114         vmcs_clear(loaded_vmcs->vmcs);
2115         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
2116                 vmcs_clear(loaded_vmcs->shadow_vmcs);
2117         loaded_vmcs->cpu = -1;
2118         loaded_vmcs->launched = 0;
2119 }
2120
2121 static void vmcs_load(struct vmcs *vmcs)
2122 {
2123         u64 phys_addr = __pa(vmcs);
2124         bool error;
2125
2126         if (static_branch_unlikely(&enable_evmcs))
2127                 return evmcs_load(phys_addr);
2128
2129         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) CC_SET(na)
2130                       : CC_OUT(na) (error) : "a"(&phys_addr), "m"(phys_addr)
2131                       : "memory");
2132         if (unlikely(error))
2133                 printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
2134                        vmcs, phys_addr);
2135 }
2136
2137 #ifdef CONFIG_KEXEC_CORE
2138 /*
2139  * This bitmap is used to indicate whether the vmclear
2140  * operation is enabled on all cpus. All disabled by
2141  * default.
2142  */
2143 static cpumask_t crash_vmclear_enabled_bitmap = CPU_MASK_NONE;
2144
2145 static inline void crash_enable_local_vmclear(int cpu)
2146 {
2147         cpumask_set_cpu(cpu, &crash_vmclear_enabled_bitmap);
2148 }
2149
2150 static inline void crash_disable_local_vmclear(int cpu)
2151 {
2152         cpumask_clear_cpu(cpu, &crash_vmclear_enabled_bitmap);
2153 }
2154
2155 static inline int crash_local_vmclear_enabled(int cpu)
2156 {
2157         return cpumask_test_cpu(cpu, &crash_vmclear_enabled_bitmap);
2158 }
2159
2160 static void crash_vmclear_local_loaded_vmcss(void)
2161 {
2162         int cpu = raw_smp_processor_id();
2163         struct loaded_vmcs *v;
2164
2165         if (!crash_local_vmclear_enabled(cpu))
2166                 return;
2167
2168         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
2169                             loaded_vmcss_on_cpu_link)
2170                 vmcs_clear(v->vmcs);
2171 }
2172 #else
2173 static inline void crash_enable_local_vmclear(int cpu) { }
2174 static inline void crash_disable_local_vmclear(int cpu) { }
2175 #endif /* CONFIG_KEXEC_CORE */
2176
2177 static void __loaded_vmcs_clear(void *arg)
2178 {
2179         struct loaded_vmcs *loaded_vmcs = arg;
2180         int cpu = raw_smp_processor_id();
2181
2182         if (loaded_vmcs->cpu != cpu)
2183                 return; /* vcpu migration can race with cpu offline */
2184         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
2185                 per_cpu(current_vmcs, cpu) = NULL;
2186         crash_disable_local_vmclear(cpu);
2187         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
2188
2189         /*
2190          * we should ensure updating loaded_vmcs->loaded_vmcss_on_cpu_link
2191          * is before setting loaded_vmcs->vcpu to -1 which is done in
2192          * loaded_vmcs_init. Otherwise, other cpu can see vcpu = -1 fist
2193          * then adds the vmcs into percpu list before it is deleted.
2194          */
2195         smp_wmb();
2196
2197         loaded_vmcs_init(loaded_vmcs);
2198         crash_enable_local_vmclear(cpu);
2199 }
2200
2201 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
2202 {
2203         int cpu = loaded_vmcs->cpu;
2204
2205         if (cpu != -1)
2206                 smp_call_function_single(cpu,
2207                          __loaded_vmcs_clear, loaded_vmcs, 1);
2208 }
2209
2210 static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr)
2211 {
2212         if (vpid == 0)
2213                 return true;
2214
2215         if (cpu_has_vmx_invvpid_individual_addr()) {
2216                 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
2217                 return true;
2218         }
2219
2220         return false;
2221 }
2222
2223 static inline void vpid_sync_vcpu_single(int vpid)
2224 {
2225         if (vpid == 0)
2226                 return;
2227
2228         if (cpu_has_vmx_invvpid_single())
2229                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
2230 }
2231
2232 static inline void vpid_sync_vcpu_global(void)
2233 {
2234         if (cpu_has_vmx_invvpid_global())
2235                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
2236 }
2237
2238 static inline void vpid_sync_context(int vpid)
2239 {
2240         if (cpu_has_vmx_invvpid_single())
2241                 vpid_sync_vcpu_single(vpid);
2242         else
2243                 vpid_sync_vcpu_global();
2244 }
2245
2246 static inline void ept_sync_global(void)
2247 {
2248         __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
2249 }
2250
2251 static inline void ept_sync_context(u64 eptp)
2252 {
2253         if (cpu_has_vmx_invept_context())
2254                 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
2255         else
2256                 ept_sync_global();
2257 }
2258
2259 static __always_inline void vmcs_check16(unsigned long field)
2260 {
2261         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
2262                          "16-bit accessor invalid for 64-bit field");
2263         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2264                          "16-bit accessor invalid for 64-bit high field");
2265         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2266                          "16-bit accessor invalid for 32-bit high field");
2267         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2268                          "16-bit accessor invalid for natural width field");
2269 }
2270
2271 static __always_inline void vmcs_check32(unsigned long field)
2272 {
2273         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2274                          "32-bit accessor invalid for 16-bit field");
2275         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2276                          "32-bit accessor invalid for natural width field");
2277 }
2278
2279 static __always_inline void vmcs_check64(unsigned long field)
2280 {
2281         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2282                          "64-bit accessor invalid for 16-bit field");
2283         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2284                          "64-bit accessor invalid for 64-bit high field");
2285         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2286                          "64-bit accessor invalid for 32-bit field");
2287         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2288                          "64-bit accessor invalid for natural width field");
2289 }
2290
2291 static __always_inline void vmcs_checkl(unsigned long field)
2292 {
2293         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2294                          "Natural width accessor invalid for 16-bit field");
2295         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
2296                          "Natural width accessor invalid for 64-bit field");
2297         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2298                          "Natural width accessor invalid for 64-bit high field");
2299         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2300                          "Natural width accessor invalid for 32-bit field");
2301 }
2302
2303 static __always_inline unsigned long __vmcs_readl(unsigned long field)
2304 {
2305         unsigned long value;
2306
2307         asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
2308                       : "=a"(value) : "d"(field) : "cc");
2309         return value;
2310 }
2311
2312 static __always_inline u16 vmcs_read16(unsigned long field)
2313 {
2314         vmcs_check16(field);
2315         if (static_branch_unlikely(&enable_evmcs))
2316                 return evmcs_read16(field);
2317         return __vmcs_readl(field);
2318 }
2319
2320 static __always_inline u32 vmcs_read32(unsigned long field)
2321 {
2322         vmcs_check32(field);
2323         if (static_branch_unlikely(&enable_evmcs))
2324                 return evmcs_read32(field);
2325         return __vmcs_readl(field);
2326 }
2327
2328 static __always_inline u64 vmcs_read64(unsigned long field)
2329 {
2330         vmcs_check64(field);
2331         if (static_branch_unlikely(&enable_evmcs))
2332                 return evmcs_read64(field);
2333 #ifdef CONFIG_X86_64
2334         return __vmcs_readl(field);
2335 #else
2336         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
2337 #endif
2338 }
2339
2340 static __always_inline unsigned long vmcs_readl(unsigned long field)
2341 {
2342         vmcs_checkl(field);
2343         if (static_branch_unlikely(&enable_evmcs))
2344                 return evmcs_read64(field);
2345         return __vmcs_readl(field);
2346 }
2347
2348 static noinline void vmwrite_error(unsigned long field, unsigned long value)
2349 {
2350         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
2351                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
2352         dump_stack();
2353 }
2354
2355 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
2356 {
2357         bool error;
2358
2359         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) CC_SET(na)
2360                       : CC_OUT(na) (error) : "a"(value), "d"(field));
2361         if (unlikely(error))
2362                 vmwrite_error(field, value);
2363 }
2364
2365 static __always_inline void vmcs_write16(unsigned long field, u16 value)
2366 {
2367         vmcs_check16(field);
2368         if (static_branch_unlikely(&enable_evmcs))
2369                 return evmcs_write16(field, value);
2370
2371         __vmcs_writel(field, value);
2372 }
2373
2374 static __always_inline void vmcs_write32(unsigned long field, u32 value)
2375 {
2376         vmcs_check32(field);
2377         if (static_branch_unlikely(&enable_evmcs))
2378                 return evmcs_write32(field, value);
2379
2380         __vmcs_writel(field, value);
2381 }
2382
2383 static __always_inline void vmcs_write64(unsigned long field, u64 value)
2384 {
2385         vmcs_check64(field);
2386         if (static_branch_unlikely(&enable_evmcs))
2387                 return evmcs_write64(field, value);
2388
2389         __vmcs_writel(field, value);
2390 #ifndef CONFIG_X86_64
2391         asm volatile ("");
2392         __vmcs_writel(field+1, value >> 32);
2393 #endif
2394 }
2395
2396 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
2397 {
2398         vmcs_checkl(field);
2399         if (static_branch_unlikely(&enable_evmcs))
2400                 return evmcs_write64(field, value);
2401
2402         __vmcs_writel(field, value);
2403 }
2404
2405 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
2406 {
2407         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
2408                          "vmcs_clear_bits does not support 64-bit fields");
2409         if (static_branch_unlikely(&enable_evmcs))
2410                 return evmcs_write32(field, evmcs_read32(field) & ~mask);
2411
2412         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
2413 }
2414
2415 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
2416 {
2417         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
2418                          "vmcs_set_bits does not support 64-bit fields");
2419         if (static_branch_unlikely(&enable_evmcs))
2420                 return evmcs_write32(field, evmcs_read32(field) | mask);
2421
2422         __vmcs_writel(field, __vmcs_readl(field) | mask);
2423 }
2424
2425 static inline void vm_entry_controls_reset_shadow(struct vcpu_vmx *vmx)
2426 {
2427         vmx->vm_entry_controls_shadow = vmcs_read32(VM_ENTRY_CONTROLS);
2428 }
2429
2430 static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
2431 {
2432         vmcs_write32(VM_ENTRY_CONTROLS, val);
2433         vmx->vm_entry_controls_shadow = val;
2434 }
2435
2436 static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
2437 {
2438         if (vmx->vm_entry_controls_shadow != val)
2439                 vm_entry_controls_init(vmx, val);
2440 }
2441
2442 static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
2443 {
2444         return vmx->vm_entry_controls_shadow;
2445 }
2446
2447
2448 static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
2449 {
2450         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
2451 }
2452
2453 static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
2454 {
2455         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
2456 }
2457
2458 static inline void vm_exit_controls_reset_shadow(struct vcpu_vmx *vmx)
2459 {
2460         vmx->vm_exit_controls_shadow = vmcs_read32(VM_EXIT_CONTROLS);
2461 }
2462
2463 static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
2464 {
2465         vmcs_write32(VM_EXIT_CONTROLS, val);
2466         vmx->vm_exit_controls_shadow = val;
2467 }
2468
2469 static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
2470 {
2471         if (vmx->vm_exit_controls_shadow != val)
2472                 vm_exit_controls_init(vmx, val);
2473 }
2474
2475 static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
2476 {
2477         return vmx->vm_exit_controls_shadow;
2478 }
2479
2480
2481 static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
2482 {
2483         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
2484 }
2485
2486 static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
2487 {
2488         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
2489 }
2490
2491 static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
2492 {
2493         vmx->segment_cache.bitmask = 0;
2494 }
2495
2496 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
2497                                        unsigned field)
2498 {
2499         bool ret;
2500         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
2501
2502         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
2503                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
2504                 vmx->segment_cache.bitmask = 0;
2505         }
2506         ret = vmx->segment_cache.bitmask & mask;
2507         vmx->segment_cache.bitmask |= mask;
2508         return ret;
2509 }
2510
2511 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
2512 {
2513         u16 *p = &vmx->segment_cache.seg[seg].selector;
2514
2515         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
2516                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
2517         return *p;
2518 }
2519
2520 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
2521 {
2522         ulong *p = &vmx->segment_cache.seg[seg].base;
2523
2524         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
2525                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
2526         return *p;
2527 }
2528
2529 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
2530 {
2531         u32 *p = &vmx->segment_cache.seg[seg].limit;
2532
2533         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
2534                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
2535         return *p;
2536 }
2537
2538 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
2539 {
2540         u32 *p = &vmx->segment_cache.seg[seg].ar;
2541
2542         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
2543                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
2544         return *p;
2545 }
2546
2547 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
2548 {
2549         u32 eb;
2550
2551         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
2552              (1u << DB_VECTOR) | (1u << AC_VECTOR);
2553         /*
2554          * Guest access to VMware backdoor ports could legitimately
2555          * trigger #GP because of TSS I/O permission bitmap.
2556          * We intercept those #GP and allow access to them anyway
2557          * as VMware does.
2558          */
2559         if (enable_vmware_backdoor)
2560                 eb |= (1u << GP_VECTOR);
2561         if ((vcpu->guest_debug &
2562              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
2563             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
2564                 eb |= 1u << BP_VECTOR;
2565         if (to_vmx(vcpu)->rmode.vm86_active)
2566                 eb = ~0;
2567         if (enable_ept)
2568                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
2569
2570         /* When we are running a nested L2 guest and L1 specified for it a
2571          * certain exception bitmap, we must trap the same exceptions and pass
2572          * them to L1. When running L2, we will only handle the exceptions
2573          * specified above if L1 did not want them.
2574          */
2575         if (is_guest_mode(vcpu))
2576                 eb |= get_vmcs12(vcpu)->exception_bitmap;
2577
2578         vmcs_write32(EXCEPTION_BITMAP, eb);
2579 }
2580
2581 /*
2582  * Check if MSR is intercepted for currently loaded MSR bitmap.
2583  */
2584 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
2585 {
2586         unsigned long *msr_bitmap;
2587         int f = sizeof(unsigned long);
2588
2589         if (!cpu_has_vmx_msr_bitmap())
2590                 return true;
2591
2592         msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
2593
2594         if (msr <= 0x1fff) {
2595                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
2596         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2597                 msr &= 0x1fff;
2598                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
2599         }
2600
2601         return true;
2602 }
2603
2604 /*
2605  * Check if MSR is intercepted for L01 MSR bitmap.
2606  */
2607 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
2608 {
2609         unsigned long *msr_bitmap;
2610         int f = sizeof(unsigned long);
2611
2612         if (!cpu_has_vmx_msr_bitmap())
2613                 return true;
2614
2615         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
2616
2617         if (msr <= 0x1fff) {
2618                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
2619         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2620                 msr &= 0x1fff;
2621                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
2622         }
2623
2624         return true;
2625 }
2626
2627 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2628                 unsigned long entry, unsigned long exit)
2629 {
2630         vm_entry_controls_clearbit(vmx, entry);
2631         vm_exit_controls_clearbit(vmx, exit);
2632 }
2633
2634 static int find_msr(struct vmx_msrs *m, unsigned int msr)
2635 {
2636         unsigned int i;
2637
2638         for (i = 0; i < m->nr; ++i) {
2639                 if (m->val[i].index == msr)
2640                         return i;
2641         }
2642         return -ENOENT;
2643 }
2644
2645 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
2646 {
2647         int i;
2648         struct msr_autoload *m = &vmx->msr_autoload;
2649
2650         switch (msr) {
2651         case MSR_EFER:
2652                 if (cpu_has_load_ia32_efer) {
2653                         clear_atomic_switch_msr_special(vmx,
2654                                         VM_ENTRY_LOAD_IA32_EFER,
2655                                         VM_EXIT_LOAD_IA32_EFER);
2656                         return;
2657                 }
2658                 break;
2659         case MSR_CORE_PERF_GLOBAL_CTRL:
2660                 if (cpu_has_load_perf_global_ctrl) {
2661                         clear_atomic_switch_msr_special(vmx,
2662                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2663                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
2664                         return;
2665                 }
2666                 break;
2667         }
2668         i = find_msr(&m->guest, msr);
2669         if (i < 0)
2670                 goto skip_guest;
2671         --m->guest.nr;
2672         m->guest.val[i] = m->guest.val[m->guest.nr];
2673         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
2674
2675 skip_guest:
2676         i = find_msr(&m->host, msr);
2677         if (i < 0)
2678                 return;
2679
2680         --m->host.nr;
2681         m->host.val[i] = m->host.val[m->host.nr];
2682         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
2683 }
2684
2685 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2686                 unsigned long entry, unsigned long exit,
2687                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
2688                 u64 guest_val, u64 host_val)
2689 {
2690         vmcs_write64(guest_val_vmcs, guest_val);
2691         vmcs_write64(host_val_vmcs, host_val);
2692         vm_entry_controls_setbit(vmx, entry);
2693         vm_exit_controls_setbit(vmx, exit);
2694 }
2695
2696 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
2697                                   u64 guest_val, u64 host_val, bool entry_only)
2698 {
2699         int i, j = 0;
2700         struct msr_autoload *m = &vmx->msr_autoload;
2701
2702         switch (msr) {
2703         case MSR_EFER:
2704                 if (cpu_has_load_ia32_efer) {
2705                         add_atomic_switch_msr_special(vmx,
2706                                         VM_ENTRY_LOAD_IA32_EFER,
2707                                         VM_EXIT_LOAD_IA32_EFER,
2708                                         GUEST_IA32_EFER,
2709                                         HOST_IA32_EFER,
2710                                         guest_val, host_val);
2711                         return;
2712                 }
2713                 break;
2714         case MSR_CORE_PERF_GLOBAL_CTRL:
2715                 if (cpu_has_load_perf_global_ctrl) {
2716                         add_atomic_switch_msr_special(vmx,
2717                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2718                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
2719                                         GUEST_IA32_PERF_GLOBAL_CTRL,
2720                                         HOST_IA32_PERF_GLOBAL_CTRL,
2721                                         guest_val, host_val);
2722                         return;
2723                 }
2724                 break;
2725         case MSR_IA32_PEBS_ENABLE:
2726                 /* PEBS needs a quiescent period after being disabled (to write
2727                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
2728                  * provide that period, so a CPU could write host's record into
2729                  * guest's memory.
2730                  */
2731                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
2732         }
2733
2734         i = find_msr(&m->guest, msr);
2735         if (!entry_only)
2736                 j = find_msr(&m->host, msr);
2737
2738         if (i == NR_AUTOLOAD_MSRS || j == NR_AUTOLOAD_MSRS) {
2739                 printk_once(KERN_WARNING "Not enough msr switch entries. "
2740                                 "Can't add msr %x\n", msr);
2741                 return;
2742         }
2743         if (i < 0) {
2744                 i = m->guest.nr++;
2745                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
2746         }
2747         m->guest.val[i].index = msr;
2748         m->guest.val[i].value = guest_val;
2749
2750         if (entry_only)
2751                 return;
2752
2753         if (j < 0) {
2754                 j = m->host.nr++;
2755                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
2756         }
2757         m->host.val[j].index = msr;
2758         m->host.val[j].value = host_val;
2759 }
2760
2761 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
2762 {
2763         u64 guest_efer = vmx->vcpu.arch.efer;
2764         u64 ignore_bits = 0;
2765
2766         if (!enable_ept) {
2767                 /*
2768                  * NX is needed to handle CR0.WP=1, CR4.SMEP=1.  Testing
2769                  * host CPUID is more efficient than testing guest CPUID
2770                  * or CR4.  Host SMEP is anyway a requirement for guest SMEP.
2771                  */
2772                 if (boot_cpu_has(X86_FEATURE_SMEP))
2773                         guest_efer |= EFER_NX;
2774                 else if (!(guest_efer & EFER_NX))
2775                         ignore_bits |= EFER_NX;
2776         }
2777
2778         /*
2779          * LMA and LME handled by hardware; SCE meaningless outside long mode.
2780          */
2781         ignore_bits |= EFER_SCE;
2782 #ifdef CONFIG_X86_64
2783         ignore_bits |= EFER_LMA | EFER_LME;
2784         /* SCE is meaningful only in long mode on Intel */
2785         if (guest_efer & EFER_LMA)
2786                 ignore_bits &= ~(u64)EFER_SCE;
2787 #endif
2788
2789         clear_atomic_switch_msr(vmx, MSR_EFER);
2790
2791         /*
2792          * On EPT, we can't emulate NX, so we must switch EFER atomically.
2793          * On CPUs that support "load IA32_EFER", always switch EFER
2794          * atomically, since it's faster than switching it manually.
2795          */
2796         if (cpu_has_load_ia32_efer ||
2797             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
2798                 if (!(guest_efer & EFER_LMA))
2799                         guest_efer &= ~EFER_LME;
2800                 if (guest_efer != host_efer)
2801                         add_atomic_switch_msr(vmx, MSR_EFER,
2802                                               guest_efer, host_efer, false);
2803                 return false;
2804         } else {
2805                 guest_efer &= ~ignore_bits;
2806                 guest_efer |= host_efer & ignore_bits;
2807
2808                 vmx->guest_msrs[efer_offset].data = guest_efer;
2809                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
2810
2811                 return true;
2812         }
2813 }
2814
2815 #ifdef CONFIG_X86_32
2816 /*
2817  * On 32-bit kernels, VM exits still load the FS and GS bases from the
2818  * VMCS rather than the segment table.  KVM uses this helper to figure
2819  * out the current bases to poke them into the VMCS before entry.
2820  */
2821 static unsigned long segment_base(u16 selector)
2822 {
2823         struct desc_struct *table;
2824         unsigned long v;
2825
2826         if (!(selector & ~SEGMENT_RPL_MASK))
2827                 return 0;
2828
2829         table = get_current_gdt_ro();
2830
2831         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2832                 u16 ldt_selector = kvm_read_ldt();
2833
2834                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
2835                         return 0;
2836
2837                 table = (struct desc_struct *)segment_base(ldt_selector);
2838         }
2839         v = get_desc_base(&table[selector >> 3]);
2840         return v;
2841 }
2842 #endif
2843
2844 static void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
2845 {
2846         struct vcpu_vmx *vmx = to_vmx(vcpu);
2847         struct vmcs_host_state *host_state;
2848 #ifdef CONFIG_X86_64
2849         int cpu = raw_smp_processor_id();
2850 #endif
2851         unsigned long fs_base, gs_base;
2852         u16 fs_sel, gs_sel;
2853         int i;
2854
2855         if (vmx->loaded_cpu_state)
2856                 return;
2857
2858         vmx->loaded_cpu_state = vmx->loaded_vmcs;
2859         host_state = &vmx->loaded_cpu_state->host_state;
2860
2861         /*
2862          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
2863          * allow segment selectors with cpl > 0 or ti == 1.
2864          */
2865         host_state->ldt_sel = kvm_read_ldt();
2866
2867 #ifdef CONFIG_X86_64
2868         savesegment(ds, host_state->ds_sel);
2869         savesegment(es, host_state->es_sel);
2870
2871         gs_base = cpu_kernelmode_gs_base(cpu);
2872         if (likely(is_64bit_mm(current->mm))) {
2873                 save_fsgs_for_kvm();
2874                 fs_sel = current->thread.fsindex;
2875                 gs_sel = current->thread.gsindex;
2876                 fs_base = current->thread.fsbase;
2877                 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
2878         } else {
2879                 savesegment(fs, fs_sel);
2880                 savesegment(gs, gs_sel);
2881                 fs_base = read_msr(MSR_FS_BASE);
2882                 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
2883         }
2884
2885         if (is_long_mode(&vmx->vcpu))
2886                 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2887 #else
2888         savesegment(fs, fs_sel);
2889         savesegment(gs, gs_sel);
2890         fs_base = segment_base(fs_sel);
2891         gs_base = segment_base(gs_sel);
2892 #endif
2893
2894         if (unlikely(fs_sel != host_state->fs_sel)) {
2895                 if (!(fs_sel & 7))
2896                         vmcs_write16(HOST_FS_SELECTOR, fs_sel);
2897                 else
2898                         vmcs_write16(HOST_FS_SELECTOR, 0);
2899                 host_state->fs_sel = fs_sel;
2900         }
2901         if (unlikely(gs_sel != host_state->gs_sel)) {
2902                 if (!(gs_sel & 7))
2903                         vmcs_write16(HOST_GS_SELECTOR, gs_sel);
2904                 else
2905                         vmcs_write16(HOST_GS_SELECTOR, 0);
2906                 host_state->gs_sel = gs_sel;
2907         }
2908         if (unlikely(fs_base != host_state->fs_base)) {
2909                 vmcs_writel(HOST_FS_BASE, fs_base);
2910                 host_state->fs_base = fs_base;
2911         }
2912         if (unlikely(gs_base != host_state->gs_base)) {
2913                 vmcs_writel(HOST_GS_BASE, gs_base);
2914                 host_state->gs_base = gs_base;
2915         }
2916
2917         for (i = 0; i < vmx->save_nmsrs; ++i)
2918                 kvm_set_shared_msr(vmx->guest_msrs[i].index,
2919                                    vmx->guest_msrs[i].data,
2920                                    vmx->guest_msrs[i].mask);
2921 }
2922
2923 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
2924 {
2925         struct vmcs_host_state *host_state;
2926
2927         if (!vmx->loaded_cpu_state)
2928                 return;
2929
2930         WARN_ON_ONCE(vmx->loaded_cpu_state != vmx->loaded_vmcs);
2931         host_state = &vmx->loaded_cpu_state->host_state;
2932
2933         ++vmx->vcpu.stat.host_state_reload;
2934         vmx->loaded_cpu_state = NULL;
2935
2936 #ifdef CONFIG_X86_64
2937         if (is_long_mode(&vmx->vcpu))
2938                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2939 #endif
2940         if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
2941                 kvm_load_ldt(host_state->ldt_sel);
2942 #ifdef CONFIG_X86_64
2943                 load_gs_index(host_state->gs_sel);
2944 #else
2945                 loadsegment(gs, host_state->gs_sel);
2946 #endif
2947         }
2948         if (host_state->fs_sel & 7)
2949                 loadsegment(fs, host_state->fs_sel);
2950 #ifdef CONFIG_X86_64
2951         if (unlikely(host_state->ds_sel | host_state->es_sel)) {
2952                 loadsegment(ds, host_state->ds_sel);
2953                 loadsegment(es, host_state->es_sel);
2954         }
2955 #endif
2956         invalidate_tss_limit();
2957 #ifdef CONFIG_X86_64
2958         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
2959 #endif
2960         load_fixmap_gdt(raw_smp_processor_id());
2961 }
2962
2963 #ifdef CONFIG_X86_64
2964 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
2965 {
2966         if (is_long_mode(&vmx->vcpu)) {
2967                 preempt_disable();
2968                 if (vmx->loaded_cpu_state)
2969                         rdmsrl(MSR_KERNEL_GS_BASE,
2970                                vmx->msr_guest_kernel_gs_base);
2971                 preempt_enable();
2972         }
2973         return vmx->msr_guest_kernel_gs_base;
2974 }
2975
2976 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
2977 {
2978         if (is_long_mode(&vmx->vcpu)) {
2979                 preempt_disable();
2980                 if (vmx->loaded_cpu_state)
2981                         wrmsrl(MSR_KERNEL_GS_BASE, data);
2982                 preempt_enable();
2983         }
2984         vmx->msr_guest_kernel_gs_base = data;
2985 }
2986 #endif
2987
2988 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
2989 {
2990         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
2991         struct pi_desc old, new;
2992         unsigned int dest;
2993
2994         /*
2995          * In case of hot-plug or hot-unplug, we may have to undo
2996          * vmx_vcpu_pi_put even if there is no assigned device.  And we
2997          * always keep PI.NDST up to date for simplicity: it makes the
2998          * code easier, and CPU migration is not a fast path.
2999          */
3000         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
3001                 return;
3002
3003         /*
3004          * First handle the simple case where no cmpxchg is necessary; just
3005          * allow posting non-urgent interrupts.
3006          *
3007          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
3008          * PI.NDST: pi_post_block will do it for us and the wakeup_handler
3009          * expects the VCPU to be on the blocked_vcpu_list that matches
3010          * PI.NDST.
3011          */
3012         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR ||
3013             vcpu->cpu == cpu) {
3014                 pi_clear_sn(pi_desc);
3015                 return;
3016         }
3017
3018         /* The full case.  */
3019         do {
3020                 old.control = new.control = pi_desc->control;
3021
3022                 dest = cpu_physical_id(cpu);
3023
3024                 if (x2apic_enabled())
3025                         new.ndst = dest;
3026                 else
3027                         new.ndst = (dest << 8) & 0xFF00;
3028
3029                 new.sn = 0;
3030         } while (cmpxchg64(&pi_desc->control, old.control,
3031                            new.control) != old.control);
3032 }
3033
3034 static void decache_tsc_multiplier(struct vcpu_vmx *vmx)
3035 {
3036         vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
3037         vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
3038 }
3039
3040 /*
3041  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
3042  * vcpu mutex is already taken.
3043  */
3044 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3045 {
3046         struct vcpu_vmx *vmx = to_vmx(vcpu);
3047         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
3048
3049         if (!already_loaded) {
3050                 loaded_vmcs_clear(vmx->loaded_vmcs);
3051                 local_irq_disable();
3052                 crash_disable_local_vmclear(cpu);
3053
3054                 /*
3055                  * Read loaded_vmcs->cpu should be before fetching
3056                  * loaded_vmcs->loaded_vmcss_on_cpu_link.
3057                  * See the comments in __loaded_vmcs_clear().
3058                  */
3059                 smp_rmb();
3060
3061                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
3062                          &per_cpu(loaded_vmcss_on_cpu, cpu));
3063                 crash_enable_local_vmclear(cpu);
3064                 local_irq_enable();
3065         }
3066
3067         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
3068                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
3069                 vmcs_load(vmx->loaded_vmcs->vmcs);
3070                 indirect_branch_prediction_barrier();
3071         }
3072
3073         if (!already_loaded) {
3074                 void *gdt = get_current_gdt_ro();
3075                 unsigned long sysenter_esp;
3076
3077                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3078
3079                 /*
3080                  * Linux uses per-cpu TSS and GDT, so set these when switching
3081                  * processors.  See 22.2.4.
3082                  */
3083                 vmcs_writel(HOST_TR_BASE,
3084                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
3085                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
3086
3087                 /*
3088                  * VM exits change the host TR limit to 0x67 after a VM
3089                  * exit.  This is okay, since 0x67 covers everything except
3090                  * the IO bitmap and have have code to handle the IO bitmap
3091                  * being lost after a VM exit.
3092                  */
3093                 BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
3094
3095                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
3096                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
3097
3098                 vmx->loaded_vmcs->cpu = cpu;
3099         }
3100
3101         /* Setup TSC multiplier */
3102         if (kvm_has_tsc_control &&
3103             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
3104                 decache_tsc_multiplier(vmx);
3105
3106         vmx_vcpu_pi_load(vcpu, cpu);
3107         vmx->host_pkru = read_pkru();
3108         vmx->host_debugctlmsr = get_debugctlmsr();
3109 }
3110
3111 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
3112 {
3113         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
3114
3115         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
3116                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
3117                 !kvm_vcpu_apicv_active(vcpu))
3118                 return;
3119
3120         /* Set SN when the vCPU is preempted */
3121         if (vcpu->preempted)
3122                 pi_set_sn(pi_desc);
3123 }
3124
3125 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
3126 {
3127         vmx_vcpu_pi_put(vcpu);
3128
3129         vmx_prepare_switch_to_host(to_vmx(vcpu));
3130 }
3131
3132 static bool emulation_required(struct kvm_vcpu *vcpu)
3133 {
3134         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
3135 }
3136
3137 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
3138
3139 /*
3140  * Return the cr0 value that a nested guest would read. This is a combination
3141  * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
3142  * its hypervisor (cr0_read_shadow).
3143  */
3144 static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
3145 {
3146         return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
3147                 (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
3148 }
3149 static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
3150 {
3151         return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
3152                 (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
3153 }
3154
3155 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
3156 {
3157         unsigned long rflags, save_rflags;
3158
3159         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
3160                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
3161                 rflags = vmcs_readl(GUEST_RFLAGS);
3162                 if (to_vmx(vcpu)->rmode.vm86_active) {
3163                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
3164                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
3165                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
3166                 }
3167                 to_vmx(vcpu)->rflags = rflags;
3168         }
3169         return to_vmx(vcpu)->rflags;
3170 }
3171
3172 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
3173 {
3174         unsigned long old_rflags = vmx_get_rflags(vcpu);
3175
3176         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
3177         to_vmx(vcpu)->rflags = rflags;
3178         if (to_vmx(vcpu)->rmode.vm86_active) {
3179                 to_vmx(vcpu)->rmode.save_rflags = rflags;
3180                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
3181         }
3182         vmcs_writel(GUEST_RFLAGS, rflags);
3183
3184         if ((old_rflags ^ to_vmx(vcpu)->rflags) & X86_EFLAGS_VM)
3185                 to_vmx(vcpu)->emulation_required = emulation_required(vcpu);
3186 }
3187
3188 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
3189 {
3190         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3191         int ret = 0;
3192
3193         if (interruptibility & GUEST_INTR_STATE_STI)
3194                 ret |= KVM_X86_SHADOW_INT_STI;
3195         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
3196                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
3197
3198         return ret;
3199 }
3200
3201 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
3202 {
3203         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3204         u32 interruptibility = interruptibility_old;
3205
3206         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
3207
3208         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
3209                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
3210         else if (mask & KVM_X86_SHADOW_INT_STI)
3211                 interruptibility |= GUEST_INTR_STATE_STI;
3212
3213         if ((interruptibility != interruptibility_old))
3214                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
3215 }
3216
3217 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
3218 {
3219         unsigned long rip;
3220
3221         rip = kvm_rip_read(vcpu);
3222         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3223         kvm_rip_write(vcpu, rip);
3224
3225         /* skipping an emulated instruction also counts */
3226         vmx_set_interrupt_shadow(vcpu, 0);
3227 }
3228
3229 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
3230                                                unsigned long exit_qual)
3231 {
3232         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3233         unsigned int nr = vcpu->arch.exception.nr;
3234         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3235
3236         if (vcpu->arch.exception.has_error_code) {
3237                 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
3238                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3239         }
3240
3241         if (kvm_exception_is_soft(nr))
3242                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3243         else
3244                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3245
3246         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
3247             vmx_get_nmi_mask(vcpu))
3248                 intr_info |= INTR_INFO_UNBLOCK_NMI;
3249
3250         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
3251 }
3252
3253 /*
3254  * KVM wants to inject page-faults which it got to the guest. This function
3255  * checks whether in a nested guest, we need to inject them to L1 or L2.
3256  */
3257 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
3258 {
3259         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3260         unsigned int nr = vcpu->arch.exception.nr;
3261
3262         if (nr == PF_VECTOR) {
3263                 if (vcpu->arch.exception.nested_apf) {
3264                         *exit_qual = vcpu->arch.apf.nested_apf_token;
3265                         return 1;
3266                 }
3267                 /*
3268                  * FIXME: we must not write CR2 when L1 intercepts an L2 #PF exception.
3269                  * The fix is to add the ancillary datum (CR2 or DR6) to structs
3270                  * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6
3271                  * can be written only when inject_pending_event runs.  This should be
3272                  * conditional on a new capability---if the capability is disabled,
3273                  * kvm_multiple_exception would write the ancillary information to
3274                  * CR2 or DR6, for backwards ABI-compatibility.
3275                  */
3276                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
3277                                                     vcpu->arch.exception.error_code)) {
3278                         *exit_qual = vcpu->arch.cr2;
3279                         return 1;
3280                 }
3281         } else {
3282                 if (vmcs12->exception_bitmap & (1u << nr)) {
3283                         if (nr == DB_VECTOR)
3284                                 *exit_qual = vcpu->arch.dr6;
3285                         else
3286                                 *exit_qual = 0;
3287                         return 1;
3288                 }
3289         }
3290
3291         return 0;
3292 }
3293
3294 static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
3295 {
3296         /*
3297          * Ensure that we clear the HLT state in the VMCS.  We don't need to
3298          * explicitly skip the instruction because if the HLT state is set,
3299          * then the instruction is already executing and RIP has already been
3300          * advanced.
3301          */
3302         if (kvm_hlt_in_guest(vcpu->kvm) &&
3303                         vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
3304                 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
3305 }
3306
3307 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
3308 {
3309         struct vcpu_vmx *vmx = to_vmx(vcpu);
3310         unsigned nr = vcpu->arch.exception.nr;
3311         bool has_error_code = vcpu->arch.exception.has_error_code;
3312         u32 error_code = vcpu->arch.exception.error_code;
3313         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3314
3315         if (has_error_code) {
3316                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
3317                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3318         }
3319
3320         if (vmx->rmode.vm86_active) {
3321                 int inc_eip = 0;
3322                 if (kvm_exception_is_soft(nr))
3323                         inc_eip = vcpu->arch.event_exit_inst_len;
3324                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
3325                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3326                 return;
3327         }
3328
3329         WARN_ON_ONCE(vmx->emulation_required);
3330
3331         if (kvm_exception_is_soft(nr)) {
3332                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
3333                              vmx->vcpu.arch.event_exit_inst_len);
3334                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3335         } else
3336                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3337
3338         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
3339
3340         vmx_clear_hlt(vcpu);
3341 }
3342
3343 static bool vmx_rdtscp_supported(void)
3344 {
3345         return cpu_has_vmx_rdtscp();
3346 }
3347
3348 static bool vmx_invpcid_supported(void)
3349 {
3350         return cpu_has_vmx_invpcid();
3351 }
3352
3353 /*
3354  * Swap MSR entry in host/guest MSR entry array.
3355  */
3356 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
3357 {
3358         struct shared_msr_entry tmp;
3359
3360         tmp = vmx->guest_msrs[to];
3361         vmx->guest_msrs[to] = vmx->guest_msrs[from];
3362         vmx->guest_msrs[from] = tmp;
3363 }
3364
3365 /*
3366  * Set up the vmcs to automatically save and restore system
3367  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
3368  * mode, as fiddling with msrs is very expensive.
3369  */
3370 static void setup_msrs(struct vcpu_vmx *vmx)
3371 {
3372         int save_nmsrs, index;
3373
3374         save_nmsrs = 0;
3375 #ifdef CONFIG_X86_64
3376         if (is_long_mode(&vmx->vcpu)) {
3377                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
3378                 if (index >= 0)
3379                         move_msr_up(vmx, index, save_nmsrs++);
3380                 index = __find_msr_index(vmx, MSR_LSTAR);
3381                 if (index >= 0)
3382                         move_msr_up(vmx, index, save_nmsrs++);
3383                 index = __find_msr_index(vmx, MSR_CSTAR);
3384                 if (index >= 0)
3385                         move_msr_up(vmx, index, save_nmsrs++);
3386                 index = __find_msr_index(vmx, MSR_TSC_AUX);
3387                 if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
3388                         move_msr_up(vmx, index, save_nmsrs++);
3389                 /*
3390                  * MSR_STAR is only needed on long mode guests, and only
3391                  * if efer.sce is enabled.
3392                  */
3393                 index = __find_msr_index(vmx, MSR_STAR);
3394                 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
3395                         move_msr_up(vmx, index, save_nmsrs++);
3396         }
3397 #endif
3398         index = __find_msr_index(vmx, MSR_EFER);
3399         if (index >= 0 && update_transition_efer(vmx, index))
3400                 move_msr_up(vmx, index, save_nmsrs++);
3401
3402         vmx->save_nmsrs = save_nmsrs;
3403
3404         if (cpu_has_vmx_msr_bitmap())
3405                 vmx_update_msr_bitmap(&vmx->vcpu);
3406 }
3407
3408 static u64 vmx_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
3409 {
3410         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3411
3412         if (is_guest_mode(vcpu) &&
3413             (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING))
3414                 return vcpu->arch.tsc_offset - vmcs12->tsc_offset;
3415
3416         return vcpu->arch.tsc_offset;
3417 }
3418
3419 /*
3420  * writes 'offset' into guest's timestamp counter offset register
3421  */
3422 static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
3423 {
3424         if (is_guest_mode(vcpu)) {
3425                 /*
3426                  * We're here if L1 chose not to trap WRMSR to TSC. According
3427                  * to the spec, this should set L1's TSC; The offset that L1
3428                  * set for L2 remains unchanged, and still needs to be added
3429                  * to the newly set TSC to get L2's TSC.
3430                  */
3431                 struct vmcs12 *vmcs12;
3432                 /* recalculate vmcs02.TSC_OFFSET: */
3433                 vmcs12 = get_vmcs12(vcpu);
3434                 vmcs_write64(TSC_OFFSET, offset +
3435                         (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING) ?
3436                          vmcs12->tsc_offset : 0));
3437         } else {
3438                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
3439                                            vmcs_read64(TSC_OFFSET), offset);
3440                 vmcs_write64(TSC_OFFSET, offset);
3441         }
3442 }
3443
3444 /*
3445  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
3446  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
3447  * all guests if the "nested" module option is off, and can also be disabled
3448  * for a single guest by disabling its VMX cpuid bit.
3449  */
3450 static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
3451 {
3452         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
3453 }
3454
3455 /*
3456  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
3457  * returned for the various VMX controls MSRs when nested VMX is enabled.
3458  * The same values should also be used to verify that vmcs12 control fields are
3459  * valid during nested entry from L1 to L2.
3460  * Each of these control msrs has a low and high 32-bit half: A low bit is on
3461  * if the corresponding bit in the (32-bit) control field *must* be on, and a
3462  * bit in the high half is on if the corresponding bit in the control field
3463  * may be on. See also vmx_control_verify().
3464  */
3465 static void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, bool apicv)
3466 {
3467         if (!nested) {
3468                 memset(msrs, 0, sizeof(*msrs));
3469                 return;
3470         }
3471
3472         /*
3473          * Note that as a general rule, the high half of the MSRs (bits in
3474          * the control fields which may be 1) should be initialized by the
3475          * intersection of the underlying hardware's MSR (i.e., features which
3476          * can be supported) and the list of features we want to expose -
3477          * because they are known to be properly supported in our code.
3478          * Also, usually, the low half of the MSRs (bits which must be 1) can
3479          * be set to 0, meaning that L1 may turn off any of these bits. The
3480          * reason is that if one of these bits is necessary, it will appear
3481          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
3482          * fields of vmcs01 and vmcs02, will turn these bits off - and
3483          * nested_vmx_exit_reflected() will not pass related exits to L1.
3484          * These rules have exceptions below.
3485          */
3486
3487         /* pin-based controls */
3488         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
3489                 msrs->pinbased_ctls_low,
3490                 msrs->pinbased_ctls_high);
3491         msrs->pinbased_ctls_low |=
3492                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3493         msrs->pinbased_ctls_high &=
3494                 PIN_BASED_EXT_INTR_MASK |
3495                 PIN_BASED_NMI_EXITING |
3496                 PIN_BASED_VIRTUAL_NMIS |
3497                 (apicv ? PIN_BASED_POSTED_INTR : 0);
3498         msrs->pinbased_ctls_high |=
3499                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
3500                 PIN_BASED_VMX_PREEMPTION_TIMER;
3501
3502         /* exit controls */
3503         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
3504                 msrs->exit_ctls_low,
3505                 msrs->exit_ctls_high);
3506         msrs->exit_ctls_low =
3507                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
3508
3509         msrs->exit_ctls_high &=
3510 #ifdef CONFIG_X86_64
3511                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
3512 #endif
3513                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
3514         msrs->exit_ctls_high |=
3515                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
3516                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
3517                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
3518
3519         if (kvm_mpx_supported())
3520                 msrs->exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
3521
3522         /* We support free control of debug control saving. */
3523         msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
3524
3525         /* entry controls */
3526         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
3527                 msrs->entry_ctls_low,
3528                 msrs->entry_ctls_high);
3529         msrs->entry_ctls_low =
3530                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
3531         msrs->entry_ctls_high &=
3532 #ifdef CONFIG_X86_64
3533                 VM_ENTRY_IA32E_MODE |
3534 #endif
3535                 VM_ENTRY_LOAD_IA32_PAT;
3536         msrs->entry_ctls_high |=
3537                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
3538         if (kvm_mpx_supported())
3539                 msrs->entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
3540
3541         /* We support free control of debug control loading. */
3542         msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
3543
3544         /* cpu-based controls */
3545         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
3546                 msrs->procbased_ctls_low,
3547                 msrs->procbased_ctls_high);
3548         msrs->procbased_ctls_low =
3549                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3550         msrs->procbased_ctls_high &=
3551                 CPU_BASED_VIRTUAL_INTR_PENDING |
3552                 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
3553                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
3554                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
3555                 CPU_BASED_CR3_STORE_EXITING |
3556 #ifdef CONFIG_X86_64
3557                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
3558 #endif
3559                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
3560                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
3561                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
3562                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
3563                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
3564         /*
3565          * We can allow some features even when not supported by the
3566          * hardware. For example, L1 can specify an MSR bitmap - and we
3567          * can use it to avoid exits to L1 - even when L0 runs L2
3568          * without MSR bitmaps.
3569          */
3570         msrs->procbased_ctls_high |=
3571                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
3572                 CPU_BASED_USE_MSR_BITMAPS;
3573
3574         /* We support free control of CR3 access interception. */
3575         msrs->procbased_ctls_low &=
3576                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
3577
3578         /*
3579          * secondary cpu-based controls.  Do not include those that
3580          * depend on CPUID bits, they are added later by vmx_cpuid_update.
3581          */
3582         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
3583                 msrs->secondary_ctls_low,
3584                 msrs->secondary_ctls_high);
3585         msrs->secondary_ctls_low = 0;
3586         msrs->secondary_ctls_high &=
3587                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
3588                 SECONDARY_EXEC_DESC |
3589                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3590                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
3591                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
3592                 SECONDARY_EXEC_WBINVD_EXITING;
3593         /*
3594          * We can emulate "VMCS shadowing," even if the hardware
3595          * doesn't support it.
3596          */
3597         msrs->secondary_ctls_high |=
3598                 SECONDARY_EXEC_SHADOW_VMCS;
3599
3600         if (enable_ept) {
3601                 /* nested EPT: emulate EPT also to L1 */
3602                 msrs->secondary_ctls_high |=
3603                         SECONDARY_EXEC_ENABLE_EPT;
3604                 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
3605                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
3606                 if (cpu_has_vmx_ept_execute_only())
3607                         msrs->ept_caps |=
3608                                 VMX_EPT_EXECUTE_ONLY_BIT;
3609                 msrs->ept_caps &= vmx_capability.ept;
3610                 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
3611                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
3612                         VMX_EPT_1GB_PAGE_BIT;
3613                 if (enable_ept_ad_bits) {
3614                         msrs->secondary_ctls_high |=
3615                                 SECONDARY_EXEC_ENABLE_PML;
3616                         msrs->ept_caps |= VMX_EPT_AD_BIT;
3617                 }
3618         }
3619
3620         if (cpu_has_vmx_vmfunc()) {
3621                 msrs->secondary_ctls_high |=
3622                         SECONDARY_EXEC_ENABLE_VMFUNC;
3623                 /*
3624                  * Advertise EPTP switching unconditionally
3625                  * since we emulate it
3626                  */
3627                 if (enable_ept)
3628                         msrs->vmfunc_controls =
3629                                 VMX_VMFUNC_EPTP_SWITCHING;
3630         }
3631
3632         /*
3633          * Old versions of KVM use the single-context version without
3634          * checking for support, so declare that it is supported even
3635          * though it is treated as global context.  The alternative is
3636          * not failing the single-context invvpid, and it is worse.
3637          */
3638         if (enable_vpid) {