Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[muen/linux.git] / arch / x86 / hyperv / mmu.c
1 #define pr_fmt(fmt)  "Hyper-V: " fmt
2
3 #include <linux/hyperv.h>
4 #include <linux/log2.h>
5 #include <linux/slab.h>
6 #include <linux/types.h>
7
8 #include <asm/fpu/api.h>
9 #include <asm/mshyperv.h>
10 #include <asm/msr.h>
11 #include <asm/tlbflush.h>
12 #include <asm/tlb.h>
13
14 #define CREATE_TRACE_POINTS
15 #include <asm/trace/hyperv.h>
16
17 /* Each gva in gva_list encodes up to 4096 pages to flush */
18 #define HV_TLB_FLUSH_UNIT (4096 * PAGE_SIZE)
19
20 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
21                                       const struct flush_tlb_info *info);
22
23 /*
24  * Fills in gva_list starting from offset. Returns the number of items added.
25  */
26 static inline int fill_gva_list(u64 gva_list[], int offset,
27                                 unsigned long start, unsigned long end)
28 {
29         int gva_n = offset;
30         unsigned long cur = start, diff;
31
32         do {
33                 diff = end > cur ? end - cur : 0;
34
35                 gva_list[gva_n] = cur & PAGE_MASK;
36                 /*
37                  * Lower 12 bits encode the number of additional
38                  * pages to flush (in addition to the 'cur' page).
39                  */
40                 if (diff >= HV_TLB_FLUSH_UNIT)
41                         gva_list[gva_n] |= ~PAGE_MASK;
42                 else if (diff)
43                         gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT;
44
45                 cur += HV_TLB_FLUSH_UNIT;
46                 gva_n++;
47
48         } while (cur < end);
49
50         return gva_n - offset;
51 }
52
53 static void hyperv_flush_tlb_others(const struct cpumask *cpus,
54                                     const struct flush_tlb_info *info)
55 {
56         int cpu, vcpu, gva_n, max_gvas;
57         struct hv_tlb_flush **flush_pcpu;
58         struct hv_tlb_flush *flush;
59         u64 status = U64_MAX;
60         unsigned long flags;
61
62         trace_hyperv_mmu_flush_tlb_others(cpus, info);
63
64         if (!hv_hypercall_pg)
65                 goto do_native;
66
67         if (cpumask_empty(cpus))
68                 return;
69
70         local_irq_save(flags);
71
72         flush_pcpu = (struct hv_tlb_flush **)
73                      this_cpu_ptr(hyperv_pcpu_input_arg);
74
75         flush = *flush_pcpu;
76
77         if (unlikely(!flush)) {
78                 local_irq_restore(flags);
79                 goto do_native;
80         }
81
82         if (info->mm) {
83                 /*
84                  * AddressSpace argument must match the CR3 with PCID bits
85                  * stripped out.
86                  */
87                 flush->address_space = virt_to_phys(info->mm->pgd);
88                 flush->address_space &= CR3_ADDR_MASK;
89                 flush->flags = 0;
90         } else {
91                 flush->address_space = 0;
92                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
93         }
94
95         flush->processor_mask = 0;
96         if (cpumask_equal(cpus, cpu_present_mask)) {
97                 flush->flags |= HV_FLUSH_ALL_PROCESSORS;
98         } else {
99                 /*
100                  * From the supplied CPU set we need to figure out if we can get
101                  * away with cheaper HVCALL_FLUSH_VIRTUAL_ADDRESS_{LIST,SPACE}
102                  * hypercalls. This is possible when the highest VP number in
103                  * the set is < 64. As VP numbers are usually in ascending order
104                  * and match Linux CPU ids, here is an optimization: we check
105                  * the VP number for the highest bit in the supplied set first
106                  * so we can quickly find out if using *_EX hypercalls is a
107                  * must. We will also check all VP numbers when walking the
108                  * supplied CPU set to remain correct in all cases.
109                  */
110                 if (hv_cpu_number_to_vp_number(cpumask_last(cpus)) >= 64)
111                         goto do_ex_hypercall;
112
113                 for_each_cpu(cpu, cpus) {
114                         vcpu = hv_cpu_number_to_vp_number(cpu);
115                         if (vcpu == VP_INVAL) {
116                                 local_irq_restore(flags);
117                                 goto do_native;
118                         }
119
120                         if (vcpu >= 64)
121                                 goto do_ex_hypercall;
122
123                         __set_bit(vcpu, (unsigned long *)
124                                   &flush->processor_mask);
125                 }
126         }
127
128         /*
129          * We can flush not more than max_gvas with one hypercall. Flush the
130          * whole address space if we were asked to do more.
131          */
132         max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]);
133
134         if (info->end == TLB_FLUSH_ALL) {
135                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
136                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
137                                          flush, NULL);
138         } else if (info->end &&
139                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
140                 status = hv_do_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE,
141                                          flush, NULL);
142         } else {
143                 gva_n = fill_gva_list(flush->gva_list, 0,
144                                       info->start, info->end);
145                 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST,
146                                              gva_n, 0, flush, NULL);
147         }
148         goto check_status;
149
150 do_ex_hypercall:
151         status = hyperv_flush_tlb_others_ex(cpus, info);
152
153 check_status:
154         local_irq_restore(flags);
155
156         if (!(status & HV_HYPERCALL_RESULT_MASK))
157                 return;
158 do_native:
159         native_flush_tlb_others(cpus, info);
160 }
161
162 static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
163                                       const struct flush_tlb_info *info)
164 {
165         int nr_bank = 0, max_gvas, gva_n;
166         struct hv_tlb_flush_ex **flush_pcpu;
167         struct hv_tlb_flush_ex *flush;
168         u64 status;
169
170         if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED))
171                 return U64_MAX;
172
173         flush_pcpu = (struct hv_tlb_flush_ex **)
174                      this_cpu_ptr(hyperv_pcpu_input_arg);
175
176         flush = *flush_pcpu;
177
178         if (info->mm) {
179                 /*
180                  * AddressSpace argument must match the CR3 with PCID bits
181                  * stripped out.
182                  */
183                 flush->address_space = virt_to_phys(info->mm->pgd);
184                 flush->address_space &= CR3_ADDR_MASK;
185                 flush->flags = 0;
186         } else {
187                 flush->address_space = 0;
188                 flush->flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES;
189         }
190
191         flush->hv_vp_set.valid_bank_mask = 0;
192
193         flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
194         nr_bank = cpumask_to_vpset(&(flush->hv_vp_set), cpus);
195         if (nr_bank < 0)
196                 return U64_MAX;
197
198         /*
199          * We can flush not more than max_gvas with one hypercall. Flush the
200          * whole address space if we were asked to do more.
201          */
202         max_gvas =
203                 (PAGE_SIZE - sizeof(*flush) - nr_bank *
204                  sizeof(flush->hv_vp_set.bank_contents[0])) /
205                 sizeof(flush->gva_list[0]);
206
207         if (info->end == TLB_FLUSH_ALL) {
208                 flush->flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY;
209                 status = hv_do_rep_hypercall(
210                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
211                         0, nr_bank, flush, NULL);
212         } else if (info->end &&
213                    ((info->end - info->start)/HV_TLB_FLUSH_UNIT) > max_gvas) {
214                 status = hv_do_rep_hypercall(
215                         HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX,
216                         0, nr_bank, flush, NULL);
217         } else {
218                 gva_n = fill_gva_list(flush->gva_list, nr_bank,
219                                       info->start, info->end);
220                 status = hv_do_rep_hypercall(
221                         HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX,
222                         gva_n, nr_bank, flush, NULL);
223         }
224
225         return status;
226 }
227
228 void hyperv_setup_mmu_ops(void)
229 {
230         if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
231                 return;
232
233         pr_info("Using hypercall for remote TLB flush\n");
234         pv_ops.mmu.flush_tlb_others = hyperv_flush_tlb_others;
235         pv_ops.mmu.tlb_remove_table = tlb_remove_table;
236 }