nvmm_x86_svm.c revision 1.36
1 /* $NetBSD: nvmm_x86_svm.c,v 1.36 2019/04/03 17:32:58 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.36 2019/04/03 17:32:58 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 #include <sys/mman.h> 42 43 #include <uvm/uvm.h> 44 #include <uvm/uvm_page.h> 45 46 #include <x86/cputypes.h> 47 #include <x86/specialreg.h> 48 #include <x86/pmap.h> 49 #include <x86/dbregs.h> 50 #include <x86/cpu_counter.h> 51 #include <machine/cpuvar.h> 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/x86/nvmm_x86.h> 56 57 int svm_vmrun(paddr_t, uint64_t *); 58 59 #define MSR_VM_HSAVE_PA 0xC0010117 60 61 /* -------------------------------------------------------------------------- */ 62 63 #define VMCB_EXITCODE_CR0_READ 0x0000 64 #define VMCB_EXITCODE_CR1_READ 0x0001 65 #define VMCB_EXITCODE_CR2_READ 0x0002 66 #define VMCB_EXITCODE_CR3_READ 0x0003 67 #define VMCB_EXITCODE_CR4_READ 0x0004 68 #define VMCB_EXITCODE_CR5_READ 0x0005 69 #define VMCB_EXITCODE_CR6_READ 0x0006 70 #define VMCB_EXITCODE_CR7_READ 0x0007 71 #define VMCB_EXITCODE_CR8_READ 0x0008 72 #define VMCB_EXITCODE_CR9_READ 0x0009 73 #define VMCB_EXITCODE_CR10_READ 0x000A 74 #define VMCB_EXITCODE_CR11_READ 0x000B 75 #define VMCB_EXITCODE_CR12_READ 0x000C 76 #define VMCB_EXITCODE_CR13_READ 0x000D 77 #define VMCB_EXITCODE_CR14_READ 0x000E 78 #define VMCB_EXITCODE_CR15_READ 0x000F 79 #define VMCB_EXITCODE_CR0_WRITE 0x0010 80 #define VMCB_EXITCODE_CR1_WRITE 0x0011 81 #define VMCB_EXITCODE_CR2_WRITE 0x0012 82 #define VMCB_EXITCODE_CR3_WRITE 0x0013 83 #define VMCB_EXITCODE_CR4_WRITE 0x0014 84 #define VMCB_EXITCODE_CR5_WRITE 0x0015 85 #define VMCB_EXITCODE_CR6_WRITE 0x0016 86 #define VMCB_EXITCODE_CR7_WRITE 0x0017 87 #define VMCB_EXITCODE_CR8_WRITE 0x0018 88 #define VMCB_EXITCODE_CR9_WRITE 0x0019 89 #define VMCB_EXITCODE_CR10_WRITE 0x001A 90 #define VMCB_EXITCODE_CR11_WRITE 0x001B 91 #define VMCB_EXITCODE_CR12_WRITE 0x001C 92 #define VMCB_EXITCODE_CR13_WRITE 0x001D 93 #define VMCB_EXITCODE_CR14_WRITE 0x001E 94 #define VMCB_EXITCODE_CR15_WRITE 0x001F 95 #define VMCB_EXITCODE_DR0_READ 0x0020 96 #define VMCB_EXITCODE_DR1_READ 0x0021 97 #define VMCB_EXITCODE_DR2_READ 0x0022 98 #define VMCB_EXITCODE_DR3_READ 0x0023 99 #define VMCB_EXITCODE_DR4_READ 0x0024 100 #define VMCB_EXITCODE_DR5_READ 0x0025 101 #define VMCB_EXITCODE_DR6_READ 0x0026 102 #define VMCB_EXITCODE_DR7_READ 0x0027 103 #define VMCB_EXITCODE_DR8_READ 0x0028 104 #define VMCB_EXITCODE_DR9_READ 0x0029 105 #define VMCB_EXITCODE_DR10_READ 0x002A 106 #define VMCB_EXITCODE_DR11_READ 0x002B 107 #define VMCB_EXITCODE_DR12_READ 0x002C 108 #define VMCB_EXITCODE_DR13_READ 0x002D 109 #define VMCB_EXITCODE_DR14_READ 0x002E 110 #define VMCB_EXITCODE_DR15_READ 0x002F 111 #define VMCB_EXITCODE_DR0_WRITE 0x0030 112 #define VMCB_EXITCODE_DR1_WRITE 0x0031 113 #define VMCB_EXITCODE_DR2_WRITE 0x0032 114 #define VMCB_EXITCODE_DR3_WRITE 0x0033 115 #define VMCB_EXITCODE_DR4_WRITE 0x0034 116 #define VMCB_EXITCODE_DR5_WRITE 0x0035 117 #define VMCB_EXITCODE_DR6_WRITE 0x0036 118 #define VMCB_EXITCODE_DR7_WRITE 0x0037 119 #define VMCB_EXITCODE_DR8_WRITE 0x0038 120 #define VMCB_EXITCODE_DR9_WRITE 0x0039 121 #define VMCB_EXITCODE_DR10_WRITE 0x003A 122 #define VMCB_EXITCODE_DR11_WRITE 0x003B 123 #define VMCB_EXITCODE_DR12_WRITE 0x003C 124 #define VMCB_EXITCODE_DR13_WRITE 0x003D 125 #define VMCB_EXITCODE_DR14_WRITE 0x003E 126 #define VMCB_EXITCODE_DR15_WRITE 0x003F 127 #define VMCB_EXITCODE_EXCP0 0x0040 128 #define VMCB_EXITCODE_EXCP1 0x0041 129 #define VMCB_EXITCODE_EXCP2 0x0042 130 #define VMCB_EXITCODE_EXCP3 0x0043 131 #define VMCB_EXITCODE_EXCP4 0x0044 132 #define VMCB_EXITCODE_EXCP5 0x0045 133 #define VMCB_EXITCODE_EXCP6 0x0046 134 #define VMCB_EXITCODE_EXCP7 0x0047 135 #define VMCB_EXITCODE_EXCP8 0x0048 136 #define VMCB_EXITCODE_EXCP9 0x0049 137 #define VMCB_EXITCODE_EXCP10 0x004A 138 #define VMCB_EXITCODE_EXCP11 0x004B 139 #define VMCB_EXITCODE_EXCP12 0x004C 140 #define VMCB_EXITCODE_EXCP13 0x004D 141 #define VMCB_EXITCODE_EXCP14 0x004E 142 #define VMCB_EXITCODE_EXCP15 0x004F 143 #define VMCB_EXITCODE_EXCP16 0x0050 144 #define VMCB_EXITCODE_EXCP17 0x0051 145 #define VMCB_EXITCODE_EXCP18 0x0052 146 #define VMCB_EXITCODE_EXCP19 0x0053 147 #define VMCB_EXITCODE_EXCP20 0x0054 148 #define VMCB_EXITCODE_EXCP21 0x0055 149 #define VMCB_EXITCODE_EXCP22 0x0056 150 #define VMCB_EXITCODE_EXCP23 0x0057 151 #define VMCB_EXITCODE_EXCP24 0x0058 152 #define VMCB_EXITCODE_EXCP25 0x0059 153 #define VMCB_EXITCODE_EXCP26 0x005A 154 #define VMCB_EXITCODE_EXCP27 0x005B 155 #define VMCB_EXITCODE_EXCP28 0x005C 156 #define VMCB_EXITCODE_EXCP29 0x005D 157 #define VMCB_EXITCODE_EXCP30 0x005E 158 #define VMCB_EXITCODE_EXCP31 0x005F 159 #define VMCB_EXITCODE_INTR 0x0060 160 #define VMCB_EXITCODE_NMI 0x0061 161 #define VMCB_EXITCODE_SMI 0x0062 162 #define VMCB_EXITCODE_INIT 0x0063 163 #define VMCB_EXITCODE_VINTR 0x0064 164 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 165 #define VMCB_EXITCODE_IDTR_READ 0x0066 166 #define VMCB_EXITCODE_GDTR_READ 0x0067 167 #define VMCB_EXITCODE_LDTR_READ 0x0068 168 #define VMCB_EXITCODE_TR_READ 0x0069 169 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 170 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 171 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 172 #define VMCB_EXITCODE_TR_WRITE 0x006D 173 #define VMCB_EXITCODE_RDTSC 0x006E 174 #define VMCB_EXITCODE_RDPMC 0x006F 175 #define VMCB_EXITCODE_PUSHF 0x0070 176 #define VMCB_EXITCODE_POPF 0x0071 177 #define VMCB_EXITCODE_CPUID 0x0072 178 #define VMCB_EXITCODE_RSM 0x0073 179 #define VMCB_EXITCODE_IRET 0x0074 180 #define VMCB_EXITCODE_SWINT 0x0075 181 #define VMCB_EXITCODE_INVD 0x0076 182 #define VMCB_EXITCODE_PAUSE 0x0077 183 #define VMCB_EXITCODE_HLT 0x0078 184 #define VMCB_EXITCODE_INVLPG 0x0079 185 #define VMCB_EXITCODE_INVLPGA 0x007A 186 #define VMCB_EXITCODE_IOIO 0x007B 187 #define VMCB_EXITCODE_MSR 0x007C 188 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 189 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 190 #define VMCB_EXITCODE_SHUTDOWN 0x007F 191 #define VMCB_EXITCODE_VMRUN 0x0080 192 #define VMCB_EXITCODE_VMMCALL 0x0081 193 #define VMCB_EXITCODE_VMLOAD 0x0082 194 #define VMCB_EXITCODE_VMSAVE 0x0083 195 #define VMCB_EXITCODE_STGI 0x0084 196 #define VMCB_EXITCODE_CLGI 0x0085 197 #define VMCB_EXITCODE_SKINIT 0x0086 198 #define VMCB_EXITCODE_RDTSCP 0x0087 199 #define VMCB_EXITCODE_ICEBP 0x0088 200 #define VMCB_EXITCODE_WBINVD 0x0089 201 #define VMCB_EXITCODE_MONITOR 0x008A 202 #define VMCB_EXITCODE_MWAIT 0x008B 203 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 204 #define VMCB_EXITCODE_XSETBV 0x008D 205 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 206 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 207 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 208 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 209 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 210 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 211 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 212 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 213 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 214 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 215 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 216 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 217 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 218 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 219 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 220 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 221 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 222 #define VMCB_EXITCODE_NPF 0x0400 223 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 224 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 225 #define VMCB_EXITCODE_VMGEXIT 0x0403 226 #define VMCB_EXITCODE_INVALID -1 227 228 /* -------------------------------------------------------------------------- */ 229 230 struct vmcb_ctrl { 231 uint32_t intercept_cr; 232 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 233 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 234 235 uint32_t intercept_dr; 236 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 237 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 238 239 uint32_t intercept_vec; 240 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 241 242 uint32_t intercept_misc1; 243 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 244 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 245 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 246 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 247 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 248 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 249 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 250 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 251 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 252 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 253 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 254 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 255 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 256 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 257 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 258 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 259 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 260 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 261 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 262 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 263 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 264 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 265 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 266 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 267 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 268 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 269 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 270 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 271 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 272 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 273 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 274 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 275 276 uint32_t intercept_misc2; 277 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 278 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 279 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 280 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 281 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 282 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 283 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 284 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 285 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 286 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 287 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 288 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(12) 289 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 290 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 291 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 292 293 uint8_t rsvd1[40]; 294 uint16_t pause_filt_thresh; 295 uint16_t pause_filt_cnt; 296 uint64_t iopm_base_pa; 297 uint64_t msrpm_base_pa; 298 uint64_t tsc_offset; 299 uint32_t guest_asid; 300 301 uint32_t tlb_ctrl; 302 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 303 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 304 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 305 306 uint64_t v; 307 #define VMCB_CTRL_V_TPR __BITS(3,0) 308 #define VMCB_CTRL_V_IRQ __BIT(8) 309 #define VMCB_CTRL_V_VGIF __BIT(9) 310 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 311 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 312 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 313 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 314 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 315 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 316 317 uint64_t intr; 318 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 319 320 uint64_t exitcode; 321 uint64_t exitinfo1; 322 uint64_t exitinfo2; 323 324 uint64_t exitintinfo; 325 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 326 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 327 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 328 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 329 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 330 331 uint64_t enable1; 332 #define VMCB_CTRL_ENABLE_NP __BIT(0) 333 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 334 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 335 336 uint64_t avic; 337 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 338 339 uint64_t ghcb; 340 341 uint64_t eventinj; 342 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 343 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 344 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 345 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 346 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 347 348 uint64_t n_cr3; 349 350 uint64_t enable2; 351 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 352 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 353 354 uint32_t vmcb_clean; 355 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 356 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 357 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 358 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 359 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 360 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 361 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 362 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 363 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 364 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 365 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 366 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 367 368 uint32_t rsvd2; 369 uint64_t nrip; 370 uint8_t inst_len; 371 uint8_t inst_bytes[15]; 372 uint64_t avic_abpp; 373 uint64_t rsvd3; 374 uint64_t avic_ltp; 375 376 uint64_t avic_phys; 377 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 378 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 379 380 uint64_t rsvd4; 381 uint64_t vmcb_ptr; 382 383 uint8_t pad[752]; 384 } __packed; 385 386 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 387 388 struct vmcb_segment { 389 uint16_t selector; 390 uint16_t attrib; /* hidden */ 391 uint32_t limit; /* hidden */ 392 uint64_t base; /* hidden */ 393 } __packed; 394 395 CTASSERT(sizeof(struct vmcb_segment) == 16); 396 397 struct vmcb_state { 398 struct vmcb_segment es; 399 struct vmcb_segment cs; 400 struct vmcb_segment ss; 401 struct vmcb_segment ds; 402 struct vmcb_segment fs; 403 struct vmcb_segment gs; 404 struct vmcb_segment gdt; 405 struct vmcb_segment ldt; 406 struct vmcb_segment idt; 407 struct vmcb_segment tr; 408 uint8_t rsvd1[43]; 409 uint8_t cpl; 410 uint8_t rsvd2[4]; 411 uint64_t efer; 412 uint8_t rsvd3[112]; 413 uint64_t cr4; 414 uint64_t cr3; 415 uint64_t cr0; 416 uint64_t dr7; 417 uint64_t dr6; 418 uint64_t rflags; 419 uint64_t rip; 420 uint8_t rsvd4[88]; 421 uint64_t rsp; 422 uint8_t rsvd5[24]; 423 uint64_t rax; 424 uint64_t star; 425 uint64_t lstar; 426 uint64_t cstar; 427 uint64_t sfmask; 428 uint64_t kernelgsbase; 429 uint64_t sysenter_cs; 430 uint64_t sysenter_esp; 431 uint64_t sysenter_eip; 432 uint64_t cr2; 433 uint8_t rsvd6[32]; 434 uint64_t g_pat; 435 uint64_t dbgctl; 436 uint64_t br_from; 437 uint64_t br_to; 438 uint64_t int_from; 439 uint64_t int_to; 440 uint8_t pad[2408]; 441 } __packed; 442 443 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 444 445 struct vmcb { 446 struct vmcb_ctrl ctrl; 447 struct vmcb_state state; 448 } __packed; 449 450 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 451 CTASSERT(offsetof(struct vmcb, state) == 0x400); 452 453 /* -------------------------------------------------------------------------- */ 454 455 struct svm_hsave { 456 paddr_t pa; 457 }; 458 459 static struct svm_hsave hsave[MAXCPUS]; 460 461 static uint8_t *svm_asidmap __read_mostly; 462 static uint32_t svm_maxasid __read_mostly; 463 static kmutex_t svm_asidlock __cacheline_aligned; 464 465 static bool svm_decode_assist __read_mostly; 466 static uint32_t svm_ctrl_tlb_flush __read_mostly; 467 468 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 469 static uint64_t svm_xcr0_mask __read_mostly; 470 471 #define SVM_NCPUIDS 32 472 473 #define VMCB_NPAGES 1 474 475 #define MSRBM_NPAGES 2 476 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 477 478 #define IOBM_NPAGES 3 479 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 480 481 /* Does not include EFER_LMSLE. */ 482 #define EFER_VALID \ 483 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 484 485 #define EFER_TLB_FLUSH \ 486 (EFER_NXE|EFER_LMA|EFER_LME) 487 #define CR0_TLB_FLUSH \ 488 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 489 #define CR4_TLB_FLUSH \ 490 (CR4_PGE|CR4_PAE|CR4_PSE) 491 492 /* -------------------------------------------------------------------------- */ 493 494 struct svm_machdata { 495 bool cpuidpresent[SVM_NCPUIDS]; 496 struct nvmm_x86_conf_cpuid cpuid[SVM_NCPUIDS]; 497 volatile uint64_t mach_htlb_gen; 498 }; 499 500 static const size_t svm_conf_sizes[NVMM_X86_NCONF] = { 501 [NVMM_X86_CONF_CPUID] = sizeof(struct nvmm_x86_conf_cpuid) 502 }; 503 504 struct svm_cpudata { 505 /* General */ 506 bool shared_asid; 507 bool gtlb_want_flush; 508 bool gtsc_want_update; 509 uint64_t vcpu_htlb_gen; 510 511 /* VMCB */ 512 struct vmcb *vmcb; 513 paddr_t vmcb_pa; 514 515 /* I/O bitmap */ 516 uint8_t *iobm; 517 paddr_t iobm_pa; 518 519 /* MSR bitmap */ 520 uint8_t *msrbm; 521 paddr_t msrbm_pa; 522 523 /* Host state */ 524 uint64_t hxcr0; 525 uint64_t star; 526 uint64_t lstar; 527 uint64_t cstar; 528 uint64_t sfmask; 529 uint64_t fsbase; 530 uint64_t kernelgsbase; 531 bool ts_set; 532 struct xsave_header hfpu __aligned(64); 533 534 /* Event state */ 535 bool int_window_exit; 536 bool nmi_window_exit; 537 538 /* Guest state */ 539 uint64_t gxcr0; 540 uint64_t gprs[NVMM_X64_NGPR]; 541 uint64_t drs[NVMM_X64_NDR]; 542 uint64_t gtsc; 543 struct xsave_header gfpu __aligned(64); 544 }; 545 546 static void 547 svm_vmcb_cache_default(struct vmcb *vmcb) 548 { 549 vmcb->ctrl.vmcb_clean = 550 VMCB_CTRL_VMCB_CLEAN_I | 551 VMCB_CTRL_VMCB_CLEAN_IOPM | 552 VMCB_CTRL_VMCB_CLEAN_ASID | 553 VMCB_CTRL_VMCB_CLEAN_TPR | 554 VMCB_CTRL_VMCB_CLEAN_NP | 555 VMCB_CTRL_VMCB_CLEAN_CR | 556 VMCB_CTRL_VMCB_CLEAN_DR | 557 VMCB_CTRL_VMCB_CLEAN_DT | 558 VMCB_CTRL_VMCB_CLEAN_SEG | 559 VMCB_CTRL_VMCB_CLEAN_CR2 | 560 VMCB_CTRL_VMCB_CLEAN_LBR | 561 VMCB_CTRL_VMCB_CLEAN_AVIC; 562 } 563 564 static void 565 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 566 { 567 if (flags & NVMM_X64_STATE_SEGS) { 568 vmcb->ctrl.vmcb_clean &= 569 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 570 } 571 if (flags & NVMM_X64_STATE_CRS) { 572 vmcb->ctrl.vmcb_clean &= 573 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 574 VMCB_CTRL_VMCB_CLEAN_TPR); 575 } 576 if (flags & NVMM_X64_STATE_DRS) { 577 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 578 } 579 if (flags & NVMM_X64_STATE_MSRS) { 580 /* CR for EFER, NP for PAT. */ 581 vmcb->ctrl.vmcb_clean &= 582 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 583 } 584 } 585 586 static inline void 587 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 588 { 589 vmcb->ctrl.vmcb_clean &= ~flags; 590 } 591 592 static inline void 593 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 594 { 595 vmcb->ctrl.vmcb_clean = 0; 596 } 597 598 #define SVM_EVENT_TYPE_HW_INT 0 599 #define SVM_EVENT_TYPE_NMI 2 600 #define SVM_EVENT_TYPE_EXC 3 601 #define SVM_EVENT_TYPE_SW_INT 4 602 603 static void 604 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 605 { 606 struct svm_cpudata *cpudata = vcpu->cpudata; 607 struct vmcb *vmcb = cpudata->vmcb; 608 609 if (nmi) { 610 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 611 cpudata->nmi_window_exit = true; 612 } else { 613 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 614 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 615 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 616 cpudata->int_window_exit = true; 617 } 618 619 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 620 } 621 622 static void 623 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 624 { 625 struct svm_cpudata *cpudata = vcpu->cpudata; 626 struct vmcb *vmcb = cpudata->vmcb; 627 628 if (nmi) { 629 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 630 cpudata->nmi_window_exit = false; 631 } else { 632 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 633 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 634 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 635 cpudata->int_window_exit = false; 636 } 637 638 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 639 } 640 641 static inline int 642 svm_event_has_error(uint64_t vector) 643 { 644 switch (vector) { 645 case 8: /* #DF */ 646 case 10: /* #TS */ 647 case 11: /* #NP */ 648 case 12: /* #SS */ 649 case 13: /* #GP */ 650 case 14: /* #PF */ 651 case 17: /* #AC */ 652 case 30: /* #SX */ 653 return 1; 654 default: 655 return 0; 656 } 657 } 658 659 static int 660 svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 661 struct nvmm_event *event) 662 { 663 struct svm_cpudata *cpudata = vcpu->cpudata; 664 struct vmcb *vmcb = cpudata->vmcb; 665 int type = 0, err = 0; 666 667 if (event->vector >= 256) { 668 return EINVAL; 669 } 670 671 switch (event->type) { 672 case NVMM_EVENT_INTERRUPT_HW: 673 type = SVM_EVENT_TYPE_HW_INT; 674 if (event->vector == 2) { 675 type = SVM_EVENT_TYPE_NMI; 676 } 677 if (type == SVM_EVENT_TYPE_NMI) { 678 if (cpudata->nmi_window_exit) { 679 return EAGAIN; 680 } 681 svm_event_waitexit_enable(vcpu, true); 682 } else { 683 if (((vmcb->state.rflags & PSL_I) == 0) || 684 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0)) { 685 svm_event_waitexit_enable(vcpu, false); 686 return EAGAIN; 687 } 688 } 689 err = 0; 690 break; 691 case NVMM_EVENT_INTERRUPT_SW: 692 return EINVAL; 693 case NVMM_EVENT_EXCEPTION: 694 type = SVM_EVENT_TYPE_EXC; 695 if (event->vector == 2 || event->vector >= 32) 696 return EINVAL; 697 if (event->vector == 3 || event->vector == 0) 698 return EINVAL; 699 err = svm_event_has_error(event->vector); 700 break; 701 default: 702 return EINVAL; 703 } 704 705 vmcb->ctrl.eventinj = 706 __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) | 707 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 708 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 709 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 710 __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE); 711 712 return 0; 713 } 714 715 static void 716 svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 717 { 718 struct nvmm_event event; 719 int ret __diagused; 720 721 event.type = NVMM_EVENT_EXCEPTION; 722 event.vector = 6; 723 event.u.error = 0; 724 725 ret = svm_vcpu_inject(mach, vcpu, &event); 726 KASSERT(ret == 0); 727 } 728 729 static void 730 svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 731 { 732 struct nvmm_event event; 733 int ret __diagused; 734 735 event.type = NVMM_EVENT_EXCEPTION; 736 event.vector = 13; 737 event.u.error = 0; 738 739 ret = svm_vcpu_inject(mach, vcpu, &event); 740 KASSERT(ret == 0); 741 } 742 743 static inline void 744 svm_inkernel_advance(struct vmcb *vmcb) 745 { 746 /* 747 * Maybe we should also apply single-stepping and debug exceptions. 748 * Matters for guest-ring3, because it can execute 'cpuid' under a 749 * debugger. 750 */ 751 vmcb->state.rip = vmcb->ctrl.nrip; 752 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 753 } 754 755 static void 756 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 757 { 758 struct svm_cpudata *cpudata = vcpu->cpudata; 759 uint64_t cr4; 760 761 switch (eax) { 762 case 0x00000001: 763 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 764 765 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 766 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 767 CPUID_LOCAL_APIC_ID); 768 769 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 770 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 771 772 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 773 774 /* CPUID2_OSXSAVE depends on CR4. */ 775 cr4 = cpudata->vmcb->state.cr4; 776 if (!(cr4 & CR4_OSXSAVE)) { 777 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 778 } 779 break; 780 case 0x00000005: 781 case 0x00000006: 782 cpudata->vmcb->state.rax = 0; 783 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 784 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 785 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 786 break; 787 case 0x00000007: 788 cpudata->vmcb->state.rax &= nvmm_cpuid_00000007.eax; 789 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 790 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 791 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 792 break; 793 case 0x0000000D: 794 if (svm_xcr0_mask == 0) { 795 break; 796 } 797 switch (ecx) { 798 case 0: 799 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 800 if (cpudata->gxcr0 & XCR0_SSE) { 801 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 802 } else { 803 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 804 } 805 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 806 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave); 807 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 808 break; 809 case 1: 810 cpudata->vmcb->state.rax &= ~CPUID_PES1_XSAVES; 811 break; 812 } 813 break; 814 case 0x40000000: 815 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 816 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 817 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 818 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 819 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 820 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 821 break; 822 case 0x80000001: 823 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 824 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 825 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 826 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 827 break; 828 default: 829 break; 830 } 831 } 832 833 static void 834 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 835 struct nvmm_exit *exit) 836 { 837 struct svm_machdata *machdata = mach->machdata; 838 struct svm_cpudata *cpudata = vcpu->cpudata; 839 struct nvmm_x86_conf_cpuid *cpuid; 840 uint64_t eax, ecx; 841 u_int descs[4]; 842 size_t i; 843 844 eax = cpudata->vmcb->state.rax; 845 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 846 x86_cpuid2(eax, ecx, descs); 847 848 cpudata->vmcb->state.rax = descs[0]; 849 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 850 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 851 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 852 853 for (i = 0; i < SVM_NCPUIDS; i++) { 854 cpuid = &machdata->cpuid[i]; 855 if (!machdata->cpuidpresent[i]) { 856 continue; 857 } 858 if (cpuid->leaf != eax) { 859 continue; 860 } 861 862 /* del */ 863 cpudata->vmcb->state.rax &= ~cpuid->del.eax; 864 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->del.ebx; 865 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->del.ecx; 866 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->del.edx; 867 868 /* set */ 869 cpudata->vmcb->state.rax |= cpuid->set.eax; 870 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->set.ebx; 871 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->set.ecx; 872 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->set.edx; 873 874 break; 875 } 876 877 /* Overwrite non-tunable leaves. */ 878 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 879 880 svm_inkernel_advance(cpudata->vmcb); 881 exit->reason = NVMM_EXIT_NONE; 882 } 883 884 static void 885 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 886 struct nvmm_exit *exit) 887 { 888 struct svm_cpudata *cpudata = vcpu->cpudata; 889 struct vmcb *vmcb = cpudata->vmcb; 890 891 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 892 svm_event_waitexit_disable(vcpu, false); 893 } 894 895 svm_inkernel_advance(cpudata->vmcb); 896 exit->reason = NVMM_EXIT_HALTED; 897 } 898 899 #define SVM_EXIT_IO_PORT __BITS(31,16) 900 #define SVM_EXIT_IO_SEG __BITS(12,10) 901 #define SVM_EXIT_IO_A64 __BIT(9) 902 #define SVM_EXIT_IO_A32 __BIT(8) 903 #define SVM_EXIT_IO_A16 __BIT(7) 904 #define SVM_EXIT_IO_SZ32 __BIT(6) 905 #define SVM_EXIT_IO_SZ16 __BIT(5) 906 #define SVM_EXIT_IO_SZ8 __BIT(4) 907 #define SVM_EXIT_IO_REP __BIT(3) 908 #define SVM_EXIT_IO_STR __BIT(2) 909 #define SVM_EXIT_IO_IN __BIT(0) 910 911 static void 912 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 913 struct nvmm_exit *exit) 914 { 915 struct svm_cpudata *cpudata = vcpu->cpudata; 916 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 917 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 918 919 exit->reason = NVMM_EXIT_IO; 920 921 if (info & SVM_EXIT_IO_IN) { 922 exit->u.io.type = NVMM_EXIT_IO_IN; 923 } else { 924 exit->u.io.type = NVMM_EXIT_IO_OUT; 925 } 926 927 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 928 929 if (svm_decode_assist) { 930 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 931 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 932 } else { 933 exit->u.io.seg = -1; 934 } 935 936 if (info & SVM_EXIT_IO_A64) { 937 exit->u.io.address_size = 8; 938 } else if (info & SVM_EXIT_IO_A32) { 939 exit->u.io.address_size = 4; 940 } else if (info & SVM_EXIT_IO_A16) { 941 exit->u.io.address_size = 2; 942 } 943 944 if (info & SVM_EXIT_IO_SZ32) { 945 exit->u.io.operand_size = 4; 946 } else if (info & SVM_EXIT_IO_SZ16) { 947 exit->u.io.operand_size = 2; 948 } else if (info & SVM_EXIT_IO_SZ8) { 949 exit->u.io.operand_size = 1; 950 } 951 952 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 953 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 954 exit->u.io.npc = nextpc; 955 } 956 957 static const uint64_t msr_ignore_list[] = { 958 0xc0010055, /* MSR_CMPHALT */ 959 MSR_DE_CFG, 960 MSR_IC_CFG, 961 MSR_UCODE_AMD_PATCHLEVEL 962 }; 963 964 static bool 965 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 966 struct nvmm_exit *exit) 967 { 968 struct svm_cpudata *cpudata = vcpu->cpudata; 969 struct vmcb *vmcb = cpudata->vmcb; 970 uint64_t val; 971 size_t i; 972 973 switch (exit->u.msr.type) { 974 case NVMM_EXIT_MSR_RDMSR: 975 if (exit->u.msr.msr == MSR_NB_CFG) { 976 val = NB_CFG_INITAPICCPUIDLO; 977 vmcb->state.rax = (val & 0xFFFFFFFF); 978 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 979 goto handled; 980 } 981 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 982 if (msr_ignore_list[i] != exit->u.msr.msr) 983 continue; 984 val = 0; 985 vmcb->state.rax = (val & 0xFFFFFFFF); 986 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 987 goto handled; 988 } 989 break; 990 case NVMM_EXIT_MSR_WRMSR: 991 if (exit->u.msr.msr == MSR_EFER) { 992 if (__predict_false(exit->u.msr.val & ~EFER_VALID)) { 993 goto error; 994 } 995 if ((vmcb->state.efer ^ exit->u.msr.val) & 996 EFER_TLB_FLUSH) { 997 cpudata->gtlb_want_flush = true; 998 } 999 vmcb->state.efer = exit->u.msr.val | EFER_SVME; 1000 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1001 goto handled; 1002 } 1003 if (exit->u.msr.msr == MSR_TSC) { 1004 cpudata->gtsc = exit->u.msr.val; 1005 cpudata->gtsc_want_update = true; 1006 goto handled; 1007 } 1008 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1009 if (msr_ignore_list[i] != exit->u.msr.msr) 1010 continue; 1011 goto handled; 1012 } 1013 break; 1014 } 1015 1016 return false; 1017 1018 handled: 1019 svm_inkernel_advance(cpudata->vmcb); 1020 return true; 1021 1022 error: 1023 svm_inject_gp(mach, vcpu); 1024 return true; 1025 } 1026 1027 static void 1028 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1029 struct nvmm_exit *exit) 1030 { 1031 struct svm_cpudata *cpudata = vcpu->cpudata; 1032 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1033 1034 if (info == 0) { 1035 exit->u.msr.type = NVMM_EXIT_MSR_RDMSR; 1036 } else { 1037 exit->u.msr.type = NVMM_EXIT_MSR_WRMSR; 1038 } 1039 1040 exit->u.msr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1041 1042 if (info == 1) { 1043 uint64_t rdx, rax; 1044 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1045 rax = cpudata->vmcb->state.rax; 1046 exit->u.msr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1047 } else { 1048 exit->u.msr.val = 0; 1049 } 1050 1051 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1052 exit->reason = NVMM_EXIT_NONE; 1053 return; 1054 } 1055 1056 exit->reason = NVMM_EXIT_MSR; 1057 exit->u.msr.npc = cpudata->vmcb->ctrl.nrip; 1058 } 1059 1060 static void 1061 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1062 struct nvmm_exit *exit) 1063 { 1064 struct svm_cpudata *cpudata = vcpu->cpudata; 1065 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1066 1067 exit->reason = NVMM_EXIT_MEMORY; 1068 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1069 exit->u.mem.prot = PROT_WRITE; 1070 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1071 exit->u.mem.prot = PROT_EXEC; 1072 else 1073 exit->u.mem.prot = PROT_READ; 1074 exit->u.mem.gpa = gpa; 1075 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1076 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1077 sizeof(exit->u.mem.inst_bytes)); 1078 } 1079 1080 static void 1081 svm_exit_insn(struct vmcb *vmcb, struct nvmm_exit *exit, uint64_t reason) 1082 { 1083 exit->u.insn.npc = vmcb->ctrl.nrip; 1084 exit->reason = reason; 1085 } 1086 1087 static void 1088 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1089 struct nvmm_exit *exit) 1090 { 1091 struct svm_cpudata *cpudata = vcpu->cpudata; 1092 struct vmcb *vmcb = cpudata->vmcb; 1093 uint64_t val; 1094 1095 exit->reason = NVMM_EXIT_NONE; 1096 1097 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1098 (vmcb->state.rax & 0xFFFFFFFF); 1099 1100 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1101 goto error; 1102 } else if (__predict_false(vmcb->state.cpl != 0)) { 1103 goto error; 1104 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1105 goto error; 1106 } else if (__predict_false((val & XCR0_X87) == 0)) { 1107 goto error; 1108 } 1109 1110 cpudata->gxcr0 = val; 1111 1112 svm_inkernel_advance(cpudata->vmcb); 1113 return; 1114 1115 error: 1116 svm_inject_gp(mach, vcpu); 1117 } 1118 1119 /* -------------------------------------------------------------------------- */ 1120 1121 static void 1122 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1123 { 1124 struct svm_cpudata *cpudata = vcpu->cpudata; 1125 1126 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1127 1128 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1129 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1130 1131 if (svm_xcr0_mask != 0) { 1132 cpudata->hxcr0 = rdxcr(0); 1133 wrxcr(0, cpudata->gxcr0); 1134 } 1135 } 1136 1137 static void 1138 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1139 { 1140 struct svm_cpudata *cpudata = vcpu->cpudata; 1141 1142 if (svm_xcr0_mask != 0) { 1143 cpudata->gxcr0 = rdxcr(0); 1144 wrxcr(0, cpudata->hxcr0); 1145 } 1146 1147 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1148 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1149 1150 if (cpudata->ts_set) { 1151 stts(); 1152 } 1153 } 1154 1155 static void 1156 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1157 { 1158 struct svm_cpudata *cpudata = vcpu->cpudata; 1159 1160 x86_dbregs_save(curlwp); 1161 1162 ldr7(0); 1163 1164 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1165 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1166 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1167 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1168 } 1169 1170 static void 1171 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1172 { 1173 struct svm_cpudata *cpudata = vcpu->cpudata; 1174 1175 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1176 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1177 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1178 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1179 1180 x86_dbregs_restore(curlwp); 1181 } 1182 1183 static void 1184 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1185 { 1186 struct svm_cpudata *cpudata = vcpu->cpudata; 1187 1188 cpudata->fsbase = rdmsr(MSR_FSBASE); 1189 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1190 } 1191 1192 static void 1193 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1194 { 1195 struct svm_cpudata *cpudata = vcpu->cpudata; 1196 1197 wrmsr(MSR_STAR, cpudata->star); 1198 wrmsr(MSR_LSTAR, cpudata->lstar); 1199 wrmsr(MSR_CSTAR, cpudata->cstar); 1200 wrmsr(MSR_SFMASK, cpudata->sfmask); 1201 wrmsr(MSR_FSBASE, cpudata->fsbase); 1202 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1203 } 1204 1205 /* -------------------------------------------------------------------------- */ 1206 1207 static inline void 1208 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1209 { 1210 struct svm_cpudata *cpudata = vcpu->cpudata; 1211 1212 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1213 cpudata->gtlb_want_flush = true; 1214 } 1215 } 1216 1217 static inline void 1218 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1219 { 1220 /* 1221 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1222 * executing on this hCPU and the hTLB already got flushed, or it 1223 * was executing on another hCPU in which case the catchup is done 1224 * in svm_gtlb_catchup(). 1225 */ 1226 } 1227 1228 static inline uint64_t 1229 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1230 { 1231 struct vmcb *vmcb = cpudata->vmcb; 1232 uint64_t machgen; 1233 1234 machgen = machdata->mach_htlb_gen; 1235 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1236 return machgen; 1237 } 1238 1239 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1240 return machgen; 1241 } 1242 1243 static inline void 1244 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1245 { 1246 struct vmcb *vmcb = cpudata->vmcb; 1247 1248 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1249 cpudata->vcpu_htlb_gen = machgen; 1250 } 1251 } 1252 1253 static int 1254 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1255 struct nvmm_exit *exit) 1256 { 1257 struct svm_machdata *machdata = mach->machdata; 1258 struct svm_cpudata *cpudata = vcpu->cpudata; 1259 struct vmcb *vmcb = cpudata->vmcb; 1260 uint64_t machgen; 1261 int hcpu, s; 1262 1263 kpreempt_disable(); 1264 hcpu = cpu_number(); 1265 1266 svm_gtlb_catchup(vcpu, hcpu); 1267 svm_htlb_catchup(vcpu, hcpu); 1268 1269 if (vcpu->hcpu_last != hcpu) { 1270 svm_vmcb_cache_flush_all(vmcb); 1271 cpudata->gtsc_want_update = true; 1272 } 1273 1274 svm_vcpu_guest_dbregs_enter(vcpu); 1275 svm_vcpu_guest_misc_enter(vcpu); 1276 1277 while (1) { 1278 if (cpudata->gtlb_want_flush) { 1279 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1280 } else { 1281 vmcb->ctrl.tlb_ctrl = 0; 1282 } 1283 1284 if (__predict_false(cpudata->gtsc_want_update)) { 1285 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1286 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1287 } 1288 1289 s = splhigh(); 1290 machgen = svm_htlb_flush(machdata, cpudata); 1291 svm_vcpu_guest_fpu_enter(vcpu); 1292 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1293 svm_vcpu_guest_fpu_leave(vcpu); 1294 svm_htlb_flush_ack(cpudata, machgen); 1295 splx(s); 1296 1297 svm_vmcb_cache_default(vmcb); 1298 1299 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1300 cpudata->gtlb_want_flush = false; 1301 cpudata->gtsc_want_update = false; 1302 vcpu->hcpu_last = hcpu; 1303 } 1304 1305 switch (vmcb->ctrl.exitcode) { 1306 case VMCB_EXITCODE_INTR: 1307 case VMCB_EXITCODE_NMI: 1308 exit->reason = NVMM_EXIT_NONE; 1309 break; 1310 case VMCB_EXITCODE_VINTR: 1311 svm_event_waitexit_disable(vcpu, false); 1312 exit->reason = NVMM_EXIT_INT_READY; 1313 break; 1314 case VMCB_EXITCODE_IRET: 1315 svm_event_waitexit_disable(vcpu, true); 1316 exit->reason = NVMM_EXIT_NMI_READY; 1317 break; 1318 case VMCB_EXITCODE_CPUID: 1319 svm_exit_cpuid(mach, vcpu, exit); 1320 break; 1321 case VMCB_EXITCODE_HLT: 1322 svm_exit_hlt(mach, vcpu, exit); 1323 break; 1324 case VMCB_EXITCODE_IOIO: 1325 svm_exit_io(mach, vcpu, exit); 1326 break; 1327 case VMCB_EXITCODE_MSR: 1328 svm_exit_msr(mach, vcpu, exit); 1329 break; 1330 case VMCB_EXITCODE_SHUTDOWN: 1331 exit->reason = NVMM_EXIT_SHUTDOWN; 1332 break; 1333 case VMCB_EXITCODE_RDPMC: 1334 case VMCB_EXITCODE_RSM: 1335 case VMCB_EXITCODE_INVLPGA: 1336 case VMCB_EXITCODE_VMRUN: 1337 case VMCB_EXITCODE_VMMCALL: 1338 case VMCB_EXITCODE_VMLOAD: 1339 case VMCB_EXITCODE_VMSAVE: 1340 case VMCB_EXITCODE_STGI: 1341 case VMCB_EXITCODE_CLGI: 1342 case VMCB_EXITCODE_SKINIT: 1343 case VMCB_EXITCODE_RDTSCP: 1344 svm_inject_ud(mach, vcpu); 1345 exit->reason = NVMM_EXIT_NONE; 1346 break; 1347 case VMCB_EXITCODE_MONITOR: 1348 svm_exit_insn(vmcb, exit, NVMM_EXIT_MONITOR); 1349 break; 1350 case VMCB_EXITCODE_MWAIT: 1351 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT); 1352 break; 1353 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1354 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT_COND); 1355 break; 1356 case VMCB_EXITCODE_XSETBV: 1357 svm_exit_xsetbv(mach, vcpu, exit); 1358 break; 1359 case VMCB_EXITCODE_NPF: 1360 svm_exit_npf(mach, vcpu, exit); 1361 break; 1362 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1363 default: 1364 exit->reason = NVMM_EXIT_INVALID; 1365 break; 1366 } 1367 1368 /* If no reason to return to userland, keep rolling. */ 1369 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1370 break; 1371 } 1372 if (curcpu()->ci_data.cpu_softints != 0) { 1373 break; 1374 } 1375 if (curlwp->l_flag & LW_USERRET) { 1376 break; 1377 } 1378 if (exit->reason != NVMM_EXIT_NONE) { 1379 break; 1380 } 1381 } 1382 1383 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1384 1385 svm_vcpu_guest_misc_leave(vcpu); 1386 svm_vcpu_guest_dbregs_leave(vcpu); 1387 1388 kpreempt_enable(); 1389 1390 exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1391 VMCB_CTRL_V_TPR); 1392 exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags; 1393 1394 exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] = 1395 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1396 exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] = 1397 cpudata->int_window_exit; 1398 exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] = 1399 cpudata->nmi_window_exit; 1400 1401 return 0; 1402 } 1403 1404 /* -------------------------------------------------------------------------- */ 1405 1406 static int 1407 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1408 { 1409 struct pglist pglist; 1410 paddr_t _pa; 1411 vaddr_t _va; 1412 size_t i; 1413 int ret; 1414 1415 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1416 &pglist, 1, 0); 1417 if (ret != 0) 1418 return ENOMEM; 1419 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1420 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1421 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1422 if (_va == 0) 1423 goto error; 1424 1425 for (i = 0; i < npages; i++) { 1426 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1427 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1428 } 1429 pmap_update(pmap_kernel()); 1430 1431 memset((void *)_va, 0, npages * PAGE_SIZE); 1432 1433 *pa = _pa; 1434 *va = _va; 1435 return 0; 1436 1437 error: 1438 for (i = 0; i < npages; i++) { 1439 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1440 } 1441 return ENOMEM; 1442 } 1443 1444 static void 1445 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1446 { 1447 size_t i; 1448 1449 pmap_kremove(va, npages * PAGE_SIZE); 1450 pmap_update(pmap_kernel()); 1451 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1452 for (i = 0; i < npages; i++) { 1453 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1454 } 1455 } 1456 1457 /* -------------------------------------------------------------------------- */ 1458 1459 #define SVM_MSRBM_READ __BIT(0) 1460 #define SVM_MSRBM_WRITE __BIT(1) 1461 1462 static void 1463 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1464 { 1465 uint64_t byte; 1466 uint8_t bitoff; 1467 1468 if (msr < 0x00002000) { 1469 /* Range 1 */ 1470 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1471 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1472 /* Range 2 */ 1473 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1474 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1475 /* Range 3 */ 1476 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1477 } else { 1478 panic("%s: wrong range", __func__); 1479 } 1480 1481 bitoff = (msr & 0x3) << 1; 1482 1483 if (read) { 1484 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1485 } 1486 if (write) { 1487 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1488 } 1489 } 1490 1491 #define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1492 #define SVM_SEG_ATTRIB_S __BIT(4) 1493 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1494 #define SVM_SEG_ATTRIB_P __BIT(7) 1495 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1496 #define SVM_SEG_ATTRIB_L __BIT(9) 1497 #define SVM_SEG_ATTRIB_DEF __BIT(10) 1498 #define SVM_SEG_ATTRIB_G __BIT(11) 1499 1500 static void 1501 svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1502 struct vmcb_segment *vseg) 1503 { 1504 vseg->selector = seg->selector; 1505 vseg->attrib = 1506 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1507 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1508 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1509 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1510 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1511 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1512 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1513 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1514 vseg->limit = seg->limit; 1515 vseg->base = seg->base; 1516 } 1517 1518 static void 1519 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1520 { 1521 seg->selector = vseg->selector; 1522 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1523 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1524 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1525 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1526 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1527 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1528 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1529 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1530 seg->limit = vseg->limit; 1531 seg->base = vseg->base; 1532 } 1533 1534 static inline bool 1535 svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1536 uint64_t flags) 1537 { 1538 if (flags & NVMM_X64_STATE_CRS) { 1539 if ((vmcb->state.cr0 ^ 1540 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1541 return true; 1542 } 1543 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1544 return true; 1545 } 1546 if ((vmcb->state.cr4 ^ 1547 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1548 return true; 1549 } 1550 } 1551 1552 if (flags & NVMM_X64_STATE_MSRS) { 1553 if ((vmcb->state.efer ^ 1554 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1555 return true; 1556 } 1557 } 1558 1559 return false; 1560 } 1561 1562 static void 1563 svm_vcpu_setstate(struct nvmm_cpu *vcpu, const void *data, uint64_t flags) 1564 { 1565 const struct nvmm_x64_state *state = data; 1566 struct svm_cpudata *cpudata = vcpu->cpudata; 1567 struct vmcb *vmcb = cpudata->vmcb; 1568 struct fxsave *fpustate; 1569 1570 if (svm_state_tlb_flush(vmcb, state, flags)) { 1571 cpudata->gtlb_want_flush = true; 1572 } 1573 1574 if (flags & NVMM_X64_STATE_SEGS) { 1575 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1576 &vmcb->state.cs); 1577 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1578 &vmcb->state.ds); 1579 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1580 &vmcb->state.es); 1581 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1582 &vmcb->state.fs); 1583 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1584 &vmcb->state.gs); 1585 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1586 &vmcb->state.ss); 1587 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1588 &vmcb->state.gdt); 1589 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1590 &vmcb->state.idt); 1591 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1592 &vmcb->state.ldt); 1593 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1594 &vmcb->state.tr); 1595 1596 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1597 } 1598 1599 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1600 if (flags & NVMM_X64_STATE_GPRS) { 1601 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1602 1603 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1604 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1605 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1606 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1607 } 1608 1609 if (flags & NVMM_X64_STATE_CRS) { 1610 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1611 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1612 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1613 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1614 1615 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1616 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1617 VMCB_CTRL_V_TPR); 1618 1619 if (svm_xcr0_mask != 0) { 1620 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1621 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1622 cpudata->gxcr0 &= svm_xcr0_mask; 1623 cpudata->gxcr0 |= XCR0_X87; 1624 } 1625 } 1626 1627 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1628 if (flags & NVMM_X64_STATE_DRS) { 1629 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1630 1631 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1632 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1633 } 1634 1635 if (flags & NVMM_X64_STATE_MSRS) { 1636 /* 1637 * EFER_SVME is mandatory. 1638 */ 1639 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1640 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1641 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1642 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1643 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1644 vmcb->state.kernelgsbase = 1645 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1646 vmcb->state.sysenter_cs = 1647 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1648 vmcb->state.sysenter_esp = 1649 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1650 vmcb->state.sysenter_eip = 1651 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1652 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1653 1654 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 1655 cpudata->gtsc_want_update = true; 1656 } 1657 1658 if (flags & NVMM_X64_STATE_MISC) { 1659 if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { 1660 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1661 } else { 1662 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1663 } 1664 1665 if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { 1666 svm_event_waitexit_enable(vcpu, false); 1667 } else { 1668 svm_event_waitexit_disable(vcpu, false); 1669 } 1670 1671 if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { 1672 svm_event_waitexit_enable(vcpu, true); 1673 } else { 1674 svm_event_waitexit_disable(vcpu, true); 1675 } 1676 } 1677 1678 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1679 if (flags & NVMM_X64_STATE_FPU) { 1680 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1681 sizeof(state->fpu)); 1682 1683 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1684 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1685 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1686 1687 if (svm_xcr0_mask != 0) { 1688 /* Reset XSTATE_BV, to force a reload. */ 1689 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1690 } 1691 } 1692 1693 svm_vmcb_cache_update(vmcb, flags); 1694 } 1695 1696 static void 1697 svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1698 { 1699 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1700 struct svm_cpudata *cpudata = vcpu->cpudata; 1701 struct vmcb *vmcb = cpudata->vmcb; 1702 1703 if (flags & NVMM_X64_STATE_SEGS) { 1704 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1705 &vmcb->state.cs); 1706 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1707 &vmcb->state.ds); 1708 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1709 &vmcb->state.es); 1710 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1711 &vmcb->state.fs); 1712 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1713 &vmcb->state.gs); 1714 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1715 &vmcb->state.ss); 1716 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1717 &vmcb->state.gdt); 1718 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1719 &vmcb->state.idt); 1720 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1721 &vmcb->state.ldt); 1722 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1723 &vmcb->state.tr); 1724 1725 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1726 } 1727 1728 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1729 if (flags & NVMM_X64_STATE_GPRS) { 1730 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1731 1732 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1733 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1734 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1735 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1736 } 1737 1738 if (flags & NVMM_X64_STATE_CRS) { 1739 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1740 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1741 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1742 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1743 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1744 VMCB_CTRL_V_TPR); 1745 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1746 } 1747 1748 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1749 if (flags & NVMM_X64_STATE_DRS) { 1750 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1751 1752 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1753 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1754 } 1755 1756 if (flags & NVMM_X64_STATE_MSRS) { 1757 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1758 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1759 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1760 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1761 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1762 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1763 vmcb->state.kernelgsbase; 1764 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1765 vmcb->state.sysenter_cs; 1766 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1767 vmcb->state.sysenter_esp; 1768 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1769 vmcb->state.sysenter_eip; 1770 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1771 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 1772 1773 /* Hide SVME. */ 1774 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1775 } 1776 1777 if (flags & NVMM_X64_STATE_MISC) { 1778 state->misc[NVMM_X64_MISC_INT_SHADOW] = 1779 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1780 state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = 1781 cpudata->int_window_exit; 1782 state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = 1783 cpudata->nmi_window_exit; 1784 } 1785 1786 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1787 if (flags & NVMM_X64_STATE_FPU) { 1788 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 1789 sizeof(state->fpu)); 1790 } 1791 } 1792 1793 /* -------------------------------------------------------------------------- */ 1794 1795 static void 1796 svm_asid_alloc(struct nvmm_cpu *vcpu) 1797 { 1798 struct svm_cpudata *cpudata = vcpu->cpudata; 1799 struct vmcb *vmcb = cpudata->vmcb; 1800 size_t i, oct, bit; 1801 1802 mutex_enter(&svm_asidlock); 1803 1804 for (i = 0; i < svm_maxasid; i++) { 1805 oct = i / 8; 1806 bit = i % 8; 1807 1808 if (svm_asidmap[oct] & __BIT(bit)) { 1809 continue; 1810 } 1811 1812 svm_asidmap[oct] |= __BIT(bit); 1813 vmcb->ctrl.guest_asid = i; 1814 mutex_exit(&svm_asidlock); 1815 return; 1816 } 1817 1818 /* 1819 * No free ASID. Use the last one, which is shared and requires 1820 * special TLB handling. 1821 */ 1822 cpudata->shared_asid = true; 1823 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1824 mutex_exit(&svm_asidlock); 1825 } 1826 1827 static void 1828 svm_asid_free(struct nvmm_cpu *vcpu) 1829 { 1830 struct svm_cpudata *cpudata = vcpu->cpudata; 1831 struct vmcb *vmcb = cpudata->vmcb; 1832 size_t oct, bit; 1833 1834 if (cpudata->shared_asid) { 1835 return; 1836 } 1837 1838 oct = vmcb->ctrl.guest_asid / 8; 1839 bit = vmcb->ctrl.guest_asid % 8; 1840 1841 mutex_enter(&svm_asidlock); 1842 svm_asidmap[oct] &= ~__BIT(bit); 1843 mutex_exit(&svm_asidlock); 1844 } 1845 1846 static void 1847 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1848 { 1849 struct svm_cpudata *cpudata = vcpu->cpudata; 1850 struct vmcb *vmcb = cpudata->vmcb; 1851 1852 /* Allow reads/writes of Control Registers. */ 1853 vmcb->ctrl.intercept_cr = 0; 1854 1855 /* Allow reads/writes of Debug Registers. */ 1856 vmcb->ctrl.intercept_dr = 0; 1857 1858 /* Allow exceptions 0 to 31. */ 1859 vmcb->ctrl.intercept_vec = 0; 1860 1861 /* 1862 * Allow: 1863 * - SMI [smm interrupts] 1864 * - VINTR [virtual interrupts] 1865 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1866 * - RIDTR [reads of IDTR] 1867 * - RGDTR [reads of GDTR] 1868 * - RLDTR [reads of LDTR] 1869 * - RTR [reads of TR] 1870 * - WIDTR [writes of IDTR] 1871 * - WGDTR [writes of GDTR] 1872 * - WLDTR [writes of LDTR] 1873 * - WTR [writes of TR] 1874 * - RDTSC [rdtsc instruction] 1875 * - PUSHF [pushf instruction] 1876 * - POPF [popf instruction] 1877 * - IRET [iret instruction] 1878 * - INTN [int $n instructions] 1879 * - INVD [invd instruction] 1880 * - PAUSE [pause instruction] 1881 * - INVLPG [invplg instruction] 1882 * - TASKSW [task switches] 1883 * 1884 * Intercept the rest below. 1885 */ 1886 vmcb->ctrl.intercept_misc1 = 1887 VMCB_CTRL_INTERCEPT_INTR | 1888 VMCB_CTRL_INTERCEPT_NMI | 1889 VMCB_CTRL_INTERCEPT_INIT | 1890 VMCB_CTRL_INTERCEPT_RDPMC | 1891 VMCB_CTRL_INTERCEPT_CPUID | 1892 VMCB_CTRL_INTERCEPT_RSM | 1893 VMCB_CTRL_INTERCEPT_HLT | 1894 VMCB_CTRL_INTERCEPT_INVLPGA | 1895 VMCB_CTRL_INTERCEPT_IOIO_PROT | 1896 VMCB_CTRL_INTERCEPT_MSR_PROT | 1897 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 1898 VMCB_CTRL_INTERCEPT_SHUTDOWN; 1899 1900 /* 1901 * Allow: 1902 * - ICEBP [icebp instruction] 1903 * - WBINVD [wbinvd instruction] 1904 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 1905 * 1906 * Intercept the rest below. 1907 */ 1908 vmcb->ctrl.intercept_misc2 = 1909 VMCB_CTRL_INTERCEPT_VMRUN | 1910 VMCB_CTRL_INTERCEPT_VMMCALL | 1911 VMCB_CTRL_INTERCEPT_VMLOAD | 1912 VMCB_CTRL_INTERCEPT_VMSAVE | 1913 VMCB_CTRL_INTERCEPT_STGI | 1914 VMCB_CTRL_INTERCEPT_CLGI | 1915 VMCB_CTRL_INTERCEPT_SKINIT | 1916 VMCB_CTRL_INTERCEPT_RDTSCP | 1917 VMCB_CTRL_INTERCEPT_MONITOR | 1918 VMCB_CTRL_INTERCEPT_MWAIT | 1919 VMCB_CTRL_INTERCEPT_XSETBV; 1920 1921 /* Intercept all I/O accesses. */ 1922 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 1923 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 1924 1925 /* Allow direct access to certain MSRs. */ 1926 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 1927 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 1928 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 1929 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 1930 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 1931 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 1932 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 1933 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 1934 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 1935 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 1936 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 1937 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 1938 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 1939 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 1940 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 1941 1942 /* Generate ASID. */ 1943 svm_asid_alloc(vcpu); 1944 1945 /* Virtual TPR. */ 1946 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 1947 1948 /* Enable Nested Paging. */ 1949 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 1950 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 1951 1952 /* Init XSAVE header. */ 1953 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1954 cpudata->gfpu.xsh_xcomp_bv = 0; 1955 1956 /* These MSRs are static. */ 1957 cpudata->star = rdmsr(MSR_STAR); 1958 cpudata->lstar = rdmsr(MSR_LSTAR); 1959 cpudata->cstar = rdmsr(MSR_CSTAR); 1960 cpudata->sfmask = rdmsr(MSR_SFMASK); 1961 1962 /* Install the RESET state. */ 1963 svm_vcpu_setstate(vcpu, &nvmm_x86_reset_state, NVMM_X64_STATE_ALL); 1964 } 1965 1966 static int 1967 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1968 { 1969 struct svm_cpudata *cpudata; 1970 int error; 1971 1972 /* Allocate the SVM cpudata. */ 1973 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 1974 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 1975 UVM_KMF_WIRED|UVM_KMF_ZERO); 1976 vcpu->cpudata = cpudata; 1977 1978 /* VMCB */ 1979 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 1980 VMCB_NPAGES); 1981 if (error) 1982 goto error; 1983 1984 /* I/O Bitmap */ 1985 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 1986 IOBM_NPAGES); 1987 if (error) 1988 goto error; 1989 1990 /* MSR Bitmap */ 1991 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 1992 MSRBM_NPAGES); 1993 if (error) 1994 goto error; 1995 1996 /* Init the VCPU info. */ 1997 svm_vcpu_init(mach, vcpu); 1998 1999 return 0; 2000 2001 error: 2002 if (cpudata->vmcb_pa) { 2003 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2004 VMCB_NPAGES); 2005 } 2006 if (cpudata->iobm_pa) { 2007 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2008 IOBM_NPAGES); 2009 } 2010 if (cpudata->msrbm_pa) { 2011 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2012 MSRBM_NPAGES); 2013 } 2014 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2015 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2016 return error; 2017 } 2018 2019 static void 2020 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2021 { 2022 struct svm_cpudata *cpudata = vcpu->cpudata; 2023 2024 svm_asid_free(vcpu); 2025 2026 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2027 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2028 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2029 2030 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2031 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2032 } 2033 2034 /* -------------------------------------------------------------------------- */ 2035 2036 static void 2037 svm_tlb_flush(struct pmap *pm) 2038 { 2039 struct nvmm_machine *mach = pm->pm_data; 2040 struct svm_machdata *machdata = mach->machdata; 2041 2042 atomic_inc_64(&machdata->mach_htlb_gen); 2043 2044 /* Generates IPIs, which cause #VMEXITs. */ 2045 pmap_tlb_shootdown(pmap_kernel(), -1, PG_G, TLBSHOOT_UPDATE); 2046 } 2047 2048 static void 2049 svm_machine_create(struct nvmm_machine *mach) 2050 { 2051 struct svm_machdata *machdata; 2052 2053 /* Fill in pmap info. */ 2054 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2055 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2056 2057 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2058 mach->machdata = machdata; 2059 2060 /* Start with an hTLB flush everywhere. */ 2061 machdata->mach_htlb_gen = 1; 2062 } 2063 2064 static void 2065 svm_machine_destroy(struct nvmm_machine *mach) 2066 { 2067 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2068 } 2069 2070 static int 2071 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2072 { 2073 struct nvmm_x86_conf_cpuid *cpuid = data; 2074 struct svm_machdata *machdata = (struct svm_machdata *)mach->machdata; 2075 size_t i; 2076 2077 if (__predict_false(op != NVMM_X86_CONF_CPUID)) { 2078 return EINVAL; 2079 } 2080 2081 if (__predict_false((cpuid->set.eax & cpuid->del.eax) || 2082 (cpuid->set.ebx & cpuid->del.ebx) || 2083 (cpuid->set.ecx & cpuid->del.ecx) || 2084 (cpuid->set.edx & cpuid->del.edx))) { 2085 return EINVAL; 2086 } 2087 2088 /* If already here, replace. */ 2089 for (i = 0; i < SVM_NCPUIDS; i++) { 2090 if (!machdata->cpuidpresent[i]) { 2091 continue; 2092 } 2093 if (machdata->cpuid[i].leaf == cpuid->leaf) { 2094 memcpy(&machdata->cpuid[i], cpuid, 2095 sizeof(struct nvmm_x86_conf_cpuid)); 2096 return 0; 2097 } 2098 } 2099 2100 /* Not here, insert. */ 2101 for (i = 0; i < SVM_NCPUIDS; i++) { 2102 if (!machdata->cpuidpresent[i]) { 2103 machdata->cpuidpresent[i] = true; 2104 memcpy(&machdata->cpuid[i], cpuid, 2105 sizeof(struct nvmm_x86_conf_cpuid)); 2106 return 0; 2107 } 2108 } 2109 2110 return ENOBUFS; 2111 } 2112 2113 /* -------------------------------------------------------------------------- */ 2114 2115 static bool 2116 svm_ident(void) 2117 { 2118 u_int descs[4]; 2119 uint64_t msr; 2120 2121 if (cpu_vendor != CPUVENDOR_AMD) { 2122 return false; 2123 } 2124 if (!(cpu_feature[3] & CPUID_SVM)) { 2125 return false; 2126 } 2127 2128 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2129 return false; 2130 } 2131 x86_cpuid(0x8000000a, descs); 2132 2133 /* Want Nested Paging. */ 2134 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2135 return false; 2136 } 2137 2138 /* Want nRIP. */ 2139 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2140 return false; 2141 } 2142 2143 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2144 2145 msr = rdmsr(MSR_VMCR); 2146 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2147 return false; 2148 } 2149 2150 return true; 2151 } 2152 2153 static void 2154 svm_init_asid(uint32_t maxasid) 2155 { 2156 size_t i, j, allocsz; 2157 2158 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2159 2160 /* Arbitrarily limit. */ 2161 maxasid = uimin(maxasid, 8192); 2162 2163 svm_maxasid = maxasid; 2164 allocsz = roundup(maxasid, 8) / 8; 2165 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2166 2167 /* ASID 0 is reserved for the host. */ 2168 svm_asidmap[0] |= __BIT(0); 2169 2170 /* ASID n-1 is special, we share it. */ 2171 i = (maxasid - 1) / 8; 2172 j = (maxasid - 1) % 8; 2173 svm_asidmap[i] |= __BIT(j); 2174 } 2175 2176 static void 2177 svm_change_cpu(void *arg1, void *arg2) 2178 { 2179 bool enable = (bool)arg1; 2180 uint64_t msr; 2181 2182 msr = rdmsr(MSR_VMCR); 2183 if (msr & VMCR_SVMED) { 2184 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2185 } 2186 2187 if (!enable) { 2188 wrmsr(MSR_VM_HSAVE_PA, 0); 2189 } 2190 2191 msr = rdmsr(MSR_EFER); 2192 if (enable) { 2193 msr |= EFER_SVME; 2194 } else { 2195 msr &= ~EFER_SVME; 2196 } 2197 wrmsr(MSR_EFER, msr); 2198 2199 if (enable) { 2200 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2201 } 2202 } 2203 2204 static void 2205 svm_init(void) 2206 { 2207 CPU_INFO_ITERATOR cii; 2208 struct cpu_info *ci; 2209 struct vm_page *pg; 2210 u_int descs[4]; 2211 uint64_t xc; 2212 2213 x86_cpuid(0x8000000a, descs); 2214 2215 /* The guest TLB flush command. */ 2216 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2217 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2218 } else { 2219 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2220 } 2221 2222 /* Init the ASID. */ 2223 svm_init_asid(descs[1]); 2224 2225 /* Init the XCR0 mask. */ 2226 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2227 2228 memset(hsave, 0, sizeof(hsave)); 2229 for (CPU_INFO_FOREACH(cii, ci)) { 2230 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2231 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2232 } 2233 2234 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2235 xc_wait(xc); 2236 } 2237 2238 static void 2239 svm_fini_asid(void) 2240 { 2241 size_t allocsz; 2242 2243 allocsz = roundup(svm_maxasid, 8) / 8; 2244 kmem_free(svm_asidmap, allocsz); 2245 2246 mutex_destroy(&svm_asidlock); 2247 } 2248 2249 static void 2250 svm_fini(void) 2251 { 2252 uint64_t xc; 2253 size_t i; 2254 2255 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2256 xc_wait(xc); 2257 2258 for (i = 0; i < MAXCPUS; i++) { 2259 if (hsave[i].pa != 0) 2260 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2261 } 2262 2263 svm_fini_asid(); 2264 } 2265 2266 static void 2267 svm_capability(struct nvmm_capability *cap) 2268 { 2269 cap->u.x86.xcr0_mask = svm_xcr0_mask; 2270 cap->u.x86.mxcsr_mask = x86_fpu_mxcsr_mask; 2271 cap->u.x86.conf_cpuid_maxops = SVM_NCPUIDS; 2272 } 2273 2274 const struct nvmm_impl nvmm_x86_svm = { 2275 .ident = svm_ident, 2276 .init = svm_init, 2277 .fini = svm_fini, 2278 .capability = svm_capability, 2279 .conf_max = NVMM_X86_NCONF, 2280 .conf_sizes = svm_conf_sizes, 2281 .state_size = sizeof(struct nvmm_x64_state), 2282 .machine_create = svm_machine_create, 2283 .machine_destroy = svm_machine_destroy, 2284 .machine_configure = svm_machine_configure, 2285 .vcpu_create = svm_vcpu_create, 2286 .vcpu_destroy = svm_vcpu_destroy, 2287 .vcpu_setstate = svm_vcpu_setstate, 2288 .vcpu_getstate = svm_vcpu_getstate, 2289 .vcpu_inject = svm_vcpu_inject, 2290 .vcpu_run = svm_vcpu_run 2291 }; 2292
Indexes created Wed Oct 01 13:09:50 GMT 2025