nvmm_x86_svm.c revision 1.10
1 /* $NetBSD: nvmm_x86_svm.c,v 1.10 2019/01/06 16:10:51 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.10 2019/01/06 16:10:51 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 42 #include <uvm/uvm.h> 43 #include <uvm/uvm_page.h> 44 45 #include <x86/cputypes.h> 46 #include <x86/specialreg.h> 47 #include <x86/pmap.h> 48 #include <x86/dbregs.h> 49 #include <machine/cpuvar.h> 50 51 #include <dev/nvmm/nvmm.h> 52 #include <dev/nvmm/nvmm_internal.h> 53 #include <dev/nvmm/x86/nvmm_x86.h> 54 55 int svm_vmrun(paddr_t, uint64_t *); 56 57 #define MSR_VM_HSAVE_PA 0xC0010117 58 59 /* -------------------------------------------------------------------------- */ 60 61 #define VMCB_EXITCODE_CR0_READ 0x0000 62 #define VMCB_EXITCODE_CR1_READ 0x0001 63 #define VMCB_EXITCODE_CR2_READ 0x0002 64 #define VMCB_EXITCODE_CR3_READ 0x0003 65 #define VMCB_EXITCODE_CR4_READ 0x0004 66 #define VMCB_EXITCODE_CR5_READ 0x0005 67 #define VMCB_EXITCODE_CR6_READ 0x0006 68 #define VMCB_EXITCODE_CR7_READ 0x0007 69 #define VMCB_EXITCODE_CR8_READ 0x0008 70 #define VMCB_EXITCODE_CR9_READ 0x0009 71 #define VMCB_EXITCODE_CR10_READ 0x000A 72 #define VMCB_EXITCODE_CR11_READ 0x000B 73 #define VMCB_EXITCODE_CR12_READ 0x000C 74 #define VMCB_EXITCODE_CR13_READ 0x000D 75 #define VMCB_EXITCODE_CR14_READ 0x000E 76 #define VMCB_EXITCODE_CR15_READ 0x000F 77 #define VMCB_EXITCODE_CR0_WRITE 0x0010 78 #define VMCB_EXITCODE_CR1_WRITE 0x0011 79 #define VMCB_EXITCODE_CR2_WRITE 0x0012 80 #define VMCB_EXITCODE_CR3_WRITE 0x0013 81 #define VMCB_EXITCODE_CR4_WRITE 0x0014 82 #define VMCB_EXITCODE_CR5_WRITE 0x0015 83 #define VMCB_EXITCODE_CR6_WRITE 0x0016 84 #define VMCB_EXITCODE_CR7_WRITE 0x0017 85 #define VMCB_EXITCODE_CR8_WRITE 0x0018 86 #define VMCB_EXITCODE_CR9_WRITE 0x0019 87 #define VMCB_EXITCODE_CR10_WRITE 0x001A 88 #define VMCB_EXITCODE_CR11_WRITE 0x001B 89 #define VMCB_EXITCODE_CR12_WRITE 0x001C 90 #define VMCB_EXITCODE_CR13_WRITE 0x001D 91 #define VMCB_EXITCODE_CR14_WRITE 0x001E 92 #define VMCB_EXITCODE_CR15_WRITE 0x001F 93 #define VMCB_EXITCODE_DR0_READ 0x0020 94 #define VMCB_EXITCODE_DR1_READ 0x0021 95 #define VMCB_EXITCODE_DR2_READ 0x0022 96 #define VMCB_EXITCODE_DR3_READ 0x0023 97 #define VMCB_EXITCODE_DR4_READ 0x0024 98 #define VMCB_EXITCODE_DR5_READ 0x0025 99 #define VMCB_EXITCODE_DR6_READ 0x0026 100 #define VMCB_EXITCODE_DR7_READ 0x0027 101 #define VMCB_EXITCODE_DR8_READ 0x0028 102 #define VMCB_EXITCODE_DR9_READ 0x0029 103 #define VMCB_EXITCODE_DR10_READ 0x002A 104 #define VMCB_EXITCODE_DR11_READ 0x002B 105 #define VMCB_EXITCODE_DR12_READ 0x002C 106 #define VMCB_EXITCODE_DR13_READ 0x002D 107 #define VMCB_EXITCODE_DR14_READ 0x002E 108 #define VMCB_EXITCODE_DR15_READ 0x002F 109 #define VMCB_EXITCODE_DR0_WRITE 0x0030 110 #define VMCB_EXITCODE_DR1_WRITE 0x0031 111 #define VMCB_EXITCODE_DR2_WRITE 0x0032 112 #define VMCB_EXITCODE_DR3_WRITE 0x0033 113 #define VMCB_EXITCODE_DR4_WRITE 0x0034 114 #define VMCB_EXITCODE_DR5_WRITE 0x0035 115 #define VMCB_EXITCODE_DR6_WRITE 0x0036 116 #define VMCB_EXITCODE_DR7_WRITE 0x0037 117 #define VMCB_EXITCODE_DR8_WRITE 0x0038 118 #define VMCB_EXITCODE_DR9_WRITE 0x0039 119 #define VMCB_EXITCODE_DR10_WRITE 0x003A 120 #define VMCB_EXITCODE_DR11_WRITE 0x003B 121 #define VMCB_EXITCODE_DR12_WRITE 0x003C 122 #define VMCB_EXITCODE_DR13_WRITE 0x003D 123 #define VMCB_EXITCODE_DR14_WRITE 0x003E 124 #define VMCB_EXITCODE_DR15_WRITE 0x003F 125 #define VMCB_EXITCODE_EXCP0 0x0040 126 #define VMCB_EXITCODE_EXCP1 0x0041 127 #define VMCB_EXITCODE_EXCP2 0x0042 128 #define VMCB_EXITCODE_EXCP3 0x0043 129 #define VMCB_EXITCODE_EXCP4 0x0044 130 #define VMCB_EXITCODE_EXCP5 0x0045 131 #define VMCB_EXITCODE_EXCP6 0x0046 132 #define VMCB_EXITCODE_EXCP7 0x0047 133 #define VMCB_EXITCODE_EXCP8 0x0048 134 #define VMCB_EXITCODE_EXCP9 0x0049 135 #define VMCB_EXITCODE_EXCP10 0x004A 136 #define VMCB_EXITCODE_EXCP11 0x004B 137 #define VMCB_EXITCODE_EXCP12 0x004C 138 #define VMCB_EXITCODE_EXCP13 0x004D 139 #define VMCB_EXITCODE_EXCP14 0x004E 140 #define VMCB_EXITCODE_EXCP15 0x004F 141 #define VMCB_EXITCODE_EXCP16 0x0050 142 #define VMCB_EXITCODE_EXCP17 0x0051 143 #define VMCB_EXITCODE_EXCP18 0x0052 144 #define VMCB_EXITCODE_EXCP19 0x0053 145 #define VMCB_EXITCODE_EXCP20 0x0054 146 #define VMCB_EXITCODE_EXCP21 0x0055 147 #define VMCB_EXITCODE_EXCP22 0x0056 148 #define VMCB_EXITCODE_EXCP23 0x0057 149 #define VMCB_EXITCODE_EXCP24 0x0058 150 #define VMCB_EXITCODE_EXCP25 0x0059 151 #define VMCB_EXITCODE_EXCP26 0x005A 152 #define VMCB_EXITCODE_EXCP27 0x005B 153 #define VMCB_EXITCODE_EXCP28 0x005C 154 #define VMCB_EXITCODE_EXCP29 0x005D 155 #define VMCB_EXITCODE_EXCP30 0x005E 156 #define VMCB_EXITCODE_EXCP31 0x005F 157 #define VMCB_EXITCODE_INTR 0x0060 158 #define VMCB_EXITCODE_NMI 0x0061 159 #define VMCB_EXITCODE_SMI 0x0062 160 #define VMCB_EXITCODE_INIT 0x0063 161 #define VMCB_EXITCODE_VINTR 0x0064 162 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 163 #define VMCB_EXITCODE_IDTR_READ 0x0066 164 #define VMCB_EXITCODE_GDTR_READ 0x0067 165 #define VMCB_EXITCODE_LDTR_READ 0x0068 166 #define VMCB_EXITCODE_TR_READ 0x0069 167 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 168 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 169 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 170 #define VMCB_EXITCODE_TR_WRITE 0x006D 171 #define VMCB_EXITCODE_RDTSC 0x006E 172 #define VMCB_EXITCODE_RDPMC 0x006F 173 #define VMCB_EXITCODE_PUSHF 0x0070 174 #define VMCB_EXITCODE_POPF 0x0071 175 #define VMCB_EXITCODE_CPUID 0x0072 176 #define VMCB_EXITCODE_RSM 0x0073 177 #define VMCB_EXITCODE_IRET 0x0074 178 #define VMCB_EXITCODE_SWINT 0x0075 179 #define VMCB_EXITCODE_INVD 0x0076 180 #define VMCB_EXITCODE_PAUSE 0x0077 181 #define VMCB_EXITCODE_HLT 0x0078 182 #define VMCB_EXITCODE_INVLPG 0x0079 183 #define VMCB_EXITCODE_INVLPGA 0x007A 184 #define VMCB_EXITCODE_IOIO 0x007B 185 #define VMCB_EXITCODE_MSR 0x007C 186 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 187 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 188 #define VMCB_EXITCODE_SHUTDOWN 0x007F 189 #define VMCB_EXITCODE_VMRUN 0x0080 190 #define VMCB_EXITCODE_VMMCALL 0x0081 191 #define VMCB_EXITCODE_VMLOAD 0x0082 192 #define VMCB_EXITCODE_VMSAVE 0x0083 193 #define VMCB_EXITCODE_STGI 0x0084 194 #define VMCB_EXITCODE_CLGI 0x0085 195 #define VMCB_EXITCODE_SKINIT 0x0086 196 #define VMCB_EXITCODE_RDTSCP 0x0087 197 #define VMCB_EXITCODE_ICEBP 0x0088 198 #define VMCB_EXITCODE_WBINVD 0x0089 199 #define VMCB_EXITCODE_MONITOR 0x008A 200 #define VMCB_EXITCODE_MWAIT 0x008B 201 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 202 #define VMCB_EXITCODE_XSETBV 0x008D 203 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 204 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 205 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 206 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 207 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 208 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 209 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 210 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 211 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 212 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 213 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 214 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 215 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 216 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 217 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 218 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 219 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 220 #define VMCB_EXITCODE_NPF 0x0400 221 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 222 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 223 #define VMCB_EXITCODE_VMGEXIT 0x0403 224 #define VMCB_EXITCODE_INVALID -1 225 226 /* -------------------------------------------------------------------------- */ 227 228 struct vmcb_ctrl { 229 uint32_t intercept_cr; 230 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 231 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 232 233 uint32_t intercept_dr; 234 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 235 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 236 237 uint32_t intercept_vec; 238 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 239 240 uint32_t intercept_misc1; 241 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 242 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 243 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 244 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 245 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 246 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 247 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 248 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 249 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 250 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 251 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 252 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 253 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 254 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 255 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 256 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 257 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 258 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 259 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 260 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 261 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 262 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 263 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 264 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 265 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 266 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 267 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 268 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 269 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 270 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 271 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 272 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 273 274 uint32_t intercept_misc2; 275 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 276 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 277 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 278 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 279 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 280 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 281 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 282 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 283 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 284 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 285 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 286 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(12) 287 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 288 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 289 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 290 291 uint8_t rsvd1[40]; 292 uint16_t pause_filt_thresh; 293 uint16_t pause_filt_cnt; 294 uint64_t iopm_base_pa; 295 uint64_t msrpm_base_pa; 296 uint64_t tsc_offset; 297 uint32_t guest_asid; 298 299 uint32_t tlb_ctrl; 300 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 301 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 302 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 303 304 uint64_t v; 305 #define VMCB_CTRL_V_TPR __BITS(7,0) 306 #define VMCB_CTRL_V_IRQ __BIT(8) 307 #define VMCB_CTRL_V_VGIF __BIT(9) 308 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 309 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 310 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 311 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 312 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 313 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 314 315 uint64_t intr; 316 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 317 #define VMCB_CTRL_GUEST_INTR_MASK __BIT(1) 318 319 uint64_t exitcode; 320 uint64_t exitinfo1; 321 uint64_t exitinfo2; 322 323 uint64_t exitintinfo; 324 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 325 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 326 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 327 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 328 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 329 330 uint64_t enable1; 331 #define VMCB_CTRL_ENABLE_NP __BIT(0) 332 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 333 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 334 335 uint64_t avic; 336 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 337 338 uint64_t ghcb; 339 340 uint64_t eventinj; 341 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 342 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 343 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 344 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 345 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 346 347 uint64_t n_cr3; 348 349 uint64_t enable2; 350 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 351 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 352 353 uint32_t vmcb_clean; 354 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 355 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 356 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 357 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 358 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 359 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 360 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 361 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 362 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 363 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 364 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 365 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 366 367 uint32_t rsvd2; 368 uint64_t nrip; 369 uint8_t inst_len; 370 uint8_t inst_bytes[15]; 371 uint8_t pad[800]; 372 } __packed; 373 374 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 375 376 struct vmcb_segment { 377 uint16_t selector; 378 uint16_t attrib; /* hidden */ 379 uint32_t limit; /* hidden */ 380 uint64_t base; /* hidden */ 381 } __packed; 382 383 CTASSERT(sizeof(struct vmcb_segment) == 16); 384 385 struct vmcb_state { 386 struct vmcb_segment es; 387 struct vmcb_segment cs; 388 struct vmcb_segment ss; 389 struct vmcb_segment ds; 390 struct vmcb_segment fs; 391 struct vmcb_segment gs; 392 struct vmcb_segment gdt; 393 struct vmcb_segment ldt; 394 struct vmcb_segment idt; 395 struct vmcb_segment tr; 396 uint8_t rsvd1[43]; 397 uint8_t cpl; 398 uint8_t rsvd2[4]; 399 uint64_t efer; 400 uint8_t rsvd3[112]; 401 uint64_t cr4; 402 uint64_t cr3; 403 uint64_t cr0; 404 uint64_t dr7; 405 uint64_t dr6; 406 uint64_t rflags; 407 uint64_t rip; 408 uint8_t rsvd4[88]; 409 uint64_t rsp; 410 uint8_t rsvd5[24]; 411 uint64_t rax; 412 uint64_t star; 413 uint64_t lstar; 414 uint64_t cstar; 415 uint64_t sfmask; 416 uint64_t kernelgsbase; 417 uint64_t sysenter_cs; 418 uint64_t sysenter_esp; 419 uint64_t sysenter_eip; 420 uint64_t cr2; 421 uint8_t rsvd6[32]; 422 uint64_t g_pat; 423 uint64_t dbgctl; 424 uint64_t br_from; 425 uint64_t br_to; 426 uint64_t int_from; 427 uint64_t int_to; 428 uint8_t pad[2408]; 429 } __packed; 430 431 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 432 433 struct vmcb { 434 struct vmcb_ctrl ctrl; 435 struct vmcb_state state; 436 } __packed; 437 438 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 439 CTASSERT(offsetof(struct vmcb, state) == 0x400); 440 441 /* -------------------------------------------------------------------------- */ 442 443 struct svm_hsave { 444 paddr_t pa; 445 }; 446 447 static struct svm_hsave hsave[MAXCPUS]; 448 449 static uint8_t *svm_asidmap __read_mostly; 450 static uint32_t svm_maxasid __read_mostly; 451 static kmutex_t svm_asidlock __cacheline_aligned; 452 453 static bool svm_decode_assist __read_mostly; 454 static uint32_t svm_ctrl_tlb_flush __read_mostly; 455 456 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 457 static uint64_t svm_xcr0_mask __read_mostly; 458 459 #define SVM_NCPUIDS 32 460 461 #define VMCB_NPAGES 1 462 463 #define MSRBM_NPAGES 2 464 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 465 466 #define IOBM_NPAGES 3 467 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 468 469 /* Does not include EFER_LMSLE. */ 470 #define EFER_VALID \ 471 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 472 473 #define EFER_TLB_FLUSH \ 474 (EFER_NXE|EFER_LMA|EFER_LME) 475 #define CR0_TLB_FLUSH \ 476 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 477 #define CR4_TLB_FLUSH \ 478 (CR4_PGE|CR4_PAE|CR4_PSE) 479 480 /* -------------------------------------------------------------------------- */ 481 482 struct svm_machdata { 483 bool cpuidpresent[SVM_NCPUIDS]; 484 struct nvmm_x86_conf_cpuid cpuid[SVM_NCPUIDS]; 485 }; 486 487 static const size_t svm_conf_sizes[NVMM_X86_NCONF] = { 488 [NVMM_X86_CONF_CPUID] = sizeof(struct nvmm_x86_conf_cpuid) 489 }; 490 491 struct svm_cpudata { 492 /* x64-specific */ 493 struct nvmm_x64_state state; 494 495 /* General */ 496 bool shared_asid; 497 bool tlb_want_flush; 498 499 /* VMCB */ 500 struct vmcb *vmcb; 501 paddr_t vmcb_pa; 502 503 /* I/O bitmap */ 504 uint8_t *iobm; 505 paddr_t iobm_pa; 506 507 /* MSR bitmap */ 508 uint8_t *msrbm; 509 paddr_t msrbm_pa; 510 511 /* Host state */ 512 uint64_t xcr0; 513 uint64_t star; 514 uint64_t lstar; 515 uint64_t cstar; 516 uint64_t sfmask; 517 uint64_t cr2; 518 bool ts_set; 519 struct xsave_header hfpu __aligned(16); 520 521 /* Event state */ 522 bool int_window_exit; 523 bool nmi_window_exit; 524 525 /* Guest state */ 526 uint64_t tsc_offset; 527 struct xsave_header gfpu __aligned(16); 528 }; 529 530 #define SVM_EVENT_TYPE_HW_INT 0 531 #define SVM_EVENT_TYPE_NMI 2 532 #define SVM_EVENT_TYPE_EXC 3 533 #define SVM_EVENT_TYPE_SW_INT 4 534 535 static void 536 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 537 { 538 struct svm_cpudata *cpudata = vcpu->cpudata; 539 struct vmcb *vmcb = cpudata->vmcb; 540 541 if (nmi) { 542 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 543 cpudata->nmi_window_exit = true; 544 } else { 545 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 546 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 547 cpudata->int_window_exit = true; 548 } 549 } 550 551 static void 552 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 553 { 554 struct svm_cpudata *cpudata = vcpu->cpudata; 555 struct vmcb *vmcb = cpudata->vmcb; 556 557 if (nmi) { 558 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 559 cpudata->nmi_window_exit = false; 560 } else { 561 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 562 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 563 cpudata->int_window_exit = false; 564 } 565 } 566 567 static inline int 568 svm_event_has_error(uint64_t vector) 569 { 570 switch (vector) { 571 case 8: /* #DF */ 572 case 10: /* #TS */ 573 case 11: /* #NP */ 574 case 12: /* #SS */ 575 case 13: /* #GP */ 576 case 14: /* #PF */ 577 case 17: /* #AC */ 578 case 30: /* #SX */ 579 return 1; 580 default: 581 return 0; 582 } 583 } 584 585 static int 586 svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 587 struct nvmm_event *event) 588 { 589 struct svm_cpudata *cpudata = vcpu->cpudata; 590 struct vmcb *vmcb = cpudata->vmcb; 591 int type = 0, err = 0; 592 593 if (event->vector >= 256) { 594 return EINVAL; 595 } 596 597 switch (event->type) { 598 case NVMM_EVENT_INTERRUPT_HW: 599 type = SVM_EVENT_TYPE_HW_INT; 600 if (event->vector == 2) { 601 type = SVM_EVENT_TYPE_NMI; 602 } 603 if (type == SVM_EVENT_TYPE_NMI) { 604 if (cpudata->nmi_window_exit) { 605 return EAGAIN; 606 } 607 svm_event_waitexit_enable(vcpu, true); 608 } else { 609 if (((vmcb->state.rflags & PSL_I) == 0) || 610 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0)) { 611 svm_event_waitexit_enable(vcpu, false); 612 return EAGAIN; 613 } 614 } 615 err = 0; 616 break; 617 case NVMM_EVENT_INTERRUPT_SW: 618 type = SVM_EVENT_TYPE_SW_INT; 619 err = 0; 620 break; 621 case NVMM_EVENT_EXCEPTION: 622 type = SVM_EVENT_TYPE_EXC; 623 if (event->vector == 2 || event->vector >= 32) 624 return EINVAL; 625 err = svm_event_has_error(event->vector); 626 break; 627 default: 628 return EINVAL; 629 } 630 631 vmcb->ctrl.eventinj = 632 __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) | 633 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 634 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 635 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 636 __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE); 637 638 return 0; 639 } 640 641 static void 642 svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 643 { 644 struct nvmm_event event; 645 int ret __diagused; 646 647 event.type = NVMM_EVENT_EXCEPTION; 648 event.vector = 6; 649 event.u.error = 0; 650 651 ret = svm_vcpu_inject(mach, vcpu, &event); 652 KASSERT(ret == 0); 653 } 654 655 static void 656 svm_inject_db(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 657 { 658 struct nvmm_event event; 659 int ret __diagused; 660 661 event.type = NVMM_EVENT_EXCEPTION; 662 event.vector = 1; 663 event.u.error = 0; 664 665 ret = svm_vcpu_inject(mach, vcpu, &event); 666 KASSERT(ret == 0); 667 } 668 669 static void 670 svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 671 { 672 struct nvmm_event event; 673 int ret __diagused; 674 675 event.type = NVMM_EVENT_EXCEPTION; 676 event.vector = 13; 677 event.u.error = 0; 678 679 ret = svm_vcpu_inject(mach, vcpu, &event); 680 KASSERT(ret == 0); 681 } 682 683 static void 684 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 685 { 686 struct svm_cpudata *cpudata = vcpu->cpudata; 687 struct nvmm_x64_state *state = &cpudata->state; 688 689 switch (eax) { 690 case 0x00000001: /* APIC number in RBX. The rest is tunable. */ 691 state->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 692 state->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 693 CPUID_LOCAL_APIC_ID); 694 break; 695 case 0x0000000D: /* FPU description. Not tunable. */ 696 if (ecx != 0 || svm_xcr0_mask == 0) { 697 break; 698 } 699 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 700 if (state->crs[NVMM_X64_CR_XCR0] & XCR0_SSE) { 701 state->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 702 } else { 703 state->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 704 } 705 state->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 706 state->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave); 707 state->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 708 break; 709 case 0x40000000: 710 memcpy(&state->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 711 memcpy(&state->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 712 memcpy(&state->gprs[NVMM_X64_GPR_RDX], " ___", 4); 713 break; 714 case 0x80000001: /* No SVM in ECX. The rest is tunable. */ 715 state->gprs[NVMM_X64_GPR_RCX] &= ~CPUID_SVM; 716 break; 717 default: 718 break; 719 } 720 } 721 722 static void 723 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 724 struct nvmm_exit *exit) 725 { 726 struct svm_machdata *machdata = mach->machdata; 727 struct svm_cpudata *cpudata = vcpu->cpudata; 728 struct nvmm_x64_state *state = &cpudata->state; 729 struct nvmm_x86_conf_cpuid *cpuid; 730 uint64_t eax, ecx; 731 u_int descs[4]; 732 size_t i; 733 734 eax = cpudata->vmcb->state.rax; 735 ecx = state->gprs[NVMM_X64_GPR_RCX]; 736 x86_cpuid2(eax, ecx, descs); 737 738 cpudata->vmcb->state.rax = descs[0]; 739 state->gprs[NVMM_X64_GPR_RBX] = descs[1]; 740 state->gprs[NVMM_X64_GPR_RCX] = descs[2]; 741 state->gprs[NVMM_X64_GPR_RDX] = descs[3]; 742 743 for (i = 0; i < SVM_NCPUIDS; i++) { 744 cpuid = &machdata->cpuid[i]; 745 if (!machdata->cpuidpresent[i]) { 746 continue; 747 } 748 if (cpuid->leaf != eax) { 749 continue; 750 } 751 752 /* del */ 753 cpudata->vmcb->state.rax &= ~cpuid->del.eax; 754 state->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->del.ebx; 755 state->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->del.ecx; 756 state->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->del.edx; 757 758 /* set */ 759 cpudata->vmcb->state.rax |= cpuid->set.eax; 760 state->gprs[NVMM_X64_GPR_RBX] |= cpuid->set.ebx; 761 state->gprs[NVMM_X64_GPR_RCX] |= cpuid->set.ecx; 762 state->gprs[NVMM_X64_GPR_RDX] |= cpuid->set.edx; 763 764 break; 765 } 766 767 /* Overwrite non-tunable leaves. */ 768 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 769 770 /* For now we omit DBREGS. */ 771 if (__predict_false(cpudata->vmcb->state.rflags & PSL_T)) { 772 svm_inject_db(mach, vcpu); 773 } 774 775 cpudata->vmcb->state.rip = cpudata->vmcb->ctrl.nrip; 776 exit->reason = NVMM_EXIT_NONE; 777 } 778 779 static void 780 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 781 struct nvmm_exit *exit) 782 { 783 struct svm_cpudata *cpudata = vcpu->cpudata; 784 785 exit->reason = NVMM_EXIT_HLT; 786 exit->u.hlt.npc = cpudata->vmcb->ctrl.nrip; 787 } 788 789 #define SVM_EXIT_IO_PORT __BITS(31,16) 790 #define SVM_EXIT_IO_SEG __BITS(12,10) 791 #define SVM_EXIT_IO_A64 __BIT(9) 792 #define SVM_EXIT_IO_A32 __BIT(8) 793 #define SVM_EXIT_IO_A16 __BIT(7) 794 #define SVM_EXIT_IO_SZ32 __BIT(6) 795 #define SVM_EXIT_IO_SZ16 __BIT(5) 796 #define SVM_EXIT_IO_SZ8 __BIT(4) 797 #define SVM_EXIT_IO_REP __BIT(3) 798 #define SVM_EXIT_IO_STR __BIT(2) 799 #define SVM_EXIT_IO_IN __BIT(0) 800 801 static const int seg_to_nvmm[] = { 802 [0] = NVMM_X64_SEG_ES, 803 [1] = NVMM_X64_SEG_CS, 804 [2] = NVMM_X64_SEG_SS, 805 [3] = NVMM_X64_SEG_DS, 806 [4] = NVMM_X64_SEG_FS, 807 [5] = NVMM_X64_SEG_GS 808 }; 809 810 static void 811 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 812 struct nvmm_exit *exit) 813 { 814 struct svm_cpudata *cpudata = vcpu->cpudata; 815 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 816 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 817 818 exit->reason = NVMM_EXIT_IO; 819 820 if (info & SVM_EXIT_IO_IN) { 821 exit->u.io.type = NVMM_EXIT_IO_IN; 822 } else { 823 exit->u.io.type = NVMM_EXIT_IO_OUT; 824 } 825 826 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 827 828 if (svm_decode_assist) { 829 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 830 exit->u.io.seg = seg_to_nvmm[__SHIFTOUT(info, SVM_EXIT_IO_SEG)]; 831 } else { 832 exit->u.io.seg = -1; 833 } 834 835 if (info & SVM_EXIT_IO_A64) { 836 exit->u.io.address_size = 8; 837 } else if (info & SVM_EXIT_IO_A32) { 838 exit->u.io.address_size = 4; 839 } else if (info & SVM_EXIT_IO_A16) { 840 exit->u.io.address_size = 2; 841 } 842 843 if (info & SVM_EXIT_IO_SZ32) { 844 exit->u.io.operand_size = 4; 845 } else if (info & SVM_EXIT_IO_SZ16) { 846 exit->u.io.operand_size = 2; 847 } else if (info & SVM_EXIT_IO_SZ8) { 848 exit->u.io.operand_size = 1; 849 } 850 851 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 852 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 853 exit->u.io.npc = nextpc; 854 } 855 856 static const uint64_t msr_ignore_list[] = { 857 0xc0010055, /* MSR_CMPHALT */ 858 MSR_DE_CFG, 859 MSR_IC_CFG, 860 MSR_UCODE_AMD_PATCHLEVEL 861 }; 862 863 static bool 864 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 865 struct nvmm_exit *exit) 866 { 867 struct svm_cpudata *cpudata = vcpu->cpudata; 868 struct nvmm_x64_state *state = &cpudata->state; 869 uint64_t val; 870 size_t i; 871 872 switch (exit->u.msr.type) { 873 case NVMM_EXIT_MSR_RDMSR: 874 if (exit->u.msr.msr == MSR_CR_PAT) { 875 val = cpudata->vmcb->state.g_pat; 876 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 877 state->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 878 goto handled; 879 } 880 if (exit->u.msr.msr == MSR_NB_CFG) { 881 val = NB_CFG_INITAPICCPUIDLO; 882 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 883 state->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 884 goto handled; 885 } 886 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 887 if (msr_ignore_list[i] != exit->u.msr.msr) 888 continue; 889 val = 0; 890 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 891 state->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 892 goto handled; 893 } 894 break; 895 case NVMM_EXIT_MSR_WRMSR: 896 if (exit->u.msr.msr == MSR_EFER) { 897 if (__predict_false(exit->u.msr.val & ~EFER_VALID)) { 898 svm_inject_gp(mach, vcpu); 899 goto handled; 900 } 901 if ((cpudata->vmcb->state.efer ^ exit->u.msr.val) & 902 EFER_TLB_FLUSH) { 903 cpudata->tlb_want_flush = true; 904 } 905 cpudata->vmcb->state.efer = exit->u.msr.val | EFER_SVME; 906 goto handled; 907 } 908 if (exit->u.msr.msr == MSR_CR_PAT) { 909 cpudata->vmcb->state.g_pat = exit->u.msr.val; 910 goto handled; 911 } 912 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 913 if (msr_ignore_list[i] != exit->u.msr.msr) 914 continue; 915 goto handled; 916 } 917 break; 918 } 919 920 return false; 921 922 handled: 923 cpudata->vmcb->state.rip = cpudata->vmcb->ctrl.nrip; 924 return true; 925 } 926 927 static void 928 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 929 struct nvmm_exit *exit) 930 { 931 struct svm_cpudata *cpudata = vcpu->cpudata; 932 struct nvmm_x64_state *state = &cpudata->state; 933 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 934 935 if (info == 0) { 936 exit->u.msr.type = NVMM_EXIT_MSR_RDMSR; 937 } else { 938 exit->u.msr.type = NVMM_EXIT_MSR_WRMSR; 939 } 940 941 exit->u.msr.msr = state->gprs[NVMM_X64_GPR_RCX]; 942 943 if (info == 1) { 944 uint64_t rdx, rax; 945 rdx = state->gprs[NVMM_X64_GPR_RDX]; 946 rax = cpudata->vmcb->state.rax; 947 exit->u.msr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 948 } else { 949 exit->u.msr.val = 0; 950 } 951 952 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 953 exit->reason = NVMM_EXIT_NONE; 954 return; 955 } 956 957 exit->reason = NVMM_EXIT_MSR; 958 exit->u.msr.npc = cpudata->vmcb->ctrl.nrip; 959 } 960 961 static void 962 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 963 struct nvmm_exit *exit) 964 { 965 struct svm_cpudata *cpudata = vcpu->cpudata; 966 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 967 int error; 968 969 error = uvm_fault(&mach->vm->vm_map, gpa, VM_PROT_ALL); 970 971 if (error) { 972 exit->reason = NVMM_EXIT_MEMORY; 973 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 974 exit->u.mem.perm = NVMM_EXIT_MEMORY_WRITE; 975 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 976 exit->u.mem.perm = NVMM_EXIT_MEMORY_EXEC; 977 else 978 exit->u.mem.perm = NVMM_EXIT_MEMORY_READ; 979 exit->u.mem.gpa = gpa; 980 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 981 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 982 sizeof(exit->u.mem.inst_bytes)); 983 exit->u.mem.npc = cpudata->vmcb->ctrl.nrip; 984 } else { 985 exit->reason = NVMM_EXIT_NONE; 986 } 987 } 988 989 static void 990 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 991 struct nvmm_exit *exit) 992 { 993 struct svm_cpudata *cpudata = vcpu->cpudata; 994 struct nvmm_x64_state *state = &cpudata->state; 995 struct vmcb *vmcb = cpudata->vmcb; 996 uint64_t val; 997 998 exit->reason = NVMM_EXIT_NONE; 999 1000 val = (state->gprs[NVMM_X64_GPR_RDX] << 32) | 1001 (vmcb->state.rax & 0xFFFFFFFF); 1002 1003 if (__predict_false(state->gprs[NVMM_X64_GPR_RCX] != 0)) { 1004 goto error; 1005 } else if (__predict_false(vmcb->state.cpl != 0)) { 1006 goto error; 1007 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1008 goto error; 1009 } else if (__predict_false((val & XCR0_X87) == 0)) { 1010 goto error; 1011 } 1012 1013 state->crs[NVMM_X64_CR_XCR0] = val; 1014 1015 cpudata->vmcb->state.rip = cpudata->vmcb->ctrl.nrip; 1016 return; 1017 1018 error: 1019 svm_inject_gp(mach, vcpu); 1020 } 1021 1022 static void 1023 svm_vmcb_cache_default(struct vmcb *vmcb) 1024 { 1025 vmcb->ctrl.vmcb_clean = 1026 VMCB_CTRL_VMCB_CLEAN_I | 1027 VMCB_CTRL_VMCB_CLEAN_IOPM | 1028 VMCB_CTRL_VMCB_CLEAN_ASID | 1029 VMCB_CTRL_VMCB_CLEAN_LBR | 1030 VMCB_CTRL_VMCB_CLEAN_AVIC; 1031 } 1032 1033 static void 1034 svm_vmcb_cache_flush(struct vmcb *vmcb) 1035 { 1036 vmcb->ctrl.vmcb_clean = 0; 1037 } 1038 1039 static void 1040 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1041 { 1042 struct svm_cpudata *cpudata = vcpu->cpudata; 1043 1044 if (x86_xsave_features != 0) { 1045 cpudata->xcr0 = rdxcr(0); 1046 wrxcr(0, cpudata->state.crs[NVMM_X64_CR_XCR0]); 1047 } 1048 1049 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1050 1051 fpu_area_save(&cpudata->hfpu); 1052 fpu_area_restore(&cpudata->gfpu); 1053 } 1054 1055 static void 1056 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1057 { 1058 struct svm_cpudata *cpudata = vcpu->cpudata; 1059 1060 fpu_area_save(&cpudata->gfpu); 1061 fpu_area_restore(&cpudata->hfpu); 1062 1063 if (cpudata->ts_set) { 1064 stts(); 1065 } 1066 1067 if (x86_xsave_features != 0) { 1068 cpudata->state.crs[NVMM_X64_CR_XCR0] = rdxcr(0); 1069 wrxcr(0, cpudata->xcr0); 1070 } 1071 } 1072 1073 static void 1074 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1075 { 1076 struct svm_cpudata *cpudata = vcpu->cpudata; 1077 struct nvmm_x64_state *state = &cpudata->state; 1078 1079 x86_dbregs_save(curlwp); 1080 1081 ldr0(state->drs[NVMM_X64_DR_DR0]); 1082 ldr1(state->drs[NVMM_X64_DR_DR1]); 1083 ldr2(state->drs[NVMM_X64_DR_DR2]); 1084 ldr3(state->drs[NVMM_X64_DR_DR3]); 1085 } 1086 1087 static void 1088 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1089 { 1090 struct svm_cpudata *cpudata = vcpu->cpudata; 1091 struct nvmm_x64_state *state = &cpudata->state; 1092 1093 state->drs[NVMM_X64_DR_DR0] = rdr0(); 1094 state->drs[NVMM_X64_DR_DR1] = rdr1(); 1095 state->drs[NVMM_X64_DR_DR2] = rdr2(); 1096 state->drs[NVMM_X64_DR_DR3] = rdr3(); 1097 1098 x86_dbregs_restore(curlwp); 1099 } 1100 1101 static void 1102 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1103 { 1104 struct svm_cpudata *cpudata = vcpu->cpudata; 1105 1106 /* Save the fixed Host MSRs. */ 1107 cpudata->star = rdmsr(MSR_STAR); 1108 cpudata->lstar = rdmsr(MSR_LSTAR); 1109 cpudata->cstar = rdmsr(MSR_CSTAR); 1110 cpudata->sfmask = rdmsr(MSR_SFMASK); 1111 1112 /* Save the Host CR2. */ 1113 cpudata->cr2 = rcr2(); 1114 } 1115 1116 static void 1117 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1118 { 1119 struct svm_cpudata *cpudata = vcpu->cpudata; 1120 1121 /* Restore the fixed Host MSRs. */ 1122 wrmsr(MSR_STAR, cpudata->star); 1123 wrmsr(MSR_LSTAR, cpudata->lstar); 1124 wrmsr(MSR_CSTAR, cpudata->cstar); 1125 wrmsr(MSR_SFMASK, cpudata->sfmask); 1126 1127 /* Restore the Host CR2. */ 1128 lcr2(cpudata->cr2); 1129 } 1130 1131 static int 1132 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1133 struct nvmm_exit *exit) 1134 { 1135 struct svm_cpudata *cpudata = vcpu->cpudata; 1136 struct vmcb *vmcb = cpudata->vmcb; 1137 bool tlb_need_flush = false; 1138 int hcpu, s; 1139 1140 kpreempt_disable(); 1141 hcpu = cpu_number(); 1142 1143 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1144 tlb_need_flush = true; 1145 } 1146 1147 if (cpudata->tlb_want_flush || tlb_need_flush) { 1148 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1149 } else { 1150 vmcb->ctrl.tlb_ctrl = 0; 1151 } 1152 1153 if (vcpu->hcpu_last != hcpu) { 1154 vmcb->ctrl.tsc_offset = cpudata->tsc_offset + 1155 curcpu()->ci_data.cpu_cc_skew; 1156 svm_vmcb_cache_flush(vmcb); 1157 } 1158 1159 svm_vcpu_guest_dbregs_enter(vcpu); 1160 svm_vcpu_guest_misc_enter(vcpu); 1161 1162 while (1) { 1163 s = splhigh(); 1164 svm_vcpu_guest_fpu_enter(vcpu); 1165 svm_vmrun(cpudata->vmcb_pa, cpudata->state.gprs); 1166 svm_vcpu_guest_fpu_leave(vcpu); 1167 splx(s); 1168 1169 svm_vmcb_cache_default(vmcb); 1170 1171 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1172 if (cpudata->tlb_want_flush) { 1173 cpudata->tlb_want_flush = false; 1174 } 1175 vcpu->hcpu_last = hcpu; 1176 } 1177 1178 switch (vmcb->ctrl.exitcode) { 1179 case VMCB_EXITCODE_INTR: 1180 case VMCB_EXITCODE_NMI: 1181 exit->reason = NVMM_EXIT_NONE; 1182 break; 1183 case VMCB_EXITCODE_VINTR: 1184 svm_event_waitexit_disable(vcpu, false); 1185 exit->reason = NVMM_EXIT_INT_READY; 1186 break; 1187 case VMCB_EXITCODE_IRET: 1188 svm_event_waitexit_disable(vcpu, true); 1189 exit->reason = NVMM_EXIT_NMI_READY; 1190 break; 1191 case VMCB_EXITCODE_CPUID: 1192 svm_exit_cpuid(mach, vcpu, exit); 1193 break; 1194 case VMCB_EXITCODE_HLT: 1195 svm_exit_hlt(mach, vcpu, exit); 1196 break; 1197 case VMCB_EXITCODE_IOIO: 1198 svm_exit_io(mach, vcpu, exit); 1199 break; 1200 case VMCB_EXITCODE_MSR: 1201 svm_exit_msr(mach, vcpu, exit); 1202 break; 1203 case VMCB_EXITCODE_SHUTDOWN: 1204 exit->reason = NVMM_EXIT_SHUTDOWN; 1205 break; 1206 case VMCB_EXITCODE_RDPMC: 1207 case VMCB_EXITCODE_RSM: 1208 case VMCB_EXITCODE_INVLPGA: 1209 case VMCB_EXITCODE_VMRUN: 1210 case VMCB_EXITCODE_VMMCALL: 1211 case VMCB_EXITCODE_VMLOAD: 1212 case VMCB_EXITCODE_VMSAVE: 1213 case VMCB_EXITCODE_STGI: 1214 case VMCB_EXITCODE_CLGI: 1215 case VMCB_EXITCODE_SKINIT: 1216 case VMCB_EXITCODE_RDTSCP: 1217 svm_inject_ud(mach, vcpu); 1218 exit->reason = NVMM_EXIT_NONE; 1219 break; 1220 case VMCB_EXITCODE_MONITOR: 1221 exit->reason = NVMM_EXIT_MONITOR; 1222 break; 1223 case VMCB_EXITCODE_MWAIT: 1224 exit->reason = NVMM_EXIT_MWAIT; 1225 break; 1226 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1227 exit->reason = NVMM_EXIT_MWAIT_COND; 1228 break; 1229 case VMCB_EXITCODE_XSETBV: 1230 svm_exit_xsetbv(mach, vcpu, exit); 1231 break; 1232 case VMCB_EXITCODE_NPF: 1233 svm_exit_npf(mach, vcpu, exit); 1234 break; 1235 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1236 default: 1237 exit->reason = NVMM_EXIT_INVALID; 1238 break; 1239 } 1240 1241 if (vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V) { 1242 printf("WAS PROCESSING!\n"); 1243 } 1244 1245 /* If no reason to return to userland, keep rolling. */ 1246 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1247 break; 1248 } 1249 if (curcpu()->ci_data.cpu_softints != 0) { 1250 break; 1251 } 1252 if (curlwp->l_flag & LW_USERRET) { 1253 break; 1254 } 1255 if (exit->reason != NVMM_EXIT_NONE) { 1256 break; 1257 } 1258 } 1259 1260 svm_vcpu_guest_misc_leave(vcpu); 1261 svm_vcpu_guest_dbregs_leave(vcpu); 1262 1263 kpreempt_enable(); 1264 1265 exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1266 VMCB_CTRL_V_TPR); 1267 exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags; 1268 1269 exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] = 1270 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1271 exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] = 1272 cpudata->int_window_exit; 1273 exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] = 1274 cpudata->nmi_window_exit; 1275 1276 return 0; 1277 } 1278 1279 /* -------------------------------------------------------------------------- */ 1280 1281 static int 1282 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1283 { 1284 struct pglist pglist; 1285 paddr_t _pa; 1286 vaddr_t _va; 1287 size_t i; 1288 int ret; 1289 1290 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1291 &pglist, 1, 0); 1292 if (ret != 0) 1293 return ENOMEM; 1294 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1295 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1296 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1297 if (_va == 0) 1298 goto error; 1299 1300 for (i = 0; i < npages; i++) { 1301 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1302 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1303 } 1304 pmap_update(pmap_kernel()); 1305 1306 memset((void *)_va, 0, npages * PAGE_SIZE); 1307 1308 *pa = _pa; 1309 *va = _va; 1310 return 0; 1311 1312 error: 1313 for (i = 0; i < npages; i++) { 1314 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1315 } 1316 return ENOMEM; 1317 } 1318 1319 static void 1320 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1321 { 1322 size_t i; 1323 1324 pmap_kremove(va, npages * PAGE_SIZE); 1325 pmap_update(pmap_kernel()); 1326 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1327 for (i = 0; i < npages; i++) { 1328 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1329 } 1330 } 1331 1332 /* -------------------------------------------------------------------------- */ 1333 1334 #define SVM_MSRBM_READ __BIT(0) 1335 #define SVM_MSRBM_WRITE __BIT(1) 1336 1337 static void 1338 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1339 { 1340 uint64_t byte; 1341 uint8_t bitoff; 1342 1343 if (msr < 0x00002000) { 1344 /* Range 1 */ 1345 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1346 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1347 /* Range 2 */ 1348 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1349 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1350 /* Range 3 */ 1351 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1352 } else { 1353 panic("%s: wrong range", __func__); 1354 } 1355 1356 bitoff = (msr & 0x3) << 1; 1357 1358 if (read) { 1359 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1360 } 1361 if (write) { 1362 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1363 } 1364 } 1365 1366 static void 1367 svm_asid_alloc(struct nvmm_cpu *vcpu) 1368 { 1369 struct svm_cpudata *cpudata = vcpu->cpudata; 1370 struct vmcb *vmcb = cpudata->vmcb; 1371 size_t i, oct, bit; 1372 1373 mutex_enter(&svm_asidlock); 1374 1375 for (i = 0; i < svm_maxasid; i++) { 1376 oct = i / 8; 1377 bit = i % 8; 1378 1379 if (svm_asidmap[oct] & __BIT(bit)) { 1380 continue; 1381 } 1382 1383 svm_asidmap[oct] |= __BIT(bit); 1384 vmcb->ctrl.guest_asid = i; 1385 mutex_exit(&svm_asidlock); 1386 return; 1387 } 1388 1389 /* 1390 * No free ASID. Use the last one, which is shared and requires 1391 * special TLB handling. 1392 */ 1393 cpudata->shared_asid = true; 1394 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1395 mutex_exit(&svm_asidlock); 1396 } 1397 1398 static void 1399 svm_asid_free(struct nvmm_cpu *vcpu) 1400 { 1401 struct svm_cpudata *cpudata = vcpu->cpudata; 1402 struct vmcb *vmcb = cpudata->vmcb; 1403 size_t oct, bit; 1404 1405 if (cpudata->shared_asid) { 1406 return; 1407 } 1408 1409 oct = vmcb->ctrl.guest_asid / 8; 1410 bit = vmcb->ctrl.guest_asid % 8; 1411 1412 mutex_enter(&svm_asidlock); 1413 svm_asidmap[oct] &= ~__BIT(bit); 1414 mutex_exit(&svm_asidlock); 1415 } 1416 1417 static void 1418 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1419 { 1420 struct svm_cpudata *cpudata = vcpu->cpudata; 1421 struct vmcb *vmcb = cpudata->vmcb; 1422 1423 /* Allow reads/writes of Control Registers. */ 1424 vmcb->ctrl.intercept_cr = 0; 1425 1426 /* Allow reads/writes of Debug Registers. */ 1427 vmcb->ctrl.intercept_dr = 0; 1428 1429 /* Allow exceptions 0 to 31. */ 1430 vmcb->ctrl.intercept_vec = 0; 1431 1432 /* 1433 * Allow: 1434 * - SMI [smm interrupts] 1435 * - VINTR [virtual interrupts] 1436 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1437 * - RIDTR [reads of IDTR] 1438 * - RGDTR [reads of GDTR] 1439 * - RLDTR [reads of LDTR] 1440 * - RTR [reads of TR] 1441 * - WIDTR [writes of IDTR] 1442 * - WGDTR [writes of GDTR] 1443 * - WLDTR [writes of LDTR] 1444 * - WTR [writes of TR] 1445 * - RDTSC [rdtsc instruction] 1446 * - PUSHF [pushf instruction] 1447 * - POPF [popf instruction] 1448 * - IRET [iret instruction] 1449 * - INTN [int $n instructions] 1450 * - INVD [invd instruction] 1451 * - PAUSE [pause instruction] 1452 * - INVLPG [invplg instruction] 1453 * - TASKSW [task switches] 1454 * 1455 * Intercept the rest below. 1456 */ 1457 vmcb->ctrl.intercept_misc1 = 1458 VMCB_CTRL_INTERCEPT_INTR | 1459 VMCB_CTRL_INTERCEPT_NMI | 1460 VMCB_CTRL_INTERCEPT_INIT | 1461 VMCB_CTRL_INTERCEPT_RDPMC | 1462 VMCB_CTRL_INTERCEPT_CPUID | 1463 VMCB_CTRL_INTERCEPT_RSM | 1464 VMCB_CTRL_INTERCEPT_HLT | 1465 VMCB_CTRL_INTERCEPT_INVLPGA | 1466 VMCB_CTRL_INTERCEPT_IOIO_PROT | 1467 VMCB_CTRL_INTERCEPT_MSR_PROT | 1468 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 1469 VMCB_CTRL_INTERCEPT_SHUTDOWN; 1470 1471 /* 1472 * Allow: 1473 * - ICEBP [icebp instruction] 1474 * - WBINVD [wbinvd instruction] 1475 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 1476 * 1477 * Intercept the rest below. 1478 */ 1479 vmcb->ctrl.intercept_misc2 = 1480 VMCB_CTRL_INTERCEPT_VMRUN | 1481 VMCB_CTRL_INTERCEPT_VMMCALL | 1482 VMCB_CTRL_INTERCEPT_VMLOAD | 1483 VMCB_CTRL_INTERCEPT_VMSAVE | 1484 VMCB_CTRL_INTERCEPT_STGI | 1485 VMCB_CTRL_INTERCEPT_CLGI | 1486 VMCB_CTRL_INTERCEPT_SKINIT | 1487 VMCB_CTRL_INTERCEPT_RDTSCP | 1488 VMCB_CTRL_INTERCEPT_MONITOR | 1489 VMCB_CTRL_INTERCEPT_MWAIT | 1490 VMCB_CTRL_INTERCEPT_XSETBV; 1491 1492 /* Intercept all I/O accesses. */ 1493 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 1494 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 1495 1496 /* 1497 * Allow: 1498 * - EFER [read] 1499 * - STAR [read, write] 1500 * - LSTAR [read, write] 1501 * - CSTAR [read, write] 1502 * - SFMASK [read, write] 1503 * - KERNELGSBASE [read, write] 1504 * - SYSENTER_CS [read, write] 1505 * - SYSENTER_ESP [read, write] 1506 * - SYSENTER_EIP [read, write] 1507 * - FSBASE [read, write] 1508 * - GSBASE [read, write] 1509 * - TSC [read] 1510 * 1511 * Intercept the rest. 1512 */ 1513 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 1514 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 1515 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 1516 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 1517 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 1518 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 1519 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 1520 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 1521 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 1522 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 1523 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 1524 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 1525 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 1526 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 1527 1528 /* Generate ASID. */ 1529 svm_asid_alloc(vcpu); 1530 1531 /* Virtual TPR. */ 1532 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 1533 1534 /* Enable Nested Paging. */ 1535 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 1536 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 1537 1538 /* Must always be set. */ 1539 vmcb->state.efer = EFER_SVME; 1540 1541 /* Init XSAVE header. */ 1542 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1543 cpudata->gfpu.xsh_xcomp_bv = 0; 1544 1545 /* Bluntly hide the host TSC. */ 1546 cpudata->tsc_offset = rdtsc(); 1547 } 1548 1549 static int 1550 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1551 { 1552 struct svm_cpudata *cpudata; 1553 int error; 1554 1555 /* Allocate the SVM cpudata. */ 1556 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 1557 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 1558 UVM_KMF_WIRED|UVM_KMF_ZERO); 1559 vcpu->cpudata = cpudata; 1560 1561 /* VMCB */ 1562 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 1563 VMCB_NPAGES); 1564 if (error) 1565 goto error; 1566 1567 /* I/O Bitmap */ 1568 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 1569 IOBM_NPAGES); 1570 if (error) 1571 goto error; 1572 1573 /* MSR Bitmap */ 1574 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 1575 MSRBM_NPAGES); 1576 if (error) 1577 goto error; 1578 1579 /* Init the VCPU info. */ 1580 svm_vcpu_init(mach, vcpu); 1581 1582 return 0; 1583 1584 error: 1585 if (cpudata->vmcb_pa) { 1586 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 1587 VMCB_NPAGES); 1588 } 1589 if (cpudata->iobm_pa) { 1590 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 1591 IOBM_NPAGES); 1592 } 1593 if (cpudata->msrbm_pa) { 1594 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 1595 MSRBM_NPAGES); 1596 } 1597 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1598 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1599 return error; 1600 } 1601 1602 static void 1603 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1604 { 1605 struct svm_cpudata *cpudata = vcpu->cpudata; 1606 1607 svm_asid_free(vcpu); 1608 1609 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 1610 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 1611 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 1612 1613 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1614 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1615 } 1616 1617 #define SVM_SEG_ATTRIB_TYPE __BITS(4,0) 1618 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1619 #define SVM_SEG_ATTRIB_P __BIT(7) 1620 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1621 #define SVM_SEG_ATTRIB_LONG __BIT(9) 1622 #define SVM_SEG_ATTRIB_DEF32 __BIT(10) 1623 #define SVM_SEG_ATTRIB_GRAN __BIT(11) 1624 1625 static void 1626 svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1627 { 1628 vseg->selector = seg->selector; 1629 vseg->attrib = 1630 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1631 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1632 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1633 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1634 __SHIFTIN(seg->attrib.lng, SVM_SEG_ATTRIB_LONG) | 1635 __SHIFTIN(seg->attrib.def32, SVM_SEG_ATTRIB_DEF32) | 1636 __SHIFTIN(seg->attrib.gran, SVM_SEG_ATTRIB_GRAN); 1637 vseg->limit = seg->limit; 1638 vseg->base = seg->base; 1639 } 1640 1641 static void 1642 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1643 { 1644 seg->selector = vseg->selector; 1645 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1646 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1647 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1648 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1649 seg->attrib.lng = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_LONG); 1650 seg->attrib.def32 = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF32); 1651 seg->attrib.gran = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_GRAN); 1652 seg->limit = vseg->limit; 1653 seg->base = vseg->base; 1654 } 1655 1656 static bool 1657 svm_state_tlb_flush(struct nvmm_x64_state *cstate, 1658 struct nvmm_x64_state *nstate, uint64_t flags) 1659 { 1660 if (flags & NVMM_X64_STATE_CRS) { 1661 if ((cstate->crs[NVMM_X64_CR_CR0] ^ 1662 nstate->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1663 return true; 1664 } 1665 if (cstate->crs[NVMM_X64_CR_CR3] != 1666 nstate->crs[NVMM_X64_CR_CR3]) { 1667 return true; 1668 } 1669 if ((cstate->crs[NVMM_X64_CR_CR4] ^ 1670 nstate->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1671 return true; 1672 } 1673 } 1674 1675 if (flags & NVMM_X64_STATE_MSRS) { 1676 if ((cstate->msrs[NVMM_X64_MSR_EFER] ^ 1677 nstate->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1678 return true; 1679 } 1680 } 1681 1682 return false; 1683 } 1684 1685 static void 1686 svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1687 { 1688 struct svm_cpudata *cpudata = vcpu->cpudata; 1689 struct nvmm_x64_state *cstate = &cpudata->state; 1690 struct nvmm_x64_state *nstate = (struct nvmm_x64_state *)data; 1691 struct vmcb *vmcb = cpudata->vmcb; 1692 struct fxsave *fpustate; 1693 1694 if (svm_state_tlb_flush(cstate, nstate, flags)) { 1695 cpudata->tlb_want_flush = true; 1696 } 1697 1698 if (flags & NVMM_X64_STATE_SEGS) { 1699 memcpy(cstate->segs, nstate->segs, sizeof(nstate->segs)); 1700 1701 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_CS], 1702 &vmcb->state.cs); 1703 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_DS], 1704 &vmcb->state.ds); 1705 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_ES], 1706 &vmcb->state.es); 1707 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_FS], 1708 &vmcb->state.fs); 1709 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_GS], 1710 &vmcb->state.gs); 1711 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_SS], 1712 &vmcb->state.ss); 1713 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_GDT], 1714 &vmcb->state.gdt); 1715 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_IDT], 1716 &vmcb->state.idt); 1717 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_LDT], 1718 &vmcb->state.ldt); 1719 svm_vcpu_setstate_seg(&cstate->segs[NVMM_X64_SEG_TR], 1720 &vmcb->state.tr); 1721 } 1722 1723 if (flags & NVMM_X64_STATE_GPRS) { 1724 memcpy(cstate->gprs, nstate->gprs, sizeof(nstate->gprs)); 1725 1726 vmcb->state.rip = cstate->gprs[NVMM_X64_GPR_RIP]; 1727 vmcb->state.rsp = cstate->gprs[NVMM_X64_GPR_RSP]; 1728 vmcb->state.rax = cstate->gprs[NVMM_X64_GPR_RAX]; 1729 vmcb->state.rflags = cstate->gprs[NVMM_X64_GPR_RFLAGS]; 1730 } 1731 1732 if (flags & NVMM_X64_STATE_CRS) { 1733 memcpy(cstate->crs, nstate->crs, sizeof(nstate->crs)); 1734 1735 vmcb->state.cr0 = cstate->crs[NVMM_X64_CR_CR0]; 1736 vmcb->state.cr2 = cstate->crs[NVMM_X64_CR_CR2]; 1737 vmcb->state.cr3 = cstate->crs[NVMM_X64_CR_CR3]; 1738 vmcb->state.cr4 = cstate->crs[NVMM_X64_CR_CR4]; 1739 1740 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1741 vmcb->ctrl.v |= __SHIFTIN(cstate->crs[NVMM_X64_CR_CR8], 1742 VMCB_CTRL_V_TPR); 1743 1744 /* Clear unsupported XCR0 bits, set mandatory X87 bit. */ 1745 if (svm_xcr0_mask != 0) { 1746 cstate->crs[NVMM_X64_CR_XCR0] &= svm_xcr0_mask; 1747 cstate->crs[NVMM_X64_CR_XCR0] |= XCR0_X87; 1748 } else { 1749 cstate->crs[NVMM_X64_CR_XCR0] = 0; 1750 } 1751 } 1752 1753 if (flags & NVMM_X64_STATE_DRS) { 1754 memcpy(cstate->drs, nstate->drs, sizeof(nstate->drs)); 1755 1756 vmcb->state.dr6 = cstate->drs[NVMM_X64_DR_DR6]; 1757 vmcb->state.dr7 = cstate->drs[NVMM_X64_DR_DR7]; 1758 } 1759 1760 if (flags & NVMM_X64_STATE_MSRS) { 1761 memcpy(cstate->msrs, nstate->msrs, sizeof(nstate->msrs)); 1762 1763 /* Bit EFER_SVME is mandatory. */ 1764 cstate->msrs[NVMM_X64_MSR_EFER] |= EFER_SVME; 1765 1766 vmcb->state.efer = cstate->msrs[NVMM_X64_MSR_EFER]; 1767 vmcb->state.star = cstate->msrs[NVMM_X64_MSR_STAR]; 1768 vmcb->state.lstar = cstate->msrs[NVMM_X64_MSR_LSTAR]; 1769 vmcb->state.cstar = cstate->msrs[NVMM_X64_MSR_CSTAR]; 1770 vmcb->state.sfmask = cstate->msrs[NVMM_X64_MSR_SFMASK]; 1771 vmcb->state.kernelgsbase = 1772 cstate->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1773 vmcb->state.sysenter_cs = 1774 cstate->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1775 vmcb->state.sysenter_esp = 1776 cstate->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1777 vmcb->state.sysenter_eip = 1778 cstate->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1779 vmcb->state.g_pat = cstate->msrs[NVMM_X64_MSR_PAT]; 1780 } 1781 1782 if (flags & NVMM_X64_STATE_MISC) { 1783 memcpy(cstate->misc, nstate->misc, sizeof(nstate->misc)); 1784 1785 vmcb->state.cpl = cstate->misc[NVMM_X64_MISC_CPL]; 1786 1787 if (cstate->misc[NVMM_X64_MISC_INT_SHADOW]) { 1788 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1789 } else { 1790 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1791 } 1792 1793 if (cstate->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { 1794 svm_event_waitexit_enable(vcpu, false); 1795 } else { 1796 svm_event_waitexit_disable(vcpu, false); 1797 } 1798 1799 if (cstate->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { 1800 svm_event_waitexit_enable(vcpu, true); 1801 } else { 1802 svm_event_waitexit_disable(vcpu, true); 1803 } 1804 } 1805 1806 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(cstate->fpu)); 1807 if (flags & NVMM_X64_STATE_FPU) { 1808 memcpy(&cstate->fpu, &nstate->fpu, sizeof(nstate->fpu)); 1809 1810 memcpy(cpudata->gfpu.xsh_fxsave, &cstate->fpu, 1811 sizeof(cstate->fpu)); 1812 1813 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1814 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1815 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1816 } 1817 } 1818 1819 static void 1820 svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1821 { 1822 struct svm_cpudata *cpudata = vcpu->cpudata; 1823 struct nvmm_x64_state *cstate = &cpudata->state; 1824 struct nvmm_x64_state *nstate = (struct nvmm_x64_state *)data; 1825 struct vmcb *vmcb = cpudata->vmcb; 1826 1827 if (flags & NVMM_X64_STATE_SEGS) { 1828 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_CS], 1829 &vmcb->state.cs); 1830 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_DS], 1831 &vmcb->state.ds); 1832 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_ES], 1833 &vmcb->state.es); 1834 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_FS], 1835 &vmcb->state.fs); 1836 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_GS], 1837 &vmcb->state.gs); 1838 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_SS], 1839 &vmcb->state.ss); 1840 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_GDT], 1841 &vmcb->state.gdt); 1842 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_IDT], 1843 &vmcb->state.idt); 1844 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_LDT], 1845 &vmcb->state.ldt); 1846 svm_vcpu_getstate_seg(&cstate->segs[NVMM_X64_SEG_TR], 1847 &vmcb->state.tr); 1848 1849 memcpy(nstate->segs, cstate->segs, sizeof(cstate->segs)); 1850 } 1851 1852 if (flags & NVMM_X64_STATE_GPRS) { 1853 cstate->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1854 cstate->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1855 cstate->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1856 cstate->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1857 1858 memcpy(nstate->gprs, cstate->gprs, sizeof(cstate->gprs)); 1859 } 1860 1861 if (flags & NVMM_X64_STATE_CRS) { 1862 cstate->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1863 cstate->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1864 cstate->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1865 cstate->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1866 cstate->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1867 VMCB_CTRL_V_TPR); 1868 1869 memcpy(nstate->crs, cstate->crs, sizeof(cstate->crs)); 1870 } 1871 1872 if (flags & NVMM_X64_STATE_DRS) { 1873 cstate->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1874 cstate->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1875 1876 memcpy(nstate->drs, cstate->drs, sizeof(cstate->drs)); 1877 } 1878 1879 if (flags & NVMM_X64_STATE_MSRS) { 1880 cstate->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1881 cstate->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1882 cstate->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1883 cstate->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1884 cstate->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1885 cstate->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1886 vmcb->state.kernelgsbase; 1887 cstate->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1888 vmcb->state.sysenter_cs; 1889 cstate->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1890 vmcb->state.sysenter_esp; 1891 cstate->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1892 vmcb->state.sysenter_eip; 1893 cstate->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1894 1895 memcpy(nstate->msrs, cstate->msrs, sizeof(cstate->msrs)); 1896 1897 /* Hide SVME. */ 1898 nstate->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1899 } 1900 1901 if (flags & NVMM_X64_STATE_MISC) { 1902 cstate->misc[NVMM_X64_MISC_CPL] = vmcb->state.cpl; 1903 1904 cstate->misc[NVMM_X64_MISC_INT_SHADOW] = 1905 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1906 cstate->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = 1907 cpudata->int_window_exit; 1908 cstate->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = 1909 cpudata->nmi_window_exit; 1910 1911 memcpy(nstate->misc, cstate->misc, sizeof(cstate->misc)); 1912 } 1913 1914 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(cstate->fpu)); 1915 if (flags & NVMM_X64_STATE_FPU) { 1916 memcpy(&cstate->fpu, cpudata->gfpu.xsh_fxsave, 1917 sizeof(cstate->fpu)); 1918 1919 memcpy(&nstate->fpu, &cstate->fpu, sizeof(cstate->fpu)); 1920 } 1921 } 1922 1923 /* -------------------------------------------------------------------------- */ 1924 1925 static void 1926 svm_tlb_flush(struct pmap *pm) 1927 { 1928 struct nvmm_machine *mach = pm->pm_data; 1929 struct svm_cpudata *cpudata; 1930 struct nvmm_cpu *vcpu; 1931 int error; 1932 size_t i; 1933 1934 /* Request TLB flushes. */ 1935 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 1936 error = nvmm_vcpu_get(mach, i, &vcpu); 1937 if (error) 1938 continue; 1939 cpudata = vcpu->cpudata; 1940 cpudata->tlb_want_flush = true; 1941 nvmm_vcpu_put(vcpu); 1942 } 1943 } 1944 1945 static void 1946 svm_machine_create(struct nvmm_machine *mach) 1947 { 1948 /* Fill in pmap info. */ 1949 mach->vm->vm_map.pmap->pm_data = (void *)mach; 1950 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 1951 1952 mach->machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 1953 } 1954 1955 static void 1956 svm_machine_destroy(struct nvmm_machine *mach) 1957 { 1958 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 1959 } 1960 1961 static int 1962 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 1963 { 1964 struct nvmm_x86_conf_cpuid *cpuid = data; 1965 struct svm_machdata *machdata = (struct svm_machdata *)mach->machdata; 1966 size_t i; 1967 1968 if (__predict_false(op != NVMM_X86_CONF_CPUID)) { 1969 return EINVAL; 1970 } 1971 1972 if (__predict_false((cpuid->set.eax & cpuid->del.eax) || 1973 (cpuid->set.ebx & cpuid->del.ebx) || 1974 (cpuid->set.ecx & cpuid->del.ecx) || 1975 (cpuid->set.edx & cpuid->del.edx))) { 1976 return EINVAL; 1977 } 1978 1979 /* If already here, replace. */ 1980 for (i = 0; i < SVM_NCPUIDS; i++) { 1981 if (!machdata->cpuidpresent[i]) { 1982 continue; 1983 } 1984 if (machdata->cpuid[i].leaf == cpuid->leaf) { 1985 memcpy(&machdata->cpuid[i], cpuid, 1986 sizeof(struct nvmm_x86_conf_cpuid)); 1987 return 0; 1988 } 1989 } 1990 1991 /* Not here, insert. */ 1992 for (i = 0; i < SVM_NCPUIDS; i++) { 1993 if (!machdata->cpuidpresent[i]) { 1994 machdata->cpuidpresent[i] = true; 1995 memcpy(&machdata->cpuid[i], cpuid, 1996 sizeof(struct nvmm_x86_conf_cpuid)); 1997 return 0; 1998 } 1999 } 2000 2001 return ENOBUFS; 2002 } 2003 2004 /* -------------------------------------------------------------------------- */ 2005 2006 static bool 2007 svm_ident(void) 2008 { 2009 u_int descs[4]; 2010 uint64_t msr; 2011 2012 if (cpu_vendor != CPUVENDOR_AMD) { 2013 return false; 2014 } 2015 if (!(cpu_feature[3] & CPUID_SVM)) { 2016 return false; 2017 } 2018 2019 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2020 return false; 2021 } 2022 x86_cpuid(0x8000000a, descs); 2023 2024 /* Want Nested Paging. */ 2025 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2026 return false; 2027 } 2028 2029 /* Want nRIP. */ 2030 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2031 return false; 2032 } 2033 2034 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2035 2036 msr = rdmsr(MSR_VMCR); 2037 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2038 return false; 2039 } 2040 2041 return true; 2042 } 2043 2044 static void 2045 svm_init_asid(uint32_t maxasid) 2046 { 2047 size_t i, j, allocsz; 2048 2049 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2050 2051 /* Arbitrarily limit. */ 2052 maxasid = uimin(maxasid, 8192); 2053 2054 svm_maxasid = maxasid; 2055 allocsz = roundup(maxasid, 8) / 8; 2056 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2057 2058 /* ASID 0 is reserved for the host. */ 2059 svm_asidmap[0] |= __BIT(0); 2060 2061 /* ASID n-1 is special, we share it. */ 2062 i = (maxasid - 1) / 8; 2063 j = (maxasid - 1) % 8; 2064 svm_asidmap[i] |= __BIT(j); 2065 } 2066 2067 static void 2068 svm_change_cpu(void *arg1, void *arg2) 2069 { 2070 bool enable = (bool)arg1; 2071 uint64_t msr; 2072 2073 msr = rdmsr(MSR_VMCR); 2074 if (msr & VMCR_SVMED) { 2075 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2076 } 2077 2078 if (!enable) { 2079 wrmsr(MSR_VM_HSAVE_PA, 0); 2080 } 2081 2082 msr = rdmsr(MSR_EFER); 2083 if (enable) { 2084 msr |= EFER_SVME; 2085 } else { 2086 msr &= ~EFER_SVME; 2087 } 2088 wrmsr(MSR_EFER, msr); 2089 2090 if (enable) { 2091 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2092 } 2093 } 2094 2095 static void 2096 svm_init(void) 2097 { 2098 CPU_INFO_ITERATOR cii; 2099 struct cpu_info *ci; 2100 struct vm_page *pg; 2101 u_int descs[4]; 2102 uint64_t xc; 2103 2104 x86_cpuid(0x8000000a, descs); 2105 2106 /* The guest TLB flush command. */ 2107 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2108 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2109 } else { 2110 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2111 } 2112 2113 /* Init the ASID. */ 2114 svm_init_asid(descs[1]); 2115 2116 /* Init the XCR0 mask. */ 2117 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2118 2119 memset(hsave, 0, sizeof(hsave)); 2120 for (CPU_INFO_FOREACH(cii, ci)) { 2121 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2122 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2123 } 2124 2125 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2126 xc_wait(xc); 2127 } 2128 2129 static void 2130 svm_fini_asid(void) 2131 { 2132 size_t allocsz; 2133 2134 allocsz = roundup(svm_maxasid, 8) / 8; 2135 kmem_free(svm_asidmap, allocsz); 2136 2137 mutex_destroy(&svm_asidlock); 2138 } 2139 2140 static void 2141 svm_fini(void) 2142 { 2143 uint64_t xc; 2144 size_t i; 2145 2146 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2147 xc_wait(xc); 2148 2149 for (i = 0; i < MAXCPUS; i++) { 2150 if (hsave[i].pa != 0) 2151 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2152 } 2153 2154 svm_fini_asid(); 2155 } 2156 2157 static void 2158 svm_capability(struct nvmm_capability *cap) 2159 { 2160 cap->u.x86.xcr0_mask = svm_xcr0_mask; 2161 cap->u.x86.mxcsr_mask = x86_fpu_mxcsr_mask; 2162 cap->u.x86.conf_cpuid_maxops = SVM_NCPUIDS; 2163 } 2164 2165 const struct nvmm_impl nvmm_x86_svm = { 2166 .ident = svm_ident, 2167 .init = svm_init, 2168 .fini = svm_fini, 2169 .capability = svm_capability, 2170 .conf_max = NVMM_X86_NCONF, 2171 .conf_sizes = svm_conf_sizes, 2172 .state_size = sizeof(struct nvmm_x64_state), 2173 .machine_create = svm_machine_create, 2174 .machine_destroy = svm_machine_destroy, 2175 .machine_configure = svm_machine_configure, 2176 .vcpu_create = svm_vcpu_create, 2177 .vcpu_destroy = svm_vcpu_destroy, 2178 .vcpu_setstate = svm_vcpu_setstate, 2179 .vcpu_getstate = svm_vcpu_getstate, 2180 .vcpu_inject = svm_vcpu_inject, 2181 .vcpu_run = svm_vcpu_run 2182 }; 2183
Indexes created Tue Sep 30 20:09:53 GMT 2025