nvmm_x86_svm.c revision 1.17
1 /* $NetBSD: nvmm_x86_svm.c,v 1.17 2019/01/24 13:05:59 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.17 2019/01/24 13:05:59 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 42 #include <uvm/uvm.h> 43 #include <uvm/uvm_page.h> 44 45 #include <x86/cputypes.h> 46 #include <x86/specialreg.h> 47 #include <x86/pmap.h> 48 #include <x86/dbregs.h> 49 #include <machine/cpuvar.h> 50 51 #include <dev/nvmm/nvmm.h> 52 #include <dev/nvmm/nvmm_internal.h> 53 #include <dev/nvmm/x86/nvmm_x86.h> 54 55 int svm_vmrun(paddr_t, uint64_t *); 56 57 #define MSR_VM_HSAVE_PA 0xC0010117 58 59 /* -------------------------------------------------------------------------- */ 60 61 #define VMCB_EXITCODE_CR0_READ 0x0000 62 #define VMCB_EXITCODE_CR1_READ 0x0001 63 #define VMCB_EXITCODE_CR2_READ 0x0002 64 #define VMCB_EXITCODE_CR3_READ 0x0003 65 #define VMCB_EXITCODE_CR4_READ 0x0004 66 #define VMCB_EXITCODE_CR5_READ 0x0005 67 #define VMCB_EXITCODE_CR6_READ 0x0006 68 #define VMCB_EXITCODE_CR7_READ 0x0007 69 #define VMCB_EXITCODE_CR8_READ 0x0008 70 #define VMCB_EXITCODE_CR9_READ 0x0009 71 #define VMCB_EXITCODE_CR10_READ 0x000A 72 #define VMCB_EXITCODE_CR11_READ 0x000B 73 #define VMCB_EXITCODE_CR12_READ 0x000C 74 #define VMCB_EXITCODE_CR13_READ 0x000D 75 #define VMCB_EXITCODE_CR14_READ 0x000E 76 #define VMCB_EXITCODE_CR15_READ 0x000F 77 #define VMCB_EXITCODE_CR0_WRITE 0x0010 78 #define VMCB_EXITCODE_CR1_WRITE 0x0011 79 #define VMCB_EXITCODE_CR2_WRITE 0x0012 80 #define VMCB_EXITCODE_CR3_WRITE 0x0013 81 #define VMCB_EXITCODE_CR4_WRITE 0x0014 82 #define VMCB_EXITCODE_CR5_WRITE 0x0015 83 #define VMCB_EXITCODE_CR6_WRITE 0x0016 84 #define VMCB_EXITCODE_CR7_WRITE 0x0017 85 #define VMCB_EXITCODE_CR8_WRITE 0x0018 86 #define VMCB_EXITCODE_CR9_WRITE 0x0019 87 #define VMCB_EXITCODE_CR10_WRITE 0x001A 88 #define VMCB_EXITCODE_CR11_WRITE 0x001B 89 #define VMCB_EXITCODE_CR12_WRITE 0x001C 90 #define VMCB_EXITCODE_CR13_WRITE 0x001D 91 #define VMCB_EXITCODE_CR14_WRITE 0x001E 92 #define VMCB_EXITCODE_CR15_WRITE 0x001F 93 #define VMCB_EXITCODE_DR0_READ 0x0020 94 #define VMCB_EXITCODE_DR1_READ 0x0021 95 #define VMCB_EXITCODE_DR2_READ 0x0022 96 #define VMCB_EXITCODE_DR3_READ 0x0023 97 #define VMCB_EXITCODE_DR4_READ 0x0024 98 #define VMCB_EXITCODE_DR5_READ 0x0025 99 #define VMCB_EXITCODE_DR6_READ 0x0026 100 #define VMCB_EXITCODE_DR7_READ 0x0027 101 #define VMCB_EXITCODE_DR8_READ 0x0028 102 #define VMCB_EXITCODE_DR9_READ 0x0029 103 #define VMCB_EXITCODE_DR10_READ 0x002A 104 #define VMCB_EXITCODE_DR11_READ 0x002B 105 #define VMCB_EXITCODE_DR12_READ 0x002C 106 #define VMCB_EXITCODE_DR13_READ 0x002D 107 #define VMCB_EXITCODE_DR14_READ 0x002E 108 #define VMCB_EXITCODE_DR15_READ 0x002F 109 #define VMCB_EXITCODE_DR0_WRITE 0x0030 110 #define VMCB_EXITCODE_DR1_WRITE 0x0031 111 #define VMCB_EXITCODE_DR2_WRITE 0x0032 112 #define VMCB_EXITCODE_DR3_WRITE 0x0033 113 #define VMCB_EXITCODE_DR4_WRITE 0x0034 114 #define VMCB_EXITCODE_DR5_WRITE 0x0035 115 #define VMCB_EXITCODE_DR6_WRITE 0x0036 116 #define VMCB_EXITCODE_DR7_WRITE 0x0037 117 #define VMCB_EXITCODE_DR8_WRITE 0x0038 118 #define VMCB_EXITCODE_DR9_WRITE 0x0039 119 #define VMCB_EXITCODE_DR10_WRITE 0x003A 120 #define VMCB_EXITCODE_DR11_WRITE 0x003B 121 #define VMCB_EXITCODE_DR12_WRITE 0x003C 122 #define VMCB_EXITCODE_DR13_WRITE 0x003D 123 #define VMCB_EXITCODE_DR14_WRITE 0x003E 124 #define VMCB_EXITCODE_DR15_WRITE 0x003F 125 #define VMCB_EXITCODE_EXCP0 0x0040 126 #define VMCB_EXITCODE_EXCP1 0x0041 127 #define VMCB_EXITCODE_EXCP2 0x0042 128 #define VMCB_EXITCODE_EXCP3 0x0043 129 #define VMCB_EXITCODE_EXCP4 0x0044 130 #define VMCB_EXITCODE_EXCP5 0x0045 131 #define VMCB_EXITCODE_EXCP6 0x0046 132 #define VMCB_EXITCODE_EXCP7 0x0047 133 #define VMCB_EXITCODE_EXCP8 0x0048 134 #define VMCB_EXITCODE_EXCP9 0x0049 135 #define VMCB_EXITCODE_EXCP10 0x004A 136 #define VMCB_EXITCODE_EXCP11 0x004B 137 #define VMCB_EXITCODE_EXCP12 0x004C 138 #define VMCB_EXITCODE_EXCP13 0x004D 139 #define VMCB_EXITCODE_EXCP14 0x004E 140 #define VMCB_EXITCODE_EXCP15 0x004F 141 #define VMCB_EXITCODE_EXCP16 0x0050 142 #define VMCB_EXITCODE_EXCP17 0x0051 143 #define VMCB_EXITCODE_EXCP18 0x0052 144 #define VMCB_EXITCODE_EXCP19 0x0053 145 #define VMCB_EXITCODE_EXCP20 0x0054 146 #define VMCB_EXITCODE_EXCP21 0x0055 147 #define VMCB_EXITCODE_EXCP22 0x0056 148 #define VMCB_EXITCODE_EXCP23 0x0057 149 #define VMCB_EXITCODE_EXCP24 0x0058 150 #define VMCB_EXITCODE_EXCP25 0x0059 151 #define VMCB_EXITCODE_EXCP26 0x005A 152 #define VMCB_EXITCODE_EXCP27 0x005B 153 #define VMCB_EXITCODE_EXCP28 0x005C 154 #define VMCB_EXITCODE_EXCP29 0x005D 155 #define VMCB_EXITCODE_EXCP30 0x005E 156 #define VMCB_EXITCODE_EXCP31 0x005F 157 #define VMCB_EXITCODE_INTR 0x0060 158 #define VMCB_EXITCODE_NMI 0x0061 159 #define VMCB_EXITCODE_SMI 0x0062 160 #define VMCB_EXITCODE_INIT 0x0063 161 #define VMCB_EXITCODE_VINTR 0x0064 162 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 163 #define VMCB_EXITCODE_IDTR_READ 0x0066 164 #define VMCB_EXITCODE_GDTR_READ 0x0067 165 #define VMCB_EXITCODE_LDTR_READ 0x0068 166 #define VMCB_EXITCODE_TR_READ 0x0069 167 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 168 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 169 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 170 #define VMCB_EXITCODE_TR_WRITE 0x006D 171 #define VMCB_EXITCODE_RDTSC 0x006E 172 #define VMCB_EXITCODE_RDPMC 0x006F 173 #define VMCB_EXITCODE_PUSHF 0x0070 174 #define VMCB_EXITCODE_POPF 0x0071 175 #define VMCB_EXITCODE_CPUID 0x0072 176 #define VMCB_EXITCODE_RSM 0x0073 177 #define VMCB_EXITCODE_IRET 0x0074 178 #define VMCB_EXITCODE_SWINT 0x0075 179 #define VMCB_EXITCODE_INVD 0x0076 180 #define VMCB_EXITCODE_PAUSE 0x0077 181 #define VMCB_EXITCODE_HLT 0x0078 182 #define VMCB_EXITCODE_INVLPG 0x0079 183 #define VMCB_EXITCODE_INVLPGA 0x007A 184 #define VMCB_EXITCODE_IOIO 0x007B 185 #define VMCB_EXITCODE_MSR 0x007C 186 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 187 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 188 #define VMCB_EXITCODE_SHUTDOWN 0x007F 189 #define VMCB_EXITCODE_VMRUN 0x0080 190 #define VMCB_EXITCODE_VMMCALL 0x0081 191 #define VMCB_EXITCODE_VMLOAD 0x0082 192 #define VMCB_EXITCODE_VMSAVE 0x0083 193 #define VMCB_EXITCODE_STGI 0x0084 194 #define VMCB_EXITCODE_CLGI 0x0085 195 #define VMCB_EXITCODE_SKINIT 0x0086 196 #define VMCB_EXITCODE_RDTSCP 0x0087 197 #define VMCB_EXITCODE_ICEBP 0x0088 198 #define VMCB_EXITCODE_WBINVD 0x0089 199 #define VMCB_EXITCODE_MONITOR 0x008A 200 #define VMCB_EXITCODE_MWAIT 0x008B 201 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 202 #define VMCB_EXITCODE_XSETBV 0x008D 203 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 204 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 205 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 206 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 207 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 208 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 209 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 210 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 211 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 212 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 213 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 214 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 215 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 216 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 217 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 218 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 219 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 220 #define VMCB_EXITCODE_NPF 0x0400 221 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 222 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 223 #define VMCB_EXITCODE_VMGEXIT 0x0403 224 #define VMCB_EXITCODE_INVALID -1 225 226 /* -------------------------------------------------------------------------- */ 227 228 struct vmcb_ctrl { 229 uint32_t intercept_cr; 230 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 231 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 232 233 uint32_t intercept_dr; 234 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 235 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 236 237 uint32_t intercept_vec; 238 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 239 240 uint32_t intercept_misc1; 241 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 242 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 243 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 244 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 245 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 246 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 247 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 248 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 249 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 250 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 251 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 252 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 253 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 254 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 255 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 256 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 257 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 258 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 259 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 260 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 261 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 262 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 263 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 264 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 265 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 266 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 267 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 268 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 269 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 270 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 271 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 272 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 273 274 uint32_t intercept_misc2; 275 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 276 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 277 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 278 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 279 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 280 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 281 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 282 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 283 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 284 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 285 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 286 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(12) 287 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 288 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 289 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 290 291 uint8_t rsvd1[40]; 292 uint16_t pause_filt_thresh; 293 uint16_t pause_filt_cnt; 294 uint64_t iopm_base_pa; 295 uint64_t msrpm_base_pa; 296 uint64_t tsc_offset; 297 uint32_t guest_asid; 298 299 uint32_t tlb_ctrl; 300 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 301 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 302 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 303 304 uint64_t v; 305 #define VMCB_CTRL_V_TPR __BITS(7,0) 306 #define VMCB_CTRL_V_IRQ __BIT(8) 307 #define VMCB_CTRL_V_VGIF __BIT(9) 308 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 309 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 310 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 311 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 312 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 313 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 314 315 uint64_t intr; 316 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 317 318 uint64_t exitcode; 319 uint64_t exitinfo1; 320 uint64_t exitinfo2; 321 322 uint64_t exitintinfo; 323 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 324 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 325 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 326 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 327 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 328 329 uint64_t enable1; 330 #define VMCB_CTRL_ENABLE_NP __BIT(0) 331 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 332 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 333 334 uint64_t avic; 335 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 336 337 uint64_t ghcb; 338 339 uint64_t eventinj; 340 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 341 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 342 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 343 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 344 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 345 346 uint64_t n_cr3; 347 348 uint64_t enable2; 349 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 350 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 351 352 uint32_t vmcb_clean; 353 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 354 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 355 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 356 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 357 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 358 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 359 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 360 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 361 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 362 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 363 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 364 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 365 366 uint32_t rsvd2; 367 uint64_t nrip; 368 uint8_t inst_len; 369 uint8_t inst_bytes[15]; 370 uint64_t avic_abpp; 371 uint64_t rsvd3; 372 uint64_t avic_ltp; 373 374 uint64_t avic_phys; 375 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 376 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 377 378 uint64_t rsvd4; 379 uint64_t vmcb_ptr; 380 381 uint8_t pad[752]; 382 } __packed; 383 384 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 385 386 struct vmcb_segment { 387 uint16_t selector; 388 uint16_t attrib; /* hidden */ 389 uint32_t limit; /* hidden */ 390 uint64_t base; /* hidden */ 391 } __packed; 392 393 CTASSERT(sizeof(struct vmcb_segment) == 16); 394 395 struct vmcb_state { 396 struct vmcb_segment es; 397 struct vmcb_segment cs; 398 struct vmcb_segment ss; 399 struct vmcb_segment ds; 400 struct vmcb_segment fs; 401 struct vmcb_segment gs; 402 struct vmcb_segment gdt; 403 struct vmcb_segment ldt; 404 struct vmcb_segment idt; 405 struct vmcb_segment tr; 406 uint8_t rsvd1[43]; 407 uint8_t cpl; 408 uint8_t rsvd2[4]; 409 uint64_t efer; 410 uint8_t rsvd3[112]; 411 uint64_t cr4; 412 uint64_t cr3; 413 uint64_t cr0; 414 uint64_t dr7; 415 uint64_t dr6; 416 uint64_t rflags; 417 uint64_t rip; 418 uint8_t rsvd4[88]; 419 uint64_t rsp; 420 uint8_t rsvd5[24]; 421 uint64_t rax; 422 uint64_t star; 423 uint64_t lstar; 424 uint64_t cstar; 425 uint64_t sfmask; 426 uint64_t kernelgsbase; 427 uint64_t sysenter_cs; 428 uint64_t sysenter_esp; 429 uint64_t sysenter_eip; 430 uint64_t cr2; 431 uint8_t rsvd6[32]; 432 uint64_t g_pat; 433 uint64_t dbgctl; 434 uint64_t br_from; 435 uint64_t br_to; 436 uint64_t int_from; 437 uint64_t int_to; 438 uint8_t pad[2408]; 439 } __packed; 440 441 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 442 443 struct vmcb { 444 struct vmcb_ctrl ctrl; 445 struct vmcb_state state; 446 } __packed; 447 448 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 449 CTASSERT(offsetof(struct vmcb, state) == 0x400); 450 451 /* -------------------------------------------------------------------------- */ 452 453 struct svm_hsave { 454 paddr_t pa; 455 }; 456 457 static struct svm_hsave hsave[MAXCPUS]; 458 459 static uint8_t *svm_asidmap __read_mostly; 460 static uint32_t svm_maxasid __read_mostly; 461 static kmutex_t svm_asidlock __cacheline_aligned; 462 463 static bool svm_decode_assist __read_mostly; 464 static uint32_t svm_ctrl_tlb_flush __read_mostly; 465 466 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 467 static uint64_t svm_xcr0_mask __read_mostly; 468 469 #define SVM_NCPUIDS 32 470 471 #define VMCB_NPAGES 1 472 473 #define MSRBM_NPAGES 2 474 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 475 476 #define IOBM_NPAGES 3 477 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 478 479 /* Does not include EFER_LMSLE. */ 480 #define EFER_VALID \ 481 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 482 483 #define EFER_TLB_FLUSH \ 484 (EFER_NXE|EFER_LMA|EFER_LME) 485 #define CR0_TLB_FLUSH \ 486 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 487 #define CR4_TLB_FLUSH \ 488 (CR4_PGE|CR4_PAE|CR4_PSE) 489 490 /* -------------------------------------------------------------------------- */ 491 492 struct svm_machdata { 493 bool cpuidpresent[SVM_NCPUIDS]; 494 struct nvmm_x86_conf_cpuid cpuid[SVM_NCPUIDS]; 495 }; 496 497 static const size_t svm_conf_sizes[NVMM_X86_NCONF] = { 498 [NVMM_X86_CONF_CPUID] = sizeof(struct nvmm_x86_conf_cpuid) 499 }; 500 501 struct svm_cpudata { 502 /* General */ 503 bool shared_asid; 504 bool tlb_want_flush; 505 506 /* VMCB */ 507 struct vmcb *vmcb; 508 paddr_t vmcb_pa; 509 510 /* I/O bitmap */ 511 uint8_t *iobm; 512 paddr_t iobm_pa; 513 514 /* MSR bitmap */ 515 uint8_t *msrbm; 516 paddr_t msrbm_pa; 517 518 /* Host state */ 519 uint64_t hxcr0; 520 uint64_t star; 521 uint64_t lstar; 522 uint64_t cstar; 523 uint64_t sfmask; 524 uint64_t fsbase; 525 uint64_t kernelgsbase; 526 bool ts_set; 527 struct xsave_header hfpu __aligned(64); 528 529 /* Event state */ 530 bool int_window_exit; 531 bool nmi_window_exit; 532 533 /* Guest state */ 534 uint64_t gxcr0; 535 uint64_t gprs[NVMM_X64_NGPR]; 536 uint64_t drs[NVMM_X64_NDR]; 537 uint64_t tsc_offset; 538 struct xsave_header gfpu __aligned(64); 539 }; 540 541 static void 542 svm_vmcb_cache_default(struct vmcb *vmcb) 543 { 544 vmcb->ctrl.vmcb_clean = 545 VMCB_CTRL_VMCB_CLEAN_I | 546 VMCB_CTRL_VMCB_CLEAN_IOPM | 547 VMCB_CTRL_VMCB_CLEAN_ASID | 548 VMCB_CTRL_VMCB_CLEAN_TPR | 549 VMCB_CTRL_VMCB_CLEAN_NP | 550 VMCB_CTRL_VMCB_CLEAN_CR | 551 VMCB_CTRL_VMCB_CLEAN_DR | 552 VMCB_CTRL_VMCB_CLEAN_DT | 553 VMCB_CTRL_VMCB_CLEAN_SEG | 554 VMCB_CTRL_VMCB_CLEAN_CR2 | 555 VMCB_CTRL_VMCB_CLEAN_LBR | 556 VMCB_CTRL_VMCB_CLEAN_AVIC; 557 } 558 559 static void 560 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 561 { 562 if (flags & NVMM_X64_STATE_SEGS) { 563 vmcb->ctrl.vmcb_clean &= 564 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 565 } 566 if (flags & NVMM_X64_STATE_CRS) { 567 vmcb->ctrl.vmcb_clean &= 568 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 569 VMCB_CTRL_VMCB_CLEAN_TPR); 570 } 571 if (flags & NVMM_X64_STATE_DRS) { 572 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 573 } 574 if (flags & NVMM_X64_STATE_MSRS) { 575 /* CR for EFER, NP for PAT. */ 576 vmcb->ctrl.vmcb_clean &= 577 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 578 } 579 if (flags & NVMM_X64_STATE_MISC) { 580 /* SEG for CPL. */ 581 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_SEG; 582 } 583 } 584 585 static inline void 586 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 587 { 588 vmcb->ctrl.vmcb_clean &= ~flags; 589 } 590 591 static inline void 592 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 593 { 594 vmcb->ctrl.vmcb_clean = 0; 595 } 596 597 #define SVM_EVENT_TYPE_HW_INT 0 598 #define SVM_EVENT_TYPE_NMI 2 599 #define SVM_EVENT_TYPE_EXC 3 600 #define SVM_EVENT_TYPE_SW_INT 4 601 602 static void 603 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 604 { 605 struct svm_cpudata *cpudata = vcpu->cpudata; 606 struct vmcb *vmcb = cpudata->vmcb; 607 608 if (nmi) { 609 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 610 cpudata->nmi_window_exit = true; 611 } else { 612 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 613 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 614 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 615 cpudata->int_window_exit = true; 616 } 617 618 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 619 } 620 621 static void 622 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 623 { 624 struct svm_cpudata *cpudata = vcpu->cpudata; 625 struct vmcb *vmcb = cpudata->vmcb; 626 627 if (nmi) { 628 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 629 cpudata->nmi_window_exit = false; 630 } else { 631 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 632 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 633 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 634 cpudata->int_window_exit = false; 635 } 636 637 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 638 } 639 640 static inline int 641 svm_event_has_error(uint64_t vector) 642 { 643 switch (vector) { 644 case 8: /* #DF */ 645 case 10: /* #TS */ 646 case 11: /* #NP */ 647 case 12: /* #SS */ 648 case 13: /* #GP */ 649 case 14: /* #PF */ 650 case 17: /* #AC */ 651 case 30: /* #SX */ 652 return 1; 653 default: 654 return 0; 655 } 656 } 657 658 static int 659 svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 660 struct nvmm_event *event) 661 { 662 struct svm_cpudata *cpudata = vcpu->cpudata; 663 struct vmcb *vmcb = cpudata->vmcb; 664 int type = 0, err = 0; 665 666 if (event->vector >= 256) { 667 return EINVAL; 668 } 669 670 switch (event->type) { 671 case NVMM_EVENT_INTERRUPT_HW: 672 type = SVM_EVENT_TYPE_HW_INT; 673 if (event->vector == 2) { 674 type = SVM_EVENT_TYPE_NMI; 675 } 676 if (type == SVM_EVENT_TYPE_NMI) { 677 if (cpudata->nmi_window_exit) { 678 return EAGAIN; 679 } 680 svm_event_waitexit_enable(vcpu, true); 681 } else { 682 if (((vmcb->state.rflags & PSL_I) == 0) || 683 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0)) { 684 svm_event_waitexit_enable(vcpu, false); 685 return EAGAIN; 686 } 687 } 688 err = 0; 689 break; 690 case NVMM_EVENT_INTERRUPT_SW: 691 type = SVM_EVENT_TYPE_SW_INT; 692 err = 0; 693 break; 694 case NVMM_EVENT_EXCEPTION: 695 type = SVM_EVENT_TYPE_EXC; 696 if (event->vector == 2 || event->vector >= 32) 697 return EINVAL; 698 err = svm_event_has_error(event->vector); 699 break; 700 default: 701 return EINVAL; 702 } 703 704 vmcb->ctrl.eventinj = 705 __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) | 706 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 707 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 708 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 709 __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE); 710 711 return 0; 712 } 713 714 static void 715 svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 716 { 717 struct nvmm_event event; 718 int ret __diagused; 719 720 event.type = NVMM_EVENT_EXCEPTION; 721 event.vector = 6; 722 event.u.error = 0; 723 724 ret = svm_vcpu_inject(mach, vcpu, &event); 725 KASSERT(ret == 0); 726 } 727 728 static void 729 svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 730 { 731 struct nvmm_event event; 732 int ret __diagused; 733 734 event.type = NVMM_EVENT_EXCEPTION; 735 event.vector = 13; 736 event.u.error = 0; 737 738 ret = svm_vcpu_inject(mach, vcpu, &event); 739 KASSERT(ret == 0); 740 } 741 742 static inline void 743 svm_inkernel_advance(struct vmcb *vmcb) 744 { 745 /* 746 * Maybe we should also apply single-stepping and debug exceptions. 747 * Matters for guest-ring3, because it can execute 'cpuid' under a 748 * debugger. 749 */ 750 vmcb->state.rip = vmcb->ctrl.nrip; 751 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 752 } 753 754 static void 755 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 756 { 757 struct svm_cpudata *cpudata = vcpu->cpudata; 758 759 switch (eax) { 760 case 0x00000001: /* APIC number in RBX. The rest is tunable. */ 761 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 762 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 763 CPUID_LOCAL_APIC_ID); 764 break; 765 case 0x0000000D: /* FPU description. Not tunable. */ 766 if (ecx != 0 || svm_xcr0_mask == 0) { 767 break; 768 } 769 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 770 if (cpudata->gxcr0 & XCR0_SSE) { 771 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 772 } else { 773 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 774 } 775 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 776 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave); 777 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 778 break; 779 case 0x40000000: 780 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 781 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 782 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 783 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 784 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 785 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 786 break; 787 case 0x80000001: /* No SVM, no RDTSCP. The rest is tunable. */ 788 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID_SVM; 789 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~CPUID_RDTSCP; 790 break; 791 default: 792 break; 793 } 794 } 795 796 static void 797 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 798 struct nvmm_exit *exit) 799 { 800 struct svm_machdata *machdata = mach->machdata; 801 struct svm_cpudata *cpudata = vcpu->cpudata; 802 struct nvmm_x86_conf_cpuid *cpuid; 803 uint64_t eax, ecx; 804 u_int descs[4]; 805 size_t i; 806 807 eax = cpudata->vmcb->state.rax; 808 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 809 x86_cpuid2(eax, ecx, descs); 810 811 cpudata->vmcb->state.rax = descs[0]; 812 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 813 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 814 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 815 816 for (i = 0; i < SVM_NCPUIDS; i++) { 817 cpuid = &machdata->cpuid[i]; 818 if (!machdata->cpuidpresent[i]) { 819 continue; 820 } 821 if (cpuid->leaf != eax) { 822 continue; 823 } 824 825 /* del */ 826 cpudata->vmcb->state.rax &= ~cpuid->del.eax; 827 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->del.ebx; 828 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->del.ecx; 829 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->del.edx; 830 831 /* set */ 832 cpudata->vmcb->state.rax |= cpuid->set.eax; 833 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->set.ebx; 834 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->set.ecx; 835 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->set.edx; 836 837 break; 838 } 839 840 /* Overwrite non-tunable leaves. */ 841 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 842 843 svm_inkernel_advance(cpudata->vmcb); 844 exit->reason = NVMM_EXIT_NONE; 845 } 846 847 static void 848 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 849 struct nvmm_exit *exit) 850 { 851 struct svm_cpudata *cpudata = vcpu->cpudata; 852 struct vmcb *vmcb = cpudata->vmcb; 853 854 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 855 svm_event_waitexit_disable(vcpu, false); 856 } 857 858 svm_inkernel_advance(cpudata->vmcb); 859 exit->reason = NVMM_EXIT_HALTED; 860 } 861 862 #define SVM_EXIT_IO_PORT __BITS(31,16) 863 #define SVM_EXIT_IO_SEG __BITS(12,10) 864 #define SVM_EXIT_IO_A64 __BIT(9) 865 #define SVM_EXIT_IO_A32 __BIT(8) 866 #define SVM_EXIT_IO_A16 __BIT(7) 867 #define SVM_EXIT_IO_SZ32 __BIT(6) 868 #define SVM_EXIT_IO_SZ16 __BIT(5) 869 #define SVM_EXIT_IO_SZ8 __BIT(4) 870 #define SVM_EXIT_IO_REP __BIT(3) 871 #define SVM_EXIT_IO_STR __BIT(2) 872 #define SVM_EXIT_IO_IN __BIT(0) 873 874 static const int seg_to_nvmm[] = { 875 [0] = NVMM_X64_SEG_ES, 876 [1] = NVMM_X64_SEG_CS, 877 [2] = NVMM_X64_SEG_SS, 878 [3] = NVMM_X64_SEG_DS, 879 [4] = NVMM_X64_SEG_FS, 880 [5] = NVMM_X64_SEG_GS 881 }; 882 883 static void 884 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 885 struct nvmm_exit *exit) 886 { 887 struct svm_cpudata *cpudata = vcpu->cpudata; 888 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 889 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 890 891 exit->reason = NVMM_EXIT_IO; 892 893 if (info & SVM_EXIT_IO_IN) { 894 exit->u.io.type = NVMM_EXIT_IO_IN; 895 } else { 896 exit->u.io.type = NVMM_EXIT_IO_OUT; 897 } 898 899 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 900 901 if (svm_decode_assist) { 902 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 903 exit->u.io.seg = seg_to_nvmm[__SHIFTOUT(info, SVM_EXIT_IO_SEG)]; 904 } else { 905 exit->u.io.seg = -1; 906 } 907 908 if (info & SVM_EXIT_IO_A64) { 909 exit->u.io.address_size = 8; 910 } else if (info & SVM_EXIT_IO_A32) { 911 exit->u.io.address_size = 4; 912 } else if (info & SVM_EXIT_IO_A16) { 913 exit->u.io.address_size = 2; 914 } 915 916 if (info & SVM_EXIT_IO_SZ32) { 917 exit->u.io.operand_size = 4; 918 } else if (info & SVM_EXIT_IO_SZ16) { 919 exit->u.io.operand_size = 2; 920 } else if (info & SVM_EXIT_IO_SZ8) { 921 exit->u.io.operand_size = 1; 922 } 923 924 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 925 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 926 exit->u.io.npc = nextpc; 927 } 928 929 static const uint64_t msr_ignore_list[] = { 930 0xc0010055, /* MSR_CMPHALT */ 931 MSR_DE_CFG, 932 MSR_IC_CFG, 933 MSR_UCODE_AMD_PATCHLEVEL 934 }; 935 936 static bool 937 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 938 struct nvmm_exit *exit) 939 { 940 struct svm_cpudata *cpudata = vcpu->cpudata; 941 uint64_t val; 942 size_t i; 943 944 switch (exit->u.msr.type) { 945 case NVMM_EXIT_MSR_RDMSR: 946 if (exit->u.msr.msr == MSR_CR_PAT) { 947 val = cpudata->vmcb->state.g_pat; 948 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 949 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 950 goto handled; 951 } 952 if (exit->u.msr.msr == MSR_NB_CFG) { 953 val = NB_CFG_INITAPICCPUIDLO; 954 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 955 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 956 goto handled; 957 } 958 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 959 if (msr_ignore_list[i] != exit->u.msr.msr) 960 continue; 961 val = 0; 962 cpudata->vmcb->state.rax = (val & 0xFFFFFFFF); 963 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 964 goto handled; 965 } 966 break; 967 case NVMM_EXIT_MSR_WRMSR: 968 if (exit->u.msr.msr == MSR_EFER) { 969 if (__predict_false(exit->u.msr.val & ~EFER_VALID)) { 970 svm_inject_gp(mach, vcpu); 971 goto handled; 972 } 973 if ((cpudata->vmcb->state.efer ^ exit->u.msr.val) & 974 EFER_TLB_FLUSH) { 975 cpudata->tlb_want_flush = true; 976 } 977 cpudata->vmcb->state.efer = exit->u.msr.val | EFER_SVME; 978 goto handled; 979 } 980 if (exit->u.msr.msr == MSR_CR_PAT) { 981 cpudata->vmcb->state.g_pat = exit->u.msr.val; 982 goto handled; 983 } 984 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 985 if (msr_ignore_list[i] != exit->u.msr.msr) 986 continue; 987 goto handled; 988 } 989 break; 990 } 991 992 return false; 993 994 handled: 995 svm_inkernel_advance(cpudata->vmcb); 996 return true; 997 } 998 999 static void 1000 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1001 struct nvmm_exit *exit) 1002 { 1003 struct svm_cpudata *cpudata = vcpu->cpudata; 1004 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1005 1006 if (info == 0) { 1007 exit->u.msr.type = NVMM_EXIT_MSR_RDMSR; 1008 } else { 1009 exit->u.msr.type = NVMM_EXIT_MSR_WRMSR; 1010 } 1011 1012 exit->u.msr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1013 1014 if (info == 1) { 1015 uint64_t rdx, rax; 1016 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1017 rax = cpudata->vmcb->state.rax; 1018 exit->u.msr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1019 } else { 1020 exit->u.msr.val = 0; 1021 } 1022 1023 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1024 exit->reason = NVMM_EXIT_NONE; 1025 return; 1026 } 1027 1028 exit->reason = NVMM_EXIT_MSR; 1029 exit->u.msr.npc = cpudata->vmcb->ctrl.nrip; 1030 } 1031 1032 static void 1033 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1034 struct nvmm_exit *exit) 1035 { 1036 struct svm_cpudata *cpudata = vcpu->cpudata; 1037 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1038 int error; 1039 1040 error = uvm_fault(&mach->vm->vm_map, gpa, VM_PROT_ALL); 1041 1042 if (error) { 1043 exit->reason = NVMM_EXIT_MEMORY; 1044 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1045 exit->u.mem.perm = NVMM_EXIT_MEMORY_WRITE; 1046 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1047 exit->u.mem.perm = NVMM_EXIT_MEMORY_EXEC; 1048 else 1049 exit->u.mem.perm = NVMM_EXIT_MEMORY_READ; 1050 exit->u.mem.gpa = gpa; 1051 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1052 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1053 sizeof(exit->u.mem.inst_bytes)); 1054 exit->u.mem.npc = cpudata->vmcb->ctrl.nrip; 1055 } else { 1056 exit->reason = NVMM_EXIT_NONE; 1057 } 1058 } 1059 1060 static void 1061 svm_exit_insn(struct vmcb *vmcb, struct nvmm_exit *exit, uint64_t reason) 1062 { 1063 exit->u.insn.npc = vmcb->ctrl.nrip; 1064 exit->reason = reason; 1065 } 1066 1067 static void 1068 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1069 struct nvmm_exit *exit) 1070 { 1071 struct svm_cpudata *cpudata = vcpu->cpudata; 1072 struct vmcb *vmcb = cpudata->vmcb; 1073 uint64_t val; 1074 1075 exit->reason = NVMM_EXIT_NONE; 1076 1077 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1078 (vmcb->state.rax & 0xFFFFFFFF); 1079 1080 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1081 goto error; 1082 } else if (__predict_false(vmcb->state.cpl != 0)) { 1083 goto error; 1084 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1085 goto error; 1086 } else if (__predict_false((val & XCR0_X87) == 0)) { 1087 goto error; 1088 } 1089 1090 cpudata->gxcr0 = val; 1091 1092 svm_inkernel_advance(cpudata->vmcb); 1093 return; 1094 1095 error: 1096 svm_inject_gp(mach, vcpu); 1097 } 1098 1099 static void 1100 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1101 { 1102 struct svm_cpudata *cpudata = vcpu->cpudata; 1103 1104 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1105 1106 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1107 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1108 1109 if (svm_xcr0_mask != 0) { 1110 cpudata->hxcr0 = rdxcr(0); 1111 wrxcr(0, cpudata->gxcr0); 1112 } 1113 } 1114 1115 static void 1116 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1117 { 1118 struct svm_cpudata *cpudata = vcpu->cpudata; 1119 1120 if (svm_xcr0_mask != 0) { 1121 cpudata->gxcr0 = rdxcr(0); 1122 wrxcr(0, cpudata->hxcr0); 1123 } 1124 1125 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1126 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1127 1128 if (cpudata->ts_set) { 1129 stts(); 1130 } 1131 } 1132 1133 static void 1134 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1135 { 1136 struct svm_cpudata *cpudata = vcpu->cpudata; 1137 1138 x86_dbregs_save(curlwp); 1139 1140 ldr7(0); 1141 1142 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1143 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1144 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1145 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1146 } 1147 1148 static void 1149 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1150 { 1151 struct svm_cpudata *cpudata = vcpu->cpudata; 1152 1153 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1154 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1155 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1156 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1157 1158 x86_dbregs_restore(curlwp); 1159 } 1160 1161 static void 1162 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1163 { 1164 struct svm_cpudata *cpudata = vcpu->cpudata; 1165 1166 cpudata->star = rdmsr(MSR_STAR); 1167 cpudata->lstar = rdmsr(MSR_LSTAR); 1168 cpudata->cstar = rdmsr(MSR_CSTAR); 1169 cpudata->sfmask = rdmsr(MSR_SFMASK); 1170 cpudata->fsbase = rdmsr(MSR_FSBASE); 1171 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1172 } 1173 1174 static void 1175 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1176 { 1177 struct svm_cpudata *cpudata = vcpu->cpudata; 1178 1179 wrmsr(MSR_STAR, cpudata->star); 1180 wrmsr(MSR_LSTAR, cpudata->lstar); 1181 wrmsr(MSR_CSTAR, cpudata->cstar); 1182 wrmsr(MSR_SFMASK, cpudata->sfmask); 1183 wrmsr(MSR_FSBASE, cpudata->fsbase); 1184 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1185 } 1186 1187 static int 1188 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1189 struct nvmm_exit *exit) 1190 { 1191 struct svm_cpudata *cpudata = vcpu->cpudata; 1192 struct vmcb *vmcb = cpudata->vmcb; 1193 bool tlb_need_flush = false; 1194 int hcpu, s; 1195 1196 kpreempt_disable(); 1197 hcpu = cpu_number(); 1198 1199 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1200 tlb_need_flush = true; 1201 } 1202 1203 if (cpudata->tlb_want_flush || tlb_need_flush) { 1204 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1205 } else { 1206 vmcb->ctrl.tlb_ctrl = 0; 1207 } 1208 1209 if (vcpu->hcpu_last != hcpu) { 1210 vmcb->ctrl.tsc_offset = cpudata->tsc_offset + 1211 curcpu()->ci_data.cpu_cc_skew; 1212 svm_vmcb_cache_flush_all(vmcb); 1213 } 1214 1215 svm_vcpu_guest_dbregs_enter(vcpu); 1216 svm_vcpu_guest_misc_enter(vcpu); 1217 1218 while (1) { 1219 s = splhigh(); 1220 svm_vcpu_guest_fpu_enter(vcpu); 1221 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1222 svm_vcpu_guest_fpu_leave(vcpu); 1223 splx(s); 1224 1225 svm_vmcb_cache_default(vmcb); 1226 1227 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1228 if (cpudata->tlb_want_flush) { 1229 cpudata->tlb_want_flush = false; 1230 } 1231 vcpu->hcpu_last = hcpu; 1232 } 1233 1234 switch (vmcb->ctrl.exitcode) { 1235 case VMCB_EXITCODE_INTR: 1236 case VMCB_EXITCODE_NMI: 1237 exit->reason = NVMM_EXIT_NONE; 1238 break; 1239 case VMCB_EXITCODE_VINTR: 1240 svm_event_waitexit_disable(vcpu, false); 1241 exit->reason = NVMM_EXIT_INT_READY; 1242 break; 1243 case VMCB_EXITCODE_IRET: 1244 svm_event_waitexit_disable(vcpu, true); 1245 exit->reason = NVMM_EXIT_NMI_READY; 1246 break; 1247 case VMCB_EXITCODE_CPUID: 1248 svm_exit_cpuid(mach, vcpu, exit); 1249 break; 1250 case VMCB_EXITCODE_HLT: 1251 svm_exit_hlt(mach, vcpu, exit); 1252 break; 1253 case VMCB_EXITCODE_IOIO: 1254 svm_exit_io(mach, vcpu, exit); 1255 break; 1256 case VMCB_EXITCODE_MSR: 1257 svm_exit_msr(mach, vcpu, exit); 1258 break; 1259 case VMCB_EXITCODE_SHUTDOWN: 1260 exit->reason = NVMM_EXIT_SHUTDOWN; 1261 break; 1262 case VMCB_EXITCODE_RDPMC: 1263 case VMCB_EXITCODE_RSM: 1264 case VMCB_EXITCODE_INVLPGA: 1265 case VMCB_EXITCODE_VMRUN: 1266 case VMCB_EXITCODE_VMMCALL: 1267 case VMCB_EXITCODE_VMLOAD: 1268 case VMCB_EXITCODE_VMSAVE: 1269 case VMCB_EXITCODE_STGI: 1270 case VMCB_EXITCODE_CLGI: 1271 case VMCB_EXITCODE_SKINIT: 1272 case VMCB_EXITCODE_RDTSCP: 1273 svm_inject_ud(mach, vcpu); 1274 exit->reason = NVMM_EXIT_NONE; 1275 break; 1276 case VMCB_EXITCODE_MONITOR: 1277 svm_exit_insn(vmcb, exit, NVMM_EXIT_MONITOR); 1278 break; 1279 case VMCB_EXITCODE_MWAIT: 1280 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT); 1281 break; 1282 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1283 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT_COND); 1284 break; 1285 case VMCB_EXITCODE_XSETBV: 1286 svm_exit_xsetbv(mach, vcpu, exit); 1287 break; 1288 case VMCB_EXITCODE_NPF: 1289 svm_exit_npf(mach, vcpu, exit); 1290 break; 1291 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1292 default: 1293 exit->reason = NVMM_EXIT_INVALID; 1294 break; 1295 } 1296 1297 /* If no reason to return to userland, keep rolling. */ 1298 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1299 break; 1300 } 1301 if (curcpu()->ci_data.cpu_softints != 0) { 1302 break; 1303 } 1304 if (curlwp->l_flag & LW_USERRET) { 1305 break; 1306 } 1307 if (exit->reason != NVMM_EXIT_NONE) { 1308 break; 1309 } 1310 } 1311 1312 svm_vcpu_guest_misc_leave(vcpu); 1313 svm_vcpu_guest_dbregs_leave(vcpu); 1314 1315 kpreempt_enable(); 1316 1317 exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1318 VMCB_CTRL_V_TPR); 1319 exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags; 1320 1321 exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] = 1322 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1323 exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] = 1324 cpudata->int_window_exit; 1325 exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] = 1326 cpudata->nmi_window_exit; 1327 1328 return 0; 1329 } 1330 1331 /* -------------------------------------------------------------------------- */ 1332 1333 static int 1334 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1335 { 1336 struct pglist pglist; 1337 paddr_t _pa; 1338 vaddr_t _va; 1339 size_t i; 1340 int ret; 1341 1342 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1343 &pglist, 1, 0); 1344 if (ret != 0) 1345 return ENOMEM; 1346 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1347 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1348 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1349 if (_va == 0) 1350 goto error; 1351 1352 for (i = 0; i < npages; i++) { 1353 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1354 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1355 } 1356 pmap_update(pmap_kernel()); 1357 1358 memset((void *)_va, 0, npages * PAGE_SIZE); 1359 1360 *pa = _pa; 1361 *va = _va; 1362 return 0; 1363 1364 error: 1365 for (i = 0; i < npages; i++) { 1366 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1367 } 1368 return ENOMEM; 1369 } 1370 1371 static void 1372 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1373 { 1374 size_t i; 1375 1376 pmap_kremove(va, npages * PAGE_SIZE); 1377 pmap_update(pmap_kernel()); 1378 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1379 for (i = 0; i < npages; i++) { 1380 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1381 } 1382 } 1383 1384 /* -------------------------------------------------------------------------- */ 1385 1386 #define SVM_MSRBM_READ __BIT(0) 1387 #define SVM_MSRBM_WRITE __BIT(1) 1388 1389 static void 1390 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1391 { 1392 uint64_t byte; 1393 uint8_t bitoff; 1394 1395 if (msr < 0x00002000) { 1396 /* Range 1 */ 1397 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1398 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1399 /* Range 2 */ 1400 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1401 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1402 /* Range 3 */ 1403 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1404 } else { 1405 panic("%s: wrong range", __func__); 1406 } 1407 1408 bitoff = (msr & 0x3) << 1; 1409 1410 if (read) { 1411 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1412 } 1413 if (write) { 1414 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1415 } 1416 } 1417 1418 static void 1419 svm_asid_alloc(struct nvmm_cpu *vcpu) 1420 { 1421 struct svm_cpudata *cpudata = vcpu->cpudata; 1422 struct vmcb *vmcb = cpudata->vmcb; 1423 size_t i, oct, bit; 1424 1425 mutex_enter(&svm_asidlock); 1426 1427 for (i = 0; i < svm_maxasid; i++) { 1428 oct = i / 8; 1429 bit = i % 8; 1430 1431 if (svm_asidmap[oct] & __BIT(bit)) { 1432 continue; 1433 } 1434 1435 svm_asidmap[oct] |= __BIT(bit); 1436 vmcb->ctrl.guest_asid = i; 1437 mutex_exit(&svm_asidlock); 1438 return; 1439 } 1440 1441 /* 1442 * No free ASID. Use the last one, which is shared and requires 1443 * special TLB handling. 1444 */ 1445 cpudata->shared_asid = true; 1446 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1447 mutex_exit(&svm_asidlock); 1448 } 1449 1450 static void 1451 svm_asid_free(struct nvmm_cpu *vcpu) 1452 { 1453 struct svm_cpudata *cpudata = vcpu->cpudata; 1454 struct vmcb *vmcb = cpudata->vmcb; 1455 size_t oct, bit; 1456 1457 if (cpudata->shared_asid) { 1458 return; 1459 } 1460 1461 oct = vmcb->ctrl.guest_asid / 8; 1462 bit = vmcb->ctrl.guest_asid % 8; 1463 1464 mutex_enter(&svm_asidlock); 1465 svm_asidmap[oct] &= ~__BIT(bit); 1466 mutex_exit(&svm_asidlock); 1467 } 1468 1469 static void 1470 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1471 { 1472 struct svm_cpudata *cpudata = vcpu->cpudata; 1473 struct vmcb *vmcb = cpudata->vmcb; 1474 1475 /* Allow reads/writes of Control Registers. */ 1476 vmcb->ctrl.intercept_cr = 0; 1477 1478 /* Allow reads/writes of Debug Registers. */ 1479 vmcb->ctrl.intercept_dr = 0; 1480 1481 /* Allow exceptions 0 to 31. */ 1482 vmcb->ctrl.intercept_vec = 0; 1483 1484 /* 1485 * Allow: 1486 * - SMI [smm interrupts] 1487 * - VINTR [virtual interrupts] 1488 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1489 * - RIDTR [reads of IDTR] 1490 * - RGDTR [reads of GDTR] 1491 * - RLDTR [reads of LDTR] 1492 * - RTR [reads of TR] 1493 * - WIDTR [writes of IDTR] 1494 * - WGDTR [writes of GDTR] 1495 * - WLDTR [writes of LDTR] 1496 * - WTR [writes of TR] 1497 * - RDTSC [rdtsc instruction] 1498 * - PUSHF [pushf instruction] 1499 * - POPF [popf instruction] 1500 * - IRET [iret instruction] 1501 * - INTN [int $n instructions] 1502 * - INVD [invd instruction] 1503 * - PAUSE [pause instruction] 1504 * - INVLPG [invplg instruction] 1505 * - TASKSW [task switches] 1506 * 1507 * Intercept the rest below. 1508 */ 1509 vmcb->ctrl.intercept_misc1 = 1510 VMCB_CTRL_INTERCEPT_INTR | 1511 VMCB_CTRL_INTERCEPT_NMI | 1512 VMCB_CTRL_INTERCEPT_INIT | 1513 VMCB_CTRL_INTERCEPT_RDPMC | 1514 VMCB_CTRL_INTERCEPT_CPUID | 1515 VMCB_CTRL_INTERCEPT_RSM | 1516 VMCB_CTRL_INTERCEPT_HLT | 1517 VMCB_CTRL_INTERCEPT_INVLPGA | 1518 VMCB_CTRL_INTERCEPT_IOIO_PROT | 1519 VMCB_CTRL_INTERCEPT_MSR_PROT | 1520 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 1521 VMCB_CTRL_INTERCEPT_SHUTDOWN; 1522 1523 /* 1524 * Allow: 1525 * - ICEBP [icebp instruction] 1526 * - WBINVD [wbinvd instruction] 1527 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 1528 * 1529 * Intercept the rest below. 1530 */ 1531 vmcb->ctrl.intercept_misc2 = 1532 VMCB_CTRL_INTERCEPT_VMRUN | 1533 VMCB_CTRL_INTERCEPT_VMMCALL | 1534 VMCB_CTRL_INTERCEPT_VMLOAD | 1535 VMCB_CTRL_INTERCEPT_VMSAVE | 1536 VMCB_CTRL_INTERCEPT_STGI | 1537 VMCB_CTRL_INTERCEPT_CLGI | 1538 VMCB_CTRL_INTERCEPT_SKINIT | 1539 VMCB_CTRL_INTERCEPT_RDTSCP | 1540 VMCB_CTRL_INTERCEPT_MONITOR | 1541 VMCB_CTRL_INTERCEPT_MWAIT | 1542 VMCB_CTRL_INTERCEPT_XSETBV; 1543 1544 /* Intercept all I/O accesses. */ 1545 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 1546 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 1547 1548 /* 1549 * Allow: 1550 * - EFER [read] 1551 * - STAR [read, write] 1552 * - LSTAR [read, write] 1553 * - CSTAR [read, write] 1554 * - SFMASK [read, write] 1555 * - KERNELGSBASE [read, write] 1556 * - SYSENTER_CS [read, write] 1557 * - SYSENTER_ESP [read, write] 1558 * - SYSENTER_EIP [read, write] 1559 * - FSBASE [read, write] 1560 * - GSBASE [read, write] 1561 * - TSC [read] 1562 * 1563 * Intercept the rest. 1564 */ 1565 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 1566 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 1567 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 1568 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 1569 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 1570 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 1571 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 1572 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 1573 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 1574 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 1575 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 1576 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 1577 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 1578 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 1579 1580 /* Generate ASID. */ 1581 svm_asid_alloc(vcpu); 1582 1583 /* Virtual TPR. */ 1584 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 1585 1586 /* Enable Nested Paging. */ 1587 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 1588 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 1589 1590 /* Must always be set. */ 1591 vmcb->state.efer = EFER_SVME; 1592 cpudata->gxcr0 = XCR0_X87; 1593 1594 /* Init XSAVE header. */ 1595 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1596 cpudata->gfpu.xsh_xcomp_bv = 0; 1597 1598 /* Bluntly hide the host TSC. */ 1599 cpudata->tsc_offset = rdtsc(); 1600 } 1601 1602 static int 1603 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1604 { 1605 struct svm_cpudata *cpudata; 1606 int error; 1607 1608 /* Allocate the SVM cpudata. */ 1609 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 1610 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 1611 UVM_KMF_WIRED|UVM_KMF_ZERO); 1612 vcpu->cpudata = cpudata; 1613 1614 /* VMCB */ 1615 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 1616 VMCB_NPAGES); 1617 if (error) 1618 goto error; 1619 1620 /* I/O Bitmap */ 1621 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 1622 IOBM_NPAGES); 1623 if (error) 1624 goto error; 1625 1626 /* MSR Bitmap */ 1627 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 1628 MSRBM_NPAGES); 1629 if (error) 1630 goto error; 1631 1632 /* Init the VCPU info. */ 1633 svm_vcpu_init(mach, vcpu); 1634 1635 return 0; 1636 1637 error: 1638 if (cpudata->vmcb_pa) { 1639 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 1640 VMCB_NPAGES); 1641 } 1642 if (cpudata->iobm_pa) { 1643 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 1644 IOBM_NPAGES); 1645 } 1646 if (cpudata->msrbm_pa) { 1647 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 1648 MSRBM_NPAGES); 1649 } 1650 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1651 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1652 return error; 1653 } 1654 1655 static void 1656 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1657 { 1658 struct svm_cpudata *cpudata = vcpu->cpudata; 1659 1660 svm_asid_free(vcpu); 1661 1662 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 1663 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 1664 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 1665 1666 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1667 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1668 } 1669 1670 #define SVM_SEG_ATTRIB_TYPE __BITS(4,0) 1671 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1672 #define SVM_SEG_ATTRIB_P __BIT(7) 1673 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1674 #define SVM_SEG_ATTRIB_LONG __BIT(9) 1675 #define SVM_SEG_ATTRIB_DEF32 __BIT(10) 1676 #define SVM_SEG_ATTRIB_GRAN __BIT(11) 1677 1678 static void 1679 svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1680 { 1681 vseg->selector = seg->selector; 1682 vseg->attrib = 1683 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1684 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1685 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1686 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1687 __SHIFTIN(seg->attrib.lng, SVM_SEG_ATTRIB_LONG) | 1688 __SHIFTIN(seg->attrib.def32, SVM_SEG_ATTRIB_DEF32) | 1689 __SHIFTIN(seg->attrib.gran, SVM_SEG_ATTRIB_GRAN); 1690 vseg->limit = seg->limit; 1691 vseg->base = seg->base; 1692 } 1693 1694 static void 1695 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1696 { 1697 seg->selector = vseg->selector; 1698 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1699 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1700 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1701 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1702 seg->attrib.lng = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_LONG); 1703 seg->attrib.def32 = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF32); 1704 seg->attrib.gran = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_GRAN); 1705 seg->limit = vseg->limit; 1706 seg->base = vseg->base; 1707 } 1708 1709 static inline bool 1710 svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, 1711 uint64_t flags) 1712 { 1713 if (flags & NVMM_X64_STATE_CRS) { 1714 if ((vmcb->state.cr0 ^ 1715 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1716 return true; 1717 } 1718 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1719 return true; 1720 } 1721 if ((vmcb->state.cr4 ^ 1722 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1723 return true; 1724 } 1725 } 1726 1727 if (flags & NVMM_X64_STATE_MSRS) { 1728 if ((vmcb->state.efer ^ 1729 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1730 return true; 1731 } 1732 } 1733 1734 return false; 1735 } 1736 1737 static void 1738 svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1739 { 1740 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1741 struct svm_cpudata *cpudata = vcpu->cpudata; 1742 struct vmcb *vmcb = cpudata->vmcb; 1743 struct fxsave *fpustate; 1744 1745 if (svm_state_tlb_flush(vmcb, state, flags)) { 1746 cpudata->tlb_want_flush = true; 1747 } 1748 1749 if (flags & NVMM_X64_STATE_SEGS) { 1750 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1751 &vmcb->state.cs); 1752 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1753 &vmcb->state.ds); 1754 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1755 &vmcb->state.es); 1756 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1757 &vmcb->state.fs); 1758 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1759 &vmcb->state.gs); 1760 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1761 &vmcb->state.ss); 1762 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1763 &vmcb->state.gdt); 1764 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1765 &vmcb->state.idt); 1766 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1767 &vmcb->state.ldt); 1768 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1769 &vmcb->state.tr); 1770 } 1771 1772 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1773 if (flags & NVMM_X64_STATE_GPRS) { 1774 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1775 1776 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1777 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1778 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1779 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1780 } 1781 1782 if (flags & NVMM_X64_STATE_CRS) { 1783 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1784 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1785 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1786 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1787 1788 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1789 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1790 VMCB_CTRL_V_TPR); 1791 1792 if (svm_xcr0_mask != 0) { 1793 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1794 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1795 cpudata->gxcr0 &= svm_xcr0_mask; 1796 cpudata->gxcr0 |= XCR0_X87; 1797 } 1798 } 1799 1800 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1801 if (flags & NVMM_X64_STATE_DRS) { 1802 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1803 1804 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1805 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1806 } 1807 1808 if (flags & NVMM_X64_STATE_MSRS) { 1809 /* Bit EFER_SVME is mandatory. */ 1810 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1811 1812 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1813 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1814 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1815 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1816 vmcb->state.kernelgsbase = 1817 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1818 vmcb->state.sysenter_cs = 1819 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1820 vmcb->state.sysenter_esp = 1821 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1822 vmcb->state.sysenter_eip = 1823 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1824 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1825 } 1826 1827 if (flags & NVMM_X64_STATE_MISC) { 1828 vmcb->state.cpl = state->misc[NVMM_X64_MISC_CPL]; 1829 1830 if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { 1831 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1832 } else { 1833 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1834 } 1835 1836 if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { 1837 svm_event_waitexit_enable(vcpu, false); 1838 } else { 1839 svm_event_waitexit_disable(vcpu, false); 1840 } 1841 1842 if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { 1843 svm_event_waitexit_enable(vcpu, true); 1844 } else { 1845 svm_event_waitexit_disable(vcpu, true); 1846 } 1847 } 1848 1849 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1850 if (flags & NVMM_X64_STATE_FPU) { 1851 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1852 sizeof(state->fpu)); 1853 1854 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1855 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1856 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1857 1858 if (svm_xcr0_mask != 0) { 1859 /* Reset XSTATE_BV, to force a reload. */ 1860 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1861 } 1862 } 1863 1864 svm_vmcb_cache_update(vmcb, flags); 1865 } 1866 1867 static void 1868 svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1869 { 1870 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1871 struct svm_cpudata *cpudata = vcpu->cpudata; 1872 struct vmcb *vmcb = cpudata->vmcb; 1873 1874 if (flags & NVMM_X64_STATE_SEGS) { 1875 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1876 &vmcb->state.cs); 1877 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1878 &vmcb->state.ds); 1879 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1880 &vmcb->state.es); 1881 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1882 &vmcb->state.fs); 1883 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1884 &vmcb->state.gs); 1885 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1886 &vmcb->state.ss); 1887 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1888 &vmcb->state.gdt); 1889 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1890 &vmcb->state.idt); 1891 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1892 &vmcb->state.ldt); 1893 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1894 &vmcb->state.tr); 1895 } 1896 1897 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1898 if (flags & NVMM_X64_STATE_GPRS) { 1899 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1900 1901 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1902 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1903 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1904 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1905 } 1906 1907 if (flags & NVMM_X64_STATE_CRS) { 1908 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1909 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1910 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1911 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1912 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1913 VMCB_CTRL_V_TPR); 1914 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1915 } 1916 1917 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1918 if (flags & NVMM_X64_STATE_DRS) { 1919 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1920 1921 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1922 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1923 } 1924 1925 if (flags & NVMM_X64_STATE_MSRS) { 1926 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1927 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1928 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1929 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1930 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1931 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1932 vmcb->state.kernelgsbase; 1933 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1934 vmcb->state.sysenter_cs; 1935 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1936 vmcb->state.sysenter_esp; 1937 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1938 vmcb->state.sysenter_eip; 1939 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1940 1941 /* Hide SVME. */ 1942 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1943 } 1944 1945 if (flags & NVMM_X64_STATE_MISC) { 1946 state->misc[NVMM_X64_MISC_CPL] = vmcb->state.cpl; 1947 1948 state->misc[NVMM_X64_MISC_INT_SHADOW] = 1949 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1950 state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = 1951 cpudata->int_window_exit; 1952 state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = 1953 cpudata->nmi_window_exit; 1954 } 1955 1956 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1957 if (flags & NVMM_X64_STATE_FPU) { 1958 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 1959 sizeof(state->fpu)); 1960 } 1961 } 1962 1963 /* -------------------------------------------------------------------------- */ 1964 1965 static void 1966 svm_tlb_flush(struct pmap *pm) 1967 { 1968 struct nvmm_machine *mach = pm->pm_data; 1969 struct svm_cpudata *cpudata; 1970 struct nvmm_cpu *vcpu; 1971 int error; 1972 size_t i; 1973 1974 /* Request TLB flushes. */ 1975 for (i = 0; i < NVMM_MAX_VCPUS; i++) { 1976 error = nvmm_vcpu_get(mach, i, &vcpu); 1977 if (error) 1978 continue; 1979 cpudata = vcpu->cpudata; 1980 cpudata->tlb_want_flush = true; 1981 nvmm_vcpu_put(vcpu); 1982 } 1983 } 1984 1985 static void 1986 svm_machine_create(struct nvmm_machine *mach) 1987 { 1988 /* Fill in pmap info. */ 1989 mach->vm->vm_map.pmap->pm_data = (void *)mach; 1990 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 1991 1992 mach->machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 1993 } 1994 1995 static void 1996 svm_machine_destroy(struct nvmm_machine *mach) 1997 { 1998 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 1999 } 2000 2001 static int 2002 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2003 { 2004 struct nvmm_x86_conf_cpuid *cpuid = data; 2005 struct svm_machdata *machdata = (struct svm_machdata *)mach->machdata; 2006 size_t i; 2007 2008 if (__predict_false(op != NVMM_X86_CONF_CPUID)) { 2009 return EINVAL; 2010 } 2011 2012 if (__predict_false((cpuid->set.eax & cpuid->del.eax) || 2013 (cpuid->set.ebx & cpuid->del.ebx) || 2014 (cpuid->set.ecx & cpuid->del.ecx) || 2015 (cpuid->set.edx & cpuid->del.edx))) { 2016 return EINVAL; 2017 } 2018 2019 /* If already here, replace. */ 2020 for (i = 0; i < SVM_NCPUIDS; i++) { 2021 if (!machdata->cpuidpresent[i]) { 2022 continue; 2023 } 2024 if (machdata->cpuid[i].leaf == cpuid->leaf) { 2025 memcpy(&machdata->cpuid[i], cpuid, 2026 sizeof(struct nvmm_x86_conf_cpuid)); 2027 return 0; 2028 } 2029 } 2030 2031 /* Not here, insert. */ 2032 for (i = 0; i < SVM_NCPUIDS; i++) { 2033 if (!machdata->cpuidpresent[i]) { 2034 machdata->cpuidpresent[i] = true; 2035 memcpy(&machdata->cpuid[i], cpuid, 2036 sizeof(struct nvmm_x86_conf_cpuid)); 2037 return 0; 2038 } 2039 } 2040 2041 return ENOBUFS; 2042 } 2043 2044 /* -------------------------------------------------------------------------- */ 2045 2046 static bool 2047 svm_ident(void) 2048 { 2049 u_int descs[4]; 2050 uint64_t msr; 2051 2052 if (cpu_vendor != CPUVENDOR_AMD) { 2053 return false; 2054 } 2055 if (!(cpu_feature[3] & CPUID_SVM)) { 2056 return false; 2057 } 2058 2059 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2060 return false; 2061 } 2062 x86_cpuid(0x8000000a, descs); 2063 2064 /* Want Nested Paging. */ 2065 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2066 return false; 2067 } 2068 2069 /* Want nRIP. */ 2070 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2071 return false; 2072 } 2073 2074 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2075 2076 msr = rdmsr(MSR_VMCR); 2077 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2078 return false; 2079 } 2080 2081 return true; 2082 } 2083 2084 static void 2085 svm_init_asid(uint32_t maxasid) 2086 { 2087 size_t i, j, allocsz; 2088 2089 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2090 2091 /* Arbitrarily limit. */ 2092 maxasid = uimin(maxasid, 8192); 2093 2094 svm_maxasid = maxasid; 2095 allocsz = roundup(maxasid, 8) / 8; 2096 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2097 2098 /* ASID 0 is reserved for the host. */ 2099 svm_asidmap[0] |= __BIT(0); 2100 2101 /* ASID n-1 is special, we share it. */ 2102 i = (maxasid - 1) / 8; 2103 j = (maxasid - 1) % 8; 2104 svm_asidmap[i] |= __BIT(j); 2105 } 2106 2107 static void 2108 svm_change_cpu(void *arg1, void *arg2) 2109 { 2110 bool enable = (bool)arg1; 2111 uint64_t msr; 2112 2113 msr = rdmsr(MSR_VMCR); 2114 if (msr & VMCR_SVMED) { 2115 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2116 } 2117 2118 if (!enable) { 2119 wrmsr(MSR_VM_HSAVE_PA, 0); 2120 } 2121 2122 msr = rdmsr(MSR_EFER); 2123 if (enable) { 2124 msr |= EFER_SVME; 2125 } else { 2126 msr &= ~EFER_SVME; 2127 } 2128 wrmsr(MSR_EFER, msr); 2129 2130 if (enable) { 2131 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2132 } 2133 } 2134 2135 static void 2136 svm_init(void) 2137 { 2138 CPU_INFO_ITERATOR cii; 2139 struct cpu_info *ci; 2140 struct vm_page *pg; 2141 u_int descs[4]; 2142 uint64_t xc; 2143 2144 x86_cpuid(0x8000000a, descs); 2145 2146 /* The guest TLB flush command. */ 2147 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2148 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2149 } else { 2150 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2151 } 2152 2153 /* Init the ASID. */ 2154 svm_init_asid(descs[1]); 2155 2156 /* Init the XCR0 mask. */ 2157 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2158 2159 memset(hsave, 0, sizeof(hsave)); 2160 for (CPU_INFO_FOREACH(cii, ci)) { 2161 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2162 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2163 } 2164 2165 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2166 xc_wait(xc); 2167 } 2168 2169 static void 2170 svm_fini_asid(void) 2171 { 2172 size_t allocsz; 2173 2174 allocsz = roundup(svm_maxasid, 8) / 8; 2175 kmem_free(svm_asidmap, allocsz); 2176 2177 mutex_destroy(&svm_asidlock); 2178 } 2179 2180 static void 2181 svm_fini(void) 2182 { 2183 uint64_t xc; 2184 size_t i; 2185 2186 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2187 xc_wait(xc); 2188 2189 for (i = 0; i < MAXCPUS; i++) { 2190 if (hsave[i].pa != 0) 2191 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2192 } 2193 2194 svm_fini_asid(); 2195 } 2196 2197 static void 2198 svm_capability(struct nvmm_capability *cap) 2199 { 2200 cap->u.x86.xcr0_mask = svm_xcr0_mask; 2201 cap->u.x86.mxcsr_mask = x86_fpu_mxcsr_mask; 2202 cap->u.x86.conf_cpuid_maxops = SVM_NCPUIDS; 2203 } 2204 2205 const struct nvmm_impl nvmm_x86_svm = { 2206 .ident = svm_ident, 2207 .init = svm_init, 2208 .fini = svm_fini, 2209 .capability = svm_capability, 2210 .conf_max = NVMM_X86_NCONF, 2211 .conf_sizes = svm_conf_sizes, 2212 .state_size = sizeof(struct nvmm_x64_state), 2213 .machine_create = svm_machine_create, 2214 .machine_destroy = svm_machine_destroy, 2215 .machine_configure = svm_machine_configure, 2216 .vcpu_create = svm_vcpu_create, 2217 .vcpu_destroy = svm_vcpu_destroy, 2218 .vcpu_setstate = svm_vcpu_setstate, 2219 .vcpu_getstate = svm_vcpu_getstate, 2220 .vcpu_inject = svm_vcpu_inject, 2221 .vcpu_run = svm_vcpu_run 2222 }; 2223
Indexes created Tue Sep 30 20:09:53 GMT 2025