1 /* $NetBSD: nvmm_x86_svm.c,v 1.96 2026/07/15 01:22:21 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.96 2026/07/15 01:22:21 riastradh Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/kernel.h> 37 #include <sys/kmem.h> 38 #include <sys/cpu.h> 39 #include <sys/xcall.h> 40 #include <sys/mman.h> 41 42 #include <uvm/uvm_extern.h> 43 #include <uvm/uvm_page.h> 44 45 #include <x86/cputypes.h> 46 #include <x86/specialreg.h> 47 #include <x86/dbregs.h> 48 #include <x86/cpu_counter.h> 49 50 #include <machine/cpuvar.h> 51 #include <machine/pmap_private.h> 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/x86/nvmm_x86.h> 56 57 int svm_vmrun(paddr_t, uint64_t *); 58 59 static inline void 60 svm_clgi(void) 61 { 62 asm volatile ("clgi" ::: "memory"); 63 } 64 65 static inline void 66 svm_stgi(void) 67 { 68 asm volatile ("stgi" ::: "memory"); 69 } 70 71 #define MSR_VM_HSAVE_PA 0xC0010117 72 73 /* -------------------------------------------------------------------------- */ 74 75 #define VMCB_EXITCODE_CR0_READ 0x0000 76 #define VMCB_EXITCODE_CR1_READ 0x0001 77 #define VMCB_EXITCODE_CR2_READ 0x0002 78 #define VMCB_EXITCODE_CR3_READ 0x0003 79 #define VMCB_EXITCODE_CR4_READ 0x0004 80 #define VMCB_EXITCODE_CR5_READ 0x0005 81 #define VMCB_EXITCODE_CR6_READ 0x0006 82 #define VMCB_EXITCODE_CR7_READ 0x0007 83 #define VMCB_EXITCODE_CR8_READ 0x0008 84 #define VMCB_EXITCODE_CR9_READ 0x0009 85 #define VMCB_EXITCODE_CR10_READ 0x000A 86 #define VMCB_EXITCODE_CR11_READ 0x000B 87 #define VMCB_EXITCODE_CR12_READ 0x000C 88 #define VMCB_EXITCODE_CR13_READ 0x000D 89 #define VMCB_EXITCODE_CR14_READ 0x000E 90 #define VMCB_EXITCODE_CR15_READ 0x000F 91 #define VMCB_EXITCODE_CR0_WRITE 0x0010 92 #define VMCB_EXITCODE_CR1_WRITE 0x0011 93 #define VMCB_EXITCODE_CR2_WRITE 0x0012 94 #define VMCB_EXITCODE_CR3_WRITE 0x0013 95 #define VMCB_EXITCODE_CR4_WRITE 0x0014 96 #define VMCB_EXITCODE_CR5_WRITE 0x0015 97 #define VMCB_EXITCODE_CR6_WRITE 0x0016 98 #define VMCB_EXITCODE_CR7_WRITE 0x0017 99 #define VMCB_EXITCODE_CR8_WRITE 0x0018 100 #define VMCB_EXITCODE_CR9_WRITE 0x0019 101 #define VMCB_EXITCODE_CR10_WRITE 0x001A 102 #define VMCB_EXITCODE_CR11_WRITE 0x001B 103 #define VMCB_EXITCODE_CR12_WRITE 0x001C 104 #define VMCB_EXITCODE_CR13_WRITE 0x001D 105 #define VMCB_EXITCODE_CR14_WRITE 0x001E 106 #define VMCB_EXITCODE_CR15_WRITE 0x001F 107 #define VMCB_EXITCODE_DR0_READ 0x0020 108 #define VMCB_EXITCODE_DR1_READ 0x0021 109 #define VMCB_EXITCODE_DR2_READ 0x0022 110 #define VMCB_EXITCODE_DR3_READ 0x0023 111 #define VMCB_EXITCODE_DR4_READ 0x0024 112 #define VMCB_EXITCODE_DR5_READ 0x0025 113 #define VMCB_EXITCODE_DR6_READ 0x0026 114 #define VMCB_EXITCODE_DR7_READ 0x0027 115 #define VMCB_EXITCODE_DR8_READ 0x0028 116 #define VMCB_EXITCODE_DR9_READ 0x0029 117 #define VMCB_EXITCODE_DR10_READ 0x002A 118 #define VMCB_EXITCODE_DR11_READ 0x002B 119 #define VMCB_EXITCODE_DR12_READ 0x002C 120 #define VMCB_EXITCODE_DR13_READ 0x002D 121 #define VMCB_EXITCODE_DR14_READ 0x002E 122 #define VMCB_EXITCODE_DR15_READ 0x002F 123 #define VMCB_EXITCODE_DR0_WRITE 0x0030 124 #define VMCB_EXITCODE_DR1_WRITE 0x0031 125 #define VMCB_EXITCODE_DR2_WRITE 0x0032 126 #define VMCB_EXITCODE_DR3_WRITE 0x0033 127 #define VMCB_EXITCODE_DR4_WRITE 0x0034 128 #define VMCB_EXITCODE_DR5_WRITE 0x0035 129 #define VMCB_EXITCODE_DR6_WRITE 0x0036 130 #define VMCB_EXITCODE_DR7_WRITE 0x0037 131 #define VMCB_EXITCODE_DR8_WRITE 0x0038 132 #define VMCB_EXITCODE_DR9_WRITE 0x0039 133 #define VMCB_EXITCODE_DR10_WRITE 0x003A 134 #define VMCB_EXITCODE_DR11_WRITE 0x003B 135 #define VMCB_EXITCODE_DR12_WRITE 0x003C 136 #define VMCB_EXITCODE_DR13_WRITE 0x003D 137 #define VMCB_EXITCODE_DR14_WRITE 0x003E 138 #define VMCB_EXITCODE_DR15_WRITE 0x003F 139 #define VMCB_EXITCODE_EXCP0 0x0040 140 #define VMCB_EXITCODE_EXCP1 0x0041 141 #define VMCB_EXITCODE_EXCP2 0x0042 142 #define VMCB_EXITCODE_EXCP3 0x0043 143 #define VMCB_EXITCODE_EXCP4 0x0044 144 #define VMCB_EXITCODE_EXCP5 0x0045 145 #define VMCB_EXITCODE_EXCP6 0x0046 146 #define VMCB_EXITCODE_EXCP7 0x0047 147 #define VMCB_EXITCODE_EXCP8 0x0048 148 #define VMCB_EXITCODE_EXCP9 0x0049 149 #define VMCB_EXITCODE_EXCP10 0x004A 150 #define VMCB_EXITCODE_EXCP11 0x004B 151 #define VMCB_EXITCODE_EXCP12 0x004C 152 #define VMCB_EXITCODE_EXCP13 0x004D 153 #define VMCB_EXITCODE_EXCP14 0x004E 154 #define VMCB_EXITCODE_EXCP15 0x004F 155 #define VMCB_EXITCODE_EXCP16 0x0050 156 #define VMCB_EXITCODE_EXCP17 0x0051 157 #define VMCB_EXITCODE_EXCP18 0x0052 158 #define VMCB_EXITCODE_EXCP19 0x0053 159 #define VMCB_EXITCODE_EXCP20 0x0054 160 #define VMCB_EXITCODE_EXCP21 0x0055 161 #define VMCB_EXITCODE_EXCP22 0x0056 162 #define VMCB_EXITCODE_EXCP23 0x0057 163 #define VMCB_EXITCODE_EXCP24 0x0058 164 #define VMCB_EXITCODE_EXCP25 0x0059 165 #define VMCB_EXITCODE_EXCP26 0x005A 166 #define VMCB_EXITCODE_EXCP27 0x005B 167 #define VMCB_EXITCODE_EXCP28 0x005C 168 #define VMCB_EXITCODE_EXCP29 0x005D 169 #define VMCB_EXITCODE_EXCP30 0x005E 170 #define VMCB_EXITCODE_EXCP31 0x005F 171 #define VMCB_EXITCODE_INTR 0x0060 172 #define VMCB_EXITCODE_NMI 0x0061 173 #define VMCB_EXITCODE_SMI 0x0062 174 #define VMCB_EXITCODE_INIT 0x0063 175 #define VMCB_EXITCODE_VINTR 0x0064 176 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 177 #define VMCB_EXITCODE_IDTR_READ 0x0066 178 #define VMCB_EXITCODE_GDTR_READ 0x0067 179 #define VMCB_EXITCODE_LDTR_READ 0x0068 180 #define VMCB_EXITCODE_TR_READ 0x0069 181 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 182 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 183 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 184 #define VMCB_EXITCODE_TR_WRITE 0x006D 185 #define VMCB_EXITCODE_RDTSC 0x006E 186 #define VMCB_EXITCODE_RDPMC 0x006F 187 #define VMCB_EXITCODE_PUSHF 0x0070 188 #define VMCB_EXITCODE_POPF 0x0071 189 #define VMCB_EXITCODE_CPUID 0x0072 190 #define VMCB_EXITCODE_RSM 0x0073 191 #define VMCB_EXITCODE_IRET 0x0074 192 #define VMCB_EXITCODE_SWINT 0x0075 193 #define VMCB_EXITCODE_INVD 0x0076 194 #define VMCB_EXITCODE_PAUSE 0x0077 195 #define VMCB_EXITCODE_HLT 0x0078 196 #define VMCB_EXITCODE_INVLPG 0x0079 197 #define VMCB_EXITCODE_INVLPGA 0x007A 198 #define VMCB_EXITCODE_IOIO 0x007B 199 #define VMCB_EXITCODE_MSR 0x007C 200 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 201 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 202 #define VMCB_EXITCODE_SHUTDOWN 0x007F 203 #define VMCB_EXITCODE_VMRUN 0x0080 204 #define VMCB_EXITCODE_VMMCALL 0x0081 205 #define VMCB_EXITCODE_VMLOAD 0x0082 206 #define VMCB_EXITCODE_VMSAVE 0x0083 207 #define VMCB_EXITCODE_STGI 0x0084 208 #define VMCB_EXITCODE_CLGI 0x0085 209 #define VMCB_EXITCODE_SKINIT 0x0086 210 #define VMCB_EXITCODE_RDTSCP 0x0087 211 #define VMCB_EXITCODE_ICEBP 0x0088 212 #define VMCB_EXITCODE_WBINVD 0x0089 213 #define VMCB_EXITCODE_MONITOR 0x008A 214 #define VMCB_EXITCODE_MWAIT 0x008B 215 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 216 #define VMCB_EXITCODE_XSETBV 0x008D 217 #define VMCB_EXITCODE_RDPRU 0x008E 218 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 219 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 220 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 221 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 222 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 223 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 224 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 225 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 226 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 227 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 228 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 229 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 230 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 231 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 232 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 233 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 234 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 235 #define VMCB_EXITCODE_INVLPGB 0x00A0 236 #define VMCB_EXITCODE_INVLPGB_ILLEGAL 0x00A1 237 #define VMCB_EXITCODE_INVPCID 0x00A2 238 #define VMCB_EXITCODE_MCOMMIT 0x00A3 239 #define VMCB_EXITCODE_TLBSYNC 0x00A4 240 #define VMCB_EXITCODE_NPF 0x0400 241 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 242 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 243 #define VMCB_EXITCODE_VMGEXIT 0x0403 244 #define VMCB_EXITCODE_BUSY -2ULL 245 #define VMCB_EXITCODE_INVALID -1ULL 246 247 /* -------------------------------------------------------------------------- */ 248 249 struct vmcb_ctrl { 250 uint32_t intercept_cr; 251 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 252 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 253 254 uint32_t intercept_dr; 255 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 256 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 257 258 uint32_t intercept_vec; 259 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 260 261 uint32_t intercept_misc1; 262 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 263 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 264 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 265 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 266 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 267 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 268 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 269 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 270 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 271 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 272 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 273 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 274 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 275 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 276 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 277 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 278 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 279 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 280 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 281 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 282 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 283 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 284 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 285 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 286 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 287 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 288 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 289 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 290 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 291 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 292 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 293 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 294 295 uint32_t intercept_misc2; 296 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 297 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 298 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 299 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 300 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 301 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 302 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 303 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 304 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 305 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 306 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 307 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(11) 308 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED __BIT(12) 309 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 310 #define VMCB_CTRL_INTERCEPT_RDPRU __BIT(14) 311 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 312 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 313 314 uint32_t intercept_misc3; 315 #define VMCB_CTRL_INTERCEPT_INVLPGB_ALL __BIT(0) 316 #define VMCB_CTRL_INTERCEPT_INVLPGB_ILL __BIT(1) 317 #define VMCB_CTRL_INTERCEPT_PCID __BIT(2) 318 #define VMCB_CTRL_INTERCEPT_MCOMMIT __BIT(3) 319 #define VMCB_CTRL_INTERCEPT_TLBSYNC __BIT(4) 320 321 uint8_t rsvd1[36]; 322 uint16_t pause_filt_thresh; 323 uint16_t pause_filt_cnt; 324 uint64_t iopm_base_pa; 325 uint64_t msrpm_base_pa; 326 uint64_t tsc_offset; 327 uint32_t guest_asid; 328 329 uint32_t tlb_ctrl; 330 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 331 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 332 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 333 334 uint64_t v; 335 #define VMCB_CTRL_V_TPR __BITS(3,0) 336 #define VMCB_CTRL_V_IRQ __BIT(8) 337 #define VMCB_CTRL_V_VGIF __BIT(9) 338 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 339 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 340 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 341 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 342 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 343 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 344 345 uint64_t intr; 346 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 347 #define VMCB_CTRL_INTR_MASK __BIT(1) 348 349 uint64_t exitcode; 350 uint64_t exitinfo1; 351 uint64_t exitinfo2; 352 353 uint64_t exitintinfo; 354 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 355 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 356 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 357 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 358 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 359 360 uint64_t enable1; 361 #define VMCB_CTRL_ENABLE_NP __BIT(0) 362 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 363 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 364 #define VMCB_CTRL_ENABLE_GMET __BIT(3) 365 #define VMCB_CTRL_ENABLE_VTE __BIT(5) 366 367 uint64_t avic; 368 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 369 370 uint64_t ghcb; 371 372 uint64_t eventinj; 373 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 374 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 375 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 376 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 377 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 378 379 uint64_t n_cr3; 380 381 uint64_t enable2; 382 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 383 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 384 385 uint32_t vmcb_clean; 386 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 387 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 388 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 389 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 390 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 391 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 392 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 393 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 394 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 395 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 396 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 397 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 398 399 uint32_t rsvd2; 400 uint64_t nrip; 401 uint8_t inst_len; 402 uint8_t inst_bytes[15]; 403 uint64_t avic_abpp; 404 uint64_t rsvd3; 405 uint64_t avic_ltp; 406 407 uint64_t avic_phys; 408 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 409 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 410 411 uint64_t rsvd4; 412 uint64_t vmsa_ptr; 413 414 uint8_t pad[752]; 415 } __packed; 416 417 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 418 419 struct vmcb_segment { 420 uint16_t selector; 421 uint16_t attrib; /* hidden */ 422 uint32_t limit; /* hidden */ 423 uint64_t base; /* hidden */ 424 } __packed; 425 426 CTASSERT(sizeof(struct vmcb_segment) == 16); 427 428 struct vmcb_state { 429 struct vmcb_segment es; 430 struct vmcb_segment cs; 431 struct vmcb_segment ss; 432 struct vmcb_segment ds; 433 struct vmcb_segment fs; 434 struct vmcb_segment gs; 435 struct vmcb_segment gdt; 436 struct vmcb_segment ldt; 437 struct vmcb_segment idt; 438 struct vmcb_segment tr; 439 uint8_t rsvd1[43]; 440 uint8_t cpl; 441 uint8_t rsvd2[4]; 442 uint64_t efer; 443 uint8_t rsvd3[112]; 444 uint64_t cr4; 445 uint64_t cr3; 446 uint64_t cr0; 447 uint64_t dr7; 448 uint64_t dr6; 449 uint64_t rflags; 450 uint64_t rip; 451 uint8_t rsvd4[88]; 452 uint64_t rsp; 453 uint8_t rsvd5[24]; 454 uint64_t rax; 455 uint64_t star; 456 uint64_t lstar; 457 uint64_t cstar; 458 uint64_t sfmask; 459 uint64_t kernelgsbase; 460 uint64_t sysenter_cs; 461 uint64_t sysenter_esp; 462 uint64_t sysenter_eip; 463 uint64_t cr2; 464 uint8_t rsvd6[32]; 465 uint64_t g_pat; 466 uint64_t dbgctl; 467 uint64_t br_from; 468 uint64_t br_to; 469 uint64_t int_from; 470 uint64_t int_to; 471 uint8_t pad[2408]; 472 } __packed; 473 474 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 475 476 struct vmcb { 477 struct vmcb_ctrl ctrl; 478 struct vmcb_state state; 479 } __packed; 480 481 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 482 CTASSERT(offsetof(struct vmcb, state) == 0x400); 483 484 /* -------------------------------------------------------------------------- */ 485 486 static void svm_vcpu_state_provide(struct nvmm_cpu *, uint64_t); 487 static void svm_vcpu_state_commit(struct nvmm_cpu *); 488 489 struct svm_hsave { 490 paddr_t pa; 491 }; 492 493 static struct svm_hsave hsave[MAXCPUS]; 494 495 static uint8_t *svm_asidmap __read_mostly; 496 static uint32_t svm_maxasid __read_mostly; 497 static kmutex_t svm_asidlock __cacheline_aligned; 498 499 static bool svm_decode_assist __read_mostly; 500 static uint32_t svm_ctrl_tlb_flush __read_mostly; 501 502 static uint64_t svm_xcr0_mask __read_mostly; 503 504 #define SVM_NCPUIDS 32 505 506 #define VMCB_NPAGES 1 507 508 #define MSRBM_NPAGES 2 509 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 510 511 #define IOBM_NPAGES 3 512 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 513 514 /* Does not include EFER_LMSLE. */ 515 #define EFER_VALID \ 516 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 517 518 #define EFER_TLB_FLUSH \ 519 (EFER_NXE|EFER_LMA|EFER_LME) 520 #define CR0_TLB_FLUSH \ 521 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 522 #define CR4_TLB_FLUSH \ 523 (CR4_PSE|CR4_PAE|CR4_PGE|CR4_PCIDE|CR4_SMEP) 524 525 #define CR4_VALID \ 526 (CR4_VME | \ 527 CR4_PVI | \ 528 CR4_TSD | \ 529 CR4_DE | \ 530 CR4_PSE | \ 531 CR4_PAE | \ 532 CR4_MCE | \ 533 CR4_PGE | \ 534 CR4_PCE | \ 535 CR4_OSFXSR | \ 536 CR4_OSXMMEXCPT | \ 537 CR4_UMIP | \ 538 /* CR4_LA57 excluded */ \ 539 /* bit 13 reserved on AMD */ \ 540 /* bit 14 reserved on AMD */ \ 541 /* bit 15 reserved on AMD */ \ 542 CR4_FSGSBASE | \ 543 CR4_PCIDE | \ 544 CR4_OSXSAVE | \ 545 /* bit 19 reserved on AMD */ \ 546 CR4_SMEP | \ 547 CR4_SMAP \ 548 /* CR4_PKE excluded */ \ 549 /* CR4_CET excluded */ \ 550 /* bits 24:63 reserved on AMD */) 551 552 /* -------------------------------------------------------------------------- */ 553 554 struct svm_machdata { 555 volatile uint64_t mach_htlb_gen; 556 }; 557 558 static const size_t svm_vcpu_conf_sizes[NVMM_X86_VCPU_NCONF] = { 559 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = 560 sizeof(struct nvmm_vcpu_conf_cpuid), 561 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = 562 sizeof(struct nvmm_vcpu_conf_tpr), 563 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK)] = 564 sizeof(uint64_t), 565 }; 566 567 struct svm_cpudata { 568 /* General */ 569 bool shared_asid; 570 bool gtlb_want_flush; 571 bool gtsc_want_update; 572 uint64_t vcpu_htlb_gen; 573 574 /* VMCB */ 575 struct vmcb *vmcb; 576 paddr_t vmcb_pa; 577 578 /* I/O bitmap */ 579 uint8_t *iobm; 580 paddr_t iobm_pa; 581 582 /* MSR bitmap */ 583 uint8_t *msrbm; 584 paddr_t msrbm_pa; 585 586 /* Host state */ 587 uint64_t hxcr0; 588 uint64_t star; 589 uint64_t lstar; 590 uint64_t cstar; 591 uint64_t sfmask; 592 uint64_t fsbase; 593 uint64_t kernelgsbase; 594 595 /* Intr state */ 596 bool int_window_exit; 597 bool nmi_window_exit; 598 bool evt_pending; 599 600 /* Guest state */ 601 uint64_t gxcr0; 602 uint64_t gprs[NVMM_X64_NGPR]; 603 uint64_t drs[NVMM_X64_NDR]; 604 uint64_t gtsc; 605 606 /* VCPU configuration. */ 607 bool cpuidpresent[SVM_NCPUIDS]; 608 struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; 609 uint64_t xcr0_mask; 610 611 /* 612 * Guest XSAVE state. Must be the last member because it may 613 * be extended variably by whatever CPU we're running on. We 614 * add a flexible array member afterward to ward UB-exploiting 615 * compilers away from memset/memcpy calls that access it. 616 */ 617 struct xsave_header gfpu __aligned(64); 618 uint8_t gfpu_ext[]; 619 }; 620 621 static void 622 svm_vmcb_cache_default(struct vmcb *vmcb) 623 { 624 vmcb->ctrl.vmcb_clean = 625 VMCB_CTRL_VMCB_CLEAN_I | 626 VMCB_CTRL_VMCB_CLEAN_IOPM | 627 VMCB_CTRL_VMCB_CLEAN_ASID | 628 VMCB_CTRL_VMCB_CLEAN_TPR | 629 VMCB_CTRL_VMCB_CLEAN_NP | 630 VMCB_CTRL_VMCB_CLEAN_CR | 631 VMCB_CTRL_VMCB_CLEAN_DR | 632 VMCB_CTRL_VMCB_CLEAN_DT | 633 VMCB_CTRL_VMCB_CLEAN_SEG | 634 VMCB_CTRL_VMCB_CLEAN_CR2 | 635 VMCB_CTRL_VMCB_CLEAN_LBR | 636 VMCB_CTRL_VMCB_CLEAN_AVIC; 637 } 638 639 static void 640 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 641 { 642 if (flags & NVMM_X64_STATE_SEGS) { 643 vmcb->ctrl.vmcb_clean &= 644 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 645 } 646 if (flags & NVMM_X64_STATE_CRS) { 647 vmcb->ctrl.vmcb_clean &= 648 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 649 VMCB_CTRL_VMCB_CLEAN_TPR); 650 } 651 if (flags & NVMM_X64_STATE_DRS) { 652 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 653 } 654 if (flags & NVMM_X64_STATE_MSRS) { 655 /* CR for EFER, NP for PAT. */ 656 vmcb->ctrl.vmcb_clean &= 657 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 658 } 659 } 660 661 static inline void 662 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 663 { 664 vmcb->ctrl.vmcb_clean &= ~flags; 665 } 666 667 static inline void 668 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 669 { 670 vmcb->ctrl.vmcb_clean = 0; 671 } 672 673 #define SVM_EVENT_TYPE_HW_INT 0 674 #define SVM_EVENT_TYPE_NMI 2 675 #define SVM_EVENT_TYPE_EXC 3 676 #define SVM_EVENT_TYPE_SW_INT 4 677 678 static void 679 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 680 { 681 struct svm_cpudata *cpudata = vcpu->cpudata; 682 struct vmcb *vmcb = cpudata->vmcb; 683 684 if (nmi) { 685 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 686 cpudata->nmi_window_exit = true; 687 } else { 688 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 689 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 690 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 691 cpudata->int_window_exit = true; 692 } 693 694 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 695 } 696 697 static void 698 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 699 { 700 struct svm_cpudata *cpudata = vcpu->cpudata; 701 struct vmcb *vmcb = cpudata->vmcb; 702 703 if (nmi) { 704 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 705 cpudata->nmi_window_exit = false; 706 } else { 707 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 708 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 709 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 710 cpudata->int_window_exit = false; 711 } 712 713 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 714 } 715 716 static inline bool 717 svm_excp_has_rf(uint8_t vector) 718 { 719 switch (vector) { 720 case 1: /* #DB */ 721 case 4: /* #OF */ 722 case 8: /* #DF */ 723 case 18: /* #MC */ 724 return false; 725 default: 726 return true; 727 } 728 } 729 730 static inline int 731 svm_excp_has_error(uint8_t vector) 732 { 733 switch (vector) { 734 case 8: /* #DF */ 735 case 10: /* #TS */ 736 case 11: /* #NP */ 737 case 12: /* #SS */ 738 case 13: /* #GP */ 739 case 14: /* #PF */ 740 case 17: /* #AC */ 741 case 21: /* #CP */ 742 case 30: /* #SX */ 743 return 1; 744 default: 745 return 0; 746 } 747 } 748 749 static int 750 svm_vcpu_inject(struct nvmm_cpu *vcpu) 751 { 752 struct nvmm_comm_page *comm = vcpu->comm; 753 struct svm_cpudata *cpudata = vcpu->cpudata; 754 struct vmcb *vmcb = cpudata->vmcb; 755 u_int evtype; 756 uint8_t vector; 757 uint64_t error; 758 int type = 0, err = 0; 759 760 evtype = comm->event.type; 761 vector = comm->event.vector; 762 error = comm->event.u.excp.error; 763 __insn_barrier(); 764 765 switch (evtype) { 766 case NVMM_VCPU_EVENT_EXCP: 767 type = SVM_EVENT_TYPE_EXC; 768 if (vector == 2 || vector >= 32) 769 return EINVAL; 770 if (vector == 3 || vector == 0) 771 return EINVAL; 772 if (svm_excp_has_rf(vector)) { 773 vmcb->state.rflags |= PSL_RF; 774 } 775 err = svm_excp_has_error(vector); 776 break; 777 case NVMM_VCPU_EVENT_INTR: 778 type = SVM_EVENT_TYPE_HW_INT; 779 if (vector == 2) { 780 type = SVM_EVENT_TYPE_NMI; 781 svm_event_waitexit_enable(vcpu, true); 782 } 783 err = 0; 784 break; 785 default: 786 return EINVAL; 787 } 788 789 vmcb->ctrl.eventinj = 790 __SHIFTIN(vector, VMCB_CTRL_EVENTINJ_VECTOR) | 791 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 792 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 793 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 794 __SHIFTIN(error, VMCB_CTRL_EVENTINJ_ERRORCODE); 795 796 cpudata->evt_pending = true; 797 798 return 0; 799 } 800 801 static void 802 svm_inject_ud(struct nvmm_cpu *vcpu) 803 { 804 struct nvmm_comm_page *comm = vcpu->comm; 805 int ret __diagused; 806 807 comm->event.type = NVMM_VCPU_EVENT_EXCP; 808 comm->event.vector = 6; 809 comm->event.u.excp.error = 0; 810 811 ret = svm_vcpu_inject(vcpu); 812 KASSERT(ret == 0); 813 } 814 815 static void 816 svm_inject_gp(struct nvmm_cpu *vcpu) 817 { 818 struct nvmm_comm_page *comm = vcpu->comm; 819 int ret __diagused; 820 821 comm->event.type = NVMM_VCPU_EVENT_EXCP; 822 comm->event.vector = 13; 823 comm->event.u.excp.error = 0; 824 825 ret = svm_vcpu_inject(vcpu); 826 KASSERT(ret == 0); 827 } 828 829 static inline int 830 svm_vcpu_event_commit(struct nvmm_cpu *vcpu) 831 { 832 if (__predict_true(!vcpu->comm->event_commit)) { 833 return 0; 834 } 835 vcpu->comm->event_commit = false; 836 return svm_vcpu_inject(vcpu); 837 } 838 839 static inline void 840 svm_inkernel_advance(struct vmcb *vmcb) 841 { 842 /* 843 * Maybe we should also apply single-stepping and debug exceptions. 844 * Matters for guest-ring3, because it can execute 'cpuid' under a 845 * debugger. 846 */ 847 vmcb->state.rip = vmcb->ctrl.nrip; 848 vmcb->state.rflags &= ~PSL_RF; 849 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 850 } 851 852 #define SVM_CPUID_MAX_BASIC 0xD 853 #define SVM_CPUID_MAX_HYPERVISOR 0x40000000 854 #define SVM_CPUID_MAX_EXTENDED 0x8000001F 855 static uint32_t svm_cpuid_max_basic __read_mostly; 856 static uint32_t svm_cpuid_max_extended __read_mostly; 857 858 static void 859 svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx) 860 { 861 u_int descs[4]; 862 863 x86_cpuid2(eax, ecx, descs); 864 cpudata->vmcb->state.rax = descs[0]; 865 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 866 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 867 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 868 } 869 870 static void 871 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 872 { 873 struct svm_cpudata *cpudata = vcpu->cpudata; 874 uint64_t cr4; 875 876 877 /* 878 * `If a value entered for CPUID.EAX is higher than the maximum 879 * input value for basic or extended function for that 880 * processor then the data for the highest basic information 881 * leaf is returned.' 882 * 883 * --Intel 64 and IA-32 Architectures Software Developer's 884 * Manual, Vol. 2A, Order Number: 325383-077US, April 2022, 885 * Sec. 3.2 `Instructions (A-L)', CPUID--CPU Identification, 886 * p. 3-214. 887 * 888 * We take the same to hold for the hypervisor range, 889 * 0x40000000-0x4fffffff. 890 * 891 * (Sync with nvmm_x86_vmx.c.) 892 */ 893 if (eax < 0x40000000) { /* basic CPUID range */ 894 if (__predict_false(eax > svm_cpuid_max_basic)) { 895 eax = svm_cpuid_max_basic; 896 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 897 } 898 } else if (eax < 0x80000000) { /* hypervisor CPUID range */ 899 if (__predict_false(eax > SVM_CPUID_MAX_HYPERVISOR)) { 900 eax = svm_cpuid_max_basic; 901 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 902 } 903 } else { /* extended CPUID range */ 904 if (__predict_false(eax > svm_cpuid_max_extended)) { 905 eax = svm_cpuid_max_basic; 906 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 907 } 908 } 909 910 switch (eax) { 911 912 /* 913 * basic CPUID range 914 */ 915 case 0x00000000: 916 cpudata->vmcb->state.rax = svm_cpuid_max_basic; 917 break; 918 case 0x00000001: 919 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 920 921 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 922 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 923 CPUID_LOCAL_APIC_ID); 924 925 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 926 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 927 928 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 929 930 /* CPUID2_OSXSAVE depends on CR4. */ 931 cr4 = cpudata->vmcb->state.cr4; 932 if (!(cr4 & CR4_OSXSAVE)) { 933 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 934 } 935 break; 936 case 0x00000002: /* Empty */ 937 case 0x00000003: /* Empty */ 938 case 0x00000004: /* Empty */ 939 case 0x00000005: /* Monitor/MWait */ 940 case 0x00000006: /* Power Management Related Features */ 941 cpudata->vmcb->state.rax = 0; 942 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 943 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 944 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 945 break; 946 case 0x00000007: /* Structured Extended Features */ 947 switch (ecx) { 948 case 0: 949 cpudata->vmcb->state.rax = 0; 950 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 951 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 952 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 953 break; 954 default: 955 cpudata->vmcb->state.rax = 0; 956 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 957 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 958 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 959 break; 960 } 961 break; 962 case 0x00000008: /* Empty */ 963 case 0x00000009: /* Empty */ 964 case 0x0000000A: /* Empty */ 965 case 0x0000000B: /* Empty */ 966 case 0x0000000C: /* Empty */ 967 cpudata->vmcb->state.rax = 0; 968 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 969 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 970 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 971 break; 972 case 0x0000000D: /* Processor Extended State Enumeration */ 973 if (cpudata->xcr0_mask == 0) { 974 cpudata->vmcb->state.rax = 0; 975 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 976 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 977 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 978 break; 979 } 980 switch (ecx) { 981 case 0: 982 cpudata->vmcb->state.rax = 983 cpudata->xcr0_mask & 0xFFFFFFFF; 984 cpudata->gprs[NVMM_X64_GPR_RBX] = 985 nvmm_x86_xsave_size(cpudata->gxcr0); 986 cpudata->gprs[NVMM_X64_GPR_RCX] = 987 nvmm_x86_xsave_size(cpudata->xcr0_mask); 988 cpudata->gprs[NVMM_X64_GPR_RDX] = 989 cpudata->xcr0_mask >> 32; 990 break; 991 case 1: 992 cpudata->vmcb->state.rax &= 993 (CPUID_PES1_XSAVEOPT | CPUID_PES1_XSAVEC | 994 CPUID_PES1_XGETBV); 995 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 996 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 997 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 998 break; 999 case 2 ... 62: 1000 /* 1001 * CPUID[EAX=0x0d,ECX=n], 2 <= n <= 62: size 1002 * and offset of nth component in XSAVE area. 1003 * If the nth bit of XCR0 is disabled in the 1004 * vCPU configuration, we return all-zero 1005 * instead. 1006 */ 1007 if ((cpudata->xcr0_mask & __BIT(ecx)) == 0) { 1008 cpudata->vmcb->state.rax = 0; 1009 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1010 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1011 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1012 } 1013 break; 1014 default: 1015 cpudata->vmcb->state.rax = 0; 1016 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1017 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1018 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1019 break; 1020 } 1021 break; 1022 1023 /* 1024 * hypervisor CPUID range 1025 */ 1026 case 0x40000000: /* Hypervisor Information */ 1027 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR; 1028 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1029 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1030 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1031 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 1032 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 1033 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 1034 break; 1035 1036 /* 1037 * extended CPUID range 1038 */ 1039 case 0x80000000: 1040 cpudata->vmcb->state.rax = svm_cpuid_max_extended; 1041 break; 1042 case 0x80000001: 1043 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 1044 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 1045 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 1046 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 1047 break; 1048 case 0x80000002: /* Extended Processor Name String */ 1049 case 0x80000003: /* Extended Processor Name String */ 1050 case 0x80000004: /* Extended Processor Name String */ 1051 case 0x80000005: /* L1 Cache and TLB Information */ 1052 case 0x80000006: /* L2 Cache and TLB and L3 Cache Information */ 1053 break; 1054 case 0x80000007: /* Processor Power Management and RAS Capabilities */ 1055 cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax; 1056 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx; 1057 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx; 1058 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx; 1059 break; 1060 case 0x80000008: /* Processor Capacity Parameters and Ext Feat Ident */ 1061 cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax; 1062 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx; 1063 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx; 1064 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx; 1065 break; 1066 case 0x80000009: /* Empty */ 1067 case 0x8000000A: /* SVM Features */ 1068 case 0x8000000B: /* Empty */ 1069 case 0x8000000C: /* Empty */ 1070 case 0x8000000D: /* Empty */ 1071 case 0x8000000E: /* Empty */ 1072 case 0x8000000F: /* Empty */ 1073 case 0x80000010: /* Empty */ 1074 case 0x80000011: /* Empty */ 1075 case 0x80000012: /* Empty */ 1076 case 0x80000013: /* Empty */ 1077 case 0x80000014: /* Empty */ 1078 case 0x80000015: /* Empty */ 1079 case 0x80000016: /* Empty */ 1080 case 0x80000017: /* Empty */ 1081 case 0x80000018: /* Empty */ 1082 cpudata->vmcb->state.rax = 0; 1083 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1084 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1085 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1086 break; 1087 case 0x80000019: /* TLB Characteristics for 1GB pages */ 1088 case 0x8000001A: /* Instruction Optimizations */ 1089 break; 1090 case 0x8000001B: /* Instruction-Based Sampling Capabilities */ 1091 case 0x8000001C: /* Lightweight Profiling Capabilities */ 1092 cpudata->vmcb->state.rax = 0; 1093 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1094 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1095 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1096 break; 1097 case 0x8000001D: /* Cache Topology Information */ 1098 case 0x8000001E: /* Processor Topology Information */ 1099 break; /* TODO? */ 1100 case 0x8000001F: /* Encrypted Memory Capabilities */ 1101 cpudata->vmcb->state.rax = 0; 1102 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1103 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1104 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1105 break; 1106 1107 default: 1108 break; 1109 } 1110 } 1111 1112 static void 1113 svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) 1114 { 1115 exit->u.insn.npc = vmcb->ctrl.nrip; 1116 exit->reason = reason; 1117 } 1118 1119 static void 1120 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1121 struct nvmm_vcpu_exit *exit) 1122 { 1123 struct svm_cpudata *cpudata = vcpu->cpudata; 1124 struct nvmm_vcpu_conf_cpuid *cpuid; 1125 uint64_t eax, ecx; 1126 size_t i; 1127 1128 eax = cpudata->vmcb->state.rax; 1129 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 1130 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 1131 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 1132 1133 for (i = 0; i < SVM_NCPUIDS; i++) { 1134 if (!cpudata->cpuidpresent[i]) { 1135 continue; 1136 } 1137 cpuid = &cpudata->cpuid[i]; 1138 if (cpuid->leaf != eax) { 1139 continue; 1140 } 1141 1142 if (cpuid->exit) { 1143 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID); 1144 return; 1145 } 1146 KASSERT(cpuid->mask); 1147 1148 /* del */ 1149 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax; 1150 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx; 1151 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx; 1152 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx; 1153 1154 /* set */ 1155 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax; 1156 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx; 1157 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx; 1158 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx; 1159 1160 break; 1161 } 1162 1163 svm_inkernel_advance(cpudata->vmcb); 1164 exit->reason = NVMM_VCPU_EXIT_NONE; 1165 } 1166 1167 static void 1168 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1169 struct nvmm_vcpu_exit *exit) 1170 { 1171 struct svm_cpudata *cpudata = vcpu->cpudata; 1172 struct vmcb *vmcb = cpudata->vmcb; 1173 1174 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 1175 svm_event_waitexit_disable(vcpu, false); 1176 } 1177 1178 svm_inkernel_advance(cpudata->vmcb); 1179 exit->reason = NVMM_VCPU_EXIT_HALTED; 1180 } 1181 1182 #define SVM_EXIT_IO_PORT __BITS(31,16) 1183 #define SVM_EXIT_IO_SEG __BITS(12,10) 1184 #define SVM_EXIT_IO_A64 __BIT(9) 1185 #define SVM_EXIT_IO_A32 __BIT(8) 1186 #define SVM_EXIT_IO_A16 __BIT(7) 1187 #define SVM_EXIT_IO_SZ32 __BIT(6) 1188 #define SVM_EXIT_IO_SZ16 __BIT(5) 1189 #define SVM_EXIT_IO_SZ8 __BIT(4) 1190 #define SVM_EXIT_IO_REP __BIT(3) 1191 #define SVM_EXIT_IO_STR __BIT(2) 1192 #define SVM_EXIT_IO_IN __BIT(0) 1193 1194 static void 1195 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1196 struct nvmm_vcpu_exit *exit) 1197 { 1198 struct svm_cpudata *cpudata = vcpu->cpudata; 1199 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1200 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 1201 1202 exit->reason = NVMM_VCPU_EXIT_IO; 1203 1204 exit->u.io.in = (info & SVM_EXIT_IO_IN) != 0; 1205 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 1206 1207 if (__predict_true(svm_decode_assist)) { 1208 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 1209 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 1210 } else { 1211 exit->u.io.seg = -1; 1212 } 1213 1214 if (info & SVM_EXIT_IO_A64) { 1215 exit->u.io.address_size = 8; 1216 } else if (info & SVM_EXIT_IO_A32) { 1217 exit->u.io.address_size = 4; 1218 } else if (info & SVM_EXIT_IO_A16) { 1219 exit->u.io.address_size = 2; 1220 } 1221 1222 if (info & SVM_EXIT_IO_SZ32) { 1223 exit->u.io.operand_size = 4; 1224 } else if (info & SVM_EXIT_IO_SZ16) { 1225 exit->u.io.operand_size = 2; 1226 } else if (info & SVM_EXIT_IO_SZ8) { 1227 exit->u.io.operand_size = 1; 1228 } 1229 1230 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 1231 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 1232 exit->u.io.npc = nextpc; 1233 1234 svm_vcpu_state_provide(vcpu, 1235 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1236 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1237 } 1238 1239 static const uint64_t msr_ignore_list[] = { 1240 0xc0010055, /* MSR_CMPHALT */ 1241 MSR_DE_CFG, 1242 MSR_IC_CFG, 1243 MSR_UCODE_AMD_PATCHLEVEL 1244 }; 1245 1246 static bool 1247 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1248 struct nvmm_vcpu_exit *exit) 1249 { 1250 struct svm_cpudata *cpudata = vcpu->cpudata; 1251 struct vmcb *vmcb = cpudata->vmcb; 1252 uint64_t val; 1253 size_t i; 1254 1255 if (exit->reason == NVMM_VCPU_EXIT_RDMSR) { 1256 if (exit->u.rdmsr.msr == MSR_EFER) { 1257 val = vmcb->state.efer & ~EFER_SVME; 1258 vmcb->state.rax = (val & 0xFFFFFFFF); 1259 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1260 goto handled; 1261 } 1262 if (exit->u.rdmsr.msr == MSR_NB_CFG) { 1263 val = NB_CFG_INITAPICCPUIDLO; 1264 vmcb->state.rax = (val & 0xFFFFFFFF); 1265 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1266 goto handled; 1267 } 1268 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1269 if (msr_ignore_list[i] != exit->u.rdmsr.msr) 1270 continue; 1271 val = 0; 1272 vmcb->state.rax = (val & 0xFFFFFFFF); 1273 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1274 goto handled; 1275 } 1276 } else { 1277 if (exit->u.wrmsr.msr == MSR_EFER) { 1278 if (__predict_false(exit->u.wrmsr.val & ~EFER_VALID)) { 1279 goto error; 1280 } 1281 if ((vmcb->state.efer ^ exit->u.wrmsr.val) & 1282 EFER_TLB_FLUSH) { 1283 cpudata->gtlb_want_flush = true; 1284 } 1285 vmcb->state.efer = exit->u.wrmsr.val | EFER_SVME; 1286 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1287 goto handled; 1288 } 1289 if (exit->u.wrmsr.msr == MSR_TSC) { 1290 cpudata->gtsc = exit->u.wrmsr.val; 1291 cpudata->gtsc_want_update = true; 1292 goto handled; 1293 } 1294 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1295 if (msr_ignore_list[i] != exit->u.wrmsr.msr) 1296 continue; 1297 goto handled; 1298 } 1299 } 1300 1301 return false; 1302 1303 handled: 1304 svm_inkernel_advance(cpudata->vmcb); 1305 return true; 1306 1307 error: 1308 svm_inject_gp(vcpu); 1309 return true; 1310 } 1311 1312 static inline void 1313 svm_exit_rdmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1314 struct nvmm_vcpu_exit *exit) 1315 { 1316 struct svm_cpudata *cpudata = vcpu->cpudata; 1317 1318 exit->reason = NVMM_VCPU_EXIT_RDMSR; 1319 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1320 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip; 1321 1322 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1323 exit->reason = NVMM_VCPU_EXIT_NONE; 1324 return; 1325 } 1326 1327 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1328 } 1329 1330 static inline void 1331 svm_exit_wrmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1332 struct nvmm_vcpu_exit *exit) 1333 { 1334 struct svm_cpudata *cpudata = vcpu->cpudata; 1335 uint64_t rdx, rax; 1336 1337 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1338 rax = cpudata->vmcb->state.rax; 1339 1340 exit->reason = NVMM_VCPU_EXIT_WRMSR; 1341 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1342 exit->u.wrmsr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1343 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip; 1344 1345 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1346 exit->reason = NVMM_VCPU_EXIT_NONE; 1347 return; 1348 } 1349 1350 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1351 } 1352 1353 static void 1354 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1355 struct nvmm_vcpu_exit *exit) 1356 { 1357 struct svm_cpudata *cpudata = vcpu->cpudata; 1358 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1359 1360 if (info == 0) { 1361 svm_exit_rdmsr(mach, vcpu, exit); 1362 } else { 1363 svm_exit_wrmsr(mach, vcpu, exit); 1364 } 1365 } 1366 1367 static void 1368 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1369 struct nvmm_vcpu_exit *exit) 1370 { 1371 struct svm_cpudata *cpudata = vcpu->cpudata; 1372 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1373 1374 exit->reason = NVMM_VCPU_EXIT_MEMORY; 1375 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1376 exit->u.mem.prot = PROT_WRITE; 1377 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_I) 1378 exit->u.mem.prot = PROT_EXEC; 1379 else 1380 exit->u.mem.prot = PROT_READ; 1381 exit->u.mem.gpa = gpa; 1382 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1383 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1384 sizeof(exit->u.mem.inst_bytes)); 1385 1386 svm_vcpu_state_provide(vcpu, 1387 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1388 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1389 } 1390 1391 static void 1392 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1393 struct nvmm_vcpu_exit *exit) 1394 { 1395 struct svm_cpudata *cpudata = vcpu->cpudata; 1396 struct vmcb *vmcb = cpudata->vmcb; 1397 uint64_t val; 1398 1399 exit->reason = NVMM_VCPU_EXIT_NONE; 1400 1401 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1402 (vmcb->state.rax & 0xFFFFFFFF); 1403 1404 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1405 goto error; 1406 } else if (__predict_false(vmcb->state.cpl != 0)) { 1407 goto error; 1408 } else if (__predict_false(cpudata->xcr0_mask == 0)) { 1409 goto error; 1410 } else if (__predict_false(!nvmm_x86_xcr0_valid(val, 1411 cpudata->xcr0_mask))) { 1412 goto error; 1413 } 1414 1415 KASSERTMSG(nvmm_x86_xcr0_valid(val, cpudata->xcr0_mask), 1416 "val=0x%"PRIx64" xcr0_mask=0x%"PRIx64" (gxcr0=0x%"PRIx64")", 1417 val, cpudata->xcr0_mask, cpudata->gxcr0); 1418 cpudata->gxcr0 = val; 1419 1420 svm_inkernel_advance(cpudata->vmcb); 1421 return; 1422 1423 error: 1424 svm_inject_gp(vcpu); 1425 } 1426 1427 static void 1428 svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code) 1429 { 1430 exit->u.inv.hwcode = code; 1431 exit->reason = NVMM_VCPU_EXIT_INVALID; 1432 } 1433 1434 /* -------------------------------------------------------------------------- */ 1435 1436 static void 1437 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1438 { 1439 struct svm_cpudata *cpudata = vcpu->cpudata; 1440 1441 /* 1442 * The guest's XCR0 had better not have any bits that aren't 1443 * allowed in the vCPU configuration, and the current XSAVE 1444 * area had better not store any either according to 1445 * cpudata->gfpu.xsh_xstate_bv. 1446 * 1447 * Note that XRSTOR will trap if XSTATE_BV has any bits that 1448 * are not set in XCR0. 1449 */ 1450 KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, 1451 "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 1452 cpudata->gxcr0, cpudata->xcr0_mask); 1453 KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, 1454 "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 1455 cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); 1456 1457 /* 1458 * Save anything in the FPU registers that this thread might 1459 * have been using to memory, and raise the IPL to IPL_VM to 1460 * block interrupt handlers that might use the FPU. This also 1461 * zeroes any FPU registers that the NetBSD host uses. 1462 * 1463 * After this point, we are free to use the FPU registers. 1464 */ 1465 fpu_kern_enter(); 1466 1467 /* 1468 * If the host CPU doesn't support XSAVE or we're simulating a 1469 * vCPU without it, just restore the x87 and SSE state. The 1470 * host should already have both x87 and SSE enabled in XCR0, 1471 * if the host uses XSAVE. 1472 */ 1473 if (cpudata->xcr0_mask == 0) { 1474 /* TODO: should we use *XSAVE64 here? */ 1475 fpu_area_restore(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); 1476 return; 1477 } 1478 1479 /* 1480 * Set XCR0 to allow access to anything the guest has 1481 * previously used and is saved to memory, _and_ to anything 1482 * the guest has asked to use in cpudata->gxcr0. 1483 * 1484 * The guest may have used some extended CPU state like the 1485 * zmmN registers, and then later disabled them in XCR0; in 1486 * that case, the state must be preserved in case the guest 1487 * later enables it in XCR0, but we can only load while all 1488 * bits in cpudata->gfpu.xsh_xstate_bv are set in XCR0. 1489 * 1490 * Similarly, the guest may _not_ have used some extended CPU 1491 * state since reset, but may have since enabled it in XCR0. 1492 * Such state will be clear in cpudata->gfpu.xsh_xstate_bv and 1493 * must be initialized afresh by the CPU, which requires the 1494 * bits be set in XCR0 to allow that. 1495 */ 1496 cpudata->hxcr0 = rdxcr(0); 1497 wrxcr(0, cpudata->xcr0_mask & 1498 (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)); 1499 1500 /* 1501 * Load the guest's saved extended CPU state from memory into 1502 * the CPU. 1503 */ 1504 /* TODO: should we use *XSAVE64 here? */ 1505 fpu_area_restore(&cpudata->gfpu, cpudata->xcr0_mask, false); 1506 1507 /* 1508 * If we temporarily set XCR0 beyond what the guest asked for 1509 * in order to restore state that is currently disabled, reduce 1510 * it down to what the guest asked for. 1511 */ 1512 if (__predict_false(cpudata->gxcr0 != (cpudata->xcr0_mask & 1513 (cpudata->gfpu.xsh_xstate_bv | cpudata->gxcr0)))) 1514 wrxcr(0, cpudata->xcr0_mask & cpudata->gxcr0); 1515 } 1516 1517 static void 1518 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1519 { 1520 struct svm_cpudata *cpudata = vcpu->cpudata; 1521 1522 /* 1523 * If the host CPU doesn't support XSAVE or we're simulating a 1524 * vCPU without it, just save the x87 and SSE state. If the 1525 * host uses XSAVE, it should still have both x87 and SSE 1526 * enabled in XCR0; if the host doesn't use XSAVE, doesn't 1527 * matter. 1528 */ 1529 if (cpudata->xcr0_mask == 0) { 1530 /* TODO: should we use *XSAVE64 here? */ 1531 fpu_area_save(&cpudata->gfpu, XCR0_X87|XCR0_SSE, false); 1532 goto leave; 1533 } 1534 1535 /* 1536 * In case the guest has cleared some XCR0 bits but used the 1537 * corresponding extended CPU state, increase XCR0 to the 1538 * maximum supported for this guest before we XSAVE. 1539 * 1540 * Note that XRSTOR will trap if XSTATE_BV has any bits that 1541 * are not set in XCR0. 1542 */ 1543 cpudata->gxcr0 = rdxcr(0); 1544 wrxcr(0, cpudata->xcr0_mask); 1545 1546 /* 1547 * Paranoia: Ensure the guest's XCR0 has no forbidden bits. 1548 * Should not be possible because we filter them on XSETBV 1549 * exits. 1550 */ 1551 KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, 1552 "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 1553 cpudata->gxcr0, cpudata->xcr0_mask); 1554 cpudata->gxcr0 &= cpudata->xcr0_mask; 1555 1556 /* 1557 * Save any extended CPU state that could be in use by the 1558 * guest. 1559 */ 1560 /* TODO: should we use *XSAVE64 here? */ 1561 fpu_area_save(&cpudata->gfpu, cpudata->xcr0_mask, false); 1562 1563 /* 1564 * If the host XCR0 is different from the maximum guest XCR0, 1565 * switch back to the host XCR0 so we can restore NetBSD's FPU 1566 * state. 1567 */ 1568 if (cpudata->xcr0_mask != cpudata->hxcr0) 1569 wrxcr(0, cpudata->hxcr0); 1570 1571 leave: /* 1572 * Restore any FPU registers that we might have saved in 1573 * svm_vcpu_guest_fpu_enter for this thread, and restore the 1574 * IPL from IPL_VM. 1575 * 1576 * After this point, we must not touch the FPU registers. 1577 */ 1578 fpu_kern_leave(); 1579 1580 /* 1581 * The guest's XCR0 had better not have any bits that aren't 1582 * allowed in the vCPU configuration, and the current XSAVE 1583 * area had better not store any either according to 1584 * cpudata->gfpu.xsh_xstate_bv. 1585 */ 1586 KASSERTMSG((cpudata->gxcr0 & ~cpudata->xcr0_mask) == 0, 1587 "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 1588 cpudata->gxcr0, cpudata->xcr0_mask); 1589 KASSERTMSG((cpudata->gfpu.xsh_xstate_bv & ~cpudata->xcr0_mask) == 0, 1590 "XSTATE_BV=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 1591 cpudata->gfpu.xsh_xstate_bv, cpudata->xcr0_mask); 1592 } 1593 1594 static void 1595 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1596 { 1597 struct svm_cpudata *cpudata = vcpu->cpudata; 1598 1599 x86_dbregs_save(curlwp); 1600 1601 ldr7(0); 1602 1603 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1604 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1605 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1606 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1607 } 1608 1609 static void 1610 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1611 { 1612 struct svm_cpudata *cpudata = vcpu->cpudata; 1613 1614 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1615 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1616 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1617 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1618 1619 x86_dbregs_restore(curlwp); 1620 } 1621 1622 static void 1623 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1624 { 1625 struct svm_cpudata *cpudata = vcpu->cpudata; 1626 1627 cpudata->fsbase = rdmsr(MSR_FSBASE); 1628 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1629 } 1630 1631 static void 1632 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1633 { 1634 struct svm_cpudata *cpudata = vcpu->cpudata; 1635 1636 wrmsr(MSR_STAR, cpudata->star); 1637 wrmsr(MSR_LSTAR, cpudata->lstar); 1638 wrmsr(MSR_CSTAR, cpudata->cstar); 1639 wrmsr(MSR_SFMASK, cpudata->sfmask); 1640 wrmsr(MSR_FSBASE, cpudata->fsbase); 1641 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1642 } 1643 1644 /* -------------------------------------------------------------------------- */ 1645 1646 static inline void 1647 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1648 { 1649 struct svm_cpudata *cpudata = vcpu->cpudata; 1650 1651 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1652 cpudata->gtlb_want_flush = true; 1653 } 1654 } 1655 1656 static inline void 1657 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1658 { 1659 /* 1660 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1661 * executing on this hCPU and the hTLB already got flushed, or it 1662 * was executing on another hCPU in which case the catchup is done 1663 * in svm_gtlb_catchup(). 1664 */ 1665 } 1666 1667 static inline uint64_t 1668 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1669 { 1670 struct vmcb *vmcb = cpudata->vmcb; 1671 uint64_t machgen; 1672 1673 machgen = machdata->mach_htlb_gen; 1674 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1675 return machgen; 1676 } 1677 1678 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1679 return machgen; 1680 } 1681 1682 static inline void 1683 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1684 { 1685 struct vmcb *vmcb = cpudata->vmcb; 1686 1687 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1688 cpudata->vcpu_htlb_gen = machgen; 1689 } 1690 } 1691 1692 static inline void 1693 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb) 1694 { 1695 cpudata->evt_pending = false; 1696 1697 if (__predict_false(vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V)) { 1698 vmcb->ctrl.eventinj = vmcb->ctrl.exitintinfo; 1699 cpudata->evt_pending = true; 1700 } 1701 } 1702 1703 static int 1704 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1705 struct nvmm_vcpu_exit *exit) 1706 { 1707 struct nvmm_comm_page *comm = vcpu->comm; 1708 struct svm_machdata *machdata = mach->machdata; 1709 struct svm_cpudata *cpudata = vcpu->cpudata; 1710 struct vmcb *vmcb = cpudata->vmcb; 1711 uint64_t machgen; 1712 int hcpu; 1713 1714 svm_vcpu_state_commit(vcpu); 1715 comm->state_cached = 0; 1716 1717 if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) { 1718 return EINVAL; 1719 } 1720 1721 kpreempt_disable(); 1722 hcpu = cpu_number(); 1723 1724 svm_gtlb_catchup(vcpu, hcpu); 1725 svm_htlb_catchup(vcpu, hcpu); 1726 1727 if (vcpu->hcpu_last != hcpu) { 1728 svm_vmcb_cache_flush_all(vmcb); 1729 cpudata->gtsc_want_update = true; 1730 } 1731 1732 svm_vcpu_guest_dbregs_enter(vcpu); 1733 svm_vcpu_guest_misc_enter(vcpu); 1734 1735 while (1) { 1736 if (cpudata->gtlb_want_flush) { 1737 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1738 } else { 1739 vmcb->ctrl.tlb_ctrl = 0; 1740 } 1741 1742 if (__predict_false(cpudata->gtsc_want_update)) { 1743 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1744 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1745 } 1746 1747 svm_vcpu_guest_fpu_enter(vcpu); 1748 svm_clgi(); 1749 machgen = svm_htlb_flush(machdata, cpudata); 1750 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1751 svm_htlb_flush_ack(cpudata, machgen); 1752 svm_stgi(); 1753 svm_vcpu_guest_fpu_leave(vcpu); 1754 1755 svm_vmcb_cache_default(vmcb); 1756 1757 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1758 cpudata->gtlb_want_flush = false; 1759 cpudata->gtsc_want_update = false; 1760 vcpu->hcpu_last = hcpu; 1761 } 1762 svm_exit_evt(cpudata, vmcb); 1763 1764 switch (vmcb->ctrl.exitcode) { 1765 case VMCB_EXITCODE_INTR: 1766 case VMCB_EXITCODE_NMI: 1767 exit->reason = NVMM_VCPU_EXIT_NONE; 1768 break; 1769 case VMCB_EXITCODE_VINTR: 1770 svm_event_waitexit_disable(vcpu, false); 1771 exit->reason = NVMM_VCPU_EXIT_INT_READY; 1772 break; 1773 case VMCB_EXITCODE_IRET: 1774 svm_event_waitexit_disable(vcpu, true); 1775 exit->reason = NVMM_VCPU_EXIT_NMI_READY; 1776 break; 1777 case VMCB_EXITCODE_CPUID: 1778 svm_exit_cpuid(mach, vcpu, exit); 1779 break; 1780 case VMCB_EXITCODE_HLT: 1781 svm_exit_hlt(mach, vcpu, exit); 1782 break; 1783 case VMCB_EXITCODE_IOIO: 1784 svm_exit_io(mach, vcpu, exit); 1785 break; 1786 case VMCB_EXITCODE_MSR: 1787 svm_exit_msr(mach, vcpu, exit); 1788 break; 1789 case VMCB_EXITCODE_SHUTDOWN: 1790 exit->reason = NVMM_VCPU_EXIT_SHUTDOWN; 1791 break; 1792 case VMCB_EXITCODE_RDPMC: 1793 case VMCB_EXITCODE_RSM: 1794 case VMCB_EXITCODE_INVLPGA: 1795 case VMCB_EXITCODE_VMRUN: 1796 case VMCB_EXITCODE_VMMCALL: 1797 case VMCB_EXITCODE_VMLOAD: 1798 case VMCB_EXITCODE_VMSAVE: 1799 case VMCB_EXITCODE_STGI: 1800 case VMCB_EXITCODE_CLGI: 1801 case VMCB_EXITCODE_SKINIT: 1802 case VMCB_EXITCODE_RDTSCP: 1803 case VMCB_EXITCODE_RDPRU: 1804 case VMCB_EXITCODE_INVLPGB: 1805 case VMCB_EXITCODE_INVPCID: 1806 case VMCB_EXITCODE_MCOMMIT: 1807 case VMCB_EXITCODE_TLBSYNC: 1808 svm_inject_ud(vcpu); 1809 exit->reason = NVMM_VCPU_EXIT_NONE; 1810 break; 1811 case VMCB_EXITCODE_MONITOR: 1812 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR); 1813 break; 1814 case VMCB_EXITCODE_MWAIT: 1815 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1816 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT); 1817 break; 1818 case VMCB_EXITCODE_XSETBV: 1819 svm_exit_xsetbv(mach, vcpu, exit); 1820 break; 1821 case VMCB_EXITCODE_NPF: 1822 svm_exit_npf(mach, vcpu, exit); 1823 break; 1824 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1825 default: 1826 svm_exit_invalid(exit, vmcb->ctrl.exitcode); 1827 break; 1828 } 1829 1830 /* If no reason to return to userland, keep rolling. */ 1831 if (nvmm_return_needed(vcpu, exit)) { 1832 break; 1833 } 1834 if (exit->reason != NVMM_VCPU_EXIT_NONE) { 1835 break; 1836 } 1837 } 1838 1839 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1840 1841 svm_vcpu_guest_misc_leave(vcpu); 1842 svm_vcpu_guest_dbregs_leave(vcpu); 1843 1844 kpreempt_enable(); 1845 1846 exit->exitstate.rflags = vmcb->state.rflags; 1847 exit->exitstate.cr8 = __SHIFTOUT(vmcb->ctrl.v, VMCB_CTRL_V_TPR); 1848 exit->exitstate.int_shadow = 1849 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1850 exit->exitstate.int_window_exiting = cpudata->int_window_exit; 1851 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit; 1852 exit->exitstate.evt_pending = cpudata->evt_pending; 1853 1854 return 0; 1855 } 1856 1857 /* -------------------------------------------------------------------------- */ 1858 1859 static int 1860 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1861 { 1862 struct pglist pglist; 1863 paddr_t _pa; 1864 vaddr_t _va; 1865 size_t i; 1866 int ret; 1867 1868 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1869 &pglist, 1, 0); 1870 if (ret != 0) 1871 return ENOMEM; 1872 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 1873 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1874 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1875 if (_va == 0) 1876 goto error; 1877 1878 for (i = 0; i < npages; i++) { 1879 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1880 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1881 } 1882 pmap_update(pmap_kernel()); 1883 1884 memset((void *)_va, 0, npages * PAGE_SIZE); 1885 1886 *pa = _pa; 1887 *va = _va; 1888 return 0; 1889 1890 error: 1891 for (i = 0; i < npages; i++) { 1892 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1893 } 1894 return ENOMEM; 1895 } 1896 1897 static void 1898 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1899 { 1900 size_t i; 1901 1902 pmap_kremove(va, npages * PAGE_SIZE); 1903 pmap_update(pmap_kernel()); 1904 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1905 for (i = 0; i < npages; i++) { 1906 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1907 } 1908 } 1909 1910 /* -------------------------------------------------------------------------- */ 1911 1912 #define SVM_MSRBM_READ __BIT(0) 1913 #define SVM_MSRBM_WRITE __BIT(1) 1914 1915 static void 1916 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1917 { 1918 uint64_t byte; 1919 uint8_t bitoff; 1920 1921 if (msr < 0x00002000) { 1922 /* Range 1 */ 1923 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1924 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1925 /* Range 2 */ 1926 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1927 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1928 /* Range 3 */ 1929 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1930 } else { 1931 panic("%s: wrong range", __func__); 1932 } 1933 1934 bitoff = (msr & 0x3) << 1; 1935 1936 if (read) { 1937 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1938 } 1939 if (write) { 1940 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1941 } 1942 } 1943 1944 #define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1945 #define SVM_SEG_ATTRIB_S __BIT(4) 1946 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1947 #define SVM_SEG_ATTRIB_P __BIT(7) 1948 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1949 #define SVM_SEG_ATTRIB_L __BIT(9) 1950 #define SVM_SEG_ATTRIB_DEF __BIT(10) 1951 #define SVM_SEG_ATTRIB_G __BIT(11) 1952 1953 static void 1954 svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1955 struct vmcb_segment *vseg) 1956 { 1957 vseg->selector = seg->selector; 1958 vseg->attrib = 1959 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1960 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1961 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1962 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1963 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1964 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1965 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1966 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1967 vseg->limit = seg->limit; 1968 vseg->base = seg->base; 1969 } 1970 1971 static void 1972 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, 1973 const struct vmcb_segment *vseg) 1974 { 1975 seg->selector = vseg->selector; 1976 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1977 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1978 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1979 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1980 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1981 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1982 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1983 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1984 seg->limit = vseg->limit; 1985 seg->base = vseg->base; 1986 } 1987 1988 static inline bool 1989 svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1990 uint64_t flags) 1991 { 1992 if (flags & NVMM_X64_STATE_CRS) { 1993 if ((vmcb->state.cr0 ^ 1994 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1995 return true; 1996 } 1997 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1998 return true; 1999 } 2000 if ((vmcb->state.cr4 ^ 2001 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 2002 return true; 2003 } 2004 } 2005 2006 if (flags & NVMM_X64_STATE_MSRS) { 2007 if ((vmcb->state.efer ^ 2008 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 2009 return true; 2010 } 2011 } 2012 2013 return false; 2014 } 2015 2016 static void 2017 svm_vcpu_setstate(struct nvmm_cpu *vcpu) 2018 { 2019 struct nvmm_comm_page *comm = vcpu->comm; 2020 const struct nvmm_x64_state *state = &comm->state; 2021 struct svm_cpudata *cpudata = vcpu->cpudata; 2022 struct vmcb *vmcb = cpudata->vmcb; 2023 struct fxsave *fpustate; 2024 uint64_t flags; 2025 2026 flags = comm->state_wanted; 2027 2028 if (svm_state_tlb_flush(vmcb, state, flags)) { 2029 cpudata->gtlb_want_flush = true; 2030 } 2031 2032 if (flags & NVMM_X64_STATE_SEGS) { 2033 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 2034 &vmcb->state.cs); 2035 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 2036 &vmcb->state.ds); 2037 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 2038 &vmcb->state.es); 2039 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 2040 &vmcb->state.fs); 2041 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 2042 &vmcb->state.gs); 2043 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 2044 &vmcb->state.ss); 2045 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 2046 &vmcb->state.gdt); 2047 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 2048 &vmcb->state.idt); 2049 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 2050 &vmcb->state.ldt); 2051 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 2052 &vmcb->state.tr); 2053 2054 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 2055 } 2056 2057 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 2058 if (flags & NVMM_X64_STATE_GPRS) { 2059 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 2060 2061 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 2062 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 2063 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 2064 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 2065 } 2066 2067 if (flags & NVMM_X64_STATE_CRS) { 2068 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 2069 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 2070 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 2071 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 2072 vmcb->state.cr4 &= CR4_VALID; 2073 2074 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 2075 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 2076 VMCB_CTRL_V_TPR); 2077 2078 if (cpudata->xcr0_mask != 0) { 2079 const uint64_t xcr0 = state->crs[NVMM_X64_CR_XCR0]; 2080 2081 cpudata->gxcr0 = nvmm_x86_munge_xcr0(xcr0, 2082 cpudata->xcr0_mask); 2083 } 2084 } 2085 2086 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 2087 if (flags & NVMM_X64_STATE_DRS) { 2088 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 2089 2090 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 2091 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 2092 } 2093 2094 if (flags & NVMM_X64_STATE_MSRS) { 2095 /* 2096 * EFER_SVME is mandatory. 2097 */ 2098 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 2099 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 2100 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 2101 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 2102 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 2103 vmcb->state.kernelgsbase = 2104 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 2105 vmcb->state.sysenter_cs = 2106 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 2107 vmcb->state.sysenter_esp = 2108 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 2109 vmcb->state.sysenter_eip = 2110 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 2111 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 2112 2113 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 2114 cpudata->gtsc_want_update = true; 2115 } 2116 2117 if (flags & NVMM_X64_STATE_INTR) { 2118 if (state->intr.int_shadow) { 2119 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 2120 } else { 2121 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 2122 } 2123 2124 if (state->intr.int_window_exiting) { 2125 svm_event_waitexit_enable(vcpu, false); 2126 } else { 2127 svm_event_waitexit_disable(vcpu, false); 2128 } 2129 2130 if (state->intr.nmi_window_exiting) { 2131 svm_event_waitexit_enable(vcpu, true); 2132 } else { 2133 svm_event_waitexit_disable(vcpu, true); 2134 } 2135 } 2136 2137 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 2138 if (flags & NVMM_X64_STATE_FPU) { 2139 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 2140 sizeof(state->fpu)); 2141 2142 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 2143 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 2144 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 2145 2146 if (cpudata->xcr0_mask != 0) { 2147 /* Reset XSTATE_BV, to force a reload. */ 2148 cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; 2149 } 2150 } 2151 /* 2152 * XXX XSAVE area -- need to allocate and map it separately 2153 * since it may exceed the comm page size 2154 */ 2155 2156 svm_vmcb_cache_update(vmcb, flags); 2157 2158 comm->state_wanted = 0; 2159 comm->state_cached |= flags; 2160 } 2161 2162 static void 2163 svm_vcpu_getstate(struct nvmm_cpu *vcpu) 2164 { 2165 struct nvmm_comm_page *comm = vcpu->comm; 2166 struct nvmm_x64_state *state = &comm->state; 2167 struct svm_cpudata *cpudata = vcpu->cpudata; 2168 const struct vmcb *vmcb = cpudata->vmcb; 2169 uint64_t flags; 2170 2171 flags = comm->state_wanted; 2172 2173 if (flags & NVMM_X64_STATE_SEGS) { 2174 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 2175 &vmcb->state.cs); 2176 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 2177 &vmcb->state.ds); 2178 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 2179 &vmcb->state.es); 2180 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 2181 &vmcb->state.fs); 2182 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 2183 &vmcb->state.gs); 2184 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 2185 &vmcb->state.ss); 2186 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 2187 &vmcb->state.gdt); 2188 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 2189 &vmcb->state.idt); 2190 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 2191 &vmcb->state.ldt); 2192 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 2193 &vmcb->state.tr); 2194 2195 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 2196 } 2197 2198 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 2199 if (flags & NVMM_X64_STATE_GPRS) { 2200 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 2201 2202 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 2203 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 2204 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 2205 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 2206 } 2207 2208 if (flags & NVMM_X64_STATE_CRS) { 2209 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 2210 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 2211 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 2212 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 2213 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 2214 VMCB_CTRL_V_TPR); 2215 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 2216 } 2217 2218 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 2219 if (flags & NVMM_X64_STATE_DRS) { 2220 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 2221 2222 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 2223 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 2224 } 2225 2226 if (flags & NVMM_X64_STATE_MSRS) { 2227 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 2228 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 2229 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 2230 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 2231 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 2232 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 2233 vmcb->state.kernelgsbase; 2234 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 2235 vmcb->state.sysenter_cs; 2236 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 2237 vmcb->state.sysenter_esp; 2238 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 2239 vmcb->state.sysenter_eip; 2240 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 2241 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 2242 2243 /* Hide SVME. */ 2244 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 2245 } 2246 2247 if (flags & NVMM_X64_STATE_INTR) { 2248 state->intr.int_shadow = 2249 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 2250 state->intr.int_window_exiting = cpudata->int_window_exit; 2251 state->intr.nmi_window_exiting = cpudata->nmi_window_exit; 2252 state->intr.evt_pending = cpudata->evt_pending; 2253 } 2254 2255 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 2256 if (flags & NVMM_X64_STATE_FPU) { 2257 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 2258 sizeof(state->fpu)); 2259 } 2260 /* 2261 * XXX XSAVE area -- need to allocate and map it separately 2262 * since it may exceed the comm page size 2263 */ 2264 2265 comm->state_wanted = 0; 2266 comm->state_cached |= flags; 2267 } 2268 2269 static void 2270 svm_vcpu_state_provide(struct nvmm_cpu *vcpu, uint64_t flags) 2271 { 2272 vcpu->comm->state_wanted = flags; 2273 svm_vcpu_getstate(vcpu); 2274 } 2275 2276 static void 2277 svm_vcpu_state_commit(struct nvmm_cpu *vcpu) 2278 { 2279 vcpu->comm->state_wanted = vcpu->comm->state_commit; 2280 vcpu->comm->state_commit = 0; 2281 svm_vcpu_setstate(vcpu); 2282 } 2283 2284 /* -------------------------------------------------------------------------- */ 2285 2286 static void 2287 svm_asid_alloc(struct nvmm_cpu *vcpu) 2288 { 2289 struct svm_cpudata *cpudata = vcpu->cpudata; 2290 struct vmcb *vmcb = cpudata->vmcb; 2291 size_t i, oct, bit; 2292 2293 mutex_enter(&svm_asidlock); 2294 2295 for (i = 0; i < svm_maxasid; i++) { 2296 oct = i / 8; 2297 bit = i % 8; 2298 2299 if (svm_asidmap[oct] & __BIT(bit)) { 2300 continue; 2301 } 2302 2303 svm_asidmap[oct] |= __BIT(bit); 2304 vmcb->ctrl.guest_asid = i; 2305 mutex_exit(&svm_asidlock); 2306 return; 2307 } 2308 2309 /* 2310 * No free ASID. Use the last one, which is shared and requires 2311 * special TLB handling. 2312 */ 2313 cpudata->shared_asid = true; 2314 vmcb->ctrl.guest_asid = svm_maxasid - 1; 2315 mutex_exit(&svm_asidlock); 2316 } 2317 2318 static void 2319 svm_asid_free(struct nvmm_cpu *vcpu) 2320 { 2321 struct svm_cpudata *cpudata = vcpu->cpudata; 2322 struct vmcb *vmcb = cpudata->vmcb; 2323 size_t oct, bit; 2324 2325 if (cpudata->shared_asid) { 2326 return; 2327 } 2328 2329 oct = vmcb->ctrl.guest_asid / 8; 2330 bit = vmcb->ctrl.guest_asid % 8; 2331 2332 mutex_enter(&svm_asidlock); 2333 svm_asidmap[oct] &= ~__BIT(bit); 2334 mutex_exit(&svm_asidlock); 2335 } 2336 2337 static void 2338 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2339 { 2340 struct svm_cpudata *cpudata = vcpu->cpudata; 2341 struct vmcb *vmcb = cpudata->vmcb; 2342 2343 /* Allow reads/writes of Control Registers. */ 2344 vmcb->ctrl.intercept_cr = 0; 2345 2346 /* Allow reads/writes of Debug Registers. */ 2347 vmcb->ctrl.intercept_dr = 0; 2348 2349 /* Allow exceptions 0 to 31. */ 2350 vmcb->ctrl.intercept_vec = 0; 2351 2352 /* 2353 * Allow: 2354 * - SMI [smm interrupts] 2355 * - VINTR [virtual interrupts] 2356 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 2357 * - RIDTR [reads of IDTR] 2358 * - RGDTR [reads of GDTR] 2359 * - RLDTR [reads of LDTR] 2360 * - RTR [reads of TR] 2361 * - WIDTR [writes of IDTR] 2362 * - WGDTR [writes of GDTR] 2363 * - WLDTR [writes of LDTR] 2364 * - WTR [writes of TR] 2365 * - RDTSC [rdtsc instruction] 2366 * - PUSHF [pushf instruction] 2367 * - POPF [popf instruction] 2368 * - IRET [iret instruction] 2369 * - INTN [int $n instructions] 2370 * - PAUSE [pause instruction] 2371 * - INVLPG [invplg instruction] 2372 * - TASKSW [task switches] 2373 * 2374 * Intercept the rest below. 2375 */ 2376 vmcb->ctrl.intercept_misc1 = 2377 VMCB_CTRL_INTERCEPT_INTR | 2378 VMCB_CTRL_INTERCEPT_NMI | 2379 VMCB_CTRL_INTERCEPT_INIT | 2380 VMCB_CTRL_INTERCEPT_RDPMC | 2381 VMCB_CTRL_INTERCEPT_CPUID | 2382 VMCB_CTRL_INTERCEPT_RSM | 2383 VMCB_CTRL_INTERCEPT_INVD | 2384 VMCB_CTRL_INTERCEPT_HLT | 2385 VMCB_CTRL_INTERCEPT_INVLPGA | 2386 VMCB_CTRL_INTERCEPT_IOIO_PROT | 2387 VMCB_CTRL_INTERCEPT_MSR_PROT | 2388 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 2389 VMCB_CTRL_INTERCEPT_SHUTDOWN; 2390 2391 /* 2392 * Allow: 2393 * - ICEBP [icebp instruction] 2394 * - WBINVD [wbinvd instruction] 2395 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 2396 * 2397 * Intercept the rest below. 2398 */ 2399 vmcb->ctrl.intercept_misc2 = 2400 VMCB_CTRL_INTERCEPT_VMRUN | 2401 VMCB_CTRL_INTERCEPT_VMMCALL | 2402 VMCB_CTRL_INTERCEPT_VMLOAD | 2403 VMCB_CTRL_INTERCEPT_VMSAVE | 2404 VMCB_CTRL_INTERCEPT_STGI | 2405 VMCB_CTRL_INTERCEPT_CLGI | 2406 VMCB_CTRL_INTERCEPT_SKINIT | 2407 VMCB_CTRL_INTERCEPT_RDTSCP | 2408 VMCB_CTRL_INTERCEPT_MONITOR | 2409 VMCB_CTRL_INTERCEPT_MWAIT | 2410 VMCB_CTRL_INTERCEPT_XSETBV | 2411 VMCB_CTRL_INTERCEPT_RDPRU; 2412 2413 /* 2414 * Intercept everything. 2415 */ 2416 vmcb->ctrl.intercept_misc3 = 2417 VMCB_CTRL_INTERCEPT_INVLPGB_ALL | 2418 VMCB_CTRL_INTERCEPT_PCID | 2419 VMCB_CTRL_INTERCEPT_MCOMMIT | 2420 VMCB_CTRL_INTERCEPT_TLBSYNC; 2421 2422 /* Intercept all I/O accesses. */ 2423 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 2424 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 2425 2426 /* Allow direct access to certain MSRs. */ 2427 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 2428 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 2429 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 2430 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 2431 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 2432 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 2433 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 2434 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 2435 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 2436 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 2437 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 2438 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 2439 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 2440 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 2441 2442 /* Generate ASID. */ 2443 svm_asid_alloc(vcpu); 2444 2445 /* Virtual TPR. */ 2446 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 2447 2448 /* Enable Nested Paging. */ 2449 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 2450 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 2451 2452 /* Init XSAVE header. */ 2453 cpudata->xcr0_mask = svm_xcr0_mask; 2454 KASSERTMSG(nvmm_x86_xcr0_valid(cpudata->xcr0_mask, svm_xcr0_mask), 2455 "cpudata->xcr0_mask=0x%"PRIx64" svm_xcr0_mask=0x%"PRIx64, 2456 cpudata->xcr0_mask, svm_xcr0_mask); 2457 cpudata->gfpu.xsh_xstate_bv = cpudata->xcr0_mask; 2458 cpudata->gfpu.xsh_xcomp_bv = 0; 2459 2460 /* These MSRs are static. */ 2461 cpudata->star = rdmsr(MSR_STAR); 2462 cpudata->lstar = rdmsr(MSR_LSTAR); 2463 cpudata->cstar = rdmsr(MSR_CSTAR); 2464 cpudata->sfmask = rdmsr(MSR_SFMASK); 2465 2466 /* Install the RESET state. */ 2467 memcpy(&vcpu->comm->state, &nvmm_x86_reset_state, 2468 sizeof(nvmm_x86_reset_state)); 2469 vcpu->comm->state_wanted = NVMM_X64_STATE_ALL; 2470 vcpu->comm->state_cached = 0; 2471 svm_vcpu_setstate(vcpu); 2472 } 2473 2474 static int 2475 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2476 { 2477 size_t xsave_size, cpudata_size; 2478 struct svm_cpudata *cpudata; 2479 int error; 2480 2481 /* 2482 * Compute the size of the SVM cpudata. We put the 2483 * variable-length XSAVE area at the end so if it's small 2484 * enough, it stays within a single page. We size the XSAVE 2485 * area for the maximum set of features supported by the CPU 2486 * which a guest can enable (which may be more than the NetBSD 2487 * host enables for itself -- hence we don't use 2488 * x86_fpu_save_size here!). 2489 */ 2490 xsave_size = nvmm_x86_xsave_size(svm_xcr0_mask); 2491 KASSERT(xsave_size < SIZE_MAX - offsetof(struct svm_cpudata, gfpu)); 2492 cpudata_size = MAX(sizeof(*cpudata), 2493 offsetof(struct svm_cpudata, gfpu) + xsave_size); 2494 2495 /* Allocate the SVM cpudata. */ 2496 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 2497 roundup(cpudata_size, PAGE_SIZE), 0, 2498 UVM_KMF_WIRED|UVM_KMF_ZERO); 2499 vcpu->cpudata = cpudata; 2500 2501 /* VMCB */ 2502 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 2503 VMCB_NPAGES); 2504 if (error) 2505 goto error; 2506 2507 /* I/O Bitmap */ 2508 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 2509 IOBM_NPAGES); 2510 if (error) 2511 goto error; 2512 2513 /* MSR Bitmap */ 2514 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 2515 MSRBM_NPAGES); 2516 if (error) 2517 goto error; 2518 2519 /* Init the VCPU info. */ 2520 svm_vcpu_init(mach, vcpu); 2521 2522 return 0; 2523 2524 error: 2525 if (cpudata->vmcb_pa) { 2526 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2527 VMCB_NPAGES); 2528 } 2529 if (cpudata->iobm_pa) { 2530 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2531 IOBM_NPAGES); 2532 } 2533 if (cpudata->msrbm_pa) { 2534 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2535 MSRBM_NPAGES); 2536 } 2537 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2538 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2539 return error; 2540 } 2541 2542 static void 2543 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2544 { 2545 struct svm_cpudata *cpudata = vcpu->cpudata; 2546 2547 svm_asid_free(vcpu); 2548 2549 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2550 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2551 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2552 2553 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2554 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2555 } 2556 2557 /* -------------------------------------------------------------------------- */ 2558 2559 static int 2560 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data) 2561 { 2562 struct nvmm_vcpu_conf_cpuid *cpuid = data; 2563 size_t i; 2564 2565 if (__predict_false(cpuid->mask && cpuid->exit)) { 2566 return EINVAL; 2567 } 2568 if (__predict_false(cpuid->mask && 2569 ((cpuid->u.mask.set.eax & cpuid->u.mask.del.eax) || 2570 (cpuid->u.mask.set.ebx & cpuid->u.mask.del.ebx) || 2571 (cpuid->u.mask.set.ecx & cpuid->u.mask.del.ecx) || 2572 (cpuid->u.mask.set.edx & cpuid->u.mask.del.edx)))) { 2573 return EINVAL; 2574 } 2575 2576 /* If unset, delete, to restore the default behavior. */ 2577 if (!cpuid->mask && !cpuid->exit) { 2578 for (i = 0; i < SVM_NCPUIDS; i++) { 2579 if (!cpudata->cpuidpresent[i]) { 2580 continue; 2581 } 2582 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2583 cpudata->cpuidpresent[i] = false; 2584 } 2585 } 2586 return 0; 2587 } 2588 2589 /* If already here, replace. */ 2590 for (i = 0; i < SVM_NCPUIDS; i++) { 2591 if (!cpudata->cpuidpresent[i]) { 2592 continue; 2593 } 2594 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2595 memcpy(&cpudata->cpuid[i], cpuid, 2596 sizeof(struct nvmm_vcpu_conf_cpuid)); 2597 return 0; 2598 } 2599 } 2600 2601 /* Not here, insert. */ 2602 for (i = 0; i < SVM_NCPUIDS; i++) { 2603 if (!cpudata->cpuidpresent[i]) { 2604 cpudata->cpuidpresent[i] = true; 2605 memcpy(&cpudata->cpuid[i], cpuid, 2606 sizeof(struct nvmm_vcpu_conf_cpuid)); 2607 return 0; 2608 } 2609 } 2610 2611 return ENOBUFS; 2612 } 2613 2614 static int 2615 svm_vcpu_configure_xcr0_mask(struct svm_cpudata *cpudata, void *data) 2616 { 2617 const uint64_t *xcr0_maskp = data; 2618 2619 /* 2620 * Refuse to enable XCR0 bits (extended CPU state components) 2621 * not supported by this system, or to set up otherwise 2622 * nonsensical masks like AVX (YMM_Hi128) but not SSE (XMM) 2623 * registers. Exception: The mask can be all-zero to disable 2624 * all XSAVE state components. 2625 */ 2626 if (*xcr0_maskp != 0 && 2627 !nvmm_x86_xcr0_valid(*xcr0_maskp, svm_xcr0_mask)) 2628 return EINVAL; 2629 2630 /* 2631 * Out of paranoia, clear any existing extended CPU state. 2632 * This operation is unlikely to be used before the guest has 2633 * begun execution at all, so the extended CPU state is 2634 * probably all zero. But in case some weird hypervisor 2635 * software tries to change the XCR0 mask dynamically, let's 2636 * avoid accidentally leaking things through any extended CPU 2637 * state. 2638 * 2639 * We could zero only the components that are getting disabled. 2640 * But if the saved state is compated (XSAVEC), we wouldd also 2641 * have to move the remaining components around in order to 2642 * avoid zeroing them. Since no software is likely to try this 2643 * anyway, we'll just zero everything to keep it simple and 2644 * avoid having to test the difficult-and-unused paths. 2645 */ 2646 memset(&cpudata->gfpu, 0, nvmm_x86_xsave_size(svm_xcr0_mask)); 2647 2648 /* 2649 * Set the XCR0 mask, and limit the guest's XCR0 to this mask. 2650 * Any extended CPU state the guest had previously been using 2651 * will be wiped out. 2652 */ 2653 cpudata->xcr0_mask = *xcr0_maskp; 2654 cpudata->gxcr0 &= cpudata->xcr0_mask; 2655 KASSERTMSG((cpudata->xcr0_mask == 0 || 2656 nvmm_x86_xcr0_valid(cpudata->gxcr0, cpudata->xcr0_mask)), 2657 "gxcr0=0x%"PRIx64" xcr0_mask=0x%"PRIx64, 2658 cpudata->gxcr0, cpudata->xcr0_mask); 2659 return 0; 2660 } 2661 2662 static int 2663 svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) 2664 { 2665 struct svm_cpudata *cpudata = vcpu->cpudata; 2666 2667 switch (op) { 2668 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): 2669 return svm_vcpu_configure_cpuid(cpudata, data); 2670 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_XCR0_MASK): 2671 return svm_vcpu_configure_xcr0_mask(cpudata, data); 2672 default: 2673 return EINVAL; 2674 } 2675 } 2676 2677 /* -------------------------------------------------------------------------- */ 2678 2679 static void 2680 svm_tlb_flush(struct pmap *pm) 2681 { 2682 struct nvmm_machine *mach = pm->pm_data; 2683 struct svm_machdata *machdata = mach->machdata; 2684 2685 atomic_inc_64(&machdata->mach_htlb_gen); 2686 2687 /* Generates IPIs, which cause #VMEXITs. */ 2688 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_NVMM); 2689 } 2690 2691 static void 2692 svm_machine_create(struct nvmm_machine *mach) 2693 { 2694 struct svm_machdata *machdata; 2695 2696 /* Fill in pmap info. */ 2697 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2698 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2699 2700 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2701 mach->machdata = machdata; 2702 2703 /* Start with an hTLB flush everywhere. */ 2704 machdata->mach_htlb_gen = 1; 2705 } 2706 2707 static void 2708 svm_machine_destroy(struct nvmm_machine *mach) 2709 { 2710 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2711 } 2712 2713 static int 2714 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2715 { 2716 panic("%s: impossible", __func__); 2717 } 2718 2719 /* -------------------------------------------------------------------------- */ 2720 2721 static bool 2722 svm_ident(void) 2723 { 2724 u_int descs[4]; 2725 uint64_t msr; 2726 2727 if (cpu_vendor != CPUVENDOR_AMD) { 2728 return false; 2729 } 2730 if (!(cpu_feature[3] & CPUID_SVM)) { 2731 printf("NVMM: SVM not supported\n"); 2732 return false; 2733 } 2734 2735 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2736 printf("NVMM: CPUID leaf not available\n"); 2737 return false; 2738 } 2739 x86_cpuid(0x8000000a, descs); 2740 2741 /* Expect revision 1. */ 2742 if (__SHIFTOUT(descs[0], CPUID_AMD_SVM_REV) != 1) { 2743 printf("NVMM: SVM revision not supported\n"); 2744 return false; 2745 } 2746 2747 /* Want Nested Paging. */ 2748 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2749 printf("NVMM: SVM-NP not supported\n"); 2750 return false; 2751 } 2752 2753 /* Want nRIP. */ 2754 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2755 printf("NVMM: SVM-NRIPS not supported\n"); 2756 return false; 2757 } 2758 2759 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2760 2761 msr = rdmsr(MSR_VMCR); 2762 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2763 printf("NVMM: SVM disabled in BIOS\n"); 2764 return false; 2765 } 2766 2767 return true; 2768 } 2769 2770 static void 2771 svm_init_asid(uint32_t maxasid) 2772 { 2773 size_t i, j, allocsz; 2774 2775 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2776 2777 /* Arbitrarily limit. */ 2778 maxasid = uimin(maxasid, 8192); 2779 2780 svm_maxasid = maxasid; 2781 allocsz = roundup(maxasid, 8) / 8; 2782 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2783 2784 /* ASID 0 is reserved for the host. */ 2785 svm_asidmap[0] |= __BIT(0); 2786 2787 /* ASID n-1 is special, we share it. */ 2788 i = (maxasid - 1) / 8; 2789 j = (maxasid - 1) % 8; 2790 svm_asidmap[i] |= __BIT(j); 2791 } 2792 2793 static void 2794 svm_change_cpu(void *arg1, void *arg2) 2795 { 2796 bool enable = arg1 != NULL; 2797 uint64_t msr; 2798 2799 msr = rdmsr(MSR_VMCR); 2800 if (msr & VMCR_SVMED) { 2801 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2802 } 2803 2804 if (!enable) { 2805 wrmsr(MSR_VM_HSAVE_PA, 0); 2806 } 2807 2808 msr = rdmsr(MSR_EFER); 2809 if (enable) { 2810 msr |= EFER_SVME; 2811 } else { 2812 msr &= ~EFER_SVME; 2813 } 2814 wrmsr(MSR_EFER, msr); 2815 2816 if (enable) { 2817 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2818 } 2819 } 2820 2821 static void 2822 svm_init(void) 2823 { 2824 CPU_INFO_ITERATOR cii; 2825 struct cpu_info *ci; 2826 struct vm_page *pg; 2827 u_int descs[4]; 2828 uint64_t xc; 2829 2830 x86_cpuid(0x8000000a, descs); 2831 2832 /* The guest TLB flush command. */ 2833 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2834 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2835 } else { 2836 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2837 } 2838 2839 /* Init the ASID. */ 2840 svm_init_asid(descs[1]); 2841 2842 /* 2843 * Init the XCR0 mask. 2844 * 2845 * x86_xsave_features is the cached result of 2846 * CPUID[EAX=0x0000000d,ECX=0].EDX:EAX, the set of all 2847 * supported XCR0 bits for user XSAVE state components on the 2848 * physical CPU. Hypervisor software can use 2849 * nvmm_vcpu_configure(NVMM_VCPU_CONF_XCR0_MASK) to restrict 2850 * the available features on a per-vCPU basis, e.g. in order to 2851 * limit guests to compatible features for migration. 2852 * 2853 * Out of paranoia, we mask off bit 63 which is reserved for 2854 * future extension which we don't understand because it's not 2855 * yet defined. 2856 */ 2857 svm_xcr0_mask = x86_xsave_features & __BITS(62, 0); 2858 KASSERTMSG((svm_xcr0_mask == 0 || 2859 nvmm_x86_xcr0_valid(svm_xcr0_mask, svm_xcr0_mask)), 2860 "svm_xcr0_mask=0x%"PRIx64, svm_xcr0_mask); 2861 2862 /* Init the max basic CPUID leaf. */ 2863 svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC); 2864 2865 /* Init the max extended CPUID leaf. */ 2866 x86_cpuid(0x80000000, descs); 2867 svm_cpuid_max_extended = uimin(descs[0], SVM_CPUID_MAX_EXTENDED); 2868 2869 memset(hsave, 0, sizeof(hsave)); 2870 for (CPU_INFO_FOREACH(cii, ci)) { 2871 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2872 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2873 } 2874 2875 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2876 xc_wait(xc); 2877 } 2878 2879 static void 2880 svm_fini_asid(void) 2881 { 2882 size_t allocsz; 2883 2884 allocsz = roundup(svm_maxasid, 8) / 8; 2885 kmem_free(svm_asidmap, allocsz); 2886 2887 mutex_destroy(&svm_asidlock); 2888 } 2889 2890 static void 2891 svm_fini(void) 2892 { 2893 uint64_t xc; 2894 size_t i; 2895 2896 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2897 xc_wait(xc); 2898 2899 for (i = 0; i < MAXCPUS; i++) { 2900 if (hsave[i].pa != 0) 2901 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2902 } 2903 2904 svm_fini_asid(); 2905 } 2906 2907 static void 2908 svm_capability(struct nvmm_capability *cap) 2909 { 2910 cap->arch.mach_conf_support = 0; 2911 cap->arch.vcpu_conf_support = 2912 NVMM_CAP_ARCH_VCPU_CONF_CPUID; 2913 cap->arch.xcr0_mask = svm_xcr0_mask; 2914 cap->arch.mxcsr_mask = x86_fpu_mxcsr_mask; 2915 cap->arch.conf_cpuid_maxops = SVM_NCPUIDS; 2916 } 2917 2918 const struct nvmm_impl nvmm_x86_svm = { 2919 .name = "x86-svm", 2920 .ident = svm_ident, 2921 .init = svm_init, 2922 .fini = svm_fini, 2923 .capability = svm_capability, 2924 .mach_conf_max = NVMM_X86_MACH_NCONF, 2925 .mach_conf_sizes = NULL, 2926 .vcpu_conf_max = NVMM_X86_VCPU_NCONF, 2927 .vcpu_conf_sizes = svm_vcpu_conf_sizes, 2928 .state_size = sizeof(struct nvmm_x64_state), 2929 .machine_create = svm_machine_create, 2930 .machine_destroy = svm_machine_destroy, 2931 .machine_configure = svm_machine_configure, 2932 .vcpu_create = svm_vcpu_create, 2933 .vcpu_destroy = svm_vcpu_destroy, 2934 .vcpu_configure = svm_vcpu_configure, 2935 .vcpu_setstate = svm_vcpu_setstate, 2936 .vcpu_getstate = svm_vcpu_getstate, 2937 .vcpu_inject = svm_vcpu_inject, 2938 .vcpu_run = svm_vcpu_run 2939 }; 2940
Indexes created Sun Oct 25 00:26:45 UTC 2026