nvmm_x86_svm.c revision 1.70
1 /* $NetBSD: nvmm_x86_svm.c,v 1.70 2020/08/20 11:09:56 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.70 2020/08/20 11:09:56 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 #include <sys/mman.h> 42 43 #include <uvm/uvm.h> 44 #include <uvm/uvm_page.h> 45 46 #include <x86/cputypes.h> 47 #include <x86/specialreg.h> 48 #include <x86/pmap.h> 49 #include <x86/dbregs.h> 50 #include <x86/cpu_counter.h> 51 #include <machine/cpuvar.h> 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/x86/nvmm_x86.h> 56 57 int svm_vmrun(paddr_t, uint64_t *); 58 59 static inline void 60 svm_clgi(void) 61 { 62 asm volatile ("clgi" ::: "memory"); 63 } 64 65 static inline void 66 svm_stgi(void) 67 { 68 asm volatile ("stgi" ::: "memory"); 69 } 70 71 #define MSR_VM_HSAVE_PA 0xC0010117 72 73 /* -------------------------------------------------------------------------- */ 74 75 #define VMCB_EXITCODE_CR0_READ 0x0000 76 #define VMCB_EXITCODE_CR1_READ 0x0001 77 #define VMCB_EXITCODE_CR2_READ 0x0002 78 #define VMCB_EXITCODE_CR3_READ 0x0003 79 #define VMCB_EXITCODE_CR4_READ 0x0004 80 #define VMCB_EXITCODE_CR5_READ 0x0005 81 #define VMCB_EXITCODE_CR6_READ 0x0006 82 #define VMCB_EXITCODE_CR7_READ 0x0007 83 #define VMCB_EXITCODE_CR8_READ 0x0008 84 #define VMCB_EXITCODE_CR9_READ 0x0009 85 #define VMCB_EXITCODE_CR10_READ 0x000A 86 #define VMCB_EXITCODE_CR11_READ 0x000B 87 #define VMCB_EXITCODE_CR12_READ 0x000C 88 #define VMCB_EXITCODE_CR13_READ 0x000D 89 #define VMCB_EXITCODE_CR14_READ 0x000E 90 #define VMCB_EXITCODE_CR15_READ 0x000F 91 #define VMCB_EXITCODE_CR0_WRITE 0x0010 92 #define VMCB_EXITCODE_CR1_WRITE 0x0011 93 #define VMCB_EXITCODE_CR2_WRITE 0x0012 94 #define VMCB_EXITCODE_CR3_WRITE 0x0013 95 #define VMCB_EXITCODE_CR4_WRITE 0x0014 96 #define VMCB_EXITCODE_CR5_WRITE 0x0015 97 #define VMCB_EXITCODE_CR6_WRITE 0x0016 98 #define VMCB_EXITCODE_CR7_WRITE 0x0017 99 #define VMCB_EXITCODE_CR8_WRITE 0x0018 100 #define VMCB_EXITCODE_CR9_WRITE 0x0019 101 #define VMCB_EXITCODE_CR10_WRITE 0x001A 102 #define VMCB_EXITCODE_CR11_WRITE 0x001B 103 #define VMCB_EXITCODE_CR12_WRITE 0x001C 104 #define VMCB_EXITCODE_CR13_WRITE 0x001D 105 #define VMCB_EXITCODE_CR14_WRITE 0x001E 106 #define VMCB_EXITCODE_CR15_WRITE 0x001F 107 #define VMCB_EXITCODE_DR0_READ 0x0020 108 #define VMCB_EXITCODE_DR1_READ 0x0021 109 #define VMCB_EXITCODE_DR2_READ 0x0022 110 #define VMCB_EXITCODE_DR3_READ 0x0023 111 #define VMCB_EXITCODE_DR4_READ 0x0024 112 #define VMCB_EXITCODE_DR5_READ 0x0025 113 #define VMCB_EXITCODE_DR6_READ 0x0026 114 #define VMCB_EXITCODE_DR7_READ 0x0027 115 #define VMCB_EXITCODE_DR8_READ 0x0028 116 #define VMCB_EXITCODE_DR9_READ 0x0029 117 #define VMCB_EXITCODE_DR10_READ 0x002A 118 #define VMCB_EXITCODE_DR11_READ 0x002B 119 #define VMCB_EXITCODE_DR12_READ 0x002C 120 #define VMCB_EXITCODE_DR13_READ 0x002D 121 #define VMCB_EXITCODE_DR14_READ 0x002E 122 #define VMCB_EXITCODE_DR15_READ 0x002F 123 #define VMCB_EXITCODE_DR0_WRITE 0x0030 124 #define VMCB_EXITCODE_DR1_WRITE 0x0031 125 #define VMCB_EXITCODE_DR2_WRITE 0x0032 126 #define VMCB_EXITCODE_DR3_WRITE 0x0033 127 #define VMCB_EXITCODE_DR4_WRITE 0x0034 128 #define VMCB_EXITCODE_DR5_WRITE 0x0035 129 #define VMCB_EXITCODE_DR6_WRITE 0x0036 130 #define VMCB_EXITCODE_DR7_WRITE 0x0037 131 #define VMCB_EXITCODE_DR8_WRITE 0x0038 132 #define VMCB_EXITCODE_DR9_WRITE 0x0039 133 #define VMCB_EXITCODE_DR10_WRITE 0x003A 134 #define VMCB_EXITCODE_DR11_WRITE 0x003B 135 #define VMCB_EXITCODE_DR12_WRITE 0x003C 136 #define VMCB_EXITCODE_DR13_WRITE 0x003D 137 #define VMCB_EXITCODE_DR14_WRITE 0x003E 138 #define VMCB_EXITCODE_DR15_WRITE 0x003F 139 #define VMCB_EXITCODE_EXCP0 0x0040 140 #define VMCB_EXITCODE_EXCP1 0x0041 141 #define VMCB_EXITCODE_EXCP2 0x0042 142 #define VMCB_EXITCODE_EXCP3 0x0043 143 #define VMCB_EXITCODE_EXCP4 0x0044 144 #define VMCB_EXITCODE_EXCP5 0x0045 145 #define VMCB_EXITCODE_EXCP6 0x0046 146 #define VMCB_EXITCODE_EXCP7 0x0047 147 #define VMCB_EXITCODE_EXCP8 0x0048 148 #define VMCB_EXITCODE_EXCP9 0x0049 149 #define VMCB_EXITCODE_EXCP10 0x004A 150 #define VMCB_EXITCODE_EXCP11 0x004B 151 #define VMCB_EXITCODE_EXCP12 0x004C 152 #define VMCB_EXITCODE_EXCP13 0x004D 153 #define VMCB_EXITCODE_EXCP14 0x004E 154 #define VMCB_EXITCODE_EXCP15 0x004F 155 #define VMCB_EXITCODE_EXCP16 0x0050 156 #define VMCB_EXITCODE_EXCP17 0x0051 157 #define VMCB_EXITCODE_EXCP18 0x0052 158 #define VMCB_EXITCODE_EXCP19 0x0053 159 #define VMCB_EXITCODE_EXCP20 0x0054 160 #define VMCB_EXITCODE_EXCP21 0x0055 161 #define VMCB_EXITCODE_EXCP22 0x0056 162 #define VMCB_EXITCODE_EXCP23 0x0057 163 #define VMCB_EXITCODE_EXCP24 0x0058 164 #define VMCB_EXITCODE_EXCP25 0x0059 165 #define VMCB_EXITCODE_EXCP26 0x005A 166 #define VMCB_EXITCODE_EXCP27 0x005B 167 #define VMCB_EXITCODE_EXCP28 0x005C 168 #define VMCB_EXITCODE_EXCP29 0x005D 169 #define VMCB_EXITCODE_EXCP30 0x005E 170 #define VMCB_EXITCODE_EXCP31 0x005F 171 #define VMCB_EXITCODE_INTR 0x0060 172 #define VMCB_EXITCODE_NMI 0x0061 173 #define VMCB_EXITCODE_SMI 0x0062 174 #define VMCB_EXITCODE_INIT 0x0063 175 #define VMCB_EXITCODE_VINTR 0x0064 176 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 177 #define VMCB_EXITCODE_IDTR_READ 0x0066 178 #define VMCB_EXITCODE_GDTR_READ 0x0067 179 #define VMCB_EXITCODE_LDTR_READ 0x0068 180 #define VMCB_EXITCODE_TR_READ 0x0069 181 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 182 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 183 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 184 #define VMCB_EXITCODE_TR_WRITE 0x006D 185 #define VMCB_EXITCODE_RDTSC 0x006E 186 #define VMCB_EXITCODE_RDPMC 0x006F 187 #define VMCB_EXITCODE_PUSHF 0x0070 188 #define VMCB_EXITCODE_POPF 0x0071 189 #define VMCB_EXITCODE_CPUID 0x0072 190 #define VMCB_EXITCODE_RSM 0x0073 191 #define VMCB_EXITCODE_IRET 0x0074 192 #define VMCB_EXITCODE_SWINT 0x0075 193 #define VMCB_EXITCODE_INVD 0x0076 194 #define VMCB_EXITCODE_PAUSE 0x0077 195 #define VMCB_EXITCODE_HLT 0x0078 196 #define VMCB_EXITCODE_INVLPG 0x0079 197 #define VMCB_EXITCODE_INVLPGA 0x007A 198 #define VMCB_EXITCODE_IOIO 0x007B 199 #define VMCB_EXITCODE_MSR 0x007C 200 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 201 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 202 #define VMCB_EXITCODE_SHUTDOWN 0x007F 203 #define VMCB_EXITCODE_VMRUN 0x0080 204 #define VMCB_EXITCODE_VMMCALL 0x0081 205 #define VMCB_EXITCODE_VMLOAD 0x0082 206 #define VMCB_EXITCODE_VMSAVE 0x0083 207 #define VMCB_EXITCODE_STGI 0x0084 208 #define VMCB_EXITCODE_CLGI 0x0085 209 #define VMCB_EXITCODE_SKINIT 0x0086 210 #define VMCB_EXITCODE_RDTSCP 0x0087 211 #define VMCB_EXITCODE_ICEBP 0x0088 212 #define VMCB_EXITCODE_WBINVD 0x0089 213 #define VMCB_EXITCODE_MONITOR 0x008A 214 #define VMCB_EXITCODE_MWAIT 0x008B 215 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 216 #define VMCB_EXITCODE_XSETBV 0x008D 217 #define VMCB_EXITCODE_RDPRU 0x008E 218 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 219 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 220 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 221 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 222 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 223 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 224 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 225 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 226 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 227 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 228 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 229 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 230 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 231 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 232 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 233 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 234 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 235 #define VMCB_EXITCODE_INVLPGB 0x00A0 236 #define VMCB_EXITCODE_INVLPGB_ILLEGAL 0x00A1 237 #define VMCB_EXITCODE_INVPCID 0x00A2 238 #define VMCB_EXITCODE_MCOMMIT 0x00A3 239 #define VMCB_EXITCODE_TLBSYNC 0x00A4 240 #define VMCB_EXITCODE_NPF 0x0400 241 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 242 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 243 #define VMCB_EXITCODE_VMGEXIT 0x0403 244 #define VMCB_EXITCODE_BUSY -2ULL 245 #define VMCB_EXITCODE_INVALID -1ULL 246 247 /* -------------------------------------------------------------------------- */ 248 249 struct vmcb_ctrl { 250 uint32_t intercept_cr; 251 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 252 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 253 254 uint32_t intercept_dr; 255 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 256 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 257 258 uint32_t intercept_vec; 259 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 260 261 uint32_t intercept_misc1; 262 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 263 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 264 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 265 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 266 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 267 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 268 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 269 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 270 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 271 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 272 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 273 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 274 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 275 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 276 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 277 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 278 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 279 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 280 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 281 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 282 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 283 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 284 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 285 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 286 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 287 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 288 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 289 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 290 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 291 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 292 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 293 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 294 295 uint32_t intercept_misc2; 296 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 297 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 298 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 299 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 300 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 301 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 302 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 303 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 304 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 305 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 306 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 307 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(11) 308 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED __BIT(12) 309 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 310 #define VMCB_CTRL_INTERCEPT_RDPRU __BIT(14) 311 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 312 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 313 314 uint32_t intercept_misc3; 315 #define VMCB_CTRL_INTERCEPT_INVLPGB_ALL __BIT(0) 316 #define VMCB_CTRL_INTERCEPT_INVLPGB_ILL __BIT(1) 317 #define VMCB_CTRL_INTERCEPT_PCID __BIT(2) 318 #define VMCB_CTRL_INTERCEPT_MCOMMIT __BIT(3) 319 #define VMCB_CTRL_INTERCEPT_TLBSYNC __BIT(4) 320 321 uint8_t rsvd1[36]; 322 uint16_t pause_filt_thresh; 323 uint16_t pause_filt_cnt; 324 uint64_t iopm_base_pa; 325 uint64_t msrpm_base_pa; 326 uint64_t tsc_offset; 327 uint32_t guest_asid; 328 329 uint32_t tlb_ctrl; 330 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 331 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 332 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 333 334 uint64_t v; 335 #define VMCB_CTRL_V_TPR __BITS(3,0) 336 #define VMCB_CTRL_V_IRQ __BIT(8) 337 #define VMCB_CTRL_V_VGIF __BIT(9) 338 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 339 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 340 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 341 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 342 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 343 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 344 345 uint64_t intr; 346 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 347 #define VMCB_CTRL_INTR_MASK __BIT(1) 348 349 uint64_t exitcode; 350 uint64_t exitinfo1; 351 uint64_t exitinfo2; 352 353 uint64_t exitintinfo; 354 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 355 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 356 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 357 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 358 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 359 360 uint64_t enable1; 361 #define VMCB_CTRL_ENABLE_NP __BIT(0) 362 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 363 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 364 #define VMCB_CTRL_ENABLE_GMET __BIT(3) 365 #define VMCB_CTRL_ENABLE_VTE __BIT(5) 366 367 uint64_t avic; 368 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 369 370 uint64_t ghcb; 371 372 uint64_t eventinj; 373 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 374 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 375 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 376 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 377 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 378 379 uint64_t n_cr3; 380 381 uint64_t enable2; 382 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 383 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 384 385 uint32_t vmcb_clean; 386 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 387 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 388 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 389 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 390 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 391 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 392 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 393 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 394 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 395 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 396 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 397 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 398 399 uint32_t rsvd2; 400 uint64_t nrip; 401 uint8_t inst_len; 402 uint8_t inst_bytes[15]; 403 uint64_t avic_abpp; 404 uint64_t rsvd3; 405 uint64_t avic_ltp; 406 407 uint64_t avic_phys; 408 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 409 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 410 411 uint64_t rsvd4; 412 uint64_t vmsa_ptr; 413 414 uint8_t pad[752]; 415 } __packed; 416 417 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 418 419 struct vmcb_segment { 420 uint16_t selector; 421 uint16_t attrib; /* hidden */ 422 uint32_t limit; /* hidden */ 423 uint64_t base; /* hidden */ 424 } __packed; 425 426 CTASSERT(sizeof(struct vmcb_segment) == 16); 427 428 struct vmcb_state { 429 struct vmcb_segment es; 430 struct vmcb_segment cs; 431 struct vmcb_segment ss; 432 struct vmcb_segment ds; 433 struct vmcb_segment fs; 434 struct vmcb_segment gs; 435 struct vmcb_segment gdt; 436 struct vmcb_segment ldt; 437 struct vmcb_segment idt; 438 struct vmcb_segment tr; 439 uint8_t rsvd1[43]; 440 uint8_t cpl; 441 uint8_t rsvd2[4]; 442 uint64_t efer; 443 uint8_t rsvd3[112]; 444 uint64_t cr4; 445 uint64_t cr3; 446 uint64_t cr0; 447 uint64_t dr7; 448 uint64_t dr6; 449 uint64_t rflags; 450 uint64_t rip; 451 uint8_t rsvd4[88]; 452 uint64_t rsp; 453 uint8_t rsvd5[24]; 454 uint64_t rax; 455 uint64_t star; 456 uint64_t lstar; 457 uint64_t cstar; 458 uint64_t sfmask; 459 uint64_t kernelgsbase; 460 uint64_t sysenter_cs; 461 uint64_t sysenter_esp; 462 uint64_t sysenter_eip; 463 uint64_t cr2; 464 uint8_t rsvd6[32]; 465 uint64_t g_pat; 466 uint64_t dbgctl; 467 uint64_t br_from; 468 uint64_t br_to; 469 uint64_t int_from; 470 uint64_t int_to; 471 uint8_t pad[2408]; 472 } __packed; 473 474 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 475 476 struct vmcb { 477 struct vmcb_ctrl ctrl; 478 struct vmcb_state state; 479 } __packed; 480 481 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 482 CTASSERT(offsetof(struct vmcb, state) == 0x400); 483 484 /* -------------------------------------------------------------------------- */ 485 486 static void svm_vcpu_state_provide(struct nvmm_cpu *, uint64_t); 487 static void svm_vcpu_state_commit(struct nvmm_cpu *); 488 489 struct svm_hsave { 490 paddr_t pa; 491 }; 492 493 static struct svm_hsave hsave[MAXCPUS]; 494 495 static uint8_t *svm_asidmap __read_mostly; 496 static uint32_t svm_maxasid __read_mostly; 497 static kmutex_t svm_asidlock __cacheline_aligned; 498 499 static bool svm_decode_assist __read_mostly; 500 static uint32_t svm_ctrl_tlb_flush __read_mostly; 501 502 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 503 static uint64_t svm_xcr0_mask __read_mostly; 504 505 #define SVM_NCPUIDS 32 506 507 #define VMCB_NPAGES 1 508 509 #define MSRBM_NPAGES 2 510 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 511 512 #define IOBM_NPAGES 3 513 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 514 515 /* Does not include EFER_LMSLE. */ 516 #define EFER_VALID \ 517 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 518 519 #define EFER_TLB_FLUSH \ 520 (EFER_NXE|EFER_LMA|EFER_LME) 521 #define CR0_TLB_FLUSH \ 522 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 523 #define CR4_TLB_FLUSH \ 524 (CR4_PSE|CR4_PAE|CR4_PGE|CR4_PCIDE|CR4_SMEP) 525 526 /* -------------------------------------------------------------------------- */ 527 528 struct svm_machdata { 529 volatile uint64_t mach_htlb_gen; 530 }; 531 532 static const size_t svm_vcpu_conf_sizes[NVMM_X86_VCPU_NCONF] = { 533 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = 534 sizeof(struct nvmm_vcpu_conf_cpuid), 535 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = 536 sizeof(struct nvmm_vcpu_conf_tpr) 537 }; 538 539 struct svm_cpudata { 540 /* General */ 541 bool shared_asid; 542 bool gtlb_want_flush; 543 bool gtsc_want_update; 544 uint64_t vcpu_htlb_gen; 545 546 /* VMCB */ 547 struct vmcb *vmcb; 548 paddr_t vmcb_pa; 549 550 /* I/O bitmap */ 551 uint8_t *iobm; 552 paddr_t iobm_pa; 553 554 /* MSR bitmap */ 555 uint8_t *msrbm; 556 paddr_t msrbm_pa; 557 558 /* Host state */ 559 uint64_t hxcr0; 560 uint64_t star; 561 uint64_t lstar; 562 uint64_t cstar; 563 uint64_t sfmask; 564 uint64_t fsbase; 565 uint64_t kernelgsbase; 566 567 /* Intr state */ 568 bool int_window_exit; 569 bool nmi_window_exit; 570 bool evt_pending; 571 572 /* Guest state */ 573 uint64_t gxcr0; 574 uint64_t gprs[NVMM_X64_NGPR]; 575 uint64_t drs[NVMM_X64_NDR]; 576 uint64_t gtsc; 577 struct xsave_header gfpu __aligned(64); 578 579 /* VCPU configuration. */ 580 bool cpuidpresent[SVM_NCPUIDS]; 581 struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; 582 }; 583 584 static void 585 svm_vmcb_cache_default(struct vmcb *vmcb) 586 { 587 vmcb->ctrl.vmcb_clean = 588 VMCB_CTRL_VMCB_CLEAN_I | 589 VMCB_CTRL_VMCB_CLEAN_IOPM | 590 VMCB_CTRL_VMCB_CLEAN_ASID | 591 VMCB_CTRL_VMCB_CLEAN_TPR | 592 VMCB_CTRL_VMCB_CLEAN_NP | 593 VMCB_CTRL_VMCB_CLEAN_CR | 594 VMCB_CTRL_VMCB_CLEAN_DR | 595 VMCB_CTRL_VMCB_CLEAN_DT | 596 VMCB_CTRL_VMCB_CLEAN_SEG | 597 VMCB_CTRL_VMCB_CLEAN_CR2 | 598 VMCB_CTRL_VMCB_CLEAN_LBR | 599 VMCB_CTRL_VMCB_CLEAN_AVIC; 600 } 601 602 static void 603 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 604 { 605 if (flags & NVMM_X64_STATE_SEGS) { 606 vmcb->ctrl.vmcb_clean &= 607 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 608 } 609 if (flags & NVMM_X64_STATE_CRS) { 610 vmcb->ctrl.vmcb_clean &= 611 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 612 VMCB_CTRL_VMCB_CLEAN_TPR); 613 } 614 if (flags & NVMM_X64_STATE_DRS) { 615 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 616 } 617 if (flags & NVMM_X64_STATE_MSRS) { 618 /* CR for EFER, NP for PAT. */ 619 vmcb->ctrl.vmcb_clean &= 620 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 621 } 622 } 623 624 static inline void 625 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 626 { 627 vmcb->ctrl.vmcb_clean &= ~flags; 628 } 629 630 static inline void 631 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 632 { 633 vmcb->ctrl.vmcb_clean = 0; 634 } 635 636 #define SVM_EVENT_TYPE_HW_INT 0 637 #define SVM_EVENT_TYPE_NMI 2 638 #define SVM_EVENT_TYPE_EXC 3 639 #define SVM_EVENT_TYPE_SW_INT 4 640 641 static void 642 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 643 { 644 struct svm_cpudata *cpudata = vcpu->cpudata; 645 struct vmcb *vmcb = cpudata->vmcb; 646 647 if (nmi) { 648 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 649 cpudata->nmi_window_exit = true; 650 } else { 651 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 652 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 653 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 654 cpudata->int_window_exit = true; 655 } 656 657 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 658 } 659 660 static void 661 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 662 { 663 struct svm_cpudata *cpudata = vcpu->cpudata; 664 struct vmcb *vmcb = cpudata->vmcb; 665 666 if (nmi) { 667 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 668 cpudata->nmi_window_exit = false; 669 } else { 670 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 671 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 672 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 673 cpudata->int_window_exit = false; 674 } 675 676 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 677 } 678 679 static inline int 680 svm_event_has_error(uint8_t vector) 681 { 682 switch (vector) { 683 case 8: /* #DF */ 684 case 10: /* #TS */ 685 case 11: /* #NP */ 686 case 12: /* #SS */ 687 case 13: /* #GP */ 688 case 14: /* #PF */ 689 case 17: /* #AC */ 690 case 30: /* #SX */ 691 return 1; 692 default: 693 return 0; 694 } 695 } 696 697 static int 698 svm_vcpu_inject(struct nvmm_cpu *vcpu) 699 { 700 struct nvmm_comm_page *comm = vcpu->comm; 701 struct svm_cpudata *cpudata = vcpu->cpudata; 702 struct vmcb *vmcb = cpudata->vmcb; 703 u_int evtype; 704 uint8_t vector; 705 uint64_t error; 706 int type = 0, err = 0; 707 708 evtype = comm->event.type; 709 vector = comm->event.vector; 710 error = comm->event.u.excp.error; 711 __insn_barrier(); 712 713 switch (evtype) { 714 case NVMM_VCPU_EVENT_EXCP: 715 type = SVM_EVENT_TYPE_EXC; 716 if (vector == 2 || vector >= 32) 717 return EINVAL; 718 if (vector == 3 || vector == 0) 719 return EINVAL; 720 err = svm_event_has_error(vector); 721 break; 722 case NVMM_VCPU_EVENT_INTR: 723 type = SVM_EVENT_TYPE_HW_INT; 724 if (vector == 2) { 725 type = SVM_EVENT_TYPE_NMI; 726 svm_event_waitexit_enable(vcpu, true); 727 } 728 err = 0; 729 break; 730 default: 731 return EINVAL; 732 } 733 734 vmcb->ctrl.eventinj = 735 __SHIFTIN((uint64_t)vector, VMCB_CTRL_EVENTINJ_VECTOR) | 736 __SHIFTIN((uint64_t)type, VMCB_CTRL_EVENTINJ_TYPE) | 737 __SHIFTIN((uint64_t)err, VMCB_CTRL_EVENTINJ_EV) | 738 __SHIFTIN((uint64_t)1, VMCB_CTRL_EVENTINJ_V) | 739 __SHIFTIN((uint64_t)error, VMCB_CTRL_EVENTINJ_ERRORCODE); 740 741 cpudata->evt_pending = true; 742 743 return 0; 744 } 745 746 static void 747 svm_inject_ud(struct nvmm_cpu *vcpu) 748 { 749 struct nvmm_comm_page *comm = vcpu->comm; 750 int ret __diagused; 751 752 comm->event.type = NVMM_VCPU_EVENT_EXCP; 753 comm->event.vector = 6; 754 comm->event.u.excp.error = 0; 755 756 ret = svm_vcpu_inject(vcpu); 757 KASSERT(ret == 0); 758 } 759 760 static void 761 svm_inject_gp(struct nvmm_cpu *vcpu) 762 { 763 struct nvmm_comm_page *comm = vcpu->comm; 764 int ret __diagused; 765 766 comm->event.type = NVMM_VCPU_EVENT_EXCP; 767 comm->event.vector = 13; 768 comm->event.u.excp.error = 0; 769 770 ret = svm_vcpu_inject(vcpu); 771 KASSERT(ret == 0); 772 } 773 774 static inline int 775 svm_vcpu_event_commit(struct nvmm_cpu *vcpu) 776 { 777 if (__predict_true(!vcpu->comm->event_commit)) { 778 return 0; 779 } 780 vcpu->comm->event_commit = false; 781 return svm_vcpu_inject(vcpu); 782 } 783 784 static inline void 785 svm_inkernel_advance(struct vmcb *vmcb) 786 { 787 /* 788 * Maybe we should also apply single-stepping and debug exceptions. 789 * Matters for guest-ring3, because it can execute 'cpuid' under a 790 * debugger. 791 */ 792 vmcb->state.rip = vmcb->ctrl.nrip; 793 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 794 } 795 796 #define SVM_CPUID_MAX_BASIC 0xD 797 #define SVM_CPUID_MAX_HYPERVISOR 0x40000000 798 #define SVM_CPUID_MAX_EXTENDED 0x8000001F 799 static uint32_t svm_cpuid_max_basic __read_mostly; 800 static uint32_t svm_cpuid_max_extended __read_mostly; 801 802 static void 803 svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx) 804 { 805 u_int descs[4]; 806 807 x86_cpuid2(eax, ecx, descs); 808 cpudata->vmcb->state.rax = descs[0]; 809 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 810 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 811 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 812 } 813 814 static void 815 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 816 { 817 struct svm_cpudata *cpudata = vcpu->cpudata; 818 uint64_t cr4; 819 820 if (eax < 0x40000000) { 821 if (__predict_false(eax > svm_cpuid_max_basic)) { 822 eax = svm_cpuid_max_basic; 823 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 824 } 825 } else if (eax < 0x80000000) { 826 if (__predict_false(eax > SVM_CPUID_MAX_HYPERVISOR)) { 827 eax = svm_cpuid_max_basic; 828 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 829 } 830 } else { 831 if (__predict_false(eax > svm_cpuid_max_extended)) { 832 eax = svm_cpuid_max_basic; 833 svm_inkernel_exec_cpuid(cpudata, eax, ecx); 834 } 835 } 836 837 switch (eax) { 838 case 0x00000000: 839 cpudata->vmcb->state.rax = svm_cpuid_max_basic; 840 break; 841 case 0x00000001: 842 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 843 844 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 845 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 846 CPUID_LOCAL_APIC_ID); 847 848 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 849 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 850 851 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 852 853 /* CPUID2_OSXSAVE depends on CR4. */ 854 cr4 = cpudata->vmcb->state.cr4; 855 if (!(cr4 & CR4_OSXSAVE)) { 856 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 857 } 858 break; 859 case 0x00000002: /* Empty */ 860 case 0x00000003: /* Empty */ 861 case 0x00000004: /* Empty */ 862 case 0x00000005: /* Monitor/MWait */ 863 case 0x00000006: /* Power Management Related Features */ 864 cpudata->vmcb->state.rax = 0; 865 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 866 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 867 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 868 break; 869 case 0x00000007: /* Structured Extended Features */ 870 switch (ecx) { 871 case 0: 872 cpudata->vmcb->state.rax = 0; 873 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 874 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 875 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 876 break; 877 default: 878 cpudata->vmcb->state.rax = 0; 879 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 880 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 881 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 882 break; 883 } 884 break; 885 case 0x00000008: /* Empty */ 886 case 0x00000009: /* Empty */ 887 case 0x0000000A: /* Empty */ 888 case 0x0000000B: /* Empty */ 889 case 0x0000000C: /* Empty */ 890 cpudata->vmcb->state.rax = 0; 891 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 892 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 893 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 894 break; 895 case 0x0000000D: /* Processor Extended State Enumeration */ 896 if (svm_xcr0_mask == 0) { 897 break; 898 } 899 switch (ecx) { 900 case 0: 901 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 902 if (cpudata->gxcr0 & XCR0_SSE) { 903 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 904 } else { 905 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 906 } 907 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 908 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; 909 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 910 break; 911 case 1: 912 cpudata->vmcb->state.rax &= 913 (CPUID_PES1_XSAVEOPT | CPUID_PES1_XSAVEC | 914 CPUID_PES1_XGETBV); 915 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 916 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 917 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 918 break; 919 default: 920 cpudata->vmcb->state.rax = 0; 921 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 922 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 923 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 924 break; 925 } 926 break; 927 928 case 0x40000000: /* Hypervisor Information */ 929 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR; 930 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 931 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 932 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 933 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 934 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 935 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 936 break; 937 938 case 0x80000000: 939 cpudata->vmcb->state.rax = svm_cpuid_max_extended; 940 break; 941 case 0x80000001: 942 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 943 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 944 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 945 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 946 break; 947 case 0x80000002: /* Extended Processor Name String */ 948 case 0x80000003: /* Extended Processor Name String */ 949 case 0x80000004: /* Extended Processor Name String */ 950 case 0x80000005: /* L1 Cache and TLB Information */ 951 case 0x80000006: /* L2 Cache and TLB and L3 Cache Information */ 952 break; 953 case 0x80000007: /* Processor Power Management and RAS Capabilities */ 954 cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax; 955 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx; 956 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx; 957 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx; 958 break; 959 case 0x80000008: /* Processor Capacity Parameters and Ext Feat Ident */ 960 cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax; 961 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx; 962 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx; 963 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx; 964 break; 965 case 0x80000009: /* Empty */ 966 case 0x8000000A: /* SVM Features */ 967 case 0x8000000B: /* Empty */ 968 case 0x8000000C: /* Empty */ 969 case 0x8000000D: /* Empty */ 970 case 0x8000000E: /* Empty */ 971 case 0x8000000F: /* Empty */ 972 case 0x80000010: /* Empty */ 973 case 0x80000011: /* Empty */ 974 case 0x80000012: /* Empty */ 975 case 0x80000013: /* Empty */ 976 case 0x80000014: /* Empty */ 977 case 0x80000015: /* Empty */ 978 case 0x80000016: /* Empty */ 979 case 0x80000017: /* Empty */ 980 case 0x80000018: /* Empty */ 981 cpudata->vmcb->state.rax = 0; 982 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 983 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 984 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 985 break; 986 case 0x80000019: /* TLB Characteristics for 1GB pages */ 987 case 0x8000001A: /* Instruction Optimizations */ 988 break; 989 case 0x8000001B: /* Instruction-Based Sampling Capabilities */ 990 case 0x8000001C: /* Lightweight Profiling Capabilities */ 991 cpudata->vmcb->state.rax = 0; 992 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 993 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 994 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 995 break; 996 case 0x8000001D: /* Cache Topology Information */ 997 case 0x8000001E: /* Processor Topology Information */ 998 break; /* TODO? */ 999 case 0x8000001F: /* Encrypted Memory Capabilities */ 1000 cpudata->vmcb->state.rax = 0; 1001 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 1002 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 1003 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 1004 break; 1005 1006 default: 1007 break; 1008 } 1009 } 1010 1011 static void 1012 svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) 1013 { 1014 exit->u.insn.npc = vmcb->ctrl.nrip; 1015 exit->reason = reason; 1016 } 1017 1018 static void 1019 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1020 struct nvmm_vcpu_exit *exit) 1021 { 1022 struct svm_cpudata *cpudata = vcpu->cpudata; 1023 struct nvmm_vcpu_conf_cpuid *cpuid; 1024 uint64_t eax, ecx; 1025 u_int descs[4]; 1026 size_t i; 1027 1028 eax = cpudata->vmcb->state.rax; 1029 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 1030 x86_cpuid2(eax, ecx, descs); 1031 1032 cpudata->vmcb->state.rax = descs[0]; 1033 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 1034 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 1035 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 1036 1037 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 1038 1039 for (i = 0; i < SVM_NCPUIDS; i++) { 1040 if (!cpudata->cpuidpresent[i]) { 1041 continue; 1042 } 1043 cpuid = &cpudata->cpuid[i]; 1044 if (cpuid->leaf != eax) { 1045 continue; 1046 } 1047 1048 if (cpuid->exit) { 1049 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID); 1050 return; 1051 } 1052 KASSERT(cpuid->mask); 1053 1054 /* del */ 1055 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax; 1056 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx; 1057 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx; 1058 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx; 1059 1060 /* set */ 1061 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax; 1062 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx; 1063 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx; 1064 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx; 1065 1066 break; 1067 } 1068 1069 svm_inkernel_advance(cpudata->vmcb); 1070 exit->reason = NVMM_VCPU_EXIT_NONE; 1071 } 1072 1073 static void 1074 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1075 struct nvmm_vcpu_exit *exit) 1076 { 1077 struct svm_cpudata *cpudata = vcpu->cpudata; 1078 struct vmcb *vmcb = cpudata->vmcb; 1079 1080 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 1081 svm_event_waitexit_disable(vcpu, false); 1082 } 1083 1084 svm_inkernel_advance(cpudata->vmcb); 1085 exit->reason = NVMM_VCPU_EXIT_HALTED; 1086 } 1087 1088 #define SVM_EXIT_IO_PORT __BITS(31,16) 1089 #define SVM_EXIT_IO_SEG __BITS(12,10) 1090 #define SVM_EXIT_IO_A64 __BIT(9) 1091 #define SVM_EXIT_IO_A32 __BIT(8) 1092 #define SVM_EXIT_IO_A16 __BIT(7) 1093 #define SVM_EXIT_IO_SZ32 __BIT(6) 1094 #define SVM_EXIT_IO_SZ16 __BIT(5) 1095 #define SVM_EXIT_IO_SZ8 __BIT(4) 1096 #define SVM_EXIT_IO_REP __BIT(3) 1097 #define SVM_EXIT_IO_STR __BIT(2) 1098 #define SVM_EXIT_IO_IN __BIT(0) 1099 1100 static void 1101 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1102 struct nvmm_vcpu_exit *exit) 1103 { 1104 struct svm_cpudata *cpudata = vcpu->cpudata; 1105 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1106 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 1107 1108 exit->reason = NVMM_VCPU_EXIT_IO; 1109 1110 exit->u.io.in = (info & SVM_EXIT_IO_IN) != 0; 1111 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 1112 1113 if (svm_decode_assist) { 1114 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 1115 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 1116 } else { 1117 exit->u.io.seg = -1; 1118 } 1119 1120 if (info & SVM_EXIT_IO_A64) { 1121 exit->u.io.address_size = 8; 1122 } else if (info & SVM_EXIT_IO_A32) { 1123 exit->u.io.address_size = 4; 1124 } else if (info & SVM_EXIT_IO_A16) { 1125 exit->u.io.address_size = 2; 1126 } 1127 1128 if (info & SVM_EXIT_IO_SZ32) { 1129 exit->u.io.operand_size = 4; 1130 } else if (info & SVM_EXIT_IO_SZ16) { 1131 exit->u.io.operand_size = 2; 1132 } else if (info & SVM_EXIT_IO_SZ8) { 1133 exit->u.io.operand_size = 1; 1134 } 1135 1136 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 1137 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 1138 exit->u.io.npc = nextpc; 1139 1140 svm_vcpu_state_provide(vcpu, 1141 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1142 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1143 } 1144 1145 static const uint64_t msr_ignore_list[] = { 1146 0xc0010055, /* MSR_CMPHALT */ 1147 MSR_DE_CFG, 1148 MSR_IC_CFG, 1149 MSR_UCODE_AMD_PATCHLEVEL 1150 }; 1151 1152 static bool 1153 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1154 struct nvmm_vcpu_exit *exit) 1155 { 1156 struct svm_cpudata *cpudata = vcpu->cpudata; 1157 struct vmcb *vmcb = cpudata->vmcb; 1158 uint64_t val; 1159 size_t i; 1160 1161 if (exit->reason == NVMM_VCPU_EXIT_RDMSR) { 1162 if (exit->u.rdmsr.msr == MSR_NB_CFG) { 1163 val = NB_CFG_INITAPICCPUIDLO; 1164 vmcb->state.rax = (val & 0xFFFFFFFF); 1165 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1166 goto handled; 1167 } 1168 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1169 if (msr_ignore_list[i] != exit->u.rdmsr.msr) 1170 continue; 1171 val = 0; 1172 vmcb->state.rax = (val & 0xFFFFFFFF); 1173 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1174 goto handled; 1175 } 1176 } else { 1177 if (exit->u.wrmsr.msr == MSR_EFER) { 1178 if (__predict_false(exit->u.wrmsr.val & ~EFER_VALID)) { 1179 goto error; 1180 } 1181 if ((vmcb->state.efer ^ exit->u.wrmsr.val) & 1182 EFER_TLB_FLUSH) { 1183 cpudata->gtlb_want_flush = true; 1184 } 1185 vmcb->state.efer = exit->u.wrmsr.val | EFER_SVME; 1186 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1187 goto handled; 1188 } 1189 if (exit->u.wrmsr.msr == MSR_TSC) { 1190 cpudata->gtsc = exit->u.wrmsr.val; 1191 cpudata->gtsc_want_update = true; 1192 goto handled; 1193 } 1194 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1195 if (msr_ignore_list[i] != exit->u.wrmsr.msr) 1196 continue; 1197 goto handled; 1198 } 1199 } 1200 1201 return false; 1202 1203 handled: 1204 svm_inkernel_advance(cpudata->vmcb); 1205 return true; 1206 1207 error: 1208 svm_inject_gp(vcpu); 1209 return true; 1210 } 1211 1212 static inline void 1213 svm_exit_rdmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1214 struct nvmm_vcpu_exit *exit) 1215 { 1216 struct svm_cpudata *cpudata = vcpu->cpudata; 1217 1218 exit->reason = NVMM_VCPU_EXIT_RDMSR; 1219 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1220 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip; 1221 1222 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1223 exit->reason = NVMM_VCPU_EXIT_NONE; 1224 return; 1225 } 1226 1227 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1228 } 1229 1230 static inline void 1231 svm_exit_wrmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1232 struct nvmm_vcpu_exit *exit) 1233 { 1234 struct svm_cpudata *cpudata = vcpu->cpudata; 1235 uint64_t rdx, rax; 1236 1237 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1238 rax = cpudata->vmcb->state.rax; 1239 1240 exit->reason = NVMM_VCPU_EXIT_WRMSR; 1241 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1242 exit->u.wrmsr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1243 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip; 1244 1245 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1246 exit->reason = NVMM_VCPU_EXIT_NONE; 1247 return; 1248 } 1249 1250 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1251 } 1252 1253 static void 1254 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1255 struct nvmm_vcpu_exit *exit) 1256 { 1257 struct svm_cpudata *cpudata = vcpu->cpudata; 1258 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1259 1260 if (info == 0) { 1261 svm_exit_rdmsr(mach, vcpu, exit); 1262 } else { 1263 svm_exit_wrmsr(mach, vcpu, exit); 1264 } 1265 } 1266 1267 static void 1268 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1269 struct nvmm_vcpu_exit *exit) 1270 { 1271 struct svm_cpudata *cpudata = vcpu->cpudata; 1272 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1273 1274 exit->reason = NVMM_VCPU_EXIT_MEMORY; 1275 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1276 exit->u.mem.prot = PROT_WRITE; 1277 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1278 exit->u.mem.prot = PROT_EXEC; 1279 else 1280 exit->u.mem.prot = PROT_READ; 1281 exit->u.mem.gpa = gpa; 1282 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1283 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1284 sizeof(exit->u.mem.inst_bytes)); 1285 1286 svm_vcpu_state_provide(vcpu, 1287 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1288 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1289 } 1290 1291 static void 1292 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1293 struct nvmm_vcpu_exit *exit) 1294 { 1295 struct svm_cpudata *cpudata = vcpu->cpudata; 1296 struct vmcb *vmcb = cpudata->vmcb; 1297 uint64_t val; 1298 1299 exit->reason = NVMM_VCPU_EXIT_NONE; 1300 1301 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1302 (vmcb->state.rax & 0xFFFFFFFF); 1303 1304 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1305 goto error; 1306 } else if (__predict_false(vmcb->state.cpl != 0)) { 1307 goto error; 1308 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1309 goto error; 1310 } else if (__predict_false((val & XCR0_X87) == 0)) { 1311 goto error; 1312 } 1313 1314 cpudata->gxcr0 = val; 1315 if (svm_xcr0_mask != 0) { 1316 wrxcr(0, cpudata->gxcr0); 1317 } 1318 1319 svm_inkernel_advance(cpudata->vmcb); 1320 return; 1321 1322 error: 1323 svm_inject_gp(vcpu); 1324 } 1325 1326 static void 1327 svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code) 1328 { 1329 exit->u.inv.hwcode = code; 1330 exit->reason = NVMM_VCPU_EXIT_INVALID; 1331 } 1332 1333 /* -------------------------------------------------------------------------- */ 1334 1335 static void 1336 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1337 { 1338 struct svm_cpudata *cpudata = vcpu->cpudata; 1339 1340 fpu_kern_enter(); 1341 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1342 1343 if (svm_xcr0_mask != 0) { 1344 cpudata->hxcr0 = rdxcr(0); 1345 wrxcr(0, cpudata->gxcr0); 1346 } 1347 } 1348 1349 static void 1350 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1351 { 1352 struct svm_cpudata *cpudata = vcpu->cpudata; 1353 1354 if (svm_xcr0_mask != 0) { 1355 cpudata->gxcr0 = rdxcr(0); 1356 wrxcr(0, cpudata->hxcr0); 1357 } 1358 1359 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1360 fpu_kern_leave(); 1361 } 1362 1363 static void 1364 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1365 { 1366 struct svm_cpudata *cpudata = vcpu->cpudata; 1367 1368 x86_dbregs_save(curlwp); 1369 1370 ldr7(0); 1371 1372 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1373 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1374 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1375 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1376 } 1377 1378 static void 1379 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1380 { 1381 struct svm_cpudata *cpudata = vcpu->cpudata; 1382 1383 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1384 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1385 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1386 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1387 1388 x86_dbregs_restore(curlwp); 1389 } 1390 1391 static void 1392 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1393 { 1394 struct svm_cpudata *cpudata = vcpu->cpudata; 1395 1396 cpudata->fsbase = rdmsr(MSR_FSBASE); 1397 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1398 } 1399 1400 static void 1401 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1402 { 1403 struct svm_cpudata *cpudata = vcpu->cpudata; 1404 1405 wrmsr(MSR_STAR, cpudata->star); 1406 wrmsr(MSR_LSTAR, cpudata->lstar); 1407 wrmsr(MSR_CSTAR, cpudata->cstar); 1408 wrmsr(MSR_SFMASK, cpudata->sfmask); 1409 wrmsr(MSR_FSBASE, cpudata->fsbase); 1410 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1411 } 1412 1413 /* -------------------------------------------------------------------------- */ 1414 1415 static inline void 1416 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1417 { 1418 struct svm_cpudata *cpudata = vcpu->cpudata; 1419 1420 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1421 cpudata->gtlb_want_flush = true; 1422 } 1423 } 1424 1425 static inline void 1426 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1427 { 1428 /* 1429 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1430 * executing on this hCPU and the hTLB already got flushed, or it 1431 * was executing on another hCPU in which case the catchup is done 1432 * in svm_gtlb_catchup(). 1433 */ 1434 } 1435 1436 static inline uint64_t 1437 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1438 { 1439 struct vmcb *vmcb = cpudata->vmcb; 1440 uint64_t machgen; 1441 1442 machgen = machdata->mach_htlb_gen; 1443 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1444 return machgen; 1445 } 1446 1447 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1448 return machgen; 1449 } 1450 1451 static inline void 1452 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1453 { 1454 struct vmcb *vmcb = cpudata->vmcb; 1455 1456 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1457 cpudata->vcpu_htlb_gen = machgen; 1458 } 1459 } 1460 1461 static inline void 1462 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb) 1463 { 1464 cpudata->evt_pending = false; 1465 1466 if (__predict_false(vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V)) { 1467 vmcb->ctrl.eventinj = vmcb->ctrl.exitintinfo; 1468 cpudata->evt_pending = true; 1469 } 1470 } 1471 1472 static int 1473 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1474 struct nvmm_vcpu_exit *exit) 1475 { 1476 struct nvmm_comm_page *comm = vcpu->comm; 1477 struct svm_machdata *machdata = mach->machdata; 1478 struct svm_cpudata *cpudata = vcpu->cpudata; 1479 struct vmcb *vmcb = cpudata->vmcb; 1480 uint64_t machgen; 1481 int hcpu; 1482 1483 if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) { 1484 return EINVAL; 1485 } 1486 svm_vcpu_state_commit(vcpu); 1487 comm->state_cached = 0; 1488 1489 kpreempt_disable(); 1490 hcpu = cpu_number(); 1491 1492 svm_gtlb_catchup(vcpu, hcpu); 1493 svm_htlb_catchup(vcpu, hcpu); 1494 1495 if (vcpu->hcpu_last != hcpu) { 1496 svm_vmcb_cache_flush_all(vmcb); 1497 cpudata->gtsc_want_update = true; 1498 } 1499 1500 svm_vcpu_guest_dbregs_enter(vcpu); 1501 svm_vcpu_guest_misc_enter(vcpu); 1502 svm_vcpu_guest_fpu_enter(vcpu); 1503 1504 while (1) { 1505 if (cpudata->gtlb_want_flush) { 1506 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1507 } else { 1508 vmcb->ctrl.tlb_ctrl = 0; 1509 } 1510 1511 if (__predict_false(cpudata->gtsc_want_update)) { 1512 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1513 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1514 } 1515 1516 svm_clgi(); 1517 machgen = svm_htlb_flush(machdata, cpudata); 1518 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1519 svm_htlb_flush_ack(cpudata, machgen); 1520 svm_stgi(); 1521 1522 svm_vmcb_cache_default(vmcb); 1523 1524 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1525 cpudata->gtlb_want_flush = false; 1526 cpudata->gtsc_want_update = false; 1527 vcpu->hcpu_last = hcpu; 1528 } 1529 svm_exit_evt(cpudata, vmcb); 1530 1531 switch (vmcb->ctrl.exitcode) { 1532 case VMCB_EXITCODE_INTR: 1533 case VMCB_EXITCODE_NMI: 1534 exit->reason = NVMM_VCPU_EXIT_NONE; 1535 break; 1536 case VMCB_EXITCODE_VINTR: 1537 svm_event_waitexit_disable(vcpu, false); 1538 exit->reason = NVMM_VCPU_EXIT_INT_READY; 1539 break; 1540 case VMCB_EXITCODE_IRET: 1541 svm_event_waitexit_disable(vcpu, true); 1542 exit->reason = NVMM_VCPU_EXIT_NMI_READY; 1543 break; 1544 case VMCB_EXITCODE_CPUID: 1545 svm_exit_cpuid(mach, vcpu, exit); 1546 break; 1547 case VMCB_EXITCODE_HLT: 1548 svm_exit_hlt(mach, vcpu, exit); 1549 break; 1550 case VMCB_EXITCODE_IOIO: 1551 svm_exit_io(mach, vcpu, exit); 1552 break; 1553 case VMCB_EXITCODE_MSR: 1554 svm_exit_msr(mach, vcpu, exit); 1555 break; 1556 case VMCB_EXITCODE_SHUTDOWN: 1557 exit->reason = NVMM_VCPU_EXIT_SHUTDOWN; 1558 break; 1559 case VMCB_EXITCODE_RDPMC: 1560 case VMCB_EXITCODE_RSM: 1561 case VMCB_EXITCODE_INVLPGA: 1562 case VMCB_EXITCODE_VMRUN: 1563 case VMCB_EXITCODE_VMMCALL: 1564 case VMCB_EXITCODE_VMLOAD: 1565 case VMCB_EXITCODE_VMSAVE: 1566 case VMCB_EXITCODE_STGI: 1567 case VMCB_EXITCODE_CLGI: 1568 case VMCB_EXITCODE_SKINIT: 1569 case VMCB_EXITCODE_RDTSCP: 1570 case VMCB_EXITCODE_RDPRU: 1571 case VMCB_EXITCODE_INVLPGB: 1572 case VMCB_EXITCODE_INVPCID: 1573 case VMCB_EXITCODE_MCOMMIT: 1574 case VMCB_EXITCODE_TLBSYNC: 1575 svm_inject_ud(vcpu); 1576 exit->reason = NVMM_VCPU_EXIT_NONE; 1577 break; 1578 case VMCB_EXITCODE_MONITOR: 1579 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR); 1580 break; 1581 case VMCB_EXITCODE_MWAIT: 1582 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1583 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT); 1584 break; 1585 case VMCB_EXITCODE_XSETBV: 1586 svm_exit_xsetbv(mach, vcpu, exit); 1587 break; 1588 case VMCB_EXITCODE_NPF: 1589 svm_exit_npf(mach, vcpu, exit); 1590 break; 1591 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1592 default: 1593 svm_exit_invalid(exit, vmcb->ctrl.exitcode); 1594 break; 1595 } 1596 1597 /* If no reason to return to userland, keep rolling. */ 1598 if (nvmm_return_needed()) { 1599 break; 1600 } 1601 if (exit->reason != NVMM_VCPU_EXIT_NONE) { 1602 break; 1603 } 1604 } 1605 1606 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1607 1608 svm_vcpu_guest_fpu_leave(vcpu); 1609 svm_vcpu_guest_misc_leave(vcpu); 1610 svm_vcpu_guest_dbregs_leave(vcpu); 1611 1612 kpreempt_enable(); 1613 1614 exit->exitstate.rflags = vmcb->state.rflags; 1615 exit->exitstate.cr8 = __SHIFTOUT(vmcb->ctrl.v, VMCB_CTRL_V_TPR); 1616 exit->exitstate.int_shadow = 1617 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1618 exit->exitstate.int_window_exiting = cpudata->int_window_exit; 1619 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit; 1620 exit->exitstate.evt_pending = cpudata->evt_pending; 1621 1622 return 0; 1623 } 1624 1625 /* -------------------------------------------------------------------------- */ 1626 1627 static int 1628 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1629 { 1630 struct pglist pglist; 1631 paddr_t _pa; 1632 vaddr_t _va; 1633 size_t i; 1634 int ret; 1635 1636 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1637 &pglist, 1, 0); 1638 if (ret != 0) 1639 return ENOMEM; 1640 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 1641 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1642 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1643 if (_va == 0) 1644 goto error; 1645 1646 for (i = 0; i < npages; i++) { 1647 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1648 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1649 } 1650 pmap_update(pmap_kernel()); 1651 1652 memset((void *)_va, 0, npages * PAGE_SIZE); 1653 1654 *pa = _pa; 1655 *va = _va; 1656 return 0; 1657 1658 error: 1659 for (i = 0; i < npages; i++) { 1660 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1661 } 1662 return ENOMEM; 1663 } 1664 1665 static void 1666 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1667 { 1668 size_t i; 1669 1670 pmap_kremove(va, npages * PAGE_SIZE); 1671 pmap_update(pmap_kernel()); 1672 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1673 for (i = 0; i < npages; i++) { 1674 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1675 } 1676 } 1677 1678 /* -------------------------------------------------------------------------- */ 1679 1680 #define SVM_MSRBM_READ __BIT(0) 1681 #define SVM_MSRBM_WRITE __BIT(1) 1682 1683 static void 1684 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1685 { 1686 uint64_t byte; 1687 uint8_t bitoff; 1688 1689 if (msr < 0x00002000) { 1690 /* Range 1 */ 1691 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1692 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1693 /* Range 2 */ 1694 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1695 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1696 /* Range 3 */ 1697 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1698 } else { 1699 panic("%s: wrong range", __func__); 1700 } 1701 1702 bitoff = (msr & 0x3) << 1; 1703 1704 if (read) { 1705 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1706 } 1707 if (write) { 1708 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1709 } 1710 } 1711 1712 #define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1713 #define SVM_SEG_ATTRIB_S __BIT(4) 1714 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1715 #define SVM_SEG_ATTRIB_P __BIT(7) 1716 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1717 #define SVM_SEG_ATTRIB_L __BIT(9) 1718 #define SVM_SEG_ATTRIB_DEF __BIT(10) 1719 #define SVM_SEG_ATTRIB_G __BIT(11) 1720 1721 static void 1722 svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1723 struct vmcb_segment *vseg) 1724 { 1725 vseg->selector = seg->selector; 1726 vseg->attrib = 1727 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1728 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1729 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1730 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1731 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1732 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1733 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1734 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1735 vseg->limit = seg->limit; 1736 vseg->base = seg->base; 1737 } 1738 1739 static void 1740 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1741 { 1742 seg->selector = vseg->selector; 1743 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1744 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1745 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1746 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1747 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1748 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1749 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1750 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1751 seg->limit = vseg->limit; 1752 seg->base = vseg->base; 1753 } 1754 1755 static inline bool 1756 svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1757 uint64_t flags) 1758 { 1759 if (flags & NVMM_X64_STATE_CRS) { 1760 if ((vmcb->state.cr0 ^ 1761 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1762 return true; 1763 } 1764 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1765 return true; 1766 } 1767 if ((vmcb->state.cr4 ^ 1768 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1769 return true; 1770 } 1771 } 1772 1773 if (flags & NVMM_X64_STATE_MSRS) { 1774 if ((vmcb->state.efer ^ 1775 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1776 return true; 1777 } 1778 } 1779 1780 return false; 1781 } 1782 1783 static void 1784 svm_vcpu_setstate(struct nvmm_cpu *vcpu) 1785 { 1786 struct nvmm_comm_page *comm = vcpu->comm; 1787 const struct nvmm_x64_state *state = &comm->state; 1788 struct svm_cpudata *cpudata = vcpu->cpudata; 1789 struct vmcb *vmcb = cpudata->vmcb; 1790 struct fxsave *fpustate; 1791 uint64_t flags; 1792 1793 flags = comm->state_wanted; 1794 1795 if (svm_state_tlb_flush(vmcb, state, flags)) { 1796 cpudata->gtlb_want_flush = true; 1797 } 1798 1799 if (flags & NVMM_X64_STATE_SEGS) { 1800 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1801 &vmcb->state.cs); 1802 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1803 &vmcb->state.ds); 1804 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1805 &vmcb->state.es); 1806 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1807 &vmcb->state.fs); 1808 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1809 &vmcb->state.gs); 1810 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1811 &vmcb->state.ss); 1812 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1813 &vmcb->state.gdt); 1814 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1815 &vmcb->state.idt); 1816 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1817 &vmcb->state.ldt); 1818 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1819 &vmcb->state.tr); 1820 1821 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1822 } 1823 1824 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1825 if (flags & NVMM_X64_STATE_GPRS) { 1826 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1827 1828 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1829 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1830 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1831 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1832 } 1833 1834 if (flags & NVMM_X64_STATE_CRS) { 1835 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1836 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1837 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1838 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1839 1840 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1841 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1842 VMCB_CTRL_V_TPR); 1843 1844 if (svm_xcr0_mask != 0) { 1845 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1846 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1847 cpudata->gxcr0 &= svm_xcr0_mask; 1848 cpudata->gxcr0 |= XCR0_X87; 1849 } 1850 } 1851 1852 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1853 if (flags & NVMM_X64_STATE_DRS) { 1854 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1855 1856 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1857 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1858 } 1859 1860 if (flags & NVMM_X64_STATE_MSRS) { 1861 /* 1862 * EFER_SVME is mandatory. 1863 */ 1864 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1865 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1866 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1867 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1868 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1869 vmcb->state.kernelgsbase = 1870 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1871 vmcb->state.sysenter_cs = 1872 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1873 vmcb->state.sysenter_esp = 1874 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1875 vmcb->state.sysenter_eip = 1876 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1877 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1878 1879 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 1880 cpudata->gtsc_want_update = true; 1881 } 1882 1883 if (flags & NVMM_X64_STATE_INTR) { 1884 if (state->intr.int_shadow) { 1885 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1886 } else { 1887 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1888 } 1889 1890 if (state->intr.int_window_exiting) { 1891 svm_event_waitexit_enable(vcpu, false); 1892 } else { 1893 svm_event_waitexit_disable(vcpu, false); 1894 } 1895 1896 if (state->intr.nmi_window_exiting) { 1897 svm_event_waitexit_enable(vcpu, true); 1898 } else { 1899 svm_event_waitexit_disable(vcpu, true); 1900 } 1901 } 1902 1903 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1904 if (flags & NVMM_X64_STATE_FPU) { 1905 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1906 sizeof(state->fpu)); 1907 1908 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1909 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1910 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1911 1912 if (svm_xcr0_mask != 0) { 1913 /* Reset XSTATE_BV, to force a reload. */ 1914 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1915 } 1916 } 1917 1918 svm_vmcb_cache_update(vmcb, flags); 1919 1920 comm->state_wanted = 0; 1921 comm->state_cached |= flags; 1922 } 1923 1924 static void 1925 svm_vcpu_getstate(struct nvmm_cpu *vcpu) 1926 { 1927 struct nvmm_comm_page *comm = vcpu->comm; 1928 struct nvmm_x64_state *state = &comm->state; 1929 struct svm_cpudata *cpudata = vcpu->cpudata; 1930 struct vmcb *vmcb = cpudata->vmcb; 1931 uint64_t flags; 1932 1933 flags = comm->state_wanted; 1934 1935 if (flags & NVMM_X64_STATE_SEGS) { 1936 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1937 &vmcb->state.cs); 1938 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1939 &vmcb->state.ds); 1940 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1941 &vmcb->state.es); 1942 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1943 &vmcb->state.fs); 1944 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1945 &vmcb->state.gs); 1946 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1947 &vmcb->state.ss); 1948 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1949 &vmcb->state.gdt); 1950 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1951 &vmcb->state.idt); 1952 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1953 &vmcb->state.ldt); 1954 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1955 &vmcb->state.tr); 1956 1957 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1958 } 1959 1960 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1961 if (flags & NVMM_X64_STATE_GPRS) { 1962 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1963 1964 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1965 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1966 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1967 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1968 } 1969 1970 if (flags & NVMM_X64_STATE_CRS) { 1971 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1972 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1973 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1974 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1975 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1976 VMCB_CTRL_V_TPR); 1977 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1978 } 1979 1980 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1981 if (flags & NVMM_X64_STATE_DRS) { 1982 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1983 1984 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1985 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1986 } 1987 1988 if (flags & NVMM_X64_STATE_MSRS) { 1989 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1990 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1991 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1992 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1993 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1994 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1995 vmcb->state.kernelgsbase; 1996 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1997 vmcb->state.sysenter_cs; 1998 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1999 vmcb->state.sysenter_esp; 2000 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 2001 vmcb->state.sysenter_eip; 2002 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 2003 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 2004 2005 /* Hide SVME. */ 2006 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 2007 } 2008 2009 if (flags & NVMM_X64_STATE_INTR) { 2010 state->intr.int_shadow = 2011 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 2012 state->intr.int_window_exiting = cpudata->int_window_exit; 2013 state->intr.nmi_window_exiting = cpudata->nmi_window_exit; 2014 state->intr.evt_pending = cpudata->evt_pending; 2015 } 2016 2017 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 2018 if (flags & NVMM_X64_STATE_FPU) { 2019 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 2020 sizeof(state->fpu)); 2021 } 2022 2023 comm->state_wanted = 0; 2024 comm->state_cached |= flags; 2025 } 2026 2027 static void 2028 svm_vcpu_state_provide(struct nvmm_cpu *vcpu, uint64_t flags) 2029 { 2030 vcpu->comm->state_wanted = flags; 2031 svm_vcpu_getstate(vcpu); 2032 } 2033 2034 static void 2035 svm_vcpu_state_commit(struct nvmm_cpu *vcpu) 2036 { 2037 vcpu->comm->state_wanted = vcpu->comm->state_commit; 2038 vcpu->comm->state_commit = 0; 2039 svm_vcpu_setstate(vcpu); 2040 } 2041 2042 /* -------------------------------------------------------------------------- */ 2043 2044 static void 2045 svm_asid_alloc(struct nvmm_cpu *vcpu) 2046 { 2047 struct svm_cpudata *cpudata = vcpu->cpudata; 2048 struct vmcb *vmcb = cpudata->vmcb; 2049 size_t i, oct, bit; 2050 2051 mutex_enter(&svm_asidlock); 2052 2053 for (i = 0; i < svm_maxasid; i++) { 2054 oct = i / 8; 2055 bit = i % 8; 2056 2057 if (svm_asidmap[oct] & __BIT(bit)) { 2058 continue; 2059 } 2060 2061 svm_asidmap[oct] |= __BIT(bit); 2062 vmcb->ctrl.guest_asid = i; 2063 mutex_exit(&svm_asidlock); 2064 return; 2065 } 2066 2067 /* 2068 * No free ASID. Use the last one, which is shared and requires 2069 * special TLB handling. 2070 */ 2071 cpudata->shared_asid = true; 2072 vmcb->ctrl.guest_asid = svm_maxasid - 1; 2073 mutex_exit(&svm_asidlock); 2074 } 2075 2076 static void 2077 svm_asid_free(struct nvmm_cpu *vcpu) 2078 { 2079 struct svm_cpudata *cpudata = vcpu->cpudata; 2080 struct vmcb *vmcb = cpudata->vmcb; 2081 size_t oct, bit; 2082 2083 if (cpudata->shared_asid) { 2084 return; 2085 } 2086 2087 oct = vmcb->ctrl.guest_asid / 8; 2088 bit = vmcb->ctrl.guest_asid % 8; 2089 2090 mutex_enter(&svm_asidlock); 2091 svm_asidmap[oct] &= ~__BIT(bit); 2092 mutex_exit(&svm_asidlock); 2093 } 2094 2095 static void 2096 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2097 { 2098 struct svm_cpudata *cpudata = vcpu->cpudata; 2099 struct vmcb *vmcb = cpudata->vmcb; 2100 2101 /* Allow reads/writes of Control Registers. */ 2102 vmcb->ctrl.intercept_cr = 0; 2103 2104 /* Allow reads/writes of Debug Registers. */ 2105 vmcb->ctrl.intercept_dr = 0; 2106 2107 /* Allow exceptions 0 to 31. */ 2108 vmcb->ctrl.intercept_vec = 0; 2109 2110 /* 2111 * Allow: 2112 * - SMI [smm interrupts] 2113 * - VINTR [virtual interrupts] 2114 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 2115 * - RIDTR [reads of IDTR] 2116 * - RGDTR [reads of GDTR] 2117 * - RLDTR [reads of LDTR] 2118 * - RTR [reads of TR] 2119 * - WIDTR [writes of IDTR] 2120 * - WGDTR [writes of GDTR] 2121 * - WLDTR [writes of LDTR] 2122 * - WTR [writes of TR] 2123 * - RDTSC [rdtsc instruction] 2124 * - PUSHF [pushf instruction] 2125 * - POPF [popf instruction] 2126 * - IRET [iret instruction] 2127 * - INTN [int $n instructions] 2128 * - INVD [invd instruction] 2129 * - PAUSE [pause instruction] 2130 * - INVLPG [invplg instruction] 2131 * - TASKSW [task switches] 2132 * 2133 * Intercept the rest below. 2134 */ 2135 vmcb->ctrl.intercept_misc1 = 2136 VMCB_CTRL_INTERCEPT_INTR | 2137 VMCB_CTRL_INTERCEPT_NMI | 2138 VMCB_CTRL_INTERCEPT_INIT | 2139 VMCB_CTRL_INTERCEPT_RDPMC | 2140 VMCB_CTRL_INTERCEPT_CPUID | 2141 VMCB_CTRL_INTERCEPT_RSM | 2142 VMCB_CTRL_INTERCEPT_HLT | 2143 VMCB_CTRL_INTERCEPT_INVLPGA | 2144 VMCB_CTRL_INTERCEPT_IOIO_PROT | 2145 VMCB_CTRL_INTERCEPT_MSR_PROT | 2146 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 2147 VMCB_CTRL_INTERCEPT_SHUTDOWN; 2148 2149 /* 2150 * Allow: 2151 * - ICEBP [icebp instruction] 2152 * - WBINVD [wbinvd instruction] 2153 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 2154 * 2155 * Intercept the rest below. 2156 */ 2157 vmcb->ctrl.intercept_misc2 = 2158 VMCB_CTRL_INTERCEPT_VMRUN | 2159 VMCB_CTRL_INTERCEPT_VMMCALL | 2160 VMCB_CTRL_INTERCEPT_VMLOAD | 2161 VMCB_CTRL_INTERCEPT_VMSAVE | 2162 VMCB_CTRL_INTERCEPT_STGI | 2163 VMCB_CTRL_INTERCEPT_CLGI | 2164 VMCB_CTRL_INTERCEPT_SKINIT | 2165 VMCB_CTRL_INTERCEPT_RDTSCP | 2166 VMCB_CTRL_INTERCEPT_MONITOR | 2167 VMCB_CTRL_INTERCEPT_MWAIT | 2168 VMCB_CTRL_INTERCEPT_XSETBV | 2169 VMCB_CTRL_INTERCEPT_RDPRU; 2170 2171 /* 2172 * Intercept everything. 2173 */ 2174 vmcb->ctrl.intercept_misc3 = 2175 VMCB_CTRL_INTERCEPT_INVLPGB_ALL | 2176 VMCB_CTRL_INTERCEPT_PCID | 2177 VMCB_CTRL_INTERCEPT_MCOMMIT | 2178 VMCB_CTRL_INTERCEPT_TLBSYNC; 2179 2180 /* Intercept all I/O accesses. */ 2181 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 2182 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 2183 2184 /* Allow direct access to certain MSRs. */ 2185 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 2186 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 2187 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 2188 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 2189 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 2190 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 2191 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 2192 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 2193 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 2194 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 2195 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 2196 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 2197 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 2198 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 2199 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 2200 2201 /* Generate ASID. */ 2202 svm_asid_alloc(vcpu); 2203 2204 /* Virtual TPR. */ 2205 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 2206 2207 /* Enable Nested Paging. */ 2208 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 2209 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 2210 2211 /* Init XSAVE header. */ 2212 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 2213 cpudata->gfpu.xsh_xcomp_bv = 0; 2214 2215 /* These MSRs are static. */ 2216 cpudata->star = rdmsr(MSR_STAR); 2217 cpudata->lstar = rdmsr(MSR_LSTAR); 2218 cpudata->cstar = rdmsr(MSR_CSTAR); 2219 cpudata->sfmask = rdmsr(MSR_SFMASK); 2220 2221 /* Install the RESET state. */ 2222 memcpy(&vcpu->comm->state, &nvmm_x86_reset_state, 2223 sizeof(nvmm_x86_reset_state)); 2224 vcpu->comm->state_wanted = NVMM_X64_STATE_ALL; 2225 vcpu->comm->state_cached = 0; 2226 svm_vcpu_setstate(vcpu); 2227 } 2228 2229 static int 2230 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2231 { 2232 struct svm_cpudata *cpudata; 2233 int error; 2234 2235 /* Allocate the SVM cpudata. */ 2236 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 2237 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 2238 UVM_KMF_WIRED|UVM_KMF_ZERO); 2239 vcpu->cpudata = cpudata; 2240 2241 /* VMCB */ 2242 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 2243 VMCB_NPAGES); 2244 if (error) 2245 goto error; 2246 2247 /* I/O Bitmap */ 2248 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 2249 IOBM_NPAGES); 2250 if (error) 2251 goto error; 2252 2253 /* MSR Bitmap */ 2254 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 2255 MSRBM_NPAGES); 2256 if (error) 2257 goto error; 2258 2259 /* Init the VCPU info. */ 2260 svm_vcpu_init(mach, vcpu); 2261 2262 return 0; 2263 2264 error: 2265 if (cpudata->vmcb_pa) { 2266 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2267 VMCB_NPAGES); 2268 } 2269 if (cpudata->iobm_pa) { 2270 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2271 IOBM_NPAGES); 2272 } 2273 if (cpudata->msrbm_pa) { 2274 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2275 MSRBM_NPAGES); 2276 } 2277 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2278 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2279 return error; 2280 } 2281 2282 static void 2283 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2284 { 2285 struct svm_cpudata *cpudata = vcpu->cpudata; 2286 2287 svm_asid_free(vcpu); 2288 2289 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2290 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2291 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2292 2293 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2294 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2295 } 2296 2297 /* -------------------------------------------------------------------------- */ 2298 2299 static int 2300 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data) 2301 { 2302 struct nvmm_vcpu_conf_cpuid *cpuid = data; 2303 size_t i; 2304 2305 if (__predict_false(cpuid->mask && cpuid->exit)) { 2306 return EINVAL; 2307 } 2308 if (__predict_false(cpuid->mask && 2309 ((cpuid->u.mask.set.eax & cpuid->u.mask.del.eax) || 2310 (cpuid->u.mask.set.ebx & cpuid->u.mask.del.ebx) || 2311 (cpuid->u.mask.set.ecx & cpuid->u.mask.del.ecx) || 2312 (cpuid->u.mask.set.edx & cpuid->u.mask.del.edx)))) { 2313 return EINVAL; 2314 } 2315 2316 /* If unset, delete, to restore the default behavior. */ 2317 if (!cpuid->mask && !cpuid->exit) { 2318 for (i = 0; i < SVM_NCPUIDS; i++) { 2319 if (!cpudata->cpuidpresent[i]) { 2320 continue; 2321 } 2322 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2323 cpudata->cpuidpresent[i] = false; 2324 } 2325 } 2326 return 0; 2327 } 2328 2329 /* If already here, replace. */ 2330 for (i = 0; i < SVM_NCPUIDS; i++) { 2331 if (!cpudata->cpuidpresent[i]) { 2332 continue; 2333 } 2334 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2335 memcpy(&cpudata->cpuid[i], cpuid, 2336 sizeof(struct nvmm_vcpu_conf_cpuid)); 2337 return 0; 2338 } 2339 } 2340 2341 /* Not here, insert. */ 2342 for (i = 0; i < SVM_NCPUIDS; i++) { 2343 if (!cpudata->cpuidpresent[i]) { 2344 cpudata->cpuidpresent[i] = true; 2345 memcpy(&cpudata->cpuid[i], cpuid, 2346 sizeof(struct nvmm_vcpu_conf_cpuid)); 2347 return 0; 2348 } 2349 } 2350 2351 return ENOBUFS; 2352 } 2353 2354 static int 2355 svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) 2356 { 2357 struct svm_cpudata *cpudata = vcpu->cpudata; 2358 2359 switch (op) { 2360 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): 2361 return svm_vcpu_configure_cpuid(cpudata, data); 2362 default: 2363 return EINVAL; 2364 } 2365 } 2366 2367 /* -------------------------------------------------------------------------- */ 2368 2369 static void 2370 svm_tlb_flush(struct pmap *pm) 2371 { 2372 struct nvmm_machine *mach = pm->pm_data; 2373 struct svm_machdata *machdata = mach->machdata; 2374 2375 atomic_inc_64(&machdata->mach_htlb_gen); 2376 2377 /* Generates IPIs, which cause #VMEXITs. */ 2378 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_NVMM); 2379 } 2380 2381 static void 2382 svm_machine_create(struct nvmm_machine *mach) 2383 { 2384 struct svm_machdata *machdata; 2385 2386 /* Fill in pmap info. */ 2387 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2388 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2389 2390 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2391 mach->machdata = machdata; 2392 2393 /* Start with an hTLB flush everywhere. */ 2394 machdata->mach_htlb_gen = 1; 2395 } 2396 2397 static void 2398 svm_machine_destroy(struct nvmm_machine *mach) 2399 { 2400 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2401 } 2402 2403 static int 2404 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2405 { 2406 panic("%s: impossible", __func__); 2407 } 2408 2409 /* -------------------------------------------------------------------------- */ 2410 2411 static bool 2412 svm_ident(void) 2413 { 2414 u_int descs[4]; 2415 uint64_t msr; 2416 2417 if (cpu_vendor != CPUVENDOR_AMD) { 2418 return false; 2419 } 2420 if (!(cpu_feature[3] & CPUID_SVM)) { 2421 printf("NVMM: SVM not supported\n"); 2422 return false; 2423 } 2424 2425 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2426 printf("NVMM: CPUID leaf not available\n"); 2427 return false; 2428 } 2429 x86_cpuid(0x8000000a, descs); 2430 2431 /* Want Nested Paging. */ 2432 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2433 printf("NVMM: SVM-NP not supported\n"); 2434 return false; 2435 } 2436 2437 /* Want nRIP. */ 2438 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2439 printf("NVMM: SVM-NRIPS not supported\n"); 2440 return false; 2441 } 2442 2443 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2444 2445 msr = rdmsr(MSR_VMCR); 2446 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2447 printf("NVMM: SVM disabled in BIOS\n"); 2448 return false; 2449 } 2450 2451 return true; 2452 } 2453 2454 static void 2455 svm_init_asid(uint32_t maxasid) 2456 { 2457 size_t i, j, allocsz; 2458 2459 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2460 2461 /* Arbitrarily limit. */ 2462 maxasid = uimin(maxasid, 8192); 2463 2464 svm_maxasid = maxasid; 2465 allocsz = roundup(maxasid, 8) / 8; 2466 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2467 2468 /* ASID 0 is reserved for the host. */ 2469 svm_asidmap[0] |= __BIT(0); 2470 2471 /* ASID n-1 is special, we share it. */ 2472 i = (maxasid - 1) / 8; 2473 j = (maxasid - 1) % 8; 2474 svm_asidmap[i] |= __BIT(j); 2475 } 2476 2477 static void 2478 svm_change_cpu(void *arg1, void *arg2) 2479 { 2480 bool enable = arg1 != NULL; 2481 uint64_t msr; 2482 2483 msr = rdmsr(MSR_VMCR); 2484 if (msr & VMCR_SVMED) { 2485 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2486 } 2487 2488 if (!enable) { 2489 wrmsr(MSR_VM_HSAVE_PA, 0); 2490 } 2491 2492 msr = rdmsr(MSR_EFER); 2493 if (enable) { 2494 msr |= EFER_SVME; 2495 } else { 2496 msr &= ~EFER_SVME; 2497 } 2498 wrmsr(MSR_EFER, msr); 2499 2500 if (enable) { 2501 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2502 } 2503 } 2504 2505 static void 2506 svm_init(void) 2507 { 2508 CPU_INFO_ITERATOR cii; 2509 struct cpu_info *ci; 2510 struct vm_page *pg; 2511 u_int descs[4]; 2512 uint64_t xc; 2513 2514 x86_cpuid(0x8000000a, descs); 2515 2516 /* The guest TLB flush command. */ 2517 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2518 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2519 } else { 2520 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2521 } 2522 2523 /* Init the ASID. */ 2524 svm_init_asid(descs[1]); 2525 2526 /* Init the XCR0 mask. */ 2527 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2528 2529 /* Init the max basic CPUID leaf. */ 2530 svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC); 2531 2532 /* Init the max extended CPUID leaf. */ 2533 x86_cpuid(0x80000000, descs); 2534 svm_cpuid_max_extended = uimin(descs[0], SVM_CPUID_MAX_EXTENDED); 2535 2536 memset(hsave, 0, sizeof(hsave)); 2537 for (CPU_INFO_FOREACH(cii, ci)) { 2538 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2539 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2540 } 2541 2542 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2543 xc_wait(xc); 2544 } 2545 2546 static void 2547 svm_fini_asid(void) 2548 { 2549 size_t allocsz; 2550 2551 allocsz = roundup(svm_maxasid, 8) / 8; 2552 kmem_free(svm_asidmap, allocsz); 2553 2554 mutex_destroy(&svm_asidlock); 2555 } 2556 2557 static void 2558 svm_fini(void) 2559 { 2560 uint64_t xc; 2561 size_t i; 2562 2563 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2564 xc_wait(xc); 2565 2566 for (i = 0; i < MAXCPUS; i++) { 2567 if (hsave[i].pa != 0) 2568 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2569 } 2570 2571 svm_fini_asid(); 2572 } 2573 2574 static void 2575 svm_capability(struct nvmm_capability *cap) 2576 { 2577 cap->arch.mach_conf_support = 0; 2578 cap->arch.vcpu_conf_support = 2579 NVMM_CAP_ARCH_VCPU_CONF_CPUID; 2580 cap->arch.xcr0_mask = svm_xcr0_mask; 2581 cap->arch.mxcsr_mask = x86_fpu_mxcsr_mask; 2582 cap->arch.conf_cpuid_maxops = SVM_NCPUIDS; 2583 } 2584 2585 const struct nvmm_impl nvmm_x86_svm = { 2586 .name = "x86-svm", 2587 .ident = svm_ident, 2588 .init = svm_init, 2589 .fini = svm_fini, 2590 .capability = svm_capability, 2591 .mach_conf_max = NVMM_X86_MACH_NCONF, 2592 .mach_conf_sizes = NULL, 2593 .vcpu_conf_max = NVMM_X86_VCPU_NCONF, 2594 .vcpu_conf_sizes = svm_vcpu_conf_sizes, 2595 .state_size = sizeof(struct nvmm_x64_state), 2596 .machine_create = svm_machine_create, 2597 .machine_destroy = svm_machine_destroy, 2598 .machine_configure = svm_machine_configure, 2599 .vcpu_create = svm_vcpu_create, 2600 .vcpu_destroy = svm_vcpu_destroy, 2601 .vcpu_configure = svm_vcpu_configure, 2602 .vcpu_setstate = svm_vcpu_setstate, 2603 .vcpu_getstate = svm_vcpu_getstate, 2604 .vcpu_inject = svm_vcpu_inject, 2605 .vcpu_run = svm_vcpu_run 2606 }; 2607
Indexes created Tue Sep 30 20:09:53 GMT 2025