nvmm_x86_svm.c revision 1.29
1 /* $NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 42 #include <uvm/uvm.h> 43 #include <uvm/uvm_page.h> 44 45 #include <x86/cputypes.h> 46 #include <x86/specialreg.h> 47 #include <x86/pmap.h> 48 #include <x86/dbregs.h> 49 #include <x86/cpu_counter.h> 50 #include <machine/cpuvar.h> 51 52 #include <dev/nvmm/nvmm.h> 53 #include <dev/nvmm/nvmm_internal.h> 54 #include <dev/nvmm/x86/nvmm_x86.h> 55 56 int svm_vmrun(paddr_t, uint64_t *); 57 58 #define MSR_VM_HSAVE_PA 0xC0010117 59 60 /* -------------------------------------------------------------------------- */ 61 62 #define VMCB_EXITCODE_CR0_READ 0x0000 63 #define VMCB_EXITCODE_CR1_READ 0x0001 64 #define VMCB_EXITCODE_CR2_READ 0x0002 65 #define VMCB_EXITCODE_CR3_READ 0x0003 66 #define VMCB_EXITCODE_CR4_READ 0x0004 67 #define VMCB_EXITCODE_CR5_READ 0x0005 68 #define VMCB_EXITCODE_CR6_READ 0x0006 69 #define VMCB_EXITCODE_CR7_READ 0x0007 70 #define VMCB_EXITCODE_CR8_READ 0x0008 71 #define VMCB_EXITCODE_CR9_READ 0x0009 72 #define VMCB_EXITCODE_CR10_READ 0x000A 73 #define VMCB_EXITCODE_CR11_READ 0x000B 74 #define VMCB_EXITCODE_CR12_READ 0x000C 75 #define VMCB_EXITCODE_CR13_READ 0x000D 76 #define VMCB_EXITCODE_CR14_READ 0x000E 77 #define VMCB_EXITCODE_CR15_READ 0x000F 78 #define VMCB_EXITCODE_CR0_WRITE 0x0010 79 #define VMCB_EXITCODE_CR1_WRITE 0x0011 80 #define VMCB_EXITCODE_CR2_WRITE 0x0012 81 #define VMCB_EXITCODE_CR3_WRITE 0x0013 82 #define VMCB_EXITCODE_CR4_WRITE 0x0014 83 #define VMCB_EXITCODE_CR5_WRITE 0x0015 84 #define VMCB_EXITCODE_CR6_WRITE 0x0016 85 #define VMCB_EXITCODE_CR7_WRITE 0x0017 86 #define VMCB_EXITCODE_CR8_WRITE 0x0018 87 #define VMCB_EXITCODE_CR9_WRITE 0x0019 88 #define VMCB_EXITCODE_CR10_WRITE 0x001A 89 #define VMCB_EXITCODE_CR11_WRITE 0x001B 90 #define VMCB_EXITCODE_CR12_WRITE 0x001C 91 #define VMCB_EXITCODE_CR13_WRITE 0x001D 92 #define VMCB_EXITCODE_CR14_WRITE 0x001E 93 #define VMCB_EXITCODE_CR15_WRITE 0x001F 94 #define VMCB_EXITCODE_DR0_READ 0x0020 95 #define VMCB_EXITCODE_DR1_READ 0x0021 96 #define VMCB_EXITCODE_DR2_READ 0x0022 97 #define VMCB_EXITCODE_DR3_READ 0x0023 98 #define VMCB_EXITCODE_DR4_READ 0x0024 99 #define VMCB_EXITCODE_DR5_READ 0x0025 100 #define VMCB_EXITCODE_DR6_READ 0x0026 101 #define VMCB_EXITCODE_DR7_READ 0x0027 102 #define VMCB_EXITCODE_DR8_READ 0x0028 103 #define VMCB_EXITCODE_DR9_READ 0x0029 104 #define VMCB_EXITCODE_DR10_READ 0x002A 105 #define VMCB_EXITCODE_DR11_READ 0x002B 106 #define VMCB_EXITCODE_DR12_READ 0x002C 107 #define VMCB_EXITCODE_DR13_READ 0x002D 108 #define VMCB_EXITCODE_DR14_READ 0x002E 109 #define VMCB_EXITCODE_DR15_READ 0x002F 110 #define VMCB_EXITCODE_DR0_WRITE 0x0030 111 #define VMCB_EXITCODE_DR1_WRITE 0x0031 112 #define VMCB_EXITCODE_DR2_WRITE 0x0032 113 #define VMCB_EXITCODE_DR3_WRITE 0x0033 114 #define VMCB_EXITCODE_DR4_WRITE 0x0034 115 #define VMCB_EXITCODE_DR5_WRITE 0x0035 116 #define VMCB_EXITCODE_DR6_WRITE 0x0036 117 #define VMCB_EXITCODE_DR7_WRITE 0x0037 118 #define VMCB_EXITCODE_DR8_WRITE 0x0038 119 #define VMCB_EXITCODE_DR9_WRITE 0x0039 120 #define VMCB_EXITCODE_DR10_WRITE 0x003A 121 #define VMCB_EXITCODE_DR11_WRITE 0x003B 122 #define VMCB_EXITCODE_DR12_WRITE 0x003C 123 #define VMCB_EXITCODE_DR13_WRITE 0x003D 124 #define VMCB_EXITCODE_DR14_WRITE 0x003E 125 #define VMCB_EXITCODE_DR15_WRITE 0x003F 126 #define VMCB_EXITCODE_EXCP0 0x0040 127 #define VMCB_EXITCODE_EXCP1 0x0041 128 #define VMCB_EXITCODE_EXCP2 0x0042 129 #define VMCB_EXITCODE_EXCP3 0x0043 130 #define VMCB_EXITCODE_EXCP4 0x0044 131 #define VMCB_EXITCODE_EXCP5 0x0045 132 #define VMCB_EXITCODE_EXCP6 0x0046 133 #define VMCB_EXITCODE_EXCP7 0x0047 134 #define VMCB_EXITCODE_EXCP8 0x0048 135 #define VMCB_EXITCODE_EXCP9 0x0049 136 #define VMCB_EXITCODE_EXCP10 0x004A 137 #define VMCB_EXITCODE_EXCP11 0x004B 138 #define VMCB_EXITCODE_EXCP12 0x004C 139 #define VMCB_EXITCODE_EXCP13 0x004D 140 #define VMCB_EXITCODE_EXCP14 0x004E 141 #define VMCB_EXITCODE_EXCP15 0x004F 142 #define VMCB_EXITCODE_EXCP16 0x0050 143 #define VMCB_EXITCODE_EXCP17 0x0051 144 #define VMCB_EXITCODE_EXCP18 0x0052 145 #define VMCB_EXITCODE_EXCP19 0x0053 146 #define VMCB_EXITCODE_EXCP20 0x0054 147 #define VMCB_EXITCODE_EXCP21 0x0055 148 #define VMCB_EXITCODE_EXCP22 0x0056 149 #define VMCB_EXITCODE_EXCP23 0x0057 150 #define VMCB_EXITCODE_EXCP24 0x0058 151 #define VMCB_EXITCODE_EXCP25 0x0059 152 #define VMCB_EXITCODE_EXCP26 0x005A 153 #define VMCB_EXITCODE_EXCP27 0x005B 154 #define VMCB_EXITCODE_EXCP28 0x005C 155 #define VMCB_EXITCODE_EXCP29 0x005D 156 #define VMCB_EXITCODE_EXCP30 0x005E 157 #define VMCB_EXITCODE_EXCP31 0x005F 158 #define VMCB_EXITCODE_INTR 0x0060 159 #define VMCB_EXITCODE_NMI 0x0061 160 #define VMCB_EXITCODE_SMI 0x0062 161 #define VMCB_EXITCODE_INIT 0x0063 162 #define VMCB_EXITCODE_VINTR 0x0064 163 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 164 #define VMCB_EXITCODE_IDTR_READ 0x0066 165 #define VMCB_EXITCODE_GDTR_READ 0x0067 166 #define VMCB_EXITCODE_LDTR_READ 0x0068 167 #define VMCB_EXITCODE_TR_READ 0x0069 168 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 169 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 170 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 171 #define VMCB_EXITCODE_TR_WRITE 0x006D 172 #define VMCB_EXITCODE_RDTSC 0x006E 173 #define VMCB_EXITCODE_RDPMC 0x006F 174 #define VMCB_EXITCODE_PUSHF 0x0070 175 #define VMCB_EXITCODE_POPF 0x0071 176 #define VMCB_EXITCODE_CPUID 0x0072 177 #define VMCB_EXITCODE_RSM 0x0073 178 #define VMCB_EXITCODE_IRET 0x0074 179 #define VMCB_EXITCODE_SWINT 0x0075 180 #define VMCB_EXITCODE_INVD 0x0076 181 #define VMCB_EXITCODE_PAUSE 0x0077 182 #define VMCB_EXITCODE_HLT 0x0078 183 #define VMCB_EXITCODE_INVLPG 0x0079 184 #define VMCB_EXITCODE_INVLPGA 0x007A 185 #define VMCB_EXITCODE_IOIO 0x007B 186 #define VMCB_EXITCODE_MSR 0x007C 187 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 188 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 189 #define VMCB_EXITCODE_SHUTDOWN 0x007F 190 #define VMCB_EXITCODE_VMRUN 0x0080 191 #define VMCB_EXITCODE_VMMCALL 0x0081 192 #define VMCB_EXITCODE_VMLOAD 0x0082 193 #define VMCB_EXITCODE_VMSAVE 0x0083 194 #define VMCB_EXITCODE_STGI 0x0084 195 #define VMCB_EXITCODE_CLGI 0x0085 196 #define VMCB_EXITCODE_SKINIT 0x0086 197 #define VMCB_EXITCODE_RDTSCP 0x0087 198 #define VMCB_EXITCODE_ICEBP 0x0088 199 #define VMCB_EXITCODE_WBINVD 0x0089 200 #define VMCB_EXITCODE_MONITOR 0x008A 201 #define VMCB_EXITCODE_MWAIT 0x008B 202 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 203 #define VMCB_EXITCODE_XSETBV 0x008D 204 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 205 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 206 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 207 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 208 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 209 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 210 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 211 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 212 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 213 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 214 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 215 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 216 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 217 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 218 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 219 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 220 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 221 #define VMCB_EXITCODE_NPF 0x0400 222 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 223 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 224 #define VMCB_EXITCODE_VMGEXIT 0x0403 225 #define VMCB_EXITCODE_INVALID -1 226 227 /* -------------------------------------------------------------------------- */ 228 229 struct vmcb_ctrl { 230 uint32_t intercept_cr; 231 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 232 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 233 234 uint32_t intercept_dr; 235 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 236 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 237 238 uint32_t intercept_vec; 239 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 240 241 uint32_t intercept_misc1; 242 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 243 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 244 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 245 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 246 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 247 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 248 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 249 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 250 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 251 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 252 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 253 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 254 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 255 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 256 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 257 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 258 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 259 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 260 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 261 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 262 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 263 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 264 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 265 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 266 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 267 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 268 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 269 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 270 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 271 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 272 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 273 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 274 275 uint32_t intercept_misc2; 276 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 277 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 278 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 279 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 280 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 281 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 282 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 283 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 284 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 285 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 286 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 287 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(12) 288 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 289 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 290 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 291 292 uint8_t rsvd1[40]; 293 uint16_t pause_filt_thresh; 294 uint16_t pause_filt_cnt; 295 uint64_t iopm_base_pa; 296 uint64_t msrpm_base_pa; 297 uint64_t tsc_offset; 298 uint32_t guest_asid; 299 300 uint32_t tlb_ctrl; 301 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 302 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 303 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 304 305 uint64_t v; 306 #define VMCB_CTRL_V_TPR __BITS(7,0) 307 #define VMCB_CTRL_V_IRQ __BIT(8) 308 #define VMCB_CTRL_V_VGIF __BIT(9) 309 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 310 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 311 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 312 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 313 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 314 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 315 316 uint64_t intr; 317 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 318 319 uint64_t exitcode; 320 uint64_t exitinfo1; 321 uint64_t exitinfo2; 322 323 uint64_t exitintinfo; 324 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 325 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 326 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 327 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 328 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 329 330 uint64_t enable1; 331 #define VMCB_CTRL_ENABLE_NP __BIT(0) 332 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 333 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 334 335 uint64_t avic; 336 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 337 338 uint64_t ghcb; 339 340 uint64_t eventinj; 341 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 342 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 343 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 344 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 345 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 346 347 uint64_t n_cr3; 348 349 uint64_t enable2; 350 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 351 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 352 353 uint32_t vmcb_clean; 354 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 355 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 356 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 357 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 358 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 359 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 360 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 361 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 362 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 363 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 364 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 365 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 366 367 uint32_t rsvd2; 368 uint64_t nrip; 369 uint8_t inst_len; 370 uint8_t inst_bytes[15]; 371 uint64_t avic_abpp; 372 uint64_t rsvd3; 373 uint64_t avic_ltp; 374 375 uint64_t avic_phys; 376 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 377 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 378 379 uint64_t rsvd4; 380 uint64_t vmcb_ptr; 381 382 uint8_t pad[752]; 383 } __packed; 384 385 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 386 387 struct vmcb_segment { 388 uint16_t selector; 389 uint16_t attrib; /* hidden */ 390 uint32_t limit; /* hidden */ 391 uint64_t base; /* hidden */ 392 } __packed; 393 394 CTASSERT(sizeof(struct vmcb_segment) == 16); 395 396 struct vmcb_state { 397 struct vmcb_segment es; 398 struct vmcb_segment cs; 399 struct vmcb_segment ss; 400 struct vmcb_segment ds; 401 struct vmcb_segment fs; 402 struct vmcb_segment gs; 403 struct vmcb_segment gdt; 404 struct vmcb_segment ldt; 405 struct vmcb_segment idt; 406 struct vmcb_segment tr; 407 uint8_t rsvd1[43]; 408 uint8_t cpl; 409 uint8_t rsvd2[4]; 410 uint64_t efer; 411 uint8_t rsvd3[112]; 412 uint64_t cr4; 413 uint64_t cr3; 414 uint64_t cr0; 415 uint64_t dr7; 416 uint64_t dr6; 417 uint64_t rflags; 418 uint64_t rip; 419 uint8_t rsvd4[88]; 420 uint64_t rsp; 421 uint8_t rsvd5[24]; 422 uint64_t rax; 423 uint64_t star; 424 uint64_t lstar; 425 uint64_t cstar; 426 uint64_t sfmask; 427 uint64_t kernelgsbase; 428 uint64_t sysenter_cs; 429 uint64_t sysenter_esp; 430 uint64_t sysenter_eip; 431 uint64_t cr2; 432 uint8_t rsvd6[32]; 433 uint64_t g_pat; 434 uint64_t dbgctl; 435 uint64_t br_from; 436 uint64_t br_to; 437 uint64_t int_from; 438 uint64_t int_to; 439 uint8_t pad[2408]; 440 } __packed; 441 442 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 443 444 struct vmcb { 445 struct vmcb_ctrl ctrl; 446 struct vmcb_state state; 447 } __packed; 448 449 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 450 CTASSERT(offsetof(struct vmcb, state) == 0x400); 451 452 /* -------------------------------------------------------------------------- */ 453 454 struct svm_hsave { 455 paddr_t pa; 456 }; 457 458 static struct svm_hsave hsave[MAXCPUS]; 459 460 static uint8_t *svm_asidmap __read_mostly; 461 static uint32_t svm_maxasid __read_mostly; 462 static kmutex_t svm_asidlock __cacheline_aligned; 463 464 static bool svm_decode_assist __read_mostly; 465 static uint32_t svm_ctrl_tlb_flush __read_mostly; 466 467 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 468 static uint64_t svm_xcr0_mask __read_mostly; 469 470 #define SVM_NCPUIDS 32 471 472 #define VMCB_NPAGES 1 473 474 #define MSRBM_NPAGES 2 475 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 476 477 #define IOBM_NPAGES 3 478 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 479 480 /* Does not include EFER_LMSLE. */ 481 #define EFER_VALID \ 482 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 483 484 #define EFER_TLB_FLUSH \ 485 (EFER_NXE|EFER_LMA|EFER_LME) 486 #define CR0_TLB_FLUSH \ 487 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 488 #define CR4_TLB_FLUSH \ 489 (CR4_PGE|CR4_PAE|CR4_PSE) 490 491 /* -------------------------------------------------------------------------- */ 492 493 struct svm_machdata { 494 bool cpuidpresent[SVM_NCPUIDS]; 495 struct nvmm_x86_conf_cpuid cpuid[SVM_NCPUIDS]; 496 volatile uint64_t mach_htlb_gen; 497 }; 498 499 static const size_t svm_conf_sizes[NVMM_X86_NCONF] = { 500 [NVMM_X86_CONF_CPUID] = sizeof(struct nvmm_x86_conf_cpuid) 501 }; 502 503 struct svm_cpudata { 504 /* General */ 505 bool shared_asid; 506 bool gtlb_want_flush; 507 uint64_t vcpu_htlb_gen; 508 509 /* VMCB */ 510 struct vmcb *vmcb; 511 paddr_t vmcb_pa; 512 513 /* I/O bitmap */ 514 uint8_t *iobm; 515 paddr_t iobm_pa; 516 517 /* MSR bitmap */ 518 uint8_t *msrbm; 519 paddr_t msrbm_pa; 520 521 /* Host state */ 522 uint64_t hxcr0; 523 uint64_t star; 524 uint64_t lstar; 525 uint64_t cstar; 526 uint64_t sfmask; 527 uint64_t fsbase; 528 uint64_t kernelgsbase; 529 bool ts_set; 530 struct xsave_header hfpu __aligned(64); 531 532 /* Event state */ 533 bool int_window_exit; 534 bool nmi_window_exit; 535 536 /* Guest state */ 537 uint64_t gxcr0; 538 uint64_t gprs[NVMM_X64_NGPR]; 539 uint64_t drs[NVMM_X64_NDR]; 540 uint64_t tsc_offset; 541 struct xsave_header gfpu __aligned(64); 542 }; 543 544 static void 545 svm_vmcb_cache_default(struct vmcb *vmcb) 546 { 547 vmcb->ctrl.vmcb_clean = 548 VMCB_CTRL_VMCB_CLEAN_I | 549 VMCB_CTRL_VMCB_CLEAN_IOPM | 550 VMCB_CTRL_VMCB_CLEAN_ASID | 551 VMCB_CTRL_VMCB_CLEAN_TPR | 552 VMCB_CTRL_VMCB_CLEAN_NP | 553 VMCB_CTRL_VMCB_CLEAN_CR | 554 VMCB_CTRL_VMCB_CLEAN_DR | 555 VMCB_CTRL_VMCB_CLEAN_DT | 556 VMCB_CTRL_VMCB_CLEAN_SEG | 557 VMCB_CTRL_VMCB_CLEAN_CR2 | 558 VMCB_CTRL_VMCB_CLEAN_LBR | 559 VMCB_CTRL_VMCB_CLEAN_AVIC; 560 } 561 562 static void 563 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 564 { 565 if (flags & NVMM_X64_STATE_SEGS) { 566 vmcb->ctrl.vmcb_clean &= 567 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 568 } 569 if (flags & NVMM_X64_STATE_CRS) { 570 vmcb->ctrl.vmcb_clean &= 571 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 572 VMCB_CTRL_VMCB_CLEAN_TPR); 573 } 574 if (flags & NVMM_X64_STATE_DRS) { 575 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 576 } 577 if (flags & NVMM_X64_STATE_MSRS) { 578 /* CR for EFER, NP for PAT. */ 579 vmcb->ctrl.vmcb_clean &= 580 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 581 } 582 } 583 584 static inline void 585 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 586 { 587 vmcb->ctrl.vmcb_clean &= ~flags; 588 } 589 590 static inline void 591 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 592 { 593 vmcb->ctrl.vmcb_clean = 0; 594 } 595 596 #define SVM_EVENT_TYPE_HW_INT 0 597 #define SVM_EVENT_TYPE_NMI 2 598 #define SVM_EVENT_TYPE_EXC 3 599 #define SVM_EVENT_TYPE_SW_INT 4 600 601 static void 602 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 603 { 604 struct svm_cpudata *cpudata = vcpu->cpudata; 605 struct vmcb *vmcb = cpudata->vmcb; 606 607 if (nmi) { 608 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 609 cpudata->nmi_window_exit = true; 610 } else { 611 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 612 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 613 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 614 cpudata->int_window_exit = true; 615 } 616 617 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 618 } 619 620 static void 621 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 622 { 623 struct svm_cpudata *cpudata = vcpu->cpudata; 624 struct vmcb *vmcb = cpudata->vmcb; 625 626 if (nmi) { 627 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 628 cpudata->nmi_window_exit = false; 629 } else { 630 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 631 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 632 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 633 cpudata->int_window_exit = false; 634 } 635 636 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 637 } 638 639 static inline int 640 svm_event_has_error(uint64_t vector) 641 { 642 switch (vector) { 643 case 8: /* #DF */ 644 case 10: /* #TS */ 645 case 11: /* #NP */ 646 case 12: /* #SS */ 647 case 13: /* #GP */ 648 case 14: /* #PF */ 649 case 17: /* #AC */ 650 case 30: /* #SX */ 651 return 1; 652 default: 653 return 0; 654 } 655 } 656 657 static int 658 svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 659 struct nvmm_event *event) 660 { 661 struct svm_cpudata *cpudata = vcpu->cpudata; 662 struct vmcb *vmcb = cpudata->vmcb; 663 int type = 0, err = 0; 664 665 if (event->vector >= 256) { 666 return EINVAL; 667 } 668 669 switch (event->type) { 670 case NVMM_EVENT_INTERRUPT_HW: 671 type = SVM_EVENT_TYPE_HW_INT; 672 if (event->vector == 2) { 673 type = SVM_EVENT_TYPE_NMI; 674 } 675 if (type == SVM_EVENT_TYPE_NMI) { 676 if (cpudata->nmi_window_exit) { 677 return EAGAIN; 678 } 679 svm_event_waitexit_enable(vcpu, true); 680 } else { 681 if (((vmcb->state.rflags & PSL_I) == 0) || 682 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0)) { 683 svm_event_waitexit_enable(vcpu, false); 684 return EAGAIN; 685 } 686 } 687 err = 0; 688 break; 689 case NVMM_EVENT_INTERRUPT_SW: 690 return EINVAL; 691 case NVMM_EVENT_EXCEPTION: 692 type = SVM_EVENT_TYPE_EXC; 693 if (event->vector == 2 || event->vector >= 32) 694 return EINVAL; 695 if (event->vector == 3 || event->vector == 0) 696 return EINVAL; 697 err = svm_event_has_error(event->vector); 698 break; 699 default: 700 return EINVAL; 701 } 702 703 vmcb->ctrl.eventinj = 704 __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) | 705 __SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) | 706 __SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) | 707 __SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) | 708 __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE); 709 710 return 0; 711 } 712 713 static void 714 svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 715 { 716 struct nvmm_event event; 717 int ret __diagused; 718 719 event.type = NVMM_EVENT_EXCEPTION; 720 event.vector = 6; 721 event.u.error = 0; 722 723 ret = svm_vcpu_inject(mach, vcpu, &event); 724 KASSERT(ret == 0); 725 } 726 727 static void 728 svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 729 { 730 struct nvmm_event event; 731 int ret __diagused; 732 733 event.type = NVMM_EVENT_EXCEPTION; 734 event.vector = 13; 735 event.u.error = 0; 736 737 ret = svm_vcpu_inject(mach, vcpu, &event); 738 KASSERT(ret == 0); 739 } 740 741 static inline void 742 svm_inkernel_advance(struct vmcb *vmcb) 743 { 744 /* 745 * Maybe we should also apply single-stepping and debug exceptions. 746 * Matters for guest-ring3, because it can execute 'cpuid' under a 747 * debugger. 748 */ 749 vmcb->state.rip = vmcb->ctrl.nrip; 750 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 751 } 752 753 static void 754 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 755 { 756 struct svm_cpudata *cpudata = vcpu->cpudata; 757 uint64_t cr4; 758 759 switch (eax) { 760 case 0x00000001: 761 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 762 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 763 CPUID_LOCAL_APIC_ID); 764 765 /* CPUID2_OSXSAVE depends on CR4. */ 766 cr4 = cpudata->vmcb->state.cr4; 767 if (!(cr4 & CR4_OSXSAVE)) { 768 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 769 } 770 break; 771 case 0x0000000D: 772 if (svm_xcr0_mask == 0) { 773 break; 774 } 775 switch (ecx) { 776 case 0: 777 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 778 if (cpudata->gxcr0 & XCR0_SSE) { 779 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 780 } else { 781 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 782 } 783 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 784 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave); 785 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 786 break; 787 case 1: 788 cpudata->vmcb->state.rax &= ~CPUID_PES1_XSAVES; 789 break; 790 } 791 break; 792 case 0x40000000: 793 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 794 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 795 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 796 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 797 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 798 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 799 break; 800 case 0x80000001: 801 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID_SVM; 802 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~CPUID_RDTSCP; 803 break; 804 default: 805 break; 806 } 807 } 808 809 static void 810 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 811 struct nvmm_exit *exit) 812 { 813 struct svm_machdata *machdata = mach->machdata; 814 struct svm_cpudata *cpudata = vcpu->cpudata; 815 struct nvmm_x86_conf_cpuid *cpuid; 816 uint64_t eax, ecx; 817 u_int descs[4]; 818 size_t i; 819 820 eax = cpudata->vmcb->state.rax; 821 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 822 x86_cpuid2(eax, ecx, descs); 823 824 cpudata->vmcb->state.rax = descs[0]; 825 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 826 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 827 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 828 829 for (i = 0; i < SVM_NCPUIDS; i++) { 830 cpuid = &machdata->cpuid[i]; 831 if (!machdata->cpuidpresent[i]) { 832 continue; 833 } 834 if (cpuid->leaf != eax) { 835 continue; 836 } 837 838 /* del */ 839 cpudata->vmcb->state.rax &= ~cpuid->del.eax; 840 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->del.ebx; 841 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->del.ecx; 842 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->del.edx; 843 844 /* set */ 845 cpudata->vmcb->state.rax |= cpuid->set.eax; 846 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->set.ebx; 847 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->set.ecx; 848 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->set.edx; 849 850 break; 851 } 852 853 /* Overwrite non-tunable leaves. */ 854 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 855 856 svm_inkernel_advance(cpudata->vmcb); 857 exit->reason = NVMM_EXIT_NONE; 858 } 859 860 static void 861 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 862 struct nvmm_exit *exit) 863 { 864 struct svm_cpudata *cpudata = vcpu->cpudata; 865 struct vmcb *vmcb = cpudata->vmcb; 866 867 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 868 svm_event_waitexit_disable(vcpu, false); 869 } 870 871 svm_inkernel_advance(cpudata->vmcb); 872 exit->reason = NVMM_EXIT_HALTED; 873 } 874 875 #define SVM_EXIT_IO_PORT __BITS(31,16) 876 #define SVM_EXIT_IO_SEG __BITS(12,10) 877 #define SVM_EXIT_IO_A64 __BIT(9) 878 #define SVM_EXIT_IO_A32 __BIT(8) 879 #define SVM_EXIT_IO_A16 __BIT(7) 880 #define SVM_EXIT_IO_SZ32 __BIT(6) 881 #define SVM_EXIT_IO_SZ16 __BIT(5) 882 #define SVM_EXIT_IO_SZ8 __BIT(4) 883 #define SVM_EXIT_IO_REP __BIT(3) 884 #define SVM_EXIT_IO_STR __BIT(2) 885 #define SVM_EXIT_IO_IN __BIT(0) 886 887 static const int seg_to_nvmm[] = { 888 [0] = NVMM_X64_SEG_ES, 889 [1] = NVMM_X64_SEG_CS, 890 [2] = NVMM_X64_SEG_SS, 891 [3] = NVMM_X64_SEG_DS, 892 [4] = NVMM_X64_SEG_FS, 893 [5] = NVMM_X64_SEG_GS 894 }; 895 896 static void 897 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 898 struct nvmm_exit *exit) 899 { 900 struct svm_cpudata *cpudata = vcpu->cpudata; 901 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 902 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 903 904 exit->reason = NVMM_EXIT_IO; 905 906 if (info & SVM_EXIT_IO_IN) { 907 exit->u.io.type = NVMM_EXIT_IO_IN; 908 } else { 909 exit->u.io.type = NVMM_EXIT_IO_OUT; 910 } 911 912 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 913 914 if (svm_decode_assist) { 915 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 916 exit->u.io.seg = seg_to_nvmm[__SHIFTOUT(info, SVM_EXIT_IO_SEG)]; 917 } else { 918 exit->u.io.seg = -1; 919 } 920 921 if (info & SVM_EXIT_IO_A64) { 922 exit->u.io.address_size = 8; 923 } else if (info & SVM_EXIT_IO_A32) { 924 exit->u.io.address_size = 4; 925 } else if (info & SVM_EXIT_IO_A16) { 926 exit->u.io.address_size = 2; 927 } 928 929 if (info & SVM_EXIT_IO_SZ32) { 930 exit->u.io.operand_size = 4; 931 } else if (info & SVM_EXIT_IO_SZ16) { 932 exit->u.io.operand_size = 2; 933 } else if (info & SVM_EXIT_IO_SZ8) { 934 exit->u.io.operand_size = 1; 935 } 936 937 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 938 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 939 exit->u.io.npc = nextpc; 940 } 941 942 static const uint64_t msr_ignore_list[] = { 943 0xc0010055, /* MSR_CMPHALT */ 944 MSR_DE_CFG, 945 MSR_IC_CFG, 946 MSR_UCODE_AMD_PATCHLEVEL 947 }; 948 949 static bool 950 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 951 struct nvmm_exit *exit) 952 { 953 struct svm_cpudata *cpudata = vcpu->cpudata; 954 struct vmcb *vmcb = cpudata->vmcb; 955 uint64_t val; 956 size_t i; 957 958 switch (exit->u.msr.type) { 959 case NVMM_EXIT_MSR_RDMSR: 960 if (exit->u.msr.msr == MSR_NB_CFG) { 961 val = NB_CFG_INITAPICCPUIDLO; 962 vmcb->state.rax = (val & 0xFFFFFFFF); 963 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 964 goto handled; 965 } 966 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 967 if (msr_ignore_list[i] != exit->u.msr.msr) 968 continue; 969 val = 0; 970 vmcb->state.rax = (val & 0xFFFFFFFF); 971 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 972 goto handled; 973 } 974 break; 975 case NVMM_EXIT_MSR_WRMSR: 976 if (exit->u.msr.msr == MSR_EFER) { 977 if (__predict_false(exit->u.msr.val & ~EFER_VALID)) { 978 goto error; 979 } 980 if ((vmcb->state.efer ^ exit->u.msr.val) & 981 EFER_TLB_FLUSH) { 982 cpudata->gtlb_want_flush = true; 983 } 984 vmcb->state.efer = exit->u.msr.val | EFER_SVME; 985 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 986 goto handled; 987 } 988 if (exit->u.msr.msr == MSR_TSC) { 989 cpudata->tsc_offset = exit->u.msr.val - cpu_counter(); 990 vmcb->ctrl.tsc_offset = cpudata->tsc_offset + 991 curcpu()->ci_data.cpu_cc_skew; 992 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 993 goto handled; 994 } 995 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 996 if (msr_ignore_list[i] != exit->u.msr.msr) 997 continue; 998 goto handled; 999 } 1000 break; 1001 } 1002 1003 return false; 1004 1005 handled: 1006 svm_inkernel_advance(cpudata->vmcb); 1007 return true; 1008 1009 error: 1010 svm_inject_gp(mach, vcpu); 1011 return true; 1012 } 1013 1014 static void 1015 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1016 struct nvmm_exit *exit) 1017 { 1018 struct svm_cpudata *cpudata = vcpu->cpudata; 1019 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1020 1021 if (info == 0) { 1022 exit->u.msr.type = NVMM_EXIT_MSR_RDMSR; 1023 } else { 1024 exit->u.msr.type = NVMM_EXIT_MSR_WRMSR; 1025 } 1026 1027 exit->u.msr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1028 1029 if (info == 1) { 1030 uint64_t rdx, rax; 1031 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1032 rax = cpudata->vmcb->state.rax; 1033 exit->u.msr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1034 } else { 1035 exit->u.msr.val = 0; 1036 } 1037 1038 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1039 exit->reason = NVMM_EXIT_NONE; 1040 return; 1041 } 1042 1043 exit->reason = NVMM_EXIT_MSR; 1044 exit->u.msr.npc = cpudata->vmcb->ctrl.nrip; 1045 } 1046 1047 static void 1048 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1049 struct nvmm_exit *exit) 1050 { 1051 struct svm_cpudata *cpudata = vcpu->cpudata; 1052 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1053 1054 exit->reason = NVMM_EXIT_MEMORY; 1055 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1056 exit->u.mem.perm = NVMM_EXIT_MEMORY_WRITE; 1057 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1058 exit->u.mem.perm = NVMM_EXIT_MEMORY_EXEC; 1059 else 1060 exit->u.mem.perm = NVMM_EXIT_MEMORY_READ; 1061 exit->u.mem.gpa = gpa; 1062 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1063 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1064 sizeof(exit->u.mem.inst_bytes)); 1065 } 1066 1067 static void 1068 svm_exit_insn(struct vmcb *vmcb, struct nvmm_exit *exit, uint64_t reason) 1069 { 1070 exit->u.insn.npc = vmcb->ctrl.nrip; 1071 exit->reason = reason; 1072 } 1073 1074 static void 1075 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1076 struct nvmm_exit *exit) 1077 { 1078 struct svm_cpudata *cpudata = vcpu->cpudata; 1079 struct vmcb *vmcb = cpudata->vmcb; 1080 uint64_t val; 1081 1082 exit->reason = NVMM_EXIT_NONE; 1083 1084 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1085 (vmcb->state.rax & 0xFFFFFFFF); 1086 1087 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1088 goto error; 1089 } else if (__predict_false(vmcb->state.cpl != 0)) { 1090 goto error; 1091 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1092 goto error; 1093 } else if (__predict_false((val & XCR0_X87) == 0)) { 1094 goto error; 1095 } 1096 1097 cpudata->gxcr0 = val; 1098 1099 svm_inkernel_advance(cpudata->vmcb); 1100 return; 1101 1102 error: 1103 svm_inject_gp(mach, vcpu); 1104 } 1105 1106 /* -------------------------------------------------------------------------- */ 1107 1108 static void 1109 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1110 { 1111 struct svm_cpudata *cpudata = vcpu->cpudata; 1112 1113 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1114 1115 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1116 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1117 1118 if (svm_xcr0_mask != 0) { 1119 cpudata->hxcr0 = rdxcr(0); 1120 wrxcr(0, cpudata->gxcr0); 1121 } 1122 } 1123 1124 static void 1125 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1126 { 1127 struct svm_cpudata *cpudata = vcpu->cpudata; 1128 1129 if (svm_xcr0_mask != 0) { 1130 cpudata->gxcr0 = rdxcr(0); 1131 wrxcr(0, cpudata->hxcr0); 1132 } 1133 1134 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1135 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1136 1137 if (cpudata->ts_set) { 1138 stts(); 1139 } 1140 } 1141 1142 static void 1143 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1144 { 1145 struct svm_cpudata *cpudata = vcpu->cpudata; 1146 1147 x86_dbregs_save(curlwp); 1148 1149 ldr7(0); 1150 1151 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1152 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1153 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1154 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1155 } 1156 1157 static void 1158 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1159 { 1160 struct svm_cpudata *cpudata = vcpu->cpudata; 1161 1162 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1163 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1164 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1165 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1166 1167 x86_dbregs_restore(curlwp); 1168 } 1169 1170 static void 1171 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1172 { 1173 struct svm_cpudata *cpudata = vcpu->cpudata; 1174 1175 cpudata->fsbase = rdmsr(MSR_FSBASE); 1176 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1177 } 1178 1179 static void 1180 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1181 { 1182 struct svm_cpudata *cpudata = vcpu->cpudata; 1183 1184 wrmsr(MSR_STAR, cpudata->star); 1185 wrmsr(MSR_LSTAR, cpudata->lstar); 1186 wrmsr(MSR_CSTAR, cpudata->cstar); 1187 wrmsr(MSR_SFMASK, cpudata->sfmask); 1188 wrmsr(MSR_FSBASE, cpudata->fsbase); 1189 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1190 } 1191 1192 /* -------------------------------------------------------------------------- */ 1193 1194 static inline void 1195 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1196 { 1197 struct svm_cpudata *cpudata = vcpu->cpudata; 1198 1199 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1200 cpudata->gtlb_want_flush = true; 1201 } 1202 } 1203 1204 static inline void 1205 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1206 { 1207 /* 1208 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1209 * executing on this hCPU and the hTLB already got flushed, or it 1210 * was executing on another hCPU in which case the catchup is done 1211 * in svm_gtlb_catchup(). 1212 */ 1213 } 1214 1215 static inline uint64_t 1216 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1217 { 1218 struct vmcb *vmcb = cpudata->vmcb; 1219 uint64_t machgen; 1220 1221 machgen = machdata->mach_htlb_gen; 1222 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1223 return machgen; 1224 } 1225 1226 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1227 return machgen; 1228 } 1229 1230 static inline void 1231 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1232 { 1233 struct vmcb *vmcb = cpudata->vmcb; 1234 1235 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1236 cpudata->vcpu_htlb_gen = machgen; 1237 } 1238 } 1239 1240 static int 1241 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1242 struct nvmm_exit *exit) 1243 { 1244 struct svm_machdata *machdata = mach->machdata; 1245 struct svm_cpudata *cpudata = vcpu->cpudata; 1246 struct vmcb *vmcb = cpudata->vmcb; 1247 uint64_t machgen; 1248 int hcpu, s; 1249 1250 kpreempt_disable(); 1251 hcpu = cpu_number(); 1252 1253 svm_gtlb_catchup(vcpu, hcpu); 1254 svm_htlb_catchup(vcpu, hcpu); 1255 1256 if (vcpu->hcpu_last != hcpu) { 1257 vmcb->ctrl.tsc_offset = cpudata->tsc_offset + 1258 curcpu()->ci_data.cpu_cc_skew; 1259 svm_vmcb_cache_flush_all(vmcb); 1260 } 1261 1262 svm_vcpu_guest_dbregs_enter(vcpu); 1263 svm_vcpu_guest_misc_enter(vcpu); 1264 1265 while (1) { 1266 if (cpudata->gtlb_want_flush) { 1267 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1268 } else { 1269 vmcb->ctrl.tlb_ctrl = 0; 1270 } 1271 1272 s = splhigh(); 1273 machgen = svm_htlb_flush(machdata, cpudata); 1274 svm_vcpu_guest_fpu_enter(vcpu); 1275 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1276 svm_vcpu_guest_fpu_leave(vcpu); 1277 svm_htlb_flush_ack(cpudata, machgen); 1278 splx(s); 1279 1280 svm_vmcb_cache_default(vmcb); 1281 1282 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1283 cpudata->gtlb_want_flush = false; 1284 vcpu->hcpu_last = hcpu; 1285 } 1286 1287 switch (vmcb->ctrl.exitcode) { 1288 case VMCB_EXITCODE_INTR: 1289 case VMCB_EXITCODE_NMI: 1290 exit->reason = NVMM_EXIT_NONE; 1291 break; 1292 case VMCB_EXITCODE_VINTR: 1293 svm_event_waitexit_disable(vcpu, false); 1294 exit->reason = NVMM_EXIT_INT_READY; 1295 break; 1296 case VMCB_EXITCODE_IRET: 1297 svm_event_waitexit_disable(vcpu, true); 1298 exit->reason = NVMM_EXIT_NMI_READY; 1299 break; 1300 case VMCB_EXITCODE_CPUID: 1301 svm_exit_cpuid(mach, vcpu, exit); 1302 break; 1303 case VMCB_EXITCODE_HLT: 1304 svm_exit_hlt(mach, vcpu, exit); 1305 break; 1306 case VMCB_EXITCODE_IOIO: 1307 svm_exit_io(mach, vcpu, exit); 1308 break; 1309 case VMCB_EXITCODE_MSR: 1310 svm_exit_msr(mach, vcpu, exit); 1311 break; 1312 case VMCB_EXITCODE_SHUTDOWN: 1313 exit->reason = NVMM_EXIT_SHUTDOWN; 1314 break; 1315 case VMCB_EXITCODE_RDPMC: 1316 case VMCB_EXITCODE_RSM: 1317 case VMCB_EXITCODE_INVLPGA: 1318 case VMCB_EXITCODE_VMRUN: 1319 case VMCB_EXITCODE_VMMCALL: 1320 case VMCB_EXITCODE_VMLOAD: 1321 case VMCB_EXITCODE_VMSAVE: 1322 case VMCB_EXITCODE_STGI: 1323 case VMCB_EXITCODE_CLGI: 1324 case VMCB_EXITCODE_SKINIT: 1325 case VMCB_EXITCODE_RDTSCP: 1326 svm_inject_ud(mach, vcpu); 1327 exit->reason = NVMM_EXIT_NONE; 1328 break; 1329 case VMCB_EXITCODE_MONITOR: 1330 svm_exit_insn(vmcb, exit, NVMM_EXIT_MONITOR); 1331 break; 1332 case VMCB_EXITCODE_MWAIT: 1333 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT); 1334 break; 1335 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1336 svm_exit_insn(vmcb, exit, NVMM_EXIT_MWAIT_COND); 1337 break; 1338 case VMCB_EXITCODE_XSETBV: 1339 svm_exit_xsetbv(mach, vcpu, exit); 1340 break; 1341 case VMCB_EXITCODE_NPF: 1342 svm_exit_npf(mach, vcpu, exit); 1343 break; 1344 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1345 default: 1346 exit->reason = NVMM_EXIT_INVALID; 1347 break; 1348 } 1349 1350 /* If no reason to return to userland, keep rolling. */ 1351 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1352 break; 1353 } 1354 if (curcpu()->ci_data.cpu_softints != 0) { 1355 break; 1356 } 1357 if (curlwp->l_flag & LW_USERRET) { 1358 break; 1359 } 1360 if (exit->reason != NVMM_EXIT_NONE) { 1361 break; 1362 } 1363 } 1364 1365 svm_vcpu_guest_misc_leave(vcpu); 1366 svm_vcpu_guest_dbregs_leave(vcpu); 1367 1368 kpreempt_enable(); 1369 1370 exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1371 VMCB_CTRL_V_TPR); 1372 exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags; 1373 1374 exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] = 1375 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1376 exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] = 1377 cpudata->int_window_exit; 1378 exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] = 1379 cpudata->nmi_window_exit; 1380 1381 return 0; 1382 } 1383 1384 /* -------------------------------------------------------------------------- */ 1385 1386 static int 1387 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1388 { 1389 struct pglist pglist; 1390 paddr_t _pa; 1391 vaddr_t _va; 1392 size_t i; 1393 int ret; 1394 1395 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1396 &pglist, 1, 0); 1397 if (ret != 0) 1398 return ENOMEM; 1399 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1400 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1401 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1402 if (_va == 0) 1403 goto error; 1404 1405 for (i = 0; i < npages; i++) { 1406 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1407 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1408 } 1409 pmap_update(pmap_kernel()); 1410 1411 memset((void *)_va, 0, npages * PAGE_SIZE); 1412 1413 *pa = _pa; 1414 *va = _va; 1415 return 0; 1416 1417 error: 1418 for (i = 0; i < npages; i++) { 1419 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1420 } 1421 return ENOMEM; 1422 } 1423 1424 static void 1425 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1426 { 1427 size_t i; 1428 1429 pmap_kremove(va, npages * PAGE_SIZE); 1430 pmap_update(pmap_kernel()); 1431 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1432 for (i = 0; i < npages; i++) { 1433 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1434 } 1435 } 1436 1437 /* -------------------------------------------------------------------------- */ 1438 1439 #define SVM_MSRBM_READ __BIT(0) 1440 #define SVM_MSRBM_WRITE __BIT(1) 1441 1442 static void 1443 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1444 { 1445 uint64_t byte; 1446 uint8_t bitoff; 1447 1448 if (msr < 0x00002000) { 1449 /* Range 1 */ 1450 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1451 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1452 /* Range 2 */ 1453 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1454 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1455 /* Range 3 */ 1456 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1457 } else { 1458 panic("%s: wrong range", __func__); 1459 } 1460 1461 bitoff = (msr & 0x3) << 1; 1462 1463 if (read) { 1464 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1465 } 1466 if (write) { 1467 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1468 } 1469 } 1470 1471 static void 1472 svm_asid_alloc(struct nvmm_cpu *vcpu) 1473 { 1474 struct svm_cpudata *cpudata = vcpu->cpudata; 1475 struct vmcb *vmcb = cpudata->vmcb; 1476 size_t i, oct, bit; 1477 1478 mutex_enter(&svm_asidlock); 1479 1480 for (i = 0; i < svm_maxasid; i++) { 1481 oct = i / 8; 1482 bit = i % 8; 1483 1484 if (svm_asidmap[oct] & __BIT(bit)) { 1485 continue; 1486 } 1487 1488 svm_asidmap[oct] |= __BIT(bit); 1489 vmcb->ctrl.guest_asid = i; 1490 mutex_exit(&svm_asidlock); 1491 return; 1492 } 1493 1494 /* 1495 * No free ASID. Use the last one, which is shared and requires 1496 * special TLB handling. 1497 */ 1498 cpudata->shared_asid = true; 1499 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1500 mutex_exit(&svm_asidlock); 1501 } 1502 1503 static void 1504 svm_asid_free(struct nvmm_cpu *vcpu) 1505 { 1506 struct svm_cpudata *cpudata = vcpu->cpudata; 1507 struct vmcb *vmcb = cpudata->vmcb; 1508 size_t oct, bit; 1509 1510 if (cpudata->shared_asid) { 1511 return; 1512 } 1513 1514 oct = vmcb->ctrl.guest_asid / 8; 1515 bit = vmcb->ctrl.guest_asid % 8; 1516 1517 mutex_enter(&svm_asidlock); 1518 svm_asidmap[oct] &= ~__BIT(bit); 1519 mutex_exit(&svm_asidlock); 1520 } 1521 1522 static void 1523 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1524 { 1525 struct svm_cpudata *cpudata = vcpu->cpudata; 1526 struct vmcb *vmcb = cpudata->vmcb; 1527 1528 /* Allow reads/writes of Control Registers. */ 1529 vmcb->ctrl.intercept_cr = 0; 1530 1531 /* Allow reads/writes of Debug Registers. */ 1532 vmcb->ctrl.intercept_dr = 0; 1533 1534 /* Allow exceptions 0 to 31. */ 1535 vmcb->ctrl.intercept_vec = 0; 1536 1537 /* 1538 * Allow: 1539 * - SMI [smm interrupts] 1540 * - VINTR [virtual interrupts] 1541 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1542 * - RIDTR [reads of IDTR] 1543 * - RGDTR [reads of GDTR] 1544 * - RLDTR [reads of LDTR] 1545 * - RTR [reads of TR] 1546 * - WIDTR [writes of IDTR] 1547 * - WGDTR [writes of GDTR] 1548 * - WLDTR [writes of LDTR] 1549 * - WTR [writes of TR] 1550 * - RDTSC [rdtsc instruction] 1551 * - PUSHF [pushf instruction] 1552 * - POPF [popf instruction] 1553 * - IRET [iret instruction] 1554 * - INTN [int $n instructions] 1555 * - INVD [invd instruction] 1556 * - PAUSE [pause instruction] 1557 * - INVLPG [invplg instruction] 1558 * - TASKSW [task switches] 1559 * 1560 * Intercept the rest below. 1561 */ 1562 vmcb->ctrl.intercept_misc1 = 1563 VMCB_CTRL_INTERCEPT_INTR | 1564 VMCB_CTRL_INTERCEPT_NMI | 1565 VMCB_CTRL_INTERCEPT_INIT | 1566 VMCB_CTRL_INTERCEPT_RDPMC | 1567 VMCB_CTRL_INTERCEPT_CPUID | 1568 VMCB_CTRL_INTERCEPT_RSM | 1569 VMCB_CTRL_INTERCEPT_HLT | 1570 VMCB_CTRL_INTERCEPT_INVLPGA | 1571 VMCB_CTRL_INTERCEPT_IOIO_PROT | 1572 VMCB_CTRL_INTERCEPT_MSR_PROT | 1573 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 1574 VMCB_CTRL_INTERCEPT_SHUTDOWN; 1575 1576 /* 1577 * Allow: 1578 * - ICEBP [icebp instruction] 1579 * - WBINVD [wbinvd instruction] 1580 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 1581 * 1582 * Intercept the rest below. 1583 */ 1584 vmcb->ctrl.intercept_misc2 = 1585 VMCB_CTRL_INTERCEPT_VMRUN | 1586 VMCB_CTRL_INTERCEPT_VMMCALL | 1587 VMCB_CTRL_INTERCEPT_VMLOAD | 1588 VMCB_CTRL_INTERCEPT_VMSAVE | 1589 VMCB_CTRL_INTERCEPT_STGI | 1590 VMCB_CTRL_INTERCEPT_CLGI | 1591 VMCB_CTRL_INTERCEPT_SKINIT | 1592 VMCB_CTRL_INTERCEPT_RDTSCP | 1593 VMCB_CTRL_INTERCEPT_MONITOR | 1594 VMCB_CTRL_INTERCEPT_MWAIT | 1595 VMCB_CTRL_INTERCEPT_XSETBV; 1596 1597 /* Intercept all I/O accesses. */ 1598 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 1599 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 1600 1601 /* 1602 * Allow: 1603 * - EFER [read] 1604 * - STAR [read, write] 1605 * - LSTAR [read, write] 1606 * - CSTAR [read, write] 1607 * - SFMASK [read, write] 1608 * - KERNELGSBASE [read, write] 1609 * - SYSENTER_CS [read, write] 1610 * - SYSENTER_ESP [read, write] 1611 * - SYSENTER_EIP [read, write] 1612 * - FSBASE [read, write] 1613 * - GSBASE [read, write] 1614 * - PAT [read, write] 1615 * - TSC [read] 1616 * 1617 * Intercept the rest. 1618 */ 1619 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 1620 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 1621 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 1622 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 1623 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 1624 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 1625 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 1626 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 1627 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 1628 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 1629 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 1630 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 1631 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 1632 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 1633 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 1634 1635 /* Generate ASID. */ 1636 svm_asid_alloc(vcpu); 1637 1638 /* Virtual TPR. */ 1639 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 1640 1641 /* Enable Nested Paging. */ 1642 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 1643 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 1644 1645 /* Must always be set. */ 1646 vmcb->state.efer = EFER_SVME; 1647 cpudata->gxcr0 = XCR0_X87; 1648 1649 /* Init XSAVE header. */ 1650 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1651 cpudata->gfpu.xsh_xcomp_bv = 0; 1652 1653 /* Set guest TSC to zero, more or less. */ 1654 cpudata->tsc_offset = -cpu_counter(); 1655 1656 /* These MSRs are static. */ 1657 cpudata->star = rdmsr(MSR_STAR); 1658 cpudata->lstar = rdmsr(MSR_LSTAR); 1659 cpudata->cstar = rdmsr(MSR_CSTAR); 1660 cpudata->sfmask = rdmsr(MSR_SFMASK); 1661 } 1662 1663 static int 1664 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1665 { 1666 struct svm_cpudata *cpudata; 1667 int error; 1668 1669 /* Allocate the SVM cpudata. */ 1670 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 1671 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 1672 UVM_KMF_WIRED|UVM_KMF_ZERO); 1673 vcpu->cpudata = cpudata; 1674 1675 /* VMCB */ 1676 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 1677 VMCB_NPAGES); 1678 if (error) 1679 goto error; 1680 1681 /* I/O Bitmap */ 1682 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 1683 IOBM_NPAGES); 1684 if (error) 1685 goto error; 1686 1687 /* MSR Bitmap */ 1688 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 1689 MSRBM_NPAGES); 1690 if (error) 1691 goto error; 1692 1693 /* Init the VCPU info. */ 1694 svm_vcpu_init(mach, vcpu); 1695 1696 return 0; 1697 1698 error: 1699 if (cpudata->vmcb_pa) { 1700 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 1701 VMCB_NPAGES); 1702 } 1703 if (cpudata->iobm_pa) { 1704 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 1705 IOBM_NPAGES); 1706 } 1707 if (cpudata->msrbm_pa) { 1708 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 1709 MSRBM_NPAGES); 1710 } 1711 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1712 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1713 return error; 1714 } 1715 1716 static void 1717 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1718 { 1719 struct svm_cpudata *cpudata = vcpu->cpudata; 1720 1721 svm_asid_free(vcpu); 1722 1723 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 1724 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 1725 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 1726 1727 uvm_km_free(kernel_map, (vaddr_t)cpudata, 1728 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 1729 } 1730 1731 #define SVM_SEG_ATTRIB_TYPE __BITS(4,0) 1732 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1733 #define SVM_SEG_ATTRIB_P __BIT(7) 1734 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1735 #define SVM_SEG_ATTRIB_LONG __BIT(9) 1736 #define SVM_SEG_ATTRIB_DEF32 __BIT(10) 1737 #define SVM_SEG_ATTRIB_GRAN __BIT(11) 1738 1739 static void 1740 svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1741 { 1742 vseg->selector = seg->selector; 1743 vseg->attrib = 1744 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1745 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1746 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1747 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1748 __SHIFTIN(seg->attrib.lng, SVM_SEG_ATTRIB_LONG) | 1749 __SHIFTIN(seg->attrib.def32, SVM_SEG_ATTRIB_DEF32) | 1750 __SHIFTIN(seg->attrib.gran, SVM_SEG_ATTRIB_GRAN); 1751 vseg->limit = seg->limit; 1752 vseg->base = seg->base; 1753 } 1754 1755 static void 1756 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1757 { 1758 seg->selector = vseg->selector; 1759 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1760 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1761 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1762 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1763 seg->attrib.lng = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_LONG); 1764 seg->attrib.def32 = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF32); 1765 seg->attrib.gran = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_GRAN); 1766 seg->limit = vseg->limit; 1767 seg->base = vseg->base; 1768 } 1769 1770 static inline bool 1771 svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state, 1772 uint64_t flags) 1773 { 1774 if (flags & NVMM_X64_STATE_CRS) { 1775 if ((vmcb->state.cr0 ^ 1776 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1777 return true; 1778 } 1779 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1780 return true; 1781 } 1782 if ((vmcb->state.cr4 ^ 1783 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1784 return true; 1785 } 1786 } 1787 1788 if (flags & NVMM_X64_STATE_MSRS) { 1789 if ((vmcb->state.efer ^ 1790 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1791 return true; 1792 } 1793 } 1794 1795 return false; 1796 } 1797 1798 static void 1799 svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1800 { 1801 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1802 struct svm_cpudata *cpudata = vcpu->cpudata; 1803 struct vmcb *vmcb = cpudata->vmcb; 1804 struct fxsave *fpustate; 1805 1806 if (svm_state_tlb_flush(vmcb, state, flags)) { 1807 cpudata->gtlb_want_flush = true; 1808 } 1809 1810 if (flags & NVMM_X64_STATE_SEGS) { 1811 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1812 &vmcb->state.cs); 1813 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1814 &vmcb->state.ds); 1815 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1816 &vmcb->state.es); 1817 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1818 &vmcb->state.fs); 1819 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1820 &vmcb->state.gs); 1821 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1822 &vmcb->state.ss); 1823 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1824 &vmcb->state.gdt); 1825 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1826 &vmcb->state.idt); 1827 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1828 &vmcb->state.ldt); 1829 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1830 &vmcb->state.tr); 1831 1832 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1833 } 1834 1835 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1836 if (flags & NVMM_X64_STATE_GPRS) { 1837 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1838 1839 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1840 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1841 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1842 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1843 } 1844 1845 if (flags & NVMM_X64_STATE_CRS) { 1846 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1847 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1848 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1849 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1850 1851 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1852 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1853 VMCB_CTRL_V_TPR); 1854 1855 if (svm_xcr0_mask != 0) { 1856 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1857 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1858 cpudata->gxcr0 &= svm_xcr0_mask; 1859 cpudata->gxcr0 |= XCR0_X87; 1860 } 1861 } 1862 1863 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1864 if (flags & NVMM_X64_STATE_DRS) { 1865 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1866 1867 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1868 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1869 } 1870 1871 if (flags & NVMM_X64_STATE_MSRS) { 1872 /* Bit EFER_SVME is mandatory. */ 1873 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1874 1875 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1876 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1877 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1878 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1879 vmcb->state.kernelgsbase = 1880 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1881 vmcb->state.sysenter_cs = 1882 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1883 vmcb->state.sysenter_esp = 1884 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1885 vmcb->state.sysenter_eip = 1886 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1887 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1888 } 1889 1890 if (flags & NVMM_X64_STATE_MISC) { 1891 if (state->misc[NVMM_X64_MISC_INT_SHADOW]) { 1892 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1893 } else { 1894 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1895 } 1896 1897 if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) { 1898 svm_event_waitexit_enable(vcpu, false); 1899 } else { 1900 svm_event_waitexit_disable(vcpu, false); 1901 } 1902 1903 if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) { 1904 svm_event_waitexit_enable(vcpu, true); 1905 } else { 1906 svm_event_waitexit_disable(vcpu, true); 1907 } 1908 } 1909 1910 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1911 if (flags & NVMM_X64_STATE_FPU) { 1912 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1913 sizeof(state->fpu)); 1914 1915 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1916 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1917 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1918 1919 if (svm_xcr0_mask != 0) { 1920 /* Reset XSTATE_BV, to force a reload. */ 1921 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1922 } 1923 } 1924 1925 svm_vmcb_cache_update(vmcb, flags); 1926 } 1927 1928 static void 1929 svm_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags) 1930 { 1931 struct nvmm_x64_state *state = (struct nvmm_x64_state *)data; 1932 struct svm_cpudata *cpudata = vcpu->cpudata; 1933 struct vmcb *vmcb = cpudata->vmcb; 1934 1935 if (flags & NVMM_X64_STATE_SEGS) { 1936 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1937 &vmcb->state.cs); 1938 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1939 &vmcb->state.ds); 1940 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1941 &vmcb->state.es); 1942 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1943 &vmcb->state.fs); 1944 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1945 &vmcb->state.gs); 1946 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1947 &vmcb->state.ss); 1948 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1949 &vmcb->state.gdt); 1950 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1951 &vmcb->state.idt); 1952 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1953 &vmcb->state.ldt); 1954 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1955 &vmcb->state.tr); 1956 1957 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1958 } 1959 1960 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1961 if (flags & NVMM_X64_STATE_GPRS) { 1962 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1963 1964 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1965 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1966 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1967 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1968 } 1969 1970 if (flags & NVMM_X64_STATE_CRS) { 1971 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1972 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1973 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1974 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1975 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1976 VMCB_CTRL_V_TPR); 1977 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1978 } 1979 1980 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1981 if (flags & NVMM_X64_STATE_DRS) { 1982 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1983 1984 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1985 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1986 } 1987 1988 if (flags & NVMM_X64_STATE_MSRS) { 1989 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1990 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1991 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1992 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1993 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1994 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1995 vmcb->state.kernelgsbase; 1996 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1997 vmcb->state.sysenter_cs; 1998 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1999 vmcb->state.sysenter_esp; 2000 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 2001 vmcb->state.sysenter_eip; 2002 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 2003 2004 /* Hide SVME. */ 2005 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 2006 } 2007 2008 if (flags & NVMM_X64_STATE_MISC) { 2009 state->misc[NVMM_X64_MISC_INT_SHADOW] = 2010 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 2011 state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] = 2012 cpudata->int_window_exit; 2013 state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] = 2014 cpudata->nmi_window_exit; 2015 } 2016 2017 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 2018 if (flags & NVMM_X64_STATE_FPU) { 2019 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 2020 sizeof(state->fpu)); 2021 } 2022 } 2023 2024 /* -------------------------------------------------------------------------- */ 2025 2026 static void 2027 svm_tlb_flush(struct pmap *pm) 2028 { 2029 struct nvmm_machine *mach = pm->pm_data; 2030 struct svm_machdata *machdata = mach->machdata; 2031 2032 atomic_inc_64(&machdata->mach_htlb_gen); 2033 2034 /* Generates IPIs, which cause #VMEXITs. */ 2035 pmap_tlb_shootdown(pmap_kernel(), -1, PG_G, TLBSHOOT_UPDATE); 2036 } 2037 2038 static void 2039 svm_machine_create(struct nvmm_machine *mach) 2040 { 2041 struct svm_machdata *machdata; 2042 2043 /* Fill in pmap info. */ 2044 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2045 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2046 2047 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2048 mach->machdata = machdata; 2049 2050 /* Start with an hTLB flush everywhere. */ 2051 machdata->mach_htlb_gen = 1; 2052 } 2053 2054 static void 2055 svm_machine_destroy(struct nvmm_machine *mach) 2056 { 2057 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2058 } 2059 2060 static int 2061 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2062 { 2063 struct nvmm_x86_conf_cpuid *cpuid = data; 2064 struct svm_machdata *machdata = (struct svm_machdata *)mach->machdata; 2065 size_t i; 2066 2067 if (__predict_false(op != NVMM_X86_CONF_CPUID)) { 2068 return EINVAL; 2069 } 2070 2071 if (__predict_false((cpuid->set.eax & cpuid->del.eax) || 2072 (cpuid->set.ebx & cpuid->del.ebx) || 2073 (cpuid->set.ecx & cpuid->del.ecx) || 2074 (cpuid->set.edx & cpuid->del.edx))) { 2075 return EINVAL; 2076 } 2077 2078 /* If already here, replace. */ 2079 for (i = 0; i < SVM_NCPUIDS; i++) { 2080 if (!machdata->cpuidpresent[i]) { 2081 continue; 2082 } 2083 if (machdata->cpuid[i].leaf == cpuid->leaf) { 2084 memcpy(&machdata->cpuid[i], cpuid, 2085 sizeof(struct nvmm_x86_conf_cpuid)); 2086 return 0; 2087 } 2088 } 2089 2090 /* Not here, insert. */ 2091 for (i = 0; i < SVM_NCPUIDS; i++) { 2092 if (!machdata->cpuidpresent[i]) { 2093 machdata->cpuidpresent[i] = true; 2094 memcpy(&machdata->cpuid[i], cpuid, 2095 sizeof(struct nvmm_x86_conf_cpuid)); 2096 return 0; 2097 } 2098 } 2099 2100 return ENOBUFS; 2101 } 2102 2103 /* -------------------------------------------------------------------------- */ 2104 2105 static bool 2106 svm_ident(void) 2107 { 2108 u_int descs[4]; 2109 uint64_t msr; 2110 2111 if (cpu_vendor != CPUVENDOR_AMD) { 2112 return false; 2113 } 2114 if (!(cpu_feature[3] & CPUID_SVM)) { 2115 return false; 2116 } 2117 2118 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2119 return false; 2120 } 2121 x86_cpuid(0x8000000a, descs); 2122 2123 /* Want Nested Paging. */ 2124 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2125 return false; 2126 } 2127 2128 /* Want nRIP. */ 2129 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2130 return false; 2131 } 2132 2133 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2134 2135 msr = rdmsr(MSR_VMCR); 2136 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2137 return false; 2138 } 2139 2140 return true; 2141 } 2142 2143 static void 2144 svm_init_asid(uint32_t maxasid) 2145 { 2146 size_t i, j, allocsz; 2147 2148 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2149 2150 /* Arbitrarily limit. */ 2151 maxasid = uimin(maxasid, 8192); 2152 2153 svm_maxasid = maxasid; 2154 allocsz = roundup(maxasid, 8) / 8; 2155 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2156 2157 /* ASID 0 is reserved for the host. */ 2158 svm_asidmap[0] |= __BIT(0); 2159 2160 /* ASID n-1 is special, we share it. */ 2161 i = (maxasid - 1) / 8; 2162 j = (maxasid - 1) % 8; 2163 svm_asidmap[i] |= __BIT(j); 2164 } 2165 2166 static void 2167 svm_change_cpu(void *arg1, void *arg2) 2168 { 2169 bool enable = (bool)arg1; 2170 uint64_t msr; 2171 2172 msr = rdmsr(MSR_VMCR); 2173 if (msr & VMCR_SVMED) { 2174 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2175 } 2176 2177 if (!enable) { 2178 wrmsr(MSR_VM_HSAVE_PA, 0); 2179 } 2180 2181 msr = rdmsr(MSR_EFER); 2182 if (enable) { 2183 msr |= EFER_SVME; 2184 } else { 2185 msr &= ~EFER_SVME; 2186 } 2187 wrmsr(MSR_EFER, msr); 2188 2189 if (enable) { 2190 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2191 } 2192 } 2193 2194 static void 2195 svm_init(void) 2196 { 2197 CPU_INFO_ITERATOR cii; 2198 struct cpu_info *ci; 2199 struct vm_page *pg; 2200 u_int descs[4]; 2201 uint64_t xc; 2202 2203 x86_cpuid(0x8000000a, descs); 2204 2205 /* The guest TLB flush command. */ 2206 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2207 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2208 } else { 2209 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2210 } 2211 2212 /* Init the ASID. */ 2213 svm_init_asid(descs[1]); 2214 2215 /* Init the XCR0 mask. */ 2216 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2217 2218 memset(hsave, 0, sizeof(hsave)); 2219 for (CPU_INFO_FOREACH(cii, ci)) { 2220 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2221 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2222 } 2223 2224 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2225 xc_wait(xc); 2226 } 2227 2228 static void 2229 svm_fini_asid(void) 2230 { 2231 size_t allocsz; 2232 2233 allocsz = roundup(svm_maxasid, 8) / 8; 2234 kmem_free(svm_asidmap, allocsz); 2235 2236 mutex_destroy(&svm_asidlock); 2237 } 2238 2239 static void 2240 svm_fini(void) 2241 { 2242 uint64_t xc; 2243 size_t i; 2244 2245 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2246 xc_wait(xc); 2247 2248 for (i = 0; i < MAXCPUS; i++) { 2249 if (hsave[i].pa != 0) 2250 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2251 } 2252 2253 svm_fini_asid(); 2254 } 2255 2256 static void 2257 svm_capability(struct nvmm_capability *cap) 2258 { 2259 cap->u.x86.xcr0_mask = svm_xcr0_mask; 2260 cap->u.x86.mxcsr_mask = x86_fpu_mxcsr_mask; 2261 cap->u.x86.conf_cpuid_maxops = SVM_NCPUIDS; 2262 } 2263 2264 const struct nvmm_impl nvmm_x86_svm = { 2265 .ident = svm_ident, 2266 .init = svm_init, 2267 .fini = svm_fini, 2268 .capability = svm_capability, 2269 .conf_max = NVMM_X86_NCONF, 2270 .conf_sizes = svm_conf_sizes, 2271 .state_size = sizeof(struct nvmm_x64_state), 2272 .machine_create = svm_machine_create, 2273 .machine_destroy = svm_machine_destroy, 2274 .machine_configure = svm_machine_configure, 2275 .vcpu_create = svm_vcpu_create, 2276 .vcpu_destroy = svm_vcpu_destroy, 2277 .vcpu_setstate = svm_vcpu_setstate, 2278 .vcpu_getstate = svm_vcpu_getstate, 2279 .vcpu_inject = svm_vcpu_inject, 2280 .vcpu_run = svm_vcpu_run 2281 }; 2282
Indexes created Tue Sep 30 17:09:57 GMT 2025