nvmm_x86_svm.c revision 1.46.4.5
1 /* $NetBSD: nvmm_x86_svm.c,v 1.46.4.5 2020/05/21 10:52:58 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.5 2020/05/21 10:52:58 martin Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 #include <sys/mman.h> 42 43 #include <uvm/uvm.h> 44 #include <uvm/uvm_page.h> 45 46 #include <x86/cputypes.h> 47 #include <x86/specialreg.h> 48 #include <x86/pmap.h> 49 #include <x86/dbregs.h> 50 #include <x86/cpu_counter.h> 51 #include <machine/cpuvar.h> 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/x86/nvmm_x86.h> 56 57 int svm_vmrun(paddr_t, uint64_t *); 58 59 #define MSR_VM_HSAVE_PA 0xC0010117 60 61 /* -------------------------------------------------------------------------- */ 62 63 #define VMCB_EXITCODE_CR0_READ 0x0000 64 #define VMCB_EXITCODE_CR1_READ 0x0001 65 #define VMCB_EXITCODE_CR2_READ 0x0002 66 #define VMCB_EXITCODE_CR3_READ 0x0003 67 #define VMCB_EXITCODE_CR4_READ 0x0004 68 #define VMCB_EXITCODE_CR5_READ 0x0005 69 #define VMCB_EXITCODE_CR6_READ 0x0006 70 #define VMCB_EXITCODE_CR7_READ 0x0007 71 #define VMCB_EXITCODE_CR8_READ 0x0008 72 #define VMCB_EXITCODE_CR9_READ 0x0009 73 #define VMCB_EXITCODE_CR10_READ 0x000A 74 #define VMCB_EXITCODE_CR11_READ 0x000B 75 #define VMCB_EXITCODE_CR12_READ 0x000C 76 #define VMCB_EXITCODE_CR13_READ 0x000D 77 #define VMCB_EXITCODE_CR14_READ 0x000E 78 #define VMCB_EXITCODE_CR15_READ 0x000F 79 #define VMCB_EXITCODE_CR0_WRITE 0x0010 80 #define VMCB_EXITCODE_CR1_WRITE 0x0011 81 #define VMCB_EXITCODE_CR2_WRITE 0x0012 82 #define VMCB_EXITCODE_CR3_WRITE 0x0013 83 #define VMCB_EXITCODE_CR4_WRITE 0x0014 84 #define VMCB_EXITCODE_CR5_WRITE 0x0015 85 #define VMCB_EXITCODE_CR6_WRITE 0x0016 86 #define VMCB_EXITCODE_CR7_WRITE 0x0017 87 #define VMCB_EXITCODE_CR8_WRITE 0x0018 88 #define VMCB_EXITCODE_CR9_WRITE 0x0019 89 #define VMCB_EXITCODE_CR10_WRITE 0x001A 90 #define VMCB_EXITCODE_CR11_WRITE 0x001B 91 #define VMCB_EXITCODE_CR12_WRITE 0x001C 92 #define VMCB_EXITCODE_CR13_WRITE 0x001D 93 #define VMCB_EXITCODE_CR14_WRITE 0x001E 94 #define VMCB_EXITCODE_CR15_WRITE 0x001F 95 #define VMCB_EXITCODE_DR0_READ 0x0020 96 #define VMCB_EXITCODE_DR1_READ 0x0021 97 #define VMCB_EXITCODE_DR2_READ 0x0022 98 #define VMCB_EXITCODE_DR3_READ 0x0023 99 #define VMCB_EXITCODE_DR4_READ 0x0024 100 #define VMCB_EXITCODE_DR5_READ 0x0025 101 #define VMCB_EXITCODE_DR6_READ 0x0026 102 #define VMCB_EXITCODE_DR7_READ 0x0027 103 #define VMCB_EXITCODE_DR8_READ 0x0028 104 #define VMCB_EXITCODE_DR9_READ 0x0029 105 #define VMCB_EXITCODE_DR10_READ 0x002A 106 #define VMCB_EXITCODE_DR11_READ 0x002B 107 #define VMCB_EXITCODE_DR12_READ 0x002C 108 #define VMCB_EXITCODE_DR13_READ 0x002D 109 #define VMCB_EXITCODE_DR14_READ 0x002E 110 #define VMCB_EXITCODE_DR15_READ 0x002F 111 #define VMCB_EXITCODE_DR0_WRITE 0x0030 112 #define VMCB_EXITCODE_DR1_WRITE 0x0031 113 #define VMCB_EXITCODE_DR2_WRITE 0x0032 114 #define VMCB_EXITCODE_DR3_WRITE 0x0033 115 #define VMCB_EXITCODE_DR4_WRITE 0x0034 116 #define VMCB_EXITCODE_DR5_WRITE 0x0035 117 #define VMCB_EXITCODE_DR6_WRITE 0x0036 118 #define VMCB_EXITCODE_DR7_WRITE 0x0037 119 #define VMCB_EXITCODE_DR8_WRITE 0x0038 120 #define VMCB_EXITCODE_DR9_WRITE 0x0039 121 #define VMCB_EXITCODE_DR10_WRITE 0x003A 122 #define VMCB_EXITCODE_DR11_WRITE 0x003B 123 #define VMCB_EXITCODE_DR12_WRITE 0x003C 124 #define VMCB_EXITCODE_DR13_WRITE 0x003D 125 #define VMCB_EXITCODE_DR14_WRITE 0x003E 126 #define VMCB_EXITCODE_DR15_WRITE 0x003F 127 #define VMCB_EXITCODE_EXCP0 0x0040 128 #define VMCB_EXITCODE_EXCP1 0x0041 129 #define VMCB_EXITCODE_EXCP2 0x0042 130 #define VMCB_EXITCODE_EXCP3 0x0043 131 #define VMCB_EXITCODE_EXCP4 0x0044 132 #define VMCB_EXITCODE_EXCP5 0x0045 133 #define VMCB_EXITCODE_EXCP6 0x0046 134 #define VMCB_EXITCODE_EXCP7 0x0047 135 #define VMCB_EXITCODE_EXCP8 0x0048 136 #define VMCB_EXITCODE_EXCP9 0x0049 137 #define VMCB_EXITCODE_EXCP10 0x004A 138 #define VMCB_EXITCODE_EXCP11 0x004B 139 #define VMCB_EXITCODE_EXCP12 0x004C 140 #define VMCB_EXITCODE_EXCP13 0x004D 141 #define VMCB_EXITCODE_EXCP14 0x004E 142 #define VMCB_EXITCODE_EXCP15 0x004F 143 #define VMCB_EXITCODE_EXCP16 0x0050 144 #define VMCB_EXITCODE_EXCP17 0x0051 145 #define VMCB_EXITCODE_EXCP18 0x0052 146 #define VMCB_EXITCODE_EXCP19 0x0053 147 #define VMCB_EXITCODE_EXCP20 0x0054 148 #define VMCB_EXITCODE_EXCP21 0x0055 149 #define VMCB_EXITCODE_EXCP22 0x0056 150 #define VMCB_EXITCODE_EXCP23 0x0057 151 #define VMCB_EXITCODE_EXCP24 0x0058 152 #define VMCB_EXITCODE_EXCP25 0x0059 153 #define VMCB_EXITCODE_EXCP26 0x005A 154 #define VMCB_EXITCODE_EXCP27 0x005B 155 #define VMCB_EXITCODE_EXCP28 0x005C 156 #define VMCB_EXITCODE_EXCP29 0x005D 157 #define VMCB_EXITCODE_EXCP30 0x005E 158 #define VMCB_EXITCODE_EXCP31 0x005F 159 #define VMCB_EXITCODE_INTR 0x0060 160 #define VMCB_EXITCODE_NMI 0x0061 161 #define VMCB_EXITCODE_SMI 0x0062 162 #define VMCB_EXITCODE_INIT 0x0063 163 #define VMCB_EXITCODE_VINTR 0x0064 164 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 165 #define VMCB_EXITCODE_IDTR_READ 0x0066 166 #define VMCB_EXITCODE_GDTR_READ 0x0067 167 #define VMCB_EXITCODE_LDTR_READ 0x0068 168 #define VMCB_EXITCODE_TR_READ 0x0069 169 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 170 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 171 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 172 #define VMCB_EXITCODE_TR_WRITE 0x006D 173 #define VMCB_EXITCODE_RDTSC 0x006E 174 #define VMCB_EXITCODE_RDPMC 0x006F 175 #define VMCB_EXITCODE_PUSHF 0x0070 176 #define VMCB_EXITCODE_POPF 0x0071 177 #define VMCB_EXITCODE_CPUID 0x0072 178 #define VMCB_EXITCODE_RSM 0x0073 179 #define VMCB_EXITCODE_IRET 0x0074 180 #define VMCB_EXITCODE_SWINT 0x0075 181 #define VMCB_EXITCODE_INVD 0x0076 182 #define VMCB_EXITCODE_PAUSE 0x0077 183 #define VMCB_EXITCODE_HLT 0x0078 184 #define VMCB_EXITCODE_INVLPG 0x0079 185 #define VMCB_EXITCODE_INVLPGA 0x007A 186 #define VMCB_EXITCODE_IOIO 0x007B 187 #define VMCB_EXITCODE_MSR 0x007C 188 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 189 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 190 #define VMCB_EXITCODE_SHUTDOWN 0x007F 191 #define VMCB_EXITCODE_VMRUN 0x0080 192 #define VMCB_EXITCODE_VMMCALL 0x0081 193 #define VMCB_EXITCODE_VMLOAD 0x0082 194 #define VMCB_EXITCODE_VMSAVE 0x0083 195 #define VMCB_EXITCODE_STGI 0x0084 196 #define VMCB_EXITCODE_CLGI 0x0085 197 #define VMCB_EXITCODE_SKINIT 0x0086 198 #define VMCB_EXITCODE_RDTSCP 0x0087 199 #define VMCB_EXITCODE_ICEBP 0x0088 200 #define VMCB_EXITCODE_WBINVD 0x0089 201 #define VMCB_EXITCODE_MONITOR 0x008A 202 #define VMCB_EXITCODE_MWAIT 0x008B 203 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 204 #define VMCB_EXITCODE_XSETBV 0x008D 205 #define VMCB_EXITCODE_RDPRU 0x008E 206 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 207 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 208 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 209 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 210 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 211 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 212 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 213 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 214 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 215 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 216 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 217 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 218 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 219 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 220 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 221 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 222 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 223 #define VMCB_EXITCODE_MCOMMIT 0x00A3 224 #define VMCB_EXITCODE_NPF 0x0400 225 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 226 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 227 #define VMCB_EXITCODE_VMGEXIT 0x0403 228 #define VMCB_EXITCODE_INVALID -1 229 230 /* -------------------------------------------------------------------------- */ 231 232 struct vmcb_ctrl { 233 uint32_t intercept_cr; 234 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 235 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 236 237 uint32_t intercept_dr; 238 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 239 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 240 241 uint32_t intercept_vec; 242 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 243 244 uint32_t intercept_misc1; 245 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 246 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 247 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 248 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 249 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 250 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 251 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 252 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 253 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 254 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 255 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 256 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 257 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 258 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 259 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 260 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 261 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 262 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 263 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 264 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 265 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 266 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 267 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 268 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 269 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 270 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 271 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 272 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 273 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 274 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 275 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 276 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 277 278 uint32_t intercept_misc2; 279 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 280 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 281 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 282 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 283 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 284 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 285 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 286 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 287 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 288 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 289 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 290 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(11) 291 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED __BIT(12) 292 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 293 #define VMCB_CTRL_INTERCEPT_RDPRU __BIT(14) 294 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 295 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 296 297 uint32_t intercept_misc3; 298 #define VMCB_CTRL_INTERCEPT_MCOMMIT __BIT(3) 299 300 uint8_t rsvd1[36]; 301 uint16_t pause_filt_thresh; 302 uint16_t pause_filt_cnt; 303 uint64_t iopm_base_pa; 304 uint64_t msrpm_base_pa; 305 uint64_t tsc_offset; 306 uint32_t guest_asid; 307 308 uint32_t tlb_ctrl; 309 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 310 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 311 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 312 313 uint64_t v; 314 #define VMCB_CTRL_V_TPR __BITS(3,0) 315 #define VMCB_CTRL_V_IRQ __BIT(8) 316 #define VMCB_CTRL_V_VGIF __BIT(9) 317 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 318 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 319 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 320 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 321 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 322 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 323 324 uint64_t intr; 325 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 326 327 uint64_t exitcode; 328 uint64_t exitinfo1; 329 uint64_t exitinfo2; 330 331 uint64_t exitintinfo; 332 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 333 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 334 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 335 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 336 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 337 338 uint64_t enable1; 339 #define VMCB_CTRL_ENABLE_NP __BIT(0) 340 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 341 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 342 #define VMCB_CTRL_ENABLE_GMET __BIT(3) 343 #define VMCB_CTRL_ENABLE_VTE __BIT(5) 344 345 uint64_t avic; 346 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 347 348 uint64_t ghcb; 349 350 uint64_t eventinj; 351 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 352 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 353 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 354 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 355 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 356 357 uint64_t n_cr3; 358 359 uint64_t enable2; 360 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 361 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 362 363 uint32_t vmcb_clean; 364 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 365 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 366 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 367 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 368 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 369 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 370 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 371 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 372 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 373 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 374 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 375 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 376 377 uint32_t rsvd2; 378 uint64_t nrip; 379 uint8_t inst_len; 380 uint8_t inst_bytes[15]; 381 uint64_t avic_abpp; 382 uint64_t rsvd3; 383 uint64_t avic_ltp; 384 385 uint64_t avic_phys; 386 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 387 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 388 389 uint64_t rsvd4; 390 uint64_t vmcb_ptr; 391 392 uint8_t pad[752]; 393 } __packed; 394 395 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 396 397 struct vmcb_segment { 398 uint16_t selector; 399 uint16_t attrib; /* hidden */ 400 uint32_t limit; /* hidden */ 401 uint64_t base; /* hidden */ 402 } __packed; 403 404 CTASSERT(sizeof(struct vmcb_segment) == 16); 405 406 struct vmcb_state { 407 struct vmcb_segment es; 408 struct vmcb_segment cs; 409 struct vmcb_segment ss; 410 struct vmcb_segment ds; 411 struct vmcb_segment fs; 412 struct vmcb_segment gs; 413 struct vmcb_segment gdt; 414 struct vmcb_segment ldt; 415 struct vmcb_segment idt; 416 struct vmcb_segment tr; 417 uint8_t rsvd1[43]; 418 uint8_t cpl; 419 uint8_t rsvd2[4]; 420 uint64_t efer; 421 uint8_t rsvd3[112]; 422 uint64_t cr4; 423 uint64_t cr3; 424 uint64_t cr0; 425 uint64_t dr7; 426 uint64_t dr6; 427 uint64_t rflags; 428 uint64_t rip; 429 uint8_t rsvd4[88]; 430 uint64_t rsp; 431 uint8_t rsvd5[24]; 432 uint64_t rax; 433 uint64_t star; 434 uint64_t lstar; 435 uint64_t cstar; 436 uint64_t sfmask; 437 uint64_t kernelgsbase; 438 uint64_t sysenter_cs; 439 uint64_t sysenter_esp; 440 uint64_t sysenter_eip; 441 uint64_t cr2; 442 uint8_t rsvd6[32]; 443 uint64_t g_pat; 444 uint64_t dbgctl; 445 uint64_t br_from; 446 uint64_t br_to; 447 uint64_t int_from; 448 uint64_t int_to; 449 uint8_t pad[2408]; 450 } __packed; 451 452 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 453 454 struct vmcb { 455 struct vmcb_ctrl ctrl; 456 struct vmcb_state state; 457 } __packed; 458 459 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 460 CTASSERT(offsetof(struct vmcb, state) == 0x400); 461 462 /* -------------------------------------------------------------------------- */ 463 464 static void svm_vcpu_state_provide(struct nvmm_cpu *, uint64_t); 465 static void svm_vcpu_state_commit(struct nvmm_cpu *); 466 467 struct svm_hsave { 468 paddr_t pa; 469 }; 470 471 static struct svm_hsave hsave[MAXCPUS]; 472 473 static uint8_t *svm_asidmap __read_mostly; 474 static uint32_t svm_maxasid __read_mostly; 475 static kmutex_t svm_asidlock __cacheline_aligned; 476 477 static bool svm_decode_assist __read_mostly; 478 static uint32_t svm_ctrl_tlb_flush __read_mostly; 479 480 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 481 static uint64_t svm_xcr0_mask __read_mostly; 482 483 #define SVM_NCPUIDS 32 484 485 #define VMCB_NPAGES 1 486 487 #define MSRBM_NPAGES 2 488 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 489 490 #define IOBM_NPAGES 3 491 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 492 493 /* Does not include EFER_LMSLE. */ 494 #define EFER_VALID \ 495 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 496 497 #define EFER_TLB_FLUSH \ 498 (EFER_NXE|EFER_LMA|EFER_LME) 499 #define CR0_TLB_FLUSH \ 500 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 501 #define CR4_TLB_FLUSH \ 502 (CR4_PGE|CR4_PAE|CR4_PSE) 503 504 /* -------------------------------------------------------------------------- */ 505 506 struct svm_machdata { 507 volatile uint64_t mach_htlb_gen; 508 }; 509 510 static const size_t svm_vcpu_conf_sizes[NVMM_X86_VCPU_NCONF] = { 511 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = 512 sizeof(struct nvmm_vcpu_conf_cpuid), 513 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = 514 sizeof(struct nvmm_vcpu_conf_tpr) 515 }; 516 517 struct svm_cpudata { 518 /* General */ 519 bool shared_asid; 520 bool gtlb_want_flush; 521 bool gtsc_want_update; 522 uint64_t vcpu_htlb_gen; 523 524 /* VMCB */ 525 struct vmcb *vmcb; 526 paddr_t vmcb_pa; 527 528 /* I/O bitmap */ 529 uint8_t *iobm; 530 paddr_t iobm_pa; 531 532 /* MSR bitmap */ 533 uint8_t *msrbm; 534 paddr_t msrbm_pa; 535 536 /* Host state */ 537 uint64_t hxcr0; 538 uint64_t star; 539 uint64_t lstar; 540 uint64_t cstar; 541 uint64_t sfmask; 542 uint64_t fsbase; 543 uint64_t kernelgsbase; 544 bool ts_set; 545 struct xsave_header hfpu __aligned(64); 546 547 /* Intr state */ 548 bool int_window_exit; 549 bool nmi_window_exit; 550 bool evt_pending; 551 552 /* Guest state */ 553 uint64_t gxcr0; 554 uint64_t gprs[NVMM_X64_NGPR]; 555 uint64_t drs[NVMM_X64_NDR]; 556 uint64_t gtsc; 557 struct xsave_header gfpu __aligned(64); 558 559 /* VCPU configuration. */ 560 bool cpuidpresent[SVM_NCPUIDS]; 561 struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; 562 }; 563 564 static void 565 svm_vmcb_cache_default(struct vmcb *vmcb) 566 { 567 vmcb->ctrl.vmcb_clean = 568 VMCB_CTRL_VMCB_CLEAN_I | 569 VMCB_CTRL_VMCB_CLEAN_IOPM | 570 VMCB_CTRL_VMCB_CLEAN_ASID | 571 VMCB_CTRL_VMCB_CLEAN_TPR | 572 VMCB_CTRL_VMCB_CLEAN_NP | 573 VMCB_CTRL_VMCB_CLEAN_CR | 574 VMCB_CTRL_VMCB_CLEAN_DR | 575 VMCB_CTRL_VMCB_CLEAN_DT | 576 VMCB_CTRL_VMCB_CLEAN_SEG | 577 VMCB_CTRL_VMCB_CLEAN_CR2 | 578 VMCB_CTRL_VMCB_CLEAN_LBR | 579 VMCB_CTRL_VMCB_CLEAN_AVIC; 580 } 581 582 static void 583 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 584 { 585 if (flags & NVMM_X64_STATE_SEGS) { 586 vmcb->ctrl.vmcb_clean &= 587 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 588 } 589 if (flags & NVMM_X64_STATE_CRS) { 590 vmcb->ctrl.vmcb_clean &= 591 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 592 VMCB_CTRL_VMCB_CLEAN_TPR); 593 } 594 if (flags & NVMM_X64_STATE_DRS) { 595 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 596 } 597 if (flags & NVMM_X64_STATE_MSRS) { 598 /* CR for EFER, NP for PAT. */ 599 vmcb->ctrl.vmcb_clean &= 600 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 601 } 602 } 603 604 static inline void 605 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 606 { 607 vmcb->ctrl.vmcb_clean &= ~flags; 608 } 609 610 static inline void 611 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 612 { 613 vmcb->ctrl.vmcb_clean = 0; 614 } 615 616 #define SVM_EVENT_TYPE_HW_INT 0 617 #define SVM_EVENT_TYPE_NMI 2 618 #define SVM_EVENT_TYPE_EXC 3 619 #define SVM_EVENT_TYPE_SW_INT 4 620 621 static void 622 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 623 { 624 struct svm_cpudata *cpudata = vcpu->cpudata; 625 struct vmcb *vmcb = cpudata->vmcb; 626 627 if (nmi) { 628 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 629 cpudata->nmi_window_exit = true; 630 } else { 631 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 632 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 633 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 634 cpudata->int_window_exit = true; 635 } 636 637 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 638 } 639 640 static void 641 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 642 { 643 struct svm_cpudata *cpudata = vcpu->cpudata; 644 struct vmcb *vmcb = cpudata->vmcb; 645 646 if (nmi) { 647 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 648 cpudata->nmi_window_exit = false; 649 } else { 650 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 651 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 652 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 653 cpudata->int_window_exit = false; 654 } 655 656 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 657 } 658 659 static inline int 660 svm_event_has_error(uint8_t vector) 661 { 662 switch (vector) { 663 case 8: /* #DF */ 664 case 10: /* #TS */ 665 case 11: /* #NP */ 666 case 12: /* #SS */ 667 case 13: /* #GP */ 668 case 14: /* #PF */ 669 case 17: /* #AC */ 670 case 30: /* #SX */ 671 return 1; 672 default: 673 return 0; 674 } 675 } 676 677 static int 678 svm_vcpu_inject(struct nvmm_cpu *vcpu) 679 { 680 struct nvmm_comm_page *comm = vcpu->comm; 681 struct svm_cpudata *cpudata = vcpu->cpudata; 682 struct vmcb *vmcb = cpudata->vmcb; 683 u_int evtype; 684 uint8_t vector; 685 uint64_t error; 686 int type = 0, err = 0; 687 688 evtype = comm->event.type; 689 vector = comm->event.vector; 690 error = comm->event.u.excp.error; 691 __insn_barrier(); 692 693 switch (evtype) { 694 case NVMM_VCPU_EVENT_EXCP: 695 type = SVM_EVENT_TYPE_EXC; 696 if (vector == 2 || vector >= 32) 697 return EINVAL; 698 if (vector == 3 || vector == 0) 699 return EINVAL; 700 err = svm_event_has_error(vector); 701 break; 702 case NVMM_VCPU_EVENT_INTR: 703 type = SVM_EVENT_TYPE_HW_INT; 704 if (vector == 2) { 705 type = SVM_EVENT_TYPE_NMI; 706 svm_event_waitexit_enable(vcpu, true); 707 } 708 err = 0; 709 break; 710 default: 711 return EINVAL; 712 } 713 714 vmcb->ctrl.eventinj = 715 __SHIFTIN((uint64_t)vector, VMCB_CTRL_EVENTINJ_VECTOR) | 716 __SHIFTIN((uint64_t)type, VMCB_CTRL_EVENTINJ_TYPE) | 717 __SHIFTIN((uint64_t)err, VMCB_CTRL_EVENTINJ_EV) | 718 __SHIFTIN((uint64_t)1, VMCB_CTRL_EVENTINJ_V) | 719 __SHIFTIN((uint64_t)error, VMCB_CTRL_EVENTINJ_ERRORCODE); 720 721 cpudata->evt_pending = true; 722 723 return 0; 724 } 725 726 static void 727 svm_inject_ud(struct nvmm_cpu *vcpu) 728 { 729 struct nvmm_comm_page *comm = vcpu->comm; 730 int ret __diagused; 731 732 comm->event.type = NVMM_VCPU_EVENT_EXCP; 733 comm->event.vector = 6; 734 comm->event.u.excp.error = 0; 735 736 ret = svm_vcpu_inject(vcpu); 737 KASSERT(ret == 0); 738 } 739 740 static void 741 svm_inject_gp(struct nvmm_cpu *vcpu) 742 { 743 struct nvmm_comm_page *comm = vcpu->comm; 744 int ret __diagused; 745 746 comm->event.type = NVMM_VCPU_EVENT_EXCP; 747 comm->event.vector = 13; 748 comm->event.u.excp.error = 0; 749 750 ret = svm_vcpu_inject(vcpu); 751 KASSERT(ret == 0); 752 } 753 754 static inline int 755 svm_vcpu_event_commit(struct nvmm_cpu *vcpu) 756 { 757 if (__predict_true(!vcpu->comm->event_commit)) { 758 return 0; 759 } 760 vcpu->comm->event_commit = false; 761 return svm_vcpu_inject(vcpu); 762 } 763 764 static inline void 765 svm_inkernel_advance(struct vmcb *vmcb) 766 { 767 /* 768 * Maybe we should also apply single-stepping and debug exceptions. 769 * Matters for guest-ring3, because it can execute 'cpuid' under a 770 * debugger. 771 */ 772 vmcb->state.rip = vmcb->ctrl.nrip; 773 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 774 } 775 776 #define SVM_CPUID_MAX_HYPERVISOR 0x40000000 777 778 static void 779 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 780 { 781 struct svm_cpudata *cpudata = vcpu->cpudata; 782 uint64_t cr4; 783 784 switch (eax) { 785 case 0x00000001: 786 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 787 788 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 789 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 790 CPUID_LOCAL_APIC_ID); 791 792 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 793 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 794 795 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 796 797 /* CPUID2_OSXSAVE depends on CR4. */ 798 cr4 = cpudata->vmcb->state.cr4; 799 if (!(cr4 & CR4_OSXSAVE)) { 800 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 801 } 802 break; 803 case 0x00000002: /* Empty */ 804 case 0x00000003: /* Empty */ 805 case 0x00000004: /* Empty */ 806 case 0x00000005: /* Monitor/MWait */ 807 case 0x00000006: /* Power Management Related Features */ 808 cpudata->vmcb->state.rax = 0; 809 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 810 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 811 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 812 break; 813 case 0x00000007: /* Structured Extended Features */ 814 cpudata->vmcb->state.rax &= nvmm_cpuid_00000007.eax; 815 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 816 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 817 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 818 break; 819 case 0x00000008: /* Empty */ 820 case 0x00000009: /* Empty */ 821 case 0x0000000A: /* Empty */ 822 case 0x0000000B: /* Empty */ 823 case 0x0000000C: /* Empty */ 824 cpudata->vmcb->state.rax = 0; 825 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 826 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 827 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 828 break; 829 case 0x0000000D: /* Processor Extended State Enumeration */ 830 if (svm_xcr0_mask == 0) { 831 break; 832 } 833 switch (ecx) { 834 case 0: 835 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 836 if (cpudata->gxcr0 & XCR0_SSE) { 837 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 838 } else { 839 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 840 } 841 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 842 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; 843 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 844 break; 845 case 1: 846 cpudata->vmcb->state.rax &= 847 (CPUID_PES1_XSAVEOPT | CPUID_PES1_XSAVEC | 848 CPUID_PES1_XGETBV); 849 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 850 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 851 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 852 break; 853 default: 854 cpudata->vmcb->state.rax = 0; 855 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 856 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 857 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 858 break; 859 } 860 break; 861 862 case 0x40000000: /* Hypervisor Information */ 863 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR; 864 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 865 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 866 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 867 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 868 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 869 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 870 break; 871 872 case 0x80000001: 873 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 874 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 875 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 876 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 877 break; 878 default: 879 break; 880 } 881 } 882 883 static void 884 svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) 885 { 886 exit->u.insn.npc = vmcb->ctrl.nrip; 887 exit->reason = reason; 888 } 889 890 static void 891 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 892 struct nvmm_vcpu_exit *exit) 893 { 894 struct svm_cpudata *cpudata = vcpu->cpudata; 895 struct nvmm_vcpu_conf_cpuid *cpuid; 896 uint64_t eax, ecx; 897 u_int descs[4]; 898 size_t i; 899 900 eax = cpudata->vmcb->state.rax; 901 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 902 x86_cpuid2(eax, ecx, descs); 903 904 cpudata->vmcb->state.rax = descs[0]; 905 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 906 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 907 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 908 909 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 910 911 for (i = 0; i < SVM_NCPUIDS; i++) { 912 if (!cpudata->cpuidpresent[i]) { 913 continue; 914 } 915 cpuid = &cpudata->cpuid[i]; 916 if (cpuid->leaf != eax) { 917 continue; 918 } 919 920 if (cpuid->exit) { 921 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID); 922 return; 923 } 924 KASSERT(cpuid->mask); 925 926 /* del */ 927 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax; 928 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx; 929 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx; 930 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx; 931 932 /* set */ 933 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax; 934 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx; 935 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx; 936 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx; 937 938 break; 939 } 940 941 svm_inkernel_advance(cpudata->vmcb); 942 exit->reason = NVMM_VCPU_EXIT_NONE; 943 } 944 945 static void 946 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 947 struct nvmm_vcpu_exit *exit) 948 { 949 struct svm_cpudata *cpudata = vcpu->cpudata; 950 struct vmcb *vmcb = cpudata->vmcb; 951 952 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 953 svm_event_waitexit_disable(vcpu, false); 954 } 955 956 svm_inkernel_advance(cpudata->vmcb); 957 exit->reason = NVMM_VCPU_EXIT_HALTED; 958 } 959 960 #define SVM_EXIT_IO_PORT __BITS(31,16) 961 #define SVM_EXIT_IO_SEG __BITS(12,10) 962 #define SVM_EXIT_IO_A64 __BIT(9) 963 #define SVM_EXIT_IO_A32 __BIT(8) 964 #define SVM_EXIT_IO_A16 __BIT(7) 965 #define SVM_EXIT_IO_SZ32 __BIT(6) 966 #define SVM_EXIT_IO_SZ16 __BIT(5) 967 #define SVM_EXIT_IO_SZ8 __BIT(4) 968 #define SVM_EXIT_IO_REP __BIT(3) 969 #define SVM_EXIT_IO_STR __BIT(2) 970 #define SVM_EXIT_IO_IN __BIT(0) 971 972 static void 973 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 974 struct nvmm_vcpu_exit *exit) 975 { 976 struct svm_cpudata *cpudata = vcpu->cpudata; 977 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 978 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 979 980 exit->reason = NVMM_VCPU_EXIT_IO; 981 982 exit->u.io.in = (info & SVM_EXIT_IO_IN) != 0; 983 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 984 985 if (svm_decode_assist) { 986 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 987 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 988 } else { 989 exit->u.io.seg = -1; 990 } 991 992 if (info & SVM_EXIT_IO_A64) { 993 exit->u.io.address_size = 8; 994 } else if (info & SVM_EXIT_IO_A32) { 995 exit->u.io.address_size = 4; 996 } else if (info & SVM_EXIT_IO_A16) { 997 exit->u.io.address_size = 2; 998 } 999 1000 if (info & SVM_EXIT_IO_SZ32) { 1001 exit->u.io.operand_size = 4; 1002 } else if (info & SVM_EXIT_IO_SZ16) { 1003 exit->u.io.operand_size = 2; 1004 } else if (info & SVM_EXIT_IO_SZ8) { 1005 exit->u.io.operand_size = 1; 1006 } 1007 1008 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 1009 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 1010 exit->u.io.npc = nextpc; 1011 1012 svm_vcpu_state_provide(vcpu, 1013 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1014 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1015 } 1016 1017 static const uint64_t msr_ignore_list[] = { 1018 0xc0010055, /* MSR_CMPHALT */ 1019 MSR_DE_CFG, 1020 MSR_IC_CFG, 1021 MSR_UCODE_AMD_PATCHLEVEL 1022 }; 1023 1024 static bool 1025 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1026 struct nvmm_vcpu_exit *exit) 1027 { 1028 struct svm_cpudata *cpudata = vcpu->cpudata; 1029 struct vmcb *vmcb = cpudata->vmcb; 1030 uint64_t val; 1031 size_t i; 1032 1033 if (exit->reason == NVMM_VCPU_EXIT_RDMSR) { 1034 if (exit->u.rdmsr.msr == MSR_NB_CFG) { 1035 val = NB_CFG_INITAPICCPUIDLO; 1036 vmcb->state.rax = (val & 0xFFFFFFFF); 1037 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1038 goto handled; 1039 } 1040 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1041 if (msr_ignore_list[i] != exit->u.rdmsr.msr) 1042 continue; 1043 val = 0; 1044 vmcb->state.rax = (val & 0xFFFFFFFF); 1045 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1046 goto handled; 1047 } 1048 } else { 1049 if (exit->u.wrmsr.msr == MSR_EFER) { 1050 if (__predict_false(exit->u.wrmsr.val & ~EFER_VALID)) { 1051 goto error; 1052 } 1053 if ((vmcb->state.efer ^ exit->u.wrmsr.val) & 1054 EFER_TLB_FLUSH) { 1055 cpudata->gtlb_want_flush = true; 1056 } 1057 vmcb->state.efer = exit->u.wrmsr.val | EFER_SVME; 1058 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1059 goto handled; 1060 } 1061 if (exit->u.wrmsr.msr == MSR_TSC) { 1062 cpudata->gtsc = exit->u.wrmsr.val; 1063 cpudata->gtsc_want_update = true; 1064 goto handled; 1065 } 1066 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1067 if (msr_ignore_list[i] != exit->u.wrmsr.msr) 1068 continue; 1069 goto handled; 1070 } 1071 } 1072 1073 return false; 1074 1075 handled: 1076 svm_inkernel_advance(cpudata->vmcb); 1077 return true; 1078 1079 error: 1080 svm_inject_gp(vcpu); 1081 return true; 1082 } 1083 1084 static inline void 1085 svm_exit_rdmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1086 struct nvmm_vcpu_exit *exit) 1087 { 1088 struct svm_cpudata *cpudata = vcpu->cpudata; 1089 1090 exit->reason = NVMM_VCPU_EXIT_RDMSR; 1091 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1092 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip; 1093 1094 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1095 exit->reason = NVMM_VCPU_EXIT_NONE; 1096 return; 1097 } 1098 1099 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1100 } 1101 1102 static inline void 1103 svm_exit_wrmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1104 struct nvmm_vcpu_exit *exit) 1105 { 1106 struct svm_cpudata *cpudata = vcpu->cpudata; 1107 uint64_t rdx, rax; 1108 1109 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1110 rax = cpudata->vmcb->state.rax; 1111 1112 exit->reason = NVMM_VCPU_EXIT_WRMSR; 1113 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1114 exit->u.wrmsr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1115 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip; 1116 1117 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1118 exit->reason = NVMM_VCPU_EXIT_NONE; 1119 return; 1120 } 1121 1122 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1123 } 1124 1125 static void 1126 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1127 struct nvmm_vcpu_exit *exit) 1128 { 1129 struct svm_cpudata *cpudata = vcpu->cpudata; 1130 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1131 1132 if (info == 0) { 1133 svm_exit_rdmsr(mach, vcpu, exit); 1134 } else { 1135 svm_exit_wrmsr(mach, vcpu, exit); 1136 } 1137 } 1138 1139 static void 1140 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1141 struct nvmm_vcpu_exit *exit) 1142 { 1143 struct svm_cpudata *cpudata = vcpu->cpudata; 1144 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1145 1146 exit->reason = NVMM_VCPU_EXIT_MEMORY; 1147 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1148 exit->u.mem.prot = PROT_WRITE; 1149 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1150 exit->u.mem.prot = PROT_EXEC; 1151 else 1152 exit->u.mem.prot = PROT_READ; 1153 exit->u.mem.gpa = gpa; 1154 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1155 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1156 sizeof(exit->u.mem.inst_bytes)); 1157 1158 svm_vcpu_state_provide(vcpu, 1159 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1160 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1161 } 1162 1163 static void 1164 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1165 struct nvmm_vcpu_exit *exit) 1166 { 1167 struct svm_cpudata *cpudata = vcpu->cpudata; 1168 struct vmcb *vmcb = cpudata->vmcb; 1169 uint64_t val; 1170 1171 exit->reason = NVMM_VCPU_EXIT_NONE; 1172 1173 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1174 (vmcb->state.rax & 0xFFFFFFFF); 1175 1176 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1177 goto error; 1178 } else if (__predict_false(vmcb->state.cpl != 0)) { 1179 goto error; 1180 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1181 goto error; 1182 } else if (__predict_false((val & XCR0_X87) == 0)) { 1183 goto error; 1184 } 1185 1186 cpudata->gxcr0 = val; 1187 1188 svm_inkernel_advance(cpudata->vmcb); 1189 return; 1190 1191 error: 1192 svm_inject_gp(vcpu); 1193 } 1194 1195 static void 1196 svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code) 1197 { 1198 exit->u.inv.hwcode = code; 1199 exit->reason = NVMM_VCPU_EXIT_INVALID; 1200 } 1201 1202 /* -------------------------------------------------------------------------- */ 1203 1204 static void 1205 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1206 { 1207 struct svm_cpudata *cpudata = vcpu->cpudata; 1208 1209 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1210 1211 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1212 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1213 1214 if (svm_xcr0_mask != 0) { 1215 cpudata->hxcr0 = rdxcr(0); 1216 wrxcr(0, cpudata->gxcr0); 1217 } 1218 } 1219 1220 static void 1221 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1222 { 1223 struct svm_cpudata *cpudata = vcpu->cpudata; 1224 1225 if (svm_xcr0_mask != 0) { 1226 cpudata->gxcr0 = rdxcr(0); 1227 wrxcr(0, cpudata->hxcr0); 1228 } 1229 1230 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1231 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1232 1233 if (cpudata->ts_set) { 1234 stts(); 1235 } 1236 } 1237 1238 static void 1239 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1240 { 1241 struct svm_cpudata *cpudata = vcpu->cpudata; 1242 1243 x86_dbregs_save(curlwp); 1244 1245 ldr7(0); 1246 1247 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1248 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1249 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1250 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1251 } 1252 1253 static void 1254 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1255 { 1256 struct svm_cpudata *cpudata = vcpu->cpudata; 1257 1258 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1259 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1260 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1261 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1262 1263 x86_dbregs_restore(curlwp); 1264 } 1265 1266 static void 1267 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1268 { 1269 struct svm_cpudata *cpudata = vcpu->cpudata; 1270 1271 cpudata->fsbase = rdmsr(MSR_FSBASE); 1272 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1273 } 1274 1275 static void 1276 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1277 { 1278 struct svm_cpudata *cpudata = vcpu->cpudata; 1279 1280 wrmsr(MSR_STAR, cpudata->star); 1281 wrmsr(MSR_LSTAR, cpudata->lstar); 1282 wrmsr(MSR_CSTAR, cpudata->cstar); 1283 wrmsr(MSR_SFMASK, cpudata->sfmask); 1284 wrmsr(MSR_FSBASE, cpudata->fsbase); 1285 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1286 } 1287 1288 /* -------------------------------------------------------------------------- */ 1289 1290 static inline void 1291 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1292 { 1293 struct svm_cpudata *cpudata = vcpu->cpudata; 1294 1295 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1296 cpudata->gtlb_want_flush = true; 1297 } 1298 } 1299 1300 static inline void 1301 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1302 { 1303 /* 1304 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1305 * executing on this hCPU and the hTLB already got flushed, or it 1306 * was executing on another hCPU in which case the catchup is done 1307 * in svm_gtlb_catchup(). 1308 */ 1309 } 1310 1311 static inline uint64_t 1312 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1313 { 1314 struct vmcb *vmcb = cpudata->vmcb; 1315 uint64_t machgen; 1316 1317 machgen = machdata->mach_htlb_gen; 1318 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1319 return machgen; 1320 } 1321 1322 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1323 return machgen; 1324 } 1325 1326 static inline void 1327 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1328 { 1329 struct vmcb *vmcb = cpudata->vmcb; 1330 1331 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1332 cpudata->vcpu_htlb_gen = machgen; 1333 } 1334 } 1335 1336 static inline void 1337 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb) 1338 { 1339 cpudata->evt_pending = false; 1340 1341 if (__predict_false(vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V)) { 1342 vmcb->ctrl.eventinj = vmcb->ctrl.exitintinfo; 1343 cpudata->evt_pending = true; 1344 } 1345 } 1346 1347 static int 1348 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1349 struct nvmm_vcpu_exit *exit) 1350 { 1351 struct nvmm_comm_page *comm = vcpu->comm; 1352 struct svm_machdata *machdata = mach->machdata; 1353 struct svm_cpudata *cpudata = vcpu->cpudata; 1354 struct vmcb *vmcb = cpudata->vmcb; 1355 uint64_t machgen; 1356 int hcpu, s; 1357 1358 if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) { 1359 return EINVAL; 1360 } 1361 svm_vcpu_state_commit(vcpu); 1362 comm->state_cached = 0; 1363 1364 kpreempt_disable(); 1365 hcpu = cpu_number(); 1366 1367 svm_gtlb_catchup(vcpu, hcpu); 1368 svm_htlb_catchup(vcpu, hcpu); 1369 1370 if (vcpu->hcpu_last != hcpu) { 1371 svm_vmcb_cache_flush_all(vmcb); 1372 cpudata->gtsc_want_update = true; 1373 } 1374 1375 svm_vcpu_guest_dbregs_enter(vcpu); 1376 svm_vcpu_guest_misc_enter(vcpu); 1377 1378 while (1) { 1379 if (cpudata->gtlb_want_flush) { 1380 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1381 } else { 1382 vmcb->ctrl.tlb_ctrl = 0; 1383 } 1384 1385 if (__predict_false(cpudata->gtsc_want_update)) { 1386 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1387 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1388 } 1389 1390 s = splhigh(); 1391 machgen = svm_htlb_flush(machdata, cpudata); 1392 svm_vcpu_guest_fpu_enter(vcpu); 1393 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1394 svm_vcpu_guest_fpu_leave(vcpu); 1395 svm_htlb_flush_ack(cpudata, machgen); 1396 splx(s); 1397 1398 svm_vmcb_cache_default(vmcb); 1399 1400 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1401 cpudata->gtlb_want_flush = false; 1402 cpudata->gtsc_want_update = false; 1403 vcpu->hcpu_last = hcpu; 1404 } 1405 svm_exit_evt(cpudata, vmcb); 1406 1407 switch (vmcb->ctrl.exitcode) { 1408 case VMCB_EXITCODE_INTR: 1409 case VMCB_EXITCODE_NMI: 1410 exit->reason = NVMM_VCPU_EXIT_NONE; 1411 break; 1412 case VMCB_EXITCODE_VINTR: 1413 svm_event_waitexit_disable(vcpu, false); 1414 exit->reason = NVMM_VCPU_EXIT_INT_READY; 1415 break; 1416 case VMCB_EXITCODE_IRET: 1417 svm_event_waitexit_disable(vcpu, true); 1418 exit->reason = NVMM_VCPU_EXIT_NMI_READY; 1419 break; 1420 case VMCB_EXITCODE_CPUID: 1421 svm_exit_cpuid(mach, vcpu, exit); 1422 break; 1423 case VMCB_EXITCODE_HLT: 1424 svm_exit_hlt(mach, vcpu, exit); 1425 break; 1426 case VMCB_EXITCODE_IOIO: 1427 svm_exit_io(mach, vcpu, exit); 1428 break; 1429 case VMCB_EXITCODE_MSR: 1430 svm_exit_msr(mach, vcpu, exit); 1431 break; 1432 case VMCB_EXITCODE_SHUTDOWN: 1433 exit->reason = NVMM_VCPU_EXIT_SHUTDOWN; 1434 break; 1435 case VMCB_EXITCODE_RDPMC: 1436 case VMCB_EXITCODE_RSM: 1437 case VMCB_EXITCODE_INVLPGA: 1438 case VMCB_EXITCODE_VMRUN: 1439 case VMCB_EXITCODE_VMMCALL: 1440 case VMCB_EXITCODE_VMLOAD: 1441 case VMCB_EXITCODE_VMSAVE: 1442 case VMCB_EXITCODE_STGI: 1443 case VMCB_EXITCODE_CLGI: 1444 case VMCB_EXITCODE_SKINIT: 1445 case VMCB_EXITCODE_RDTSCP: 1446 svm_inject_ud(vcpu); 1447 exit->reason = NVMM_VCPU_EXIT_NONE; 1448 break; 1449 case VMCB_EXITCODE_MONITOR: 1450 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR); 1451 break; 1452 case VMCB_EXITCODE_MWAIT: 1453 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1454 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT); 1455 break; 1456 case VMCB_EXITCODE_XSETBV: 1457 svm_exit_xsetbv(mach, vcpu, exit); 1458 break; 1459 case VMCB_EXITCODE_NPF: 1460 svm_exit_npf(mach, vcpu, exit); 1461 break; 1462 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1463 default: 1464 svm_exit_invalid(exit, vmcb->ctrl.exitcode); 1465 break; 1466 } 1467 1468 /* If no reason to return to userland, keep rolling. */ 1469 if (curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 1470 break; 1471 } 1472 if (curcpu()->ci_data.cpu_softints != 0) { 1473 break; 1474 } 1475 if (curlwp->l_flag & LW_USERRET) { 1476 break; 1477 } 1478 if (exit->reason != NVMM_VCPU_EXIT_NONE) { 1479 break; 1480 } 1481 } 1482 1483 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1484 1485 svm_vcpu_guest_misc_leave(vcpu); 1486 svm_vcpu_guest_dbregs_leave(vcpu); 1487 1488 kpreempt_enable(); 1489 1490 exit->exitstate.rflags = vmcb->state.rflags; 1491 exit->exitstate.cr8 = __SHIFTOUT(vmcb->ctrl.v, VMCB_CTRL_V_TPR); 1492 exit->exitstate.int_shadow = 1493 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1494 exit->exitstate.int_window_exiting = cpudata->int_window_exit; 1495 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit; 1496 exit->exitstate.evt_pending = cpudata->evt_pending; 1497 1498 return 0; 1499 } 1500 1501 /* -------------------------------------------------------------------------- */ 1502 1503 static int 1504 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1505 { 1506 struct pglist pglist; 1507 paddr_t _pa; 1508 vaddr_t _va; 1509 size_t i; 1510 int ret; 1511 1512 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1513 &pglist, 1, 0); 1514 if (ret != 0) 1515 return ENOMEM; 1516 _pa = TAILQ_FIRST(&pglist)->phys_addr; 1517 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1518 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1519 if (_va == 0) 1520 goto error; 1521 1522 for (i = 0; i < npages; i++) { 1523 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1524 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1525 } 1526 pmap_update(pmap_kernel()); 1527 1528 memset((void *)_va, 0, npages * PAGE_SIZE); 1529 1530 *pa = _pa; 1531 *va = _va; 1532 return 0; 1533 1534 error: 1535 for (i = 0; i < npages; i++) { 1536 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1537 } 1538 return ENOMEM; 1539 } 1540 1541 static void 1542 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1543 { 1544 size_t i; 1545 1546 pmap_kremove(va, npages * PAGE_SIZE); 1547 pmap_update(pmap_kernel()); 1548 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1549 for (i = 0; i < npages; i++) { 1550 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1551 } 1552 } 1553 1554 /* -------------------------------------------------------------------------- */ 1555 1556 #define SVM_MSRBM_READ __BIT(0) 1557 #define SVM_MSRBM_WRITE __BIT(1) 1558 1559 static void 1560 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1561 { 1562 uint64_t byte; 1563 uint8_t bitoff; 1564 1565 if (msr < 0x00002000) { 1566 /* Range 1 */ 1567 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1568 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1569 /* Range 2 */ 1570 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1571 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1572 /* Range 3 */ 1573 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1574 } else { 1575 panic("%s: wrong range", __func__); 1576 } 1577 1578 bitoff = (msr & 0x3) << 1; 1579 1580 if (read) { 1581 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1582 } 1583 if (write) { 1584 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1585 } 1586 } 1587 1588 #define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1589 #define SVM_SEG_ATTRIB_S __BIT(4) 1590 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1591 #define SVM_SEG_ATTRIB_P __BIT(7) 1592 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1593 #define SVM_SEG_ATTRIB_L __BIT(9) 1594 #define SVM_SEG_ATTRIB_DEF __BIT(10) 1595 #define SVM_SEG_ATTRIB_G __BIT(11) 1596 1597 static void 1598 svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1599 struct vmcb_segment *vseg) 1600 { 1601 vseg->selector = seg->selector; 1602 vseg->attrib = 1603 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1604 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1605 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1606 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1607 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1608 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1609 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1610 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1611 vseg->limit = seg->limit; 1612 vseg->base = seg->base; 1613 } 1614 1615 static void 1616 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1617 { 1618 seg->selector = vseg->selector; 1619 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1620 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1621 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1622 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1623 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1624 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1625 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1626 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1627 seg->limit = vseg->limit; 1628 seg->base = vseg->base; 1629 } 1630 1631 static inline bool 1632 svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1633 uint64_t flags) 1634 { 1635 if (flags & NVMM_X64_STATE_CRS) { 1636 if ((vmcb->state.cr0 ^ 1637 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1638 return true; 1639 } 1640 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1641 return true; 1642 } 1643 if ((vmcb->state.cr4 ^ 1644 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1645 return true; 1646 } 1647 } 1648 1649 if (flags & NVMM_X64_STATE_MSRS) { 1650 if ((vmcb->state.efer ^ 1651 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1652 return true; 1653 } 1654 } 1655 1656 return false; 1657 } 1658 1659 static void 1660 svm_vcpu_setstate(struct nvmm_cpu *vcpu) 1661 { 1662 struct nvmm_comm_page *comm = vcpu->comm; 1663 const struct nvmm_x64_state *state = &comm->state; 1664 struct svm_cpudata *cpudata = vcpu->cpudata; 1665 struct vmcb *vmcb = cpudata->vmcb; 1666 struct fxsave *fpustate; 1667 uint64_t flags; 1668 1669 flags = comm->state_wanted; 1670 1671 if (svm_state_tlb_flush(vmcb, state, flags)) { 1672 cpudata->gtlb_want_flush = true; 1673 } 1674 1675 if (flags & NVMM_X64_STATE_SEGS) { 1676 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1677 &vmcb->state.cs); 1678 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1679 &vmcb->state.ds); 1680 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1681 &vmcb->state.es); 1682 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1683 &vmcb->state.fs); 1684 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1685 &vmcb->state.gs); 1686 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1687 &vmcb->state.ss); 1688 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1689 &vmcb->state.gdt); 1690 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1691 &vmcb->state.idt); 1692 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1693 &vmcb->state.ldt); 1694 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1695 &vmcb->state.tr); 1696 1697 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1698 } 1699 1700 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1701 if (flags & NVMM_X64_STATE_GPRS) { 1702 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1703 1704 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1705 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1706 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1707 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1708 } 1709 1710 if (flags & NVMM_X64_STATE_CRS) { 1711 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1712 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1713 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1714 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1715 1716 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1717 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1718 VMCB_CTRL_V_TPR); 1719 1720 if (svm_xcr0_mask != 0) { 1721 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1722 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1723 cpudata->gxcr0 &= svm_xcr0_mask; 1724 cpudata->gxcr0 |= XCR0_X87; 1725 } 1726 } 1727 1728 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1729 if (flags & NVMM_X64_STATE_DRS) { 1730 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1731 1732 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1733 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1734 } 1735 1736 if (flags & NVMM_X64_STATE_MSRS) { 1737 /* 1738 * EFER_SVME is mandatory. 1739 */ 1740 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1741 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1742 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1743 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1744 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1745 vmcb->state.kernelgsbase = 1746 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1747 vmcb->state.sysenter_cs = 1748 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1749 vmcb->state.sysenter_esp = 1750 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1751 vmcb->state.sysenter_eip = 1752 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1753 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1754 1755 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 1756 cpudata->gtsc_want_update = true; 1757 } 1758 1759 if (flags & NVMM_X64_STATE_INTR) { 1760 if (state->intr.int_shadow) { 1761 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1762 } else { 1763 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1764 } 1765 1766 if (state->intr.int_window_exiting) { 1767 svm_event_waitexit_enable(vcpu, false); 1768 } else { 1769 svm_event_waitexit_disable(vcpu, false); 1770 } 1771 1772 if (state->intr.nmi_window_exiting) { 1773 svm_event_waitexit_enable(vcpu, true); 1774 } else { 1775 svm_event_waitexit_disable(vcpu, true); 1776 } 1777 } 1778 1779 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1780 if (flags & NVMM_X64_STATE_FPU) { 1781 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1782 sizeof(state->fpu)); 1783 1784 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1785 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1786 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1787 1788 if (svm_xcr0_mask != 0) { 1789 /* Reset XSTATE_BV, to force a reload. */ 1790 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1791 } 1792 } 1793 1794 svm_vmcb_cache_update(vmcb, flags); 1795 1796 comm->state_wanted = 0; 1797 comm->state_cached |= flags; 1798 } 1799 1800 static void 1801 svm_vcpu_getstate(struct nvmm_cpu *vcpu) 1802 { 1803 struct nvmm_comm_page *comm = vcpu->comm; 1804 struct nvmm_x64_state *state = &comm->state; 1805 struct svm_cpudata *cpudata = vcpu->cpudata; 1806 struct vmcb *vmcb = cpudata->vmcb; 1807 uint64_t flags; 1808 1809 flags = comm->state_wanted; 1810 1811 if (flags & NVMM_X64_STATE_SEGS) { 1812 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1813 &vmcb->state.cs); 1814 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1815 &vmcb->state.ds); 1816 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1817 &vmcb->state.es); 1818 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1819 &vmcb->state.fs); 1820 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1821 &vmcb->state.gs); 1822 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1823 &vmcb->state.ss); 1824 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1825 &vmcb->state.gdt); 1826 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1827 &vmcb->state.idt); 1828 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1829 &vmcb->state.ldt); 1830 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1831 &vmcb->state.tr); 1832 1833 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1834 } 1835 1836 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1837 if (flags & NVMM_X64_STATE_GPRS) { 1838 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1839 1840 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1841 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1842 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1843 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1844 } 1845 1846 if (flags & NVMM_X64_STATE_CRS) { 1847 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1848 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1849 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1850 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1851 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1852 VMCB_CTRL_V_TPR); 1853 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1854 } 1855 1856 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1857 if (flags & NVMM_X64_STATE_DRS) { 1858 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1859 1860 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1861 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1862 } 1863 1864 if (flags & NVMM_X64_STATE_MSRS) { 1865 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1866 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1867 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1868 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1869 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1870 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1871 vmcb->state.kernelgsbase; 1872 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1873 vmcb->state.sysenter_cs; 1874 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1875 vmcb->state.sysenter_esp; 1876 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1877 vmcb->state.sysenter_eip; 1878 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1879 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 1880 1881 /* Hide SVME. */ 1882 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1883 } 1884 1885 if (flags & NVMM_X64_STATE_INTR) { 1886 state->intr.int_shadow = 1887 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1888 state->intr.int_window_exiting = cpudata->int_window_exit; 1889 state->intr.nmi_window_exiting = cpudata->nmi_window_exit; 1890 state->intr.evt_pending = cpudata->evt_pending; 1891 } 1892 1893 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1894 if (flags & NVMM_X64_STATE_FPU) { 1895 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 1896 sizeof(state->fpu)); 1897 } 1898 1899 comm->state_wanted = 0; 1900 comm->state_cached |= flags; 1901 } 1902 1903 static void 1904 svm_vcpu_state_provide(struct nvmm_cpu *vcpu, uint64_t flags) 1905 { 1906 vcpu->comm->state_wanted = flags; 1907 svm_vcpu_getstate(vcpu); 1908 } 1909 1910 static void 1911 svm_vcpu_state_commit(struct nvmm_cpu *vcpu) 1912 { 1913 vcpu->comm->state_wanted = vcpu->comm->state_commit; 1914 vcpu->comm->state_commit = 0; 1915 svm_vcpu_setstate(vcpu); 1916 } 1917 1918 /* -------------------------------------------------------------------------- */ 1919 1920 static void 1921 svm_asid_alloc(struct nvmm_cpu *vcpu) 1922 { 1923 struct svm_cpudata *cpudata = vcpu->cpudata; 1924 struct vmcb *vmcb = cpudata->vmcb; 1925 size_t i, oct, bit; 1926 1927 mutex_enter(&svm_asidlock); 1928 1929 for (i = 0; i < svm_maxasid; i++) { 1930 oct = i / 8; 1931 bit = i % 8; 1932 1933 if (svm_asidmap[oct] & __BIT(bit)) { 1934 continue; 1935 } 1936 1937 svm_asidmap[oct] |= __BIT(bit); 1938 vmcb->ctrl.guest_asid = i; 1939 mutex_exit(&svm_asidlock); 1940 return; 1941 } 1942 1943 /* 1944 * No free ASID. Use the last one, which is shared and requires 1945 * special TLB handling. 1946 */ 1947 cpudata->shared_asid = true; 1948 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1949 mutex_exit(&svm_asidlock); 1950 } 1951 1952 static void 1953 svm_asid_free(struct nvmm_cpu *vcpu) 1954 { 1955 struct svm_cpudata *cpudata = vcpu->cpudata; 1956 struct vmcb *vmcb = cpudata->vmcb; 1957 size_t oct, bit; 1958 1959 if (cpudata->shared_asid) { 1960 return; 1961 } 1962 1963 oct = vmcb->ctrl.guest_asid / 8; 1964 bit = vmcb->ctrl.guest_asid % 8; 1965 1966 mutex_enter(&svm_asidlock); 1967 svm_asidmap[oct] &= ~__BIT(bit); 1968 mutex_exit(&svm_asidlock); 1969 } 1970 1971 static void 1972 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1973 { 1974 struct svm_cpudata *cpudata = vcpu->cpudata; 1975 struct vmcb *vmcb = cpudata->vmcb; 1976 1977 /* Allow reads/writes of Control Registers. */ 1978 vmcb->ctrl.intercept_cr = 0; 1979 1980 /* Allow reads/writes of Debug Registers. */ 1981 vmcb->ctrl.intercept_dr = 0; 1982 1983 /* Allow exceptions 0 to 31. */ 1984 vmcb->ctrl.intercept_vec = 0; 1985 1986 /* 1987 * Allow: 1988 * - SMI [smm interrupts] 1989 * - VINTR [virtual interrupts] 1990 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 1991 * - RIDTR [reads of IDTR] 1992 * - RGDTR [reads of GDTR] 1993 * - RLDTR [reads of LDTR] 1994 * - RTR [reads of TR] 1995 * - WIDTR [writes of IDTR] 1996 * - WGDTR [writes of GDTR] 1997 * - WLDTR [writes of LDTR] 1998 * - WTR [writes of TR] 1999 * - RDTSC [rdtsc instruction] 2000 * - PUSHF [pushf instruction] 2001 * - POPF [popf instruction] 2002 * - IRET [iret instruction] 2003 * - INTN [int $n instructions] 2004 * - INVD [invd instruction] 2005 * - PAUSE [pause instruction] 2006 * - INVLPG [invplg instruction] 2007 * - TASKSW [task switches] 2008 * 2009 * Intercept the rest below. 2010 */ 2011 vmcb->ctrl.intercept_misc1 = 2012 VMCB_CTRL_INTERCEPT_INTR | 2013 VMCB_CTRL_INTERCEPT_NMI | 2014 VMCB_CTRL_INTERCEPT_INIT | 2015 VMCB_CTRL_INTERCEPT_RDPMC | 2016 VMCB_CTRL_INTERCEPT_CPUID | 2017 VMCB_CTRL_INTERCEPT_RSM | 2018 VMCB_CTRL_INTERCEPT_HLT | 2019 VMCB_CTRL_INTERCEPT_INVLPGA | 2020 VMCB_CTRL_INTERCEPT_IOIO_PROT | 2021 VMCB_CTRL_INTERCEPT_MSR_PROT | 2022 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 2023 VMCB_CTRL_INTERCEPT_SHUTDOWN; 2024 2025 /* 2026 * Allow: 2027 * - ICEBP [icebp instruction] 2028 * - WBINVD [wbinvd instruction] 2029 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 2030 * 2031 * Intercept the rest below. 2032 */ 2033 vmcb->ctrl.intercept_misc2 = 2034 VMCB_CTRL_INTERCEPT_VMRUN | 2035 VMCB_CTRL_INTERCEPT_VMMCALL | 2036 VMCB_CTRL_INTERCEPT_VMLOAD | 2037 VMCB_CTRL_INTERCEPT_VMSAVE | 2038 VMCB_CTRL_INTERCEPT_STGI | 2039 VMCB_CTRL_INTERCEPT_CLGI | 2040 VMCB_CTRL_INTERCEPT_SKINIT | 2041 VMCB_CTRL_INTERCEPT_RDTSCP | 2042 VMCB_CTRL_INTERCEPT_MONITOR | 2043 VMCB_CTRL_INTERCEPT_MWAIT | 2044 VMCB_CTRL_INTERCEPT_XSETBV; 2045 2046 /* Intercept all I/O accesses. */ 2047 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 2048 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 2049 2050 /* Allow direct access to certain MSRs. */ 2051 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 2052 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 2053 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 2054 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 2055 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 2056 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 2057 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 2058 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 2059 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 2060 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 2061 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 2062 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 2063 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 2064 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 2065 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 2066 2067 /* Generate ASID. */ 2068 svm_asid_alloc(vcpu); 2069 2070 /* Virtual TPR. */ 2071 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 2072 2073 /* Enable Nested Paging. */ 2074 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 2075 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 2076 2077 /* Init XSAVE header. */ 2078 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 2079 cpudata->gfpu.xsh_xcomp_bv = 0; 2080 2081 /* These MSRs are static. */ 2082 cpudata->star = rdmsr(MSR_STAR); 2083 cpudata->lstar = rdmsr(MSR_LSTAR); 2084 cpudata->cstar = rdmsr(MSR_CSTAR); 2085 cpudata->sfmask = rdmsr(MSR_SFMASK); 2086 2087 /* Install the RESET state. */ 2088 memcpy(&vcpu->comm->state, &nvmm_x86_reset_state, 2089 sizeof(nvmm_x86_reset_state)); 2090 vcpu->comm->state_wanted = NVMM_X64_STATE_ALL; 2091 vcpu->comm->state_cached = 0; 2092 svm_vcpu_setstate(vcpu); 2093 } 2094 2095 static int 2096 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2097 { 2098 struct svm_cpudata *cpudata; 2099 int error; 2100 2101 /* Allocate the SVM cpudata. */ 2102 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 2103 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 2104 UVM_KMF_WIRED|UVM_KMF_ZERO); 2105 vcpu->cpudata = cpudata; 2106 2107 /* VMCB */ 2108 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 2109 VMCB_NPAGES); 2110 if (error) 2111 goto error; 2112 2113 /* I/O Bitmap */ 2114 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 2115 IOBM_NPAGES); 2116 if (error) 2117 goto error; 2118 2119 /* MSR Bitmap */ 2120 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 2121 MSRBM_NPAGES); 2122 if (error) 2123 goto error; 2124 2125 /* Init the VCPU info. */ 2126 svm_vcpu_init(mach, vcpu); 2127 2128 return 0; 2129 2130 error: 2131 if (cpudata->vmcb_pa) { 2132 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2133 VMCB_NPAGES); 2134 } 2135 if (cpudata->iobm_pa) { 2136 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2137 IOBM_NPAGES); 2138 } 2139 if (cpudata->msrbm_pa) { 2140 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2141 MSRBM_NPAGES); 2142 } 2143 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2144 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2145 return error; 2146 } 2147 2148 static void 2149 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2150 { 2151 struct svm_cpudata *cpudata = vcpu->cpudata; 2152 2153 svm_asid_free(vcpu); 2154 2155 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2156 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2157 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2158 2159 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2160 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2161 } 2162 2163 /* -------------------------------------------------------------------------- */ 2164 2165 static int 2166 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data) 2167 { 2168 struct nvmm_vcpu_conf_cpuid *cpuid = data; 2169 size_t i; 2170 2171 if (__predict_false(cpuid->mask && cpuid->exit)) { 2172 return EINVAL; 2173 } 2174 if (__predict_false(cpuid->mask && 2175 ((cpuid->u.mask.set.eax & cpuid->u.mask.del.eax) || 2176 (cpuid->u.mask.set.ebx & cpuid->u.mask.del.ebx) || 2177 (cpuid->u.mask.set.ecx & cpuid->u.mask.del.ecx) || 2178 (cpuid->u.mask.set.edx & cpuid->u.mask.del.edx)))) { 2179 return EINVAL; 2180 } 2181 2182 /* If unset, delete, to restore the default behavior. */ 2183 if (!cpuid->mask && !cpuid->exit) { 2184 for (i = 0; i < SVM_NCPUIDS; i++) { 2185 if (!cpudata->cpuidpresent[i]) { 2186 continue; 2187 } 2188 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2189 cpudata->cpuidpresent[i] = false; 2190 } 2191 } 2192 return 0; 2193 } 2194 2195 /* If already here, replace. */ 2196 for (i = 0; i < SVM_NCPUIDS; i++) { 2197 if (!cpudata->cpuidpresent[i]) { 2198 continue; 2199 } 2200 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2201 memcpy(&cpudata->cpuid[i], cpuid, 2202 sizeof(struct nvmm_vcpu_conf_cpuid)); 2203 return 0; 2204 } 2205 } 2206 2207 /* Not here, insert. */ 2208 for (i = 0; i < SVM_NCPUIDS; i++) { 2209 if (!cpudata->cpuidpresent[i]) { 2210 cpudata->cpuidpresent[i] = true; 2211 memcpy(&cpudata->cpuid[i], cpuid, 2212 sizeof(struct nvmm_vcpu_conf_cpuid)); 2213 return 0; 2214 } 2215 } 2216 2217 return ENOBUFS; 2218 } 2219 2220 static int 2221 svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) 2222 { 2223 struct svm_cpudata *cpudata = vcpu->cpudata; 2224 2225 switch (op) { 2226 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): 2227 return svm_vcpu_configure_cpuid(cpudata, data); 2228 default: 2229 return EINVAL; 2230 } 2231 } 2232 2233 /* -------------------------------------------------------------------------- */ 2234 2235 static void 2236 svm_tlb_flush(struct pmap *pm) 2237 { 2238 struct nvmm_machine *mach = pm->pm_data; 2239 struct svm_machdata *machdata = mach->machdata; 2240 2241 atomic_inc_64(&machdata->mach_htlb_gen); 2242 2243 /* Generates IPIs, which cause #VMEXITs. */ 2244 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_UPDATE); 2245 } 2246 2247 static void 2248 svm_machine_create(struct nvmm_machine *mach) 2249 { 2250 struct svm_machdata *machdata; 2251 2252 /* Fill in pmap info. */ 2253 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2254 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2255 2256 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2257 mach->machdata = machdata; 2258 2259 /* Start with an hTLB flush everywhere. */ 2260 machdata->mach_htlb_gen = 1; 2261 } 2262 2263 static void 2264 svm_machine_destroy(struct nvmm_machine *mach) 2265 { 2266 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2267 } 2268 2269 static int 2270 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2271 { 2272 panic("%s: impossible", __func__); 2273 } 2274 2275 /* -------------------------------------------------------------------------- */ 2276 2277 static bool 2278 svm_ident(void) 2279 { 2280 u_int descs[4]; 2281 uint64_t msr; 2282 2283 if (cpu_vendor != CPUVENDOR_AMD) { 2284 return false; 2285 } 2286 if (!(cpu_feature[3] & CPUID_SVM)) { 2287 printf("NVMM: SVM not supported\n"); 2288 return false; 2289 } 2290 2291 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2292 printf("NVMM: CPUID leaf not available\n"); 2293 return false; 2294 } 2295 x86_cpuid(0x8000000a, descs); 2296 2297 /* Want Nested Paging. */ 2298 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2299 printf("NVMM: SVM-NP not supported\n"); 2300 return false; 2301 } 2302 2303 /* Want nRIP. */ 2304 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2305 printf("NVMM: SVM-NRIPS not supported\n"); 2306 return false; 2307 } 2308 2309 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2310 2311 msr = rdmsr(MSR_VMCR); 2312 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2313 printf("NVMM: SVM disabled in BIOS\n"); 2314 return false; 2315 } 2316 2317 return true; 2318 } 2319 2320 static void 2321 svm_init_asid(uint32_t maxasid) 2322 { 2323 size_t i, j, allocsz; 2324 2325 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2326 2327 /* Arbitrarily limit. */ 2328 maxasid = uimin(maxasid, 8192); 2329 2330 svm_maxasid = maxasid; 2331 allocsz = roundup(maxasid, 8) / 8; 2332 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2333 2334 /* ASID 0 is reserved for the host. */ 2335 svm_asidmap[0] |= __BIT(0); 2336 2337 /* ASID n-1 is special, we share it. */ 2338 i = (maxasid - 1) / 8; 2339 j = (maxasid - 1) % 8; 2340 svm_asidmap[i] |= __BIT(j); 2341 } 2342 2343 static void 2344 svm_change_cpu(void *arg1, void *arg2) 2345 { 2346 bool enable = (bool)arg1; 2347 uint64_t msr; 2348 2349 msr = rdmsr(MSR_VMCR); 2350 if (msr & VMCR_SVMED) { 2351 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2352 } 2353 2354 if (!enable) { 2355 wrmsr(MSR_VM_HSAVE_PA, 0); 2356 } 2357 2358 msr = rdmsr(MSR_EFER); 2359 if (enable) { 2360 msr |= EFER_SVME; 2361 } else { 2362 msr &= ~EFER_SVME; 2363 } 2364 wrmsr(MSR_EFER, msr); 2365 2366 if (enable) { 2367 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2368 } 2369 } 2370 2371 static void 2372 svm_init(void) 2373 { 2374 CPU_INFO_ITERATOR cii; 2375 struct cpu_info *ci; 2376 struct vm_page *pg; 2377 u_int descs[4]; 2378 uint64_t xc; 2379 2380 x86_cpuid(0x8000000a, descs); 2381 2382 /* The guest TLB flush command. */ 2383 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2384 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2385 } else { 2386 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2387 } 2388 2389 /* Init the ASID. */ 2390 svm_init_asid(descs[1]); 2391 2392 /* Init the XCR0 mask. */ 2393 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2394 2395 memset(hsave, 0, sizeof(hsave)); 2396 for (CPU_INFO_FOREACH(cii, ci)) { 2397 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2398 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2399 } 2400 2401 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2402 xc_wait(xc); 2403 } 2404 2405 static void 2406 svm_fini_asid(void) 2407 { 2408 size_t allocsz; 2409 2410 allocsz = roundup(svm_maxasid, 8) / 8; 2411 kmem_free(svm_asidmap, allocsz); 2412 2413 mutex_destroy(&svm_asidlock); 2414 } 2415 2416 static void 2417 svm_fini(void) 2418 { 2419 uint64_t xc; 2420 size_t i; 2421 2422 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2423 xc_wait(xc); 2424 2425 for (i = 0; i < MAXCPUS; i++) { 2426 if (hsave[i].pa != 0) 2427 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2428 } 2429 2430 svm_fini_asid(); 2431 } 2432 2433 static void 2434 svm_capability(struct nvmm_capability *cap) 2435 { 2436 cap->arch.mach_conf_support = 0; 2437 cap->arch.vcpu_conf_support = 2438 NVMM_CAP_ARCH_VCPU_CONF_CPUID; 2439 cap->arch.xcr0_mask = svm_xcr0_mask; 2440 cap->arch.mxcsr_mask = x86_fpu_mxcsr_mask; 2441 cap->arch.conf_cpuid_maxops = SVM_NCPUIDS; 2442 } 2443 2444 const struct nvmm_impl nvmm_x86_svm = { 2445 .ident = svm_ident, 2446 .init = svm_init, 2447 .fini = svm_fini, 2448 .capability = svm_capability, 2449 .mach_conf_max = NVMM_X86_MACH_NCONF, 2450 .mach_conf_sizes = NULL, 2451 .vcpu_conf_max = NVMM_X86_VCPU_NCONF, 2452 .vcpu_conf_sizes = svm_vcpu_conf_sizes, 2453 .state_size = sizeof(struct nvmm_x64_state), 2454 .machine_create = svm_machine_create, 2455 .machine_destroy = svm_machine_destroy, 2456 .machine_configure = svm_machine_configure, 2457 .vcpu_create = svm_vcpu_create, 2458 .vcpu_destroy = svm_vcpu_destroy, 2459 .vcpu_configure = svm_vcpu_configure, 2460 .vcpu_setstate = svm_vcpu_setstate, 2461 .vcpu_getstate = svm_vcpu_getstate, 2462 .vcpu_inject = svm_vcpu_inject, 2463 .vcpu_run = svm_vcpu_run 2464 }; 2465
Indexes created Wed Oct 29 18:09:52 GMT 2025