nvmm_x86_svm.c revision 1.46.4.8
1 /* $NetBSD: nvmm_x86_svm.c,v 1.46.4.8 2020/08/18 09:29:52 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.8 2020/08/18 09:29:52 martin Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/kmem.h> 39 #include <sys/cpu.h> 40 #include <sys/xcall.h> 41 #include <sys/mman.h> 42 43 #include <uvm/uvm.h> 44 #include <uvm/uvm_page.h> 45 46 #include <x86/cputypes.h> 47 #include <x86/specialreg.h> 48 #include <x86/pmap.h> 49 #include <x86/dbregs.h> 50 #include <x86/cpu_counter.h> 51 #include <machine/cpuvar.h> 52 53 #include <dev/nvmm/nvmm.h> 54 #include <dev/nvmm/nvmm_internal.h> 55 #include <dev/nvmm/x86/nvmm_x86.h> 56 57 int svm_vmrun(paddr_t, uint64_t *); 58 59 #define MSR_VM_HSAVE_PA 0xC0010117 60 61 /* -------------------------------------------------------------------------- */ 62 63 #define VMCB_EXITCODE_CR0_READ 0x0000 64 #define VMCB_EXITCODE_CR1_READ 0x0001 65 #define VMCB_EXITCODE_CR2_READ 0x0002 66 #define VMCB_EXITCODE_CR3_READ 0x0003 67 #define VMCB_EXITCODE_CR4_READ 0x0004 68 #define VMCB_EXITCODE_CR5_READ 0x0005 69 #define VMCB_EXITCODE_CR6_READ 0x0006 70 #define VMCB_EXITCODE_CR7_READ 0x0007 71 #define VMCB_EXITCODE_CR8_READ 0x0008 72 #define VMCB_EXITCODE_CR9_READ 0x0009 73 #define VMCB_EXITCODE_CR10_READ 0x000A 74 #define VMCB_EXITCODE_CR11_READ 0x000B 75 #define VMCB_EXITCODE_CR12_READ 0x000C 76 #define VMCB_EXITCODE_CR13_READ 0x000D 77 #define VMCB_EXITCODE_CR14_READ 0x000E 78 #define VMCB_EXITCODE_CR15_READ 0x000F 79 #define VMCB_EXITCODE_CR0_WRITE 0x0010 80 #define VMCB_EXITCODE_CR1_WRITE 0x0011 81 #define VMCB_EXITCODE_CR2_WRITE 0x0012 82 #define VMCB_EXITCODE_CR3_WRITE 0x0013 83 #define VMCB_EXITCODE_CR4_WRITE 0x0014 84 #define VMCB_EXITCODE_CR5_WRITE 0x0015 85 #define VMCB_EXITCODE_CR6_WRITE 0x0016 86 #define VMCB_EXITCODE_CR7_WRITE 0x0017 87 #define VMCB_EXITCODE_CR8_WRITE 0x0018 88 #define VMCB_EXITCODE_CR9_WRITE 0x0019 89 #define VMCB_EXITCODE_CR10_WRITE 0x001A 90 #define VMCB_EXITCODE_CR11_WRITE 0x001B 91 #define VMCB_EXITCODE_CR12_WRITE 0x001C 92 #define VMCB_EXITCODE_CR13_WRITE 0x001D 93 #define VMCB_EXITCODE_CR14_WRITE 0x001E 94 #define VMCB_EXITCODE_CR15_WRITE 0x001F 95 #define VMCB_EXITCODE_DR0_READ 0x0020 96 #define VMCB_EXITCODE_DR1_READ 0x0021 97 #define VMCB_EXITCODE_DR2_READ 0x0022 98 #define VMCB_EXITCODE_DR3_READ 0x0023 99 #define VMCB_EXITCODE_DR4_READ 0x0024 100 #define VMCB_EXITCODE_DR5_READ 0x0025 101 #define VMCB_EXITCODE_DR6_READ 0x0026 102 #define VMCB_EXITCODE_DR7_READ 0x0027 103 #define VMCB_EXITCODE_DR8_READ 0x0028 104 #define VMCB_EXITCODE_DR9_READ 0x0029 105 #define VMCB_EXITCODE_DR10_READ 0x002A 106 #define VMCB_EXITCODE_DR11_READ 0x002B 107 #define VMCB_EXITCODE_DR12_READ 0x002C 108 #define VMCB_EXITCODE_DR13_READ 0x002D 109 #define VMCB_EXITCODE_DR14_READ 0x002E 110 #define VMCB_EXITCODE_DR15_READ 0x002F 111 #define VMCB_EXITCODE_DR0_WRITE 0x0030 112 #define VMCB_EXITCODE_DR1_WRITE 0x0031 113 #define VMCB_EXITCODE_DR2_WRITE 0x0032 114 #define VMCB_EXITCODE_DR3_WRITE 0x0033 115 #define VMCB_EXITCODE_DR4_WRITE 0x0034 116 #define VMCB_EXITCODE_DR5_WRITE 0x0035 117 #define VMCB_EXITCODE_DR6_WRITE 0x0036 118 #define VMCB_EXITCODE_DR7_WRITE 0x0037 119 #define VMCB_EXITCODE_DR8_WRITE 0x0038 120 #define VMCB_EXITCODE_DR9_WRITE 0x0039 121 #define VMCB_EXITCODE_DR10_WRITE 0x003A 122 #define VMCB_EXITCODE_DR11_WRITE 0x003B 123 #define VMCB_EXITCODE_DR12_WRITE 0x003C 124 #define VMCB_EXITCODE_DR13_WRITE 0x003D 125 #define VMCB_EXITCODE_DR14_WRITE 0x003E 126 #define VMCB_EXITCODE_DR15_WRITE 0x003F 127 #define VMCB_EXITCODE_EXCP0 0x0040 128 #define VMCB_EXITCODE_EXCP1 0x0041 129 #define VMCB_EXITCODE_EXCP2 0x0042 130 #define VMCB_EXITCODE_EXCP3 0x0043 131 #define VMCB_EXITCODE_EXCP4 0x0044 132 #define VMCB_EXITCODE_EXCP5 0x0045 133 #define VMCB_EXITCODE_EXCP6 0x0046 134 #define VMCB_EXITCODE_EXCP7 0x0047 135 #define VMCB_EXITCODE_EXCP8 0x0048 136 #define VMCB_EXITCODE_EXCP9 0x0049 137 #define VMCB_EXITCODE_EXCP10 0x004A 138 #define VMCB_EXITCODE_EXCP11 0x004B 139 #define VMCB_EXITCODE_EXCP12 0x004C 140 #define VMCB_EXITCODE_EXCP13 0x004D 141 #define VMCB_EXITCODE_EXCP14 0x004E 142 #define VMCB_EXITCODE_EXCP15 0x004F 143 #define VMCB_EXITCODE_EXCP16 0x0050 144 #define VMCB_EXITCODE_EXCP17 0x0051 145 #define VMCB_EXITCODE_EXCP18 0x0052 146 #define VMCB_EXITCODE_EXCP19 0x0053 147 #define VMCB_EXITCODE_EXCP20 0x0054 148 #define VMCB_EXITCODE_EXCP21 0x0055 149 #define VMCB_EXITCODE_EXCP22 0x0056 150 #define VMCB_EXITCODE_EXCP23 0x0057 151 #define VMCB_EXITCODE_EXCP24 0x0058 152 #define VMCB_EXITCODE_EXCP25 0x0059 153 #define VMCB_EXITCODE_EXCP26 0x005A 154 #define VMCB_EXITCODE_EXCP27 0x005B 155 #define VMCB_EXITCODE_EXCP28 0x005C 156 #define VMCB_EXITCODE_EXCP29 0x005D 157 #define VMCB_EXITCODE_EXCP30 0x005E 158 #define VMCB_EXITCODE_EXCP31 0x005F 159 #define VMCB_EXITCODE_INTR 0x0060 160 #define VMCB_EXITCODE_NMI 0x0061 161 #define VMCB_EXITCODE_SMI 0x0062 162 #define VMCB_EXITCODE_INIT 0x0063 163 #define VMCB_EXITCODE_VINTR 0x0064 164 #define VMCB_EXITCODE_CR0_SEL_WRITE 0x0065 165 #define VMCB_EXITCODE_IDTR_READ 0x0066 166 #define VMCB_EXITCODE_GDTR_READ 0x0067 167 #define VMCB_EXITCODE_LDTR_READ 0x0068 168 #define VMCB_EXITCODE_TR_READ 0x0069 169 #define VMCB_EXITCODE_IDTR_WRITE 0x006A 170 #define VMCB_EXITCODE_GDTR_WRITE 0x006B 171 #define VMCB_EXITCODE_LDTR_WRITE 0x006C 172 #define VMCB_EXITCODE_TR_WRITE 0x006D 173 #define VMCB_EXITCODE_RDTSC 0x006E 174 #define VMCB_EXITCODE_RDPMC 0x006F 175 #define VMCB_EXITCODE_PUSHF 0x0070 176 #define VMCB_EXITCODE_POPF 0x0071 177 #define VMCB_EXITCODE_CPUID 0x0072 178 #define VMCB_EXITCODE_RSM 0x0073 179 #define VMCB_EXITCODE_IRET 0x0074 180 #define VMCB_EXITCODE_SWINT 0x0075 181 #define VMCB_EXITCODE_INVD 0x0076 182 #define VMCB_EXITCODE_PAUSE 0x0077 183 #define VMCB_EXITCODE_HLT 0x0078 184 #define VMCB_EXITCODE_INVLPG 0x0079 185 #define VMCB_EXITCODE_INVLPGA 0x007A 186 #define VMCB_EXITCODE_IOIO 0x007B 187 #define VMCB_EXITCODE_MSR 0x007C 188 #define VMCB_EXITCODE_TASK_SWITCH 0x007D 189 #define VMCB_EXITCODE_FERR_FREEZE 0x007E 190 #define VMCB_EXITCODE_SHUTDOWN 0x007F 191 #define VMCB_EXITCODE_VMRUN 0x0080 192 #define VMCB_EXITCODE_VMMCALL 0x0081 193 #define VMCB_EXITCODE_VMLOAD 0x0082 194 #define VMCB_EXITCODE_VMSAVE 0x0083 195 #define VMCB_EXITCODE_STGI 0x0084 196 #define VMCB_EXITCODE_CLGI 0x0085 197 #define VMCB_EXITCODE_SKINIT 0x0086 198 #define VMCB_EXITCODE_RDTSCP 0x0087 199 #define VMCB_EXITCODE_ICEBP 0x0088 200 #define VMCB_EXITCODE_WBINVD 0x0089 201 #define VMCB_EXITCODE_MONITOR 0x008A 202 #define VMCB_EXITCODE_MWAIT 0x008B 203 #define VMCB_EXITCODE_MWAIT_CONDITIONAL 0x008C 204 #define VMCB_EXITCODE_XSETBV 0x008D 205 #define VMCB_EXITCODE_RDPRU 0x008E 206 #define VMCB_EXITCODE_EFER_WRITE_TRAP 0x008F 207 #define VMCB_EXITCODE_CR0_WRITE_TRAP 0x0090 208 #define VMCB_EXITCODE_CR1_WRITE_TRAP 0x0091 209 #define VMCB_EXITCODE_CR2_WRITE_TRAP 0x0092 210 #define VMCB_EXITCODE_CR3_WRITE_TRAP 0x0093 211 #define VMCB_EXITCODE_CR4_WRITE_TRAP 0x0094 212 #define VMCB_EXITCODE_CR5_WRITE_TRAP 0x0095 213 #define VMCB_EXITCODE_CR6_WRITE_TRAP 0x0096 214 #define VMCB_EXITCODE_CR7_WRITE_TRAP 0x0097 215 #define VMCB_EXITCODE_CR8_WRITE_TRAP 0x0098 216 #define VMCB_EXITCODE_CR9_WRITE_TRAP 0x0099 217 #define VMCB_EXITCODE_CR10_WRITE_TRAP 0x009A 218 #define VMCB_EXITCODE_CR11_WRITE_TRAP 0x009B 219 #define VMCB_EXITCODE_CR12_WRITE_TRAP 0x009C 220 #define VMCB_EXITCODE_CR13_WRITE_TRAP 0x009D 221 #define VMCB_EXITCODE_CR14_WRITE_TRAP 0x009E 222 #define VMCB_EXITCODE_CR15_WRITE_TRAP 0x009F 223 #define VMCB_EXITCODE_INVLPGB 0x00A0 224 #define VMCB_EXITCODE_INVLPGB_ILLEGAL 0x00A1 225 #define VMCB_EXITCODE_INVPCID 0x00A2 226 #define VMCB_EXITCODE_MCOMMIT 0x00A3 227 #define VMCB_EXITCODE_TLBSYNC 0x00A4 228 #define VMCB_EXITCODE_NPF 0x0400 229 #define VMCB_EXITCODE_AVIC_INCOMP_IPI 0x0401 230 #define VMCB_EXITCODE_AVIC_NOACCEL 0x0402 231 #define VMCB_EXITCODE_VMGEXIT 0x0403 232 #define VMCB_EXITCODE_BUSY -2ULL 233 #define VMCB_EXITCODE_INVALID -1ULL 234 235 /* -------------------------------------------------------------------------- */ 236 237 struct vmcb_ctrl { 238 uint32_t intercept_cr; 239 #define VMCB_CTRL_INTERCEPT_RCR(x) __BIT( 0 + x) 240 #define VMCB_CTRL_INTERCEPT_WCR(x) __BIT(16 + x) 241 242 uint32_t intercept_dr; 243 #define VMCB_CTRL_INTERCEPT_RDR(x) __BIT( 0 + x) 244 #define VMCB_CTRL_INTERCEPT_WDR(x) __BIT(16 + x) 245 246 uint32_t intercept_vec; 247 #define VMCB_CTRL_INTERCEPT_VEC(x) __BIT(x) 248 249 uint32_t intercept_misc1; 250 #define VMCB_CTRL_INTERCEPT_INTR __BIT(0) 251 #define VMCB_CTRL_INTERCEPT_NMI __BIT(1) 252 #define VMCB_CTRL_INTERCEPT_SMI __BIT(2) 253 #define VMCB_CTRL_INTERCEPT_INIT __BIT(3) 254 #define VMCB_CTRL_INTERCEPT_VINTR __BIT(4) 255 #define VMCB_CTRL_INTERCEPT_CR0_SPEC __BIT(5) 256 #define VMCB_CTRL_INTERCEPT_RIDTR __BIT(6) 257 #define VMCB_CTRL_INTERCEPT_RGDTR __BIT(7) 258 #define VMCB_CTRL_INTERCEPT_RLDTR __BIT(8) 259 #define VMCB_CTRL_INTERCEPT_RTR __BIT(9) 260 #define VMCB_CTRL_INTERCEPT_WIDTR __BIT(10) 261 #define VMCB_CTRL_INTERCEPT_WGDTR __BIT(11) 262 #define VMCB_CTRL_INTERCEPT_WLDTR __BIT(12) 263 #define VMCB_CTRL_INTERCEPT_WTR __BIT(13) 264 #define VMCB_CTRL_INTERCEPT_RDTSC __BIT(14) 265 #define VMCB_CTRL_INTERCEPT_RDPMC __BIT(15) 266 #define VMCB_CTRL_INTERCEPT_PUSHF __BIT(16) 267 #define VMCB_CTRL_INTERCEPT_POPF __BIT(17) 268 #define VMCB_CTRL_INTERCEPT_CPUID __BIT(18) 269 #define VMCB_CTRL_INTERCEPT_RSM __BIT(19) 270 #define VMCB_CTRL_INTERCEPT_IRET __BIT(20) 271 #define VMCB_CTRL_INTERCEPT_INTN __BIT(21) 272 #define VMCB_CTRL_INTERCEPT_INVD __BIT(22) 273 #define VMCB_CTRL_INTERCEPT_PAUSE __BIT(23) 274 #define VMCB_CTRL_INTERCEPT_HLT __BIT(24) 275 #define VMCB_CTRL_INTERCEPT_INVLPG __BIT(25) 276 #define VMCB_CTRL_INTERCEPT_INVLPGA __BIT(26) 277 #define VMCB_CTRL_INTERCEPT_IOIO_PROT __BIT(27) 278 #define VMCB_CTRL_INTERCEPT_MSR_PROT __BIT(28) 279 #define VMCB_CTRL_INTERCEPT_TASKSW __BIT(29) 280 #define VMCB_CTRL_INTERCEPT_FERR_FREEZE __BIT(30) 281 #define VMCB_CTRL_INTERCEPT_SHUTDOWN __BIT(31) 282 283 uint32_t intercept_misc2; 284 #define VMCB_CTRL_INTERCEPT_VMRUN __BIT(0) 285 #define VMCB_CTRL_INTERCEPT_VMMCALL __BIT(1) 286 #define VMCB_CTRL_INTERCEPT_VMLOAD __BIT(2) 287 #define VMCB_CTRL_INTERCEPT_VMSAVE __BIT(3) 288 #define VMCB_CTRL_INTERCEPT_STGI __BIT(4) 289 #define VMCB_CTRL_INTERCEPT_CLGI __BIT(5) 290 #define VMCB_CTRL_INTERCEPT_SKINIT __BIT(6) 291 #define VMCB_CTRL_INTERCEPT_RDTSCP __BIT(7) 292 #define VMCB_CTRL_INTERCEPT_ICEBP __BIT(8) 293 #define VMCB_CTRL_INTERCEPT_WBINVD __BIT(9) 294 #define VMCB_CTRL_INTERCEPT_MONITOR __BIT(10) 295 #define VMCB_CTRL_INTERCEPT_MWAIT __BIT(11) 296 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED __BIT(12) 297 #define VMCB_CTRL_INTERCEPT_XSETBV __BIT(13) 298 #define VMCB_CTRL_INTERCEPT_RDPRU __BIT(14) 299 #define VMCB_CTRL_INTERCEPT_EFER_SPEC __BIT(15) 300 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x) __BIT(16 + x) 301 302 uint32_t intercept_misc3; 303 #define VMCB_CTRL_INTERCEPT_INVLPGB_ALL __BIT(0) 304 #define VMCB_CTRL_INTERCEPT_INVLPGB_ILL __BIT(1) 305 #define VMCB_CTRL_INTERCEPT_PCID __BIT(2) 306 #define VMCB_CTRL_INTERCEPT_MCOMMIT __BIT(3) 307 #define VMCB_CTRL_INTERCEPT_TLBSYNC __BIT(4) 308 309 uint8_t rsvd1[36]; 310 uint16_t pause_filt_thresh; 311 uint16_t pause_filt_cnt; 312 uint64_t iopm_base_pa; 313 uint64_t msrpm_base_pa; 314 uint64_t tsc_offset; 315 uint32_t guest_asid; 316 317 uint32_t tlb_ctrl; 318 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL 0x01 319 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST 0x03 320 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL 0x07 321 322 uint64_t v; 323 #define VMCB_CTRL_V_TPR __BITS(3,0) 324 #define VMCB_CTRL_V_IRQ __BIT(8) 325 #define VMCB_CTRL_V_VGIF __BIT(9) 326 #define VMCB_CTRL_V_INTR_PRIO __BITS(19,16) 327 #define VMCB_CTRL_V_IGN_TPR __BIT(20) 328 #define VMCB_CTRL_V_INTR_MASKING __BIT(24) 329 #define VMCB_CTRL_V_GUEST_VGIF __BIT(25) 330 #define VMCB_CTRL_V_AVIC_EN __BIT(31) 331 #define VMCB_CTRL_V_INTR_VECTOR __BITS(39,32) 332 333 uint64_t intr; 334 #define VMCB_CTRL_INTR_SHADOW __BIT(0) 335 #define VMCB_CTRL_INTR_MASK __BIT(1) 336 337 uint64_t exitcode; 338 uint64_t exitinfo1; 339 uint64_t exitinfo2; 340 341 uint64_t exitintinfo; 342 #define VMCB_CTRL_EXITINTINFO_VECTOR __BITS(7,0) 343 #define VMCB_CTRL_EXITINTINFO_TYPE __BITS(10,8) 344 #define VMCB_CTRL_EXITINTINFO_EV __BIT(11) 345 #define VMCB_CTRL_EXITINTINFO_V __BIT(31) 346 #define VMCB_CTRL_EXITINTINFO_ERRORCODE __BITS(63,32) 347 348 uint64_t enable1; 349 #define VMCB_CTRL_ENABLE_NP __BIT(0) 350 #define VMCB_CTRL_ENABLE_SEV __BIT(1) 351 #define VMCB_CTRL_ENABLE_ES_SEV __BIT(2) 352 #define VMCB_CTRL_ENABLE_GMET __BIT(3) 353 #define VMCB_CTRL_ENABLE_VTE __BIT(5) 354 355 uint64_t avic; 356 #define VMCB_CTRL_AVIC_APIC_BAR __BITS(51,0) 357 358 uint64_t ghcb; 359 360 uint64_t eventinj; 361 #define VMCB_CTRL_EVENTINJ_VECTOR __BITS(7,0) 362 #define VMCB_CTRL_EVENTINJ_TYPE __BITS(10,8) 363 #define VMCB_CTRL_EVENTINJ_EV __BIT(11) 364 #define VMCB_CTRL_EVENTINJ_V __BIT(31) 365 #define VMCB_CTRL_EVENTINJ_ERRORCODE __BITS(63,32) 366 367 uint64_t n_cr3; 368 369 uint64_t enable2; 370 #define VMCB_CTRL_ENABLE_LBR __BIT(0) 371 #define VMCB_CTRL_ENABLE_VVMSAVE __BIT(1) 372 373 uint32_t vmcb_clean; 374 #define VMCB_CTRL_VMCB_CLEAN_I __BIT(0) 375 #define VMCB_CTRL_VMCB_CLEAN_IOPM __BIT(1) 376 #define VMCB_CTRL_VMCB_CLEAN_ASID __BIT(2) 377 #define VMCB_CTRL_VMCB_CLEAN_TPR __BIT(3) 378 #define VMCB_CTRL_VMCB_CLEAN_NP __BIT(4) 379 #define VMCB_CTRL_VMCB_CLEAN_CR __BIT(5) 380 #define VMCB_CTRL_VMCB_CLEAN_DR __BIT(6) 381 #define VMCB_CTRL_VMCB_CLEAN_DT __BIT(7) 382 #define VMCB_CTRL_VMCB_CLEAN_SEG __BIT(8) 383 #define VMCB_CTRL_VMCB_CLEAN_CR2 __BIT(9) 384 #define VMCB_CTRL_VMCB_CLEAN_LBR __BIT(10) 385 #define VMCB_CTRL_VMCB_CLEAN_AVIC __BIT(11) 386 387 uint32_t rsvd2; 388 uint64_t nrip; 389 uint8_t inst_len; 390 uint8_t inst_bytes[15]; 391 uint64_t avic_abpp; 392 uint64_t rsvd3; 393 uint64_t avic_ltp; 394 395 uint64_t avic_phys; 396 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR __BITS(51,12) 397 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX __BITS(7,0) 398 399 uint64_t rsvd4; 400 uint64_t vmsa_ptr; 401 402 uint8_t pad[752]; 403 } __packed; 404 405 CTASSERT(sizeof(struct vmcb_ctrl) == 1024); 406 407 struct vmcb_segment { 408 uint16_t selector; 409 uint16_t attrib; /* hidden */ 410 uint32_t limit; /* hidden */ 411 uint64_t base; /* hidden */ 412 } __packed; 413 414 CTASSERT(sizeof(struct vmcb_segment) == 16); 415 416 struct vmcb_state { 417 struct vmcb_segment es; 418 struct vmcb_segment cs; 419 struct vmcb_segment ss; 420 struct vmcb_segment ds; 421 struct vmcb_segment fs; 422 struct vmcb_segment gs; 423 struct vmcb_segment gdt; 424 struct vmcb_segment ldt; 425 struct vmcb_segment idt; 426 struct vmcb_segment tr; 427 uint8_t rsvd1[43]; 428 uint8_t cpl; 429 uint8_t rsvd2[4]; 430 uint64_t efer; 431 uint8_t rsvd3[112]; 432 uint64_t cr4; 433 uint64_t cr3; 434 uint64_t cr0; 435 uint64_t dr7; 436 uint64_t dr6; 437 uint64_t rflags; 438 uint64_t rip; 439 uint8_t rsvd4[88]; 440 uint64_t rsp; 441 uint8_t rsvd5[24]; 442 uint64_t rax; 443 uint64_t star; 444 uint64_t lstar; 445 uint64_t cstar; 446 uint64_t sfmask; 447 uint64_t kernelgsbase; 448 uint64_t sysenter_cs; 449 uint64_t sysenter_esp; 450 uint64_t sysenter_eip; 451 uint64_t cr2; 452 uint8_t rsvd6[32]; 453 uint64_t g_pat; 454 uint64_t dbgctl; 455 uint64_t br_from; 456 uint64_t br_to; 457 uint64_t int_from; 458 uint64_t int_to; 459 uint8_t pad[2408]; 460 } __packed; 461 462 CTASSERT(sizeof(struct vmcb_state) == 0xC00); 463 464 struct vmcb { 465 struct vmcb_ctrl ctrl; 466 struct vmcb_state state; 467 } __packed; 468 469 CTASSERT(sizeof(struct vmcb) == PAGE_SIZE); 470 CTASSERT(offsetof(struct vmcb, state) == 0x400); 471 472 /* -------------------------------------------------------------------------- */ 473 474 static void svm_vcpu_state_provide(struct nvmm_cpu *, uint64_t); 475 static void svm_vcpu_state_commit(struct nvmm_cpu *); 476 477 struct svm_hsave { 478 paddr_t pa; 479 }; 480 481 static struct svm_hsave hsave[MAXCPUS]; 482 483 static uint8_t *svm_asidmap __read_mostly; 484 static uint32_t svm_maxasid __read_mostly; 485 static kmutex_t svm_asidlock __cacheline_aligned; 486 487 static bool svm_decode_assist __read_mostly; 488 static uint32_t svm_ctrl_tlb_flush __read_mostly; 489 490 #define SVM_XCR0_MASK_DEFAULT (XCR0_X87|XCR0_SSE) 491 static uint64_t svm_xcr0_mask __read_mostly; 492 493 #define SVM_NCPUIDS 32 494 495 #define VMCB_NPAGES 1 496 497 #define MSRBM_NPAGES 2 498 #define MSRBM_SIZE (MSRBM_NPAGES * PAGE_SIZE) 499 500 #define IOBM_NPAGES 3 501 #define IOBM_SIZE (IOBM_NPAGES * PAGE_SIZE) 502 503 /* Does not include EFER_LMSLE. */ 504 #define EFER_VALID \ 505 (EFER_SCE|EFER_LME|EFER_LMA|EFER_NXE|EFER_SVME|EFER_FFXSR|EFER_TCE) 506 507 #define EFER_TLB_FLUSH \ 508 (EFER_NXE|EFER_LMA|EFER_LME) 509 #define CR0_TLB_FLUSH \ 510 (CR0_PG|CR0_WP|CR0_CD|CR0_NW) 511 #define CR4_TLB_FLUSH \ 512 (CR4_PGE|CR4_PAE|CR4_PSE) 513 514 /* -------------------------------------------------------------------------- */ 515 516 struct svm_machdata { 517 volatile uint64_t mach_htlb_gen; 518 }; 519 520 static const size_t svm_vcpu_conf_sizes[NVMM_X86_VCPU_NCONF] = { 521 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID)] = 522 sizeof(struct nvmm_vcpu_conf_cpuid), 523 [NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_TPR)] = 524 sizeof(struct nvmm_vcpu_conf_tpr) 525 }; 526 527 struct svm_cpudata { 528 /* General */ 529 bool shared_asid; 530 bool gtlb_want_flush; 531 bool gtsc_want_update; 532 uint64_t vcpu_htlb_gen; 533 534 /* VMCB */ 535 struct vmcb *vmcb; 536 paddr_t vmcb_pa; 537 538 /* I/O bitmap */ 539 uint8_t *iobm; 540 paddr_t iobm_pa; 541 542 /* MSR bitmap */ 543 uint8_t *msrbm; 544 paddr_t msrbm_pa; 545 546 /* Host state */ 547 uint64_t hxcr0; 548 uint64_t star; 549 uint64_t lstar; 550 uint64_t cstar; 551 uint64_t sfmask; 552 uint64_t fsbase; 553 uint64_t kernelgsbase; 554 bool ts_set; 555 struct xsave_header hfpu __aligned(64); 556 557 /* Intr state */ 558 bool int_window_exit; 559 bool nmi_window_exit; 560 bool evt_pending; 561 562 /* Guest state */ 563 uint64_t gxcr0; 564 uint64_t gprs[NVMM_X64_NGPR]; 565 uint64_t drs[NVMM_X64_NDR]; 566 uint64_t gtsc; 567 struct xsave_header gfpu __aligned(64); 568 569 /* VCPU configuration. */ 570 bool cpuidpresent[SVM_NCPUIDS]; 571 struct nvmm_vcpu_conf_cpuid cpuid[SVM_NCPUIDS]; 572 }; 573 574 static void 575 svm_vmcb_cache_default(struct vmcb *vmcb) 576 { 577 vmcb->ctrl.vmcb_clean = 578 VMCB_CTRL_VMCB_CLEAN_I | 579 VMCB_CTRL_VMCB_CLEAN_IOPM | 580 VMCB_CTRL_VMCB_CLEAN_ASID | 581 VMCB_CTRL_VMCB_CLEAN_TPR | 582 VMCB_CTRL_VMCB_CLEAN_NP | 583 VMCB_CTRL_VMCB_CLEAN_CR | 584 VMCB_CTRL_VMCB_CLEAN_DR | 585 VMCB_CTRL_VMCB_CLEAN_DT | 586 VMCB_CTRL_VMCB_CLEAN_SEG | 587 VMCB_CTRL_VMCB_CLEAN_CR2 | 588 VMCB_CTRL_VMCB_CLEAN_LBR | 589 VMCB_CTRL_VMCB_CLEAN_AVIC; 590 } 591 592 static void 593 svm_vmcb_cache_update(struct vmcb *vmcb, uint64_t flags) 594 { 595 if (flags & NVMM_X64_STATE_SEGS) { 596 vmcb->ctrl.vmcb_clean &= 597 ~(VMCB_CTRL_VMCB_CLEAN_SEG | VMCB_CTRL_VMCB_CLEAN_DT); 598 } 599 if (flags & NVMM_X64_STATE_CRS) { 600 vmcb->ctrl.vmcb_clean &= 601 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_CR2 | 602 VMCB_CTRL_VMCB_CLEAN_TPR); 603 } 604 if (flags & NVMM_X64_STATE_DRS) { 605 vmcb->ctrl.vmcb_clean &= ~VMCB_CTRL_VMCB_CLEAN_DR; 606 } 607 if (flags & NVMM_X64_STATE_MSRS) { 608 /* CR for EFER, NP for PAT. */ 609 vmcb->ctrl.vmcb_clean &= 610 ~(VMCB_CTRL_VMCB_CLEAN_CR | VMCB_CTRL_VMCB_CLEAN_NP); 611 } 612 } 613 614 static inline void 615 svm_vmcb_cache_flush(struct vmcb *vmcb, uint64_t flags) 616 { 617 vmcb->ctrl.vmcb_clean &= ~flags; 618 } 619 620 static inline void 621 svm_vmcb_cache_flush_all(struct vmcb *vmcb) 622 { 623 vmcb->ctrl.vmcb_clean = 0; 624 } 625 626 #define SVM_EVENT_TYPE_HW_INT 0 627 #define SVM_EVENT_TYPE_NMI 2 628 #define SVM_EVENT_TYPE_EXC 3 629 #define SVM_EVENT_TYPE_SW_INT 4 630 631 static void 632 svm_event_waitexit_enable(struct nvmm_cpu *vcpu, bool nmi) 633 { 634 struct svm_cpudata *cpudata = vcpu->cpudata; 635 struct vmcb *vmcb = cpudata->vmcb; 636 637 if (nmi) { 638 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_IRET; 639 cpudata->nmi_window_exit = true; 640 } else { 641 vmcb->ctrl.intercept_misc1 |= VMCB_CTRL_INTERCEPT_VINTR; 642 vmcb->ctrl.v |= (VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 643 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 644 cpudata->int_window_exit = true; 645 } 646 647 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 648 } 649 650 static void 651 svm_event_waitexit_disable(struct nvmm_cpu *vcpu, bool nmi) 652 { 653 struct svm_cpudata *cpudata = vcpu->cpudata; 654 struct vmcb *vmcb = cpudata->vmcb; 655 656 if (nmi) { 657 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_IRET; 658 cpudata->nmi_window_exit = false; 659 } else { 660 vmcb->ctrl.intercept_misc1 &= ~VMCB_CTRL_INTERCEPT_VINTR; 661 vmcb->ctrl.v &= ~(VMCB_CTRL_V_IRQ | VMCB_CTRL_V_IGN_TPR); 662 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_TPR); 663 cpudata->int_window_exit = false; 664 } 665 666 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 667 } 668 669 static inline int 670 svm_event_has_error(uint8_t vector) 671 { 672 switch (vector) { 673 case 8: /* #DF */ 674 case 10: /* #TS */ 675 case 11: /* #NP */ 676 case 12: /* #SS */ 677 case 13: /* #GP */ 678 case 14: /* #PF */ 679 case 17: /* #AC */ 680 case 30: /* #SX */ 681 return 1; 682 default: 683 return 0; 684 } 685 } 686 687 static int 688 svm_vcpu_inject(struct nvmm_cpu *vcpu) 689 { 690 struct nvmm_comm_page *comm = vcpu->comm; 691 struct svm_cpudata *cpudata = vcpu->cpudata; 692 struct vmcb *vmcb = cpudata->vmcb; 693 u_int evtype; 694 uint8_t vector; 695 uint64_t error; 696 int type = 0, err = 0; 697 698 evtype = comm->event.type; 699 vector = comm->event.vector; 700 error = comm->event.u.excp.error; 701 __insn_barrier(); 702 703 switch (evtype) { 704 case NVMM_VCPU_EVENT_EXCP: 705 type = SVM_EVENT_TYPE_EXC; 706 if (vector == 2 || vector >= 32) 707 return EINVAL; 708 if (vector == 3 || vector == 0) 709 return EINVAL; 710 err = svm_event_has_error(vector); 711 break; 712 case NVMM_VCPU_EVENT_INTR: 713 type = SVM_EVENT_TYPE_HW_INT; 714 if (vector == 2) { 715 type = SVM_EVENT_TYPE_NMI; 716 svm_event_waitexit_enable(vcpu, true); 717 } 718 err = 0; 719 break; 720 default: 721 return EINVAL; 722 } 723 724 vmcb->ctrl.eventinj = 725 __SHIFTIN((uint64_t)vector, VMCB_CTRL_EVENTINJ_VECTOR) | 726 __SHIFTIN((uint64_t)type, VMCB_CTRL_EVENTINJ_TYPE) | 727 __SHIFTIN((uint64_t)err, VMCB_CTRL_EVENTINJ_EV) | 728 __SHIFTIN((uint64_t)1, VMCB_CTRL_EVENTINJ_V) | 729 __SHIFTIN((uint64_t)error, VMCB_CTRL_EVENTINJ_ERRORCODE); 730 731 cpudata->evt_pending = true; 732 733 return 0; 734 } 735 736 static void 737 svm_inject_ud(struct nvmm_cpu *vcpu) 738 { 739 struct nvmm_comm_page *comm = vcpu->comm; 740 int ret __diagused; 741 742 comm->event.type = NVMM_VCPU_EVENT_EXCP; 743 comm->event.vector = 6; 744 comm->event.u.excp.error = 0; 745 746 ret = svm_vcpu_inject(vcpu); 747 KASSERT(ret == 0); 748 } 749 750 static void 751 svm_inject_gp(struct nvmm_cpu *vcpu) 752 { 753 struct nvmm_comm_page *comm = vcpu->comm; 754 int ret __diagused; 755 756 comm->event.type = NVMM_VCPU_EVENT_EXCP; 757 comm->event.vector = 13; 758 comm->event.u.excp.error = 0; 759 760 ret = svm_vcpu_inject(vcpu); 761 KASSERT(ret == 0); 762 } 763 764 static inline int 765 svm_vcpu_event_commit(struct nvmm_cpu *vcpu) 766 { 767 if (__predict_true(!vcpu->comm->event_commit)) { 768 return 0; 769 } 770 vcpu->comm->event_commit = false; 771 return svm_vcpu_inject(vcpu); 772 } 773 774 static inline void 775 svm_inkernel_advance(struct vmcb *vmcb) 776 { 777 /* 778 * Maybe we should also apply single-stepping and debug exceptions. 779 * Matters for guest-ring3, because it can execute 'cpuid' under a 780 * debugger. 781 */ 782 vmcb->state.rip = vmcb->ctrl.nrip; 783 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 784 } 785 786 #define SVM_CPUID_MAX_HYPERVISOR 0x40000000 787 788 static void 789 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx) 790 { 791 struct svm_cpudata *cpudata = vcpu->cpudata; 792 uint64_t cr4; 793 794 switch (eax) { 795 case 0x00000001: 796 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax; 797 798 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID; 799 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid, 800 CPUID_LOCAL_APIC_ID); 801 802 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx; 803 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ; 804 805 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx; 806 807 /* CPUID2_OSXSAVE depends on CR4. */ 808 cr4 = cpudata->vmcb->state.cr4; 809 if (!(cr4 & CR4_OSXSAVE)) { 810 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE; 811 } 812 break; 813 case 0x00000002: /* Empty */ 814 case 0x00000003: /* Empty */ 815 case 0x00000004: /* Empty */ 816 case 0x00000005: /* Monitor/MWait */ 817 case 0x00000006: /* Power Management Related Features */ 818 cpudata->vmcb->state.rax = 0; 819 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 820 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 821 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 822 break; 823 case 0x00000007: /* Structured Extended Features */ 824 cpudata->vmcb->state.rax &= nvmm_cpuid_00000007.eax; 825 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx; 826 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx; 827 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx; 828 break; 829 case 0x00000008: /* Empty */ 830 case 0x00000009: /* Empty */ 831 case 0x0000000A: /* Empty */ 832 case 0x0000000B: /* Empty */ 833 case 0x0000000C: /* Empty */ 834 cpudata->vmcb->state.rax = 0; 835 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 836 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 837 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 838 break; 839 case 0x0000000D: /* Processor Extended State Enumeration */ 840 if (svm_xcr0_mask == 0) { 841 break; 842 } 843 switch (ecx) { 844 case 0: 845 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF; 846 if (cpudata->gxcr0 & XCR0_SSE) { 847 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave); 848 } else { 849 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87); 850 } 851 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */ 852 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64; 853 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32; 854 break; 855 case 1: 856 cpudata->vmcb->state.rax &= 857 (CPUID_PES1_XSAVEOPT | CPUID_PES1_XSAVEC | 858 CPUID_PES1_XGETBV); 859 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 860 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 861 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 862 break; 863 default: 864 cpudata->vmcb->state.rax = 0; 865 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 866 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 867 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 868 break; 869 } 870 break; 871 872 case 0x40000000: /* Hypervisor Information */ 873 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR; 874 cpudata->gprs[NVMM_X64_GPR_RBX] = 0; 875 cpudata->gprs[NVMM_X64_GPR_RCX] = 0; 876 cpudata->gprs[NVMM_X64_GPR_RDX] = 0; 877 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4); 878 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4); 879 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4); 880 break; 881 882 case 0x80000001: 883 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax; 884 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx; 885 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx; 886 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx; 887 break; 888 default: 889 break; 890 } 891 } 892 893 static void 894 svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason) 895 { 896 exit->u.insn.npc = vmcb->ctrl.nrip; 897 exit->reason = reason; 898 } 899 900 static void 901 svm_exit_cpuid(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 902 struct nvmm_vcpu_exit *exit) 903 { 904 struct svm_cpudata *cpudata = vcpu->cpudata; 905 struct nvmm_vcpu_conf_cpuid *cpuid; 906 uint64_t eax, ecx; 907 u_int descs[4]; 908 size_t i; 909 910 eax = cpudata->vmcb->state.rax; 911 ecx = cpudata->gprs[NVMM_X64_GPR_RCX]; 912 x86_cpuid2(eax, ecx, descs); 913 914 cpudata->vmcb->state.rax = descs[0]; 915 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1]; 916 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2]; 917 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3]; 918 919 svm_inkernel_handle_cpuid(vcpu, eax, ecx); 920 921 for (i = 0; i < SVM_NCPUIDS; i++) { 922 if (!cpudata->cpuidpresent[i]) { 923 continue; 924 } 925 cpuid = &cpudata->cpuid[i]; 926 if (cpuid->leaf != eax) { 927 continue; 928 } 929 930 if (cpuid->exit) { 931 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID); 932 return; 933 } 934 KASSERT(cpuid->mask); 935 936 /* del */ 937 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax; 938 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx; 939 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx; 940 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx; 941 942 /* set */ 943 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax; 944 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx; 945 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx; 946 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx; 947 948 break; 949 } 950 951 svm_inkernel_advance(cpudata->vmcb); 952 exit->reason = NVMM_VCPU_EXIT_NONE; 953 } 954 955 static void 956 svm_exit_hlt(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 957 struct nvmm_vcpu_exit *exit) 958 { 959 struct svm_cpudata *cpudata = vcpu->cpudata; 960 struct vmcb *vmcb = cpudata->vmcb; 961 962 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) { 963 svm_event_waitexit_disable(vcpu, false); 964 } 965 966 svm_inkernel_advance(cpudata->vmcb); 967 exit->reason = NVMM_VCPU_EXIT_HALTED; 968 } 969 970 #define SVM_EXIT_IO_PORT __BITS(31,16) 971 #define SVM_EXIT_IO_SEG __BITS(12,10) 972 #define SVM_EXIT_IO_A64 __BIT(9) 973 #define SVM_EXIT_IO_A32 __BIT(8) 974 #define SVM_EXIT_IO_A16 __BIT(7) 975 #define SVM_EXIT_IO_SZ32 __BIT(6) 976 #define SVM_EXIT_IO_SZ16 __BIT(5) 977 #define SVM_EXIT_IO_SZ8 __BIT(4) 978 #define SVM_EXIT_IO_REP __BIT(3) 979 #define SVM_EXIT_IO_STR __BIT(2) 980 #define SVM_EXIT_IO_IN __BIT(0) 981 982 static void 983 svm_exit_io(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 984 struct nvmm_vcpu_exit *exit) 985 { 986 struct svm_cpudata *cpudata = vcpu->cpudata; 987 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 988 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2; 989 990 exit->reason = NVMM_VCPU_EXIT_IO; 991 992 exit->u.io.in = (info & SVM_EXIT_IO_IN) != 0; 993 exit->u.io.port = __SHIFTOUT(info, SVM_EXIT_IO_PORT); 994 995 if (svm_decode_assist) { 996 KASSERT(__SHIFTOUT(info, SVM_EXIT_IO_SEG) < 6); 997 exit->u.io.seg = __SHIFTOUT(info, SVM_EXIT_IO_SEG); 998 } else { 999 exit->u.io.seg = -1; 1000 } 1001 1002 if (info & SVM_EXIT_IO_A64) { 1003 exit->u.io.address_size = 8; 1004 } else if (info & SVM_EXIT_IO_A32) { 1005 exit->u.io.address_size = 4; 1006 } else if (info & SVM_EXIT_IO_A16) { 1007 exit->u.io.address_size = 2; 1008 } 1009 1010 if (info & SVM_EXIT_IO_SZ32) { 1011 exit->u.io.operand_size = 4; 1012 } else if (info & SVM_EXIT_IO_SZ16) { 1013 exit->u.io.operand_size = 2; 1014 } else if (info & SVM_EXIT_IO_SZ8) { 1015 exit->u.io.operand_size = 1; 1016 } 1017 1018 exit->u.io.rep = (info & SVM_EXIT_IO_REP) != 0; 1019 exit->u.io.str = (info & SVM_EXIT_IO_STR) != 0; 1020 exit->u.io.npc = nextpc; 1021 1022 svm_vcpu_state_provide(vcpu, 1023 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1024 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1025 } 1026 1027 static const uint64_t msr_ignore_list[] = { 1028 0xc0010055, /* MSR_CMPHALT */ 1029 MSR_DE_CFG, 1030 MSR_IC_CFG, 1031 MSR_UCODE_AMD_PATCHLEVEL 1032 }; 1033 1034 static bool 1035 svm_inkernel_handle_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1036 struct nvmm_vcpu_exit *exit) 1037 { 1038 struct svm_cpudata *cpudata = vcpu->cpudata; 1039 struct vmcb *vmcb = cpudata->vmcb; 1040 uint64_t val; 1041 size_t i; 1042 1043 if (exit->reason == NVMM_VCPU_EXIT_RDMSR) { 1044 if (exit->u.rdmsr.msr == MSR_NB_CFG) { 1045 val = NB_CFG_INITAPICCPUIDLO; 1046 vmcb->state.rax = (val & 0xFFFFFFFF); 1047 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1048 goto handled; 1049 } 1050 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1051 if (msr_ignore_list[i] != exit->u.rdmsr.msr) 1052 continue; 1053 val = 0; 1054 vmcb->state.rax = (val & 0xFFFFFFFF); 1055 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32); 1056 goto handled; 1057 } 1058 } else { 1059 if (exit->u.wrmsr.msr == MSR_EFER) { 1060 if (__predict_false(exit->u.wrmsr.val & ~EFER_VALID)) { 1061 goto error; 1062 } 1063 if ((vmcb->state.efer ^ exit->u.wrmsr.val) & 1064 EFER_TLB_FLUSH) { 1065 cpudata->gtlb_want_flush = true; 1066 } 1067 vmcb->state.efer = exit->u.wrmsr.val | EFER_SVME; 1068 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_CR); 1069 goto handled; 1070 } 1071 if (exit->u.wrmsr.msr == MSR_TSC) { 1072 cpudata->gtsc = exit->u.wrmsr.val; 1073 cpudata->gtsc_want_update = true; 1074 goto handled; 1075 } 1076 for (i = 0; i < __arraycount(msr_ignore_list); i++) { 1077 if (msr_ignore_list[i] != exit->u.wrmsr.msr) 1078 continue; 1079 goto handled; 1080 } 1081 } 1082 1083 return false; 1084 1085 handled: 1086 svm_inkernel_advance(cpudata->vmcb); 1087 return true; 1088 1089 error: 1090 svm_inject_gp(vcpu); 1091 return true; 1092 } 1093 1094 static inline void 1095 svm_exit_rdmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1096 struct nvmm_vcpu_exit *exit) 1097 { 1098 struct svm_cpudata *cpudata = vcpu->cpudata; 1099 1100 exit->reason = NVMM_VCPU_EXIT_RDMSR; 1101 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1102 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip; 1103 1104 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1105 exit->reason = NVMM_VCPU_EXIT_NONE; 1106 return; 1107 } 1108 1109 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1110 } 1111 1112 static inline void 1113 svm_exit_wrmsr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1114 struct nvmm_vcpu_exit *exit) 1115 { 1116 struct svm_cpudata *cpudata = vcpu->cpudata; 1117 uint64_t rdx, rax; 1118 1119 rdx = cpudata->gprs[NVMM_X64_GPR_RDX]; 1120 rax = cpudata->vmcb->state.rax; 1121 1122 exit->reason = NVMM_VCPU_EXIT_WRMSR; 1123 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF); 1124 exit->u.wrmsr.val = (rdx << 32) | (rax & 0xFFFFFFFF); 1125 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip; 1126 1127 if (svm_inkernel_handle_msr(mach, vcpu, exit)) { 1128 exit->reason = NVMM_VCPU_EXIT_NONE; 1129 return; 1130 } 1131 1132 svm_vcpu_state_provide(vcpu, NVMM_X64_STATE_GPRS); 1133 } 1134 1135 static void 1136 svm_exit_msr(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1137 struct nvmm_vcpu_exit *exit) 1138 { 1139 struct svm_cpudata *cpudata = vcpu->cpudata; 1140 uint64_t info = cpudata->vmcb->ctrl.exitinfo1; 1141 1142 if (info == 0) { 1143 svm_exit_rdmsr(mach, vcpu, exit); 1144 } else { 1145 svm_exit_wrmsr(mach, vcpu, exit); 1146 } 1147 } 1148 1149 static void 1150 svm_exit_npf(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1151 struct nvmm_vcpu_exit *exit) 1152 { 1153 struct svm_cpudata *cpudata = vcpu->cpudata; 1154 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2; 1155 1156 exit->reason = NVMM_VCPU_EXIT_MEMORY; 1157 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W) 1158 exit->u.mem.prot = PROT_WRITE; 1159 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_X) 1160 exit->u.mem.prot = PROT_EXEC; 1161 else 1162 exit->u.mem.prot = PROT_READ; 1163 exit->u.mem.gpa = gpa; 1164 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len; 1165 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes, 1166 sizeof(exit->u.mem.inst_bytes)); 1167 1168 svm_vcpu_state_provide(vcpu, 1169 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 1170 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 1171 } 1172 1173 static void 1174 svm_exit_xsetbv(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1175 struct nvmm_vcpu_exit *exit) 1176 { 1177 struct svm_cpudata *cpudata = vcpu->cpudata; 1178 struct vmcb *vmcb = cpudata->vmcb; 1179 uint64_t val; 1180 1181 exit->reason = NVMM_VCPU_EXIT_NONE; 1182 1183 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) | 1184 (vmcb->state.rax & 0xFFFFFFFF); 1185 1186 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) { 1187 goto error; 1188 } else if (__predict_false(vmcb->state.cpl != 0)) { 1189 goto error; 1190 } else if (__predict_false((val & ~svm_xcr0_mask) != 0)) { 1191 goto error; 1192 } else if (__predict_false((val & XCR0_X87) == 0)) { 1193 goto error; 1194 } 1195 1196 cpudata->gxcr0 = val; 1197 1198 svm_inkernel_advance(cpudata->vmcb); 1199 return; 1200 1201 error: 1202 svm_inject_gp(vcpu); 1203 } 1204 1205 static void 1206 svm_exit_invalid(struct nvmm_vcpu_exit *exit, uint64_t code) 1207 { 1208 exit->u.inv.hwcode = code; 1209 exit->reason = NVMM_VCPU_EXIT_INVALID; 1210 } 1211 1212 /* -------------------------------------------------------------------------- */ 1213 1214 static void 1215 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu) 1216 { 1217 struct svm_cpudata *cpudata = vcpu->cpudata; 1218 1219 cpudata->ts_set = (rcr0() & CR0_TS) != 0; 1220 1221 fpu_area_save(&cpudata->hfpu, svm_xcr0_mask); 1222 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask); 1223 1224 if (svm_xcr0_mask != 0) { 1225 cpudata->hxcr0 = rdxcr(0); 1226 wrxcr(0, cpudata->gxcr0); 1227 } 1228 } 1229 1230 static void 1231 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu) 1232 { 1233 struct svm_cpudata *cpudata = vcpu->cpudata; 1234 1235 if (svm_xcr0_mask != 0) { 1236 cpudata->gxcr0 = rdxcr(0); 1237 wrxcr(0, cpudata->hxcr0); 1238 } 1239 1240 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask); 1241 fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask); 1242 1243 if (cpudata->ts_set) { 1244 stts(); 1245 } 1246 } 1247 1248 static void 1249 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu) 1250 { 1251 struct svm_cpudata *cpudata = vcpu->cpudata; 1252 1253 x86_dbregs_save(curlwp); 1254 1255 ldr7(0); 1256 1257 ldr0(cpudata->drs[NVMM_X64_DR_DR0]); 1258 ldr1(cpudata->drs[NVMM_X64_DR_DR1]); 1259 ldr2(cpudata->drs[NVMM_X64_DR_DR2]); 1260 ldr3(cpudata->drs[NVMM_X64_DR_DR3]); 1261 } 1262 1263 static void 1264 svm_vcpu_guest_dbregs_leave(struct nvmm_cpu *vcpu) 1265 { 1266 struct svm_cpudata *cpudata = vcpu->cpudata; 1267 1268 cpudata->drs[NVMM_X64_DR_DR0] = rdr0(); 1269 cpudata->drs[NVMM_X64_DR_DR1] = rdr1(); 1270 cpudata->drs[NVMM_X64_DR_DR2] = rdr2(); 1271 cpudata->drs[NVMM_X64_DR_DR3] = rdr3(); 1272 1273 x86_dbregs_restore(curlwp); 1274 } 1275 1276 static void 1277 svm_vcpu_guest_misc_enter(struct nvmm_cpu *vcpu) 1278 { 1279 struct svm_cpudata *cpudata = vcpu->cpudata; 1280 1281 cpudata->fsbase = rdmsr(MSR_FSBASE); 1282 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE); 1283 } 1284 1285 static void 1286 svm_vcpu_guest_misc_leave(struct nvmm_cpu *vcpu) 1287 { 1288 struct svm_cpudata *cpudata = vcpu->cpudata; 1289 1290 wrmsr(MSR_STAR, cpudata->star); 1291 wrmsr(MSR_LSTAR, cpudata->lstar); 1292 wrmsr(MSR_CSTAR, cpudata->cstar); 1293 wrmsr(MSR_SFMASK, cpudata->sfmask); 1294 wrmsr(MSR_FSBASE, cpudata->fsbase); 1295 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase); 1296 } 1297 1298 /* -------------------------------------------------------------------------- */ 1299 1300 static inline void 1301 svm_gtlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1302 { 1303 struct svm_cpudata *cpudata = vcpu->cpudata; 1304 1305 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) { 1306 cpudata->gtlb_want_flush = true; 1307 } 1308 } 1309 1310 static inline void 1311 svm_htlb_catchup(struct nvmm_cpu *vcpu, int hcpu) 1312 { 1313 /* 1314 * Nothing to do. If an hTLB flush was needed, either the VCPU was 1315 * executing on this hCPU and the hTLB already got flushed, or it 1316 * was executing on another hCPU in which case the catchup is done 1317 * in svm_gtlb_catchup(). 1318 */ 1319 } 1320 1321 static inline uint64_t 1322 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata) 1323 { 1324 struct vmcb *vmcb = cpudata->vmcb; 1325 uint64_t machgen; 1326 1327 machgen = machdata->mach_htlb_gen; 1328 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) { 1329 return machgen; 1330 } 1331 1332 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1333 return machgen; 1334 } 1335 1336 static inline void 1337 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen) 1338 { 1339 struct vmcb *vmcb = cpudata->vmcb; 1340 1341 if (__predict_true(vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID)) { 1342 cpudata->vcpu_htlb_gen = machgen; 1343 } 1344 } 1345 1346 static inline void 1347 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb) 1348 { 1349 cpudata->evt_pending = false; 1350 1351 if (__predict_false(vmcb->ctrl.exitintinfo & VMCB_CTRL_EXITINTINFO_V)) { 1352 vmcb->ctrl.eventinj = vmcb->ctrl.exitintinfo; 1353 cpudata->evt_pending = true; 1354 } 1355 } 1356 1357 static int 1358 svm_vcpu_run(struct nvmm_machine *mach, struct nvmm_cpu *vcpu, 1359 struct nvmm_vcpu_exit *exit) 1360 { 1361 struct nvmm_comm_page *comm = vcpu->comm; 1362 struct svm_machdata *machdata = mach->machdata; 1363 struct svm_cpudata *cpudata = vcpu->cpudata; 1364 struct vmcb *vmcb = cpudata->vmcb; 1365 uint64_t machgen; 1366 int hcpu, s; 1367 1368 if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) { 1369 return EINVAL; 1370 } 1371 svm_vcpu_state_commit(vcpu); 1372 comm->state_cached = 0; 1373 1374 kpreempt_disable(); 1375 hcpu = cpu_number(); 1376 1377 svm_gtlb_catchup(vcpu, hcpu); 1378 svm_htlb_catchup(vcpu, hcpu); 1379 1380 if (vcpu->hcpu_last != hcpu) { 1381 svm_vmcb_cache_flush_all(vmcb); 1382 cpudata->gtsc_want_update = true; 1383 } 1384 1385 svm_vcpu_guest_dbregs_enter(vcpu); 1386 svm_vcpu_guest_misc_enter(vcpu); 1387 1388 while (1) { 1389 if (cpudata->gtlb_want_flush) { 1390 vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush; 1391 } else { 1392 vmcb->ctrl.tlb_ctrl = 0; 1393 } 1394 1395 if (__predict_false(cpudata->gtsc_want_update)) { 1396 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc(); 1397 svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I); 1398 } 1399 1400 s = splhigh(); 1401 machgen = svm_htlb_flush(machdata, cpudata); 1402 svm_vcpu_guest_fpu_enter(vcpu); 1403 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs); 1404 svm_vcpu_guest_fpu_leave(vcpu); 1405 svm_htlb_flush_ack(cpudata, machgen); 1406 splx(s); 1407 1408 svm_vmcb_cache_default(vmcb); 1409 1410 if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) { 1411 cpudata->gtlb_want_flush = false; 1412 cpudata->gtsc_want_update = false; 1413 vcpu->hcpu_last = hcpu; 1414 } 1415 svm_exit_evt(cpudata, vmcb); 1416 1417 switch (vmcb->ctrl.exitcode) { 1418 case VMCB_EXITCODE_INTR: 1419 case VMCB_EXITCODE_NMI: 1420 exit->reason = NVMM_VCPU_EXIT_NONE; 1421 break; 1422 case VMCB_EXITCODE_VINTR: 1423 svm_event_waitexit_disable(vcpu, false); 1424 exit->reason = NVMM_VCPU_EXIT_INT_READY; 1425 break; 1426 case VMCB_EXITCODE_IRET: 1427 svm_event_waitexit_disable(vcpu, true); 1428 exit->reason = NVMM_VCPU_EXIT_NMI_READY; 1429 break; 1430 case VMCB_EXITCODE_CPUID: 1431 svm_exit_cpuid(mach, vcpu, exit); 1432 break; 1433 case VMCB_EXITCODE_HLT: 1434 svm_exit_hlt(mach, vcpu, exit); 1435 break; 1436 case VMCB_EXITCODE_IOIO: 1437 svm_exit_io(mach, vcpu, exit); 1438 break; 1439 case VMCB_EXITCODE_MSR: 1440 svm_exit_msr(mach, vcpu, exit); 1441 break; 1442 case VMCB_EXITCODE_SHUTDOWN: 1443 exit->reason = NVMM_VCPU_EXIT_SHUTDOWN; 1444 break; 1445 case VMCB_EXITCODE_RDPMC: 1446 case VMCB_EXITCODE_RSM: 1447 case VMCB_EXITCODE_INVLPGA: 1448 case VMCB_EXITCODE_VMRUN: 1449 case VMCB_EXITCODE_VMMCALL: 1450 case VMCB_EXITCODE_VMLOAD: 1451 case VMCB_EXITCODE_VMSAVE: 1452 case VMCB_EXITCODE_STGI: 1453 case VMCB_EXITCODE_CLGI: 1454 case VMCB_EXITCODE_SKINIT: 1455 case VMCB_EXITCODE_RDTSCP: 1456 case VMCB_EXITCODE_RDPRU: 1457 case VMCB_EXITCODE_INVLPGB: 1458 case VMCB_EXITCODE_INVPCID: 1459 case VMCB_EXITCODE_MCOMMIT: 1460 case VMCB_EXITCODE_TLBSYNC: 1461 svm_inject_ud(vcpu); 1462 exit->reason = NVMM_VCPU_EXIT_NONE; 1463 break; 1464 case VMCB_EXITCODE_MONITOR: 1465 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR); 1466 break; 1467 case VMCB_EXITCODE_MWAIT: 1468 case VMCB_EXITCODE_MWAIT_CONDITIONAL: 1469 svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT); 1470 break; 1471 case VMCB_EXITCODE_XSETBV: 1472 svm_exit_xsetbv(mach, vcpu, exit); 1473 break; 1474 case VMCB_EXITCODE_NPF: 1475 svm_exit_npf(mach, vcpu, exit); 1476 break; 1477 case VMCB_EXITCODE_FERR_FREEZE: /* ? */ 1478 default: 1479 svm_exit_invalid(exit, vmcb->ctrl.exitcode); 1480 break; 1481 } 1482 1483 /* If no reason to return to userland, keep rolling. */ 1484 if (nvmm_return_needed()) { 1485 break; 1486 } 1487 if (exit->reason != NVMM_VCPU_EXIT_NONE) { 1488 break; 1489 } 1490 } 1491 1492 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset; 1493 1494 svm_vcpu_guest_misc_leave(vcpu); 1495 svm_vcpu_guest_dbregs_leave(vcpu); 1496 1497 kpreempt_enable(); 1498 1499 exit->exitstate.rflags = vmcb->state.rflags; 1500 exit->exitstate.cr8 = __SHIFTOUT(vmcb->ctrl.v, VMCB_CTRL_V_TPR); 1501 exit->exitstate.int_shadow = 1502 ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0); 1503 exit->exitstate.int_window_exiting = cpudata->int_window_exit; 1504 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit; 1505 exit->exitstate.evt_pending = cpudata->evt_pending; 1506 1507 return 0; 1508 } 1509 1510 /* -------------------------------------------------------------------------- */ 1511 1512 static int 1513 svm_memalloc(paddr_t *pa, vaddr_t *va, size_t npages) 1514 { 1515 struct pglist pglist; 1516 paddr_t _pa; 1517 vaddr_t _va; 1518 size_t i; 1519 int ret; 1520 1521 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 1522 &pglist, 1, 0); 1523 if (ret != 0) 1524 return ENOMEM; 1525 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 1526 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 1527 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1528 if (_va == 0) 1529 goto error; 1530 1531 for (i = 0; i < npages; i++) { 1532 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 1533 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 1534 } 1535 pmap_update(pmap_kernel()); 1536 1537 memset((void *)_va, 0, npages * PAGE_SIZE); 1538 1539 *pa = _pa; 1540 *va = _va; 1541 return 0; 1542 1543 error: 1544 for (i = 0; i < npages; i++) { 1545 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 1546 } 1547 return ENOMEM; 1548 } 1549 1550 static void 1551 svm_memfree(paddr_t pa, vaddr_t va, size_t npages) 1552 { 1553 size_t i; 1554 1555 pmap_kremove(va, npages * PAGE_SIZE); 1556 pmap_update(pmap_kernel()); 1557 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 1558 for (i = 0; i < npages; i++) { 1559 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 1560 } 1561 } 1562 1563 /* -------------------------------------------------------------------------- */ 1564 1565 #define SVM_MSRBM_READ __BIT(0) 1566 #define SVM_MSRBM_WRITE __BIT(1) 1567 1568 static void 1569 svm_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write) 1570 { 1571 uint64_t byte; 1572 uint8_t bitoff; 1573 1574 if (msr < 0x00002000) { 1575 /* Range 1 */ 1576 byte = ((msr - 0x00000000) >> 2UL) + 0x0000; 1577 } else if (msr >= 0xC0000000 && msr < 0xC0002000) { 1578 /* Range 2 */ 1579 byte = ((msr - 0xC0000000) >> 2UL) + 0x0800; 1580 } else if (msr >= 0xC0010000 && msr < 0xC0012000) { 1581 /* Range 3 */ 1582 byte = ((msr - 0xC0010000) >> 2UL) + 0x1000; 1583 } else { 1584 panic("%s: wrong range", __func__); 1585 } 1586 1587 bitoff = (msr & 0x3) << 1; 1588 1589 if (read) { 1590 bitmap[byte] &= ~(SVM_MSRBM_READ << bitoff); 1591 } 1592 if (write) { 1593 bitmap[byte] &= ~(SVM_MSRBM_WRITE << bitoff); 1594 } 1595 } 1596 1597 #define SVM_SEG_ATTRIB_TYPE __BITS(3,0) 1598 #define SVM_SEG_ATTRIB_S __BIT(4) 1599 #define SVM_SEG_ATTRIB_DPL __BITS(6,5) 1600 #define SVM_SEG_ATTRIB_P __BIT(7) 1601 #define SVM_SEG_ATTRIB_AVL __BIT(8) 1602 #define SVM_SEG_ATTRIB_L __BIT(9) 1603 #define SVM_SEG_ATTRIB_DEF __BIT(10) 1604 #define SVM_SEG_ATTRIB_G __BIT(11) 1605 1606 static void 1607 svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg, 1608 struct vmcb_segment *vseg) 1609 { 1610 vseg->selector = seg->selector; 1611 vseg->attrib = 1612 __SHIFTIN(seg->attrib.type, SVM_SEG_ATTRIB_TYPE) | 1613 __SHIFTIN(seg->attrib.s, SVM_SEG_ATTRIB_S) | 1614 __SHIFTIN(seg->attrib.dpl, SVM_SEG_ATTRIB_DPL) | 1615 __SHIFTIN(seg->attrib.p, SVM_SEG_ATTRIB_P) | 1616 __SHIFTIN(seg->attrib.avl, SVM_SEG_ATTRIB_AVL) | 1617 __SHIFTIN(seg->attrib.l, SVM_SEG_ATTRIB_L) | 1618 __SHIFTIN(seg->attrib.def, SVM_SEG_ATTRIB_DEF) | 1619 __SHIFTIN(seg->attrib.g, SVM_SEG_ATTRIB_G); 1620 vseg->limit = seg->limit; 1621 vseg->base = seg->base; 1622 } 1623 1624 static void 1625 svm_vcpu_getstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg) 1626 { 1627 seg->selector = vseg->selector; 1628 seg->attrib.type = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_TYPE); 1629 seg->attrib.s = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_S); 1630 seg->attrib.dpl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DPL); 1631 seg->attrib.p = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_P); 1632 seg->attrib.avl = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_AVL); 1633 seg->attrib.l = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_L); 1634 seg->attrib.def = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_DEF); 1635 seg->attrib.g = __SHIFTOUT(vseg->attrib, SVM_SEG_ATTRIB_G); 1636 seg->limit = vseg->limit; 1637 seg->base = vseg->base; 1638 } 1639 1640 static inline bool 1641 svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state, 1642 uint64_t flags) 1643 { 1644 if (flags & NVMM_X64_STATE_CRS) { 1645 if ((vmcb->state.cr0 ^ 1646 state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) { 1647 return true; 1648 } 1649 if (vmcb->state.cr3 != state->crs[NVMM_X64_CR_CR3]) { 1650 return true; 1651 } 1652 if ((vmcb->state.cr4 ^ 1653 state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) { 1654 return true; 1655 } 1656 } 1657 1658 if (flags & NVMM_X64_STATE_MSRS) { 1659 if ((vmcb->state.efer ^ 1660 state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) { 1661 return true; 1662 } 1663 } 1664 1665 return false; 1666 } 1667 1668 static void 1669 svm_vcpu_setstate(struct nvmm_cpu *vcpu) 1670 { 1671 struct nvmm_comm_page *comm = vcpu->comm; 1672 const struct nvmm_x64_state *state = &comm->state; 1673 struct svm_cpudata *cpudata = vcpu->cpudata; 1674 struct vmcb *vmcb = cpudata->vmcb; 1675 struct fxsave *fpustate; 1676 uint64_t flags; 1677 1678 flags = comm->state_wanted; 1679 1680 if (svm_state_tlb_flush(vmcb, state, flags)) { 1681 cpudata->gtlb_want_flush = true; 1682 } 1683 1684 if (flags & NVMM_X64_STATE_SEGS) { 1685 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_CS], 1686 &vmcb->state.cs); 1687 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_DS], 1688 &vmcb->state.ds); 1689 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_ES], 1690 &vmcb->state.es); 1691 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_FS], 1692 &vmcb->state.fs); 1693 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GS], 1694 &vmcb->state.gs); 1695 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_SS], 1696 &vmcb->state.ss); 1697 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1698 &vmcb->state.gdt); 1699 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1700 &vmcb->state.idt); 1701 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1702 &vmcb->state.ldt); 1703 svm_vcpu_setstate_seg(&state->segs[NVMM_X64_SEG_TR], 1704 &vmcb->state.tr); 1705 1706 vmcb->state.cpl = state->segs[NVMM_X64_SEG_SS].attrib.dpl; 1707 } 1708 1709 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1710 if (flags & NVMM_X64_STATE_GPRS) { 1711 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs)); 1712 1713 vmcb->state.rip = state->gprs[NVMM_X64_GPR_RIP]; 1714 vmcb->state.rsp = state->gprs[NVMM_X64_GPR_RSP]; 1715 vmcb->state.rax = state->gprs[NVMM_X64_GPR_RAX]; 1716 vmcb->state.rflags = state->gprs[NVMM_X64_GPR_RFLAGS]; 1717 } 1718 1719 if (flags & NVMM_X64_STATE_CRS) { 1720 vmcb->state.cr0 = state->crs[NVMM_X64_CR_CR0]; 1721 vmcb->state.cr2 = state->crs[NVMM_X64_CR_CR2]; 1722 vmcb->state.cr3 = state->crs[NVMM_X64_CR_CR3]; 1723 vmcb->state.cr4 = state->crs[NVMM_X64_CR_CR4]; 1724 1725 vmcb->ctrl.v &= ~VMCB_CTRL_V_TPR; 1726 vmcb->ctrl.v |= __SHIFTIN(state->crs[NVMM_X64_CR_CR8], 1727 VMCB_CTRL_V_TPR); 1728 1729 if (svm_xcr0_mask != 0) { 1730 /* Clear illegal XCR0 bits, set mandatory X87 bit. */ 1731 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0]; 1732 cpudata->gxcr0 &= svm_xcr0_mask; 1733 cpudata->gxcr0 |= XCR0_X87; 1734 } 1735 } 1736 1737 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1738 if (flags & NVMM_X64_STATE_DRS) { 1739 memcpy(cpudata->drs, state->drs, sizeof(state->drs)); 1740 1741 vmcb->state.dr6 = state->drs[NVMM_X64_DR_DR6]; 1742 vmcb->state.dr7 = state->drs[NVMM_X64_DR_DR7]; 1743 } 1744 1745 if (flags & NVMM_X64_STATE_MSRS) { 1746 /* 1747 * EFER_SVME is mandatory. 1748 */ 1749 vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME; 1750 vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR]; 1751 vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR]; 1752 vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR]; 1753 vmcb->state.sfmask = state->msrs[NVMM_X64_MSR_SFMASK]; 1754 vmcb->state.kernelgsbase = 1755 state->msrs[NVMM_X64_MSR_KERNELGSBASE]; 1756 vmcb->state.sysenter_cs = 1757 state->msrs[NVMM_X64_MSR_SYSENTER_CS]; 1758 vmcb->state.sysenter_esp = 1759 state->msrs[NVMM_X64_MSR_SYSENTER_ESP]; 1760 vmcb->state.sysenter_eip = 1761 state->msrs[NVMM_X64_MSR_SYSENTER_EIP]; 1762 vmcb->state.g_pat = state->msrs[NVMM_X64_MSR_PAT]; 1763 1764 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC]; 1765 cpudata->gtsc_want_update = true; 1766 } 1767 1768 if (flags & NVMM_X64_STATE_INTR) { 1769 if (state->intr.int_shadow) { 1770 vmcb->ctrl.intr |= VMCB_CTRL_INTR_SHADOW; 1771 } else { 1772 vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW; 1773 } 1774 1775 if (state->intr.int_window_exiting) { 1776 svm_event_waitexit_enable(vcpu, false); 1777 } else { 1778 svm_event_waitexit_disable(vcpu, false); 1779 } 1780 1781 if (state->intr.nmi_window_exiting) { 1782 svm_event_waitexit_enable(vcpu, true); 1783 } else { 1784 svm_event_waitexit_disable(vcpu, true); 1785 } 1786 } 1787 1788 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1789 if (flags & NVMM_X64_STATE_FPU) { 1790 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu, 1791 sizeof(state->fpu)); 1792 1793 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave; 1794 fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask; 1795 fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask; 1796 1797 if (svm_xcr0_mask != 0) { 1798 /* Reset XSTATE_BV, to force a reload. */ 1799 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 1800 } 1801 } 1802 1803 svm_vmcb_cache_update(vmcb, flags); 1804 1805 comm->state_wanted = 0; 1806 comm->state_cached |= flags; 1807 } 1808 1809 static void 1810 svm_vcpu_getstate(struct nvmm_cpu *vcpu) 1811 { 1812 struct nvmm_comm_page *comm = vcpu->comm; 1813 struct nvmm_x64_state *state = &comm->state; 1814 struct svm_cpudata *cpudata = vcpu->cpudata; 1815 struct vmcb *vmcb = cpudata->vmcb; 1816 uint64_t flags; 1817 1818 flags = comm->state_wanted; 1819 1820 if (flags & NVMM_X64_STATE_SEGS) { 1821 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_CS], 1822 &vmcb->state.cs); 1823 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_DS], 1824 &vmcb->state.ds); 1825 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_ES], 1826 &vmcb->state.es); 1827 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_FS], 1828 &vmcb->state.fs); 1829 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GS], 1830 &vmcb->state.gs); 1831 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_SS], 1832 &vmcb->state.ss); 1833 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_GDT], 1834 &vmcb->state.gdt); 1835 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_IDT], 1836 &vmcb->state.idt); 1837 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_LDT], 1838 &vmcb->state.ldt); 1839 svm_vcpu_getstate_seg(&state->segs[NVMM_X64_SEG_TR], 1840 &vmcb->state.tr); 1841 1842 state->segs[NVMM_X64_SEG_SS].attrib.dpl = vmcb->state.cpl; 1843 } 1844 1845 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs)); 1846 if (flags & NVMM_X64_STATE_GPRS) { 1847 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs)); 1848 1849 state->gprs[NVMM_X64_GPR_RIP] = vmcb->state.rip; 1850 state->gprs[NVMM_X64_GPR_RSP] = vmcb->state.rsp; 1851 state->gprs[NVMM_X64_GPR_RAX] = vmcb->state.rax; 1852 state->gprs[NVMM_X64_GPR_RFLAGS] = vmcb->state.rflags; 1853 } 1854 1855 if (flags & NVMM_X64_STATE_CRS) { 1856 state->crs[NVMM_X64_CR_CR0] = vmcb->state.cr0; 1857 state->crs[NVMM_X64_CR_CR2] = vmcb->state.cr2; 1858 state->crs[NVMM_X64_CR_CR3] = vmcb->state.cr3; 1859 state->crs[NVMM_X64_CR_CR4] = vmcb->state.cr4; 1860 state->crs[NVMM_X64_CR_CR8] = __SHIFTOUT(vmcb->ctrl.v, 1861 VMCB_CTRL_V_TPR); 1862 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0; 1863 } 1864 1865 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs)); 1866 if (flags & NVMM_X64_STATE_DRS) { 1867 memcpy(state->drs, cpudata->drs, sizeof(state->drs)); 1868 1869 state->drs[NVMM_X64_DR_DR6] = vmcb->state.dr6; 1870 state->drs[NVMM_X64_DR_DR7] = vmcb->state.dr7; 1871 } 1872 1873 if (flags & NVMM_X64_STATE_MSRS) { 1874 state->msrs[NVMM_X64_MSR_EFER] = vmcb->state.efer; 1875 state->msrs[NVMM_X64_MSR_STAR] = vmcb->state.star; 1876 state->msrs[NVMM_X64_MSR_LSTAR] = vmcb->state.lstar; 1877 state->msrs[NVMM_X64_MSR_CSTAR] = vmcb->state.cstar; 1878 state->msrs[NVMM_X64_MSR_SFMASK] = vmcb->state.sfmask; 1879 state->msrs[NVMM_X64_MSR_KERNELGSBASE] = 1880 vmcb->state.kernelgsbase; 1881 state->msrs[NVMM_X64_MSR_SYSENTER_CS] = 1882 vmcb->state.sysenter_cs; 1883 state->msrs[NVMM_X64_MSR_SYSENTER_ESP] = 1884 vmcb->state.sysenter_esp; 1885 state->msrs[NVMM_X64_MSR_SYSENTER_EIP] = 1886 vmcb->state.sysenter_eip; 1887 state->msrs[NVMM_X64_MSR_PAT] = vmcb->state.g_pat; 1888 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc; 1889 1890 /* Hide SVME. */ 1891 state->msrs[NVMM_X64_MSR_EFER] &= ~EFER_SVME; 1892 } 1893 1894 if (flags & NVMM_X64_STATE_INTR) { 1895 state->intr.int_shadow = 1896 (vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0; 1897 state->intr.int_window_exiting = cpudata->int_window_exit; 1898 state->intr.nmi_window_exiting = cpudata->nmi_window_exit; 1899 state->intr.evt_pending = cpudata->evt_pending; 1900 } 1901 1902 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu)); 1903 if (flags & NVMM_X64_STATE_FPU) { 1904 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave, 1905 sizeof(state->fpu)); 1906 } 1907 1908 comm->state_wanted = 0; 1909 comm->state_cached |= flags; 1910 } 1911 1912 static void 1913 svm_vcpu_state_provide(struct nvmm_cpu *vcpu, uint64_t flags) 1914 { 1915 vcpu->comm->state_wanted = flags; 1916 svm_vcpu_getstate(vcpu); 1917 } 1918 1919 static void 1920 svm_vcpu_state_commit(struct nvmm_cpu *vcpu) 1921 { 1922 vcpu->comm->state_wanted = vcpu->comm->state_commit; 1923 vcpu->comm->state_commit = 0; 1924 svm_vcpu_setstate(vcpu); 1925 } 1926 1927 /* -------------------------------------------------------------------------- */ 1928 1929 static void 1930 svm_asid_alloc(struct nvmm_cpu *vcpu) 1931 { 1932 struct svm_cpudata *cpudata = vcpu->cpudata; 1933 struct vmcb *vmcb = cpudata->vmcb; 1934 size_t i, oct, bit; 1935 1936 mutex_enter(&svm_asidlock); 1937 1938 for (i = 0; i < svm_maxasid; i++) { 1939 oct = i / 8; 1940 bit = i % 8; 1941 1942 if (svm_asidmap[oct] & __BIT(bit)) { 1943 continue; 1944 } 1945 1946 svm_asidmap[oct] |= __BIT(bit); 1947 vmcb->ctrl.guest_asid = i; 1948 mutex_exit(&svm_asidlock); 1949 return; 1950 } 1951 1952 /* 1953 * No free ASID. Use the last one, which is shared and requires 1954 * special TLB handling. 1955 */ 1956 cpudata->shared_asid = true; 1957 vmcb->ctrl.guest_asid = svm_maxasid - 1; 1958 mutex_exit(&svm_asidlock); 1959 } 1960 1961 static void 1962 svm_asid_free(struct nvmm_cpu *vcpu) 1963 { 1964 struct svm_cpudata *cpudata = vcpu->cpudata; 1965 struct vmcb *vmcb = cpudata->vmcb; 1966 size_t oct, bit; 1967 1968 if (cpudata->shared_asid) { 1969 return; 1970 } 1971 1972 oct = vmcb->ctrl.guest_asid / 8; 1973 bit = vmcb->ctrl.guest_asid % 8; 1974 1975 mutex_enter(&svm_asidlock); 1976 svm_asidmap[oct] &= ~__BIT(bit); 1977 mutex_exit(&svm_asidlock); 1978 } 1979 1980 static void 1981 svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 1982 { 1983 struct svm_cpudata *cpudata = vcpu->cpudata; 1984 struct vmcb *vmcb = cpudata->vmcb; 1985 1986 /* Allow reads/writes of Control Registers. */ 1987 vmcb->ctrl.intercept_cr = 0; 1988 1989 /* Allow reads/writes of Debug Registers. */ 1990 vmcb->ctrl.intercept_dr = 0; 1991 1992 /* Allow exceptions 0 to 31. */ 1993 vmcb->ctrl.intercept_vec = 0; 1994 1995 /* 1996 * Allow: 1997 * - SMI [smm interrupts] 1998 * - VINTR [virtual interrupts] 1999 * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP] 2000 * - RIDTR [reads of IDTR] 2001 * - RGDTR [reads of GDTR] 2002 * - RLDTR [reads of LDTR] 2003 * - RTR [reads of TR] 2004 * - WIDTR [writes of IDTR] 2005 * - WGDTR [writes of GDTR] 2006 * - WLDTR [writes of LDTR] 2007 * - WTR [writes of TR] 2008 * - RDTSC [rdtsc instruction] 2009 * - PUSHF [pushf instruction] 2010 * - POPF [popf instruction] 2011 * - IRET [iret instruction] 2012 * - INTN [int $n instructions] 2013 * - INVD [invd instruction] 2014 * - PAUSE [pause instruction] 2015 * - INVLPG [invplg instruction] 2016 * - TASKSW [task switches] 2017 * 2018 * Intercept the rest below. 2019 */ 2020 vmcb->ctrl.intercept_misc1 = 2021 VMCB_CTRL_INTERCEPT_INTR | 2022 VMCB_CTRL_INTERCEPT_NMI | 2023 VMCB_CTRL_INTERCEPT_INIT | 2024 VMCB_CTRL_INTERCEPT_RDPMC | 2025 VMCB_CTRL_INTERCEPT_CPUID | 2026 VMCB_CTRL_INTERCEPT_RSM | 2027 VMCB_CTRL_INTERCEPT_HLT | 2028 VMCB_CTRL_INTERCEPT_INVLPGA | 2029 VMCB_CTRL_INTERCEPT_IOIO_PROT | 2030 VMCB_CTRL_INTERCEPT_MSR_PROT | 2031 VMCB_CTRL_INTERCEPT_FERR_FREEZE | 2032 VMCB_CTRL_INTERCEPT_SHUTDOWN; 2033 2034 /* 2035 * Allow: 2036 * - ICEBP [icebp instruction] 2037 * - WBINVD [wbinvd instruction] 2038 * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction] 2039 * 2040 * Intercept the rest below. 2041 */ 2042 vmcb->ctrl.intercept_misc2 = 2043 VMCB_CTRL_INTERCEPT_VMRUN | 2044 VMCB_CTRL_INTERCEPT_VMMCALL | 2045 VMCB_CTRL_INTERCEPT_VMLOAD | 2046 VMCB_CTRL_INTERCEPT_VMSAVE | 2047 VMCB_CTRL_INTERCEPT_STGI | 2048 VMCB_CTRL_INTERCEPT_CLGI | 2049 VMCB_CTRL_INTERCEPT_SKINIT | 2050 VMCB_CTRL_INTERCEPT_RDTSCP | 2051 VMCB_CTRL_INTERCEPT_MONITOR | 2052 VMCB_CTRL_INTERCEPT_MWAIT | 2053 VMCB_CTRL_INTERCEPT_XSETBV | 2054 VMCB_CTRL_INTERCEPT_RDPRU; 2055 2056 /* 2057 * Intercept everything. 2058 */ 2059 vmcb->ctrl.intercept_misc3 = 2060 VMCB_CTRL_INTERCEPT_INVLPGB_ALL | 2061 VMCB_CTRL_INTERCEPT_PCID | 2062 VMCB_CTRL_INTERCEPT_MCOMMIT | 2063 VMCB_CTRL_INTERCEPT_TLBSYNC; 2064 2065 /* Intercept all I/O accesses. */ 2066 memset(cpudata->iobm, 0xFF, IOBM_SIZE); 2067 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa; 2068 2069 /* Allow direct access to certain MSRs. */ 2070 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE); 2071 svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false); 2072 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true); 2073 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true); 2074 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true); 2075 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true); 2076 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true); 2077 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true); 2078 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true); 2079 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true); 2080 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true); 2081 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true); 2082 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true); 2083 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false); 2084 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa; 2085 2086 /* Generate ASID. */ 2087 svm_asid_alloc(vcpu); 2088 2089 /* Virtual TPR. */ 2090 vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING; 2091 2092 /* Enable Nested Paging. */ 2093 vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP; 2094 vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0]; 2095 2096 /* Init XSAVE header. */ 2097 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask; 2098 cpudata->gfpu.xsh_xcomp_bv = 0; 2099 2100 /* These MSRs are static. */ 2101 cpudata->star = rdmsr(MSR_STAR); 2102 cpudata->lstar = rdmsr(MSR_LSTAR); 2103 cpudata->cstar = rdmsr(MSR_CSTAR); 2104 cpudata->sfmask = rdmsr(MSR_SFMASK); 2105 2106 /* Install the RESET state. */ 2107 memcpy(&vcpu->comm->state, &nvmm_x86_reset_state, 2108 sizeof(nvmm_x86_reset_state)); 2109 vcpu->comm->state_wanted = NVMM_X64_STATE_ALL; 2110 vcpu->comm->state_cached = 0; 2111 svm_vcpu_setstate(vcpu); 2112 } 2113 2114 static int 2115 svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2116 { 2117 struct svm_cpudata *cpudata; 2118 int error; 2119 2120 /* Allocate the SVM cpudata. */ 2121 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map, 2122 roundup(sizeof(*cpudata), PAGE_SIZE), 0, 2123 UVM_KMF_WIRED|UVM_KMF_ZERO); 2124 vcpu->cpudata = cpudata; 2125 2126 /* VMCB */ 2127 error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb, 2128 VMCB_NPAGES); 2129 if (error) 2130 goto error; 2131 2132 /* I/O Bitmap */ 2133 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm, 2134 IOBM_NPAGES); 2135 if (error) 2136 goto error; 2137 2138 /* MSR Bitmap */ 2139 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm, 2140 MSRBM_NPAGES); 2141 if (error) 2142 goto error; 2143 2144 /* Init the VCPU info. */ 2145 svm_vcpu_init(mach, vcpu); 2146 2147 return 0; 2148 2149 error: 2150 if (cpudata->vmcb_pa) { 2151 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, 2152 VMCB_NPAGES); 2153 } 2154 if (cpudata->iobm_pa) { 2155 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, 2156 IOBM_NPAGES); 2157 } 2158 if (cpudata->msrbm_pa) { 2159 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, 2160 MSRBM_NPAGES); 2161 } 2162 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2163 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2164 return error; 2165 } 2166 2167 static void 2168 svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu) 2169 { 2170 struct svm_cpudata *cpudata = vcpu->cpudata; 2171 2172 svm_asid_free(vcpu); 2173 2174 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES); 2175 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES); 2176 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES); 2177 2178 uvm_km_free(kernel_map, (vaddr_t)cpudata, 2179 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED); 2180 } 2181 2182 /* -------------------------------------------------------------------------- */ 2183 2184 static int 2185 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data) 2186 { 2187 struct nvmm_vcpu_conf_cpuid *cpuid = data; 2188 size_t i; 2189 2190 if (__predict_false(cpuid->mask && cpuid->exit)) { 2191 return EINVAL; 2192 } 2193 if (__predict_false(cpuid->mask && 2194 ((cpuid->u.mask.set.eax & cpuid->u.mask.del.eax) || 2195 (cpuid->u.mask.set.ebx & cpuid->u.mask.del.ebx) || 2196 (cpuid->u.mask.set.ecx & cpuid->u.mask.del.ecx) || 2197 (cpuid->u.mask.set.edx & cpuid->u.mask.del.edx)))) { 2198 return EINVAL; 2199 } 2200 2201 /* If unset, delete, to restore the default behavior. */ 2202 if (!cpuid->mask && !cpuid->exit) { 2203 for (i = 0; i < SVM_NCPUIDS; i++) { 2204 if (!cpudata->cpuidpresent[i]) { 2205 continue; 2206 } 2207 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2208 cpudata->cpuidpresent[i] = false; 2209 } 2210 } 2211 return 0; 2212 } 2213 2214 /* If already here, replace. */ 2215 for (i = 0; i < SVM_NCPUIDS; i++) { 2216 if (!cpudata->cpuidpresent[i]) { 2217 continue; 2218 } 2219 if (cpudata->cpuid[i].leaf == cpuid->leaf) { 2220 memcpy(&cpudata->cpuid[i], cpuid, 2221 sizeof(struct nvmm_vcpu_conf_cpuid)); 2222 return 0; 2223 } 2224 } 2225 2226 /* Not here, insert. */ 2227 for (i = 0; i < SVM_NCPUIDS; i++) { 2228 if (!cpudata->cpuidpresent[i]) { 2229 cpudata->cpuidpresent[i] = true; 2230 memcpy(&cpudata->cpuid[i], cpuid, 2231 sizeof(struct nvmm_vcpu_conf_cpuid)); 2232 return 0; 2233 } 2234 } 2235 2236 return ENOBUFS; 2237 } 2238 2239 static int 2240 svm_vcpu_configure(struct nvmm_cpu *vcpu, uint64_t op, void *data) 2241 { 2242 struct svm_cpudata *cpudata = vcpu->cpudata; 2243 2244 switch (op) { 2245 case NVMM_VCPU_CONF_MD(NVMM_VCPU_CONF_CPUID): 2246 return svm_vcpu_configure_cpuid(cpudata, data); 2247 default: 2248 return EINVAL; 2249 } 2250 } 2251 2252 /* -------------------------------------------------------------------------- */ 2253 2254 static void 2255 svm_tlb_flush(struct pmap *pm) 2256 { 2257 struct nvmm_machine *mach = pm->pm_data; 2258 struct svm_machdata *machdata = mach->machdata; 2259 2260 atomic_inc_64(&machdata->mach_htlb_gen); 2261 2262 /* Generates IPIs, which cause #VMEXITs. */ 2263 pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_UPDATE); 2264 } 2265 2266 static void 2267 svm_machine_create(struct nvmm_machine *mach) 2268 { 2269 struct svm_machdata *machdata; 2270 2271 /* Fill in pmap info. */ 2272 mach->vm->vm_map.pmap->pm_data = (void *)mach; 2273 mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush; 2274 2275 machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP); 2276 mach->machdata = machdata; 2277 2278 /* Start with an hTLB flush everywhere. */ 2279 machdata->mach_htlb_gen = 1; 2280 } 2281 2282 static void 2283 svm_machine_destroy(struct nvmm_machine *mach) 2284 { 2285 kmem_free(mach->machdata, sizeof(struct svm_machdata)); 2286 } 2287 2288 static int 2289 svm_machine_configure(struct nvmm_machine *mach, uint64_t op, void *data) 2290 { 2291 panic("%s: impossible", __func__); 2292 } 2293 2294 /* -------------------------------------------------------------------------- */ 2295 2296 static bool 2297 svm_ident(void) 2298 { 2299 u_int descs[4]; 2300 uint64_t msr; 2301 2302 if (cpu_vendor != CPUVENDOR_AMD) { 2303 return false; 2304 } 2305 if (!(cpu_feature[3] & CPUID_SVM)) { 2306 printf("NVMM: SVM not supported\n"); 2307 return false; 2308 } 2309 2310 if (curcpu()->ci_max_ext_cpuid < 0x8000000a) { 2311 printf("NVMM: CPUID leaf not available\n"); 2312 return false; 2313 } 2314 x86_cpuid(0x8000000a, descs); 2315 2316 /* Want Nested Paging. */ 2317 if (!(descs[3] & CPUID_AMD_SVM_NP)) { 2318 printf("NVMM: SVM-NP not supported\n"); 2319 return false; 2320 } 2321 2322 /* Want nRIP. */ 2323 if (!(descs[3] & CPUID_AMD_SVM_NRIPS)) { 2324 printf("NVMM: SVM-NRIPS not supported\n"); 2325 return false; 2326 } 2327 2328 svm_decode_assist = (descs[3] & CPUID_AMD_SVM_DecodeAssist) != 0; 2329 2330 msr = rdmsr(MSR_VMCR); 2331 if ((msr & VMCR_SVMED) && (msr & VMCR_LOCK)) { 2332 printf("NVMM: SVM disabled in BIOS\n"); 2333 return false; 2334 } 2335 2336 return true; 2337 } 2338 2339 static void 2340 svm_init_asid(uint32_t maxasid) 2341 { 2342 size_t i, j, allocsz; 2343 2344 mutex_init(&svm_asidlock, MUTEX_DEFAULT, IPL_NONE); 2345 2346 /* Arbitrarily limit. */ 2347 maxasid = uimin(maxasid, 8192); 2348 2349 svm_maxasid = maxasid; 2350 allocsz = roundup(maxasid, 8) / 8; 2351 svm_asidmap = kmem_zalloc(allocsz, KM_SLEEP); 2352 2353 /* ASID 0 is reserved for the host. */ 2354 svm_asidmap[0] |= __BIT(0); 2355 2356 /* ASID n-1 is special, we share it. */ 2357 i = (maxasid - 1) / 8; 2358 j = (maxasid - 1) % 8; 2359 svm_asidmap[i] |= __BIT(j); 2360 } 2361 2362 static void 2363 svm_change_cpu(void *arg1, void *arg2) 2364 { 2365 bool enable = arg1 != NULL; 2366 uint64_t msr; 2367 2368 msr = rdmsr(MSR_VMCR); 2369 if (msr & VMCR_SVMED) { 2370 wrmsr(MSR_VMCR, msr & ~VMCR_SVMED); 2371 } 2372 2373 if (!enable) { 2374 wrmsr(MSR_VM_HSAVE_PA, 0); 2375 } 2376 2377 msr = rdmsr(MSR_EFER); 2378 if (enable) { 2379 msr |= EFER_SVME; 2380 } else { 2381 msr &= ~EFER_SVME; 2382 } 2383 wrmsr(MSR_EFER, msr); 2384 2385 if (enable) { 2386 wrmsr(MSR_VM_HSAVE_PA, hsave[cpu_index(curcpu())].pa); 2387 } 2388 } 2389 2390 static void 2391 svm_init(void) 2392 { 2393 CPU_INFO_ITERATOR cii; 2394 struct cpu_info *ci; 2395 struct vm_page *pg; 2396 u_int descs[4]; 2397 uint64_t xc; 2398 2399 x86_cpuid(0x8000000a, descs); 2400 2401 /* The guest TLB flush command. */ 2402 if (descs[3] & CPUID_AMD_SVM_FlushByASID) { 2403 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST; 2404 } else { 2405 svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL; 2406 } 2407 2408 /* Init the ASID. */ 2409 svm_init_asid(descs[1]); 2410 2411 /* Init the XCR0 mask. */ 2412 svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features; 2413 2414 memset(hsave, 0, sizeof(hsave)); 2415 for (CPU_INFO_FOREACH(cii, ci)) { 2416 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2417 hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg); 2418 } 2419 2420 xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL); 2421 xc_wait(xc); 2422 } 2423 2424 static void 2425 svm_fini_asid(void) 2426 { 2427 size_t allocsz; 2428 2429 allocsz = roundup(svm_maxasid, 8) / 8; 2430 kmem_free(svm_asidmap, allocsz); 2431 2432 mutex_destroy(&svm_asidlock); 2433 } 2434 2435 static void 2436 svm_fini(void) 2437 { 2438 uint64_t xc; 2439 size_t i; 2440 2441 xc = xc_broadcast(0, svm_change_cpu, (void *)false, NULL); 2442 xc_wait(xc); 2443 2444 for (i = 0; i < MAXCPUS; i++) { 2445 if (hsave[i].pa != 0) 2446 uvm_pagefree(PHYS_TO_VM_PAGE(hsave[i].pa)); 2447 } 2448 2449 svm_fini_asid(); 2450 } 2451 2452 static void 2453 svm_capability(struct nvmm_capability *cap) 2454 { 2455 cap->arch.mach_conf_support = 0; 2456 cap->arch.vcpu_conf_support = 2457 NVMM_CAP_ARCH_VCPU_CONF_CPUID; 2458 cap->arch.xcr0_mask = svm_xcr0_mask; 2459 cap->arch.mxcsr_mask = x86_fpu_mxcsr_mask; 2460 cap->arch.conf_cpuid_maxops = SVM_NCPUIDS; 2461 } 2462 2463 const struct nvmm_impl nvmm_x86_svm = { 2464 .name = "x86-svm", 2465 .ident = svm_ident, 2466 .init = svm_init, 2467 .fini = svm_fini, 2468 .capability = svm_capability, 2469 .mach_conf_max = NVMM_X86_MACH_NCONF, 2470 .mach_conf_sizes = NULL, 2471 .vcpu_conf_max = NVMM_X86_VCPU_NCONF, 2472 .vcpu_conf_sizes = svm_vcpu_conf_sizes, 2473 .state_size = sizeof(struct nvmm_x64_state), 2474 .machine_create = svm_machine_create, 2475 .machine_destroy = svm_machine_destroy, 2476 .machine_configure = svm_machine_configure, 2477 .vcpu_create = svm_vcpu_create, 2478 .vcpu_destroy = svm_vcpu_destroy, 2479 .vcpu_configure = svm_vcpu_configure, 2480 .vcpu_setstate = svm_vcpu_setstate, 2481 .vcpu_getstate = svm_vcpu_getstate, 2482 .vcpu_inject = svm_vcpu_inject, 2483 .vcpu_run = svm_vcpu_run 2484 }; 2485
Indexes created Tue Oct 28 02:10:10 GMT 2025