libnvmm_x86.c revision 1.40
1 /* $NetBSD: libnvmm_x86.c,v 1.40 2020/09/05 07:22:25 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 33 #include <stdio.h> 34 #include <stdlib.h> 35 #include <string.h> 36 #include <unistd.h> 37 #include <fcntl.h> 38 #include <errno.h> 39 #include <sys/ioctl.h> 40 #include <sys/mman.h> 41 #include <machine/vmparam.h> 42 #include <machine/pte.h> 43 #include <machine/psl.h> 44 45 #define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) 46 #define __cacheline_aligned __attribute__((__aligned__(64))) 47 48 #include <x86/specialreg.h> 49 50 /* -------------------------------------------------------------------------- */ 51 52 /* 53 * Undocumented debugging function. Helpful. 54 */ 55 int 56 nvmm_vcpu_dump(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 57 { 58 struct nvmm_x64_state *state = vcpu->state; 59 uint16_t *attr; 60 size_t i; 61 int ret; 62 63 const char *segnames[] = { 64 "ES", "CS", "SS", "DS", "FS", "GS", "GDT", "IDT", "LDT", "TR" 65 }; 66 67 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_ALL); 68 if (ret == -1) 69 return -1; 70 71 printf("+ VCPU id=%d\n", (int)vcpu->cpuid); 72 printf("| -> RAX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RAX]); 73 printf("| -> RCX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RCX]); 74 printf("| -> RDX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDX]); 75 printf("| -> RBX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBX]); 76 printf("| -> RSP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSP]); 77 printf("| -> RBP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBP]); 78 printf("| -> RSI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSI]); 79 printf("| -> RDI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDI]); 80 printf("| -> RIP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RIP]); 81 printf("| -> RFLAGS=%p\n", (void *)state->gprs[NVMM_X64_GPR_RFLAGS]); 82 for (i = 0; i < NVMM_X64_NSEG; i++) { 83 attr = (uint16_t *)&state->segs[i].attrib; 84 printf("| -> %s: sel=0x%x base=%"PRIx64", limit=%x, " 85 "attrib=%x [type=%d,l=%d,def=%d]\n", 86 segnames[i], 87 state->segs[i].selector, 88 state->segs[i].base, 89 state->segs[i].limit, 90 *attr, 91 state->segs[i].attrib.type, 92 state->segs[i].attrib.l, 93 state->segs[i].attrib.def); 94 } 95 printf("| -> MSR_EFER=%"PRIx64"\n", state->msrs[NVMM_X64_MSR_EFER]); 96 printf("| -> CR0=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR0]); 97 printf("| -> CR3=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR3]); 98 printf("| -> CR4=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR4]); 99 printf("| -> CR8=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR8]); 100 101 return 0; 102 } 103 104 /* -------------------------------------------------------------------------- */ 105 106 #define PTE32_L1_SHIFT 12 107 #define PTE32_L2_SHIFT 22 108 109 #define PTE32_L2_MASK 0xffc00000 110 #define PTE32_L1_MASK 0x003ff000 111 112 #define PTE32_L2_FRAME (PTE32_L2_MASK) 113 #define PTE32_L1_FRAME (PTE32_L2_FRAME|PTE32_L1_MASK) 114 115 #define pte32_l1idx(va) (((va) & PTE32_L1_MASK) >> PTE32_L1_SHIFT) 116 #define pte32_l2idx(va) (((va) & PTE32_L2_MASK) >> PTE32_L2_SHIFT) 117 118 #define CR3_FRAME_32BIT __BITS(31, 12) 119 120 typedef uint32_t pte_32bit_t; 121 122 static int 123 x86_gva_to_gpa_32bit(struct nvmm_machine *mach, uint64_t cr3, 124 gvaddr_t gva, gpaddr_t *gpa, bool has_pse, nvmm_prot_t *prot) 125 { 126 gpaddr_t L2gpa, L1gpa; 127 uintptr_t L2hva, L1hva; 128 pte_32bit_t *pdir, pte; 129 nvmm_prot_t pageprot; 130 131 /* We begin with an RWXU access. */ 132 *prot = NVMM_PROT_ALL; 133 134 /* Parse L2. */ 135 L2gpa = (cr3 & CR3_FRAME_32BIT); 136 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 137 return -1; 138 pdir = (pte_32bit_t *)L2hva; 139 pte = pdir[pte32_l2idx(gva)]; 140 if ((pte & PTE_P) == 0) 141 return -1; 142 if ((pte & PTE_U) == 0) 143 *prot &= ~NVMM_PROT_USER; 144 if ((pte & PTE_W) == 0) 145 *prot &= ~NVMM_PROT_WRITE; 146 if ((pte & PTE_PS) && !has_pse) 147 return -1; 148 if (pte & PTE_PS) { 149 *gpa = (pte & PTE32_L2_FRAME); 150 *gpa = *gpa + (gva & PTE32_L1_MASK); 151 return 0; 152 } 153 154 /* Parse L1. */ 155 L1gpa = (pte & PTE_FRAME); 156 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 157 return -1; 158 pdir = (pte_32bit_t *)L1hva; 159 pte = pdir[pte32_l1idx(gva)]; 160 if ((pte & PTE_P) == 0) 161 return -1; 162 if ((pte & PTE_U) == 0) 163 *prot &= ~NVMM_PROT_USER; 164 if ((pte & PTE_W) == 0) 165 *prot &= ~NVMM_PROT_WRITE; 166 if (pte & PTE_PS) 167 return -1; 168 169 *gpa = (pte & PTE_FRAME); 170 return 0; 171 } 172 173 /* -------------------------------------------------------------------------- */ 174 175 #define PTE32_PAE_L1_SHIFT 12 176 #define PTE32_PAE_L2_SHIFT 21 177 #define PTE32_PAE_L3_SHIFT 30 178 179 #define PTE32_PAE_L3_MASK 0xc0000000 180 #define PTE32_PAE_L2_MASK 0x3fe00000 181 #define PTE32_PAE_L1_MASK 0x001ff000 182 183 #define PTE32_PAE_L3_FRAME (PTE32_PAE_L3_MASK) 184 #define PTE32_PAE_L2_FRAME (PTE32_PAE_L3_FRAME|PTE32_PAE_L2_MASK) 185 #define PTE32_PAE_L1_FRAME (PTE32_PAE_L2_FRAME|PTE32_PAE_L1_MASK) 186 187 #define pte32_pae_l1idx(va) (((va) & PTE32_PAE_L1_MASK) >> PTE32_PAE_L1_SHIFT) 188 #define pte32_pae_l2idx(va) (((va) & PTE32_PAE_L2_MASK) >> PTE32_PAE_L2_SHIFT) 189 #define pte32_pae_l3idx(va) (((va) & PTE32_PAE_L3_MASK) >> PTE32_PAE_L3_SHIFT) 190 191 #define CR3_FRAME_32BIT_PAE __BITS(31, 5) 192 193 typedef uint64_t pte_32bit_pae_t; 194 195 static int 196 x86_gva_to_gpa_32bit_pae(struct nvmm_machine *mach, uint64_t cr3, 197 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 198 { 199 gpaddr_t L3gpa, L2gpa, L1gpa; 200 uintptr_t L3hva, L2hva, L1hva; 201 pte_32bit_pae_t *pdir, pte; 202 nvmm_prot_t pageprot; 203 204 /* We begin with an RWXU access. */ 205 *prot = NVMM_PROT_ALL; 206 207 /* Parse L3. */ 208 L3gpa = (cr3 & CR3_FRAME_32BIT_PAE); 209 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 210 return -1; 211 pdir = (pte_32bit_pae_t *)L3hva; 212 pte = pdir[pte32_pae_l3idx(gva)]; 213 if ((pte & PTE_P) == 0) 214 return -1; 215 if (pte & PTE_NX) 216 *prot &= ~NVMM_PROT_EXEC; 217 if (pte & PTE_PS) 218 return -1; 219 220 /* Parse L2. */ 221 L2gpa = (pte & PTE_FRAME); 222 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 223 return -1; 224 pdir = (pte_32bit_pae_t *)L2hva; 225 pte = pdir[pte32_pae_l2idx(gva)]; 226 if ((pte & PTE_P) == 0) 227 return -1; 228 if ((pte & PTE_U) == 0) 229 *prot &= ~NVMM_PROT_USER; 230 if ((pte & PTE_W) == 0) 231 *prot &= ~NVMM_PROT_WRITE; 232 if (pte & PTE_NX) 233 *prot &= ~NVMM_PROT_EXEC; 234 if (pte & PTE_PS) { 235 *gpa = (pte & PTE32_PAE_L2_FRAME); 236 *gpa = *gpa + (gva & PTE32_PAE_L1_MASK); 237 return 0; 238 } 239 240 /* Parse L1. */ 241 L1gpa = (pte & PTE_FRAME); 242 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 243 return -1; 244 pdir = (pte_32bit_pae_t *)L1hva; 245 pte = pdir[pte32_pae_l1idx(gva)]; 246 if ((pte & PTE_P) == 0) 247 return -1; 248 if ((pte & PTE_U) == 0) 249 *prot &= ~NVMM_PROT_USER; 250 if ((pte & PTE_W) == 0) 251 *prot &= ~NVMM_PROT_WRITE; 252 if (pte & PTE_NX) 253 *prot &= ~NVMM_PROT_EXEC; 254 if (pte & PTE_PS) 255 return -1; 256 257 *gpa = (pte & PTE_FRAME); 258 return 0; 259 } 260 261 /* -------------------------------------------------------------------------- */ 262 263 #define PTE64_L1_SHIFT 12 264 #define PTE64_L2_SHIFT 21 265 #define PTE64_L3_SHIFT 30 266 #define PTE64_L4_SHIFT 39 267 268 #define PTE64_L4_MASK 0x0000ff8000000000 269 #define PTE64_L3_MASK 0x0000007fc0000000 270 #define PTE64_L2_MASK 0x000000003fe00000 271 #define PTE64_L1_MASK 0x00000000001ff000 272 273 #define PTE64_L4_FRAME PTE64_L4_MASK 274 #define PTE64_L3_FRAME (PTE64_L4_FRAME|PTE64_L3_MASK) 275 #define PTE64_L2_FRAME (PTE64_L3_FRAME|PTE64_L2_MASK) 276 #define PTE64_L1_FRAME (PTE64_L2_FRAME|PTE64_L1_MASK) 277 278 #define pte64_l1idx(va) (((va) & PTE64_L1_MASK) >> PTE64_L1_SHIFT) 279 #define pte64_l2idx(va) (((va) & PTE64_L2_MASK) >> PTE64_L2_SHIFT) 280 #define pte64_l3idx(va) (((va) & PTE64_L3_MASK) >> PTE64_L3_SHIFT) 281 #define pte64_l4idx(va) (((va) & PTE64_L4_MASK) >> PTE64_L4_SHIFT) 282 283 #define CR3_FRAME_64BIT __BITS(51, 12) 284 285 typedef uint64_t pte_64bit_t; 286 287 static inline bool 288 x86_gva_64bit_canonical(gvaddr_t gva) 289 { 290 /* Bits 63:47 must have the same value. */ 291 #define SIGN_EXTEND 0xffff800000000000ULL 292 return (gva & SIGN_EXTEND) == 0 || (gva & SIGN_EXTEND) == SIGN_EXTEND; 293 } 294 295 static int 296 x86_gva_to_gpa_64bit(struct nvmm_machine *mach, uint64_t cr3, 297 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 298 { 299 gpaddr_t L4gpa, L3gpa, L2gpa, L1gpa; 300 uintptr_t L4hva, L3hva, L2hva, L1hva; 301 pte_64bit_t *pdir, pte; 302 nvmm_prot_t pageprot; 303 304 /* We begin with an RWXU access. */ 305 *prot = NVMM_PROT_ALL; 306 307 if (!x86_gva_64bit_canonical(gva)) 308 return -1; 309 310 /* Parse L4. */ 311 L4gpa = (cr3 & CR3_FRAME_64BIT); 312 if (nvmm_gpa_to_hva(mach, L4gpa, &L4hva, &pageprot) == -1) 313 return -1; 314 pdir = (pte_64bit_t *)L4hva; 315 pte = pdir[pte64_l4idx(gva)]; 316 if ((pte & PTE_P) == 0) 317 return -1; 318 if ((pte & PTE_U) == 0) 319 *prot &= ~NVMM_PROT_USER; 320 if ((pte & PTE_W) == 0) 321 *prot &= ~NVMM_PROT_WRITE; 322 if (pte & PTE_NX) 323 *prot &= ~NVMM_PROT_EXEC; 324 if (pte & PTE_PS) 325 return -1; 326 327 /* Parse L3. */ 328 L3gpa = (pte & PTE_FRAME); 329 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 330 return -1; 331 pdir = (pte_64bit_t *)L3hva; 332 pte = pdir[pte64_l3idx(gva)]; 333 if ((pte & PTE_P) == 0) 334 return -1; 335 if ((pte & PTE_U) == 0) 336 *prot &= ~NVMM_PROT_USER; 337 if ((pte & PTE_W) == 0) 338 *prot &= ~NVMM_PROT_WRITE; 339 if (pte & PTE_NX) 340 *prot &= ~NVMM_PROT_EXEC; 341 if (pte & PTE_PS) { 342 *gpa = (pte & PTE64_L3_FRAME); 343 *gpa = *gpa + (gva & (PTE64_L2_MASK|PTE64_L1_MASK)); 344 return 0; 345 } 346 347 /* Parse L2. */ 348 L2gpa = (pte & PTE_FRAME); 349 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 350 return -1; 351 pdir = (pte_64bit_t *)L2hva; 352 pte = pdir[pte64_l2idx(gva)]; 353 if ((pte & PTE_P) == 0) 354 return -1; 355 if ((pte & PTE_U) == 0) 356 *prot &= ~NVMM_PROT_USER; 357 if ((pte & PTE_W) == 0) 358 *prot &= ~NVMM_PROT_WRITE; 359 if (pte & PTE_NX) 360 *prot &= ~NVMM_PROT_EXEC; 361 if (pte & PTE_PS) { 362 *gpa = (pte & PTE64_L2_FRAME); 363 *gpa = *gpa + (gva & PTE64_L1_MASK); 364 return 0; 365 } 366 367 /* Parse L1. */ 368 L1gpa = (pte & PTE_FRAME); 369 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 370 return -1; 371 pdir = (pte_64bit_t *)L1hva; 372 pte = pdir[pte64_l1idx(gva)]; 373 if ((pte & PTE_P) == 0) 374 return -1; 375 if ((pte & PTE_U) == 0) 376 *prot &= ~NVMM_PROT_USER; 377 if ((pte & PTE_W) == 0) 378 *prot &= ~NVMM_PROT_WRITE; 379 if (pte & PTE_NX) 380 *prot &= ~NVMM_PROT_EXEC; 381 if (pte & PTE_PS) 382 return -1; 383 384 *gpa = (pte & PTE_FRAME); 385 return 0; 386 } 387 388 static inline int 389 x86_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_x64_state *state, 390 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 391 { 392 bool is_pae, is_lng, has_pse; 393 uint64_t cr3; 394 size_t off; 395 int ret; 396 397 if ((state->crs[NVMM_X64_CR_CR0] & CR0_PG) == 0) { 398 /* No paging. */ 399 *prot = NVMM_PROT_ALL; 400 *gpa = gva; 401 return 0; 402 } 403 404 off = (gva & PAGE_MASK); 405 gva &= ~PAGE_MASK; 406 407 is_pae = (state->crs[NVMM_X64_CR_CR4] & CR4_PAE) != 0; 408 is_lng = (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 409 has_pse = (state->crs[NVMM_X64_CR_CR4] & CR4_PSE) != 0; 410 cr3 = state->crs[NVMM_X64_CR_CR3]; 411 412 if (is_pae && is_lng) { 413 /* 64bit */ 414 ret = x86_gva_to_gpa_64bit(mach, cr3, gva, gpa, prot); 415 } else if (is_pae && !is_lng) { 416 /* 32bit PAE */ 417 ret = x86_gva_to_gpa_32bit_pae(mach, cr3, gva, gpa, prot); 418 } else if (!is_pae && !is_lng) { 419 /* 32bit */ 420 ret = x86_gva_to_gpa_32bit(mach, cr3, gva, gpa, has_pse, prot); 421 } else { 422 ret = -1; 423 } 424 425 if (ret == -1) { 426 errno = EFAULT; 427 } 428 429 *gpa = *gpa + off; 430 431 return ret; 432 } 433 434 int 435 nvmm_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 436 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 437 { 438 struct nvmm_x64_state *state = vcpu->state; 439 int ret; 440 441 ret = nvmm_vcpu_getstate(mach, vcpu, 442 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 443 if (ret == -1) 444 return -1; 445 446 return x86_gva_to_gpa(mach, state, gva, gpa, prot); 447 } 448 449 /* -------------------------------------------------------------------------- */ 450 451 #define DISASSEMBLER_BUG() \ 452 do { \ 453 errno = EINVAL; \ 454 return -1; \ 455 } while (0); 456 457 static inline bool 458 is_long_mode(struct nvmm_x64_state *state) 459 { 460 return (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 461 } 462 463 static inline bool 464 is_64bit(struct nvmm_x64_state *state) 465 { 466 return (state->segs[NVMM_X64_SEG_CS].attrib.l != 0); 467 } 468 469 static inline bool 470 is_32bit(struct nvmm_x64_state *state) 471 { 472 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 473 (state->segs[NVMM_X64_SEG_CS].attrib.def == 1); 474 } 475 476 static inline bool 477 is_16bit(struct nvmm_x64_state *state) 478 { 479 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 480 (state->segs[NVMM_X64_SEG_CS].attrib.def == 0); 481 } 482 483 static int 484 segment_check(struct nvmm_x64_state_seg *seg, gvaddr_t gva, size_t size) 485 { 486 uint64_t limit; 487 488 /* 489 * This is incomplete. We should check topdown, etc, really that's 490 * tiring. 491 */ 492 if (__predict_false(!seg->attrib.p)) { 493 goto error; 494 } 495 496 limit = (uint64_t)seg->limit + 1; 497 if (__predict_true(seg->attrib.g)) { 498 limit *= PAGE_SIZE; 499 } 500 501 if (__predict_false(gva + size > limit)) { 502 goto error; 503 } 504 505 return 0; 506 507 error: 508 errno = EFAULT; 509 return -1; 510 } 511 512 static inline void 513 segment_apply(struct nvmm_x64_state_seg *seg, gvaddr_t *gva) 514 { 515 *gva += seg->base; 516 } 517 518 static inline uint64_t 519 size_to_mask(size_t size) 520 { 521 switch (size) { 522 case 1: 523 return 0x00000000000000FF; 524 case 2: 525 return 0x000000000000FFFF; 526 case 4: 527 return 0x00000000FFFFFFFF; 528 case 8: 529 default: 530 return 0xFFFFFFFFFFFFFFFF; 531 } 532 } 533 534 static uint64_t 535 rep_get_cnt(struct nvmm_x64_state *state, size_t adsize) 536 { 537 uint64_t mask, cnt; 538 539 mask = size_to_mask(adsize); 540 cnt = state->gprs[NVMM_X64_GPR_RCX] & mask; 541 542 return cnt; 543 } 544 545 static void 546 rep_set_cnt(struct nvmm_x64_state *state, size_t adsize, uint64_t cnt) 547 { 548 uint64_t mask; 549 550 /* XXX: should we zero-extend? */ 551 mask = size_to_mask(adsize); 552 state->gprs[NVMM_X64_GPR_RCX] &= ~mask; 553 state->gprs[NVMM_X64_GPR_RCX] |= cnt; 554 } 555 556 static int 557 read_guest_memory(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 558 gvaddr_t gva, uint8_t *data, size_t size) 559 { 560 struct nvmm_x64_state *state = vcpu->state; 561 struct nvmm_mem mem; 562 nvmm_prot_t prot; 563 gpaddr_t gpa; 564 uintptr_t hva; 565 bool is_mmio; 566 int ret, remain; 567 568 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 569 if (__predict_false(ret == -1)) { 570 return -1; 571 } 572 if (__predict_false(!(prot & NVMM_PROT_READ))) { 573 errno = EFAULT; 574 return -1; 575 } 576 577 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 578 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 579 } else { 580 remain = 0; 581 } 582 size -= remain; 583 584 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 585 is_mmio = (ret == -1); 586 587 if (is_mmio) { 588 mem.mach = mach; 589 mem.vcpu = vcpu; 590 mem.data = data; 591 mem.gpa = gpa; 592 mem.write = false; 593 mem.size = size; 594 (*vcpu->cbs.mem)(&mem); 595 } else { 596 if (__predict_false(!(prot & NVMM_PROT_READ))) { 597 errno = EFAULT; 598 return -1; 599 } 600 memcpy(data, (uint8_t *)hva, size); 601 } 602 603 if (remain > 0) { 604 ret = read_guest_memory(mach, vcpu, gva + size, 605 data + size, remain); 606 } else { 607 ret = 0; 608 } 609 610 return ret; 611 } 612 613 static int 614 write_guest_memory(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 615 gvaddr_t gva, uint8_t *data, size_t size) 616 { 617 struct nvmm_x64_state *state = vcpu->state; 618 struct nvmm_mem mem; 619 nvmm_prot_t prot; 620 gpaddr_t gpa; 621 uintptr_t hva; 622 bool is_mmio; 623 int ret, remain; 624 625 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 626 if (__predict_false(ret == -1)) { 627 return -1; 628 } 629 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 630 errno = EFAULT; 631 return -1; 632 } 633 634 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 635 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 636 } else { 637 remain = 0; 638 } 639 size -= remain; 640 641 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 642 is_mmio = (ret == -1); 643 644 if (is_mmio) { 645 mem.mach = mach; 646 mem.vcpu = vcpu; 647 mem.data = data; 648 mem.gpa = gpa; 649 mem.write = true; 650 mem.size = size; 651 (*vcpu->cbs.mem)(&mem); 652 } else { 653 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 654 errno = EFAULT; 655 return -1; 656 } 657 memcpy((uint8_t *)hva, data, size); 658 } 659 660 if (remain > 0) { 661 ret = write_guest_memory(mach, vcpu, gva + size, 662 data + size, remain); 663 } else { 664 ret = 0; 665 } 666 667 return ret; 668 } 669 670 /* -------------------------------------------------------------------------- */ 671 672 static int fetch_segment(struct nvmm_machine *, struct nvmm_vcpu *); 673 674 #define NVMM_IO_BATCH_SIZE 32 675 676 static int 677 assist_io_batch(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 678 struct nvmm_io *io, gvaddr_t gva, uint64_t cnt) 679 { 680 uint8_t iobuf[NVMM_IO_BATCH_SIZE]; 681 size_t i, iosize, iocnt; 682 int ret; 683 684 cnt = MIN(cnt, NVMM_IO_BATCH_SIZE); 685 iosize = MIN(io->size * cnt, NVMM_IO_BATCH_SIZE); 686 iocnt = iosize / io->size; 687 688 io->data = iobuf; 689 690 if (!io->in) { 691 ret = read_guest_memory(mach, vcpu, gva, iobuf, iosize); 692 if (ret == -1) 693 return -1; 694 } 695 696 for (i = 0; i < iocnt; i++) { 697 (*vcpu->cbs.io)(io); 698 io->data += io->size; 699 } 700 701 if (io->in) { 702 ret = write_guest_memory(mach, vcpu, gva, iobuf, iosize); 703 if (ret == -1) 704 return -1; 705 } 706 707 return iocnt; 708 } 709 710 int 711 nvmm_assist_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 712 { 713 struct nvmm_x64_state *state = vcpu->state; 714 struct nvmm_vcpu_exit *exit = vcpu->exit; 715 struct nvmm_io io; 716 uint64_t cnt = 0; /* GCC */ 717 uint8_t iobuf[8]; 718 int iocnt = 1; 719 gvaddr_t gva = 0; /* GCC */ 720 int reg = 0; /* GCC */ 721 int ret, seg; 722 bool psld = false; 723 724 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_IO)) { 725 errno = EINVAL; 726 return -1; 727 } 728 729 io.mach = mach; 730 io.vcpu = vcpu; 731 io.port = exit->u.io.port; 732 io.in = exit->u.io.in; 733 io.size = exit->u.io.operand_size; 734 io.data = iobuf; 735 736 ret = nvmm_vcpu_getstate(mach, vcpu, 737 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 738 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 739 if (ret == -1) 740 return -1; 741 742 if (exit->u.io.rep) { 743 cnt = rep_get_cnt(state, exit->u.io.address_size); 744 if (__predict_false(cnt == 0)) { 745 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 746 goto out; 747 } 748 } 749 750 if (__predict_false(state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_D)) { 751 psld = true; 752 } 753 754 /* 755 * Determine GVA. 756 */ 757 if (exit->u.io.str) { 758 if (io.in) { 759 reg = NVMM_X64_GPR_RDI; 760 } else { 761 reg = NVMM_X64_GPR_RSI; 762 } 763 764 gva = state->gprs[reg]; 765 gva &= size_to_mask(exit->u.io.address_size); 766 767 if (exit->u.io.seg != -1) { 768 seg = exit->u.io.seg; 769 } else { 770 if (io.in) { 771 seg = NVMM_X64_SEG_ES; 772 } else { 773 seg = fetch_segment(mach, vcpu); 774 if (seg == -1) 775 return -1; 776 } 777 } 778 779 if (__predict_true(is_long_mode(state))) { 780 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 781 segment_apply(&state->segs[seg], &gva); 782 } 783 } else { 784 ret = segment_check(&state->segs[seg], gva, io.size); 785 if (ret == -1) 786 return -1; 787 segment_apply(&state->segs[seg], &gva); 788 } 789 790 if (exit->u.io.rep && !psld) { 791 iocnt = assist_io_batch(mach, vcpu, &io, gva, cnt); 792 if (iocnt == -1) 793 return -1; 794 goto done; 795 } 796 } 797 798 if (!io.in) { 799 if (!exit->u.io.str) { 800 memcpy(io.data, &state->gprs[NVMM_X64_GPR_RAX], io.size); 801 } else { 802 ret = read_guest_memory(mach, vcpu, gva, io.data, 803 io.size); 804 if (ret == -1) 805 return -1; 806 } 807 } 808 809 (*vcpu->cbs.io)(&io); 810 811 if (io.in) { 812 if (!exit->u.io.str) { 813 memcpy(&state->gprs[NVMM_X64_GPR_RAX], io.data, io.size); 814 if (io.size == 4) { 815 /* Zero-extend to 64 bits. */ 816 state->gprs[NVMM_X64_GPR_RAX] &= size_to_mask(4); 817 } 818 } else { 819 ret = write_guest_memory(mach, vcpu, gva, io.data, 820 io.size); 821 if (ret == -1) 822 return -1; 823 } 824 } 825 826 done: 827 if (exit->u.io.str) { 828 if (__predict_false(psld)) { 829 state->gprs[reg] -= iocnt * io.size; 830 } else { 831 state->gprs[reg] += iocnt * io.size; 832 } 833 } 834 835 if (exit->u.io.rep) { 836 cnt -= iocnt; 837 rep_set_cnt(state, exit->u.io.address_size, cnt); 838 if (cnt == 0) { 839 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 840 } 841 } else { 842 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 843 } 844 845 out: 846 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 847 if (ret == -1) 848 return -1; 849 850 return 0; 851 } 852 853 /* -------------------------------------------------------------------------- */ 854 855 struct x86_emul { 856 bool readreg; 857 bool backprop; 858 bool notouch; 859 void (*func)(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 860 }; 861 862 static void x86_func_or(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 863 static void x86_func_and(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 864 static void x86_func_xchg(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 865 static void x86_func_sub(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 866 static void x86_func_xor(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 867 static void x86_func_cmp(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 868 static void x86_func_test(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 869 static void x86_func_mov(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 870 static void x86_func_stos(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 871 static void x86_func_lods(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 872 static void x86_func_movs(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 873 874 static const struct x86_emul x86_emul_or = { 875 .readreg = true, 876 .func = x86_func_or 877 }; 878 879 static const struct x86_emul x86_emul_and = { 880 .readreg = true, 881 .func = x86_func_and 882 }; 883 884 static const struct x86_emul x86_emul_xchg = { 885 .readreg = true, 886 .backprop = true, 887 .func = x86_func_xchg 888 }; 889 890 static const struct x86_emul x86_emul_sub = { 891 .readreg = true, 892 .func = x86_func_sub 893 }; 894 895 static const struct x86_emul x86_emul_xor = { 896 .readreg = true, 897 .func = x86_func_xor 898 }; 899 900 static const struct x86_emul x86_emul_cmp = { 901 .notouch = true, 902 .func = x86_func_cmp 903 }; 904 905 static const struct x86_emul x86_emul_test = { 906 .notouch = true, 907 .func = x86_func_test 908 }; 909 910 static const struct x86_emul x86_emul_mov = { 911 .func = x86_func_mov 912 }; 913 914 static const struct x86_emul x86_emul_stos = { 915 .func = x86_func_stos 916 }; 917 918 static const struct x86_emul x86_emul_lods = { 919 .func = x86_func_lods 920 }; 921 922 static const struct x86_emul x86_emul_movs = { 923 .func = x86_func_movs 924 }; 925 926 /* Legacy prefixes. */ 927 #define LEG_LOCK 0xF0 928 #define LEG_REPN 0xF2 929 #define LEG_REP 0xF3 930 #define LEG_OVR_CS 0x2E 931 #define LEG_OVR_SS 0x36 932 #define LEG_OVR_DS 0x3E 933 #define LEG_OVR_ES 0x26 934 #define LEG_OVR_FS 0x64 935 #define LEG_OVR_GS 0x65 936 #define LEG_OPR_OVR 0x66 937 #define LEG_ADR_OVR 0x67 938 939 struct x86_legpref { 940 bool opr_ovr:1; 941 bool adr_ovr:1; 942 bool rep:1; 943 bool repn:1; 944 int8_t seg; 945 }; 946 947 struct x86_rexpref { 948 bool b:1; 949 bool x:1; 950 bool r:1; 951 bool w:1; 952 bool present:1; 953 }; 954 955 struct x86_reg { 956 int num; /* NVMM GPR state index */ 957 uint64_t mask; 958 }; 959 960 struct x86_dualreg { 961 int reg1; 962 int reg2; 963 }; 964 965 enum x86_disp_type { 966 DISP_NONE, 967 DISP_0, 968 DISP_1, 969 DISP_2, 970 DISP_4 971 }; 972 973 struct x86_disp { 974 enum x86_disp_type type; 975 uint64_t data; /* 4 bytes, but can be sign-extended */ 976 }; 977 978 struct x86_regmodrm { 979 uint8_t mod:2; 980 uint8_t reg:3; 981 uint8_t rm:3; 982 }; 983 984 struct x86_immediate { 985 uint64_t data; 986 }; 987 988 struct x86_sib { 989 uint8_t scale; 990 const struct x86_reg *idx; 991 const struct x86_reg *bas; 992 }; 993 994 enum x86_store_type { 995 STORE_NONE, 996 STORE_REG, 997 STORE_DUALREG, 998 STORE_IMM, 999 STORE_SIB, 1000 STORE_DMO 1001 }; 1002 1003 struct x86_store { 1004 enum x86_store_type type; 1005 union { 1006 const struct x86_reg *reg; 1007 struct x86_dualreg dualreg; 1008 struct x86_immediate imm; 1009 struct x86_sib sib; 1010 uint64_t dmo; 1011 } u; 1012 struct x86_disp disp; 1013 int hardseg; 1014 }; 1015 1016 struct x86_instr { 1017 uint8_t len; 1018 struct x86_legpref legpref; 1019 struct x86_rexpref rexpref; 1020 struct x86_regmodrm regmodrm; 1021 uint8_t operand_size; 1022 uint8_t address_size; 1023 uint64_t zeroextend_mask; 1024 1025 const struct x86_opcode *opcode; 1026 const struct x86_emul *emul; 1027 1028 struct x86_store src; 1029 struct x86_store dst; 1030 struct x86_store *strm; 1031 }; 1032 1033 struct x86_decode_fsm { 1034 /* vcpu */ 1035 bool is64bit; 1036 bool is32bit; 1037 bool is16bit; 1038 1039 /* fsm */ 1040 int (*fn)(struct x86_decode_fsm *, struct x86_instr *); 1041 uint8_t *buf; 1042 uint8_t *end; 1043 }; 1044 1045 struct x86_opcode { 1046 bool valid:1; 1047 bool regmodrm:1; 1048 bool regtorm:1; 1049 bool dmo:1; 1050 bool todmo:1; 1051 bool movs:1; 1052 bool stos:1; 1053 bool lods:1; 1054 bool szoverride:1; 1055 bool group1:1; 1056 bool group3:1; 1057 bool group11:1; 1058 bool immediate:1; 1059 uint8_t defsize; 1060 uint8_t flags; 1061 const struct x86_emul *emul; 1062 }; 1063 1064 struct x86_group_entry { 1065 const struct x86_emul *emul; 1066 }; 1067 1068 #define OPSIZE_BYTE 0x01 1069 #define OPSIZE_WORD 0x02 /* 2 bytes */ 1070 #define OPSIZE_DOUB 0x04 /* 4 bytes */ 1071 #define OPSIZE_QUAD 0x08 /* 8 bytes */ 1072 1073 #define FLAG_imm8 0x01 1074 #define FLAG_immz 0x02 1075 #define FLAG_ze 0x04 1076 1077 static const struct x86_group_entry group1[8] __cacheline_aligned = { 1078 [1] = { .emul = &x86_emul_or }, 1079 [4] = { .emul = &x86_emul_and }, 1080 [6] = { .emul = &x86_emul_xor }, 1081 [7] = { .emul = &x86_emul_cmp } 1082 }; 1083 1084 static const struct x86_group_entry group3[8] __cacheline_aligned = { 1085 [0] = { .emul = &x86_emul_test }, 1086 [1] = { .emul = &x86_emul_test } 1087 }; 1088 1089 static const struct x86_group_entry group11[8] __cacheline_aligned = { 1090 [0] = { .emul = &x86_emul_mov } 1091 }; 1092 1093 static const struct x86_opcode primary_opcode_table[256] __cacheline_aligned = { 1094 /* 1095 * Group1 1096 */ 1097 [0x80] = { 1098 /* Eb, Ib */ 1099 .valid = true, 1100 .regmodrm = true, 1101 .regtorm = true, 1102 .szoverride = false, 1103 .defsize = OPSIZE_BYTE, 1104 .group1 = true, 1105 .immediate = true, 1106 .emul = NULL /* group1 */ 1107 }, 1108 [0x81] = { 1109 /* Ev, Iz */ 1110 .valid = true, 1111 .regmodrm = true, 1112 .regtorm = true, 1113 .szoverride = true, 1114 .defsize = -1, 1115 .group1 = true, 1116 .immediate = true, 1117 .flags = FLAG_immz, 1118 .emul = NULL /* group1 */ 1119 }, 1120 [0x83] = { 1121 /* Ev, Ib */ 1122 .valid = true, 1123 .regmodrm = true, 1124 .regtorm = true, 1125 .szoverride = true, 1126 .defsize = -1, 1127 .group1 = true, 1128 .immediate = true, 1129 .flags = FLAG_imm8, 1130 .emul = NULL /* group1 */ 1131 }, 1132 1133 /* 1134 * Group3 1135 */ 1136 [0xF6] = { 1137 /* Eb, Ib */ 1138 .valid = true, 1139 .regmodrm = true, 1140 .regtorm = true, 1141 .szoverride = false, 1142 .defsize = OPSIZE_BYTE, 1143 .group3 = true, 1144 .immediate = true, 1145 .emul = NULL /* group3 */ 1146 }, 1147 [0xF7] = { 1148 /* Ev, Iz */ 1149 .valid = true, 1150 .regmodrm = true, 1151 .regtorm = true, 1152 .szoverride = true, 1153 .defsize = -1, 1154 .group3 = true, 1155 .immediate = true, 1156 .flags = FLAG_immz, 1157 .emul = NULL /* group3 */ 1158 }, 1159 1160 /* 1161 * Group11 1162 */ 1163 [0xC6] = { 1164 /* Eb, Ib */ 1165 .valid = true, 1166 .regmodrm = true, 1167 .regtorm = true, 1168 .szoverride = false, 1169 .defsize = OPSIZE_BYTE, 1170 .group11 = true, 1171 .immediate = true, 1172 .emul = NULL /* group11 */ 1173 }, 1174 [0xC7] = { 1175 /* Ev, Iz */ 1176 .valid = true, 1177 .regmodrm = true, 1178 .regtorm = true, 1179 .szoverride = true, 1180 .defsize = -1, 1181 .group11 = true, 1182 .immediate = true, 1183 .flags = FLAG_immz, 1184 .emul = NULL /* group11 */ 1185 }, 1186 1187 /* 1188 * OR 1189 */ 1190 [0x08] = { 1191 /* Eb, Gb */ 1192 .valid = true, 1193 .regmodrm = true, 1194 .regtorm = true, 1195 .szoverride = false, 1196 .defsize = OPSIZE_BYTE, 1197 .emul = &x86_emul_or 1198 }, 1199 [0x09] = { 1200 /* Ev, Gv */ 1201 .valid = true, 1202 .regmodrm = true, 1203 .regtorm = true, 1204 .szoverride = true, 1205 .defsize = -1, 1206 .emul = &x86_emul_or 1207 }, 1208 [0x0A] = { 1209 /* Gb, Eb */ 1210 .valid = true, 1211 .regmodrm = true, 1212 .regtorm = false, 1213 .szoverride = false, 1214 .defsize = OPSIZE_BYTE, 1215 .emul = &x86_emul_or 1216 }, 1217 [0x0B] = { 1218 /* Gv, Ev */ 1219 .valid = true, 1220 .regmodrm = true, 1221 .regtorm = false, 1222 .szoverride = true, 1223 .defsize = -1, 1224 .emul = &x86_emul_or 1225 }, 1226 1227 /* 1228 * AND 1229 */ 1230 [0x20] = { 1231 /* Eb, Gb */ 1232 .valid = true, 1233 .regmodrm = true, 1234 .regtorm = true, 1235 .szoverride = false, 1236 .defsize = OPSIZE_BYTE, 1237 .emul = &x86_emul_and 1238 }, 1239 [0x21] = { 1240 /* Ev, Gv */ 1241 .valid = true, 1242 .regmodrm = true, 1243 .regtorm = true, 1244 .szoverride = true, 1245 .defsize = -1, 1246 .emul = &x86_emul_and 1247 }, 1248 [0x22] = { 1249 /* Gb, Eb */ 1250 .valid = true, 1251 .regmodrm = true, 1252 .regtorm = false, 1253 .szoverride = false, 1254 .defsize = OPSIZE_BYTE, 1255 .emul = &x86_emul_and 1256 }, 1257 [0x23] = { 1258 /* Gv, Ev */ 1259 .valid = true, 1260 .regmodrm = true, 1261 .regtorm = false, 1262 .szoverride = true, 1263 .defsize = -1, 1264 .emul = &x86_emul_and 1265 }, 1266 1267 /* 1268 * SUB 1269 */ 1270 [0x28] = { 1271 /* Eb, Gb */ 1272 .valid = true, 1273 .regmodrm = true, 1274 .regtorm = true, 1275 .szoverride = false, 1276 .defsize = OPSIZE_BYTE, 1277 .emul = &x86_emul_sub 1278 }, 1279 [0x29] = { 1280 /* Ev, Gv */ 1281 .valid = true, 1282 .regmodrm = true, 1283 .regtorm = true, 1284 .szoverride = true, 1285 .defsize = -1, 1286 .emul = &x86_emul_sub 1287 }, 1288 [0x2A] = { 1289 /* Gb, Eb */ 1290 .valid = true, 1291 .regmodrm = true, 1292 .regtorm = false, 1293 .szoverride = false, 1294 .defsize = OPSIZE_BYTE, 1295 .emul = &x86_emul_sub 1296 }, 1297 [0x2B] = { 1298 /* Gv, Ev */ 1299 .valid = true, 1300 .regmodrm = true, 1301 .regtorm = false, 1302 .szoverride = true, 1303 .defsize = -1, 1304 .emul = &x86_emul_sub 1305 }, 1306 1307 /* 1308 * XOR 1309 */ 1310 [0x30] = { 1311 /* Eb, Gb */ 1312 .valid = true, 1313 .regmodrm = true, 1314 .regtorm = true, 1315 .szoverride = false, 1316 .defsize = OPSIZE_BYTE, 1317 .emul = &x86_emul_xor 1318 }, 1319 [0x31] = { 1320 /* Ev, Gv */ 1321 .valid = true, 1322 .regmodrm = true, 1323 .regtorm = true, 1324 .szoverride = true, 1325 .defsize = -1, 1326 .emul = &x86_emul_xor 1327 }, 1328 [0x32] = { 1329 /* Gb, Eb */ 1330 .valid = true, 1331 .regmodrm = true, 1332 .regtorm = false, 1333 .szoverride = false, 1334 .defsize = OPSIZE_BYTE, 1335 .emul = &x86_emul_xor 1336 }, 1337 [0x33] = { 1338 /* Gv, Ev */ 1339 .valid = true, 1340 .regmodrm = true, 1341 .regtorm = false, 1342 .szoverride = true, 1343 .defsize = -1, 1344 .emul = &x86_emul_xor 1345 }, 1346 1347 /* 1348 * XCHG 1349 */ 1350 [0x86] = { 1351 /* Eb, Gb */ 1352 .valid = true, 1353 .regmodrm = true, 1354 .regtorm = true, 1355 .szoverride = false, 1356 .defsize = OPSIZE_BYTE, 1357 .emul = &x86_emul_xchg 1358 }, 1359 [0x87] = { 1360 /* Ev, Gv */ 1361 .valid = true, 1362 .regmodrm = true, 1363 .regtorm = true, 1364 .szoverride = true, 1365 .defsize = -1, 1366 .emul = &x86_emul_xchg 1367 }, 1368 1369 /* 1370 * MOV 1371 */ 1372 [0x88] = { 1373 /* Eb, Gb */ 1374 .valid = true, 1375 .regmodrm = true, 1376 .regtorm = true, 1377 .szoverride = false, 1378 .defsize = OPSIZE_BYTE, 1379 .emul = &x86_emul_mov 1380 }, 1381 [0x89] = { 1382 /* Ev, Gv */ 1383 .valid = true, 1384 .regmodrm = true, 1385 .regtorm = true, 1386 .szoverride = true, 1387 .defsize = -1, 1388 .emul = &x86_emul_mov 1389 }, 1390 [0x8A] = { 1391 /* Gb, Eb */ 1392 .valid = true, 1393 .regmodrm = true, 1394 .regtorm = false, 1395 .szoverride = false, 1396 .defsize = OPSIZE_BYTE, 1397 .emul = &x86_emul_mov 1398 }, 1399 [0x8B] = { 1400 /* Gv, Ev */ 1401 .valid = true, 1402 .regmodrm = true, 1403 .regtorm = false, 1404 .szoverride = true, 1405 .defsize = -1, 1406 .emul = &x86_emul_mov 1407 }, 1408 [0xA0] = { 1409 /* AL, Ob */ 1410 .valid = true, 1411 .dmo = true, 1412 .todmo = false, 1413 .szoverride = false, 1414 .defsize = OPSIZE_BYTE, 1415 .emul = &x86_emul_mov 1416 }, 1417 [0xA1] = { 1418 /* rAX, Ov */ 1419 .valid = true, 1420 .dmo = true, 1421 .todmo = false, 1422 .szoverride = true, 1423 .defsize = -1, 1424 .emul = &x86_emul_mov 1425 }, 1426 [0xA2] = { 1427 /* Ob, AL */ 1428 .valid = true, 1429 .dmo = true, 1430 .todmo = true, 1431 .szoverride = false, 1432 .defsize = OPSIZE_BYTE, 1433 .emul = &x86_emul_mov 1434 }, 1435 [0xA3] = { 1436 /* Ov, rAX */ 1437 .valid = true, 1438 .dmo = true, 1439 .todmo = true, 1440 .szoverride = true, 1441 .defsize = -1, 1442 .emul = &x86_emul_mov 1443 }, 1444 1445 /* 1446 * MOVS 1447 */ 1448 [0xA4] = { 1449 /* Yb, Xb */ 1450 .valid = true, 1451 .movs = true, 1452 .szoverride = false, 1453 .defsize = OPSIZE_BYTE, 1454 .emul = &x86_emul_movs 1455 }, 1456 [0xA5] = { 1457 /* Yv, Xv */ 1458 .valid = true, 1459 .movs = true, 1460 .szoverride = true, 1461 .defsize = -1, 1462 .emul = &x86_emul_movs 1463 }, 1464 1465 /* 1466 * STOS 1467 */ 1468 [0xAA] = { 1469 /* Yb, AL */ 1470 .valid = true, 1471 .stos = true, 1472 .szoverride = false, 1473 .defsize = OPSIZE_BYTE, 1474 .emul = &x86_emul_stos 1475 }, 1476 [0xAB] = { 1477 /* Yv, rAX */ 1478 .valid = true, 1479 .stos = true, 1480 .szoverride = true, 1481 .defsize = -1, 1482 .emul = &x86_emul_stos 1483 }, 1484 1485 /* 1486 * LODS 1487 */ 1488 [0xAC] = { 1489 /* AL, Xb */ 1490 .valid = true, 1491 .lods = true, 1492 .szoverride = false, 1493 .defsize = OPSIZE_BYTE, 1494 .emul = &x86_emul_lods 1495 }, 1496 [0xAD] = { 1497 /* rAX, Xv */ 1498 .valid = true, 1499 .lods = true, 1500 .szoverride = true, 1501 .defsize = -1, 1502 .emul = &x86_emul_lods 1503 }, 1504 }; 1505 1506 static const struct x86_opcode secondary_opcode_table[256] __cacheline_aligned = { 1507 /* 1508 * MOVZX 1509 */ 1510 [0xB6] = { 1511 /* Gv, Eb */ 1512 .valid = true, 1513 .regmodrm = true, 1514 .regtorm = false, 1515 .szoverride = true, 1516 .defsize = OPSIZE_BYTE, 1517 .flags = FLAG_ze, 1518 .emul = &x86_emul_mov 1519 }, 1520 [0xB7] = { 1521 /* Gv, Ew */ 1522 .valid = true, 1523 .regmodrm = true, 1524 .regtorm = false, 1525 .szoverride = true, 1526 .defsize = OPSIZE_WORD, 1527 .flags = FLAG_ze, 1528 .emul = &x86_emul_mov 1529 }, 1530 }; 1531 1532 static const struct x86_reg gpr_map__rip = { NVMM_X64_GPR_RIP, 0xFFFFFFFFFFFFFFFF }; 1533 1534 /* [REX-present][enc][opsize] */ 1535 static const struct x86_reg gpr_map__special[2][4][8] __cacheline_aligned = { 1536 [false] = { 1537 /* No REX prefix. */ 1538 [0b00] = { 1539 [0] = { NVMM_X64_GPR_RAX, 0x000000000000FF00 }, /* AH */ 1540 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1541 [2] = { -1, 0 }, 1542 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1543 [4] = { -1, 0 }, 1544 [5] = { -1, 0 }, 1545 [6] = { -1, 0 }, 1546 [7] = { -1, 0 }, 1547 }, 1548 [0b01] = { 1549 [0] = { NVMM_X64_GPR_RCX, 0x000000000000FF00 }, /* CH */ 1550 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1551 [2] = { -1, 0 }, 1552 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1553 [4] = { -1, 0 }, 1554 [5] = { -1, 0 }, 1555 [6] = { -1, 0 }, 1556 [7] = { -1, 0 }, 1557 }, 1558 [0b10] = { 1559 [0] = { NVMM_X64_GPR_RDX, 0x000000000000FF00 }, /* DH */ 1560 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1561 [2] = { -1, 0 }, 1562 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1563 [4] = { -1, 0 }, 1564 [5] = { -1, 0 }, 1565 [6] = { -1, 0 }, 1566 [7] = { -1, 0 }, 1567 }, 1568 [0b11] = { 1569 [0] = { NVMM_X64_GPR_RBX, 0x000000000000FF00 }, /* BH */ 1570 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1571 [2] = { -1, 0 }, 1572 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1573 [4] = { -1, 0 }, 1574 [5] = { -1, 0 }, 1575 [6] = { -1, 0 }, 1576 [7] = { -1, 0 }, 1577 } 1578 }, 1579 [true] = { 1580 /* Has REX prefix. */ 1581 [0b00] = { 1582 [0] = { NVMM_X64_GPR_RSP, 0x00000000000000FF }, /* SPL */ 1583 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1584 [2] = { -1, 0 }, 1585 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1586 [4] = { -1, 0 }, 1587 [5] = { -1, 0 }, 1588 [6] = { -1, 0 }, 1589 [7] = { NVMM_X64_GPR_RSP, 0xFFFFFFFFFFFFFFFF }, /* RSP */ 1590 }, 1591 [0b01] = { 1592 [0] = { NVMM_X64_GPR_RBP, 0x00000000000000FF }, /* BPL */ 1593 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1594 [2] = { -1, 0 }, 1595 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1596 [4] = { -1, 0 }, 1597 [5] = { -1, 0 }, 1598 [6] = { -1, 0 }, 1599 [7] = { NVMM_X64_GPR_RBP, 0xFFFFFFFFFFFFFFFF }, /* RBP */ 1600 }, 1601 [0b10] = { 1602 [0] = { NVMM_X64_GPR_RSI, 0x00000000000000FF }, /* SIL */ 1603 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1604 [2] = { -1, 0 }, 1605 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1606 [4] = { -1, 0 }, 1607 [5] = { -1, 0 }, 1608 [6] = { -1, 0 }, 1609 [7] = { NVMM_X64_GPR_RSI, 0xFFFFFFFFFFFFFFFF }, /* RSI */ 1610 }, 1611 [0b11] = { 1612 [0] = { NVMM_X64_GPR_RDI, 0x00000000000000FF }, /* DIL */ 1613 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1614 [2] = { -1, 0 }, 1615 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1616 [4] = { -1, 0 }, 1617 [5] = { -1, 0 }, 1618 [6] = { -1, 0 }, 1619 [7] = { NVMM_X64_GPR_RDI, 0xFFFFFFFFFFFFFFFF }, /* RDI */ 1620 } 1621 } 1622 }; 1623 1624 /* [depends][enc][size] */ 1625 static const struct x86_reg gpr_map[2][8][8] __cacheline_aligned = { 1626 [false] = { 1627 /* Not extended. */ 1628 [0b000] = { 1629 [0] = { NVMM_X64_GPR_RAX, 0x00000000000000FF }, /* AL */ 1630 [1] = { NVMM_X64_GPR_RAX, 0x000000000000FFFF }, /* AX */ 1631 [2] = { -1, 0 }, 1632 [3] = { NVMM_X64_GPR_RAX, 0x00000000FFFFFFFF }, /* EAX */ 1633 [4] = { -1, 0 }, 1634 [5] = { -1, 0 }, 1635 [6] = { -1, 0 }, 1636 [7] = { NVMM_X64_GPR_RAX, 0xFFFFFFFFFFFFFFFF }, /* RAX */ 1637 }, 1638 [0b001] = { 1639 [0] = { NVMM_X64_GPR_RCX, 0x00000000000000FF }, /* CL */ 1640 [1] = { NVMM_X64_GPR_RCX, 0x000000000000FFFF }, /* CX */ 1641 [2] = { -1, 0 }, 1642 [3] = { NVMM_X64_GPR_RCX, 0x00000000FFFFFFFF }, /* ECX */ 1643 [4] = { -1, 0 }, 1644 [5] = { -1, 0 }, 1645 [6] = { -1, 0 }, 1646 [7] = { NVMM_X64_GPR_RCX, 0xFFFFFFFFFFFFFFFF }, /* RCX */ 1647 }, 1648 [0b010] = { 1649 [0] = { NVMM_X64_GPR_RDX, 0x00000000000000FF }, /* DL */ 1650 [1] = { NVMM_X64_GPR_RDX, 0x000000000000FFFF }, /* DX */ 1651 [2] = { -1, 0 }, 1652 [3] = { NVMM_X64_GPR_RDX, 0x00000000FFFFFFFF }, /* EDX */ 1653 [4] = { -1, 0 }, 1654 [5] = { -1, 0 }, 1655 [6] = { -1, 0 }, 1656 [7] = { NVMM_X64_GPR_RDX, 0xFFFFFFFFFFFFFFFF }, /* RDX */ 1657 }, 1658 [0b011] = { 1659 [0] = { NVMM_X64_GPR_RBX, 0x00000000000000FF }, /* BL */ 1660 [1] = { NVMM_X64_GPR_RBX, 0x000000000000FFFF }, /* BX */ 1661 [2] = { -1, 0 }, 1662 [3] = { NVMM_X64_GPR_RBX, 0x00000000FFFFFFFF }, /* EBX */ 1663 [4] = { -1, 0 }, 1664 [5] = { -1, 0 }, 1665 [6] = { -1, 0 }, 1666 [7] = { NVMM_X64_GPR_RBX, 0xFFFFFFFFFFFFFFFF }, /* RBX */ 1667 }, 1668 [0b100] = { 1669 [0] = { -1, 0 }, /* SPECIAL */ 1670 [1] = { -1, 0 }, /* SPECIAL */ 1671 [2] = { -1, 0 }, 1672 [3] = { -1, 0 }, /* SPECIAL */ 1673 [4] = { -1, 0 }, 1674 [5] = { -1, 0 }, 1675 [6] = { -1, 0 }, 1676 [7] = { -1, 0 }, /* SPECIAL */ 1677 }, 1678 [0b101] = { 1679 [0] = { -1, 0 }, /* SPECIAL */ 1680 [1] = { -1, 0 }, /* SPECIAL */ 1681 [2] = { -1, 0 }, 1682 [3] = { -1, 0 }, /* SPECIAL */ 1683 [4] = { -1, 0 }, 1684 [5] = { -1, 0 }, 1685 [6] = { -1, 0 }, 1686 [7] = { -1, 0 }, /* SPECIAL */ 1687 }, 1688 [0b110] = { 1689 [0] = { -1, 0 }, /* SPECIAL */ 1690 [1] = { -1, 0 }, /* SPECIAL */ 1691 [2] = { -1, 0 }, 1692 [3] = { -1, 0 }, /* SPECIAL */ 1693 [4] = { -1, 0 }, 1694 [5] = { -1, 0 }, 1695 [6] = { -1, 0 }, 1696 [7] = { -1, 0 }, /* SPECIAL */ 1697 }, 1698 [0b111] = { 1699 [0] = { -1, 0 }, /* SPECIAL */ 1700 [1] = { -1, 0 }, /* SPECIAL */ 1701 [2] = { -1, 0 }, 1702 [3] = { -1, 0 }, /* SPECIAL */ 1703 [4] = { -1, 0 }, 1704 [5] = { -1, 0 }, 1705 [6] = { -1, 0 }, 1706 [7] = { -1, 0 }, /* SPECIAL */ 1707 }, 1708 }, 1709 [true] = { 1710 /* Extended. */ 1711 [0b000] = { 1712 [0] = { NVMM_X64_GPR_R8, 0x00000000000000FF }, /* R8B */ 1713 [1] = { NVMM_X64_GPR_R8, 0x000000000000FFFF }, /* R8W */ 1714 [2] = { -1, 0 }, 1715 [3] = { NVMM_X64_GPR_R8, 0x00000000FFFFFFFF }, /* R8D */ 1716 [4] = { -1, 0 }, 1717 [5] = { -1, 0 }, 1718 [6] = { -1, 0 }, 1719 [7] = { NVMM_X64_GPR_R8, 0xFFFFFFFFFFFFFFFF }, /* R8 */ 1720 }, 1721 [0b001] = { 1722 [0] = { NVMM_X64_GPR_R9, 0x00000000000000FF }, /* R9B */ 1723 [1] = { NVMM_X64_GPR_R9, 0x000000000000FFFF }, /* R9W */ 1724 [2] = { -1, 0 }, 1725 [3] = { NVMM_X64_GPR_R9, 0x00000000FFFFFFFF }, /* R9D */ 1726 [4] = { -1, 0 }, 1727 [5] = { -1, 0 }, 1728 [6] = { -1, 0 }, 1729 [7] = { NVMM_X64_GPR_R9, 0xFFFFFFFFFFFFFFFF }, /* R9 */ 1730 }, 1731 [0b010] = { 1732 [0] = { NVMM_X64_GPR_R10, 0x00000000000000FF }, /* R10B */ 1733 [1] = { NVMM_X64_GPR_R10, 0x000000000000FFFF }, /* R10W */ 1734 [2] = { -1, 0 }, 1735 [3] = { NVMM_X64_GPR_R10, 0x00000000FFFFFFFF }, /* R10D */ 1736 [4] = { -1, 0 }, 1737 [5] = { -1, 0 }, 1738 [6] = { -1, 0 }, 1739 [7] = { NVMM_X64_GPR_R10, 0xFFFFFFFFFFFFFFFF }, /* R10 */ 1740 }, 1741 [0b011] = { 1742 [0] = { NVMM_X64_GPR_R11, 0x00000000000000FF }, /* R11B */ 1743 [1] = { NVMM_X64_GPR_R11, 0x000000000000FFFF }, /* R11W */ 1744 [2] = { -1, 0 }, 1745 [3] = { NVMM_X64_GPR_R11, 0x00000000FFFFFFFF }, /* R11D */ 1746 [4] = { -1, 0 }, 1747 [5] = { -1, 0 }, 1748 [6] = { -1, 0 }, 1749 [7] = { NVMM_X64_GPR_R11, 0xFFFFFFFFFFFFFFFF }, /* R11 */ 1750 }, 1751 [0b100] = { 1752 [0] = { NVMM_X64_GPR_R12, 0x00000000000000FF }, /* R12B */ 1753 [1] = { NVMM_X64_GPR_R12, 0x000000000000FFFF }, /* R12W */ 1754 [2] = { -1, 0 }, 1755 [3] = { NVMM_X64_GPR_R12, 0x00000000FFFFFFFF }, /* R12D */ 1756 [4] = { -1, 0 }, 1757 [5] = { -1, 0 }, 1758 [6] = { -1, 0 }, 1759 [7] = { NVMM_X64_GPR_R12, 0xFFFFFFFFFFFFFFFF }, /* R12 */ 1760 }, 1761 [0b101] = { 1762 [0] = { NVMM_X64_GPR_R13, 0x00000000000000FF }, /* R13B */ 1763 [1] = { NVMM_X64_GPR_R13, 0x000000000000FFFF }, /* R13W */ 1764 [2] = { -1, 0 }, 1765 [3] = { NVMM_X64_GPR_R13, 0x00000000FFFFFFFF }, /* R13D */ 1766 [4] = { -1, 0 }, 1767 [5] = { -1, 0 }, 1768 [6] = { -1, 0 }, 1769 [7] = { NVMM_X64_GPR_R13, 0xFFFFFFFFFFFFFFFF }, /* R13 */ 1770 }, 1771 [0b110] = { 1772 [0] = { NVMM_X64_GPR_R14, 0x00000000000000FF }, /* R14B */ 1773 [1] = { NVMM_X64_GPR_R14, 0x000000000000FFFF }, /* R14W */ 1774 [2] = { -1, 0 }, 1775 [3] = { NVMM_X64_GPR_R14, 0x00000000FFFFFFFF }, /* R14D */ 1776 [4] = { -1, 0 }, 1777 [5] = { -1, 0 }, 1778 [6] = { -1, 0 }, 1779 [7] = { NVMM_X64_GPR_R14, 0xFFFFFFFFFFFFFFFF }, /* R14 */ 1780 }, 1781 [0b111] = { 1782 [0] = { NVMM_X64_GPR_R15, 0x00000000000000FF }, /* R15B */ 1783 [1] = { NVMM_X64_GPR_R15, 0x000000000000FFFF }, /* R15W */ 1784 [2] = { -1, 0 }, 1785 [3] = { NVMM_X64_GPR_R15, 0x00000000FFFFFFFF }, /* R15D */ 1786 [4] = { -1, 0 }, 1787 [5] = { -1, 0 }, 1788 [6] = { -1, 0 }, 1789 [7] = { NVMM_X64_GPR_R15, 0xFFFFFFFFFFFFFFFF }, /* R15 */ 1790 }, 1791 } 1792 }; 1793 1794 /* [enc] */ 1795 static const int gpr_dual_reg1_rm[8] __cacheline_aligned = { 1796 [0b000] = NVMM_X64_GPR_RBX, /* BX (+SI) */ 1797 [0b001] = NVMM_X64_GPR_RBX, /* BX (+DI) */ 1798 [0b010] = NVMM_X64_GPR_RBP, /* BP (+SI) */ 1799 [0b011] = NVMM_X64_GPR_RBP, /* BP (+DI) */ 1800 [0b100] = NVMM_X64_GPR_RSI, /* SI */ 1801 [0b101] = NVMM_X64_GPR_RDI, /* DI */ 1802 [0b110] = NVMM_X64_GPR_RBP, /* BP */ 1803 [0b111] = NVMM_X64_GPR_RBX, /* BX */ 1804 }; 1805 1806 static int 1807 node_overflow(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1808 { 1809 fsm->fn = NULL; 1810 return -1; 1811 } 1812 1813 static int 1814 fsm_read(struct x86_decode_fsm *fsm, uint8_t *bytes, size_t n) 1815 { 1816 if (fsm->buf + n > fsm->end) { 1817 return -1; 1818 } 1819 memcpy(bytes, fsm->buf, n); 1820 return 0; 1821 } 1822 1823 static inline void 1824 fsm_advance(struct x86_decode_fsm *fsm, size_t n, 1825 int (*fn)(struct x86_decode_fsm *, struct x86_instr *)) 1826 { 1827 fsm->buf += n; 1828 if (fsm->buf > fsm->end) { 1829 fsm->fn = node_overflow; 1830 } else { 1831 fsm->fn = fn; 1832 } 1833 } 1834 1835 static const struct x86_reg * 1836 resolve_special_register(struct x86_instr *instr, uint8_t enc, size_t regsize) 1837 { 1838 enc &= 0b11; 1839 if (regsize == 8) { 1840 /* May be 64bit without REX */ 1841 return &gpr_map__special[1][enc][regsize-1]; 1842 } 1843 return &gpr_map__special[instr->rexpref.present][enc][regsize-1]; 1844 } 1845 1846 /* 1847 * Special node, for MOVS. Fake two displacements of zero on the source and 1848 * destination registers. 1849 */ 1850 static int 1851 node_movs(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1852 { 1853 size_t adrsize; 1854 1855 adrsize = instr->address_size; 1856 1857 /* DS:RSI */ 1858 instr->src.type = STORE_REG; 1859 instr->src.u.reg = &gpr_map__special[1][2][adrsize-1]; 1860 instr->src.disp.type = DISP_0; 1861 1862 /* ES:RDI, force ES */ 1863 instr->dst.type = STORE_REG; 1864 instr->dst.u.reg = &gpr_map__special[1][3][adrsize-1]; 1865 instr->dst.disp.type = DISP_0; 1866 instr->dst.hardseg = NVMM_X64_SEG_ES; 1867 1868 fsm_advance(fsm, 0, NULL); 1869 1870 return 0; 1871 } 1872 1873 /* 1874 * Special node, for STOS and LODS. Fake a displacement of zero on the 1875 * destination register. 1876 */ 1877 static int 1878 node_stlo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1879 { 1880 const struct x86_opcode *opcode = instr->opcode; 1881 struct x86_store *stlo, *streg; 1882 size_t adrsize, regsize; 1883 1884 adrsize = instr->address_size; 1885 regsize = instr->operand_size; 1886 1887 if (opcode->stos) { 1888 streg = &instr->src; 1889 stlo = &instr->dst; 1890 } else { 1891 streg = &instr->dst; 1892 stlo = &instr->src; 1893 } 1894 1895 streg->type = STORE_REG; 1896 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1897 1898 stlo->type = STORE_REG; 1899 if (opcode->stos) { 1900 /* ES:RDI, force ES */ 1901 stlo->u.reg = &gpr_map__special[1][3][adrsize-1]; 1902 stlo->hardseg = NVMM_X64_SEG_ES; 1903 } else { 1904 /* DS:RSI */ 1905 stlo->u.reg = &gpr_map__special[1][2][adrsize-1]; 1906 } 1907 stlo->disp.type = DISP_0; 1908 1909 fsm_advance(fsm, 0, NULL); 1910 1911 return 0; 1912 } 1913 1914 static int 1915 node_dmo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1916 { 1917 const struct x86_opcode *opcode = instr->opcode; 1918 struct x86_store *stdmo, *streg; 1919 size_t adrsize, regsize; 1920 1921 adrsize = instr->address_size; 1922 regsize = instr->operand_size; 1923 1924 if (opcode->todmo) { 1925 streg = &instr->src; 1926 stdmo = &instr->dst; 1927 } else { 1928 streg = &instr->dst; 1929 stdmo = &instr->src; 1930 } 1931 1932 streg->type = STORE_REG; 1933 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1934 1935 stdmo->type = STORE_DMO; 1936 if (fsm_read(fsm, (uint8_t *)&stdmo->u.dmo, adrsize) == -1) { 1937 return -1; 1938 } 1939 fsm_advance(fsm, adrsize, NULL); 1940 1941 return 0; 1942 } 1943 1944 static inline uint64_t 1945 sign_extend(uint64_t val, int size) 1946 { 1947 if (size == 1) { 1948 if (val & __BIT(7)) 1949 val |= 0xFFFFFFFFFFFFFF00; 1950 } else if (size == 2) { 1951 if (val & __BIT(15)) 1952 val |= 0xFFFFFFFFFFFF0000; 1953 } else if (size == 4) { 1954 if (val & __BIT(31)) 1955 val |= 0xFFFFFFFF00000000; 1956 } 1957 return val; 1958 } 1959 1960 static int 1961 node_immediate(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1962 { 1963 const struct x86_opcode *opcode = instr->opcode; 1964 struct x86_store *store; 1965 uint8_t immsize; 1966 size_t sesize = 0; 1967 1968 /* The immediate is the source */ 1969 store = &instr->src; 1970 immsize = instr->operand_size; 1971 1972 if (opcode->flags & FLAG_imm8) { 1973 sesize = immsize; 1974 immsize = 1; 1975 } else if ((opcode->flags & FLAG_immz) && (immsize == 8)) { 1976 sesize = immsize; 1977 immsize = 4; 1978 } 1979 1980 store->type = STORE_IMM; 1981 if (fsm_read(fsm, (uint8_t *)&store->u.imm.data, immsize) == -1) { 1982 return -1; 1983 } 1984 fsm_advance(fsm, immsize, NULL); 1985 1986 if (sesize != 0) { 1987 store->u.imm.data = sign_extend(store->u.imm.data, sesize); 1988 } 1989 1990 return 0; 1991 } 1992 1993 static int 1994 node_disp(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1995 { 1996 const struct x86_opcode *opcode = instr->opcode; 1997 uint64_t data = 0; 1998 size_t n; 1999 2000 if (instr->strm->disp.type == DISP_1) { 2001 n = 1; 2002 } else if (instr->strm->disp.type == DISP_2) { 2003 n = 2; 2004 } else if (instr->strm->disp.type == DISP_4) { 2005 n = 4; 2006 } else { 2007 DISASSEMBLER_BUG(); 2008 } 2009 2010 if (fsm_read(fsm, (uint8_t *)&data, n) == -1) { 2011 return -1; 2012 } 2013 2014 if (__predict_true(fsm->is64bit)) { 2015 data = sign_extend(data, n); 2016 } 2017 2018 instr->strm->disp.data = data; 2019 2020 if (opcode->immediate) { 2021 fsm_advance(fsm, n, node_immediate); 2022 } else { 2023 fsm_advance(fsm, n, NULL); 2024 } 2025 2026 return 0; 2027 } 2028 2029 /* 2030 * Special node to handle 16bit addressing encoding, which can reference two 2031 * registers at once. 2032 */ 2033 static int 2034 node_dual(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2035 { 2036 int reg1, reg2; 2037 2038 reg1 = gpr_dual_reg1_rm[instr->regmodrm.rm]; 2039 2040 if (instr->regmodrm.rm == 0b000 || 2041 instr->regmodrm.rm == 0b010) { 2042 reg2 = NVMM_X64_GPR_RSI; 2043 } else if (instr->regmodrm.rm == 0b001 || 2044 instr->regmodrm.rm == 0b011) { 2045 reg2 = NVMM_X64_GPR_RDI; 2046 } else { 2047 DISASSEMBLER_BUG(); 2048 } 2049 2050 instr->strm->type = STORE_DUALREG; 2051 instr->strm->u.dualreg.reg1 = reg1; 2052 instr->strm->u.dualreg.reg2 = reg2; 2053 2054 if (instr->strm->disp.type == DISP_NONE) { 2055 DISASSEMBLER_BUG(); 2056 } else if (instr->strm->disp.type == DISP_0) { 2057 /* Indirect register addressing mode */ 2058 if (instr->opcode->immediate) { 2059 fsm_advance(fsm, 1, node_immediate); 2060 } else { 2061 fsm_advance(fsm, 1, NULL); 2062 } 2063 } else { 2064 fsm_advance(fsm, 1, node_disp); 2065 } 2066 2067 return 0; 2068 } 2069 2070 static const struct x86_reg * 2071 get_register_idx(struct x86_instr *instr, uint8_t index) 2072 { 2073 uint8_t enc = index; 2074 const struct x86_reg *reg; 2075 size_t regsize; 2076 2077 regsize = instr->address_size; 2078 reg = &gpr_map[instr->rexpref.x][enc][regsize-1]; 2079 2080 if (reg->num == -1) { 2081 reg = resolve_special_register(instr, enc, regsize); 2082 } 2083 2084 return reg; 2085 } 2086 2087 static const struct x86_reg * 2088 get_register_bas(struct x86_instr *instr, uint8_t base) 2089 { 2090 uint8_t enc = base; 2091 const struct x86_reg *reg; 2092 size_t regsize; 2093 2094 regsize = instr->address_size; 2095 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2096 if (reg->num == -1) { 2097 reg = resolve_special_register(instr, enc, regsize); 2098 } 2099 2100 return reg; 2101 } 2102 2103 static int 2104 node_sib(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2105 { 2106 const struct x86_opcode *opcode; 2107 uint8_t scale, index, base; 2108 bool noindex, nobase; 2109 uint8_t byte; 2110 2111 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2112 return -1; 2113 } 2114 2115 scale = ((byte & 0b11000000) >> 6); 2116 index = ((byte & 0b00111000) >> 3); 2117 base = ((byte & 0b00000111) >> 0); 2118 2119 opcode = instr->opcode; 2120 2121 noindex = false; 2122 nobase = false; 2123 2124 if (index == 0b100 && !instr->rexpref.x) { 2125 /* Special case: the index is null */ 2126 noindex = true; 2127 } 2128 2129 if (instr->regmodrm.mod == 0b00 && base == 0b101) { 2130 /* Special case: the base is null + disp32 */ 2131 instr->strm->disp.type = DISP_4; 2132 nobase = true; 2133 } 2134 2135 instr->strm->type = STORE_SIB; 2136 instr->strm->u.sib.scale = (1 << scale); 2137 if (!noindex) 2138 instr->strm->u.sib.idx = get_register_idx(instr, index); 2139 if (!nobase) 2140 instr->strm->u.sib.bas = get_register_bas(instr, base); 2141 2142 /* May have a displacement, or an immediate */ 2143 if (instr->strm->disp.type == DISP_1 || 2144 instr->strm->disp.type == DISP_2 || 2145 instr->strm->disp.type == DISP_4) { 2146 fsm_advance(fsm, 1, node_disp); 2147 } else if (opcode->immediate) { 2148 fsm_advance(fsm, 1, node_immediate); 2149 } else { 2150 fsm_advance(fsm, 1, NULL); 2151 } 2152 2153 return 0; 2154 } 2155 2156 static const struct x86_reg * 2157 get_register_reg(struct x86_instr *instr, const struct x86_opcode *opcode) 2158 { 2159 uint8_t enc = instr->regmodrm.reg; 2160 const struct x86_reg *reg; 2161 size_t regsize; 2162 2163 regsize = instr->operand_size; 2164 2165 reg = &gpr_map[instr->rexpref.r][enc][regsize-1]; 2166 if (reg->num == -1) { 2167 reg = resolve_special_register(instr, enc, regsize); 2168 } 2169 2170 return reg; 2171 } 2172 2173 static const struct x86_reg * 2174 get_register_rm(struct x86_instr *instr, const struct x86_opcode *opcode) 2175 { 2176 uint8_t enc = instr->regmodrm.rm; 2177 const struct x86_reg *reg; 2178 size_t regsize; 2179 2180 if (instr->strm->disp.type == DISP_NONE) { 2181 regsize = instr->operand_size; 2182 } else { 2183 /* Indirect access, the size is that of the address. */ 2184 regsize = instr->address_size; 2185 } 2186 2187 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2188 if (reg->num == -1) { 2189 reg = resolve_special_register(instr, enc, regsize); 2190 } 2191 2192 return reg; 2193 } 2194 2195 static inline bool 2196 has_sib(struct x86_instr *instr) 2197 { 2198 return (instr->address_size != 2 && /* no SIB in 16bit addressing */ 2199 instr->regmodrm.mod != 0b11 && 2200 instr->regmodrm.rm == 0b100); 2201 } 2202 2203 static inline bool 2204 is_rip_relative(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2205 { 2206 return (fsm->is64bit && /* RIP-relative only in 64bit mode */ 2207 instr->regmodrm.mod == 0b00 && 2208 instr->regmodrm.rm == 0b101); 2209 } 2210 2211 static inline bool 2212 is_disp32_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2213 { 2214 return (!fsm->is64bit && /* no disp32-only in 64bit mode */ 2215 instr->address_size != 2 && /* no disp32-only in 16bit addressing */ 2216 instr->regmodrm.mod == 0b00 && 2217 instr->regmodrm.rm == 0b101); 2218 } 2219 2220 static inline bool 2221 is_disp16_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2222 { 2223 return (instr->address_size == 2 && /* disp16-only only in 16bit addr */ 2224 instr->regmodrm.mod == 0b00 && 2225 instr->regmodrm.rm == 0b110); 2226 } 2227 2228 static inline bool 2229 is_dual(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2230 { 2231 return (instr->address_size == 2 && 2232 instr->regmodrm.mod != 0b11 && 2233 instr->regmodrm.rm <= 0b011); 2234 } 2235 2236 static enum x86_disp_type 2237 get_disp_type(struct x86_instr *instr) 2238 { 2239 switch (instr->regmodrm.mod) { 2240 case 0b00: /* indirect */ 2241 return DISP_0; 2242 case 0b01: /* indirect+1 */ 2243 return DISP_1; 2244 case 0b10: /* indirect+{2,4} */ 2245 if (__predict_false(instr->address_size == 2)) { 2246 return DISP_2; 2247 } 2248 return DISP_4; 2249 case 0b11: /* direct */ 2250 default: /* llvm */ 2251 return DISP_NONE; 2252 } 2253 __unreachable(); 2254 } 2255 2256 static int 2257 node_regmodrm(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2258 { 2259 struct x86_store *strg, *strm; 2260 const struct x86_opcode *opcode; 2261 const struct x86_reg *reg; 2262 uint8_t byte; 2263 2264 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2265 return -1; 2266 } 2267 2268 opcode = instr->opcode; 2269 2270 instr->regmodrm.rm = ((byte & 0b00000111) >> 0); 2271 instr->regmodrm.reg = ((byte & 0b00111000) >> 3); 2272 instr->regmodrm.mod = ((byte & 0b11000000) >> 6); 2273 2274 if (opcode->regtorm) { 2275 strg = &instr->src; 2276 strm = &instr->dst; 2277 } else { /* RM to REG */ 2278 strm = &instr->src; 2279 strg = &instr->dst; 2280 } 2281 2282 /* Save for later use. */ 2283 instr->strm = strm; 2284 2285 /* 2286 * Special cases: Groups. The REG field of REGMODRM is the index in 2287 * the group. op1 gets overwritten in the Immediate node, if any. 2288 */ 2289 if (opcode->group1) { 2290 if (group1[instr->regmodrm.reg].emul == NULL) { 2291 return -1; 2292 } 2293 instr->emul = group1[instr->regmodrm.reg].emul; 2294 } else if (opcode->group3) { 2295 if (group3[instr->regmodrm.reg].emul == NULL) { 2296 return -1; 2297 } 2298 instr->emul = group3[instr->regmodrm.reg].emul; 2299 } else if (opcode->group11) { 2300 if (group11[instr->regmodrm.reg].emul == NULL) { 2301 return -1; 2302 } 2303 instr->emul = group11[instr->regmodrm.reg].emul; 2304 } 2305 2306 if (!opcode->immediate) { 2307 reg = get_register_reg(instr, opcode); 2308 if (reg == NULL) { 2309 return -1; 2310 } 2311 strg->type = STORE_REG; 2312 strg->u.reg = reg; 2313 } 2314 2315 /* The displacement applies to RM. */ 2316 strm->disp.type = get_disp_type(instr); 2317 2318 if (has_sib(instr)) { 2319 /* Overwrites RM */ 2320 fsm_advance(fsm, 1, node_sib); 2321 return 0; 2322 } 2323 2324 if (is_rip_relative(fsm, instr)) { 2325 /* Overwrites RM */ 2326 strm->type = STORE_REG; 2327 strm->u.reg = &gpr_map__rip; 2328 strm->disp.type = DISP_4; 2329 fsm_advance(fsm, 1, node_disp); 2330 return 0; 2331 } 2332 2333 if (is_disp32_only(fsm, instr)) { 2334 /* Overwrites RM */ 2335 strm->type = STORE_REG; 2336 strm->u.reg = NULL; 2337 strm->disp.type = DISP_4; 2338 fsm_advance(fsm, 1, node_disp); 2339 return 0; 2340 } 2341 2342 if (__predict_false(is_disp16_only(fsm, instr))) { 2343 /* Overwrites RM */ 2344 strm->type = STORE_REG; 2345 strm->u.reg = NULL; 2346 strm->disp.type = DISP_2; 2347 fsm_advance(fsm, 1, node_disp); 2348 return 0; 2349 } 2350 2351 if (__predict_false(is_dual(fsm, instr))) { 2352 /* Overwrites RM */ 2353 fsm_advance(fsm, 0, node_dual); 2354 return 0; 2355 } 2356 2357 reg = get_register_rm(instr, opcode); 2358 if (reg == NULL) { 2359 return -1; 2360 } 2361 strm->type = STORE_REG; 2362 strm->u.reg = reg; 2363 2364 if (strm->disp.type == DISP_NONE) { 2365 /* Direct register addressing mode */ 2366 if (opcode->immediate) { 2367 fsm_advance(fsm, 1, node_immediate); 2368 } else { 2369 fsm_advance(fsm, 1, NULL); 2370 } 2371 } else if (strm->disp.type == DISP_0) { 2372 /* Indirect register addressing mode */ 2373 if (opcode->immediate) { 2374 fsm_advance(fsm, 1, node_immediate); 2375 } else { 2376 fsm_advance(fsm, 1, NULL); 2377 } 2378 } else { 2379 fsm_advance(fsm, 1, node_disp); 2380 } 2381 2382 return 0; 2383 } 2384 2385 static size_t 2386 get_operand_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2387 { 2388 const struct x86_opcode *opcode = instr->opcode; 2389 int opsize; 2390 2391 /* Get the opsize */ 2392 if (!opcode->szoverride) { 2393 opsize = opcode->defsize; 2394 } else if (instr->rexpref.present && instr->rexpref.w) { 2395 opsize = 8; 2396 } else { 2397 if (!fsm->is16bit) { 2398 if (instr->legpref.opr_ovr) { 2399 opsize = 2; 2400 } else { 2401 opsize = 4; 2402 } 2403 } else { /* 16bit */ 2404 if (instr->legpref.opr_ovr) { 2405 opsize = 4; 2406 } else { 2407 opsize = 2; 2408 } 2409 } 2410 } 2411 2412 return opsize; 2413 } 2414 2415 static size_t 2416 get_address_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2417 { 2418 if (fsm->is64bit) { 2419 if (__predict_false(instr->legpref.adr_ovr)) { 2420 return 4; 2421 } 2422 return 8; 2423 } 2424 2425 if (fsm->is32bit) { 2426 if (__predict_false(instr->legpref.adr_ovr)) { 2427 return 2; 2428 } 2429 return 4; 2430 } 2431 2432 /* 16bit. */ 2433 if (__predict_false(instr->legpref.adr_ovr)) { 2434 return 4; 2435 } 2436 return 2; 2437 } 2438 2439 static int 2440 node_primary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2441 { 2442 const struct x86_opcode *opcode; 2443 uint8_t byte; 2444 2445 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2446 return -1; 2447 } 2448 2449 opcode = &primary_opcode_table[byte]; 2450 if (__predict_false(!opcode->valid)) { 2451 return -1; 2452 } 2453 2454 instr->opcode = opcode; 2455 instr->emul = opcode->emul; 2456 instr->operand_size = get_operand_size(fsm, instr); 2457 instr->address_size = get_address_size(fsm, instr); 2458 2459 if (fsm->is64bit && (instr->operand_size == 4)) { 2460 /* Zero-extend to 64 bits. */ 2461 instr->zeroextend_mask = ~size_to_mask(4); 2462 } 2463 2464 if (opcode->regmodrm) { 2465 fsm_advance(fsm, 1, node_regmodrm); 2466 } else if (opcode->dmo) { 2467 /* Direct-Memory Offsets */ 2468 fsm_advance(fsm, 1, node_dmo); 2469 } else if (opcode->stos || opcode->lods) { 2470 fsm_advance(fsm, 1, node_stlo); 2471 } else if (opcode->movs) { 2472 fsm_advance(fsm, 1, node_movs); 2473 } else { 2474 return -1; 2475 } 2476 2477 return 0; 2478 } 2479 2480 static int 2481 node_secondary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2482 { 2483 const struct x86_opcode *opcode; 2484 uint8_t byte; 2485 2486 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2487 return -1; 2488 } 2489 2490 opcode = &secondary_opcode_table[byte]; 2491 if (__predict_false(!opcode->valid)) { 2492 return -1; 2493 } 2494 2495 instr->opcode = opcode; 2496 instr->emul = opcode->emul; 2497 instr->operand_size = get_operand_size(fsm, instr); 2498 instr->address_size = get_address_size(fsm, instr); 2499 2500 if (fsm->is64bit && (instr->operand_size == 4)) { 2501 /* Zero-extend to 64 bits. */ 2502 instr->zeroextend_mask = ~size_to_mask(4); 2503 } 2504 2505 if (opcode->flags & FLAG_ze) { 2506 /* 2507 * Compute the mask for zero-extend. Update the operand size, 2508 * we move fewer bytes. 2509 */ 2510 instr->zeroextend_mask |= size_to_mask(instr->operand_size); 2511 instr->zeroextend_mask &= ~size_to_mask(opcode->defsize); 2512 instr->operand_size = opcode->defsize; 2513 } 2514 2515 if (opcode->regmodrm) { 2516 fsm_advance(fsm, 1, node_regmodrm); 2517 } else { 2518 return -1; 2519 } 2520 2521 return 0; 2522 } 2523 2524 static int 2525 node_main(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2526 { 2527 uint8_t byte; 2528 2529 #define ESCAPE 0x0F 2530 #define VEX_1 0xC5 2531 #define VEX_2 0xC4 2532 #define XOP 0x8F 2533 2534 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2535 return -1; 2536 } 2537 2538 /* 2539 * We don't take XOP. It is AMD-specific, and it was removed shortly 2540 * after being introduced. 2541 */ 2542 if (byte == ESCAPE) { 2543 fsm_advance(fsm, 1, node_secondary_opcode); 2544 } else if (!instr->rexpref.present) { 2545 if (byte == VEX_1) { 2546 return -1; 2547 } else if (byte == VEX_2) { 2548 return -1; 2549 } else { 2550 fsm->fn = node_primary_opcode; 2551 } 2552 } else { 2553 fsm->fn = node_primary_opcode; 2554 } 2555 2556 return 0; 2557 } 2558 2559 static int 2560 node_rex_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2561 { 2562 struct x86_rexpref *rexpref = &instr->rexpref; 2563 uint8_t byte; 2564 size_t n = 0; 2565 2566 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2567 return -1; 2568 } 2569 2570 if (byte >= 0x40 && byte <= 0x4F) { 2571 if (__predict_false(!fsm->is64bit)) { 2572 return -1; 2573 } 2574 rexpref->b = ((byte & 0x1) != 0); 2575 rexpref->x = ((byte & 0x2) != 0); 2576 rexpref->r = ((byte & 0x4) != 0); 2577 rexpref->w = ((byte & 0x8) != 0); 2578 rexpref->present = true; 2579 n = 1; 2580 } 2581 2582 fsm_advance(fsm, n, node_main); 2583 return 0; 2584 } 2585 2586 static int 2587 node_legacy_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2588 { 2589 uint8_t byte; 2590 2591 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2592 return -1; 2593 } 2594 2595 if (byte == LEG_OPR_OVR) { 2596 instr->legpref.opr_ovr = 1; 2597 } else if (byte == LEG_OVR_DS) { 2598 instr->legpref.seg = NVMM_X64_SEG_DS; 2599 } else if (byte == LEG_OVR_ES) { 2600 instr->legpref.seg = NVMM_X64_SEG_ES; 2601 } else if (byte == LEG_REP) { 2602 instr->legpref.rep = 1; 2603 } else if (byte == LEG_OVR_GS) { 2604 instr->legpref.seg = NVMM_X64_SEG_GS; 2605 } else if (byte == LEG_OVR_FS) { 2606 instr->legpref.seg = NVMM_X64_SEG_FS; 2607 } else if (byte == LEG_ADR_OVR) { 2608 instr->legpref.adr_ovr = 1; 2609 } else if (byte == LEG_OVR_CS) { 2610 instr->legpref.seg = NVMM_X64_SEG_CS; 2611 } else if (byte == LEG_OVR_SS) { 2612 instr->legpref.seg = NVMM_X64_SEG_SS; 2613 } else if (byte == LEG_REPN) { 2614 instr->legpref.repn = 1; 2615 } else if (byte == LEG_LOCK) { 2616 /* ignore */ 2617 } else { 2618 /* not a legacy prefix */ 2619 fsm_advance(fsm, 0, node_rex_prefix); 2620 return 0; 2621 } 2622 2623 fsm_advance(fsm, 1, node_legacy_prefix); 2624 return 0; 2625 } 2626 2627 static int 2628 x86_decode(uint8_t *inst_bytes, size_t inst_len, struct x86_instr *instr, 2629 struct nvmm_x64_state *state) 2630 { 2631 struct x86_decode_fsm fsm; 2632 int ret; 2633 2634 memset(instr, 0, sizeof(*instr)); 2635 instr->legpref.seg = -1; 2636 instr->src.hardseg = -1; 2637 instr->dst.hardseg = -1; 2638 2639 fsm.is64bit = is_64bit(state); 2640 fsm.is32bit = is_32bit(state); 2641 fsm.is16bit = is_16bit(state); 2642 2643 fsm.fn = node_legacy_prefix; 2644 fsm.buf = inst_bytes; 2645 fsm.end = inst_bytes + inst_len; 2646 2647 while (fsm.fn != NULL) { 2648 ret = (*fsm.fn)(&fsm, instr); 2649 if (ret == -1) 2650 return -1; 2651 } 2652 2653 instr->len = fsm.buf - inst_bytes; 2654 2655 return 0; 2656 } 2657 2658 /* -------------------------------------------------------------------------- */ 2659 2660 #define EXEC_INSTR(sz, instr) \ 2661 static uint##sz##_t \ 2662 exec_##instr##sz(uint##sz##_t op1, uint##sz##_t op2, uint64_t *rflags) \ 2663 { \ 2664 uint##sz##_t res; \ 2665 __asm __volatile ( \ 2666 #instr" %2, %3;" \ 2667 "mov %3, %1;" \ 2668 "pushfq;" \ 2669 "popq %0" \ 2670 : "=r" (*rflags), "=r" (res) \ 2671 : "r" (op1), "r" (op2)); \ 2672 return res; \ 2673 } 2674 2675 #define EXEC_DISPATCHER(instr) \ 2676 static uint64_t \ 2677 exec_##instr(uint64_t op1, uint64_t op2, uint64_t *rflags, size_t opsize) \ 2678 { \ 2679 switch (opsize) { \ 2680 case 1: \ 2681 return exec_##instr##8(op1, op2, rflags); \ 2682 case 2: \ 2683 return exec_##instr##16(op1, op2, rflags); \ 2684 case 4: \ 2685 return exec_##instr##32(op1, op2, rflags); \ 2686 default: \ 2687 return exec_##instr##64(op1, op2, rflags); \ 2688 } \ 2689 } 2690 2691 /* SUB: ret = op1 - op2 */ 2692 #define PSL_SUB_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF|PSL_AF) 2693 EXEC_INSTR(8, sub) 2694 EXEC_INSTR(16, sub) 2695 EXEC_INSTR(32, sub) 2696 EXEC_INSTR(64, sub) 2697 EXEC_DISPATCHER(sub) 2698 2699 /* OR: ret = op1 | op2 */ 2700 #define PSL_OR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2701 EXEC_INSTR(8, or) 2702 EXEC_INSTR(16, or) 2703 EXEC_INSTR(32, or) 2704 EXEC_INSTR(64, or) 2705 EXEC_DISPATCHER(or) 2706 2707 /* AND: ret = op1 & op2 */ 2708 #define PSL_AND_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2709 EXEC_INSTR(8, and) 2710 EXEC_INSTR(16, and) 2711 EXEC_INSTR(32, and) 2712 EXEC_INSTR(64, and) 2713 EXEC_DISPATCHER(and) 2714 2715 /* XOR: ret = op1 ^ op2 */ 2716 #define PSL_XOR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2717 EXEC_INSTR(8, xor) 2718 EXEC_INSTR(16, xor) 2719 EXEC_INSTR(32, xor) 2720 EXEC_INSTR(64, xor) 2721 EXEC_DISPATCHER(xor) 2722 2723 /* -------------------------------------------------------------------------- */ 2724 2725 /* 2726 * Emulation functions. We don't care about the order of the operands, except 2727 * for SUB, CMP and TEST. For these ones we look at mem->write to determine who 2728 * is op1 and who is op2. 2729 */ 2730 2731 static void 2732 x86_func_or(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2733 { 2734 uint64_t *retval = (uint64_t *)mem->data; 2735 const bool write = mem->write; 2736 uint64_t *op1, op2, fl, ret; 2737 2738 op1 = (uint64_t *)mem->data; 2739 op2 = 0; 2740 2741 /* Fetch the value to be OR'ed (op2). */ 2742 mem->data = (uint8_t *)&op2; 2743 mem->write = false; 2744 (*vcpu->cbs.mem)(mem); 2745 2746 /* Perform the OR. */ 2747 ret = exec_or(*op1, op2, &fl, mem->size); 2748 2749 if (write) { 2750 /* Write back the result. */ 2751 mem->data = (uint8_t *)&ret; 2752 mem->write = true; 2753 (*vcpu->cbs.mem)(mem); 2754 } else { 2755 /* Return data to the caller. */ 2756 *retval = ret; 2757 } 2758 2759 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_OR_MASK; 2760 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_OR_MASK); 2761 } 2762 2763 static void 2764 x86_func_and(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2765 { 2766 uint64_t *retval = (uint64_t *)mem->data; 2767 const bool write = mem->write; 2768 uint64_t *op1, op2, fl, ret; 2769 2770 op1 = (uint64_t *)mem->data; 2771 op2 = 0; 2772 2773 /* Fetch the value to be AND'ed (op2). */ 2774 mem->data = (uint8_t *)&op2; 2775 mem->write = false; 2776 (*vcpu->cbs.mem)(mem); 2777 2778 /* Perform the AND. */ 2779 ret = exec_and(*op1, op2, &fl, mem->size); 2780 2781 if (write) { 2782 /* Write back the result. */ 2783 mem->data = (uint8_t *)&ret; 2784 mem->write = true; 2785 (*vcpu->cbs.mem)(mem); 2786 } else { 2787 /* Return data to the caller. */ 2788 *retval = ret; 2789 } 2790 2791 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2792 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2793 } 2794 2795 static void 2796 x86_func_xchg(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2797 { 2798 uint64_t *op1, op2; 2799 2800 op1 = (uint64_t *)mem->data; 2801 op2 = 0; 2802 2803 /* Fetch op2. */ 2804 mem->data = (uint8_t *)&op2; 2805 mem->write = false; 2806 (*vcpu->cbs.mem)(mem); 2807 2808 /* Write op1 in op2. */ 2809 mem->data = (uint8_t *)op1; 2810 mem->write = true; 2811 (*vcpu->cbs.mem)(mem); 2812 2813 /* Write op2 in op1. */ 2814 *op1 = op2; 2815 } 2816 2817 static void 2818 x86_func_sub(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2819 { 2820 uint64_t *retval = (uint64_t *)mem->data; 2821 const bool write = mem->write; 2822 uint64_t *op1, *op2, fl, ret; 2823 uint64_t tmp; 2824 bool memop1; 2825 2826 memop1 = !mem->write; 2827 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2828 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2829 2830 /* Fetch the value to be SUB'ed (op1 or op2). */ 2831 mem->data = (uint8_t *)&tmp; 2832 mem->write = false; 2833 (*vcpu->cbs.mem)(mem); 2834 2835 /* Perform the SUB. */ 2836 ret = exec_sub(*op1, *op2, &fl, mem->size); 2837 2838 if (write) { 2839 /* Write back the result. */ 2840 mem->data = (uint8_t *)&ret; 2841 mem->write = true; 2842 (*vcpu->cbs.mem)(mem); 2843 } else { 2844 /* Return data to the caller. */ 2845 *retval = ret; 2846 } 2847 2848 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2849 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2850 } 2851 2852 static void 2853 x86_func_xor(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2854 { 2855 uint64_t *retval = (uint64_t *)mem->data; 2856 const bool write = mem->write; 2857 uint64_t *op1, op2, fl, ret; 2858 2859 op1 = (uint64_t *)mem->data; 2860 op2 = 0; 2861 2862 /* Fetch the value to be XOR'ed (op2). */ 2863 mem->data = (uint8_t *)&op2; 2864 mem->write = false; 2865 (*vcpu->cbs.mem)(mem); 2866 2867 /* Perform the XOR. */ 2868 ret = exec_xor(*op1, op2, &fl, mem->size); 2869 2870 if (write) { 2871 /* Write back the result. */ 2872 mem->data = (uint8_t *)&ret; 2873 mem->write = true; 2874 (*vcpu->cbs.mem)(mem); 2875 } else { 2876 /* Return data to the caller. */ 2877 *retval = ret; 2878 } 2879 2880 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_XOR_MASK; 2881 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_XOR_MASK); 2882 } 2883 2884 static void 2885 x86_func_cmp(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2886 { 2887 uint64_t *op1, *op2, fl; 2888 uint64_t tmp; 2889 bool memop1; 2890 2891 memop1 = !mem->write; 2892 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2893 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2894 2895 /* Fetch the value to be CMP'ed (op1 or op2). */ 2896 mem->data = (uint8_t *)&tmp; 2897 mem->write = false; 2898 (*vcpu->cbs.mem)(mem); 2899 2900 /* Perform the CMP. */ 2901 exec_sub(*op1, *op2, &fl, mem->size); 2902 2903 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2904 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2905 } 2906 2907 static void 2908 x86_func_test(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2909 { 2910 uint64_t *op1, *op2, fl; 2911 uint64_t tmp; 2912 bool memop1; 2913 2914 memop1 = !mem->write; 2915 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2916 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2917 2918 /* Fetch the value to be TEST'ed (op1 or op2). */ 2919 mem->data = (uint8_t *)&tmp; 2920 mem->write = false; 2921 (*vcpu->cbs.mem)(mem); 2922 2923 /* Perform the TEST. */ 2924 exec_and(*op1, *op2, &fl, mem->size); 2925 2926 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2927 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2928 } 2929 2930 static void 2931 x86_func_mov(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2932 { 2933 /* 2934 * Nothing special, just move without emulation. 2935 */ 2936 (*vcpu->cbs.mem)(mem); 2937 } 2938 2939 static void 2940 x86_func_stos(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2941 { 2942 /* 2943 * Just move, and update RDI. 2944 */ 2945 (*vcpu->cbs.mem)(mem); 2946 2947 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2948 gprs[NVMM_X64_GPR_RDI] -= mem->size; 2949 } else { 2950 gprs[NVMM_X64_GPR_RDI] += mem->size; 2951 } 2952 } 2953 2954 static void 2955 x86_func_lods(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2956 { 2957 /* 2958 * Just move, and update RSI. 2959 */ 2960 (*vcpu->cbs.mem)(mem); 2961 2962 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2963 gprs[NVMM_X64_GPR_RSI] -= mem->size; 2964 } else { 2965 gprs[NVMM_X64_GPR_RSI] += mem->size; 2966 } 2967 } 2968 2969 static void 2970 x86_func_movs(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2971 { 2972 /* 2973 * Special instruction: double memory operand. Don't call the cb, 2974 * because the storage has already been performed earlier. 2975 */ 2976 2977 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2978 gprs[NVMM_X64_GPR_RSI] -= mem->size; 2979 gprs[NVMM_X64_GPR_RDI] -= mem->size; 2980 } else { 2981 gprs[NVMM_X64_GPR_RSI] += mem->size; 2982 gprs[NVMM_X64_GPR_RDI] += mem->size; 2983 } 2984 } 2985 2986 /* -------------------------------------------------------------------------- */ 2987 2988 static inline uint64_t 2989 gpr_read_address(struct x86_instr *instr, struct nvmm_x64_state *state, int gpr) 2990 { 2991 uint64_t val; 2992 2993 val = state->gprs[gpr]; 2994 val &= size_to_mask(instr->address_size); 2995 2996 return val; 2997 } 2998 2999 static int 3000 store_to_gva(struct nvmm_x64_state *state, struct x86_instr *instr, 3001 struct x86_store *store, gvaddr_t *gvap, size_t size) 3002 { 3003 struct x86_sib *sib; 3004 gvaddr_t gva = 0; 3005 uint64_t reg; 3006 int ret, seg; 3007 3008 if (store->type == STORE_SIB) { 3009 sib = &store->u.sib; 3010 if (sib->bas != NULL) 3011 gva += gpr_read_address(instr, state, sib->bas->num); 3012 if (sib->idx != NULL) { 3013 reg = gpr_read_address(instr, state, sib->idx->num); 3014 gva += sib->scale * reg; 3015 } 3016 } else if (store->type == STORE_REG) { 3017 if (store->u.reg == NULL) { 3018 /* The base is null. Happens with disp32-only and 3019 * disp16-only. */ 3020 } else { 3021 gva = gpr_read_address(instr, state, store->u.reg->num); 3022 } 3023 } else if (store->type == STORE_DUALREG) { 3024 gva = gpr_read_address(instr, state, store->u.dualreg.reg1) + 3025 gpr_read_address(instr, state, store->u.dualreg.reg2); 3026 } else { 3027 gva = store->u.dmo; 3028 } 3029 3030 if (store->disp.type != DISP_NONE) { 3031 gva += store->disp.data; 3032 } 3033 3034 if (store->hardseg != -1) { 3035 seg = store->hardseg; 3036 } else { 3037 if (__predict_false(instr->legpref.seg != -1)) { 3038 seg = instr->legpref.seg; 3039 } else { 3040 seg = NVMM_X64_SEG_DS; 3041 } 3042 } 3043 3044 if (__predict_true(is_long_mode(state))) { 3045 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 3046 segment_apply(&state->segs[seg], &gva); 3047 } 3048 } else { 3049 ret = segment_check(&state->segs[seg], gva, size); 3050 if (ret == -1) 3051 return -1; 3052 segment_apply(&state->segs[seg], &gva); 3053 } 3054 3055 *gvap = gva; 3056 return 0; 3057 } 3058 3059 static int 3060 fetch_segment(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3061 { 3062 struct nvmm_x64_state *state = vcpu->state; 3063 uint8_t inst_bytes[5], byte; 3064 size_t i, fetchsize; 3065 gvaddr_t gva; 3066 int ret, seg; 3067 3068 fetchsize = sizeof(inst_bytes); 3069 3070 gva = state->gprs[NVMM_X64_GPR_RIP]; 3071 if (__predict_false(!is_long_mode(state))) { 3072 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3073 fetchsize); 3074 if (ret == -1) 3075 return -1; 3076 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3077 } 3078 3079 ret = read_guest_memory(mach, vcpu, gva, inst_bytes, fetchsize); 3080 if (ret == -1) 3081 return -1; 3082 3083 seg = NVMM_X64_SEG_DS; 3084 for (i = 0; i < fetchsize; i++) { 3085 byte = inst_bytes[i]; 3086 3087 if (byte == LEG_OVR_DS) { 3088 seg = NVMM_X64_SEG_DS; 3089 } else if (byte == LEG_OVR_ES) { 3090 seg = NVMM_X64_SEG_ES; 3091 } else if (byte == LEG_OVR_GS) { 3092 seg = NVMM_X64_SEG_GS; 3093 } else if (byte == LEG_OVR_FS) { 3094 seg = NVMM_X64_SEG_FS; 3095 } else if (byte == LEG_OVR_CS) { 3096 seg = NVMM_X64_SEG_CS; 3097 } else if (byte == LEG_OVR_SS) { 3098 seg = NVMM_X64_SEG_SS; 3099 } else if (byte == LEG_OPR_OVR) { 3100 /* nothing */ 3101 } else if (byte == LEG_ADR_OVR) { 3102 /* nothing */ 3103 } else if (byte == LEG_REP) { 3104 /* nothing */ 3105 } else if (byte == LEG_REPN) { 3106 /* nothing */ 3107 } else if (byte == LEG_LOCK) { 3108 /* nothing */ 3109 } else { 3110 return seg; 3111 } 3112 } 3113 3114 return seg; 3115 } 3116 3117 static int 3118 fetch_instruction(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3119 struct nvmm_vcpu_exit *exit) 3120 { 3121 struct nvmm_x64_state *state = vcpu->state; 3122 size_t fetchsize; 3123 gvaddr_t gva; 3124 int ret; 3125 3126 fetchsize = sizeof(exit->u.mem.inst_bytes); 3127 3128 gva = state->gprs[NVMM_X64_GPR_RIP]; 3129 if (__predict_false(!is_long_mode(state))) { 3130 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3131 fetchsize); 3132 if (ret == -1) 3133 return -1; 3134 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3135 } 3136 3137 ret = read_guest_memory(mach, vcpu, gva, exit->u.mem.inst_bytes, 3138 fetchsize); 3139 if (ret == -1) 3140 return -1; 3141 3142 exit->u.mem.inst_len = fetchsize; 3143 3144 return 0; 3145 } 3146 3147 static int 3148 assist_mem_double(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3149 struct x86_instr *instr) 3150 { 3151 struct nvmm_x64_state *state = vcpu->state; 3152 struct nvmm_mem mem; 3153 uint8_t data[8]; 3154 gvaddr_t gva; 3155 size_t size; 3156 int ret; 3157 3158 size = instr->operand_size; 3159 3160 /* Source. */ 3161 ret = store_to_gva(state, instr, &instr->src, &gva, size); 3162 if (ret == -1) 3163 return -1; 3164 ret = read_guest_memory(mach, vcpu, gva, data, size); 3165 if (ret == -1) 3166 return -1; 3167 3168 /* Destination. */ 3169 ret = store_to_gva(state, instr, &instr->dst, &gva, size); 3170 if (ret == -1) 3171 return -1; 3172 ret = write_guest_memory(mach, vcpu, gva, data, size); 3173 if (ret == -1) 3174 return -1; 3175 3176 mem.size = size; 3177 (*instr->emul->func)(vcpu, &mem, state->gprs); 3178 3179 return 0; 3180 } 3181 3182 static int 3183 assist_mem_single(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3184 struct x86_instr *instr) 3185 { 3186 struct nvmm_x64_state *state = vcpu->state; 3187 struct nvmm_vcpu_exit *exit = vcpu->exit; 3188 struct nvmm_mem mem; 3189 uint8_t membuf[8]; 3190 uint64_t val; 3191 3192 memset(membuf, 0, sizeof(membuf)); 3193 3194 mem.mach = mach; 3195 mem.vcpu = vcpu; 3196 mem.gpa = exit->u.mem.gpa; 3197 mem.size = instr->operand_size; 3198 mem.data = membuf; 3199 3200 /* Determine the direction. */ 3201 switch (instr->src.type) { 3202 case STORE_REG: 3203 if (instr->src.disp.type != DISP_NONE) { 3204 /* Indirect access. */ 3205 mem.write = false; 3206 } else { 3207 /* Direct access. */ 3208 mem.write = true; 3209 } 3210 break; 3211 case STORE_DUALREG: 3212 if (instr->src.disp.type == DISP_NONE) { 3213 DISASSEMBLER_BUG(); 3214 } 3215 mem.write = false; 3216 break; 3217 case STORE_IMM: 3218 mem.write = true; 3219 break; 3220 case STORE_SIB: 3221 mem.write = false; 3222 break; 3223 case STORE_DMO: 3224 mem.write = false; 3225 break; 3226 default: 3227 DISASSEMBLER_BUG(); 3228 } 3229 3230 if (mem.write) { 3231 switch (instr->src.type) { 3232 case STORE_REG: 3233 /* The instruction was "reg -> mem". Fetch the register 3234 * in membuf. */ 3235 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3236 DISASSEMBLER_BUG(); 3237 } 3238 val = state->gprs[instr->src.u.reg->num]; 3239 val = __SHIFTOUT(val, instr->src.u.reg->mask); 3240 memcpy(mem.data, &val, mem.size); 3241 break; 3242 case STORE_IMM: 3243 /* The instruction was "imm -> mem". Fetch the immediate 3244 * in membuf. */ 3245 memcpy(mem.data, &instr->src.u.imm.data, mem.size); 3246 break; 3247 default: 3248 DISASSEMBLER_BUG(); 3249 } 3250 } else if (instr->emul->readreg) { 3251 /* The instruction was "mem -> reg", but the value of the 3252 * register matters for the emul func. Fetch it in membuf. */ 3253 if (__predict_false(instr->dst.type != STORE_REG)) { 3254 DISASSEMBLER_BUG(); 3255 } 3256 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3257 DISASSEMBLER_BUG(); 3258 } 3259 val = state->gprs[instr->dst.u.reg->num]; 3260 val = __SHIFTOUT(val, instr->dst.u.reg->mask); 3261 memcpy(mem.data, &val, mem.size); 3262 } 3263 3264 (*instr->emul->func)(vcpu, &mem, state->gprs); 3265 3266 if (instr->emul->notouch) { 3267 /* We're done. */ 3268 return 0; 3269 } 3270 3271 if (!mem.write) { 3272 /* The instruction was "mem -> reg". The emul func has filled 3273 * membuf with the memory content. Install membuf in the 3274 * register. */ 3275 if (__predict_false(instr->dst.type != STORE_REG)) { 3276 DISASSEMBLER_BUG(); 3277 } 3278 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3279 DISASSEMBLER_BUG(); 3280 } 3281 memcpy(&val, membuf, sizeof(uint64_t)); 3282 val = __SHIFTIN(val, instr->dst.u.reg->mask); 3283 state->gprs[instr->dst.u.reg->num] &= ~instr->dst.u.reg->mask; 3284 state->gprs[instr->dst.u.reg->num] |= val; 3285 state->gprs[instr->dst.u.reg->num] &= ~instr->zeroextend_mask; 3286 } else if (instr->emul->backprop) { 3287 /* The instruction was "reg -> mem", but the memory must be 3288 * back-propagated to the register. Install membuf in the 3289 * register. */ 3290 if (__predict_false(instr->src.type != STORE_REG)) { 3291 DISASSEMBLER_BUG(); 3292 } 3293 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3294 DISASSEMBLER_BUG(); 3295 } 3296 memcpy(&val, membuf, sizeof(uint64_t)); 3297 val = __SHIFTIN(val, instr->src.u.reg->mask); 3298 state->gprs[instr->src.u.reg->num] &= ~instr->src.u.reg->mask; 3299 state->gprs[instr->src.u.reg->num] |= val; 3300 state->gprs[instr->src.u.reg->num] &= ~instr->zeroextend_mask; 3301 } 3302 3303 return 0; 3304 } 3305 3306 int 3307 nvmm_assist_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3308 { 3309 struct nvmm_x64_state *state = vcpu->state; 3310 struct nvmm_vcpu_exit *exit = vcpu->exit; 3311 struct x86_instr instr; 3312 uint64_t cnt = 0; /* GCC */ 3313 int ret; 3314 3315 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_MEMORY)) { 3316 errno = EINVAL; 3317 return -1; 3318 } 3319 3320 ret = nvmm_vcpu_getstate(mach, vcpu, 3321 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 3322 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 3323 if (ret == -1) 3324 return -1; 3325 3326 if (exit->u.mem.inst_len == 0) { 3327 /* 3328 * The instruction was not fetched from the kernel. Fetch 3329 * it ourselves. 3330 */ 3331 ret = fetch_instruction(mach, vcpu, exit); 3332 if (ret == -1) 3333 return -1; 3334 } 3335 3336 ret = x86_decode(exit->u.mem.inst_bytes, exit->u.mem.inst_len, 3337 &instr, state); 3338 if (ret == -1) { 3339 errno = ENODEV; 3340 return -1; 3341 } 3342 3343 if (instr.legpref.rep || instr.legpref.repn) { 3344 cnt = rep_get_cnt(state, instr.address_size); 3345 if (__predict_false(cnt == 0)) { 3346 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3347 goto out; 3348 } 3349 } 3350 3351 if (instr.opcode->movs) { 3352 ret = assist_mem_double(mach, vcpu, &instr); 3353 } else { 3354 ret = assist_mem_single(mach, vcpu, &instr); 3355 } 3356 if (ret == -1) { 3357 errno = ENODEV; 3358 return -1; 3359 } 3360 3361 if (instr.legpref.rep || instr.legpref.repn) { 3362 cnt -= 1; 3363 rep_set_cnt(state, instr.address_size, cnt); 3364 if (cnt == 0) { 3365 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3366 } else if (__predict_false(instr.legpref.repn)) { 3367 if (state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_Z) { 3368 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3369 } 3370 } 3371 } else { 3372 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3373 } 3374 3375 out: 3376 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 3377 if (ret == -1) 3378 return -1; 3379 3380 return 0; 3381 } 3382
Indexes created Fri Sep 26 08:10:20 GMT 2025