libnvmm_x86.c revision 1.36
1 /* $NetBSD: libnvmm_x86.c,v 1.36 2019/10/23 07:01:11 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 #include <unistd.h> 38 #include <fcntl.h> 39 #include <errno.h> 40 #include <sys/ioctl.h> 41 #include <sys/mman.h> 42 #include <machine/vmparam.h> 43 #include <machine/pte.h> 44 #include <machine/psl.h> 45 46 #define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) 47 #define __cacheline_aligned __attribute__((__aligned__(64))) 48 49 #include <x86/specialreg.h> 50 51 /* -------------------------------------------------------------------------- */ 52 53 /* 54 * Undocumented debugging function. Helpful. 55 */ 56 int 57 nvmm_vcpu_dump(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 58 { 59 struct nvmm_x64_state *state = vcpu->state; 60 uint16_t *attr; 61 size_t i; 62 int ret; 63 64 const char *segnames[] = { 65 "ES", "CS", "SS", "DS", "FS", "GS", "GDT", "IDT", "LDT", "TR" 66 }; 67 68 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_ALL); 69 if (ret == -1) 70 return -1; 71 72 printf("+ VCPU id=%d\n", (int)vcpu->cpuid); 73 printf("| -> RAX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RAX]); 74 printf("| -> RCX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RCX]); 75 printf("| -> RDX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDX]); 76 printf("| -> RBX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBX]); 77 printf("| -> RSP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSP]); 78 printf("| -> RBP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBP]); 79 printf("| -> RSI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSI]); 80 printf("| -> RDI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDI]); 81 printf("| -> RIP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RIP]); 82 printf("| -> RFLAGS=%p\n", (void *)state->gprs[NVMM_X64_GPR_RFLAGS]); 83 for (i = 0; i < NVMM_X64_NSEG; i++) { 84 attr = (uint16_t *)&state->segs[i].attrib; 85 printf("| -> %s: sel=0x%x base=%"PRIx64", limit=%x, " 86 "attrib=%x [type=%d,l=%d,def=%d]\n", 87 segnames[i], 88 state->segs[i].selector, 89 state->segs[i].base, 90 state->segs[i].limit, 91 *attr, 92 state->segs[i].attrib.type, 93 state->segs[i].attrib.l, 94 state->segs[i].attrib.def); 95 } 96 printf("| -> MSR_EFER=%"PRIx64"\n", state->msrs[NVMM_X64_MSR_EFER]); 97 printf("| -> CR0=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR0]); 98 printf("| -> CR3=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR3]); 99 printf("| -> CR4=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR4]); 100 printf("| -> CR8=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR8]); 101 102 return 0; 103 } 104 105 /* -------------------------------------------------------------------------- */ 106 107 #define PTE32_L1_SHIFT 12 108 #define PTE32_L2_SHIFT 22 109 110 #define PTE32_L2_MASK 0xffc00000 111 #define PTE32_L1_MASK 0x003ff000 112 113 #define PTE32_L2_FRAME (PTE32_L2_MASK) 114 #define PTE32_L1_FRAME (PTE32_L2_FRAME|PTE32_L1_MASK) 115 116 #define pte32_l1idx(va) (((va) & PTE32_L1_MASK) >> PTE32_L1_SHIFT) 117 #define pte32_l2idx(va) (((va) & PTE32_L2_MASK) >> PTE32_L2_SHIFT) 118 119 #define CR3_FRAME_32BIT PG_FRAME 120 121 typedef uint32_t pte_32bit_t; 122 123 static int 124 x86_gva_to_gpa_32bit(struct nvmm_machine *mach, uint64_t cr3, 125 gvaddr_t gva, gpaddr_t *gpa, bool has_pse, nvmm_prot_t *prot) 126 { 127 gpaddr_t L2gpa, L1gpa; 128 uintptr_t L2hva, L1hva; 129 pte_32bit_t *pdir, pte; 130 nvmm_prot_t pageprot; 131 132 /* We begin with an RWXU access. */ 133 *prot = NVMM_PROT_ALL; 134 135 /* Parse L2. */ 136 L2gpa = (cr3 & CR3_FRAME_32BIT); 137 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 138 return -1; 139 pdir = (pte_32bit_t *)L2hva; 140 pte = pdir[pte32_l2idx(gva)]; 141 if ((pte & PG_V) == 0) 142 return -1; 143 if ((pte & PG_u) == 0) 144 *prot &= ~NVMM_PROT_USER; 145 if ((pte & PG_KW) == 0) 146 *prot &= ~NVMM_PROT_WRITE; 147 if ((pte & PG_PS) && !has_pse) 148 return -1; 149 if (pte & PG_PS) { 150 *gpa = (pte & PTE32_L2_FRAME); 151 *gpa = *gpa + (gva & PTE32_L1_MASK); 152 return 0; 153 } 154 155 /* Parse L1. */ 156 L1gpa = (pte & PG_FRAME); 157 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 158 return -1; 159 pdir = (pte_32bit_t *)L1hva; 160 pte = pdir[pte32_l1idx(gva)]; 161 if ((pte & PG_V) == 0) 162 return -1; 163 if ((pte & PG_u) == 0) 164 *prot &= ~NVMM_PROT_USER; 165 if ((pte & PG_KW) == 0) 166 *prot &= ~NVMM_PROT_WRITE; 167 if (pte & PG_PS) 168 return -1; 169 170 *gpa = (pte & PG_FRAME); 171 return 0; 172 } 173 174 /* -------------------------------------------------------------------------- */ 175 176 #define PTE32_PAE_L1_SHIFT 12 177 #define PTE32_PAE_L2_SHIFT 21 178 #define PTE32_PAE_L3_SHIFT 30 179 180 #define PTE32_PAE_L3_MASK 0xc0000000 181 #define PTE32_PAE_L2_MASK 0x3fe00000 182 #define PTE32_PAE_L1_MASK 0x001ff000 183 184 #define PTE32_PAE_L3_FRAME (PTE32_PAE_L3_MASK) 185 #define PTE32_PAE_L2_FRAME (PTE32_PAE_L3_FRAME|PTE32_PAE_L2_MASK) 186 #define PTE32_PAE_L1_FRAME (PTE32_PAE_L2_FRAME|PTE32_PAE_L1_MASK) 187 188 #define pte32_pae_l1idx(va) (((va) & PTE32_PAE_L1_MASK) >> PTE32_PAE_L1_SHIFT) 189 #define pte32_pae_l2idx(va) (((va) & PTE32_PAE_L2_MASK) >> PTE32_PAE_L2_SHIFT) 190 #define pte32_pae_l3idx(va) (((va) & PTE32_PAE_L3_MASK) >> PTE32_PAE_L3_SHIFT) 191 192 #define CR3_FRAME_32BIT_PAE __BITS(31, 5) 193 194 typedef uint64_t pte_32bit_pae_t; 195 196 static int 197 x86_gva_to_gpa_32bit_pae(struct nvmm_machine *mach, uint64_t cr3, 198 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 199 { 200 gpaddr_t L3gpa, L2gpa, L1gpa; 201 uintptr_t L3hva, L2hva, L1hva; 202 pte_32bit_pae_t *pdir, pte; 203 nvmm_prot_t pageprot; 204 205 /* We begin with an RWXU access. */ 206 *prot = NVMM_PROT_ALL; 207 208 /* Parse L3. */ 209 L3gpa = (cr3 & CR3_FRAME_32BIT_PAE); 210 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 211 return -1; 212 pdir = (pte_32bit_pae_t *)L3hva; 213 pte = pdir[pte32_pae_l3idx(gva)]; 214 if ((pte & PG_V) == 0) 215 return -1; 216 if (pte & PG_NX) 217 *prot &= ~NVMM_PROT_EXEC; 218 if (pte & PG_PS) 219 return -1; 220 221 /* Parse L2. */ 222 L2gpa = (pte & PG_FRAME); 223 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 224 return -1; 225 pdir = (pte_32bit_pae_t *)L2hva; 226 pte = pdir[pte32_pae_l2idx(gva)]; 227 if ((pte & PG_V) == 0) 228 return -1; 229 if ((pte & PG_u) == 0) 230 *prot &= ~NVMM_PROT_USER; 231 if ((pte & PG_KW) == 0) 232 *prot &= ~NVMM_PROT_WRITE; 233 if (pte & PG_NX) 234 *prot &= ~NVMM_PROT_EXEC; 235 if (pte & PG_PS) { 236 *gpa = (pte & PTE32_PAE_L2_FRAME); 237 *gpa = *gpa + (gva & PTE32_PAE_L1_MASK); 238 return 0; 239 } 240 241 /* Parse L1. */ 242 L1gpa = (pte & PG_FRAME); 243 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 244 return -1; 245 pdir = (pte_32bit_pae_t *)L1hva; 246 pte = pdir[pte32_pae_l1idx(gva)]; 247 if ((pte & PG_V) == 0) 248 return -1; 249 if ((pte & PG_u) == 0) 250 *prot &= ~NVMM_PROT_USER; 251 if ((pte & PG_KW) == 0) 252 *prot &= ~NVMM_PROT_WRITE; 253 if (pte & PG_NX) 254 *prot &= ~NVMM_PROT_EXEC; 255 if (pte & PG_PS) 256 return -1; 257 258 *gpa = (pte & PG_FRAME); 259 return 0; 260 } 261 262 /* -------------------------------------------------------------------------- */ 263 264 #define PTE64_L1_SHIFT 12 265 #define PTE64_L2_SHIFT 21 266 #define PTE64_L3_SHIFT 30 267 #define PTE64_L4_SHIFT 39 268 269 #define PTE64_L4_MASK 0x0000ff8000000000 270 #define PTE64_L3_MASK 0x0000007fc0000000 271 #define PTE64_L2_MASK 0x000000003fe00000 272 #define PTE64_L1_MASK 0x00000000001ff000 273 274 #define PTE64_L4_FRAME PTE64_L4_MASK 275 #define PTE64_L3_FRAME (PTE64_L4_FRAME|PTE64_L3_MASK) 276 #define PTE64_L2_FRAME (PTE64_L3_FRAME|PTE64_L2_MASK) 277 #define PTE64_L1_FRAME (PTE64_L2_FRAME|PTE64_L1_MASK) 278 279 #define pte64_l1idx(va) (((va) & PTE64_L1_MASK) >> PTE64_L1_SHIFT) 280 #define pte64_l2idx(va) (((va) & PTE64_L2_MASK) >> PTE64_L2_SHIFT) 281 #define pte64_l3idx(va) (((va) & PTE64_L3_MASK) >> PTE64_L3_SHIFT) 282 #define pte64_l4idx(va) (((va) & PTE64_L4_MASK) >> PTE64_L4_SHIFT) 283 284 #define CR3_FRAME_64BIT PG_FRAME 285 286 typedef uint64_t pte_64bit_t; 287 288 static inline bool 289 x86_gva_64bit_canonical(gvaddr_t gva) 290 { 291 /* Bits 63:47 must have the same value. */ 292 #define SIGN_EXTEND 0xffff800000000000ULL 293 return (gva & SIGN_EXTEND) == 0 || (gva & SIGN_EXTEND) == SIGN_EXTEND; 294 } 295 296 static int 297 x86_gva_to_gpa_64bit(struct nvmm_machine *mach, uint64_t cr3, 298 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 299 { 300 gpaddr_t L4gpa, L3gpa, L2gpa, L1gpa; 301 uintptr_t L4hva, L3hva, L2hva, L1hva; 302 pte_64bit_t *pdir, pte; 303 nvmm_prot_t pageprot; 304 305 /* We begin with an RWXU access. */ 306 *prot = NVMM_PROT_ALL; 307 308 if (!x86_gva_64bit_canonical(gva)) 309 return -1; 310 311 /* Parse L4. */ 312 L4gpa = (cr3 & CR3_FRAME_64BIT); 313 if (nvmm_gpa_to_hva(mach, L4gpa, &L4hva, &pageprot) == -1) 314 return -1; 315 pdir = (pte_64bit_t *)L4hva; 316 pte = pdir[pte64_l4idx(gva)]; 317 if ((pte & PG_V) == 0) 318 return -1; 319 if ((pte & PG_u) == 0) 320 *prot &= ~NVMM_PROT_USER; 321 if ((pte & PG_KW) == 0) 322 *prot &= ~NVMM_PROT_WRITE; 323 if (pte & PG_NX) 324 *prot &= ~NVMM_PROT_EXEC; 325 if (pte & PG_PS) 326 return -1; 327 328 /* Parse L3. */ 329 L3gpa = (pte & PG_FRAME); 330 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 331 return -1; 332 pdir = (pte_64bit_t *)L3hva; 333 pte = pdir[pte64_l3idx(gva)]; 334 if ((pte & PG_V) == 0) 335 return -1; 336 if ((pte & PG_u) == 0) 337 *prot &= ~NVMM_PROT_USER; 338 if ((pte & PG_KW) == 0) 339 *prot &= ~NVMM_PROT_WRITE; 340 if (pte & PG_NX) 341 *prot &= ~NVMM_PROT_EXEC; 342 if (pte & PG_PS) { 343 *gpa = (pte & PTE64_L3_FRAME); 344 *gpa = *gpa + (gva & (PTE64_L2_MASK|PTE64_L1_MASK)); 345 return 0; 346 } 347 348 /* Parse L2. */ 349 L2gpa = (pte & PG_FRAME); 350 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 351 return -1; 352 pdir = (pte_64bit_t *)L2hva; 353 pte = pdir[pte64_l2idx(gva)]; 354 if ((pte & PG_V) == 0) 355 return -1; 356 if ((pte & PG_u) == 0) 357 *prot &= ~NVMM_PROT_USER; 358 if ((pte & PG_KW) == 0) 359 *prot &= ~NVMM_PROT_WRITE; 360 if (pte & PG_NX) 361 *prot &= ~NVMM_PROT_EXEC; 362 if (pte & PG_PS) { 363 *gpa = (pte & PTE64_L2_FRAME); 364 *gpa = *gpa + (gva & PTE64_L1_MASK); 365 return 0; 366 } 367 368 /* Parse L1. */ 369 L1gpa = (pte & PG_FRAME); 370 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 371 return -1; 372 pdir = (pte_64bit_t *)L1hva; 373 pte = pdir[pte64_l1idx(gva)]; 374 if ((pte & PG_V) == 0) 375 return -1; 376 if ((pte & PG_u) == 0) 377 *prot &= ~NVMM_PROT_USER; 378 if ((pte & PG_KW) == 0) 379 *prot &= ~NVMM_PROT_WRITE; 380 if (pte & PG_NX) 381 *prot &= ~NVMM_PROT_EXEC; 382 if (pte & PG_PS) 383 return -1; 384 385 *gpa = (pte & PG_FRAME); 386 return 0; 387 } 388 389 static inline int 390 x86_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_x64_state *state, 391 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 392 { 393 bool is_pae, is_lng, has_pse; 394 uint64_t cr3; 395 size_t off; 396 int ret; 397 398 if ((state->crs[NVMM_X64_CR_CR0] & CR0_PG) == 0) { 399 /* No paging. */ 400 *prot = NVMM_PROT_ALL; 401 *gpa = gva; 402 return 0; 403 } 404 405 off = (gva & PAGE_MASK); 406 gva &= ~PAGE_MASK; 407 408 is_pae = (state->crs[NVMM_X64_CR_CR4] & CR4_PAE) != 0; 409 is_lng = (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 410 has_pse = (state->crs[NVMM_X64_CR_CR4] & CR4_PSE) != 0; 411 cr3 = state->crs[NVMM_X64_CR_CR3]; 412 413 if (is_pae && is_lng) { 414 /* 64bit */ 415 ret = x86_gva_to_gpa_64bit(mach, cr3, gva, gpa, prot); 416 } else if (is_pae && !is_lng) { 417 /* 32bit PAE */ 418 ret = x86_gva_to_gpa_32bit_pae(mach, cr3, gva, gpa, prot); 419 } else if (!is_pae && !is_lng) { 420 /* 32bit */ 421 ret = x86_gva_to_gpa_32bit(mach, cr3, gva, gpa, has_pse, prot); 422 } else { 423 ret = -1; 424 } 425 426 if (ret == -1) { 427 errno = EFAULT; 428 } 429 430 *gpa = *gpa + off; 431 432 return ret; 433 } 434 435 int 436 nvmm_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 437 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 438 { 439 struct nvmm_x64_state *state = vcpu->state; 440 int ret; 441 442 ret = nvmm_vcpu_getstate(mach, vcpu, 443 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 444 if (ret == -1) 445 return -1; 446 447 return x86_gva_to_gpa(mach, state, gva, gpa, prot); 448 } 449 450 /* -------------------------------------------------------------------------- */ 451 452 #define DISASSEMBLER_BUG() \ 453 do { \ 454 errno = EINVAL; \ 455 return -1; \ 456 } while (0); 457 458 static inline bool 459 is_long_mode(struct nvmm_x64_state *state) 460 { 461 return (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 462 } 463 464 static inline bool 465 is_64bit(struct nvmm_x64_state *state) 466 { 467 return (state->segs[NVMM_X64_SEG_CS].attrib.l != 0); 468 } 469 470 static inline bool 471 is_32bit(struct nvmm_x64_state *state) 472 { 473 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 474 (state->segs[NVMM_X64_SEG_CS].attrib.def == 1); 475 } 476 477 static inline bool 478 is_16bit(struct nvmm_x64_state *state) 479 { 480 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 481 (state->segs[NVMM_X64_SEG_CS].attrib.def == 0); 482 } 483 484 static int 485 segment_check(struct nvmm_x64_state_seg *seg, gvaddr_t gva, size_t size) 486 { 487 uint64_t limit; 488 489 /* 490 * This is incomplete. We should check topdown, etc, really that's 491 * tiring. 492 */ 493 if (__predict_false(!seg->attrib.p)) { 494 goto error; 495 } 496 497 limit = (uint64_t)seg->limit + 1; 498 if (__predict_true(seg->attrib.g)) { 499 limit *= PAGE_SIZE; 500 } 501 502 if (__predict_false(gva + size > limit)) { 503 goto error; 504 } 505 506 return 0; 507 508 error: 509 errno = EFAULT; 510 return -1; 511 } 512 513 static inline void 514 segment_apply(struct nvmm_x64_state_seg *seg, gvaddr_t *gva) 515 { 516 *gva += seg->base; 517 } 518 519 static inline uint64_t 520 size_to_mask(size_t size) 521 { 522 switch (size) { 523 case 1: 524 return 0x00000000000000FF; 525 case 2: 526 return 0x000000000000FFFF; 527 case 4: 528 return 0x00000000FFFFFFFF; 529 case 8: 530 default: 531 return 0xFFFFFFFFFFFFFFFF; 532 } 533 } 534 535 static uint64_t 536 rep_get_cnt(struct nvmm_x64_state *state, size_t adsize) 537 { 538 uint64_t mask, cnt; 539 540 mask = size_to_mask(adsize); 541 cnt = state->gprs[NVMM_X64_GPR_RCX] & mask; 542 543 return cnt; 544 } 545 546 static void 547 rep_set_cnt(struct nvmm_x64_state *state, size_t adsize, uint64_t cnt) 548 { 549 uint64_t mask; 550 551 /* XXX: should we zero-extend? */ 552 mask = size_to_mask(adsize); 553 state->gprs[NVMM_X64_GPR_RCX] &= ~mask; 554 state->gprs[NVMM_X64_GPR_RCX] |= cnt; 555 } 556 557 static int 558 read_guest_memory(struct nvmm_machine *mach, struct nvmm_x64_state *state, 559 gvaddr_t gva, uint8_t *data, size_t size) 560 { 561 struct nvmm_mem mem; 562 nvmm_prot_t prot; 563 gpaddr_t gpa; 564 uintptr_t hva; 565 bool is_mmio; 566 int ret, remain; 567 568 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 569 if (__predict_false(ret == -1)) { 570 return -1; 571 } 572 if (__predict_false(!(prot & NVMM_PROT_READ))) { 573 errno = EFAULT; 574 return -1; 575 } 576 577 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 578 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 579 } else { 580 remain = 0; 581 } 582 size -= remain; 583 584 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 585 is_mmio = (ret == -1); 586 587 if (is_mmio) { 588 mem.data = data; 589 mem.gpa = gpa; 590 mem.write = false; 591 mem.size = size; 592 (*mach->cbs.mem)(&mem); 593 } else { 594 if (__predict_false(!(prot & NVMM_PROT_READ))) { 595 errno = EFAULT; 596 return -1; 597 } 598 memcpy(data, (uint8_t *)hva, size); 599 } 600 601 if (remain > 0) { 602 ret = read_guest_memory(mach, state, gva + size, 603 data + size, remain); 604 } else { 605 ret = 0; 606 } 607 608 return ret; 609 } 610 611 static int 612 write_guest_memory(struct nvmm_machine *mach, struct nvmm_x64_state *state, 613 gvaddr_t gva, uint8_t *data, size_t size) 614 { 615 struct nvmm_mem mem; 616 nvmm_prot_t prot; 617 gpaddr_t gpa; 618 uintptr_t hva; 619 bool is_mmio; 620 int ret, remain; 621 622 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 623 if (__predict_false(ret == -1)) { 624 return -1; 625 } 626 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 627 errno = EFAULT; 628 return -1; 629 } 630 631 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 632 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 633 } else { 634 remain = 0; 635 } 636 size -= remain; 637 638 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 639 is_mmio = (ret == -1); 640 641 if (is_mmio) { 642 mem.data = data; 643 mem.gpa = gpa; 644 mem.write = true; 645 mem.size = size; 646 (*mach->cbs.mem)(&mem); 647 } else { 648 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 649 errno = EFAULT; 650 return -1; 651 } 652 memcpy((uint8_t *)hva, data, size); 653 } 654 655 if (remain > 0) { 656 ret = write_guest_memory(mach, state, gva + size, 657 data + size, remain); 658 } else { 659 ret = 0; 660 } 661 662 return ret; 663 } 664 665 /* -------------------------------------------------------------------------- */ 666 667 static int fetch_segment(struct nvmm_machine *, struct nvmm_x64_state *); 668 669 #define NVMM_IO_BATCH_SIZE 32 670 671 static int 672 assist_io_batch(struct nvmm_machine *mach, struct nvmm_x64_state *state, 673 struct nvmm_io *io, gvaddr_t gva, uint64_t cnt) 674 { 675 uint8_t iobuf[NVMM_IO_BATCH_SIZE]; 676 size_t i, iosize, iocnt; 677 int ret; 678 679 cnt = MIN(cnt, NVMM_IO_BATCH_SIZE); 680 iosize = MIN(io->size * cnt, NVMM_IO_BATCH_SIZE); 681 iocnt = iosize / io->size; 682 683 io->data = iobuf; 684 685 if (!io->in) { 686 ret = read_guest_memory(mach, state, gva, iobuf, iosize); 687 if (ret == -1) 688 return -1; 689 } 690 691 for (i = 0; i < iocnt; i++) { 692 (*mach->cbs.io)(io); 693 io->data += io->size; 694 } 695 696 if (io->in) { 697 ret = write_guest_memory(mach, state, gva, iobuf, iosize); 698 if (ret == -1) 699 return -1; 700 } 701 702 return iocnt; 703 } 704 705 int 706 nvmm_assist_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 707 { 708 struct nvmm_x64_state *state = vcpu->state; 709 struct nvmm_vcpu_exit *exit = vcpu->exit; 710 struct nvmm_io io; 711 uint64_t cnt = 0; /* GCC */ 712 uint8_t iobuf[8]; 713 int iocnt = 1; 714 gvaddr_t gva = 0; /* GCC */ 715 int reg = 0; /* GCC */ 716 int ret, seg; 717 bool psld = false; 718 719 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_IO)) { 720 errno = EINVAL; 721 return -1; 722 } 723 724 io.port = exit->u.io.port; 725 io.in = exit->u.io.in; 726 io.size = exit->u.io.operand_size; 727 io.data = iobuf; 728 729 ret = nvmm_vcpu_getstate(mach, vcpu, 730 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 731 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 732 if (ret == -1) 733 return -1; 734 735 if (exit->u.io.rep) { 736 cnt = rep_get_cnt(state, exit->u.io.address_size); 737 if (__predict_false(cnt == 0)) { 738 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 739 goto out; 740 } 741 } 742 743 if (__predict_false(state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_D)) { 744 psld = true; 745 } 746 747 /* 748 * Determine GVA. 749 */ 750 if (exit->u.io.str) { 751 if (io.in) { 752 reg = NVMM_X64_GPR_RDI; 753 } else { 754 reg = NVMM_X64_GPR_RSI; 755 } 756 757 gva = state->gprs[reg]; 758 gva &= size_to_mask(exit->u.io.address_size); 759 760 if (exit->u.io.seg != -1) { 761 seg = exit->u.io.seg; 762 } else { 763 if (io.in) { 764 seg = NVMM_X64_SEG_ES; 765 } else { 766 seg = fetch_segment(mach, state); 767 if (seg == -1) 768 return -1; 769 } 770 } 771 772 if (__predict_true(is_long_mode(state))) { 773 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 774 segment_apply(&state->segs[seg], &gva); 775 } 776 } else { 777 ret = segment_check(&state->segs[seg], gva, io.size); 778 if (ret == -1) 779 return -1; 780 segment_apply(&state->segs[seg], &gva); 781 } 782 783 if (exit->u.io.rep && !psld) { 784 iocnt = assist_io_batch(mach, state, &io, gva, cnt); 785 if (iocnt == -1) 786 return -1; 787 goto done; 788 } 789 } 790 791 if (!io.in) { 792 if (!exit->u.io.str) { 793 memcpy(io.data, &state->gprs[NVMM_X64_GPR_RAX], io.size); 794 } else { 795 ret = read_guest_memory(mach, state, gva, io.data, 796 io.size); 797 if (ret == -1) 798 return -1; 799 } 800 } 801 802 (*mach->cbs.io)(&io); 803 804 if (io.in) { 805 if (!exit->u.io.str) { 806 memcpy(&state->gprs[NVMM_X64_GPR_RAX], io.data, io.size); 807 if (io.size == 4) { 808 /* Zero-extend to 64 bits. */ 809 state->gprs[NVMM_X64_GPR_RAX] &= size_to_mask(4); 810 } 811 } else { 812 ret = write_guest_memory(mach, state, gva, io.data, 813 io.size); 814 if (ret == -1) 815 return -1; 816 } 817 } 818 819 done: 820 if (exit->u.io.str) { 821 if (__predict_false(psld)) { 822 state->gprs[reg] -= iocnt * io.size; 823 } else { 824 state->gprs[reg] += iocnt * io.size; 825 } 826 } 827 828 if (exit->u.io.rep) { 829 cnt -= iocnt; 830 rep_set_cnt(state, exit->u.io.address_size, cnt); 831 if (cnt == 0) { 832 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 833 } 834 } else { 835 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 836 } 837 838 out: 839 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 840 if (ret == -1) 841 return -1; 842 843 return 0; 844 } 845 846 /* -------------------------------------------------------------------------- */ 847 848 struct x86_emul { 849 bool readreg; 850 bool backprop; 851 bool notouch; 852 void (*func)(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 853 }; 854 855 static void x86_func_or(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 856 static void x86_func_and(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 857 static void x86_func_xchg(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 858 static void x86_func_sub(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 859 static void x86_func_xor(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 860 static void x86_func_cmp(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 861 static void x86_func_test(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 862 static void x86_func_mov(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 863 static void x86_func_stos(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 864 static void x86_func_lods(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 865 static void x86_func_movs(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 866 867 static const struct x86_emul x86_emul_or = { 868 .readreg = true, 869 .func = x86_func_or 870 }; 871 872 static const struct x86_emul x86_emul_and = { 873 .readreg = true, 874 .func = x86_func_and 875 }; 876 877 static const struct x86_emul x86_emul_xchg = { 878 .readreg = true, 879 .backprop = true, 880 .func = x86_func_xchg 881 }; 882 883 static const struct x86_emul x86_emul_sub = { 884 .readreg = true, 885 .func = x86_func_sub 886 }; 887 888 static const struct x86_emul x86_emul_xor = { 889 .readreg = true, 890 .func = x86_func_xor 891 }; 892 893 static const struct x86_emul x86_emul_cmp = { 894 .notouch = true, 895 .func = x86_func_cmp 896 }; 897 898 static const struct x86_emul x86_emul_test = { 899 .notouch = true, 900 .func = x86_func_test 901 }; 902 903 static const struct x86_emul x86_emul_mov = { 904 .func = x86_func_mov 905 }; 906 907 static const struct x86_emul x86_emul_stos = { 908 .func = x86_func_stos 909 }; 910 911 static const struct x86_emul x86_emul_lods = { 912 .func = x86_func_lods 913 }; 914 915 static const struct x86_emul x86_emul_movs = { 916 .func = x86_func_movs 917 }; 918 919 /* Legacy prefixes. */ 920 #define LEG_LOCK 0xF0 921 #define LEG_REPN 0xF2 922 #define LEG_REP 0xF3 923 #define LEG_OVR_CS 0x2E 924 #define LEG_OVR_SS 0x36 925 #define LEG_OVR_DS 0x3E 926 #define LEG_OVR_ES 0x26 927 #define LEG_OVR_FS 0x64 928 #define LEG_OVR_GS 0x65 929 #define LEG_OPR_OVR 0x66 930 #define LEG_ADR_OVR 0x67 931 932 struct x86_legpref { 933 bool opr_ovr:1; 934 bool adr_ovr:1; 935 bool rep:1; 936 bool repn:1; 937 int8_t seg; 938 }; 939 940 struct x86_rexpref { 941 bool b:1; 942 bool x:1; 943 bool r:1; 944 bool w:1; 945 bool present:1; 946 }; 947 948 struct x86_reg { 949 int num; /* NVMM GPR state index */ 950 uint64_t mask; 951 }; 952 953 struct x86_dualreg { 954 int reg1; 955 int reg2; 956 }; 957 958 enum x86_disp_type { 959 DISP_NONE, 960 DISP_0, 961 DISP_1, 962 DISP_2, 963 DISP_4 964 }; 965 966 struct x86_disp { 967 enum x86_disp_type type; 968 uint64_t data; /* 4 bytes, but can be sign-extended */ 969 }; 970 971 struct x86_regmodrm { 972 uint8_t mod:2; 973 uint8_t reg:3; 974 uint8_t rm:3; 975 }; 976 977 struct x86_immediate { 978 uint64_t data; 979 }; 980 981 struct x86_sib { 982 uint8_t scale; 983 const struct x86_reg *idx; 984 const struct x86_reg *bas; 985 }; 986 987 enum x86_store_type { 988 STORE_NONE, 989 STORE_REG, 990 STORE_DUALREG, 991 STORE_IMM, 992 STORE_SIB, 993 STORE_DMO 994 }; 995 996 struct x86_store { 997 enum x86_store_type type; 998 union { 999 const struct x86_reg *reg; 1000 struct x86_dualreg dualreg; 1001 struct x86_immediate imm; 1002 struct x86_sib sib; 1003 uint64_t dmo; 1004 } u; 1005 struct x86_disp disp; 1006 int hardseg; 1007 }; 1008 1009 struct x86_instr { 1010 uint8_t len; 1011 struct x86_legpref legpref; 1012 struct x86_rexpref rexpref; 1013 struct x86_regmodrm regmodrm; 1014 uint8_t operand_size; 1015 uint8_t address_size; 1016 uint64_t zeroextend_mask; 1017 1018 const struct x86_opcode *opcode; 1019 const struct x86_emul *emul; 1020 1021 struct x86_store src; 1022 struct x86_store dst; 1023 struct x86_store *strm; 1024 }; 1025 1026 struct x86_decode_fsm { 1027 /* vcpu */ 1028 bool is64bit; 1029 bool is32bit; 1030 bool is16bit; 1031 1032 /* fsm */ 1033 int (*fn)(struct x86_decode_fsm *, struct x86_instr *); 1034 uint8_t *buf; 1035 uint8_t *end; 1036 }; 1037 1038 struct x86_opcode { 1039 bool valid:1; 1040 bool regmodrm:1; 1041 bool regtorm:1; 1042 bool dmo:1; 1043 bool todmo:1; 1044 bool movs:1; 1045 bool stos:1; 1046 bool lods:1; 1047 bool szoverride:1; 1048 bool group1:1; 1049 bool group3:1; 1050 bool group11:1; 1051 bool immediate:1; 1052 uint8_t defsize; 1053 uint8_t flags; 1054 const struct x86_emul *emul; 1055 }; 1056 1057 struct x86_group_entry { 1058 const struct x86_emul *emul; 1059 }; 1060 1061 #define OPSIZE_BYTE 0x01 1062 #define OPSIZE_WORD 0x02 /* 2 bytes */ 1063 #define OPSIZE_DOUB 0x04 /* 4 bytes */ 1064 #define OPSIZE_QUAD 0x08 /* 8 bytes */ 1065 1066 #define FLAG_imm8 0x01 1067 #define FLAG_immz 0x02 1068 #define FLAG_ze 0x04 1069 1070 static const struct x86_group_entry group1[8] __cacheline_aligned = { 1071 [1] = { .emul = &x86_emul_or }, 1072 [4] = { .emul = &x86_emul_and }, 1073 [6] = { .emul = &x86_emul_xor }, 1074 [7] = { .emul = &x86_emul_cmp } 1075 }; 1076 1077 static const struct x86_group_entry group3[8] __cacheline_aligned = { 1078 [0] = { .emul = &x86_emul_test }, 1079 [1] = { .emul = &x86_emul_test } 1080 }; 1081 1082 static const struct x86_group_entry group11[8] __cacheline_aligned = { 1083 [0] = { .emul = &x86_emul_mov } 1084 }; 1085 1086 static const struct x86_opcode primary_opcode_table[256] __cacheline_aligned = { 1087 /* 1088 * Group1 1089 */ 1090 [0x80] = { 1091 /* Eb, Ib */ 1092 .valid = true, 1093 .regmodrm = true, 1094 .regtorm = true, 1095 .szoverride = false, 1096 .defsize = OPSIZE_BYTE, 1097 .group1 = true, 1098 .immediate = true, 1099 .emul = NULL /* group1 */ 1100 }, 1101 [0x81] = { 1102 /* Ev, Iz */ 1103 .valid = true, 1104 .regmodrm = true, 1105 .regtorm = true, 1106 .szoverride = true, 1107 .defsize = -1, 1108 .group1 = true, 1109 .immediate = true, 1110 .flags = FLAG_immz, 1111 .emul = NULL /* group1 */ 1112 }, 1113 [0x83] = { 1114 /* Ev, Ib */ 1115 .valid = true, 1116 .regmodrm = true, 1117 .regtorm = true, 1118 .szoverride = true, 1119 .defsize = -1, 1120 .group1 = true, 1121 .immediate = true, 1122 .flags = FLAG_imm8, 1123 .emul = NULL /* group1 */ 1124 }, 1125 1126 /* 1127 * Group3 1128 */ 1129 [0xF6] = { 1130 /* Eb, Ib */ 1131 .valid = true, 1132 .regmodrm = true, 1133 .regtorm = true, 1134 .szoverride = false, 1135 .defsize = OPSIZE_BYTE, 1136 .group3 = true, 1137 .immediate = true, 1138 .emul = NULL /* group3 */ 1139 }, 1140 [0xF7] = { 1141 /* Ev, Iz */ 1142 .valid = true, 1143 .regmodrm = true, 1144 .regtorm = true, 1145 .szoverride = true, 1146 .defsize = -1, 1147 .group3 = true, 1148 .immediate = true, 1149 .flags = FLAG_immz, 1150 .emul = NULL /* group3 */ 1151 }, 1152 1153 /* 1154 * Group11 1155 */ 1156 [0xC6] = { 1157 /* Eb, Ib */ 1158 .valid = true, 1159 .regmodrm = true, 1160 .regtorm = true, 1161 .szoverride = false, 1162 .defsize = OPSIZE_BYTE, 1163 .group11 = true, 1164 .immediate = true, 1165 .emul = NULL /* group11 */ 1166 }, 1167 [0xC7] = { 1168 /* Ev, Iz */ 1169 .valid = true, 1170 .regmodrm = true, 1171 .regtorm = true, 1172 .szoverride = true, 1173 .defsize = -1, 1174 .group11 = true, 1175 .immediate = true, 1176 .flags = FLAG_immz, 1177 .emul = NULL /* group11 */ 1178 }, 1179 1180 /* 1181 * OR 1182 */ 1183 [0x08] = { 1184 /* Eb, Gb */ 1185 .valid = true, 1186 .regmodrm = true, 1187 .regtorm = true, 1188 .szoverride = false, 1189 .defsize = OPSIZE_BYTE, 1190 .emul = &x86_emul_or 1191 }, 1192 [0x09] = { 1193 /* Ev, Gv */ 1194 .valid = true, 1195 .regmodrm = true, 1196 .regtorm = true, 1197 .szoverride = true, 1198 .defsize = -1, 1199 .emul = &x86_emul_or 1200 }, 1201 [0x0A] = { 1202 /* Gb, Eb */ 1203 .valid = true, 1204 .regmodrm = true, 1205 .regtorm = false, 1206 .szoverride = false, 1207 .defsize = OPSIZE_BYTE, 1208 .emul = &x86_emul_or 1209 }, 1210 [0x0B] = { 1211 /* Gv, Ev */ 1212 .valid = true, 1213 .regmodrm = true, 1214 .regtorm = false, 1215 .szoverride = true, 1216 .defsize = -1, 1217 .emul = &x86_emul_or 1218 }, 1219 1220 /* 1221 * AND 1222 */ 1223 [0x20] = { 1224 /* Eb, Gb */ 1225 .valid = true, 1226 .regmodrm = true, 1227 .regtorm = true, 1228 .szoverride = false, 1229 .defsize = OPSIZE_BYTE, 1230 .emul = &x86_emul_and 1231 }, 1232 [0x21] = { 1233 /* Ev, Gv */ 1234 .valid = true, 1235 .regmodrm = true, 1236 .regtorm = true, 1237 .szoverride = true, 1238 .defsize = -1, 1239 .emul = &x86_emul_and 1240 }, 1241 [0x22] = { 1242 /* Gb, Eb */ 1243 .valid = true, 1244 .regmodrm = true, 1245 .regtorm = false, 1246 .szoverride = false, 1247 .defsize = OPSIZE_BYTE, 1248 .emul = &x86_emul_and 1249 }, 1250 [0x23] = { 1251 /* Gv, Ev */ 1252 .valid = true, 1253 .regmodrm = true, 1254 .regtorm = false, 1255 .szoverride = true, 1256 .defsize = -1, 1257 .emul = &x86_emul_and 1258 }, 1259 1260 /* 1261 * SUB 1262 */ 1263 [0x28] = { 1264 /* Eb, Gb */ 1265 .valid = true, 1266 .regmodrm = true, 1267 .regtorm = true, 1268 .szoverride = false, 1269 .defsize = OPSIZE_BYTE, 1270 .emul = &x86_emul_sub 1271 }, 1272 [0x29] = { 1273 /* Ev, Gv */ 1274 .valid = true, 1275 .regmodrm = true, 1276 .regtorm = true, 1277 .szoverride = true, 1278 .defsize = -1, 1279 .emul = &x86_emul_sub 1280 }, 1281 [0x2A] = { 1282 /* Gb, Eb */ 1283 .valid = true, 1284 .regmodrm = true, 1285 .regtorm = false, 1286 .szoverride = false, 1287 .defsize = OPSIZE_BYTE, 1288 .emul = &x86_emul_sub 1289 }, 1290 [0x2B] = { 1291 /* Gv, Ev */ 1292 .valid = true, 1293 .regmodrm = true, 1294 .regtorm = false, 1295 .szoverride = true, 1296 .defsize = -1, 1297 .emul = &x86_emul_sub 1298 }, 1299 1300 /* 1301 * XOR 1302 */ 1303 [0x30] = { 1304 /* Eb, Gb */ 1305 .valid = true, 1306 .regmodrm = true, 1307 .regtorm = true, 1308 .szoverride = false, 1309 .defsize = OPSIZE_BYTE, 1310 .emul = &x86_emul_xor 1311 }, 1312 [0x31] = { 1313 /* Ev, Gv */ 1314 .valid = true, 1315 .regmodrm = true, 1316 .regtorm = true, 1317 .szoverride = true, 1318 .defsize = -1, 1319 .emul = &x86_emul_xor 1320 }, 1321 [0x32] = { 1322 /* Gb, Eb */ 1323 .valid = true, 1324 .regmodrm = true, 1325 .regtorm = false, 1326 .szoverride = false, 1327 .defsize = OPSIZE_BYTE, 1328 .emul = &x86_emul_xor 1329 }, 1330 [0x33] = { 1331 /* Gv, Ev */ 1332 .valid = true, 1333 .regmodrm = true, 1334 .regtorm = false, 1335 .szoverride = true, 1336 .defsize = -1, 1337 .emul = &x86_emul_xor 1338 }, 1339 1340 /* 1341 * XCHG 1342 */ 1343 [0x86] = { 1344 /* Eb, Gb */ 1345 .valid = true, 1346 .regmodrm = true, 1347 .regtorm = true, 1348 .szoverride = false, 1349 .defsize = OPSIZE_BYTE, 1350 .emul = &x86_emul_xchg 1351 }, 1352 [0x87] = { 1353 /* Ev, Gv */ 1354 .valid = true, 1355 .regmodrm = true, 1356 .regtorm = true, 1357 .szoverride = true, 1358 .defsize = -1, 1359 .emul = &x86_emul_xchg 1360 }, 1361 1362 /* 1363 * MOV 1364 */ 1365 [0x88] = { 1366 /* Eb, Gb */ 1367 .valid = true, 1368 .regmodrm = true, 1369 .regtorm = true, 1370 .szoverride = false, 1371 .defsize = OPSIZE_BYTE, 1372 .emul = &x86_emul_mov 1373 }, 1374 [0x89] = { 1375 /* Ev, Gv */ 1376 .valid = true, 1377 .regmodrm = true, 1378 .regtorm = true, 1379 .szoverride = true, 1380 .defsize = -1, 1381 .emul = &x86_emul_mov 1382 }, 1383 [0x8A] = { 1384 /* Gb, Eb */ 1385 .valid = true, 1386 .regmodrm = true, 1387 .regtorm = false, 1388 .szoverride = false, 1389 .defsize = OPSIZE_BYTE, 1390 .emul = &x86_emul_mov 1391 }, 1392 [0x8B] = { 1393 /* Gv, Ev */ 1394 .valid = true, 1395 .regmodrm = true, 1396 .regtorm = false, 1397 .szoverride = true, 1398 .defsize = -1, 1399 .emul = &x86_emul_mov 1400 }, 1401 [0xA0] = { 1402 /* AL, Ob */ 1403 .valid = true, 1404 .dmo = true, 1405 .todmo = false, 1406 .szoverride = false, 1407 .defsize = OPSIZE_BYTE, 1408 .emul = &x86_emul_mov 1409 }, 1410 [0xA1] = { 1411 /* rAX, Ov */ 1412 .valid = true, 1413 .dmo = true, 1414 .todmo = false, 1415 .szoverride = true, 1416 .defsize = -1, 1417 .emul = &x86_emul_mov 1418 }, 1419 [0xA2] = { 1420 /* Ob, AL */ 1421 .valid = true, 1422 .dmo = true, 1423 .todmo = true, 1424 .szoverride = false, 1425 .defsize = OPSIZE_BYTE, 1426 .emul = &x86_emul_mov 1427 }, 1428 [0xA3] = { 1429 /* Ov, rAX */ 1430 .valid = true, 1431 .dmo = true, 1432 .todmo = true, 1433 .szoverride = true, 1434 .defsize = -1, 1435 .emul = &x86_emul_mov 1436 }, 1437 1438 /* 1439 * MOVS 1440 */ 1441 [0xA4] = { 1442 /* Yb, Xb */ 1443 .valid = true, 1444 .movs = true, 1445 .szoverride = false, 1446 .defsize = OPSIZE_BYTE, 1447 .emul = &x86_emul_movs 1448 }, 1449 [0xA5] = { 1450 /* Yv, Xv */ 1451 .valid = true, 1452 .movs = true, 1453 .szoverride = true, 1454 .defsize = -1, 1455 .emul = &x86_emul_movs 1456 }, 1457 1458 /* 1459 * STOS 1460 */ 1461 [0xAA] = { 1462 /* Yb, AL */ 1463 .valid = true, 1464 .stos = true, 1465 .szoverride = false, 1466 .defsize = OPSIZE_BYTE, 1467 .emul = &x86_emul_stos 1468 }, 1469 [0xAB] = { 1470 /* Yv, rAX */ 1471 .valid = true, 1472 .stos = true, 1473 .szoverride = true, 1474 .defsize = -1, 1475 .emul = &x86_emul_stos 1476 }, 1477 1478 /* 1479 * LODS 1480 */ 1481 [0xAC] = { 1482 /* AL, Xb */ 1483 .valid = true, 1484 .lods = true, 1485 .szoverride = false, 1486 .defsize = OPSIZE_BYTE, 1487 .emul = &x86_emul_lods 1488 }, 1489 [0xAD] = { 1490 /* rAX, Xv */ 1491 .valid = true, 1492 .lods = true, 1493 .szoverride = true, 1494 .defsize = -1, 1495 .emul = &x86_emul_lods 1496 }, 1497 }; 1498 1499 static const struct x86_opcode secondary_opcode_table[256] __cacheline_aligned = { 1500 /* 1501 * MOVZX 1502 */ 1503 [0xB6] = { 1504 /* Gv, Eb */ 1505 .valid = true, 1506 .regmodrm = true, 1507 .regtorm = false, 1508 .szoverride = true, 1509 .defsize = OPSIZE_BYTE, 1510 .flags = FLAG_ze, 1511 .emul = &x86_emul_mov 1512 }, 1513 [0xB7] = { 1514 /* Gv, Ew */ 1515 .valid = true, 1516 .regmodrm = true, 1517 .regtorm = false, 1518 .szoverride = true, 1519 .defsize = OPSIZE_WORD, 1520 .flags = FLAG_ze, 1521 .emul = &x86_emul_mov 1522 }, 1523 }; 1524 1525 static const struct x86_reg gpr_map__rip = { NVMM_X64_GPR_RIP, 0xFFFFFFFFFFFFFFFF }; 1526 1527 /* [REX-present][enc][opsize] */ 1528 static const struct x86_reg gpr_map__special[2][4][8] __cacheline_aligned = { 1529 [false] = { 1530 /* No REX prefix. */ 1531 [0b00] = { 1532 [0] = { NVMM_X64_GPR_RAX, 0x000000000000FF00 }, /* AH */ 1533 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1534 [2] = { -1, 0 }, 1535 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1536 [4] = { -1, 0 }, 1537 [5] = { -1, 0 }, 1538 [6] = { -1, 0 }, 1539 [7] = { -1, 0 }, 1540 }, 1541 [0b01] = { 1542 [0] = { NVMM_X64_GPR_RCX, 0x000000000000FF00 }, /* CH */ 1543 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1544 [2] = { -1, 0 }, 1545 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1546 [4] = { -1, 0 }, 1547 [5] = { -1, 0 }, 1548 [6] = { -1, 0 }, 1549 [7] = { -1, 0 }, 1550 }, 1551 [0b10] = { 1552 [0] = { NVMM_X64_GPR_RDX, 0x000000000000FF00 }, /* DH */ 1553 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1554 [2] = { -1, 0 }, 1555 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1556 [4] = { -1, 0 }, 1557 [5] = { -1, 0 }, 1558 [6] = { -1, 0 }, 1559 [7] = { -1, 0 }, 1560 }, 1561 [0b11] = { 1562 [0] = { NVMM_X64_GPR_RBX, 0x000000000000FF00 }, /* BH */ 1563 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1564 [2] = { -1, 0 }, 1565 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1566 [4] = { -1, 0 }, 1567 [5] = { -1, 0 }, 1568 [6] = { -1, 0 }, 1569 [7] = { -1, 0 }, 1570 } 1571 }, 1572 [true] = { 1573 /* Has REX prefix. */ 1574 [0b00] = { 1575 [0] = { NVMM_X64_GPR_RSP, 0x00000000000000FF }, /* SPL */ 1576 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1577 [2] = { -1, 0 }, 1578 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1579 [4] = { -1, 0 }, 1580 [5] = { -1, 0 }, 1581 [6] = { -1, 0 }, 1582 [7] = { NVMM_X64_GPR_RSP, 0xFFFFFFFFFFFFFFFF }, /* RSP */ 1583 }, 1584 [0b01] = { 1585 [0] = { NVMM_X64_GPR_RBP, 0x00000000000000FF }, /* BPL */ 1586 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1587 [2] = { -1, 0 }, 1588 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1589 [4] = { -1, 0 }, 1590 [5] = { -1, 0 }, 1591 [6] = { -1, 0 }, 1592 [7] = { NVMM_X64_GPR_RBP, 0xFFFFFFFFFFFFFFFF }, /* RBP */ 1593 }, 1594 [0b10] = { 1595 [0] = { NVMM_X64_GPR_RSI, 0x00000000000000FF }, /* SIL */ 1596 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1597 [2] = { -1, 0 }, 1598 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1599 [4] = { -1, 0 }, 1600 [5] = { -1, 0 }, 1601 [6] = { -1, 0 }, 1602 [7] = { NVMM_X64_GPR_RSI, 0xFFFFFFFFFFFFFFFF }, /* RSI */ 1603 }, 1604 [0b11] = { 1605 [0] = { NVMM_X64_GPR_RDI, 0x00000000000000FF }, /* DIL */ 1606 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1607 [2] = { -1, 0 }, 1608 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1609 [4] = { -1, 0 }, 1610 [5] = { -1, 0 }, 1611 [6] = { -1, 0 }, 1612 [7] = { NVMM_X64_GPR_RDI, 0xFFFFFFFFFFFFFFFF }, /* RDI */ 1613 } 1614 } 1615 }; 1616 1617 /* [depends][enc][size] */ 1618 static const struct x86_reg gpr_map[2][8][8] __cacheline_aligned = { 1619 [false] = { 1620 /* Not extended. */ 1621 [0b000] = { 1622 [0] = { NVMM_X64_GPR_RAX, 0x00000000000000FF }, /* AL */ 1623 [1] = { NVMM_X64_GPR_RAX, 0x000000000000FFFF }, /* AX */ 1624 [2] = { -1, 0 }, 1625 [3] = { NVMM_X64_GPR_RAX, 0x00000000FFFFFFFF }, /* EAX */ 1626 [4] = { -1, 0 }, 1627 [5] = { -1, 0 }, 1628 [6] = { -1, 0 }, 1629 [7] = { NVMM_X64_GPR_RAX, 0xFFFFFFFFFFFFFFFF }, /* RAX */ 1630 }, 1631 [0b001] = { 1632 [0] = { NVMM_X64_GPR_RCX, 0x00000000000000FF }, /* CL */ 1633 [1] = { NVMM_X64_GPR_RCX, 0x000000000000FFFF }, /* CX */ 1634 [2] = { -1, 0 }, 1635 [3] = { NVMM_X64_GPR_RCX, 0x00000000FFFFFFFF }, /* ECX */ 1636 [4] = { -1, 0 }, 1637 [5] = { -1, 0 }, 1638 [6] = { -1, 0 }, 1639 [7] = { NVMM_X64_GPR_RCX, 0xFFFFFFFFFFFFFFFF }, /* RCX */ 1640 }, 1641 [0b010] = { 1642 [0] = { NVMM_X64_GPR_RDX, 0x00000000000000FF }, /* DL */ 1643 [1] = { NVMM_X64_GPR_RDX, 0x000000000000FFFF }, /* DX */ 1644 [2] = { -1, 0 }, 1645 [3] = { NVMM_X64_GPR_RDX, 0x00000000FFFFFFFF }, /* EDX */ 1646 [4] = { -1, 0 }, 1647 [5] = { -1, 0 }, 1648 [6] = { -1, 0 }, 1649 [7] = { NVMM_X64_GPR_RDX, 0xFFFFFFFFFFFFFFFF }, /* RDX */ 1650 }, 1651 [0b011] = { 1652 [0] = { NVMM_X64_GPR_RBX, 0x00000000000000FF }, /* BL */ 1653 [1] = { NVMM_X64_GPR_RBX, 0x000000000000FFFF }, /* BX */ 1654 [2] = { -1, 0 }, 1655 [3] = { NVMM_X64_GPR_RBX, 0x00000000FFFFFFFF }, /* EBX */ 1656 [4] = { -1, 0 }, 1657 [5] = { -1, 0 }, 1658 [6] = { -1, 0 }, 1659 [7] = { NVMM_X64_GPR_RBX, 0xFFFFFFFFFFFFFFFF }, /* RBX */ 1660 }, 1661 [0b100] = { 1662 [0] = { -1, 0 }, /* SPECIAL */ 1663 [1] = { -1, 0 }, /* SPECIAL */ 1664 [2] = { -1, 0 }, 1665 [3] = { -1, 0 }, /* SPECIAL */ 1666 [4] = { -1, 0 }, 1667 [5] = { -1, 0 }, 1668 [6] = { -1, 0 }, 1669 [7] = { -1, 0 }, /* SPECIAL */ 1670 }, 1671 [0b101] = { 1672 [0] = { -1, 0 }, /* SPECIAL */ 1673 [1] = { -1, 0 }, /* SPECIAL */ 1674 [2] = { -1, 0 }, 1675 [3] = { -1, 0 }, /* SPECIAL */ 1676 [4] = { -1, 0 }, 1677 [5] = { -1, 0 }, 1678 [6] = { -1, 0 }, 1679 [7] = { -1, 0 }, /* SPECIAL */ 1680 }, 1681 [0b110] = { 1682 [0] = { -1, 0 }, /* SPECIAL */ 1683 [1] = { -1, 0 }, /* SPECIAL */ 1684 [2] = { -1, 0 }, 1685 [3] = { -1, 0 }, /* SPECIAL */ 1686 [4] = { -1, 0 }, 1687 [5] = { -1, 0 }, 1688 [6] = { -1, 0 }, 1689 [7] = { -1, 0 }, /* SPECIAL */ 1690 }, 1691 [0b111] = { 1692 [0] = { -1, 0 }, /* SPECIAL */ 1693 [1] = { -1, 0 }, /* SPECIAL */ 1694 [2] = { -1, 0 }, 1695 [3] = { -1, 0 }, /* SPECIAL */ 1696 [4] = { -1, 0 }, 1697 [5] = { -1, 0 }, 1698 [6] = { -1, 0 }, 1699 [7] = { -1, 0 }, /* SPECIAL */ 1700 }, 1701 }, 1702 [true] = { 1703 /* Extended. */ 1704 [0b000] = { 1705 [0] = { NVMM_X64_GPR_R8, 0x00000000000000FF }, /* R8B */ 1706 [1] = { NVMM_X64_GPR_R8, 0x000000000000FFFF }, /* R8W */ 1707 [2] = { -1, 0 }, 1708 [3] = { NVMM_X64_GPR_R8, 0x00000000FFFFFFFF }, /* R8D */ 1709 [4] = { -1, 0 }, 1710 [5] = { -1, 0 }, 1711 [6] = { -1, 0 }, 1712 [7] = { NVMM_X64_GPR_R8, 0xFFFFFFFFFFFFFFFF }, /* R8 */ 1713 }, 1714 [0b001] = { 1715 [0] = { NVMM_X64_GPR_R9, 0x00000000000000FF }, /* R9B */ 1716 [1] = { NVMM_X64_GPR_R9, 0x000000000000FFFF }, /* R9W */ 1717 [2] = { -1, 0 }, 1718 [3] = { NVMM_X64_GPR_R9, 0x00000000FFFFFFFF }, /* R9D */ 1719 [4] = { -1, 0 }, 1720 [5] = { -1, 0 }, 1721 [6] = { -1, 0 }, 1722 [7] = { NVMM_X64_GPR_R9, 0xFFFFFFFFFFFFFFFF }, /* R9 */ 1723 }, 1724 [0b010] = { 1725 [0] = { NVMM_X64_GPR_R10, 0x00000000000000FF }, /* R10B */ 1726 [1] = { NVMM_X64_GPR_R10, 0x000000000000FFFF }, /* R10W */ 1727 [2] = { -1, 0 }, 1728 [3] = { NVMM_X64_GPR_R10, 0x00000000FFFFFFFF }, /* R10D */ 1729 [4] = { -1, 0 }, 1730 [5] = { -1, 0 }, 1731 [6] = { -1, 0 }, 1732 [7] = { NVMM_X64_GPR_R10, 0xFFFFFFFFFFFFFFFF }, /* R10 */ 1733 }, 1734 [0b011] = { 1735 [0] = { NVMM_X64_GPR_R11, 0x00000000000000FF }, /* R11B */ 1736 [1] = { NVMM_X64_GPR_R11, 0x000000000000FFFF }, /* R11W */ 1737 [2] = { -1, 0 }, 1738 [3] = { NVMM_X64_GPR_R11, 0x00000000FFFFFFFF }, /* R11D */ 1739 [4] = { -1, 0 }, 1740 [5] = { -1, 0 }, 1741 [6] = { -1, 0 }, 1742 [7] = { NVMM_X64_GPR_R11, 0xFFFFFFFFFFFFFFFF }, /* R11 */ 1743 }, 1744 [0b100] = { 1745 [0] = { NVMM_X64_GPR_R12, 0x00000000000000FF }, /* R12B */ 1746 [1] = { NVMM_X64_GPR_R12, 0x000000000000FFFF }, /* R12W */ 1747 [2] = { -1, 0 }, 1748 [3] = { NVMM_X64_GPR_R12, 0x00000000FFFFFFFF }, /* R12D */ 1749 [4] = { -1, 0 }, 1750 [5] = { -1, 0 }, 1751 [6] = { -1, 0 }, 1752 [7] = { NVMM_X64_GPR_R12, 0xFFFFFFFFFFFFFFFF }, /* R12 */ 1753 }, 1754 [0b101] = { 1755 [0] = { NVMM_X64_GPR_R13, 0x00000000000000FF }, /* R13B */ 1756 [1] = { NVMM_X64_GPR_R13, 0x000000000000FFFF }, /* R13W */ 1757 [2] = { -1, 0 }, 1758 [3] = { NVMM_X64_GPR_R13, 0x00000000FFFFFFFF }, /* R13D */ 1759 [4] = { -1, 0 }, 1760 [5] = { -1, 0 }, 1761 [6] = { -1, 0 }, 1762 [7] = { NVMM_X64_GPR_R13, 0xFFFFFFFFFFFFFFFF }, /* R13 */ 1763 }, 1764 [0b110] = { 1765 [0] = { NVMM_X64_GPR_R14, 0x00000000000000FF }, /* R14B */ 1766 [1] = { NVMM_X64_GPR_R14, 0x000000000000FFFF }, /* R14W */ 1767 [2] = { -1, 0 }, 1768 [3] = { NVMM_X64_GPR_R14, 0x00000000FFFFFFFF }, /* R14D */ 1769 [4] = { -1, 0 }, 1770 [5] = { -1, 0 }, 1771 [6] = { -1, 0 }, 1772 [7] = { NVMM_X64_GPR_R14, 0xFFFFFFFFFFFFFFFF }, /* R14 */ 1773 }, 1774 [0b111] = { 1775 [0] = { NVMM_X64_GPR_R15, 0x00000000000000FF }, /* R15B */ 1776 [1] = { NVMM_X64_GPR_R15, 0x000000000000FFFF }, /* R15W */ 1777 [2] = { -1, 0 }, 1778 [3] = { NVMM_X64_GPR_R15, 0x00000000FFFFFFFF }, /* R15D */ 1779 [4] = { -1, 0 }, 1780 [5] = { -1, 0 }, 1781 [6] = { -1, 0 }, 1782 [7] = { NVMM_X64_GPR_R15, 0xFFFFFFFFFFFFFFFF }, /* R15 */ 1783 }, 1784 } 1785 }; 1786 1787 /* [enc] */ 1788 static const int gpr_dual_reg1_rm[8] __cacheline_aligned = { 1789 [0b000] = NVMM_X64_GPR_RBX, /* BX (+SI) */ 1790 [0b001] = NVMM_X64_GPR_RBX, /* BX (+DI) */ 1791 [0b010] = NVMM_X64_GPR_RBP, /* BP (+SI) */ 1792 [0b011] = NVMM_X64_GPR_RBP, /* BP (+DI) */ 1793 [0b100] = NVMM_X64_GPR_RSI, /* SI */ 1794 [0b101] = NVMM_X64_GPR_RDI, /* DI */ 1795 [0b110] = NVMM_X64_GPR_RBP, /* BP */ 1796 [0b111] = NVMM_X64_GPR_RBX, /* BX */ 1797 }; 1798 1799 static int 1800 node_overflow(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1801 { 1802 fsm->fn = NULL; 1803 return -1; 1804 } 1805 1806 static int 1807 fsm_read(struct x86_decode_fsm *fsm, uint8_t *bytes, size_t n) 1808 { 1809 if (fsm->buf + n > fsm->end) { 1810 return -1; 1811 } 1812 memcpy(bytes, fsm->buf, n); 1813 return 0; 1814 } 1815 1816 static inline void 1817 fsm_advance(struct x86_decode_fsm *fsm, size_t n, 1818 int (*fn)(struct x86_decode_fsm *, struct x86_instr *)) 1819 { 1820 fsm->buf += n; 1821 if (fsm->buf > fsm->end) { 1822 fsm->fn = node_overflow; 1823 } else { 1824 fsm->fn = fn; 1825 } 1826 } 1827 1828 static const struct x86_reg * 1829 resolve_special_register(struct x86_instr *instr, uint8_t enc, size_t regsize) 1830 { 1831 enc &= 0b11; 1832 if (regsize == 8) { 1833 /* May be 64bit without REX */ 1834 return &gpr_map__special[1][enc][regsize-1]; 1835 } 1836 return &gpr_map__special[instr->rexpref.present][enc][regsize-1]; 1837 } 1838 1839 /* 1840 * Special node, for MOVS. Fake two displacements of zero on the source and 1841 * destination registers. 1842 */ 1843 static int 1844 node_movs(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1845 { 1846 size_t adrsize; 1847 1848 adrsize = instr->address_size; 1849 1850 /* DS:RSI */ 1851 instr->src.type = STORE_REG; 1852 instr->src.u.reg = &gpr_map__special[1][2][adrsize-1]; 1853 instr->src.disp.type = DISP_0; 1854 1855 /* ES:RDI, force ES */ 1856 instr->dst.type = STORE_REG; 1857 instr->dst.u.reg = &gpr_map__special[1][3][adrsize-1]; 1858 instr->dst.disp.type = DISP_0; 1859 instr->dst.hardseg = NVMM_X64_SEG_ES; 1860 1861 fsm_advance(fsm, 0, NULL); 1862 1863 return 0; 1864 } 1865 1866 /* 1867 * Special node, for STOS and LODS. Fake a displacement of zero on the 1868 * destination register. 1869 */ 1870 static int 1871 node_stlo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1872 { 1873 const struct x86_opcode *opcode = instr->opcode; 1874 struct x86_store *stlo, *streg; 1875 size_t adrsize, regsize; 1876 1877 adrsize = instr->address_size; 1878 regsize = instr->operand_size; 1879 1880 if (opcode->stos) { 1881 streg = &instr->src; 1882 stlo = &instr->dst; 1883 } else { 1884 streg = &instr->dst; 1885 stlo = &instr->src; 1886 } 1887 1888 streg->type = STORE_REG; 1889 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1890 1891 stlo->type = STORE_REG; 1892 if (opcode->stos) { 1893 /* ES:RDI, force ES */ 1894 stlo->u.reg = &gpr_map__special[1][3][adrsize-1]; 1895 stlo->hardseg = NVMM_X64_SEG_ES; 1896 } else { 1897 /* DS:RSI */ 1898 stlo->u.reg = &gpr_map__special[1][2][adrsize-1]; 1899 } 1900 stlo->disp.type = DISP_0; 1901 1902 fsm_advance(fsm, 0, NULL); 1903 1904 return 0; 1905 } 1906 1907 static int 1908 node_dmo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1909 { 1910 const struct x86_opcode *opcode = instr->opcode; 1911 struct x86_store *stdmo, *streg; 1912 size_t adrsize, regsize; 1913 1914 adrsize = instr->address_size; 1915 regsize = instr->operand_size; 1916 1917 if (opcode->todmo) { 1918 streg = &instr->src; 1919 stdmo = &instr->dst; 1920 } else { 1921 streg = &instr->dst; 1922 stdmo = &instr->src; 1923 } 1924 1925 streg->type = STORE_REG; 1926 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1927 1928 stdmo->type = STORE_DMO; 1929 if (fsm_read(fsm, (uint8_t *)&stdmo->u.dmo, adrsize) == -1) { 1930 return -1; 1931 } 1932 fsm_advance(fsm, adrsize, NULL); 1933 1934 return 0; 1935 } 1936 1937 static inline uint64_t 1938 sign_extend(uint64_t val, int size) 1939 { 1940 if (size == 1) { 1941 if (val & __BIT(7)) 1942 val |= 0xFFFFFFFFFFFFFF00; 1943 } else if (size == 2) { 1944 if (val & __BIT(15)) 1945 val |= 0xFFFFFFFFFFFF0000; 1946 } else if (size == 4) { 1947 if (val & __BIT(31)) 1948 val |= 0xFFFFFFFF00000000; 1949 } 1950 return val; 1951 } 1952 1953 static int 1954 node_immediate(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1955 { 1956 const struct x86_opcode *opcode = instr->opcode; 1957 struct x86_store *store; 1958 uint8_t immsize; 1959 size_t sesize = 0; 1960 1961 /* The immediate is the source */ 1962 store = &instr->src; 1963 immsize = instr->operand_size; 1964 1965 if (opcode->flags & FLAG_imm8) { 1966 sesize = immsize; 1967 immsize = 1; 1968 } else if ((opcode->flags & FLAG_immz) && (immsize == 8)) { 1969 sesize = immsize; 1970 immsize = 4; 1971 } 1972 1973 store->type = STORE_IMM; 1974 if (fsm_read(fsm, (uint8_t *)&store->u.imm.data, immsize) == -1) { 1975 return -1; 1976 } 1977 fsm_advance(fsm, immsize, NULL); 1978 1979 if (sesize != 0) { 1980 store->u.imm.data = sign_extend(store->u.imm.data, sesize); 1981 } 1982 1983 return 0; 1984 } 1985 1986 static int 1987 node_disp(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1988 { 1989 const struct x86_opcode *opcode = instr->opcode; 1990 uint64_t data = 0; 1991 size_t n; 1992 1993 if (instr->strm->disp.type == DISP_1) { 1994 n = 1; 1995 } else if (instr->strm->disp.type == DISP_2) { 1996 n = 2; 1997 } else if (instr->strm->disp.type == DISP_4) { 1998 n = 4; 1999 } else { 2000 DISASSEMBLER_BUG(); 2001 } 2002 2003 if (fsm_read(fsm, (uint8_t *)&data, n) == -1) { 2004 return -1; 2005 } 2006 2007 if (__predict_true(fsm->is64bit)) { 2008 data = sign_extend(data, n); 2009 } 2010 2011 instr->strm->disp.data = data; 2012 2013 if (opcode->immediate) { 2014 fsm_advance(fsm, n, node_immediate); 2015 } else { 2016 fsm_advance(fsm, n, NULL); 2017 } 2018 2019 return 0; 2020 } 2021 2022 /* 2023 * Special node to handle 16bit addressing encoding, which can reference two 2024 * registers at once. 2025 */ 2026 static int 2027 node_dual(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2028 { 2029 int reg1, reg2; 2030 2031 reg1 = gpr_dual_reg1_rm[instr->regmodrm.rm]; 2032 2033 if (instr->regmodrm.rm == 0b000 || 2034 instr->regmodrm.rm == 0b010) { 2035 reg2 = NVMM_X64_GPR_RSI; 2036 } else if (instr->regmodrm.rm == 0b001 || 2037 instr->regmodrm.rm == 0b011) { 2038 reg2 = NVMM_X64_GPR_RDI; 2039 } else { 2040 DISASSEMBLER_BUG(); 2041 } 2042 2043 instr->strm->type = STORE_DUALREG; 2044 instr->strm->u.dualreg.reg1 = reg1; 2045 instr->strm->u.dualreg.reg2 = reg2; 2046 2047 if (instr->strm->disp.type == DISP_NONE) { 2048 DISASSEMBLER_BUG(); 2049 } else if (instr->strm->disp.type == DISP_0) { 2050 /* Indirect register addressing mode */ 2051 if (instr->opcode->immediate) { 2052 fsm_advance(fsm, 1, node_immediate); 2053 } else { 2054 fsm_advance(fsm, 1, NULL); 2055 } 2056 } else { 2057 fsm_advance(fsm, 1, node_disp); 2058 } 2059 2060 return 0; 2061 } 2062 2063 static const struct x86_reg * 2064 get_register_idx(struct x86_instr *instr, uint8_t index) 2065 { 2066 uint8_t enc = index; 2067 const struct x86_reg *reg; 2068 size_t regsize; 2069 2070 regsize = instr->address_size; 2071 reg = &gpr_map[instr->rexpref.x][enc][regsize-1]; 2072 2073 if (reg->num == -1) { 2074 reg = resolve_special_register(instr, enc, regsize); 2075 } 2076 2077 return reg; 2078 } 2079 2080 static const struct x86_reg * 2081 get_register_bas(struct x86_instr *instr, uint8_t base) 2082 { 2083 uint8_t enc = base; 2084 const struct x86_reg *reg; 2085 size_t regsize; 2086 2087 regsize = instr->address_size; 2088 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2089 if (reg->num == -1) { 2090 reg = resolve_special_register(instr, enc, regsize); 2091 } 2092 2093 return reg; 2094 } 2095 2096 static int 2097 node_sib(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2098 { 2099 const struct x86_opcode *opcode; 2100 uint8_t scale, index, base; 2101 bool noindex, nobase; 2102 uint8_t byte; 2103 2104 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2105 return -1; 2106 } 2107 2108 scale = ((byte & 0b11000000) >> 6); 2109 index = ((byte & 0b00111000) >> 3); 2110 base = ((byte & 0b00000111) >> 0); 2111 2112 opcode = instr->opcode; 2113 2114 noindex = false; 2115 nobase = false; 2116 2117 if (index == 0b100 && !instr->rexpref.x) { 2118 /* Special case: the index is null */ 2119 noindex = true; 2120 } 2121 2122 if (instr->regmodrm.mod == 0b00 && base == 0b101) { 2123 /* Special case: the base is null + disp32 */ 2124 instr->strm->disp.type = DISP_4; 2125 nobase = true; 2126 } 2127 2128 instr->strm->type = STORE_SIB; 2129 instr->strm->u.sib.scale = (1 << scale); 2130 if (!noindex) 2131 instr->strm->u.sib.idx = get_register_idx(instr, index); 2132 if (!nobase) 2133 instr->strm->u.sib.bas = get_register_bas(instr, base); 2134 2135 /* May have a displacement, or an immediate */ 2136 if (instr->strm->disp.type == DISP_1 || 2137 instr->strm->disp.type == DISP_2 || 2138 instr->strm->disp.type == DISP_4) { 2139 fsm_advance(fsm, 1, node_disp); 2140 } else if (opcode->immediate) { 2141 fsm_advance(fsm, 1, node_immediate); 2142 } else { 2143 fsm_advance(fsm, 1, NULL); 2144 } 2145 2146 return 0; 2147 } 2148 2149 static const struct x86_reg * 2150 get_register_reg(struct x86_instr *instr, const struct x86_opcode *opcode) 2151 { 2152 uint8_t enc = instr->regmodrm.reg; 2153 const struct x86_reg *reg; 2154 size_t regsize; 2155 2156 regsize = instr->operand_size; 2157 2158 reg = &gpr_map[instr->rexpref.r][enc][regsize-1]; 2159 if (reg->num == -1) { 2160 reg = resolve_special_register(instr, enc, regsize); 2161 } 2162 2163 return reg; 2164 } 2165 2166 static const struct x86_reg * 2167 get_register_rm(struct x86_instr *instr, const struct x86_opcode *opcode) 2168 { 2169 uint8_t enc = instr->regmodrm.rm; 2170 const struct x86_reg *reg; 2171 size_t regsize; 2172 2173 if (instr->strm->disp.type == DISP_NONE) { 2174 regsize = instr->operand_size; 2175 } else { 2176 /* Indirect access, the size is that of the address. */ 2177 regsize = instr->address_size; 2178 } 2179 2180 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2181 if (reg->num == -1) { 2182 reg = resolve_special_register(instr, enc, regsize); 2183 } 2184 2185 return reg; 2186 } 2187 2188 static inline bool 2189 has_sib(struct x86_instr *instr) 2190 { 2191 return (instr->address_size != 2 && /* no SIB in 16bit addressing */ 2192 instr->regmodrm.mod != 0b11 && 2193 instr->regmodrm.rm == 0b100); 2194 } 2195 2196 static inline bool 2197 is_rip_relative(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2198 { 2199 return (fsm->is64bit && /* RIP-relative only in 64bit mode */ 2200 instr->regmodrm.mod == 0b00 && 2201 instr->regmodrm.rm == 0b101); 2202 } 2203 2204 static inline bool 2205 is_disp32_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2206 { 2207 return (!fsm->is64bit && /* no disp32-only in 64bit mode */ 2208 instr->address_size != 2 && /* no disp32-only in 16bit addressing */ 2209 instr->regmodrm.mod == 0b00 && 2210 instr->regmodrm.rm == 0b101); 2211 } 2212 2213 static inline bool 2214 is_disp16_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2215 { 2216 return (instr->address_size == 2 && /* disp16-only only in 16bit addr */ 2217 instr->regmodrm.mod == 0b00 && 2218 instr->regmodrm.rm == 0b110); 2219 } 2220 2221 static inline bool 2222 is_dual(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2223 { 2224 return (instr->address_size == 2 && 2225 instr->regmodrm.mod != 0b11 && 2226 instr->regmodrm.rm <= 0b011); 2227 } 2228 2229 static enum x86_disp_type 2230 get_disp_type(struct x86_instr *instr) 2231 { 2232 switch (instr->regmodrm.mod) { 2233 case 0b00: /* indirect */ 2234 return DISP_0; 2235 case 0b01: /* indirect+1 */ 2236 return DISP_1; 2237 case 0b10: /* indirect+{2,4} */ 2238 if (__predict_false(instr->address_size == 2)) { 2239 return DISP_2; 2240 } 2241 return DISP_4; 2242 case 0b11: /* direct */ 2243 default: /* llvm */ 2244 return DISP_NONE; 2245 } 2246 } 2247 2248 static int 2249 node_regmodrm(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2250 { 2251 struct x86_store *strg, *strm; 2252 const struct x86_opcode *opcode; 2253 const struct x86_reg *reg; 2254 uint8_t byte; 2255 2256 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2257 return -1; 2258 } 2259 2260 opcode = instr->opcode; 2261 2262 instr->regmodrm.rm = ((byte & 0b00000111) >> 0); 2263 instr->regmodrm.reg = ((byte & 0b00111000) >> 3); 2264 instr->regmodrm.mod = ((byte & 0b11000000) >> 6); 2265 2266 if (opcode->regtorm) { 2267 strg = &instr->src; 2268 strm = &instr->dst; 2269 } else { /* RM to REG */ 2270 strm = &instr->src; 2271 strg = &instr->dst; 2272 } 2273 2274 /* Save for later use. */ 2275 instr->strm = strm; 2276 2277 /* 2278 * Special cases: Groups. The REG field of REGMODRM is the index in 2279 * the group. op1 gets overwritten in the Immediate node, if any. 2280 */ 2281 if (opcode->group1) { 2282 if (group1[instr->regmodrm.reg].emul == NULL) { 2283 return -1; 2284 } 2285 instr->emul = group1[instr->regmodrm.reg].emul; 2286 } else if (opcode->group3) { 2287 if (group3[instr->regmodrm.reg].emul == NULL) { 2288 return -1; 2289 } 2290 instr->emul = group3[instr->regmodrm.reg].emul; 2291 } else if (opcode->group11) { 2292 if (group11[instr->regmodrm.reg].emul == NULL) { 2293 return -1; 2294 } 2295 instr->emul = group11[instr->regmodrm.reg].emul; 2296 } 2297 2298 if (!opcode->immediate) { 2299 reg = get_register_reg(instr, opcode); 2300 if (reg == NULL) { 2301 return -1; 2302 } 2303 strg->type = STORE_REG; 2304 strg->u.reg = reg; 2305 } 2306 2307 /* The displacement applies to RM. */ 2308 strm->disp.type = get_disp_type(instr); 2309 2310 if (has_sib(instr)) { 2311 /* Overwrites RM */ 2312 fsm_advance(fsm, 1, node_sib); 2313 return 0; 2314 } 2315 2316 if (is_rip_relative(fsm, instr)) { 2317 /* Overwrites RM */ 2318 strm->type = STORE_REG; 2319 strm->u.reg = &gpr_map__rip; 2320 strm->disp.type = DISP_4; 2321 fsm_advance(fsm, 1, node_disp); 2322 return 0; 2323 } 2324 2325 if (is_disp32_only(fsm, instr)) { 2326 /* Overwrites RM */ 2327 strm->type = STORE_REG; 2328 strm->u.reg = NULL; 2329 strm->disp.type = DISP_4; 2330 fsm_advance(fsm, 1, node_disp); 2331 return 0; 2332 } 2333 2334 if (__predict_false(is_disp16_only(fsm, instr))) { 2335 /* Overwrites RM */ 2336 strm->type = STORE_REG; 2337 strm->u.reg = NULL; 2338 strm->disp.type = DISP_2; 2339 fsm_advance(fsm, 1, node_disp); 2340 return 0; 2341 } 2342 2343 if (__predict_false(is_dual(fsm, instr))) { 2344 /* Overwrites RM */ 2345 fsm_advance(fsm, 0, node_dual); 2346 return 0; 2347 } 2348 2349 reg = get_register_rm(instr, opcode); 2350 if (reg == NULL) { 2351 return -1; 2352 } 2353 strm->type = STORE_REG; 2354 strm->u.reg = reg; 2355 2356 if (strm->disp.type == DISP_NONE) { 2357 /* Direct register addressing mode */ 2358 if (opcode->immediate) { 2359 fsm_advance(fsm, 1, node_immediate); 2360 } else { 2361 fsm_advance(fsm, 1, NULL); 2362 } 2363 } else if (strm->disp.type == DISP_0) { 2364 /* Indirect register addressing mode */ 2365 if (opcode->immediate) { 2366 fsm_advance(fsm, 1, node_immediate); 2367 } else { 2368 fsm_advance(fsm, 1, NULL); 2369 } 2370 } else { 2371 fsm_advance(fsm, 1, node_disp); 2372 } 2373 2374 return 0; 2375 } 2376 2377 static size_t 2378 get_operand_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2379 { 2380 const struct x86_opcode *opcode = instr->opcode; 2381 int opsize; 2382 2383 /* Get the opsize */ 2384 if (!opcode->szoverride) { 2385 opsize = opcode->defsize; 2386 } else if (instr->rexpref.present && instr->rexpref.w) { 2387 opsize = 8; 2388 } else { 2389 if (!fsm->is16bit) { 2390 if (instr->legpref.opr_ovr) { 2391 opsize = 2; 2392 } else { 2393 opsize = 4; 2394 } 2395 } else { /* 16bit */ 2396 if (instr->legpref.opr_ovr) { 2397 opsize = 4; 2398 } else { 2399 opsize = 2; 2400 } 2401 } 2402 } 2403 2404 return opsize; 2405 } 2406 2407 static size_t 2408 get_address_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2409 { 2410 if (fsm->is64bit) { 2411 if (__predict_false(instr->legpref.adr_ovr)) { 2412 return 4; 2413 } 2414 return 8; 2415 } 2416 2417 if (fsm->is32bit) { 2418 if (__predict_false(instr->legpref.adr_ovr)) { 2419 return 2; 2420 } 2421 return 4; 2422 } 2423 2424 /* 16bit. */ 2425 if (__predict_false(instr->legpref.adr_ovr)) { 2426 return 4; 2427 } 2428 return 2; 2429 } 2430 2431 static int 2432 node_primary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2433 { 2434 const struct x86_opcode *opcode; 2435 uint8_t byte; 2436 2437 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2438 return -1; 2439 } 2440 2441 opcode = &primary_opcode_table[byte]; 2442 if (__predict_false(!opcode->valid)) { 2443 return -1; 2444 } 2445 2446 instr->opcode = opcode; 2447 instr->emul = opcode->emul; 2448 instr->operand_size = get_operand_size(fsm, instr); 2449 instr->address_size = get_address_size(fsm, instr); 2450 2451 if (fsm->is64bit && (instr->operand_size == 4)) { 2452 /* Zero-extend to 64 bits. */ 2453 instr->zeroextend_mask = ~size_to_mask(4); 2454 } 2455 2456 if (opcode->regmodrm) { 2457 fsm_advance(fsm, 1, node_regmodrm); 2458 } else if (opcode->dmo) { 2459 /* Direct-Memory Offsets */ 2460 fsm_advance(fsm, 1, node_dmo); 2461 } else if (opcode->stos || opcode->lods) { 2462 fsm_advance(fsm, 1, node_stlo); 2463 } else if (opcode->movs) { 2464 fsm_advance(fsm, 1, node_movs); 2465 } else { 2466 return -1; 2467 } 2468 2469 return 0; 2470 } 2471 2472 static int 2473 node_secondary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2474 { 2475 const struct x86_opcode *opcode; 2476 uint8_t byte; 2477 2478 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2479 return -1; 2480 } 2481 2482 opcode = &secondary_opcode_table[byte]; 2483 if (__predict_false(!opcode->valid)) { 2484 return -1; 2485 } 2486 2487 instr->opcode = opcode; 2488 instr->emul = opcode->emul; 2489 instr->operand_size = get_operand_size(fsm, instr); 2490 instr->address_size = get_address_size(fsm, instr); 2491 2492 if (fsm->is64bit && (instr->operand_size == 4)) { 2493 /* Zero-extend to 64 bits. */ 2494 instr->zeroextend_mask = ~size_to_mask(4); 2495 } 2496 2497 if (opcode->flags & FLAG_ze) { 2498 /* 2499 * Compute the mask for zero-extend. Update the operand size, 2500 * we move fewer bytes. 2501 */ 2502 instr->zeroextend_mask |= size_to_mask(instr->operand_size); 2503 instr->zeroextend_mask &= ~size_to_mask(opcode->defsize); 2504 instr->operand_size = opcode->defsize; 2505 } 2506 2507 if (opcode->regmodrm) { 2508 fsm_advance(fsm, 1, node_regmodrm); 2509 } else { 2510 return -1; 2511 } 2512 2513 return 0; 2514 } 2515 2516 static int 2517 node_main(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2518 { 2519 uint8_t byte; 2520 2521 #define ESCAPE 0x0F 2522 #define VEX_1 0xC5 2523 #define VEX_2 0xC4 2524 #define XOP 0x8F 2525 2526 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2527 return -1; 2528 } 2529 2530 /* 2531 * We don't take XOP. It is AMD-specific, and it was removed shortly 2532 * after being introduced. 2533 */ 2534 if (byte == ESCAPE) { 2535 fsm_advance(fsm, 1, node_secondary_opcode); 2536 } else if (!instr->rexpref.present) { 2537 if (byte == VEX_1) { 2538 return -1; 2539 } else if (byte == VEX_2) { 2540 return -1; 2541 } else { 2542 fsm->fn = node_primary_opcode; 2543 } 2544 } else { 2545 fsm->fn = node_primary_opcode; 2546 } 2547 2548 return 0; 2549 } 2550 2551 static int 2552 node_rex_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2553 { 2554 struct x86_rexpref *rexpref = &instr->rexpref; 2555 uint8_t byte; 2556 size_t n = 0; 2557 2558 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2559 return -1; 2560 } 2561 2562 if (byte >= 0x40 && byte <= 0x4F) { 2563 if (__predict_false(!fsm->is64bit)) { 2564 return -1; 2565 } 2566 rexpref->b = ((byte & 0x1) != 0); 2567 rexpref->x = ((byte & 0x2) != 0); 2568 rexpref->r = ((byte & 0x4) != 0); 2569 rexpref->w = ((byte & 0x8) != 0); 2570 rexpref->present = true; 2571 n = 1; 2572 } 2573 2574 fsm_advance(fsm, n, node_main); 2575 return 0; 2576 } 2577 2578 static int 2579 node_legacy_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2580 { 2581 uint8_t byte; 2582 2583 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2584 return -1; 2585 } 2586 2587 if (byte == LEG_OPR_OVR) { 2588 instr->legpref.opr_ovr = 1; 2589 } else if (byte == LEG_OVR_DS) { 2590 instr->legpref.seg = NVMM_X64_SEG_DS; 2591 } else if (byte == LEG_OVR_ES) { 2592 instr->legpref.seg = NVMM_X64_SEG_ES; 2593 } else if (byte == LEG_REP) { 2594 instr->legpref.rep = 1; 2595 } else if (byte == LEG_OVR_GS) { 2596 instr->legpref.seg = NVMM_X64_SEG_GS; 2597 } else if (byte == LEG_OVR_FS) { 2598 instr->legpref.seg = NVMM_X64_SEG_FS; 2599 } else if (byte == LEG_ADR_OVR) { 2600 instr->legpref.adr_ovr = 1; 2601 } else if (byte == LEG_OVR_CS) { 2602 instr->legpref.seg = NVMM_X64_SEG_CS; 2603 } else if (byte == LEG_OVR_SS) { 2604 instr->legpref.seg = NVMM_X64_SEG_SS; 2605 } else if (byte == LEG_REPN) { 2606 instr->legpref.repn = 1; 2607 } else if (byte == LEG_LOCK) { 2608 /* ignore */ 2609 } else { 2610 /* not a legacy prefix */ 2611 fsm_advance(fsm, 0, node_rex_prefix); 2612 return 0; 2613 } 2614 2615 fsm_advance(fsm, 1, node_legacy_prefix); 2616 return 0; 2617 } 2618 2619 static int 2620 x86_decode(uint8_t *inst_bytes, size_t inst_len, struct x86_instr *instr, 2621 struct nvmm_x64_state *state) 2622 { 2623 struct x86_decode_fsm fsm; 2624 int ret; 2625 2626 memset(instr, 0, sizeof(*instr)); 2627 instr->legpref.seg = -1; 2628 instr->src.hardseg = -1; 2629 instr->dst.hardseg = -1; 2630 2631 fsm.is64bit = is_64bit(state); 2632 fsm.is32bit = is_32bit(state); 2633 fsm.is16bit = is_16bit(state); 2634 2635 fsm.fn = node_legacy_prefix; 2636 fsm.buf = inst_bytes; 2637 fsm.end = inst_bytes + inst_len; 2638 2639 while (fsm.fn != NULL) { 2640 ret = (*fsm.fn)(&fsm, instr); 2641 if (ret == -1) 2642 return -1; 2643 } 2644 2645 instr->len = fsm.buf - inst_bytes; 2646 2647 return 0; 2648 } 2649 2650 /* -------------------------------------------------------------------------- */ 2651 2652 #define EXEC_INSTR(sz, instr) \ 2653 static uint##sz##_t \ 2654 exec_##instr##sz(uint##sz##_t op1, uint##sz##_t op2, uint64_t *rflags) \ 2655 { \ 2656 uint##sz##_t res; \ 2657 __asm __volatile ( \ 2658 #instr" %2, %3;" \ 2659 "mov %3, %1;" \ 2660 "pushfq;" \ 2661 "popq %0" \ 2662 : "=r" (*rflags), "=r" (res) \ 2663 : "r" (op1), "r" (op2)); \ 2664 return res; \ 2665 } 2666 2667 #define EXEC_DISPATCHER(instr) \ 2668 static uint64_t \ 2669 exec_##instr(uint64_t op1, uint64_t op2, uint64_t *rflags, size_t opsize) \ 2670 { \ 2671 switch (opsize) { \ 2672 case 1: \ 2673 return exec_##instr##8(op1, op2, rflags); \ 2674 case 2: \ 2675 return exec_##instr##16(op1, op2, rflags); \ 2676 case 4: \ 2677 return exec_##instr##32(op1, op2, rflags); \ 2678 default: \ 2679 return exec_##instr##64(op1, op2, rflags); \ 2680 } \ 2681 } 2682 2683 /* SUB: ret = op1 - op2 */ 2684 #define PSL_SUB_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF|PSL_AF) 2685 EXEC_INSTR(8, sub) 2686 EXEC_INSTR(16, sub) 2687 EXEC_INSTR(32, sub) 2688 EXEC_INSTR(64, sub) 2689 EXEC_DISPATCHER(sub) 2690 2691 /* OR: ret = op1 | op2 */ 2692 #define PSL_OR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2693 EXEC_INSTR(8, or) 2694 EXEC_INSTR(16, or) 2695 EXEC_INSTR(32, or) 2696 EXEC_INSTR(64, or) 2697 EXEC_DISPATCHER(or) 2698 2699 /* AND: ret = op1 & op2 */ 2700 #define PSL_AND_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2701 EXEC_INSTR(8, and) 2702 EXEC_INSTR(16, and) 2703 EXEC_INSTR(32, and) 2704 EXEC_INSTR(64, and) 2705 EXEC_DISPATCHER(and) 2706 2707 /* XOR: ret = op1 ^ op2 */ 2708 #define PSL_XOR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2709 EXEC_INSTR(8, xor) 2710 EXEC_INSTR(16, xor) 2711 EXEC_INSTR(32, xor) 2712 EXEC_INSTR(64, xor) 2713 EXEC_DISPATCHER(xor) 2714 2715 /* -------------------------------------------------------------------------- */ 2716 2717 /* 2718 * Emulation functions. We don't care about the order of the operands, except 2719 * for SUB, CMP and TEST. For these ones we look at mem->write to determine who 2720 * is op1 and who is op2. 2721 */ 2722 2723 static void 2724 x86_func_or(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2725 { 2726 uint64_t *retval = (uint64_t *)mem->data; 2727 const bool write = mem->write; 2728 uint64_t *op1, op2, fl, ret; 2729 2730 op1 = (uint64_t *)mem->data; 2731 op2 = 0; 2732 2733 /* Fetch the value to be OR'ed (op2). */ 2734 mem->data = (uint8_t *)&op2; 2735 mem->write = false; 2736 (*mach->cbs.mem)(mem); 2737 2738 /* Perform the OR. */ 2739 ret = exec_or(*op1, op2, &fl, mem->size); 2740 2741 if (write) { 2742 /* Write back the result. */ 2743 mem->data = (uint8_t *)&ret; 2744 mem->write = true; 2745 (*mach->cbs.mem)(mem); 2746 } else { 2747 /* Return data to the caller. */ 2748 *retval = ret; 2749 } 2750 2751 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_OR_MASK; 2752 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_OR_MASK); 2753 } 2754 2755 static void 2756 x86_func_and(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2757 { 2758 uint64_t *retval = (uint64_t *)mem->data; 2759 const bool write = mem->write; 2760 uint64_t *op1, op2, fl, ret; 2761 2762 op1 = (uint64_t *)mem->data; 2763 op2 = 0; 2764 2765 /* Fetch the value to be AND'ed (op2). */ 2766 mem->data = (uint8_t *)&op2; 2767 mem->write = false; 2768 (*mach->cbs.mem)(mem); 2769 2770 /* Perform the AND. */ 2771 ret = exec_and(*op1, op2, &fl, mem->size); 2772 2773 if (write) { 2774 /* Write back the result. */ 2775 mem->data = (uint8_t *)&ret; 2776 mem->write = true; 2777 (*mach->cbs.mem)(mem); 2778 } else { 2779 /* Return data to the caller. */ 2780 *retval = ret; 2781 } 2782 2783 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2784 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2785 } 2786 2787 static void 2788 x86_func_xchg(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2789 { 2790 uint64_t *op1, op2; 2791 2792 op1 = (uint64_t *)mem->data; 2793 op2 = 0; 2794 2795 /* Fetch op2. */ 2796 mem->data = (uint8_t *)&op2; 2797 mem->write = false; 2798 (*mach->cbs.mem)(mem); 2799 2800 /* Write op1 in op2. */ 2801 mem->data = (uint8_t *)op1; 2802 mem->write = true; 2803 (*mach->cbs.mem)(mem); 2804 2805 /* Write op2 in op1. */ 2806 *op1 = op2; 2807 } 2808 2809 static void 2810 x86_func_sub(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2811 { 2812 uint64_t *retval = (uint64_t *)mem->data; 2813 const bool write = mem->write; 2814 uint64_t *op1, *op2, fl, ret; 2815 uint64_t tmp; 2816 bool memop1; 2817 2818 memop1 = !mem->write; 2819 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2820 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2821 2822 /* Fetch the value to be SUB'ed (op1 or op2). */ 2823 mem->data = (uint8_t *)&tmp; 2824 mem->write = false; 2825 (*mach->cbs.mem)(mem); 2826 2827 /* Perform the SUB. */ 2828 ret = exec_sub(*op1, *op2, &fl, mem->size); 2829 2830 if (write) { 2831 /* Write back the result. */ 2832 mem->data = (uint8_t *)&ret; 2833 mem->write = true; 2834 (*mach->cbs.mem)(mem); 2835 } else { 2836 /* Return data to the caller. */ 2837 *retval = ret; 2838 } 2839 2840 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2841 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2842 } 2843 2844 static void 2845 x86_func_xor(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2846 { 2847 uint64_t *retval = (uint64_t *)mem->data; 2848 const bool write = mem->write; 2849 uint64_t *op1, op2, fl, ret; 2850 2851 op1 = (uint64_t *)mem->data; 2852 op2 = 0; 2853 2854 /* Fetch the value to be XOR'ed (op2). */ 2855 mem->data = (uint8_t *)&op2; 2856 mem->write = false; 2857 (*mach->cbs.mem)(mem); 2858 2859 /* Perform the XOR. */ 2860 ret = exec_xor(*op1, op2, &fl, mem->size); 2861 2862 if (write) { 2863 /* Write back the result. */ 2864 mem->data = (uint8_t *)&ret; 2865 mem->write = true; 2866 (*mach->cbs.mem)(mem); 2867 } else { 2868 /* Return data to the caller. */ 2869 *retval = ret; 2870 } 2871 2872 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_XOR_MASK; 2873 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_XOR_MASK); 2874 } 2875 2876 static void 2877 x86_func_cmp(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2878 { 2879 uint64_t *op1, *op2, fl; 2880 uint64_t tmp; 2881 bool memop1; 2882 2883 memop1 = !mem->write; 2884 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2885 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2886 2887 /* Fetch the value to be CMP'ed (op1 or op2). */ 2888 mem->data = (uint8_t *)&tmp; 2889 mem->write = false; 2890 (*mach->cbs.mem)(mem); 2891 2892 /* Perform the CMP. */ 2893 exec_sub(*op1, *op2, &fl, mem->size); 2894 2895 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2896 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2897 } 2898 2899 static void 2900 x86_func_test(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2901 { 2902 uint64_t *op1, *op2, fl; 2903 uint64_t tmp; 2904 bool memop1; 2905 2906 memop1 = !mem->write; 2907 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2908 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2909 2910 /* Fetch the value to be TEST'ed (op1 or op2). */ 2911 mem->data = (uint8_t *)&tmp; 2912 mem->write = false; 2913 (*mach->cbs.mem)(mem); 2914 2915 /* Perform the TEST. */ 2916 exec_and(*op1, *op2, &fl, mem->size); 2917 2918 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2919 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2920 } 2921 2922 static void 2923 x86_func_mov(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2924 { 2925 /* 2926 * Nothing special, just move without emulation. 2927 */ 2928 (*mach->cbs.mem)(mem); 2929 } 2930 2931 static void 2932 x86_func_stos(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2933 { 2934 /* 2935 * Just move, and update RDI. 2936 */ 2937 (*mach->cbs.mem)(mem); 2938 2939 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2940 gprs[NVMM_X64_GPR_RDI] -= mem->size; 2941 } else { 2942 gprs[NVMM_X64_GPR_RDI] += mem->size; 2943 } 2944 } 2945 2946 static void 2947 x86_func_lods(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2948 { 2949 /* 2950 * Just move, and update RSI. 2951 */ 2952 (*mach->cbs.mem)(mem); 2953 2954 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2955 gprs[NVMM_X64_GPR_RSI] -= mem->size; 2956 } else { 2957 gprs[NVMM_X64_GPR_RSI] += mem->size; 2958 } 2959 } 2960 2961 static void 2962 x86_func_movs(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2963 { 2964 /* 2965 * Special instruction: double memory operand. Don't call the cb, 2966 * because the storage has already been performed earlier. 2967 */ 2968 2969 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2970 gprs[NVMM_X64_GPR_RSI] -= mem->size; 2971 gprs[NVMM_X64_GPR_RDI] -= mem->size; 2972 } else { 2973 gprs[NVMM_X64_GPR_RSI] += mem->size; 2974 gprs[NVMM_X64_GPR_RDI] += mem->size; 2975 } 2976 } 2977 2978 /* -------------------------------------------------------------------------- */ 2979 2980 static inline uint64_t 2981 gpr_read_address(struct x86_instr *instr, struct nvmm_x64_state *state, int gpr) 2982 { 2983 uint64_t val; 2984 2985 val = state->gprs[gpr]; 2986 val &= size_to_mask(instr->address_size); 2987 2988 return val; 2989 } 2990 2991 static int 2992 store_to_gva(struct nvmm_x64_state *state, struct x86_instr *instr, 2993 struct x86_store *store, gvaddr_t *gvap, size_t size) 2994 { 2995 struct x86_sib *sib; 2996 gvaddr_t gva = 0; 2997 uint64_t reg; 2998 int ret, seg; 2999 3000 if (store->type == STORE_SIB) { 3001 sib = &store->u.sib; 3002 if (sib->bas != NULL) 3003 gva += gpr_read_address(instr, state, sib->bas->num); 3004 if (sib->idx != NULL) { 3005 reg = gpr_read_address(instr, state, sib->idx->num); 3006 gva += sib->scale * reg; 3007 } 3008 } else if (store->type == STORE_REG) { 3009 if (store->u.reg == NULL) { 3010 /* The base is null. Happens with disp32-only and 3011 * disp16-only. */ 3012 } else { 3013 gva = gpr_read_address(instr, state, store->u.reg->num); 3014 } 3015 } else if (store->type == STORE_DUALREG) { 3016 gva = gpr_read_address(instr, state, store->u.dualreg.reg1) + 3017 gpr_read_address(instr, state, store->u.dualreg.reg2); 3018 } else { 3019 gva = store->u.dmo; 3020 } 3021 3022 if (store->disp.type != DISP_NONE) { 3023 gva += store->disp.data; 3024 } 3025 3026 if (store->hardseg != -1) { 3027 seg = store->hardseg; 3028 } else { 3029 if (__predict_false(instr->legpref.seg != -1)) { 3030 seg = instr->legpref.seg; 3031 } else { 3032 seg = NVMM_X64_SEG_DS; 3033 } 3034 } 3035 3036 if (__predict_true(is_long_mode(state))) { 3037 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 3038 segment_apply(&state->segs[seg], &gva); 3039 } 3040 } else { 3041 ret = segment_check(&state->segs[seg], gva, size); 3042 if (ret == -1) 3043 return -1; 3044 segment_apply(&state->segs[seg], &gva); 3045 } 3046 3047 *gvap = gva; 3048 return 0; 3049 } 3050 3051 static int 3052 fetch_segment(struct nvmm_machine *mach, struct nvmm_x64_state *state) 3053 { 3054 uint8_t inst_bytes[5], byte; 3055 size_t i, fetchsize; 3056 gvaddr_t gva; 3057 int ret, seg; 3058 3059 fetchsize = sizeof(inst_bytes); 3060 3061 gva = state->gprs[NVMM_X64_GPR_RIP]; 3062 if (__predict_false(!is_long_mode(state))) { 3063 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3064 fetchsize); 3065 if (ret == -1) 3066 return -1; 3067 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3068 } 3069 3070 ret = read_guest_memory(mach, state, gva, inst_bytes, fetchsize); 3071 if (ret == -1) 3072 return -1; 3073 3074 seg = NVMM_X64_SEG_DS; 3075 for (i = 0; i < fetchsize; i++) { 3076 byte = inst_bytes[i]; 3077 3078 if (byte == LEG_OVR_DS) { 3079 seg = NVMM_X64_SEG_DS; 3080 } else if (byte == LEG_OVR_ES) { 3081 seg = NVMM_X64_SEG_ES; 3082 } else if (byte == LEG_OVR_GS) { 3083 seg = NVMM_X64_SEG_GS; 3084 } else if (byte == LEG_OVR_FS) { 3085 seg = NVMM_X64_SEG_FS; 3086 } else if (byte == LEG_OVR_CS) { 3087 seg = NVMM_X64_SEG_CS; 3088 } else if (byte == LEG_OVR_SS) { 3089 seg = NVMM_X64_SEG_SS; 3090 } else if (byte == LEG_OPR_OVR) { 3091 /* nothing */ 3092 } else if (byte == LEG_ADR_OVR) { 3093 /* nothing */ 3094 } else if (byte == LEG_REP) { 3095 /* nothing */ 3096 } else if (byte == LEG_REPN) { 3097 /* nothing */ 3098 } else if (byte == LEG_LOCK) { 3099 /* nothing */ 3100 } else { 3101 return seg; 3102 } 3103 } 3104 3105 return seg; 3106 } 3107 3108 static int 3109 fetch_instruction(struct nvmm_machine *mach, struct nvmm_x64_state *state, 3110 struct nvmm_vcpu_exit *exit) 3111 { 3112 size_t fetchsize; 3113 gvaddr_t gva; 3114 int ret; 3115 3116 fetchsize = sizeof(exit->u.mem.inst_bytes); 3117 3118 gva = state->gprs[NVMM_X64_GPR_RIP]; 3119 if (__predict_false(!is_long_mode(state))) { 3120 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3121 fetchsize); 3122 if (ret == -1) 3123 return -1; 3124 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3125 } 3126 3127 ret = read_guest_memory(mach, state, gva, exit->u.mem.inst_bytes, 3128 fetchsize); 3129 if (ret == -1) 3130 return -1; 3131 3132 exit->u.mem.inst_len = fetchsize; 3133 3134 return 0; 3135 } 3136 3137 static int 3138 assist_mem_double(struct nvmm_machine *mach, struct nvmm_x64_state *state, 3139 struct x86_instr *instr) 3140 { 3141 struct nvmm_mem mem; 3142 uint8_t data[8]; 3143 gvaddr_t gva; 3144 size_t size; 3145 int ret; 3146 3147 size = instr->operand_size; 3148 3149 /* Source. */ 3150 ret = store_to_gva(state, instr, &instr->src, &gva, size); 3151 if (ret == -1) 3152 return -1; 3153 ret = read_guest_memory(mach, state, gva, data, size); 3154 if (ret == -1) 3155 return -1; 3156 3157 /* Destination. */ 3158 ret = store_to_gva(state, instr, &instr->dst, &gva, size); 3159 if (ret == -1) 3160 return -1; 3161 ret = write_guest_memory(mach, state, gva, data, size); 3162 if (ret == -1) 3163 return -1; 3164 3165 mem.size = size; 3166 (*instr->emul->func)(mach, &mem, state->gprs); 3167 3168 return 0; 3169 } 3170 3171 static int 3172 assist_mem_single(struct nvmm_machine *mach, struct nvmm_x64_state *state, 3173 struct x86_instr *instr, struct nvmm_vcpu_exit *exit) 3174 { 3175 struct nvmm_mem mem; 3176 uint8_t membuf[8]; 3177 uint64_t val; 3178 3179 memset(membuf, 0, sizeof(membuf)); 3180 3181 mem.gpa = exit->u.mem.gpa; 3182 mem.size = instr->operand_size; 3183 mem.data = membuf; 3184 3185 /* Determine the direction. */ 3186 switch (instr->src.type) { 3187 case STORE_REG: 3188 if (instr->src.disp.type != DISP_NONE) { 3189 /* Indirect access. */ 3190 mem.write = false; 3191 } else { 3192 /* Direct access. */ 3193 mem.write = true; 3194 } 3195 break; 3196 case STORE_DUALREG: 3197 if (instr->src.disp.type == DISP_NONE) { 3198 DISASSEMBLER_BUG(); 3199 } 3200 mem.write = false; 3201 break; 3202 case STORE_IMM: 3203 mem.write = true; 3204 break; 3205 case STORE_SIB: 3206 mem.write = false; 3207 break; 3208 case STORE_DMO: 3209 mem.write = false; 3210 break; 3211 default: 3212 DISASSEMBLER_BUG(); 3213 } 3214 3215 if (mem.write) { 3216 switch (instr->src.type) { 3217 case STORE_REG: 3218 /* The instruction was "reg -> mem". Fetch the register 3219 * in membuf. */ 3220 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3221 DISASSEMBLER_BUG(); 3222 } 3223 val = state->gprs[instr->src.u.reg->num]; 3224 val = __SHIFTOUT(val, instr->src.u.reg->mask); 3225 memcpy(mem.data, &val, mem.size); 3226 break; 3227 case STORE_IMM: 3228 /* The instruction was "imm -> mem". Fetch the immediate 3229 * in membuf. */ 3230 memcpy(mem.data, &instr->src.u.imm.data, mem.size); 3231 break; 3232 default: 3233 DISASSEMBLER_BUG(); 3234 } 3235 } else if (instr->emul->readreg) { 3236 /* The instruction was "mem -> reg", but the value of the 3237 * register matters for the emul func. Fetch it in membuf. */ 3238 if (__predict_false(instr->dst.type != STORE_REG)) { 3239 DISASSEMBLER_BUG(); 3240 } 3241 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3242 DISASSEMBLER_BUG(); 3243 } 3244 val = state->gprs[instr->dst.u.reg->num]; 3245 val = __SHIFTOUT(val, instr->dst.u.reg->mask); 3246 memcpy(mem.data, &val, mem.size); 3247 } 3248 3249 (*instr->emul->func)(mach, &mem, state->gprs); 3250 3251 if (instr->emul->notouch) { 3252 /* We're done. */ 3253 return 0; 3254 } 3255 3256 if (!mem.write) { 3257 /* The instruction was "mem -> reg". The emul func has filled 3258 * membuf with the memory content. Install membuf in the 3259 * register. */ 3260 if (__predict_false(instr->dst.type != STORE_REG)) { 3261 DISASSEMBLER_BUG(); 3262 } 3263 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3264 DISASSEMBLER_BUG(); 3265 } 3266 memcpy(&val, membuf, sizeof(uint64_t)); 3267 val = __SHIFTIN(val, instr->dst.u.reg->mask); 3268 state->gprs[instr->dst.u.reg->num] &= ~instr->dst.u.reg->mask; 3269 state->gprs[instr->dst.u.reg->num] |= val; 3270 state->gprs[instr->dst.u.reg->num] &= ~instr->zeroextend_mask; 3271 } else if (instr->emul->backprop) { 3272 /* The instruction was "reg -> mem", but the memory must be 3273 * back-propagated to the register. Install membuf in the 3274 * register. */ 3275 if (__predict_false(instr->src.type != STORE_REG)) { 3276 DISASSEMBLER_BUG(); 3277 } 3278 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3279 DISASSEMBLER_BUG(); 3280 } 3281 memcpy(&val, membuf, sizeof(uint64_t)); 3282 val = __SHIFTIN(val, instr->src.u.reg->mask); 3283 state->gprs[instr->src.u.reg->num] &= ~instr->src.u.reg->mask; 3284 state->gprs[instr->src.u.reg->num] |= val; 3285 state->gprs[instr->src.u.reg->num] &= ~instr->zeroextend_mask; 3286 } 3287 3288 return 0; 3289 } 3290 3291 int 3292 nvmm_assist_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3293 { 3294 struct nvmm_x64_state *state = vcpu->state; 3295 struct nvmm_vcpu_exit *exit = vcpu->exit; 3296 struct x86_instr instr; 3297 uint64_t cnt = 0; /* GCC */ 3298 int ret; 3299 3300 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_MEMORY)) { 3301 errno = EINVAL; 3302 return -1; 3303 } 3304 3305 ret = nvmm_vcpu_getstate(mach, vcpu, 3306 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 3307 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 3308 if (ret == -1) 3309 return -1; 3310 3311 if (exit->u.mem.inst_len == 0) { 3312 /* 3313 * The instruction was not fetched from the kernel. Fetch 3314 * it ourselves. 3315 */ 3316 ret = fetch_instruction(mach, state, exit); 3317 if (ret == -1) 3318 return -1; 3319 } 3320 3321 ret = x86_decode(exit->u.mem.inst_bytes, exit->u.mem.inst_len, 3322 &instr, state); 3323 if (ret == -1) { 3324 errno = ENODEV; 3325 return -1; 3326 } 3327 3328 if (instr.legpref.rep || instr.legpref.repn) { 3329 cnt = rep_get_cnt(state, instr.address_size); 3330 if (__predict_false(cnt == 0)) { 3331 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3332 goto out; 3333 } 3334 } 3335 3336 if (instr.opcode->movs) { 3337 ret = assist_mem_double(mach, state, &instr); 3338 } else { 3339 ret = assist_mem_single(mach, state, &instr, exit); 3340 } 3341 if (ret == -1) { 3342 errno = ENODEV; 3343 return -1; 3344 } 3345 3346 if (instr.legpref.rep || instr.legpref.repn) { 3347 cnt -= 1; 3348 rep_set_cnt(state, instr.address_size, cnt); 3349 if (cnt == 0) { 3350 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3351 } else if (__predict_false(instr.legpref.repn)) { 3352 if (state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_Z) { 3353 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3354 } 3355 } 3356 } else { 3357 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3358 } 3359 3360 out: 3361 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 3362 if (ret == -1) 3363 return -1; 3364 3365 return 0; 3366 } 3367
Indexes created Wed Sep 24 05:09:52 GMT 2025