libnvmm_x86.c revision 1.31.2.2
1 1.31.2.2 christos /* $NetBSD: libnvmm_x86.c,v 1.31.2.2 2019/06/10 22:05:25 christos Exp $ */ 2 1.31.2.2 christos 3 1.31.2.2 christos /* 4 1.31.2.2 christos * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 1.31.2.2 christos * All rights reserved. 6 1.31.2.2 christos * 7 1.31.2.2 christos * This code is derived from software contributed to The NetBSD Foundation 8 1.31.2.2 christos * by Maxime Villard. 9 1.31.2.2 christos * 10 1.31.2.2 christos * Redistribution and use in source and binary forms, with or without 11 1.31.2.2 christos * modification, are permitted provided that the following conditions 12 1.31.2.2 christos * are met: 13 1.31.2.2 christos * 1. Redistributions of source code must retain the above copyright 14 1.31.2.2 christos * notice, this list of conditions and the following disclaimer. 15 1.31.2.2 christos * 2. Redistributions in binary form must reproduce the above copyright 16 1.31.2.2 christos * notice, this list of conditions and the following disclaimer in the 17 1.31.2.2 christos * documentation and/or other materials provided with the distribution. 18 1.31.2.2 christos * 19 1.31.2.2 christos * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.31.2.2 christos * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.31.2.2 christos * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.31.2.2 christos * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.31.2.2 christos * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.31.2.2 christos * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.31.2.2 christos * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.31.2.2 christos * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.31.2.2 christos * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.31.2.2 christos * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.31.2.2 christos * POSSIBILITY OF SUCH DAMAGE. 30 1.31.2.2 christos */ 31 1.31.2.2 christos 32 1.31.2.2 christos #include <sys/cdefs.h> 33 1.31.2.2 christos 34 1.31.2.2 christos #include <stdio.h> 35 1.31.2.2 christos #include <stdlib.h> 36 1.31.2.2 christos #include <string.h> 37 1.31.2.2 christos #include <unistd.h> 38 1.31.2.2 christos #include <fcntl.h> 39 1.31.2.2 christos #include <errno.h> 40 1.31.2.2 christos #include <sys/ioctl.h> 41 1.31.2.2 christos #include <sys/mman.h> 42 1.31.2.2 christos #include <machine/vmparam.h> 43 1.31.2.2 christos #include <machine/pte.h> 44 1.31.2.2 christos #include <machine/psl.h> 45 1.31.2.2 christos 46 1.31.2.2 christos #define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) 47 1.31.2.2 christos #define __cacheline_aligned __attribute__((__aligned__(64))) 48 1.31.2.2 christos 49 1.31.2.2 christos #include <x86/specialreg.h> 50 1.31.2.2 christos 51 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 52 1.31.2.2 christos 53 1.31.2.2 christos /* 54 1.31.2.2 christos * Undocumented debugging function. Helpful. 55 1.31.2.2 christos */ 56 1.31.2.2 christos int 57 1.31.2.2 christos nvmm_vcpu_dump(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 58 1.31.2.2 christos { 59 1.31.2.2 christos struct nvmm_x64_state *state = vcpu->state; 60 1.31.2.2 christos uint16_t *attr; 61 1.31.2.2 christos size_t i; 62 1.31.2.2 christos int ret; 63 1.31.2.2 christos 64 1.31.2.2 christos const char *segnames[] = { 65 1.31.2.2 christos "ES", "CS", "SS", "DS", "FS", "GS", "GDT", "IDT", "LDT", "TR" 66 1.31.2.2 christos }; 67 1.31.2.2 christos 68 1.31.2.2 christos ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_ALL); 69 1.31.2.2 christos if (ret == -1) 70 1.31.2.2 christos return -1; 71 1.31.2.2 christos 72 1.31.2.2 christos printf("+ VCPU id=%d\n", (int)vcpu->cpuid); 73 1.31.2.2 christos printf("| -> RIP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RIP]); 74 1.31.2.2 christos printf("| -> RSP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSP]); 75 1.31.2.2 christos printf("| -> RAX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RAX]); 76 1.31.2.2 christos printf("| -> RBX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBX]); 77 1.31.2.2 christos printf("| -> RCX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RCX]); 78 1.31.2.2 christos printf("| -> RFLAGS=%p\n", (void *)state->gprs[NVMM_X64_GPR_RFLAGS]); 79 1.31.2.2 christos for (i = 0; i < NVMM_X64_NSEG; i++) { 80 1.31.2.2 christos attr = (uint16_t *)&state->segs[i].attrib; 81 1.31.2.2 christos printf("| -> %s: sel=0x%x base=%"PRIx64", limit=%x, attrib=%x\n", 82 1.31.2.2 christos segnames[i], 83 1.31.2.2 christos state->segs[i].selector, 84 1.31.2.2 christos state->segs[i].base, 85 1.31.2.2 christos state->segs[i].limit, 86 1.31.2.2 christos *attr); 87 1.31.2.2 christos } 88 1.31.2.2 christos printf("| -> MSR_EFER=%"PRIx64"\n", state->msrs[NVMM_X64_MSR_EFER]); 89 1.31.2.2 christos printf("| -> CR0=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR0]); 90 1.31.2.2 christos printf("| -> CR3=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR3]); 91 1.31.2.2 christos printf("| -> CR4=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR4]); 92 1.31.2.2 christos printf("| -> CR8=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR8]); 93 1.31.2.2 christos 94 1.31.2.2 christos return 0; 95 1.31.2.2 christos } 96 1.31.2.2 christos 97 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 98 1.31.2.2 christos 99 1.31.2.2 christos #define PTE32_L1_SHIFT 12 100 1.31.2.2 christos #define PTE32_L2_SHIFT 22 101 1.31.2.2 christos 102 1.31.2.2 christos #define PTE32_L2_MASK 0xffc00000 103 1.31.2.2 christos #define PTE32_L1_MASK 0x003ff000 104 1.31.2.2 christos 105 1.31.2.2 christos #define PTE32_L2_FRAME (PTE32_L2_MASK) 106 1.31.2.2 christos #define PTE32_L1_FRAME (PTE32_L2_FRAME|PTE32_L1_MASK) 107 1.31.2.2 christos 108 1.31.2.2 christos #define pte32_l1idx(va) (((va) & PTE32_L1_MASK) >> PTE32_L1_SHIFT) 109 1.31.2.2 christos #define pte32_l2idx(va) (((va) & PTE32_L2_MASK) >> PTE32_L2_SHIFT) 110 1.31.2.2 christos 111 1.31.2.2 christos #define CR3_FRAME_32BIT PG_FRAME 112 1.31.2.2 christos 113 1.31.2.2 christos typedef uint32_t pte_32bit_t; 114 1.31.2.2 christos 115 1.31.2.2 christos static int 116 1.31.2.2 christos x86_gva_to_gpa_32bit(struct nvmm_machine *mach, uint64_t cr3, 117 1.31.2.2 christos gvaddr_t gva, gpaddr_t *gpa, bool has_pse, nvmm_prot_t *prot) 118 1.31.2.2 christos { 119 1.31.2.2 christos gpaddr_t L2gpa, L1gpa; 120 1.31.2.2 christos uintptr_t L2hva, L1hva; 121 1.31.2.2 christos pte_32bit_t *pdir, pte; 122 1.31.2.2 christos nvmm_prot_t pageprot; 123 1.31.2.2 christos 124 1.31.2.2 christos /* We begin with an RWXU access. */ 125 1.31.2.2 christos *prot = NVMM_PROT_ALL; 126 1.31.2.2 christos 127 1.31.2.2 christos /* Parse L2. */ 128 1.31.2.2 christos L2gpa = (cr3 & CR3_FRAME_32BIT); 129 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 130 1.31.2.2 christos return -1; 131 1.31.2.2 christos pdir = (pte_32bit_t *)L2hva; 132 1.31.2.2 christos pte = pdir[pte32_l2idx(gva)]; 133 1.31.2.2 christos if ((pte & PG_V) == 0) 134 1.31.2.2 christos return -1; 135 1.31.2.2 christos if ((pte & PG_u) == 0) 136 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 137 1.31.2.2 christos if ((pte & PG_KW) == 0) 138 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 139 1.31.2.2 christos if ((pte & PG_PS) && !has_pse) 140 1.31.2.2 christos return -1; 141 1.31.2.2 christos if (pte & PG_PS) { 142 1.31.2.2 christos *gpa = (pte & PTE32_L2_FRAME); 143 1.31.2.2 christos *gpa = *gpa + (gva & PTE32_L1_MASK); 144 1.31.2.2 christos return 0; 145 1.31.2.2 christos } 146 1.31.2.2 christos 147 1.31.2.2 christos /* Parse L1. */ 148 1.31.2.2 christos L1gpa = (pte & PG_FRAME); 149 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 150 1.31.2.2 christos return -1; 151 1.31.2.2 christos pdir = (pte_32bit_t *)L1hva; 152 1.31.2.2 christos pte = pdir[pte32_l1idx(gva)]; 153 1.31.2.2 christos if ((pte & PG_V) == 0) 154 1.31.2.2 christos return -1; 155 1.31.2.2 christos if ((pte & PG_u) == 0) 156 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 157 1.31.2.2 christos if ((pte & PG_KW) == 0) 158 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 159 1.31.2.2 christos if (pte & PG_PS) 160 1.31.2.2 christos return -1; 161 1.31.2.2 christos 162 1.31.2.2 christos *gpa = (pte & PG_FRAME); 163 1.31.2.2 christos return 0; 164 1.31.2.2 christos } 165 1.31.2.2 christos 166 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 167 1.31.2.2 christos 168 1.31.2.2 christos #define PTE32_PAE_L1_SHIFT 12 169 1.31.2.2 christos #define PTE32_PAE_L2_SHIFT 21 170 1.31.2.2 christos #define PTE32_PAE_L3_SHIFT 30 171 1.31.2.2 christos 172 1.31.2.2 christos #define PTE32_PAE_L3_MASK 0xc0000000 173 1.31.2.2 christos #define PTE32_PAE_L2_MASK 0x3fe00000 174 1.31.2.2 christos #define PTE32_PAE_L1_MASK 0x001ff000 175 1.31.2.2 christos 176 1.31.2.2 christos #define PTE32_PAE_L3_FRAME (PTE32_PAE_L3_MASK) 177 1.31.2.2 christos #define PTE32_PAE_L2_FRAME (PTE32_PAE_L3_FRAME|PTE32_PAE_L2_MASK) 178 1.31.2.2 christos #define PTE32_PAE_L1_FRAME (PTE32_PAE_L2_FRAME|PTE32_PAE_L1_MASK) 179 1.31.2.2 christos 180 1.31.2.2 christos #define pte32_pae_l1idx(va) (((va) & PTE32_PAE_L1_MASK) >> PTE32_PAE_L1_SHIFT) 181 1.31.2.2 christos #define pte32_pae_l2idx(va) (((va) & PTE32_PAE_L2_MASK) >> PTE32_PAE_L2_SHIFT) 182 1.31.2.2 christos #define pte32_pae_l3idx(va) (((va) & PTE32_PAE_L3_MASK) >> PTE32_PAE_L3_SHIFT) 183 1.31.2.2 christos 184 1.31.2.2 christos #define CR3_FRAME_32BIT_PAE __BITS(31, 5) 185 1.31.2.2 christos 186 1.31.2.2 christos typedef uint64_t pte_32bit_pae_t; 187 1.31.2.2 christos 188 1.31.2.2 christos static int 189 1.31.2.2 christos x86_gva_to_gpa_32bit_pae(struct nvmm_machine *mach, uint64_t cr3, 190 1.31.2.2 christos gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 191 1.31.2.2 christos { 192 1.31.2.2 christos gpaddr_t L3gpa, L2gpa, L1gpa; 193 1.31.2.2 christos uintptr_t L3hva, L2hva, L1hva; 194 1.31.2.2 christos pte_32bit_pae_t *pdir, pte; 195 1.31.2.2 christos nvmm_prot_t pageprot; 196 1.31.2.2 christos 197 1.31.2.2 christos /* We begin with an RWXU access. */ 198 1.31.2.2 christos *prot = NVMM_PROT_ALL; 199 1.31.2.2 christos 200 1.31.2.2 christos /* Parse L3. */ 201 1.31.2.2 christos L3gpa = (cr3 & CR3_FRAME_32BIT_PAE); 202 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 203 1.31.2.2 christos return -1; 204 1.31.2.2 christos pdir = (pte_32bit_pae_t *)L3hva; 205 1.31.2.2 christos pte = pdir[pte32_pae_l3idx(gva)]; 206 1.31.2.2 christos if ((pte & PG_V) == 0) 207 1.31.2.2 christos return -1; 208 1.31.2.2 christos if (pte & PG_NX) 209 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 210 1.31.2.2 christos if (pte & PG_PS) 211 1.31.2.2 christos return -1; 212 1.31.2.2 christos 213 1.31.2.2 christos /* Parse L2. */ 214 1.31.2.2 christos L2gpa = (pte & PG_FRAME); 215 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 216 1.31.2.2 christos return -1; 217 1.31.2.2 christos pdir = (pte_32bit_pae_t *)L2hva; 218 1.31.2.2 christos pte = pdir[pte32_pae_l2idx(gva)]; 219 1.31.2.2 christos if ((pte & PG_V) == 0) 220 1.31.2.2 christos return -1; 221 1.31.2.2 christos if ((pte & PG_u) == 0) 222 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 223 1.31.2.2 christos if ((pte & PG_KW) == 0) 224 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 225 1.31.2.2 christos if (pte & PG_NX) 226 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 227 1.31.2.2 christos if (pte & PG_PS) { 228 1.31.2.2 christos *gpa = (pte & PTE32_PAE_L2_FRAME); 229 1.31.2.2 christos *gpa = *gpa + (gva & PTE32_PAE_L1_MASK); 230 1.31.2.2 christos return 0; 231 1.31.2.2 christos } 232 1.31.2.2 christos 233 1.31.2.2 christos /* Parse L1. */ 234 1.31.2.2 christos L1gpa = (pte & PG_FRAME); 235 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 236 1.31.2.2 christos return -1; 237 1.31.2.2 christos pdir = (pte_32bit_pae_t *)L1hva; 238 1.31.2.2 christos pte = pdir[pte32_pae_l1idx(gva)]; 239 1.31.2.2 christos if ((pte & PG_V) == 0) 240 1.31.2.2 christos return -1; 241 1.31.2.2 christos if ((pte & PG_u) == 0) 242 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 243 1.31.2.2 christos if ((pte & PG_KW) == 0) 244 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 245 1.31.2.2 christos if (pte & PG_NX) 246 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 247 1.31.2.2 christos if (pte & PG_PS) 248 1.31.2.2 christos return -1; 249 1.31.2.2 christos 250 1.31.2.2 christos *gpa = (pte & PG_FRAME); 251 1.31.2.2 christos return 0; 252 1.31.2.2 christos } 253 1.31.2.2 christos 254 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 255 1.31.2.2 christos 256 1.31.2.2 christos #define PTE64_L1_SHIFT 12 257 1.31.2.2 christos #define PTE64_L2_SHIFT 21 258 1.31.2.2 christos #define PTE64_L3_SHIFT 30 259 1.31.2.2 christos #define PTE64_L4_SHIFT 39 260 1.31.2.2 christos 261 1.31.2.2 christos #define PTE64_L4_MASK 0x0000ff8000000000 262 1.31.2.2 christos #define PTE64_L3_MASK 0x0000007fc0000000 263 1.31.2.2 christos #define PTE64_L2_MASK 0x000000003fe00000 264 1.31.2.2 christos #define PTE64_L1_MASK 0x00000000001ff000 265 1.31.2.2 christos 266 1.31.2.2 christos #define PTE64_L4_FRAME PTE64_L4_MASK 267 1.31.2.2 christos #define PTE64_L3_FRAME (PTE64_L4_FRAME|PTE64_L3_MASK) 268 1.31.2.2 christos #define PTE64_L2_FRAME (PTE64_L3_FRAME|PTE64_L2_MASK) 269 1.31.2.2 christos #define PTE64_L1_FRAME (PTE64_L2_FRAME|PTE64_L1_MASK) 270 1.31.2.2 christos 271 1.31.2.2 christos #define pte64_l1idx(va) (((va) & PTE64_L1_MASK) >> PTE64_L1_SHIFT) 272 1.31.2.2 christos #define pte64_l2idx(va) (((va) & PTE64_L2_MASK) >> PTE64_L2_SHIFT) 273 1.31.2.2 christos #define pte64_l3idx(va) (((va) & PTE64_L3_MASK) >> PTE64_L3_SHIFT) 274 1.31.2.2 christos #define pte64_l4idx(va) (((va) & PTE64_L4_MASK) >> PTE64_L4_SHIFT) 275 1.31.2.2 christos 276 1.31.2.2 christos #define CR3_FRAME_64BIT PG_FRAME 277 1.31.2.2 christos 278 1.31.2.2 christos typedef uint64_t pte_64bit_t; 279 1.31.2.2 christos 280 1.31.2.2 christos static inline bool 281 1.31.2.2 christos x86_gva_64bit_canonical(gvaddr_t gva) 282 1.31.2.2 christos { 283 1.31.2.2 christos /* Bits 63:47 must have the same value. */ 284 1.31.2.2 christos #define SIGN_EXTEND 0xffff800000000000ULL 285 1.31.2.2 christos return (gva & SIGN_EXTEND) == 0 || (gva & SIGN_EXTEND) == SIGN_EXTEND; 286 1.31.2.2 christos } 287 1.31.2.2 christos 288 1.31.2.2 christos static int 289 1.31.2.2 christos x86_gva_to_gpa_64bit(struct nvmm_machine *mach, uint64_t cr3, 290 1.31.2.2 christos gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 291 1.31.2.2 christos { 292 1.31.2.2 christos gpaddr_t L4gpa, L3gpa, L2gpa, L1gpa; 293 1.31.2.2 christos uintptr_t L4hva, L3hva, L2hva, L1hva; 294 1.31.2.2 christos pte_64bit_t *pdir, pte; 295 1.31.2.2 christos nvmm_prot_t pageprot; 296 1.31.2.2 christos 297 1.31.2.2 christos /* We begin with an RWXU access. */ 298 1.31.2.2 christos *prot = NVMM_PROT_ALL; 299 1.31.2.2 christos 300 1.31.2.2 christos if (!x86_gva_64bit_canonical(gva)) 301 1.31.2.2 christos return -1; 302 1.31.2.2 christos 303 1.31.2.2 christos /* Parse L4. */ 304 1.31.2.2 christos L4gpa = (cr3 & CR3_FRAME_64BIT); 305 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L4gpa, &L4hva, &pageprot) == -1) 306 1.31.2.2 christos return -1; 307 1.31.2.2 christos pdir = (pte_64bit_t *)L4hva; 308 1.31.2.2 christos pte = pdir[pte64_l4idx(gva)]; 309 1.31.2.2 christos if ((pte & PG_V) == 0) 310 1.31.2.2 christos return -1; 311 1.31.2.2 christos if ((pte & PG_u) == 0) 312 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 313 1.31.2.2 christos if ((pte & PG_KW) == 0) 314 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 315 1.31.2.2 christos if (pte & PG_NX) 316 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 317 1.31.2.2 christos if (pte & PG_PS) 318 1.31.2.2 christos return -1; 319 1.31.2.2 christos 320 1.31.2.2 christos /* Parse L3. */ 321 1.31.2.2 christos L3gpa = (pte & PG_FRAME); 322 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 323 1.31.2.2 christos return -1; 324 1.31.2.2 christos pdir = (pte_64bit_t *)L3hva; 325 1.31.2.2 christos pte = pdir[pte64_l3idx(gva)]; 326 1.31.2.2 christos if ((pte & PG_V) == 0) 327 1.31.2.2 christos return -1; 328 1.31.2.2 christos if ((pte & PG_u) == 0) 329 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 330 1.31.2.2 christos if ((pte & PG_KW) == 0) 331 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 332 1.31.2.2 christos if (pte & PG_NX) 333 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 334 1.31.2.2 christos if (pte & PG_PS) { 335 1.31.2.2 christos *gpa = (pte & PTE64_L3_FRAME); 336 1.31.2.2 christos *gpa = *gpa + (gva & (PTE64_L2_MASK|PTE64_L1_MASK)); 337 1.31.2.2 christos return 0; 338 1.31.2.2 christos } 339 1.31.2.2 christos 340 1.31.2.2 christos /* Parse L2. */ 341 1.31.2.2 christos L2gpa = (pte & PG_FRAME); 342 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 343 1.31.2.2 christos return -1; 344 1.31.2.2 christos pdir = (pte_64bit_t *)L2hva; 345 1.31.2.2 christos pte = pdir[pte64_l2idx(gva)]; 346 1.31.2.2 christos if ((pte & PG_V) == 0) 347 1.31.2.2 christos return -1; 348 1.31.2.2 christos if ((pte & PG_u) == 0) 349 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 350 1.31.2.2 christos if ((pte & PG_KW) == 0) 351 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 352 1.31.2.2 christos if (pte & PG_NX) 353 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 354 1.31.2.2 christos if (pte & PG_PS) { 355 1.31.2.2 christos *gpa = (pte & PTE64_L2_FRAME); 356 1.31.2.2 christos *gpa = *gpa + (gva & PTE64_L1_MASK); 357 1.31.2.2 christos return 0; 358 1.31.2.2 christos } 359 1.31.2.2 christos 360 1.31.2.2 christos /* Parse L1. */ 361 1.31.2.2 christos L1gpa = (pte & PG_FRAME); 362 1.31.2.2 christos if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 363 1.31.2.2 christos return -1; 364 1.31.2.2 christos pdir = (pte_64bit_t *)L1hva; 365 1.31.2.2 christos pte = pdir[pte64_l1idx(gva)]; 366 1.31.2.2 christos if ((pte & PG_V) == 0) 367 1.31.2.2 christos return -1; 368 1.31.2.2 christos if ((pte & PG_u) == 0) 369 1.31.2.2 christos *prot &= ~NVMM_PROT_USER; 370 1.31.2.2 christos if ((pte & PG_KW) == 0) 371 1.31.2.2 christos *prot &= ~NVMM_PROT_WRITE; 372 1.31.2.2 christos if (pte & PG_NX) 373 1.31.2.2 christos *prot &= ~NVMM_PROT_EXEC; 374 1.31.2.2 christos if (pte & PG_PS) 375 1.31.2.2 christos return -1; 376 1.31.2.2 christos 377 1.31.2.2 christos *gpa = (pte & PG_FRAME); 378 1.31.2.2 christos return 0; 379 1.31.2.2 christos } 380 1.31.2.2 christos 381 1.31.2.2 christos static inline int 382 1.31.2.2 christos x86_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_x64_state *state, 383 1.31.2.2 christos gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 384 1.31.2.2 christos { 385 1.31.2.2 christos bool is_pae, is_lng, has_pse; 386 1.31.2.2 christos uint64_t cr3; 387 1.31.2.2 christos size_t off; 388 1.31.2.2 christos int ret; 389 1.31.2.2 christos 390 1.31.2.2 christos if ((state->crs[NVMM_X64_CR_CR0] & CR0_PG) == 0) { 391 1.31.2.2 christos /* No paging. */ 392 1.31.2.2 christos *prot = NVMM_PROT_ALL; 393 1.31.2.2 christos *gpa = gva; 394 1.31.2.2 christos return 0; 395 1.31.2.2 christos } 396 1.31.2.2 christos 397 1.31.2.2 christos off = (gva & PAGE_MASK); 398 1.31.2.2 christos gva &= ~PAGE_MASK; 399 1.31.2.2 christos 400 1.31.2.2 christos is_pae = (state->crs[NVMM_X64_CR_CR4] & CR4_PAE) != 0; 401 1.31.2.2 christos is_lng = (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 402 1.31.2.2 christos has_pse = (state->crs[NVMM_X64_CR_CR4] & CR4_PSE) != 0; 403 1.31.2.2 christos cr3 = state->crs[NVMM_X64_CR_CR3]; 404 1.31.2.2 christos 405 1.31.2.2 christos if (is_pae && is_lng) { 406 1.31.2.2 christos /* 64bit */ 407 1.31.2.2 christos ret = x86_gva_to_gpa_64bit(mach, cr3, gva, gpa, prot); 408 1.31.2.2 christos } else if (is_pae && !is_lng) { 409 1.31.2.2 christos /* 32bit PAE */ 410 1.31.2.2 christos ret = x86_gva_to_gpa_32bit_pae(mach, cr3, gva, gpa, prot); 411 1.31.2.2 christos } else if (!is_pae && !is_lng) { 412 1.31.2.2 christos /* 32bit */ 413 1.31.2.2 christos ret = x86_gva_to_gpa_32bit(mach, cr3, gva, gpa, has_pse, prot); 414 1.31.2.2 christos } else { 415 1.31.2.2 christos ret = -1; 416 1.31.2.2 christos } 417 1.31.2.2 christos 418 1.31.2.2 christos if (ret == -1) { 419 1.31.2.2 christos errno = EFAULT; 420 1.31.2.2 christos } 421 1.31.2.2 christos 422 1.31.2.2 christos *gpa = *gpa + off; 423 1.31.2.2 christos 424 1.31.2.2 christos return ret; 425 1.31.2.2 christos } 426 1.31.2.2 christos 427 1.31.2.2 christos int 428 1.31.2.2 christos nvmm_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 429 1.31.2.2 christos gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 430 1.31.2.2 christos { 431 1.31.2.2 christos struct nvmm_x64_state *state = vcpu->state; 432 1.31.2.2 christos int ret; 433 1.31.2.2 christos 434 1.31.2.2 christos ret = nvmm_vcpu_getstate(mach, vcpu, 435 1.31.2.2 christos NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 436 1.31.2.2 christos if (ret == -1) 437 1.31.2.2 christos return -1; 438 1.31.2.2 christos 439 1.31.2.2 christos return x86_gva_to_gpa(mach, state, gva, gpa, prot); 440 1.31.2.2 christos } 441 1.31.2.2 christos 442 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 443 1.31.2.2 christos 444 1.31.2.2 christos static inline bool 445 1.31.2.2 christos is_long_mode(struct nvmm_x64_state *state) 446 1.31.2.2 christos { 447 1.31.2.2 christos return (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 448 1.31.2.2 christos } 449 1.31.2.2 christos 450 1.31.2.2 christos static inline bool 451 1.31.2.2 christos is_64bit(struct nvmm_x64_state *state) 452 1.31.2.2 christos { 453 1.31.2.2 christos return (state->segs[NVMM_X64_SEG_CS].attrib.l != 0); 454 1.31.2.2 christos } 455 1.31.2.2 christos 456 1.31.2.2 christos static inline bool 457 1.31.2.2 christos is_32bit(struct nvmm_x64_state *state) 458 1.31.2.2 christos { 459 1.31.2.2 christos return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 460 1.31.2.2 christos (state->segs[NVMM_X64_SEG_CS].attrib.def == 1); 461 1.31.2.2 christos } 462 1.31.2.2 christos 463 1.31.2.2 christos static inline bool 464 1.31.2.2 christos is_16bit(struct nvmm_x64_state *state) 465 1.31.2.2 christos { 466 1.31.2.2 christos return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 467 1.31.2.2 christos (state->segs[NVMM_X64_SEG_CS].attrib.def == 0); 468 1.31.2.2 christos } 469 1.31.2.2 christos 470 1.31.2.2 christos static int 471 1.31.2.2 christos segment_check(struct nvmm_x64_state_seg *seg, gvaddr_t gva, size_t size) 472 1.31.2.2 christos { 473 1.31.2.2 christos uint64_t limit; 474 1.31.2.2 christos 475 1.31.2.2 christos /* 476 1.31.2.2 christos * This is incomplete. We should check topdown, etc, really that's 477 1.31.2.2 christos * tiring. 478 1.31.2.2 christos */ 479 1.31.2.2 christos if (__predict_false(!seg->attrib.p)) { 480 1.31.2.2 christos goto error; 481 1.31.2.2 christos } 482 1.31.2.2 christos 483 1.31.2.2 christos limit = (uint64_t)seg->limit + 1; 484 1.31.2.2 christos if (__predict_true(seg->attrib.g)) { 485 1.31.2.2 christos limit *= PAGE_SIZE; 486 1.31.2.2 christos } 487 1.31.2.2 christos 488 1.31.2.2 christos if (__predict_false(gva + size > limit)) { 489 1.31.2.2 christos goto error; 490 1.31.2.2 christos } 491 1.31.2.2 christos 492 1.31.2.2 christos return 0; 493 1.31.2.2 christos 494 1.31.2.2 christos error: 495 1.31.2.2 christos errno = EFAULT; 496 1.31.2.2 christos return -1; 497 1.31.2.2 christos } 498 1.31.2.2 christos 499 1.31.2.2 christos static inline void 500 1.31.2.2 christos segment_apply(struct nvmm_x64_state_seg *seg, gvaddr_t *gva) 501 1.31.2.2 christos { 502 1.31.2.2 christos *gva += seg->base; 503 1.31.2.2 christos } 504 1.31.2.2 christos 505 1.31.2.2 christos static inline uint64_t 506 1.31.2.2 christos size_to_mask(size_t size) 507 1.31.2.2 christos { 508 1.31.2.2 christos switch (size) { 509 1.31.2.2 christos case 1: 510 1.31.2.2 christos return 0x00000000000000FF; 511 1.31.2.2 christos case 2: 512 1.31.2.2 christos return 0x000000000000FFFF; 513 1.31.2.2 christos case 4: 514 1.31.2.2 christos return 0x00000000FFFFFFFF; 515 1.31.2.2 christos case 8: 516 1.31.2.2 christos default: 517 1.31.2.2 christos return 0xFFFFFFFFFFFFFFFF; 518 1.31.2.2 christos } 519 1.31.2.2 christos } 520 1.31.2.2 christos 521 1.31.2.2 christos static uint64_t 522 1.31.2.2 christos rep_get_cnt(struct nvmm_x64_state *state, size_t adsize) 523 1.31.2.2 christos { 524 1.31.2.2 christos uint64_t mask, cnt; 525 1.31.2.2 christos 526 1.31.2.2 christos mask = size_to_mask(adsize); 527 1.31.2.2 christos cnt = state->gprs[NVMM_X64_GPR_RCX] & mask; 528 1.31.2.2 christos 529 1.31.2.2 christos return cnt; 530 1.31.2.2 christos } 531 1.31.2.2 christos 532 1.31.2.2 christos static void 533 1.31.2.2 christos rep_set_cnt(struct nvmm_x64_state *state, size_t adsize, uint64_t cnt) 534 1.31.2.2 christos { 535 1.31.2.2 christos uint64_t mask; 536 1.31.2.2 christos 537 1.31.2.2 christos /* XXX: should we zero-extend? */ 538 1.31.2.2 christos mask = size_to_mask(adsize); 539 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RCX] &= ~mask; 540 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RCX] |= cnt; 541 1.31.2.2 christos } 542 1.31.2.2 christos 543 1.31.2.2 christos static int 544 1.31.2.2 christos read_guest_memory(struct nvmm_machine *mach, struct nvmm_x64_state *state, 545 1.31.2.2 christos gvaddr_t gva, uint8_t *data, size_t size) 546 1.31.2.2 christos { 547 1.31.2.2 christos struct nvmm_mem mem; 548 1.31.2.2 christos nvmm_prot_t prot; 549 1.31.2.2 christos gpaddr_t gpa; 550 1.31.2.2 christos uintptr_t hva; 551 1.31.2.2 christos bool is_mmio; 552 1.31.2.2 christos int ret, remain; 553 1.31.2.2 christos 554 1.31.2.2 christos ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 555 1.31.2.2 christos if (__predict_false(ret == -1)) { 556 1.31.2.2 christos return -1; 557 1.31.2.2 christos } 558 1.31.2.2 christos if (__predict_false(!(prot & NVMM_PROT_READ))) { 559 1.31.2.2 christos errno = EFAULT; 560 1.31.2.2 christos return -1; 561 1.31.2.2 christos } 562 1.31.2.2 christos 563 1.31.2.2 christos if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 564 1.31.2.2 christos remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 565 1.31.2.2 christos } else { 566 1.31.2.2 christos remain = 0; 567 1.31.2.2 christos } 568 1.31.2.2 christos size -= remain; 569 1.31.2.2 christos 570 1.31.2.2 christos ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 571 1.31.2.2 christos is_mmio = (ret == -1); 572 1.31.2.2 christos 573 1.31.2.2 christos if (is_mmio) { 574 1.31.2.2 christos mem.data = data; 575 1.31.2.2 christos mem.gpa = gpa; 576 1.31.2.2 christos mem.write = false; 577 1.31.2.2 christos mem.size = size; 578 1.31.2.2 christos (*mach->cbs.mem)(&mem); 579 1.31.2.2 christos } else { 580 1.31.2.2 christos if (__predict_false(!(prot & NVMM_PROT_READ))) { 581 1.31.2.2 christos errno = EFAULT; 582 1.31.2.2 christos return -1; 583 1.31.2.2 christos } 584 1.31.2.2 christos memcpy(data, (uint8_t *)hva, size); 585 1.31.2.2 christos } 586 1.31.2.2 christos 587 1.31.2.2 christos if (remain > 0) { 588 1.31.2.2 christos ret = read_guest_memory(mach, state, gva + size, 589 1.31.2.2 christos data + size, remain); 590 1.31.2.2 christos } else { 591 1.31.2.2 christos ret = 0; 592 1.31.2.2 christos } 593 1.31.2.2 christos 594 1.31.2.2 christos return ret; 595 1.31.2.2 christos } 596 1.31.2.2 christos 597 1.31.2.2 christos static int 598 1.31.2.2 christos write_guest_memory(struct nvmm_machine *mach, struct nvmm_x64_state *state, 599 1.31.2.2 christos gvaddr_t gva, uint8_t *data, size_t size) 600 1.31.2.2 christos { 601 1.31.2.2 christos struct nvmm_mem mem; 602 1.31.2.2 christos nvmm_prot_t prot; 603 1.31.2.2 christos gpaddr_t gpa; 604 1.31.2.2 christos uintptr_t hva; 605 1.31.2.2 christos bool is_mmio; 606 1.31.2.2 christos int ret, remain; 607 1.31.2.2 christos 608 1.31.2.2 christos ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 609 1.31.2.2 christos if (__predict_false(ret == -1)) { 610 1.31.2.2 christos return -1; 611 1.31.2.2 christos } 612 1.31.2.2 christos if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 613 1.31.2.2 christos errno = EFAULT; 614 1.31.2.2 christos return -1; 615 1.31.2.2 christos } 616 1.31.2.2 christos 617 1.31.2.2 christos if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 618 1.31.2.2 christos remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 619 1.31.2.2 christos } else { 620 1.31.2.2 christos remain = 0; 621 1.31.2.2 christos } 622 1.31.2.2 christos size -= remain; 623 1.31.2.2 christos 624 1.31.2.2 christos ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 625 1.31.2.2 christos is_mmio = (ret == -1); 626 1.31.2.2 christos 627 1.31.2.2 christos if (is_mmio) { 628 1.31.2.2 christos mem.data = data; 629 1.31.2.2 christos mem.gpa = gpa; 630 1.31.2.2 christos mem.write = true; 631 1.31.2.2 christos mem.size = size; 632 1.31.2.2 christos (*mach->cbs.mem)(&mem); 633 1.31.2.2 christos } else { 634 1.31.2.2 christos if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 635 1.31.2.2 christos errno = EFAULT; 636 1.31.2.2 christos return -1; 637 1.31.2.2 christos } 638 1.31.2.2 christos memcpy((uint8_t *)hva, data, size); 639 1.31.2.2 christos } 640 1.31.2.2 christos 641 1.31.2.2 christos if (remain > 0) { 642 1.31.2.2 christos ret = write_guest_memory(mach, state, gva + size, 643 1.31.2.2 christos data + size, remain); 644 1.31.2.2 christos } else { 645 1.31.2.2 christos ret = 0; 646 1.31.2.2 christos } 647 1.31.2.2 christos 648 1.31.2.2 christos return ret; 649 1.31.2.2 christos } 650 1.31.2.2 christos 651 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 652 1.31.2.2 christos 653 1.31.2.2 christos static int fetch_segment(struct nvmm_machine *, struct nvmm_x64_state *); 654 1.31.2.2 christos 655 1.31.2.2 christos #define NVMM_IO_BATCH_SIZE 32 656 1.31.2.2 christos 657 1.31.2.2 christos static int 658 1.31.2.2 christos assist_io_batch(struct nvmm_machine *mach, struct nvmm_x64_state *state, 659 1.31.2.2 christos struct nvmm_io *io, gvaddr_t gva, uint64_t cnt) 660 1.31.2.2 christos { 661 1.31.2.2 christos uint8_t iobuf[NVMM_IO_BATCH_SIZE]; 662 1.31.2.2 christos size_t i, iosize, iocnt; 663 1.31.2.2 christos int ret; 664 1.31.2.2 christos 665 1.31.2.2 christos cnt = MIN(cnt, NVMM_IO_BATCH_SIZE); 666 1.31.2.2 christos iosize = MIN(io->size * cnt, NVMM_IO_BATCH_SIZE); 667 1.31.2.2 christos iocnt = iosize / io->size; 668 1.31.2.2 christos 669 1.31.2.2 christos io->data = iobuf; 670 1.31.2.2 christos 671 1.31.2.2 christos if (!io->in) { 672 1.31.2.2 christos ret = read_guest_memory(mach, state, gva, iobuf, iosize); 673 1.31.2.2 christos if (ret == -1) 674 1.31.2.2 christos return -1; 675 1.31.2.2 christos } 676 1.31.2.2 christos 677 1.31.2.2 christos for (i = 0; i < iocnt; i++) { 678 1.31.2.2 christos (*mach->cbs.io)(io); 679 1.31.2.2 christos io->data += io->size; 680 1.31.2.2 christos } 681 1.31.2.2 christos 682 1.31.2.2 christos if (io->in) { 683 1.31.2.2 christos ret = write_guest_memory(mach, state, gva, iobuf, iosize); 684 1.31.2.2 christos if (ret == -1) 685 1.31.2.2 christos return -1; 686 1.31.2.2 christos } 687 1.31.2.2 christos 688 1.31.2.2 christos return iocnt; 689 1.31.2.2 christos } 690 1.31.2.2 christos 691 1.31.2.2 christos int 692 1.31.2.2 christos nvmm_assist_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 693 1.31.2.2 christos { 694 1.31.2.2 christos struct nvmm_x64_state *state = vcpu->state; 695 1.31.2.2 christos struct nvmm_exit *exit = vcpu->exit; 696 1.31.2.2 christos struct nvmm_io io; 697 1.31.2.2 christos uint64_t cnt = 0; /* GCC */ 698 1.31.2.2 christos uint8_t iobuf[8]; 699 1.31.2.2 christos int iocnt = 1; 700 1.31.2.2 christos gvaddr_t gva = 0; /* GCC */ 701 1.31.2.2 christos int reg = 0; /* GCC */ 702 1.31.2.2 christos int ret, seg; 703 1.31.2.2 christos bool psld = false; 704 1.31.2.2 christos 705 1.31.2.2 christos if (__predict_false(exit->reason != NVMM_EXIT_IO)) { 706 1.31.2.2 christos errno = EINVAL; 707 1.31.2.2 christos return -1; 708 1.31.2.2 christos } 709 1.31.2.2 christos 710 1.31.2.2 christos io.port = exit->u.io.port; 711 1.31.2.2 christos io.in = (exit->u.io.type == NVMM_EXIT_IO_IN); 712 1.31.2.2 christos io.size = exit->u.io.operand_size; 713 1.31.2.2 christos io.data = iobuf; 714 1.31.2.2 christos 715 1.31.2.2 christos ret = nvmm_vcpu_getstate(mach, vcpu, 716 1.31.2.2 christos NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 717 1.31.2.2 christos NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 718 1.31.2.2 christos if (ret == -1) 719 1.31.2.2 christos return -1; 720 1.31.2.2 christos 721 1.31.2.2 christos if (exit->u.io.rep) { 722 1.31.2.2 christos cnt = rep_get_cnt(state, exit->u.io.address_size); 723 1.31.2.2 christos if (__predict_false(cnt == 0)) { 724 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 725 1.31.2.2 christos goto out; 726 1.31.2.2 christos } 727 1.31.2.2 christos } 728 1.31.2.2 christos 729 1.31.2.2 christos if (__predict_false(state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_D)) { 730 1.31.2.2 christos psld = true; 731 1.31.2.2 christos } 732 1.31.2.2 christos 733 1.31.2.2 christos /* 734 1.31.2.2 christos * Determine GVA. 735 1.31.2.2 christos */ 736 1.31.2.2 christos if (exit->u.io.str) { 737 1.31.2.2 christos if (io.in) { 738 1.31.2.2 christos reg = NVMM_X64_GPR_RDI; 739 1.31.2.2 christos } else { 740 1.31.2.2 christos reg = NVMM_X64_GPR_RSI; 741 1.31.2.2 christos } 742 1.31.2.2 christos 743 1.31.2.2 christos gva = state->gprs[reg]; 744 1.31.2.2 christos gva &= size_to_mask(exit->u.io.address_size); 745 1.31.2.2 christos 746 1.31.2.2 christos if (exit->u.io.seg != -1) { 747 1.31.2.2 christos seg = exit->u.io.seg; 748 1.31.2.2 christos } else { 749 1.31.2.2 christos if (io.in) { 750 1.31.2.2 christos seg = NVMM_X64_SEG_ES; 751 1.31.2.2 christos } else { 752 1.31.2.2 christos seg = fetch_segment(mach, state); 753 1.31.2.2 christos if (seg == -1) 754 1.31.2.2 christos return -1; 755 1.31.2.2 christos } 756 1.31.2.2 christos } 757 1.31.2.2 christos 758 1.31.2.2 christos if (__predict_true(is_long_mode(state))) { 759 1.31.2.2 christos if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 760 1.31.2.2 christos segment_apply(&state->segs[seg], &gva); 761 1.31.2.2 christos } 762 1.31.2.2 christos } else { 763 1.31.2.2 christos ret = segment_check(&state->segs[seg], gva, io.size); 764 1.31.2.2 christos if (ret == -1) 765 1.31.2.2 christos return -1; 766 1.31.2.2 christos segment_apply(&state->segs[seg], &gva); 767 1.31.2.2 christos } 768 1.31.2.2 christos 769 1.31.2.2 christos if (exit->u.io.rep && !psld) { 770 1.31.2.2 christos iocnt = assist_io_batch(mach, state, &io, gva, cnt); 771 1.31.2.2 christos if (iocnt == -1) 772 1.31.2.2 christos return -1; 773 1.31.2.2 christos goto done; 774 1.31.2.2 christos } 775 1.31.2.2 christos } 776 1.31.2.2 christos 777 1.31.2.2 christos if (!io.in) { 778 1.31.2.2 christos if (!exit->u.io.str) { 779 1.31.2.2 christos memcpy(io.data, &state->gprs[NVMM_X64_GPR_RAX], io.size); 780 1.31.2.2 christos } else { 781 1.31.2.2 christos ret = read_guest_memory(mach, state, gva, io.data, 782 1.31.2.2 christos io.size); 783 1.31.2.2 christos if (ret == -1) 784 1.31.2.2 christos return -1; 785 1.31.2.2 christos } 786 1.31.2.2 christos } 787 1.31.2.2 christos 788 1.31.2.2 christos (*mach->cbs.io)(&io); 789 1.31.2.2 christos 790 1.31.2.2 christos if (io.in) { 791 1.31.2.2 christos if (!exit->u.io.str) { 792 1.31.2.2 christos memcpy(&state->gprs[NVMM_X64_GPR_RAX], io.data, io.size); 793 1.31.2.2 christos if (io.size == 4) { 794 1.31.2.2 christos /* Zero-extend to 64 bits. */ 795 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RAX] &= size_to_mask(4); 796 1.31.2.2 christos } 797 1.31.2.2 christos } else { 798 1.31.2.2 christos ret = write_guest_memory(mach, state, gva, io.data, 799 1.31.2.2 christos io.size); 800 1.31.2.2 christos if (ret == -1) 801 1.31.2.2 christos return -1; 802 1.31.2.2 christos } 803 1.31.2.2 christos } 804 1.31.2.2 christos 805 1.31.2.2 christos done: 806 1.31.2.2 christos if (exit->u.io.str) { 807 1.31.2.2 christos if (__predict_false(psld)) { 808 1.31.2.2 christos state->gprs[reg] -= iocnt * io.size; 809 1.31.2.2 christos } else { 810 1.31.2.2 christos state->gprs[reg] += iocnt * io.size; 811 1.31.2.2 christos } 812 1.31.2.2 christos } 813 1.31.2.2 christos 814 1.31.2.2 christos if (exit->u.io.rep) { 815 1.31.2.2 christos cnt -= iocnt; 816 1.31.2.2 christos rep_set_cnt(state, exit->u.io.address_size, cnt); 817 1.31.2.2 christos if (cnt == 0) { 818 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 819 1.31.2.2 christos } 820 1.31.2.2 christos } else { 821 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 822 1.31.2.2 christos } 823 1.31.2.2 christos 824 1.31.2.2 christos out: 825 1.31.2.2 christos ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 826 1.31.2.2 christos if (ret == -1) 827 1.31.2.2 christos return -1; 828 1.31.2.2 christos 829 1.31.2.2 christos return 0; 830 1.31.2.2 christos } 831 1.31.2.2 christos 832 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 833 1.31.2.2 christos 834 1.31.2.2 christos struct x86_emul { 835 1.31.2.2 christos bool read; 836 1.31.2.2 christos bool notouch; 837 1.31.2.2 christos void (*func)(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 838 1.31.2.2 christos }; 839 1.31.2.2 christos 840 1.31.2.2 christos static void x86_func_or(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 841 1.31.2.2 christos static void x86_func_and(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 842 1.31.2.2 christos static void x86_func_sub(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 843 1.31.2.2 christos static void x86_func_xor(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 844 1.31.2.2 christos static void x86_func_cmp(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 845 1.31.2.2 christos static void x86_func_test(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 846 1.31.2.2 christos static void x86_func_mov(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 847 1.31.2.2 christos static void x86_func_stos(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 848 1.31.2.2 christos static void x86_func_lods(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 849 1.31.2.2 christos static void x86_func_movs(struct nvmm_machine *, struct nvmm_mem *, uint64_t *); 850 1.31.2.2 christos 851 1.31.2.2 christos static const struct x86_emul x86_emul_or = { 852 1.31.2.2 christos .read = true, 853 1.31.2.2 christos .func = x86_func_or 854 1.31.2.2 christos }; 855 1.31.2.2 christos 856 1.31.2.2 christos static const struct x86_emul x86_emul_and = { 857 1.31.2.2 christos .read = true, 858 1.31.2.2 christos .func = x86_func_and 859 1.31.2.2 christos }; 860 1.31.2.2 christos 861 1.31.2.2 christos static const struct x86_emul x86_emul_sub = { 862 1.31.2.2 christos .read = true, 863 1.31.2.2 christos .func = x86_func_sub 864 1.31.2.2 christos }; 865 1.31.2.2 christos 866 1.31.2.2 christos static const struct x86_emul x86_emul_xor = { 867 1.31.2.2 christos .read = true, 868 1.31.2.2 christos .func = x86_func_xor 869 1.31.2.2 christos }; 870 1.31.2.2 christos 871 1.31.2.2 christos static const struct x86_emul x86_emul_cmp = { 872 1.31.2.2 christos .notouch = true, 873 1.31.2.2 christos .func = x86_func_cmp 874 1.31.2.2 christos }; 875 1.31.2.2 christos 876 1.31.2.2 christos static const struct x86_emul x86_emul_test = { 877 1.31.2.2 christos .notouch = true, 878 1.31.2.2 christos .func = x86_func_test 879 1.31.2.2 christos }; 880 1.31.2.2 christos 881 1.31.2.2 christos static const struct x86_emul x86_emul_mov = { 882 1.31.2.2 christos .func = x86_func_mov 883 1.31.2.2 christos }; 884 1.31.2.2 christos 885 1.31.2.2 christos static const struct x86_emul x86_emul_stos = { 886 1.31.2.2 christos .func = x86_func_stos 887 1.31.2.2 christos }; 888 1.31.2.2 christos 889 1.31.2.2 christos static const struct x86_emul x86_emul_lods = { 890 1.31.2.2 christos .func = x86_func_lods 891 1.31.2.2 christos }; 892 1.31.2.2 christos 893 1.31.2.2 christos static const struct x86_emul x86_emul_movs = { 894 1.31.2.2 christos .func = x86_func_movs 895 1.31.2.2 christos }; 896 1.31.2.2 christos 897 1.31.2.2 christos /* Legacy prefixes. */ 898 1.31.2.2 christos #define LEG_LOCK 0xF0 899 1.31.2.2 christos #define LEG_REPN 0xF2 900 1.31.2.2 christos #define LEG_REP 0xF3 901 1.31.2.2 christos #define LEG_OVR_CS 0x2E 902 1.31.2.2 christos #define LEG_OVR_SS 0x36 903 1.31.2.2 christos #define LEG_OVR_DS 0x3E 904 1.31.2.2 christos #define LEG_OVR_ES 0x26 905 1.31.2.2 christos #define LEG_OVR_FS 0x64 906 1.31.2.2 christos #define LEG_OVR_GS 0x65 907 1.31.2.2 christos #define LEG_OPR_OVR 0x66 908 1.31.2.2 christos #define LEG_ADR_OVR 0x67 909 1.31.2.2 christos 910 1.31.2.2 christos struct x86_legpref { 911 1.31.2.2 christos bool opr_ovr:1; 912 1.31.2.2 christos bool adr_ovr:1; 913 1.31.2.2 christos bool rep:1; 914 1.31.2.2 christos bool repn:1; 915 1.31.2.2 christos int8_t seg; 916 1.31.2.2 christos }; 917 1.31.2.2 christos 918 1.31.2.2 christos struct x86_rexpref { 919 1.31.2.2 christos bool b:1; 920 1.31.2.2 christos bool x:1; 921 1.31.2.2 christos bool r:1; 922 1.31.2.2 christos bool w:1; 923 1.31.2.2 christos bool present:1; 924 1.31.2.2 christos }; 925 1.31.2.2 christos 926 1.31.2.2 christos struct x86_reg { 927 1.31.2.2 christos int num; /* NVMM GPR state index */ 928 1.31.2.2 christos uint64_t mask; 929 1.31.2.2 christos }; 930 1.31.2.2 christos 931 1.31.2.2 christos enum x86_disp_type { 932 1.31.2.2 christos DISP_NONE, 933 1.31.2.2 christos DISP_0, 934 1.31.2.2 christos DISP_1, 935 1.31.2.2 christos DISP_4 936 1.31.2.2 christos }; 937 1.31.2.2 christos 938 1.31.2.2 christos struct x86_disp { 939 1.31.2.2 christos enum x86_disp_type type; 940 1.31.2.2 christos uint64_t data; /* 4 bytes, but can be sign-extended */ 941 1.31.2.2 christos }; 942 1.31.2.2 christos 943 1.31.2.2 christos enum REGMODRM__Mod { 944 1.31.2.2 christos MOD_DIS0, /* also, register indirect */ 945 1.31.2.2 christos MOD_DIS1, 946 1.31.2.2 christos MOD_DIS4, 947 1.31.2.2 christos MOD_REG 948 1.31.2.2 christos }; 949 1.31.2.2 christos 950 1.31.2.2 christos enum REGMODRM__Reg { 951 1.31.2.2 christos REG_000, /* these fields are indexes to the register map */ 952 1.31.2.2 christos REG_001, 953 1.31.2.2 christos REG_010, 954 1.31.2.2 christos REG_011, 955 1.31.2.2 christos REG_100, 956 1.31.2.2 christos REG_101, 957 1.31.2.2 christos REG_110, 958 1.31.2.2 christos REG_111 959 1.31.2.2 christos }; 960 1.31.2.2 christos 961 1.31.2.2 christos enum REGMODRM__Rm { 962 1.31.2.2 christos RM_000, /* reg */ 963 1.31.2.2 christos RM_001, /* reg */ 964 1.31.2.2 christos RM_010, /* reg */ 965 1.31.2.2 christos RM_011, /* reg */ 966 1.31.2.2 christos RM_RSP_SIB, /* reg or SIB, depending on the MOD */ 967 1.31.2.2 christos RM_RBP_DISP32, /* reg or displacement-only (= RIP-relative on amd64) */ 968 1.31.2.2 christos RM_110, 969 1.31.2.2 christos RM_111 970 1.31.2.2 christos }; 971 1.31.2.2 christos 972 1.31.2.2 christos struct x86_regmodrm { 973 1.31.2.2 christos uint8_t mod:2; 974 1.31.2.2 christos uint8_t reg:3; 975 1.31.2.2 christos uint8_t rm:3; 976 1.31.2.2 christos }; 977 1.31.2.2 christos 978 1.31.2.2 christos struct x86_immediate { 979 1.31.2.2 christos uint64_t data; 980 1.31.2.2 christos }; 981 1.31.2.2 christos 982 1.31.2.2 christos struct x86_sib { 983 1.31.2.2 christos uint8_t scale; 984 1.31.2.2 christos const struct x86_reg *idx; 985 1.31.2.2 christos const struct x86_reg *bas; 986 1.31.2.2 christos }; 987 1.31.2.2 christos 988 1.31.2.2 christos enum x86_store_type { 989 1.31.2.2 christos STORE_NONE, 990 1.31.2.2 christos STORE_REG, 991 1.31.2.2 christos STORE_IMM, 992 1.31.2.2 christos STORE_SIB, 993 1.31.2.2 christos STORE_DMO 994 1.31.2.2 christos }; 995 1.31.2.2 christos 996 1.31.2.2 christos struct x86_store { 997 1.31.2.2 christos enum x86_store_type type; 998 1.31.2.2 christos union { 999 1.31.2.2 christos const struct x86_reg *reg; 1000 1.31.2.2 christos struct x86_immediate imm; 1001 1.31.2.2 christos struct x86_sib sib; 1002 1.31.2.2 christos uint64_t dmo; 1003 1.31.2.2 christos } u; 1004 1.31.2.2 christos struct x86_disp disp; 1005 1.31.2.2 christos int hardseg; 1006 1.31.2.2 christos }; 1007 1.31.2.2 christos 1008 1.31.2.2 christos struct x86_instr { 1009 1.31.2.2 christos uint8_t len; 1010 1.31.2.2 christos struct x86_legpref legpref; 1011 1.31.2.2 christos struct x86_rexpref rexpref; 1012 1.31.2.2 christos struct x86_regmodrm regmodrm; 1013 1.31.2.2 christos uint8_t operand_size; 1014 1.31.2.2 christos uint8_t address_size; 1015 1.31.2.2 christos uint64_t zeroextend_mask; 1016 1.31.2.2 christos 1017 1.31.2.2 christos const struct x86_opcode *opcode; 1018 1.31.2.2 christos const struct x86_emul *emul; 1019 1.31.2.2 christos 1020 1.31.2.2 christos struct x86_store src; 1021 1.31.2.2 christos struct x86_store dst; 1022 1.31.2.2 christos struct x86_store *strm; 1023 1.31.2.2 christos }; 1024 1.31.2.2 christos 1025 1.31.2.2 christos struct x86_decode_fsm { 1026 1.31.2.2 christos /* vcpu */ 1027 1.31.2.2 christos bool is64bit; 1028 1.31.2.2 christos bool is32bit; 1029 1.31.2.2 christos bool is16bit; 1030 1.31.2.2 christos 1031 1.31.2.2 christos /* fsm */ 1032 1.31.2.2 christos int (*fn)(struct x86_decode_fsm *, struct x86_instr *); 1033 1.31.2.2 christos uint8_t *buf; 1034 1.31.2.2 christos uint8_t *end; 1035 1.31.2.2 christos }; 1036 1.31.2.2 christos 1037 1.31.2.2 christos struct x86_opcode { 1038 1.31.2.2 christos bool valid:1; 1039 1.31.2.2 christos bool regmodrm:1; 1040 1.31.2.2 christos bool regtorm:1; 1041 1.31.2.2 christos bool dmo:1; 1042 1.31.2.2 christos bool todmo:1; 1043 1.31.2.2 christos bool movs:1; 1044 1.31.2.2 christos bool stos:1; 1045 1.31.2.2 christos bool lods:1; 1046 1.31.2.2 christos bool szoverride:1; 1047 1.31.2.2 christos bool group1:1; 1048 1.31.2.2 christos bool group3:1; 1049 1.31.2.2 christos bool group11:1; 1050 1.31.2.2 christos bool immediate:1; 1051 1.31.2.2 christos uint8_t defsize; 1052 1.31.2.2 christos uint8_t flags; 1053 1.31.2.2 christos const struct x86_emul *emul; 1054 1.31.2.2 christos }; 1055 1.31.2.2 christos 1056 1.31.2.2 christos struct x86_group_entry { 1057 1.31.2.2 christos const struct x86_emul *emul; 1058 1.31.2.2 christos }; 1059 1.31.2.2 christos 1060 1.31.2.2 christos #define OPSIZE_BYTE 0x01 1061 1.31.2.2 christos #define OPSIZE_WORD 0x02 /* 2 bytes */ 1062 1.31.2.2 christos #define OPSIZE_DOUB 0x04 /* 4 bytes */ 1063 1.31.2.2 christos #define OPSIZE_QUAD 0x08 /* 8 bytes */ 1064 1.31.2.2 christos 1065 1.31.2.2 christos #define FLAG_imm8 0x01 1066 1.31.2.2 christos #define FLAG_immz 0x02 1067 1.31.2.2 christos #define FLAG_ze 0x04 1068 1.31.2.2 christos 1069 1.31.2.2 christos static const struct x86_group_entry group1[8] __cacheline_aligned = { 1070 1.31.2.2 christos [1] = { .emul = &x86_emul_or }, 1071 1.31.2.2 christos [4] = { .emul = &x86_emul_and }, 1072 1.31.2.2 christos [6] = { .emul = &x86_emul_xor }, 1073 1.31.2.2 christos [7] = { .emul = &x86_emul_cmp } 1074 1.31.2.2 christos }; 1075 1.31.2.2 christos 1076 1.31.2.2 christos static const struct x86_group_entry group3[8] __cacheline_aligned = { 1077 1.31.2.2 christos [0] = { .emul = &x86_emul_test }, 1078 1.31.2.2 christos [1] = { .emul = &x86_emul_test } 1079 1.31.2.2 christos }; 1080 1.31.2.2 christos 1081 1.31.2.2 christos static const struct x86_group_entry group11[8] __cacheline_aligned = { 1082 1.31.2.2 christos [0] = { .emul = &x86_emul_mov } 1083 1.31.2.2 christos }; 1084 1.31.2.2 christos 1085 1.31.2.2 christos static const struct x86_opcode primary_opcode_table[256] __cacheline_aligned = { 1086 1.31.2.2 christos /* 1087 1.31.2.2 christos * Group1 1088 1.31.2.2 christos */ 1089 1.31.2.2 christos [0x80] = { 1090 1.31.2.2 christos /* Eb, Ib */ 1091 1.31.2.2 christos .valid = true, 1092 1.31.2.2 christos .regmodrm = true, 1093 1.31.2.2 christos .regtorm = true, 1094 1.31.2.2 christos .szoverride = false, 1095 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1096 1.31.2.2 christos .group1 = true, 1097 1.31.2.2 christos .immediate = true, 1098 1.31.2.2 christos .emul = NULL /* group1 */ 1099 1.31.2.2 christos }, 1100 1.31.2.2 christos [0x81] = { 1101 1.31.2.2 christos /* Ev, Iz */ 1102 1.31.2.2 christos .valid = true, 1103 1.31.2.2 christos .regmodrm = true, 1104 1.31.2.2 christos .regtorm = true, 1105 1.31.2.2 christos .szoverride = true, 1106 1.31.2.2 christos .defsize = -1, 1107 1.31.2.2 christos .group1 = true, 1108 1.31.2.2 christos .immediate = true, 1109 1.31.2.2 christos .flags = FLAG_immz, 1110 1.31.2.2 christos .emul = NULL /* group1 */ 1111 1.31.2.2 christos }, 1112 1.31.2.2 christos [0x83] = { 1113 1.31.2.2 christos /* Ev, Ib */ 1114 1.31.2.2 christos .valid = true, 1115 1.31.2.2 christos .regmodrm = true, 1116 1.31.2.2 christos .regtorm = true, 1117 1.31.2.2 christos .szoverride = true, 1118 1.31.2.2 christos .defsize = -1, 1119 1.31.2.2 christos .group1 = true, 1120 1.31.2.2 christos .immediate = true, 1121 1.31.2.2 christos .flags = FLAG_imm8, 1122 1.31.2.2 christos .emul = NULL /* group1 */ 1123 1.31.2.2 christos }, 1124 1.31.2.2 christos 1125 1.31.2.2 christos /* 1126 1.31.2.2 christos * Group3 1127 1.31.2.2 christos */ 1128 1.31.2.2 christos [0xF6] = { 1129 1.31.2.2 christos /* Eb, Ib */ 1130 1.31.2.2 christos .valid = true, 1131 1.31.2.2 christos .regmodrm = true, 1132 1.31.2.2 christos .regtorm = true, 1133 1.31.2.2 christos .szoverride = false, 1134 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1135 1.31.2.2 christos .group3 = true, 1136 1.31.2.2 christos .immediate = true, 1137 1.31.2.2 christos .emul = NULL /* group3 */ 1138 1.31.2.2 christos }, 1139 1.31.2.2 christos [0xF7] = { 1140 1.31.2.2 christos /* Ev, Iz */ 1141 1.31.2.2 christos .valid = true, 1142 1.31.2.2 christos .regmodrm = true, 1143 1.31.2.2 christos .regtorm = true, 1144 1.31.2.2 christos .szoverride = true, 1145 1.31.2.2 christos .defsize = -1, 1146 1.31.2.2 christos .group3 = true, 1147 1.31.2.2 christos .immediate = true, 1148 1.31.2.2 christos .flags = FLAG_immz, 1149 1.31.2.2 christos .emul = NULL /* group3 */ 1150 1.31.2.2 christos }, 1151 1.31.2.2 christos 1152 1.31.2.2 christos /* 1153 1.31.2.2 christos * Group11 1154 1.31.2.2 christos */ 1155 1.31.2.2 christos [0xC6] = { 1156 1.31.2.2 christos /* Eb, Ib */ 1157 1.31.2.2 christos .valid = true, 1158 1.31.2.2 christos .regmodrm = true, 1159 1.31.2.2 christos .regtorm = true, 1160 1.31.2.2 christos .szoverride = false, 1161 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1162 1.31.2.2 christos .group11 = true, 1163 1.31.2.2 christos .immediate = true, 1164 1.31.2.2 christos .emul = NULL /* group11 */ 1165 1.31.2.2 christos }, 1166 1.31.2.2 christos [0xC7] = { 1167 1.31.2.2 christos /* Ev, Iz */ 1168 1.31.2.2 christos .valid = true, 1169 1.31.2.2 christos .regmodrm = true, 1170 1.31.2.2 christos .regtorm = true, 1171 1.31.2.2 christos .szoverride = true, 1172 1.31.2.2 christos .defsize = -1, 1173 1.31.2.2 christos .group11 = true, 1174 1.31.2.2 christos .immediate = true, 1175 1.31.2.2 christos .flags = FLAG_immz, 1176 1.31.2.2 christos .emul = NULL /* group11 */ 1177 1.31.2.2 christos }, 1178 1.31.2.2 christos 1179 1.31.2.2 christos /* 1180 1.31.2.2 christos * OR 1181 1.31.2.2 christos */ 1182 1.31.2.2 christos [0x08] = { 1183 1.31.2.2 christos /* Eb, Gb */ 1184 1.31.2.2 christos .valid = true, 1185 1.31.2.2 christos .regmodrm = true, 1186 1.31.2.2 christos .regtorm = true, 1187 1.31.2.2 christos .szoverride = false, 1188 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1189 1.31.2.2 christos .emul = &x86_emul_or 1190 1.31.2.2 christos }, 1191 1.31.2.2 christos [0x09] = { 1192 1.31.2.2 christos /* Ev, Gv */ 1193 1.31.2.2 christos .valid = true, 1194 1.31.2.2 christos .regmodrm = true, 1195 1.31.2.2 christos .regtorm = true, 1196 1.31.2.2 christos .szoverride = true, 1197 1.31.2.2 christos .defsize = -1, 1198 1.31.2.2 christos .emul = &x86_emul_or 1199 1.31.2.2 christos }, 1200 1.31.2.2 christos [0x0A] = { 1201 1.31.2.2 christos /* Gb, Eb */ 1202 1.31.2.2 christos .valid = true, 1203 1.31.2.2 christos .regmodrm = true, 1204 1.31.2.2 christos .regtorm = false, 1205 1.31.2.2 christos .szoverride = false, 1206 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1207 1.31.2.2 christos .emul = &x86_emul_or 1208 1.31.2.2 christos }, 1209 1.31.2.2 christos [0x0B] = { 1210 1.31.2.2 christos /* Gv, Ev */ 1211 1.31.2.2 christos .valid = true, 1212 1.31.2.2 christos .regmodrm = true, 1213 1.31.2.2 christos .regtorm = false, 1214 1.31.2.2 christos .szoverride = true, 1215 1.31.2.2 christos .defsize = -1, 1216 1.31.2.2 christos .emul = &x86_emul_or 1217 1.31.2.2 christos }, 1218 1.31.2.2 christos 1219 1.31.2.2 christos /* 1220 1.31.2.2 christos * AND 1221 1.31.2.2 christos */ 1222 1.31.2.2 christos [0x20] = { 1223 1.31.2.2 christos /* Eb, Gb */ 1224 1.31.2.2 christos .valid = true, 1225 1.31.2.2 christos .regmodrm = true, 1226 1.31.2.2 christos .regtorm = true, 1227 1.31.2.2 christos .szoverride = false, 1228 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1229 1.31.2.2 christos .emul = &x86_emul_and 1230 1.31.2.2 christos }, 1231 1.31.2.2 christos [0x21] = { 1232 1.31.2.2 christos /* Ev, Gv */ 1233 1.31.2.2 christos .valid = true, 1234 1.31.2.2 christos .regmodrm = true, 1235 1.31.2.2 christos .regtorm = true, 1236 1.31.2.2 christos .szoverride = true, 1237 1.31.2.2 christos .defsize = -1, 1238 1.31.2.2 christos .emul = &x86_emul_and 1239 1.31.2.2 christos }, 1240 1.31.2.2 christos [0x22] = { 1241 1.31.2.2 christos /* Gb, Eb */ 1242 1.31.2.2 christos .valid = true, 1243 1.31.2.2 christos .regmodrm = true, 1244 1.31.2.2 christos .regtorm = false, 1245 1.31.2.2 christos .szoverride = false, 1246 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1247 1.31.2.2 christos .emul = &x86_emul_and 1248 1.31.2.2 christos }, 1249 1.31.2.2 christos [0x23] = { 1250 1.31.2.2 christos /* Gv, Ev */ 1251 1.31.2.2 christos .valid = true, 1252 1.31.2.2 christos .regmodrm = true, 1253 1.31.2.2 christos .regtorm = false, 1254 1.31.2.2 christos .szoverride = true, 1255 1.31.2.2 christos .defsize = -1, 1256 1.31.2.2 christos .emul = &x86_emul_and 1257 1.31.2.2 christos }, 1258 1.31.2.2 christos 1259 1.31.2.2 christos /* 1260 1.31.2.2 christos * SUB 1261 1.31.2.2 christos */ 1262 1.31.2.2 christos [0x28] = { 1263 1.31.2.2 christos /* Eb, Gb */ 1264 1.31.2.2 christos .valid = true, 1265 1.31.2.2 christos .regmodrm = true, 1266 1.31.2.2 christos .regtorm = true, 1267 1.31.2.2 christos .szoverride = false, 1268 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1269 1.31.2.2 christos .emul = &x86_emul_sub 1270 1.31.2.2 christos }, 1271 1.31.2.2 christos [0x29] = { 1272 1.31.2.2 christos /* Ev, Gv */ 1273 1.31.2.2 christos .valid = true, 1274 1.31.2.2 christos .regmodrm = true, 1275 1.31.2.2 christos .regtorm = true, 1276 1.31.2.2 christos .szoverride = true, 1277 1.31.2.2 christos .defsize = -1, 1278 1.31.2.2 christos .emul = &x86_emul_sub 1279 1.31.2.2 christos }, 1280 1.31.2.2 christos [0x2A] = { 1281 1.31.2.2 christos /* Gb, Eb */ 1282 1.31.2.2 christos .valid = true, 1283 1.31.2.2 christos .regmodrm = true, 1284 1.31.2.2 christos .regtorm = false, 1285 1.31.2.2 christos .szoverride = false, 1286 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1287 1.31.2.2 christos .emul = &x86_emul_sub 1288 1.31.2.2 christos }, 1289 1.31.2.2 christos [0x2B] = { 1290 1.31.2.2 christos /* Gv, Ev */ 1291 1.31.2.2 christos .valid = true, 1292 1.31.2.2 christos .regmodrm = true, 1293 1.31.2.2 christos .regtorm = false, 1294 1.31.2.2 christos .szoverride = true, 1295 1.31.2.2 christos .defsize = -1, 1296 1.31.2.2 christos .emul = &x86_emul_sub 1297 1.31.2.2 christos }, 1298 1.31.2.2 christos 1299 1.31.2.2 christos /* 1300 1.31.2.2 christos * XOR 1301 1.31.2.2 christos */ 1302 1.31.2.2 christos [0x30] = { 1303 1.31.2.2 christos /* Eb, Gb */ 1304 1.31.2.2 christos .valid = true, 1305 1.31.2.2 christos .regmodrm = true, 1306 1.31.2.2 christos .regtorm = true, 1307 1.31.2.2 christos .szoverride = false, 1308 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1309 1.31.2.2 christos .emul = &x86_emul_xor 1310 1.31.2.2 christos }, 1311 1.31.2.2 christos [0x31] = { 1312 1.31.2.2 christos /* Ev, Gv */ 1313 1.31.2.2 christos .valid = true, 1314 1.31.2.2 christos .regmodrm = true, 1315 1.31.2.2 christos .regtorm = true, 1316 1.31.2.2 christos .szoverride = true, 1317 1.31.2.2 christos .defsize = -1, 1318 1.31.2.2 christos .emul = &x86_emul_xor 1319 1.31.2.2 christos }, 1320 1.31.2.2 christos [0x32] = { 1321 1.31.2.2 christos /* Gb, Eb */ 1322 1.31.2.2 christos .valid = true, 1323 1.31.2.2 christos .regmodrm = true, 1324 1.31.2.2 christos .regtorm = false, 1325 1.31.2.2 christos .szoverride = false, 1326 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1327 1.31.2.2 christos .emul = &x86_emul_xor 1328 1.31.2.2 christos }, 1329 1.31.2.2 christos [0x33] = { 1330 1.31.2.2 christos /* Gv, Ev */ 1331 1.31.2.2 christos .valid = true, 1332 1.31.2.2 christos .regmodrm = true, 1333 1.31.2.2 christos .regtorm = false, 1334 1.31.2.2 christos .szoverride = true, 1335 1.31.2.2 christos .defsize = -1, 1336 1.31.2.2 christos .emul = &x86_emul_xor 1337 1.31.2.2 christos }, 1338 1.31.2.2 christos 1339 1.31.2.2 christos /* 1340 1.31.2.2 christos * MOV 1341 1.31.2.2 christos */ 1342 1.31.2.2 christos [0x88] = { 1343 1.31.2.2 christos /* Eb, Gb */ 1344 1.31.2.2 christos .valid = true, 1345 1.31.2.2 christos .regmodrm = true, 1346 1.31.2.2 christos .regtorm = true, 1347 1.31.2.2 christos .szoverride = false, 1348 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1349 1.31.2.2 christos .emul = &x86_emul_mov 1350 1.31.2.2 christos }, 1351 1.31.2.2 christos [0x89] = { 1352 1.31.2.2 christos /* Ev, Gv */ 1353 1.31.2.2 christos .valid = true, 1354 1.31.2.2 christos .regmodrm = true, 1355 1.31.2.2 christos .regtorm = true, 1356 1.31.2.2 christos .szoverride = true, 1357 1.31.2.2 christos .defsize = -1, 1358 1.31.2.2 christos .emul = &x86_emul_mov 1359 1.31.2.2 christos }, 1360 1.31.2.2 christos [0x8A] = { 1361 1.31.2.2 christos /* Gb, Eb */ 1362 1.31.2.2 christos .valid = true, 1363 1.31.2.2 christos .regmodrm = true, 1364 1.31.2.2 christos .regtorm = false, 1365 1.31.2.2 christos .szoverride = false, 1366 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1367 1.31.2.2 christos .emul = &x86_emul_mov 1368 1.31.2.2 christos }, 1369 1.31.2.2 christos [0x8B] = { 1370 1.31.2.2 christos /* Gv, Ev */ 1371 1.31.2.2 christos .valid = true, 1372 1.31.2.2 christos .regmodrm = true, 1373 1.31.2.2 christos .regtorm = false, 1374 1.31.2.2 christos .szoverride = true, 1375 1.31.2.2 christos .defsize = -1, 1376 1.31.2.2 christos .emul = &x86_emul_mov 1377 1.31.2.2 christos }, 1378 1.31.2.2 christos [0xA0] = { 1379 1.31.2.2 christos /* AL, Ob */ 1380 1.31.2.2 christos .valid = true, 1381 1.31.2.2 christos .dmo = true, 1382 1.31.2.2 christos .todmo = false, 1383 1.31.2.2 christos .szoverride = false, 1384 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1385 1.31.2.2 christos .emul = &x86_emul_mov 1386 1.31.2.2 christos }, 1387 1.31.2.2 christos [0xA1] = { 1388 1.31.2.2 christos /* rAX, Ov */ 1389 1.31.2.2 christos .valid = true, 1390 1.31.2.2 christos .dmo = true, 1391 1.31.2.2 christos .todmo = false, 1392 1.31.2.2 christos .szoverride = true, 1393 1.31.2.2 christos .defsize = -1, 1394 1.31.2.2 christos .emul = &x86_emul_mov 1395 1.31.2.2 christos }, 1396 1.31.2.2 christos [0xA2] = { 1397 1.31.2.2 christos /* Ob, AL */ 1398 1.31.2.2 christos .valid = true, 1399 1.31.2.2 christos .dmo = true, 1400 1.31.2.2 christos .todmo = true, 1401 1.31.2.2 christos .szoverride = false, 1402 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1403 1.31.2.2 christos .emul = &x86_emul_mov 1404 1.31.2.2 christos }, 1405 1.31.2.2 christos [0xA3] = { 1406 1.31.2.2 christos /* Ov, rAX */ 1407 1.31.2.2 christos .valid = true, 1408 1.31.2.2 christos .dmo = true, 1409 1.31.2.2 christos .todmo = true, 1410 1.31.2.2 christos .szoverride = true, 1411 1.31.2.2 christos .defsize = -1, 1412 1.31.2.2 christos .emul = &x86_emul_mov 1413 1.31.2.2 christos }, 1414 1.31.2.2 christos 1415 1.31.2.2 christos /* 1416 1.31.2.2 christos * MOVS 1417 1.31.2.2 christos */ 1418 1.31.2.2 christos [0xA4] = { 1419 1.31.2.2 christos /* Yb, Xb */ 1420 1.31.2.2 christos .valid = true, 1421 1.31.2.2 christos .movs = true, 1422 1.31.2.2 christos .szoverride = false, 1423 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1424 1.31.2.2 christos .emul = &x86_emul_movs 1425 1.31.2.2 christos }, 1426 1.31.2.2 christos [0xA5] = { 1427 1.31.2.2 christos /* Yv, Xv */ 1428 1.31.2.2 christos .valid = true, 1429 1.31.2.2 christos .movs = true, 1430 1.31.2.2 christos .szoverride = true, 1431 1.31.2.2 christos .defsize = -1, 1432 1.31.2.2 christos .emul = &x86_emul_movs 1433 1.31.2.2 christos }, 1434 1.31.2.2 christos 1435 1.31.2.2 christos /* 1436 1.31.2.2 christos * STOS 1437 1.31.2.2 christos */ 1438 1.31.2.2 christos [0xAA] = { 1439 1.31.2.2 christos /* Yb, AL */ 1440 1.31.2.2 christos .valid = true, 1441 1.31.2.2 christos .stos = true, 1442 1.31.2.2 christos .szoverride = false, 1443 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1444 1.31.2.2 christos .emul = &x86_emul_stos 1445 1.31.2.2 christos }, 1446 1.31.2.2 christos [0xAB] = { 1447 1.31.2.2 christos /* Yv, rAX */ 1448 1.31.2.2 christos .valid = true, 1449 1.31.2.2 christos .stos = true, 1450 1.31.2.2 christos .szoverride = true, 1451 1.31.2.2 christos .defsize = -1, 1452 1.31.2.2 christos .emul = &x86_emul_stos 1453 1.31.2.2 christos }, 1454 1.31.2.2 christos 1455 1.31.2.2 christos /* 1456 1.31.2.2 christos * LODS 1457 1.31.2.2 christos */ 1458 1.31.2.2 christos [0xAC] = { 1459 1.31.2.2 christos /* AL, Xb */ 1460 1.31.2.2 christos .valid = true, 1461 1.31.2.2 christos .lods = true, 1462 1.31.2.2 christos .szoverride = false, 1463 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1464 1.31.2.2 christos .emul = &x86_emul_lods 1465 1.31.2.2 christos }, 1466 1.31.2.2 christos [0xAD] = { 1467 1.31.2.2 christos /* rAX, Xv */ 1468 1.31.2.2 christos .valid = true, 1469 1.31.2.2 christos .lods = true, 1470 1.31.2.2 christos .szoverride = true, 1471 1.31.2.2 christos .defsize = -1, 1472 1.31.2.2 christos .emul = &x86_emul_lods 1473 1.31.2.2 christos }, 1474 1.31.2.2 christos }; 1475 1.31.2.2 christos 1476 1.31.2.2 christos static const struct x86_opcode secondary_opcode_table[256] __cacheline_aligned = { 1477 1.31.2.2 christos /* 1478 1.31.2.2 christos * MOVZX 1479 1.31.2.2 christos */ 1480 1.31.2.2 christos [0xB6] = { 1481 1.31.2.2 christos /* Gv, Eb */ 1482 1.31.2.2 christos .valid = true, 1483 1.31.2.2 christos .regmodrm = true, 1484 1.31.2.2 christos .regtorm = false, 1485 1.31.2.2 christos .szoverride = true, 1486 1.31.2.2 christos .defsize = OPSIZE_BYTE, 1487 1.31.2.2 christos .flags = FLAG_ze, 1488 1.31.2.2 christos .emul = &x86_emul_mov 1489 1.31.2.2 christos }, 1490 1.31.2.2 christos [0xB7] = { 1491 1.31.2.2 christos /* Gv, Ew */ 1492 1.31.2.2 christos .valid = true, 1493 1.31.2.2 christos .regmodrm = true, 1494 1.31.2.2 christos .regtorm = false, 1495 1.31.2.2 christos .szoverride = true, 1496 1.31.2.2 christos .defsize = OPSIZE_WORD, 1497 1.31.2.2 christos .flags = FLAG_ze, 1498 1.31.2.2 christos .emul = &x86_emul_mov 1499 1.31.2.2 christos }, 1500 1.31.2.2 christos }; 1501 1.31.2.2 christos 1502 1.31.2.2 christos static const struct x86_reg gpr_map__rip = { NVMM_X64_GPR_RIP, 0xFFFFFFFFFFFFFFFF }; 1503 1.31.2.2 christos 1504 1.31.2.2 christos /* [REX-present][enc][opsize] */ 1505 1.31.2.2 christos static const struct x86_reg gpr_map__special[2][4][8] __cacheline_aligned = { 1506 1.31.2.2 christos [false] = { 1507 1.31.2.2 christos /* No REX prefix. */ 1508 1.31.2.2 christos [0b00] = { 1509 1.31.2.2 christos [0] = { NVMM_X64_GPR_RAX, 0x000000000000FF00 }, /* AH */ 1510 1.31.2.2 christos [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1511 1.31.2.2 christos [2] = { -1, 0 }, 1512 1.31.2.2 christos [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1513 1.31.2.2 christos [4] = { -1, 0 }, 1514 1.31.2.2 christos [5] = { -1, 0 }, 1515 1.31.2.2 christos [6] = { -1, 0 }, 1516 1.31.2.2 christos [7] = { -1, 0 }, 1517 1.31.2.2 christos }, 1518 1.31.2.2 christos [0b01] = { 1519 1.31.2.2 christos [0] = { NVMM_X64_GPR_RCX, 0x000000000000FF00 }, /* CH */ 1520 1.31.2.2 christos [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1521 1.31.2.2 christos [2] = { -1, 0 }, 1522 1.31.2.2 christos [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1523 1.31.2.2 christos [4] = { -1, 0 }, 1524 1.31.2.2 christos [5] = { -1, 0 }, 1525 1.31.2.2 christos [6] = { -1, 0 }, 1526 1.31.2.2 christos [7] = { -1, 0 }, 1527 1.31.2.2 christos }, 1528 1.31.2.2 christos [0b10] = { 1529 1.31.2.2 christos [0] = { NVMM_X64_GPR_RDX, 0x000000000000FF00 }, /* DH */ 1530 1.31.2.2 christos [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1531 1.31.2.2 christos [2] = { -1, 0 }, 1532 1.31.2.2 christos [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1533 1.31.2.2 christos [4] = { -1, 0 }, 1534 1.31.2.2 christos [5] = { -1, 0 }, 1535 1.31.2.2 christos [6] = { -1, 0 }, 1536 1.31.2.2 christos [7] = { -1, 0 }, 1537 1.31.2.2 christos }, 1538 1.31.2.2 christos [0b11] = { 1539 1.31.2.2 christos [0] = { NVMM_X64_GPR_RBX, 0x000000000000FF00 }, /* BH */ 1540 1.31.2.2 christos [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1541 1.31.2.2 christos [2] = { -1, 0 }, 1542 1.31.2.2 christos [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1543 1.31.2.2 christos [4] = { -1, 0 }, 1544 1.31.2.2 christos [5] = { -1, 0 }, 1545 1.31.2.2 christos [6] = { -1, 0 }, 1546 1.31.2.2 christos [7] = { -1, 0 }, 1547 1.31.2.2 christos } 1548 1.31.2.2 christos }, 1549 1.31.2.2 christos [true] = { 1550 1.31.2.2 christos /* Has REX prefix. */ 1551 1.31.2.2 christos [0b00] = { 1552 1.31.2.2 christos [0] = { NVMM_X64_GPR_RSP, 0x00000000000000FF }, /* SPL */ 1553 1.31.2.2 christos [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1554 1.31.2.2 christos [2] = { -1, 0 }, 1555 1.31.2.2 christos [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1556 1.31.2.2 christos [4] = { -1, 0 }, 1557 1.31.2.2 christos [5] = { -1, 0 }, 1558 1.31.2.2 christos [6] = { -1, 0 }, 1559 1.31.2.2 christos [7] = { NVMM_X64_GPR_RSP, 0xFFFFFFFFFFFFFFFF }, /* RSP */ 1560 1.31.2.2 christos }, 1561 1.31.2.2 christos [0b01] = { 1562 1.31.2.2 christos [0] = { NVMM_X64_GPR_RBP, 0x00000000000000FF }, /* BPL */ 1563 1.31.2.2 christos [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1564 1.31.2.2 christos [2] = { -1, 0 }, 1565 1.31.2.2 christos [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1566 1.31.2.2 christos [4] = { -1, 0 }, 1567 1.31.2.2 christos [5] = { -1, 0 }, 1568 1.31.2.2 christos [6] = { -1, 0 }, 1569 1.31.2.2 christos [7] = { NVMM_X64_GPR_RBP, 0xFFFFFFFFFFFFFFFF }, /* RBP */ 1570 1.31.2.2 christos }, 1571 1.31.2.2 christos [0b10] = { 1572 1.31.2.2 christos [0] = { NVMM_X64_GPR_RSI, 0x00000000000000FF }, /* SIL */ 1573 1.31.2.2 christos [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1574 1.31.2.2 christos [2] = { -1, 0 }, 1575 1.31.2.2 christos [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1576 1.31.2.2 christos [4] = { -1, 0 }, 1577 1.31.2.2 christos [5] = { -1, 0 }, 1578 1.31.2.2 christos [6] = { -1, 0 }, 1579 1.31.2.2 christos [7] = { NVMM_X64_GPR_RSI, 0xFFFFFFFFFFFFFFFF }, /* RSI */ 1580 1.31.2.2 christos }, 1581 1.31.2.2 christos [0b11] = { 1582 1.31.2.2 christos [0] = { NVMM_X64_GPR_RDI, 0x00000000000000FF }, /* DIL */ 1583 1.31.2.2 christos [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1584 1.31.2.2 christos [2] = { -1, 0 }, 1585 1.31.2.2 christos [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1586 1.31.2.2 christos [4] = { -1, 0 }, 1587 1.31.2.2 christos [5] = { -1, 0 }, 1588 1.31.2.2 christos [6] = { -1, 0 }, 1589 1.31.2.2 christos [7] = { NVMM_X64_GPR_RDI, 0xFFFFFFFFFFFFFFFF }, /* RDI */ 1590 1.31.2.2 christos } 1591 1.31.2.2 christos } 1592 1.31.2.2 christos }; 1593 1.31.2.2 christos 1594 1.31.2.2 christos /* [depends][enc][size] */ 1595 1.31.2.2 christos static const struct x86_reg gpr_map[2][8][8] __cacheline_aligned = { 1596 1.31.2.2 christos [false] = { 1597 1.31.2.2 christos /* Not extended. */ 1598 1.31.2.2 christos [0b000] = { 1599 1.31.2.2 christos [0] = { NVMM_X64_GPR_RAX, 0x00000000000000FF }, /* AL */ 1600 1.31.2.2 christos [1] = { NVMM_X64_GPR_RAX, 0x000000000000FFFF }, /* AX */ 1601 1.31.2.2 christos [2] = { -1, 0 }, 1602 1.31.2.2 christos [3] = { NVMM_X64_GPR_RAX, 0x00000000FFFFFFFF }, /* EAX */ 1603 1.31.2.2 christos [4] = { -1, 0 }, 1604 1.31.2.2 christos [5] = { -1, 0 }, 1605 1.31.2.2 christos [6] = { -1, 0 }, 1606 1.31.2.2 christos [7] = { NVMM_X64_GPR_RAX, 0xFFFFFFFFFFFFFFFF }, /* RAX */ 1607 1.31.2.2 christos }, 1608 1.31.2.2 christos [0b001] = { 1609 1.31.2.2 christos [0] = { NVMM_X64_GPR_RCX, 0x00000000000000FF }, /* CL */ 1610 1.31.2.2 christos [1] = { NVMM_X64_GPR_RCX, 0x000000000000FFFF }, /* CX */ 1611 1.31.2.2 christos [2] = { -1, 0 }, 1612 1.31.2.2 christos [3] = { NVMM_X64_GPR_RCX, 0x00000000FFFFFFFF }, /* ECX */ 1613 1.31.2.2 christos [4] = { -1, 0 }, 1614 1.31.2.2 christos [5] = { -1, 0 }, 1615 1.31.2.2 christos [6] = { -1, 0 }, 1616 1.31.2.2 christos [7] = { NVMM_X64_GPR_RCX, 0xFFFFFFFFFFFFFFFF }, /* RCX */ 1617 1.31.2.2 christos }, 1618 1.31.2.2 christos [0b010] = { 1619 1.31.2.2 christos [0] = { NVMM_X64_GPR_RDX, 0x00000000000000FF }, /* DL */ 1620 1.31.2.2 christos [1] = { NVMM_X64_GPR_RDX, 0x000000000000FFFF }, /* DX */ 1621 1.31.2.2 christos [2] = { -1, 0 }, 1622 1.31.2.2 christos [3] = { NVMM_X64_GPR_RDX, 0x00000000FFFFFFFF }, /* EDX */ 1623 1.31.2.2 christos [4] = { -1, 0 }, 1624 1.31.2.2 christos [5] = { -1, 0 }, 1625 1.31.2.2 christos [6] = { -1, 0 }, 1626 1.31.2.2 christos [7] = { NVMM_X64_GPR_RDX, 0xFFFFFFFFFFFFFFFF }, /* RDX */ 1627 1.31.2.2 christos }, 1628 1.31.2.2 christos [0b011] = { 1629 1.31.2.2 christos [0] = { NVMM_X64_GPR_RBX, 0x00000000000000FF }, /* BL */ 1630 1.31.2.2 christos [1] = { NVMM_X64_GPR_RBX, 0x000000000000FFFF }, /* BX */ 1631 1.31.2.2 christos [2] = { -1, 0 }, 1632 1.31.2.2 christos [3] = { NVMM_X64_GPR_RBX, 0x00000000FFFFFFFF }, /* EBX */ 1633 1.31.2.2 christos [4] = { -1, 0 }, 1634 1.31.2.2 christos [5] = { -1, 0 }, 1635 1.31.2.2 christos [6] = { -1, 0 }, 1636 1.31.2.2 christos [7] = { NVMM_X64_GPR_RBX, 0xFFFFFFFFFFFFFFFF }, /* RBX */ 1637 1.31.2.2 christos }, 1638 1.31.2.2 christos [0b100] = { 1639 1.31.2.2 christos [0] = { -1, 0 }, /* SPECIAL */ 1640 1.31.2.2 christos [1] = { -1, 0 }, /* SPECIAL */ 1641 1.31.2.2 christos [2] = { -1, 0 }, 1642 1.31.2.2 christos [3] = { -1, 0 }, /* SPECIAL */ 1643 1.31.2.2 christos [4] = { -1, 0 }, 1644 1.31.2.2 christos [5] = { -1, 0 }, 1645 1.31.2.2 christos [6] = { -1, 0 }, 1646 1.31.2.2 christos [7] = { -1, 0 }, /* SPECIAL */ 1647 1.31.2.2 christos }, 1648 1.31.2.2 christos [0b101] = { 1649 1.31.2.2 christos [0] = { -1, 0 }, /* SPECIAL */ 1650 1.31.2.2 christos [1] = { -1, 0 }, /* SPECIAL */ 1651 1.31.2.2 christos [2] = { -1, 0 }, 1652 1.31.2.2 christos [3] = { -1, 0 }, /* SPECIAL */ 1653 1.31.2.2 christos [4] = { -1, 0 }, 1654 1.31.2.2 christos [5] = { -1, 0 }, 1655 1.31.2.2 christos [6] = { -1, 0 }, 1656 1.31.2.2 christos [7] = { -1, 0 }, /* SPECIAL */ 1657 1.31.2.2 christos }, 1658 1.31.2.2 christos [0b110] = { 1659 1.31.2.2 christos [0] = { -1, 0 }, /* SPECIAL */ 1660 1.31.2.2 christos [1] = { -1, 0 }, /* SPECIAL */ 1661 1.31.2.2 christos [2] = { -1, 0 }, 1662 1.31.2.2 christos [3] = { -1, 0 }, /* SPECIAL */ 1663 1.31.2.2 christos [4] = { -1, 0 }, 1664 1.31.2.2 christos [5] = { -1, 0 }, 1665 1.31.2.2 christos [6] = { -1, 0 }, 1666 1.31.2.2 christos [7] = { -1, 0 }, /* SPECIAL */ 1667 1.31.2.2 christos }, 1668 1.31.2.2 christos [0b111] = { 1669 1.31.2.2 christos [0] = { -1, 0 }, /* SPECIAL */ 1670 1.31.2.2 christos [1] = { -1, 0 }, /* SPECIAL */ 1671 1.31.2.2 christos [2] = { -1, 0 }, 1672 1.31.2.2 christos [3] = { -1, 0 }, /* SPECIAL */ 1673 1.31.2.2 christos [4] = { -1, 0 }, 1674 1.31.2.2 christos [5] = { -1, 0 }, 1675 1.31.2.2 christos [6] = { -1, 0 }, 1676 1.31.2.2 christos [7] = { -1, 0 }, /* SPECIAL */ 1677 1.31.2.2 christos }, 1678 1.31.2.2 christos }, 1679 1.31.2.2 christos [true] = { 1680 1.31.2.2 christos /* Extended. */ 1681 1.31.2.2 christos [0b000] = { 1682 1.31.2.2 christos [0] = { NVMM_X64_GPR_R8, 0x00000000000000FF }, /* R8B */ 1683 1.31.2.2 christos [1] = { NVMM_X64_GPR_R8, 0x000000000000FFFF }, /* R8W */ 1684 1.31.2.2 christos [2] = { -1, 0 }, 1685 1.31.2.2 christos [3] = { NVMM_X64_GPR_R8, 0x00000000FFFFFFFF }, /* R8D */ 1686 1.31.2.2 christos [4] = { -1, 0 }, 1687 1.31.2.2 christos [5] = { -1, 0 }, 1688 1.31.2.2 christos [6] = { -1, 0 }, 1689 1.31.2.2 christos [7] = { NVMM_X64_GPR_R8, 0xFFFFFFFFFFFFFFFF }, /* R8 */ 1690 1.31.2.2 christos }, 1691 1.31.2.2 christos [0b001] = { 1692 1.31.2.2 christos [0] = { NVMM_X64_GPR_R9, 0x00000000000000FF }, /* R9B */ 1693 1.31.2.2 christos [1] = { NVMM_X64_GPR_R9, 0x000000000000FFFF }, /* R9W */ 1694 1.31.2.2 christos [2] = { -1, 0 }, 1695 1.31.2.2 christos [3] = { NVMM_X64_GPR_R9, 0x00000000FFFFFFFF }, /* R9D */ 1696 1.31.2.2 christos [4] = { -1, 0 }, 1697 1.31.2.2 christos [5] = { -1, 0 }, 1698 1.31.2.2 christos [6] = { -1, 0 }, 1699 1.31.2.2 christos [7] = { NVMM_X64_GPR_R9, 0xFFFFFFFFFFFFFFFF }, /* R9 */ 1700 1.31.2.2 christos }, 1701 1.31.2.2 christos [0b010] = { 1702 1.31.2.2 christos [0] = { NVMM_X64_GPR_R10, 0x00000000000000FF }, /* R10B */ 1703 1.31.2.2 christos [1] = { NVMM_X64_GPR_R10, 0x000000000000FFFF }, /* R10W */ 1704 1.31.2.2 christos [2] = { -1, 0 }, 1705 1.31.2.2 christos [3] = { NVMM_X64_GPR_R10, 0x00000000FFFFFFFF }, /* R10D */ 1706 1.31.2.2 christos [4] = { -1, 0 }, 1707 1.31.2.2 christos [5] = { -1, 0 }, 1708 1.31.2.2 christos [6] = { -1, 0 }, 1709 1.31.2.2 christos [7] = { NVMM_X64_GPR_R10, 0xFFFFFFFFFFFFFFFF }, /* R10 */ 1710 1.31.2.2 christos }, 1711 1.31.2.2 christos [0b011] = { 1712 1.31.2.2 christos [0] = { NVMM_X64_GPR_R11, 0x00000000000000FF }, /* R11B */ 1713 1.31.2.2 christos [1] = { NVMM_X64_GPR_R11, 0x000000000000FFFF }, /* R11W */ 1714 1.31.2.2 christos [2] = { -1, 0 }, 1715 1.31.2.2 christos [3] = { NVMM_X64_GPR_R11, 0x00000000FFFFFFFF }, /* R11D */ 1716 1.31.2.2 christos [4] = { -1, 0 }, 1717 1.31.2.2 christos [5] = { -1, 0 }, 1718 1.31.2.2 christos [6] = { -1, 0 }, 1719 1.31.2.2 christos [7] = { NVMM_X64_GPR_R11, 0xFFFFFFFFFFFFFFFF }, /* R11 */ 1720 1.31.2.2 christos }, 1721 1.31.2.2 christos [0b100] = { 1722 1.31.2.2 christos [0] = { NVMM_X64_GPR_R12, 0x00000000000000FF }, /* R12B */ 1723 1.31.2.2 christos [1] = { NVMM_X64_GPR_R12, 0x000000000000FFFF }, /* R12W */ 1724 1.31.2.2 christos [2] = { -1, 0 }, 1725 1.31.2.2 christos [3] = { NVMM_X64_GPR_R12, 0x00000000FFFFFFFF }, /* R12D */ 1726 1.31.2.2 christos [4] = { -1, 0 }, 1727 1.31.2.2 christos [5] = { -1, 0 }, 1728 1.31.2.2 christos [6] = { -1, 0 }, 1729 1.31.2.2 christos [7] = { NVMM_X64_GPR_R12, 0xFFFFFFFFFFFFFFFF }, /* R12 */ 1730 1.31.2.2 christos }, 1731 1.31.2.2 christos [0b101] = { 1732 1.31.2.2 christos [0] = { NVMM_X64_GPR_R13, 0x00000000000000FF }, /* R13B */ 1733 1.31.2.2 christos [1] = { NVMM_X64_GPR_R13, 0x000000000000FFFF }, /* R13W */ 1734 1.31.2.2 christos [2] = { -1, 0 }, 1735 1.31.2.2 christos [3] = { NVMM_X64_GPR_R13, 0x00000000FFFFFFFF }, /* R13D */ 1736 1.31.2.2 christos [4] = { -1, 0 }, 1737 1.31.2.2 christos [5] = { -1, 0 }, 1738 1.31.2.2 christos [6] = { -1, 0 }, 1739 1.31.2.2 christos [7] = { NVMM_X64_GPR_R13, 0xFFFFFFFFFFFFFFFF }, /* R13 */ 1740 1.31.2.2 christos }, 1741 1.31.2.2 christos [0b110] = { 1742 1.31.2.2 christos [0] = { NVMM_X64_GPR_R14, 0x00000000000000FF }, /* R14B */ 1743 1.31.2.2 christos [1] = { NVMM_X64_GPR_R14, 0x000000000000FFFF }, /* R14W */ 1744 1.31.2.2 christos [2] = { -1, 0 }, 1745 1.31.2.2 christos [3] = { NVMM_X64_GPR_R14, 0x00000000FFFFFFFF }, /* R14D */ 1746 1.31.2.2 christos [4] = { -1, 0 }, 1747 1.31.2.2 christos [5] = { -1, 0 }, 1748 1.31.2.2 christos [6] = { -1, 0 }, 1749 1.31.2.2 christos [7] = { NVMM_X64_GPR_R14, 0xFFFFFFFFFFFFFFFF }, /* R14 */ 1750 1.31.2.2 christos }, 1751 1.31.2.2 christos [0b111] = { 1752 1.31.2.2 christos [0] = { NVMM_X64_GPR_R15, 0x00000000000000FF }, /* R15B */ 1753 1.31.2.2 christos [1] = { NVMM_X64_GPR_R15, 0x000000000000FFFF }, /* R15W */ 1754 1.31.2.2 christos [2] = { -1, 0 }, 1755 1.31.2.2 christos [3] = { NVMM_X64_GPR_R15, 0x00000000FFFFFFFF }, /* R15D */ 1756 1.31.2.2 christos [4] = { -1, 0 }, 1757 1.31.2.2 christos [5] = { -1, 0 }, 1758 1.31.2.2 christos [6] = { -1, 0 }, 1759 1.31.2.2 christos [7] = { NVMM_X64_GPR_R15, 0xFFFFFFFFFFFFFFFF }, /* R15 */ 1760 1.31.2.2 christos }, 1761 1.31.2.2 christos } 1762 1.31.2.2 christos }; 1763 1.31.2.2 christos 1764 1.31.2.2 christos static int 1765 1.31.2.2 christos node_overflow(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1766 1.31.2.2 christos { 1767 1.31.2.2 christos fsm->fn = NULL; 1768 1.31.2.2 christos return -1; 1769 1.31.2.2 christos } 1770 1.31.2.2 christos 1771 1.31.2.2 christos static int 1772 1.31.2.2 christos fsm_read(struct x86_decode_fsm *fsm, uint8_t *bytes, size_t n) 1773 1.31.2.2 christos { 1774 1.31.2.2 christos if (fsm->buf + n > fsm->end) { 1775 1.31.2.2 christos return -1; 1776 1.31.2.2 christos } 1777 1.31.2.2 christos memcpy(bytes, fsm->buf, n); 1778 1.31.2.2 christos return 0; 1779 1.31.2.2 christos } 1780 1.31.2.2 christos 1781 1.31.2.2 christos static inline void 1782 1.31.2.2 christos fsm_advance(struct x86_decode_fsm *fsm, size_t n, 1783 1.31.2.2 christos int (*fn)(struct x86_decode_fsm *, struct x86_instr *)) 1784 1.31.2.2 christos { 1785 1.31.2.2 christos fsm->buf += n; 1786 1.31.2.2 christos if (fsm->buf > fsm->end) { 1787 1.31.2.2 christos fsm->fn = node_overflow; 1788 1.31.2.2 christos } else { 1789 1.31.2.2 christos fsm->fn = fn; 1790 1.31.2.2 christos } 1791 1.31.2.2 christos } 1792 1.31.2.2 christos 1793 1.31.2.2 christos static const struct x86_reg * 1794 1.31.2.2 christos resolve_special_register(struct x86_instr *instr, uint8_t enc, size_t regsize) 1795 1.31.2.2 christos { 1796 1.31.2.2 christos enc &= 0b11; 1797 1.31.2.2 christos if (regsize == 8) { 1798 1.31.2.2 christos /* May be 64bit without REX */ 1799 1.31.2.2 christos return &gpr_map__special[1][enc][regsize-1]; 1800 1.31.2.2 christos } 1801 1.31.2.2 christos return &gpr_map__special[instr->rexpref.present][enc][regsize-1]; 1802 1.31.2.2 christos } 1803 1.31.2.2 christos 1804 1.31.2.2 christos /* 1805 1.31.2.2 christos * Special node, for MOVS. Fake two displacements of zero on the source and 1806 1.31.2.2 christos * destination registers. 1807 1.31.2.2 christos */ 1808 1.31.2.2 christos static int 1809 1.31.2.2 christos node_movs(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1810 1.31.2.2 christos { 1811 1.31.2.2 christos size_t adrsize; 1812 1.31.2.2 christos 1813 1.31.2.2 christos adrsize = instr->address_size; 1814 1.31.2.2 christos 1815 1.31.2.2 christos /* DS:RSI */ 1816 1.31.2.2 christos instr->src.type = STORE_REG; 1817 1.31.2.2 christos instr->src.u.reg = &gpr_map__special[1][2][adrsize-1]; 1818 1.31.2.2 christos instr->src.disp.type = DISP_0; 1819 1.31.2.2 christos 1820 1.31.2.2 christos /* ES:RDI, force ES */ 1821 1.31.2.2 christos instr->dst.type = STORE_REG; 1822 1.31.2.2 christos instr->dst.u.reg = &gpr_map__special[1][3][adrsize-1]; 1823 1.31.2.2 christos instr->dst.disp.type = DISP_0; 1824 1.31.2.2 christos instr->dst.hardseg = NVMM_X64_SEG_ES; 1825 1.31.2.2 christos 1826 1.31.2.2 christos fsm_advance(fsm, 0, NULL); 1827 1.31.2.2 christos 1828 1.31.2.2 christos return 0; 1829 1.31.2.2 christos } 1830 1.31.2.2 christos 1831 1.31.2.2 christos /* 1832 1.31.2.2 christos * Special node, for STOS and LODS. Fake a displacement of zero on the 1833 1.31.2.2 christos * destination register. 1834 1.31.2.2 christos */ 1835 1.31.2.2 christos static int 1836 1.31.2.2 christos node_stlo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1837 1.31.2.2 christos { 1838 1.31.2.2 christos const struct x86_opcode *opcode = instr->opcode; 1839 1.31.2.2 christos struct x86_store *stlo, *streg; 1840 1.31.2.2 christos size_t adrsize, regsize; 1841 1.31.2.2 christos 1842 1.31.2.2 christos adrsize = instr->address_size; 1843 1.31.2.2 christos regsize = instr->operand_size; 1844 1.31.2.2 christos 1845 1.31.2.2 christos if (opcode->stos) { 1846 1.31.2.2 christos streg = &instr->src; 1847 1.31.2.2 christos stlo = &instr->dst; 1848 1.31.2.2 christos } else { 1849 1.31.2.2 christos streg = &instr->dst; 1850 1.31.2.2 christos stlo = &instr->src; 1851 1.31.2.2 christos } 1852 1.31.2.2 christos 1853 1.31.2.2 christos streg->type = STORE_REG; 1854 1.31.2.2 christos streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1855 1.31.2.2 christos 1856 1.31.2.2 christos stlo->type = STORE_REG; 1857 1.31.2.2 christos if (opcode->stos) { 1858 1.31.2.2 christos /* ES:RDI, force ES */ 1859 1.31.2.2 christos stlo->u.reg = &gpr_map__special[1][3][adrsize-1]; 1860 1.31.2.2 christos stlo->hardseg = NVMM_X64_SEG_ES; 1861 1.31.2.2 christos } else { 1862 1.31.2.2 christos /* DS:RSI */ 1863 1.31.2.2 christos stlo->u.reg = &gpr_map__special[1][2][adrsize-1]; 1864 1.31.2.2 christos } 1865 1.31.2.2 christos stlo->disp.type = DISP_0; 1866 1.31.2.2 christos 1867 1.31.2.2 christos fsm_advance(fsm, 0, NULL); 1868 1.31.2.2 christos 1869 1.31.2.2 christos return 0; 1870 1.31.2.2 christos } 1871 1.31.2.2 christos 1872 1.31.2.2 christos static int 1873 1.31.2.2 christos node_dmo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1874 1.31.2.2 christos { 1875 1.31.2.2 christos const struct x86_opcode *opcode = instr->opcode; 1876 1.31.2.2 christos struct x86_store *stdmo, *streg; 1877 1.31.2.2 christos size_t adrsize, regsize; 1878 1.31.2.2 christos 1879 1.31.2.2 christos adrsize = instr->address_size; 1880 1.31.2.2 christos regsize = instr->operand_size; 1881 1.31.2.2 christos 1882 1.31.2.2 christos if (opcode->todmo) { 1883 1.31.2.2 christos streg = &instr->src; 1884 1.31.2.2 christos stdmo = &instr->dst; 1885 1.31.2.2 christos } else { 1886 1.31.2.2 christos streg = &instr->dst; 1887 1.31.2.2 christos stdmo = &instr->src; 1888 1.31.2.2 christos } 1889 1.31.2.2 christos 1890 1.31.2.2 christos streg->type = STORE_REG; 1891 1.31.2.2 christos streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1892 1.31.2.2 christos 1893 1.31.2.2 christos stdmo->type = STORE_DMO; 1894 1.31.2.2 christos if (fsm_read(fsm, (uint8_t *)&stdmo->u.dmo, adrsize) == -1) { 1895 1.31.2.2 christos return -1; 1896 1.31.2.2 christos } 1897 1.31.2.2 christos fsm_advance(fsm, adrsize, NULL); 1898 1.31.2.2 christos 1899 1.31.2.2 christos return 0; 1900 1.31.2.2 christos } 1901 1.31.2.2 christos 1902 1.31.2.2 christos static inline uint64_t 1903 1.31.2.2 christos sign_extend(uint64_t val, int size) 1904 1.31.2.2 christos { 1905 1.31.2.2 christos if (size == 1) { 1906 1.31.2.2 christos if (val & __BIT(7)) 1907 1.31.2.2 christos val |= 0xFFFFFFFFFFFFFF00; 1908 1.31.2.2 christos } else if (size == 2) { 1909 1.31.2.2 christos if (val & __BIT(15)) 1910 1.31.2.2 christos val |= 0xFFFFFFFFFFFF0000; 1911 1.31.2.2 christos } else if (size == 4) { 1912 1.31.2.2 christos if (val & __BIT(31)) 1913 1.31.2.2 christos val |= 0xFFFFFFFF00000000; 1914 1.31.2.2 christos } 1915 1.31.2.2 christos return val; 1916 1.31.2.2 christos } 1917 1.31.2.2 christos 1918 1.31.2.2 christos static int 1919 1.31.2.2 christos node_immediate(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1920 1.31.2.2 christos { 1921 1.31.2.2 christos const struct x86_opcode *opcode = instr->opcode; 1922 1.31.2.2 christos struct x86_store *store; 1923 1.31.2.2 christos uint8_t immsize; 1924 1.31.2.2 christos size_t sesize = 0; 1925 1.31.2.2 christos 1926 1.31.2.2 christos /* The immediate is the source */ 1927 1.31.2.2 christos store = &instr->src; 1928 1.31.2.2 christos immsize = instr->operand_size; 1929 1.31.2.2 christos 1930 1.31.2.2 christos if (opcode->flags & FLAG_imm8) { 1931 1.31.2.2 christos sesize = immsize; 1932 1.31.2.2 christos immsize = 1; 1933 1.31.2.2 christos } else if ((opcode->flags & FLAG_immz) && (immsize == 8)) { 1934 1.31.2.2 christos sesize = immsize; 1935 1.31.2.2 christos immsize = 4; 1936 1.31.2.2 christos } 1937 1.31.2.2 christos 1938 1.31.2.2 christos store->type = STORE_IMM; 1939 1.31.2.2 christos if (fsm_read(fsm, (uint8_t *)&store->u.imm.data, immsize) == -1) { 1940 1.31.2.2 christos return -1; 1941 1.31.2.2 christos } 1942 1.31.2.2 christos fsm_advance(fsm, immsize, NULL); 1943 1.31.2.2 christos 1944 1.31.2.2 christos if (sesize != 0) { 1945 1.31.2.2 christos store->u.imm.data = sign_extend(store->u.imm.data, sesize); 1946 1.31.2.2 christos } 1947 1.31.2.2 christos 1948 1.31.2.2 christos return 0; 1949 1.31.2.2 christos } 1950 1.31.2.2 christos 1951 1.31.2.2 christos static int 1952 1.31.2.2 christos node_disp(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1953 1.31.2.2 christos { 1954 1.31.2.2 christos const struct x86_opcode *opcode = instr->opcode; 1955 1.31.2.2 christos uint64_t data = 0; 1956 1.31.2.2 christos size_t n; 1957 1.31.2.2 christos 1958 1.31.2.2 christos if (instr->strm->disp.type == DISP_1) { 1959 1.31.2.2 christos n = 1; 1960 1.31.2.2 christos } else { /* DISP4 */ 1961 1.31.2.2 christos n = 4; 1962 1.31.2.2 christos } 1963 1.31.2.2 christos 1964 1.31.2.2 christos if (fsm_read(fsm, (uint8_t *)&data, n) == -1) { 1965 1.31.2.2 christos return -1; 1966 1.31.2.2 christos } 1967 1.31.2.2 christos 1968 1.31.2.2 christos if (__predict_true(fsm->is64bit)) { 1969 1.31.2.2 christos data = sign_extend(data, n); 1970 1.31.2.2 christos } 1971 1.31.2.2 christos 1972 1.31.2.2 christos instr->strm->disp.data = data; 1973 1.31.2.2 christos 1974 1.31.2.2 christos if (opcode->immediate) { 1975 1.31.2.2 christos fsm_advance(fsm, n, node_immediate); 1976 1.31.2.2 christos } else { 1977 1.31.2.2 christos fsm_advance(fsm, n, NULL); 1978 1.31.2.2 christos } 1979 1.31.2.2 christos 1980 1.31.2.2 christos return 0; 1981 1.31.2.2 christos } 1982 1.31.2.2 christos 1983 1.31.2.2 christos static const struct x86_reg * 1984 1.31.2.2 christos get_register_idx(struct x86_instr *instr, uint8_t index) 1985 1.31.2.2 christos { 1986 1.31.2.2 christos uint8_t enc = index; 1987 1.31.2.2 christos const struct x86_reg *reg; 1988 1.31.2.2 christos size_t regsize; 1989 1.31.2.2 christos 1990 1.31.2.2 christos regsize = instr->address_size; 1991 1.31.2.2 christos reg = &gpr_map[instr->rexpref.x][enc][regsize-1]; 1992 1.31.2.2 christos 1993 1.31.2.2 christos if (reg->num == -1) { 1994 1.31.2.2 christos reg = resolve_special_register(instr, enc, regsize); 1995 1.31.2.2 christos } 1996 1.31.2.2 christos 1997 1.31.2.2 christos return reg; 1998 1.31.2.2 christos } 1999 1.31.2.2 christos 2000 1.31.2.2 christos static const struct x86_reg * 2001 1.31.2.2 christos get_register_bas(struct x86_instr *instr, uint8_t base) 2002 1.31.2.2 christos { 2003 1.31.2.2 christos uint8_t enc = base; 2004 1.31.2.2 christos const struct x86_reg *reg; 2005 1.31.2.2 christos size_t regsize; 2006 1.31.2.2 christos 2007 1.31.2.2 christos regsize = instr->address_size; 2008 1.31.2.2 christos reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2009 1.31.2.2 christos if (reg->num == -1) { 2010 1.31.2.2 christos reg = resolve_special_register(instr, enc, regsize); 2011 1.31.2.2 christos } 2012 1.31.2.2 christos 2013 1.31.2.2 christos return reg; 2014 1.31.2.2 christos } 2015 1.31.2.2 christos 2016 1.31.2.2 christos static int 2017 1.31.2.2 christos node_sib(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2018 1.31.2.2 christos { 2019 1.31.2.2 christos const struct x86_opcode *opcode; 2020 1.31.2.2 christos uint8_t scale, index, base; 2021 1.31.2.2 christos bool noindex, nobase; 2022 1.31.2.2 christos uint8_t byte; 2023 1.31.2.2 christos 2024 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2025 1.31.2.2 christos return -1; 2026 1.31.2.2 christos } 2027 1.31.2.2 christos 2028 1.31.2.2 christos scale = ((byte & 0b11000000) >> 6); 2029 1.31.2.2 christos index = ((byte & 0b00111000) >> 3); 2030 1.31.2.2 christos base = ((byte & 0b00000111) >> 0); 2031 1.31.2.2 christos 2032 1.31.2.2 christos opcode = instr->opcode; 2033 1.31.2.2 christos 2034 1.31.2.2 christos noindex = false; 2035 1.31.2.2 christos nobase = false; 2036 1.31.2.2 christos 2037 1.31.2.2 christos if (index == 0b100 && !instr->rexpref.x) { 2038 1.31.2.2 christos /* Special case: the index is null */ 2039 1.31.2.2 christos noindex = true; 2040 1.31.2.2 christos } 2041 1.31.2.2 christos 2042 1.31.2.2 christos if (instr->regmodrm.mod == 0b00 && base == 0b101) { 2043 1.31.2.2 christos /* Special case: the base is null + disp32 */ 2044 1.31.2.2 christos instr->strm->disp.type = DISP_4; 2045 1.31.2.2 christos nobase = true; 2046 1.31.2.2 christos } 2047 1.31.2.2 christos 2048 1.31.2.2 christos instr->strm->type = STORE_SIB; 2049 1.31.2.2 christos instr->strm->u.sib.scale = (1 << scale); 2050 1.31.2.2 christos if (!noindex) 2051 1.31.2.2 christos instr->strm->u.sib.idx = get_register_idx(instr, index); 2052 1.31.2.2 christos if (!nobase) 2053 1.31.2.2 christos instr->strm->u.sib.bas = get_register_bas(instr, base); 2054 1.31.2.2 christos 2055 1.31.2.2 christos /* May have a displacement, or an immediate */ 2056 1.31.2.2 christos if (instr->strm->disp.type == DISP_1 || instr->strm->disp.type == DISP_4) { 2057 1.31.2.2 christos fsm_advance(fsm, 1, node_disp); 2058 1.31.2.2 christos } else if (opcode->immediate) { 2059 1.31.2.2 christos fsm_advance(fsm, 1, node_immediate); 2060 1.31.2.2 christos } else { 2061 1.31.2.2 christos fsm_advance(fsm, 1, NULL); 2062 1.31.2.2 christos } 2063 1.31.2.2 christos 2064 1.31.2.2 christos return 0; 2065 1.31.2.2 christos } 2066 1.31.2.2 christos 2067 1.31.2.2 christos static const struct x86_reg * 2068 1.31.2.2 christos get_register_reg(struct x86_instr *instr, const struct x86_opcode *opcode) 2069 1.31.2.2 christos { 2070 1.31.2.2 christos uint8_t enc = instr->regmodrm.reg; 2071 1.31.2.2 christos const struct x86_reg *reg; 2072 1.31.2.2 christos size_t regsize; 2073 1.31.2.2 christos 2074 1.31.2.2 christos regsize = instr->operand_size; 2075 1.31.2.2 christos 2076 1.31.2.2 christos reg = &gpr_map[instr->rexpref.r][enc][regsize-1]; 2077 1.31.2.2 christos if (reg->num == -1) { 2078 1.31.2.2 christos reg = resolve_special_register(instr, enc, regsize); 2079 1.31.2.2 christos } 2080 1.31.2.2 christos 2081 1.31.2.2 christos return reg; 2082 1.31.2.2 christos } 2083 1.31.2.2 christos 2084 1.31.2.2 christos static const struct x86_reg * 2085 1.31.2.2 christos get_register_rm(struct x86_instr *instr, const struct x86_opcode *opcode) 2086 1.31.2.2 christos { 2087 1.31.2.2 christos uint8_t enc = instr->regmodrm.rm; 2088 1.31.2.2 christos const struct x86_reg *reg; 2089 1.31.2.2 christos size_t regsize; 2090 1.31.2.2 christos 2091 1.31.2.2 christos if (instr->strm->disp.type == DISP_NONE) { 2092 1.31.2.2 christos regsize = instr->operand_size; 2093 1.31.2.2 christos } else { 2094 1.31.2.2 christos /* Indirect access, the size is that of the address. */ 2095 1.31.2.2 christos regsize = instr->address_size; 2096 1.31.2.2 christos } 2097 1.31.2.2 christos 2098 1.31.2.2 christos reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2099 1.31.2.2 christos if (reg->num == -1) { 2100 1.31.2.2 christos reg = resolve_special_register(instr, enc, regsize); 2101 1.31.2.2 christos } 2102 1.31.2.2 christos 2103 1.31.2.2 christos return reg; 2104 1.31.2.2 christos } 2105 1.31.2.2 christos 2106 1.31.2.2 christos static inline bool 2107 1.31.2.2 christos has_sib(struct x86_instr *instr) 2108 1.31.2.2 christos { 2109 1.31.2.2 christos return (instr->regmodrm.mod != 3 && instr->regmodrm.rm == 4); 2110 1.31.2.2 christos } 2111 1.31.2.2 christos 2112 1.31.2.2 christos static inline bool 2113 1.31.2.2 christos is_rip_relative(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2114 1.31.2.2 christos { 2115 1.31.2.2 christos return (fsm->is64bit && instr->strm->disp.type == DISP_0 && 2116 1.31.2.2 christos instr->regmodrm.rm == RM_RBP_DISP32); 2117 1.31.2.2 christos } 2118 1.31.2.2 christos 2119 1.31.2.2 christos static inline bool 2120 1.31.2.2 christos is_disp32_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2121 1.31.2.2 christos { 2122 1.31.2.2 christos return (!fsm->is64bit && instr->strm->disp.type == DISP_0 && 2123 1.31.2.2 christos instr->regmodrm.rm == RM_RBP_DISP32); 2124 1.31.2.2 christos } 2125 1.31.2.2 christos 2126 1.31.2.2 christos static enum x86_disp_type 2127 1.31.2.2 christos get_disp_type(struct x86_instr *instr) 2128 1.31.2.2 christos { 2129 1.31.2.2 christos switch (instr->regmodrm.mod) { 2130 1.31.2.2 christos case MOD_DIS0: /* indirect */ 2131 1.31.2.2 christos return DISP_0; 2132 1.31.2.2 christos case MOD_DIS1: /* indirect+1 */ 2133 1.31.2.2 christos return DISP_1; 2134 1.31.2.2 christos case MOD_DIS4: /* indirect+4 */ 2135 1.31.2.2 christos return DISP_4; 2136 1.31.2.2 christos case MOD_REG: /* direct */ 2137 1.31.2.2 christos default: /* gcc */ 2138 1.31.2.2 christos return DISP_NONE; 2139 1.31.2.2 christos } 2140 1.31.2.2 christos } 2141 1.31.2.2 christos 2142 1.31.2.2 christos static int 2143 1.31.2.2 christos node_regmodrm(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2144 1.31.2.2 christos { 2145 1.31.2.2 christos struct x86_store *strg, *strm; 2146 1.31.2.2 christos const struct x86_opcode *opcode; 2147 1.31.2.2 christos const struct x86_reg *reg; 2148 1.31.2.2 christos uint8_t byte; 2149 1.31.2.2 christos 2150 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2151 1.31.2.2 christos return -1; 2152 1.31.2.2 christos } 2153 1.31.2.2 christos 2154 1.31.2.2 christos opcode = instr->opcode; 2155 1.31.2.2 christos 2156 1.31.2.2 christos instr->regmodrm.rm = ((byte & 0b00000111) >> 0); 2157 1.31.2.2 christos instr->regmodrm.reg = ((byte & 0b00111000) >> 3); 2158 1.31.2.2 christos instr->regmodrm.mod = ((byte & 0b11000000) >> 6); 2159 1.31.2.2 christos 2160 1.31.2.2 christos if (opcode->regtorm) { 2161 1.31.2.2 christos strg = &instr->src; 2162 1.31.2.2 christos strm = &instr->dst; 2163 1.31.2.2 christos } else { /* RM to REG */ 2164 1.31.2.2 christos strm = &instr->src; 2165 1.31.2.2 christos strg = &instr->dst; 2166 1.31.2.2 christos } 2167 1.31.2.2 christos 2168 1.31.2.2 christos /* Save for later use. */ 2169 1.31.2.2 christos instr->strm = strm; 2170 1.31.2.2 christos 2171 1.31.2.2 christos /* 2172 1.31.2.2 christos * Special cases: Groups. The REG field of REGMODRM is the index in 2173 1.31.2.2 christos * the group. op1 gets overwritten in the Immediate node, if any. 2174 1.31.2.2 christos */ 2175 1.31.2.2 christos if (opcode->group1) { 2176 1.31.2.2 christos if (group1[instr->regmodrm.reg].emul == NULL) { 2177 1.31.2.2 christos return -1; 2178 1.31.2.2 christos } 2179 1.31.2.2 christos instr->emul = group1[instr->regmodrm.reg].emul; 2180 1.31.2.2 christos } else if (opcode->group3) { 2181 1.31.2.2 christos if (group3[instr->regmodrm.reg].emul == NULL) { 2182 1.31.2.2 christos return -1; 2183 1.31.2.2 christos } 2184 1.31.2.2 christos instr->emul = group3[instr->regmodrm.reg].emul; 2185 1.31.2.2 christos } else if (opcode->group11) { 2186 1.31.2.2 christos if (group11[instr->regmodrm.reg].emul == NULL) { 2187 1.31.2.2 christos return -1; 2188 1.31.2.2 christos } 2189 1.31.2.2 christos instr->emul = group11[instr->regmodrm.reg].emul; 2190 1.31.2.2 christos } 2191 1.31.2.2 christos 2192 1.31.2.2 christos if (!opcode->immediate) { 2193 1.31.2.2 christos reg = get_register_reg(instr, opcode); 2194 1.31.2.2 christos if (reg == NULL) { 2195 1.31.2.2 christos return -1; 2196 1.31.2.2 christos } 2197 1.31.2.2 christos strg->type = STORE_REG; 2198 1.31.2.2 christos strg->u.reg = reg; 2199 1.31.2.2 christos } 2200 1.31.2.2 christos 2201 1.31.2.2 christos /* The displacement applies to RM. */ 2202 1.31.2.2 christos strm->disp.type = get_disp_type(instr); 2203 1.31.2.2 christos 2204 1.31.2.2 christos if (has_sib(instr)) { 2205 1.31.2.2 christos /* Overwrites RM */ 2206 1.31.2.2 christos fsm_advance(fsm, 1, node_sib); 2207 1.31.2.2 christos return 0; 2208 1.31.2.2 christos } 2209 1.31.2.2 christos 2210 1.31.2.2 christos if (is_rip_relative(fsm, instr)) { 2211 1.31.2.2 christos /* Overwrites RM */ 2212 1.31.2.2 christos strm->type = STORE_REG; 2213 1.31.2.2 christos strm->u.reg = &gpr_map__rip; 2214 1.31.2.2 christos strm->disp.type = DISP_4; 2215 1.31.2.2 christos fsm_advance(fsm, 1, node_disp); 2216 1.31.2.2 christos return 0; 2217 1.31.2.2 christos } 2218 1.31.2.2 christos 2219 1.31.2.2 christos if (is_disp32_only(fsm, instr)) { 2220 1.31.2.2 christos /* Overwrites RM */ 2221 1.31.2.2 christos strm->type = STORE_REG; 2222 1.31.2.2 christos strm->u.reg = NULL; 2223 1.31.2.2 christos strm->disp.type = DISP_4; 2224 1.31.2.2 christos fsm_advance(fsm, 1, node_disp); 2225 1.31.2.2 christos return 0; 2226 1.31.2.2 christos } 2227 1.31.2.2 christos 2228 1.31.2.2 christos reg = get_register_rm(instr, opcode); 2229 1.31.2.2 christos if (reg == NULL) { 2230 1.31.2.2 christos return -1; 2231 1.31.2.2 christos } 2232 1.31.2.2 christos strm->type = STORE_REG; 2233 1.31.2.2 christos strm->u.reg = reg; 2234 1.31.2.2 christos 2235 1.31.2.2 christos if (strm->disp.type == DISP_NONE) { 2236 1.31.2.2 christos /* Direct register addressing mode */ 2237 1.31.2.2 christos if (opcode->immediate) { 2238 1.31.2.2 christos fsm_advance(fsm, 1, node_immediate); 2239 1.31.2.2 christos } else { 2240 1.31.2.2 christos fsm_advance(fsm, 1, NULL); 2241 1.31.2.2 christos } 2242 1.31.2.2 christos } else if (strm->disp.type == DISP_0) { 2243 1.31.2.2 christos /* Indirect register addressing mode */ 2244 1.31.2.2 christos if (opcode->immediate) { 2245 1.31.2.2 christos fsm_advance(fsm, 1, node_immediate); 2246 1.31.2.2 christos } else { 2247 1.31.2.2 christos fsm_advance(fsm, 1, NULL); 2248 1.31.2.2 christos } 2249 1.31.2.2 christos } else { 2250 1.31.2.2 christos fsm_advance(fsm, 1, node_disp); 2251 1.31.2.2 christos } 2252 1.31.2.2 christos 2253 1.31.2.2 christos return 0; 2254 1.31.2.2 christos } 2255 1.31.2.2 christos 2256 1.31.2.2 christos static size_t 2257 1.31.2.2 christos get_operand_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2258 1.31.2.2 christos { 2259 1.31.2.2 christos const struct x86_opcode *opcode = instr->opcode; 2260 1.31.2.2 christos int opsize; 2261 1.31.2.2 christos 2262 1.31.2.2 christos /* Get the opsize */ 2263 1.31.2.2 christos if (!opcode->szoverride) { 2264 1.31.2.2 christos opsize = opcode->defsize; 2265 1.31.2.2 christos } else if (instr->rexpref.present && instr->rexpref.w) { 2266 1.31.2.2 christos opsize = 8; 2267 1.31.2.2 christos } else { 2268 1.31.2.2 christos if (!fsm->is16bit) { 2269 1.31.2.2 christos if (instr->legpref.opr_ovr) { 2270 1.31.2.2 christos opsize = 2; 2271 1.31.2.2 christos } else { 2272 1.31.2.2 christos opsize = 4; 2273 1.31.2.2 christos } 2274 1.31.2.2 christos } else { /* 16bit */ 2275 1.31.2.2 christos if (instr->legpref.opr_ovr) { 2276 1.31.2.2 christos opsize = 4; 2277 1.31.2.2 christos } else { 2278 1.31.2.2 christos opsize = 2; 2279 1.31.2.2 christos } 2280 1.31.2.2 christos } 2281 1.31.2.2 christos } 2282 1.31.2.2 christos 2283 1.31.2.2 christos return opsize; 2284 1.31.2.2 christos } 2285 1.31.2.2 christos 2286 1.31.2.2 christos static size_t 2287 1.31.2.2 christos get_address_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2288 1.31.2.2 christos { 2289 1.31.2.2 christos if (fsm->is64bit) { 2290 1.31.2.2 christos if (__predict_false(instr->legpref.adr_ovr)) { 2291 1.31.2.2 christos return 4; 2292 1.31.2.2 christos } 2293 1.31.2.2 christos return 8; 2294 1.31.2.2 christos } 2295 1.31.2.2 christos 2296 1.31.2.2 christos if (fsm->is32bit) { 2297 1.31.2.2 christos if (__predict_false(instr->legpref.adr_ovr)) { 2298 1.31.2.2 christos return 2; 2299 1.31.2.2 christos } 2300 1.31.2.2 christos return 4; 2301 1.31.2.2 christos } 2302 1.31.2.2 christos 2303 1.31.2.2 christos /* 16bit. */ 2304 1.31.2.2 christos if (__predict_false(instr->legpref.adr_ovr)) { 2305 1.31.2.2 christos return 4; 2306 1.31.2.2 christos } 2307 1.31.2.2 christos return 2; 2308 1.31.2.2 christos } 2309 1.31.2.2 christos 2310 1.31.2.2 christos static int 2311 1.31.2.2 christos node_primary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2312 1.31.2.2 christos { 2313 1.31.2.2 christos const struct x86_opcode *opcode; 2314 1.31.2.2 christos uint8_t byte; 2315 1.31.2.2 christos 2316 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2317 1.31.2.2 christos return -1; 2318 1.31.2.2 christos } 2319 1.31.2.2 christos 2320 1.31.2.2 christos opcode = &primary_opcode_table[byte]; 2321 1.31.2.2 christos if (__predict_false(!opcode->valid)) { 2322 1.31.2.2 christos return -1; 2323 1.31.2.2 christos } 2324 1.31.2.2 christos 2325 1.31.2.2 christos instr->opcode = opcode; 2326 1.31.2.2 christos instr->emul = opcode->emul; 2327 1.31.2.2 christos instr->operand_size = get_operand_size(fsm, instr); 2328 1.31.2.2 christos instr->address_size = get_address_size(fsm, instr); 2329 1.31.2.2 christos 2330 1.31.2.2 christos if (fsm->is64bit && (instr->operand_size == 4)) { 2331 1.31.2.2 christos /* Zero-extend to 64 bits. */ 2332 1.31.2.2 christos instr->zeroextend_mask = ~size_to_mask(4); 2333 1.31.2.2 christos } 2334 1.31.2.2 christos 2335 1.31.2.2 christos if (opcode->regmodrm) { 2336 1.31.2.2 christos fsm_advance(fsm, 1, node_regmodrm); 2337 1.31.2.2 christos } else if (opcode->dmo) { 2338 1.31.2.2 christos /* Direct-Memory Offsets */ 2339 1.31.2.2 christos fsm_advance(fsm, 1, node_dmo); 2340 1.31.2.2 christos } else if (opcode->stos || opcode->lods) { 2341 1.31.2.2 christos fsm_advance(fsm, 1, node_stlo); 2342 1.31.2.2 christos } else if (opcode->movs) { 2343 1.31.2.2 christos fsm_advance(fsm, 1, node_movs); 2344 1.31.2.2 christos } else { 2345 1.31.2.2 christos return -1; 2346 1.31.2.2 christos } 2347 1.31.2.2 christos 2348 1.31.2.2 christos return 0; 2349 1.31.2.2 christos } 2350 1.31.2.2 christos 2351 1.31.2.2 christos static int 2352 1.31.2.2 christos node_secondary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2353 1.31.2.2 christos { 2354 1.31.2.2 christos const struct x86_opcode *opcode; 2355 1.31.2.2 christos uint8_t byte; 2356 1.31.2.2 christos 2357 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2358 1.31.2.2 christos return -1; 2359 1.31.2.2 christos } 2360 1.31.2.2 christos 2361 1.31.2.2 christos opcode = &secondary_opcode_table[byte]; 2362 1.31.2.2 christos if (__predict_false(!opcode->valid)) { 2363 1.31.2.2 christos return -1; 2364 1.31.2.2 christos } 2365 1.31.2.2 christos 2366 1.31.2.2 christos instr->opcode = opcode; 2367 1.31.2.2 christos instr->emul = opcode->emul; 2368 1.31.2.2 christos instr->operand_size = get_operand_size(fsm, instr); 2369 1.31.2.2 christos instr->address_size = get_address_size(fsm, instr); 2370 1.31.2.2 christos 2371 1.31.2.2 christos if (fsm->is64bit && (instr->operand_size == 4)) { 2372 1.31.2.2 christos /* Zero-extend to 64 bits. */ 2373 1.31.2.2 christos instr->zeroextend_mask = ~size_to_mask(4); 2374 1.31.2.2 christos } 2375 1.31.2.2 christos 2376 1.31.2.2 christos if (opcode->flags & FLAG_ze) { 2377 1.31.2.2 christos /* 2378 1.31.2.2 christos * Compute the mask for zero-extend. Update the operand size, 2379 1.31.2.2 christos * we move fewer bytes. 2380 1.31.2.2 christos */ 2381 1.31.2.2 christos instr->zeroextend_mask |= size_to_mask(instr->operand_size); 2382 1.31.2.2 christos instr->zeroextend_mask &= ~size_to_mask(opcode->defsize); 2383 1.31.2.2 christos instr->operand_size = opcode->defsize; 2384 1.31.2.2 christos } 2385 1.31.2.2 christos 2386 1.31.2.2 christos if (opcode->regmodrm) { 2387 1.31.2.2 christos fsm_advance(fsm, 1, node_regmodrm); 2388 1.31.2.2 christos } else { 2389 1.31.2.2 christos return -1; 2390 1.31.2.2 christos } 2391 1.31.2.2 christos 2392 1.31.2.2 christos return 0; 2393 1.31.2.2 christos } 2394 1.31.2.2 christos 2395 1.31.2.2 christos static int 2396 1.31.2.2 christos node_main(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2397 1.31.2.2 christos { 2398 1.31.2.2 christos uint8_t byte; 2399 1.31.2.2 christos 2400 1.31.2.2 christos #define ESCAPE 0x0F 2401 1.31.2.2 christos #define VEX_1 0xC5 2402 1.31.2.2 christos #define VEX_2 0xC4 2403 1.31.2.2 christos #define XOP 0x8F 2404 1.31.2.2 christos 2405 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2406 1.31.2.2 christos return -1; 2407 1.31.2.2 christos } 2408 1.31.2.2 christos 2409 1.31.2.2 christos /* 2410 1.31.2.2 christos * We don't take XOP. It is AMD-specific, and it was removed shortly 2411 1.31.2.2 christos * after being introduced. 2412 1.31.2.2 christos */ 2413 1.31.2.2 christos if (byte == ESCAPE) { 2414 1.31.2.2 christos fsm_advance(fsm, 1, node_secondary_opcode); 2415 1.31.2.2 christos } else if (!instr->rexpref.present) { 2416 1.31.2.2 christos if (byte == VEX_1) { 2417 1.31.2.2 christos return -1; 2418 1.31.2.2 christos } else if (byte == VEX_2) { 2419 1.31.2.2 christos return -1; 2420 1.31.2.2 christos } else { 2421 1.31.2.2 christos fsm->fn = node_primary_opcode; 2422 1.31.2.2 christos } 2423 1.31.2.2 christos } else { 2424 1.31.2.2 christos fsm->fn = node_primary_opcode; 2425 1.31.2.2 christos } 2426 1.31.2.2 christos 2427 1.31.2.2 christos return 0; 2428 1.31.2.2 christos } 2429 1.31.2.2 christos 2430 1.31.2.2 christos static int 2431 1.31.2.2 christos node_rex_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2432 1.31.2.2 christos { 2433 1.31.2.2 christos struct x86_rexpref *rexpref = &instr->rexpref; 2434 1.31.2.2 christos uint8_t byte; 2435 1.31.2.2 christos size_t n = 0; 2436 1.31.2.2 christos 2437 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2438 1.31.2.2 christos return -1; 2439 1.31.2.2 christos } 2440 1.31.2.2 christos 2441 1.31.2.2 christos if (byte >= 0x40 && byte <= 0x4F) { 2442 1.31.2.2 christos if (__predict_false(!fsm->is64bit)) { 2443 1.31.2.2 christos return -1; 2444 1.31.2.2 christos } 2445 1.31.2.2 christos rexpref->b = ((byte & 0x1) != 0); 2446 1.31.2.2 christos rexpref->x = ((byte & 0x2) != 0); 2447 1.31.2.2 christos rexpref->r = ((byte & 0x4) != 0); 2448 1.31.2.2 christos rexpref->w = ((byte & 0x8) != 0); 2449 1.31.2.2 christos rexpref->present = true; 2450 1.31.2.2 christos n = 1; 2451 1.31.2.2 christos } 2452 1.31.2.2 christos 2453 1.31.2.2 christos fsm_advance(fsm, n, node_main); 2454 1.31.2.2 christos return 0; 2455 1.31.2.2 christos } 2456 1.31.2.2 christos 2457 1.31.2.2 christos static int 2458 1.31.2.2 christos node_legacy_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2459 1.31.2.2 christos { 2460 1.31.2.2 christos uint8_t byte; 2461 1.31.2.2 christos 2462 1.31.2.2 christos if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2463 1.31.2.2 christos return -1; 2464 1.31.2.2 christos } 2465 1.31.2.2 christos 2466 1.31.2.2 christos if (byte == LEG_OPR_OVR) { 2467 1.31.2.2 christos instr->legpref.opr_ovr = 1; 2468 1.31.2.2 christos } else if (byte == LEG_OVR_DS) { 2469 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_DS; 2470 1.31.2.2 christos } else if (byte == LEG_OVR_ES) { 2471 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_ES; 2472 1.31.2.2 christos } else if (byte == LEG_REP) { 2473 1.31.2.2 christos instr->legpref.rep = 1; 2474 1.31.2.2 christos } else if (byte == LEG_OVR_GS) { 2475 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_GS; 2476 1.31.2.2 christos } else if (byte == LEG_OVR_FS) { 2477 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_FS; 2478 1.31.2.2 christos } else if (byte == LEG_ADR_OVR) { 2479 1.31.2.2 christos instr->legpref.adr_ovr = 1; 2480 1.31.2.2 christos } else if (byte == LEG_OVR_CS) { 2481 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_CS; 2482 1.31.2.2 christos } else if (byte == LEG_OVR_SS) { 2483 1.31.2.2 christos instr->legpref.seg = NVMM_X64_SEG_SS; 2484 1.31.2.2 christos } else if (byte == LEG_REPN) { 2485 1.31.2.2 christos instr->legpref.repn = 1; 2486 1.31.2.2 christos } else if (byte == LEG_LOCK) { 2487 1.31.2.2 christos /* ignore */ 2488 1.31.2.2 christos } else { 2489 1.31.2.2 christos /* not a legacy prefix */ 2490 1.31.2.2 christos fsm_advance(fsm, 0, node_rex_prefix); 2491 1.31.2.2 christos return 0; 2492 1.31.2.2 christos } 2493 1.31.2.2 christos 2494 1.31.2.2 christos fsm_advance(fsm, 1, node_legacy_prefix); 2495 1.31.2.2 christos return 0; 2496 1.31.2.2 christos } 2497 1.31.2.2 christos 2498 1.31.2.2 christos static int 2499 1.31.2.2 christos x86_decode(uint8_t *inst_bytes, size_t inst_len, struct x86_instr *instr, 2500 1.31.2.2 christos struct nvmm_x64_state *state) 2501 1.31.2.2 christos { 2502 1.31.2.2 christos struct x86_decode_fsm fsm; 2503 1.31.2.2 christos int ret; 2504 1.31.2.2 christos 2505 1.31.2.2 christos memset(instr, 0, sizeof(*instr)); 2506 1.31.2.2 christos instr->legpref.seg = -1; 2507 1.31.2.2 christos instr->src.hardseg = -1; 2508 1.31.2.2 christos instr->dst.hardseg = -1; 2509 1.31.2.2 christos 2510 1.31.2.2 christos fsm.is64bit = is_64bit(state); 2511 1.31.2.2 christos fsm.is32bit = is_32bit(state); 2512 1.31.2.2 christos fsm.is16bit = is_16bit(state); 2513 1.31.2.2 christos 2514 1.31.2.2 christos fsm.fn = node_legacy_prefix; 2515 1.31.2.2 christos fsm.buf = inst_bytes; 2516 1.31.2.2 christos fsm.end = inst_bytes + inst_len; 2517 1.31.2.2 christos 2518 1.31.2.2 christos while (fsm.fn != NULL) { 2519 1.31.2.2 christos ret = (*fsm.fn)(&fsm, instr); 2520 1.31.2.2 christos if (ret == -1) 2521 1.31.2.2 christos return -1; 2522 1.31.2.2 christos } 2523 1.31.2.2 christos 2524 1.31.2.2 christos instr->len = fsm.buf - inst_bytes; 2525 1.31.2.2 christos 2526 1.31.2.2 christos return 0; 2527 1.31.2.2 christos } 2528 1.31.2.2 christos 2529 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 2530 1.31.2.2 christos 2531 1.31.2.2 christos #define EXEC_INSTR(sz, instr) \ 2532 1.31.2.2 christos static uint##sz##_t \ 2533 1.31.2.2 christos exec_##instr##sz(uint##sz##_t op1, uint##sz##_t op2, uint64_t *rflags) \ 2534 1.31.2.2 christos { \ 2535 1.31.2.2 christos uint##sz##_t res; \ 2536 1.31.2.2 christos __asm __volatile ( \ 2537 1.31.2.2 christos #instr " %2, %3;" \ 2538 1.31.2.2 christos "mov %3, %1;" \ 2539 1.31.2.2 christos "pushfq;" \ 2540 1.31.2.2 christos "popq %0" \ 2541 1.31.2.2 christos : "=r" (*rflags), "=r" (res) \ 2542 1.31.2.2 christos : "r" (op1), "r" (op2)); \ 2543 1.31.2.2 christos return res; \ 2544 1.31.2.2 christos } 2545 1.31.2.2 christos 2546 1.31.2.2 christos #define EXEC_DISPATCHER(instr) \ 2547 1.31.2.2 christos static uint64_t \ 2548 1.31.2.2 christos exec_##instr(uint64_t op1, uint64_t op2, uint64_t *rflags, size_t opsize) \ 2549 1.31.2.2 christos { \ 2550 1.31.2.2 christos switch (opsize) { \ 2551 1.31.2.2 christos case 1: \ 2552 1.31.2.2 christos return exec_##instr##8(op1, op2, rflags); \ 2553 1.31.2.2 christos case 2: \ 2554 1.31.2.2 christos return exec_##instr##16(op1, op2, rflags); \ 2555 1.31.2.2 christos case 4: \ 2556 1.31.2.2 christos return exec_##instr##32(op1, op2, rflags); \ 2557 1.31.2.2 christos default: \ 2558 1.31.2.2 christos return exec_##instr##64(op1, op2, rflags); \ 2559 1.31.2.2 christos } \ 2560 1.31.2.2 christos } 2561 1.31.2.2 christos 2562 1.31.2.2 christos /* SUB: ret = op1 - op2 */ 2563 1.31.2.2 christos #define PSL_SUB_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF|PSL_AF) 2564 1.31.2.2 christos EXEC_INSTR(8, sub) 2565 1.31.2.2 christos EXEC_INSTR(16, sub) 2566 1.31.2.2 christos EXEC_INSTR(32, sub) 2567 1.31.2.2 christos EXEC_INSTR(64, sub) 2568 1.31.2.2 christos EXEC_DISPATCHER(sub) 2569 1.31.2.2 christos 2570 1.31.2.2 christos /* OR: ret = op1 | op2 */ 2571 1.31.2.2 christos #define PSL_OR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2572 1.31.2.2 christos EXEC_INSTR(8, or) 2573 1.31.2.2 christos EXEC_INSTR(16, or) 2574 1.31.2.2 christos EXEC_INSTR(32, or) 2575 1.31.2.2 christos EXEC_INSTR(64, or) 2576 1.31.2.2 christos EXEC_DISPATCHER(or) 2577 1.31.2.2 christos 2578 1.31.2.2 christos /* AND: ret = op1 & op2 */ 2579 1.31.2.2 christos #define PSL_AND_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2580 1.31.2.2 christos EXEC_INSTR(8, and) 2581 1.31.2.2 christos EXEC_INSTR(16, and) 2582 1.31.2.2 christos EXEC_INSTR(32, and) 2583 1.31.2.2 christos EXEC_INSTR(64, and) 2584 1.31.2.2 christos EXEC_DISPATCHER(and) 2585 1.31.2.2 christos 2586 1.31.2.2 christos /* XOR: ret = op1 ^ op2 */ 2587 1.31.2.2 christos #define PSL_XOR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2588 1.31.2.2 christos EXEC_INSTR(8, xor) 2589 1.31.2.2 christos EXEC_INSTR(16, xor) 2590 1.31.2.2 christos EXEC_INSTR(32, xor) 2591 1.31.2.2 christos EXEC_INSTR(64, xor) 2592 1.31.2.2 christos EXEC_DISPATCHER(xor) 2593 1.31.2.2 christos 2594 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 2595 1.31.2.2 christos 2596 1.31.2.2 christos /* 2597 1.31.2.2 christos * Emulation functions. We don't care about the order of the operands, except 2598 1.31.2.2 christos * for SUB, CMP and TEST. For these ones we look at mem->write todetermine who 2599 1.31.2.2 christos * is op1 and who is op2. 2600 1.31.2.2 christos */ 2601 1.31.2.2 christos 2602 1.31.2.2 christos static void 2603 1.31.2.2 christos x86_func_or(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2604 1.31.2.2 christos { 2605 1.31.2.2 christos uint64_t *retval = (uint64_t *)mem->data; 2606 1.31.2.2 christos const bool write = mem->write; 2607 1.31.2.2 christos uint64_t *op1, op2, fl, ret; 2608 1.31.2.2 christos 2609 1.31.2.2 christos op1 = (uint64_t *)mem->data; 2610 1.31.2.2 christos op2 = 0; 2611 1.31.2.2 christos 2612 1.31.2.2 christos /* Fetch the value to be OR'ed (op2). */ 2613 1.31.2.2 christos mem->data = (uint8_t *)&op2; 2614 1.31.2.2 christos mem->write = false; 2615 1.31.2.2 christos (*mach->cbs.mem)(mem); 2616 1.31.2.2 christos 2617 1.31.2.2 christos /* Perform the OR. */ 2618 1.31.2.2 christos ret = exec_or(*op1, op2, &fl, mem->size); 2619 1.31.2.2 christos 2620 1.31.2.2 christos if (write) { 2621 1.31.2.2 christos /* Write back the result. */ 2622 1.31.2.2 christos mem->data = (uint8_t *)&ret; 2623 1.31.2.2 christos mem->write = true; 2624 1.31.2.2 christos (*mach->cbs.mem)(mem); 2625 1.31.2.2 christos } else { 2626 1.31.2.2 christos /* Return data to the caller. */ 2627 1.31.2.2 christos *retval = ret; 2628 1.31.2.2 christos } 2629 1.31.2.2 christos 2630 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_OR_MASK; 2631 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_OR_MASK); 2632 1.31.2.2 christos } 2633 1.31.2.2 christos 2634 1.31.2.2 christos static void 2635 1.31.2.2 christos x86_func_and(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2636 1.31.2.2 christos { 2637 1.31.2.2 christos uint64_t *retval = (uint64_t *)mem->data; 2638 1.31.2.2 christos const bool write = mem->write; 2639 1.31.2.2 christos uint64_t *op1, op2, fl, ret; 2640 1.31.2.2 christos 2641 1.31.2.2 christos op1 = (uint64_t *)mem->data; 2642 1.31.2.2 christos op2 = 0; 2643 1.31.2.2 christos 2644 1.31.2.2 christos /* Fetch the value to be AND'ed (op2). */ 2645 1.31.2.2 christos mem->data = (uint8_t *)&op2; 2646 1.31.2.2 christos mem->write = false; 2647 1.31.2.2 christos (*mach->cbs.mem)(mem); 2648 1.31.2.2 christos 2649 1.31.2.2 christos /* Perform the AND. */ 2650 1.31.2.2 christos ret = exec_and(*op1, op2, &fl, mem->size); 2651 1.31.2.2 christos 2652 1.31.2.2 christos if (write) { 2653 1.31.2.2 christos /* Write back the result. */ 2654 1.31.2.2 christos mem->data = (uint8_t *)&ret; 2655 1.31.2.2 christos mem->write = true; 2656 1.31.2.2 christos (*mach->cbs.mem)(mem); 2657 1.31.2.2 christos } else { 2658 1.31.2.2 christos /* Return data to the caller. */ 2659 1.31.2.2 christos *retval = ret; 2660 1.31.2.2 christos } 2661 1.31.2.2 christos 2662 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2663 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2664 1.31.2.2 christos } 2665 1.31.2.2 christos 2666 1.31.2.2 christos static void 2667 1.31.2.2 christos x86_func_sub(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2668 1.31.2.2 christos { 2669 1.31.2.2 christos uint64_t *retval = (uint64_t *)mem->data; 2670 1.31.2.2 christos const bool write = mem->write; 2671 1.31.2.2 christos uint64_t *op1, *op2, fl, ret; 2672 1.31.2.2 christos uint64_t tmp; 2673 1.31.2.2 christos bool memop1; 2674 1.31.2.2 christos 2675 1.31.2.2 christos memop1 = !mem->write; 2676 1.31.2.2 christos op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2677 1.31.2.2 christos op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2678 1.31.2.2 christos 2679 1.31.2.2 christos /* Fetch the value to be SUB'ed (op1 or op2). */ 2680 1.31.2.2 christos mem->data = (uint8_t *)&tmp; 2681 1.31.2.2 christos mem->write = false; 2682 1.31.2.2 christos (*mach->cbs.mem)(mem); 2683 1.31.2.2 christos 2684 1.31.2.2 christos /* Perform the SUB. */ 2685 1.31.2.2 christos ret = exec_sub(*op1, *op2, &fl, mem->size); 2686 1.31.2.2 christos 2687 1.31.2.2 christos if (write) { 2688 1.31.2.2 christos /* Write back the result. */ 2689 1.31.2.2 christos mem->data = (uint8_t *)&ret; 2690 1.31.2.2 christos mem->write = true; 2691 1.31.2.2 christos (*mach->cbs.mem)(mem); 2692 1.31.2.2 christos } else { 2693 1.31.2.2 christos /* Return data to the caller. */ 2694 1.31.2.2 christos *retval = ret; 2695 1.31.2.2 christos } 2696 1.31.2.2 christos 2697 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2698 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2699 1.31.2.2 christos } 2700 1.31.2.2 christos 2701 1.31.2.2 christos static void 2702 1.31.2.2 christos x86_func_xor(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2703 1.31.2.2 christos { 2704 1.31.2.2 christos uint64_t *retval = (uint64_t *)mem->data; 2705 1.31.2.2 christos const bool write = mem->write; 2706 1.31.2.2 christos uint64_t *op1, op2, fl, ret; 2707 1.31.2.2 christos 2708 1.31.2.2 christos op1 = (uint64_t *)mem->data; 2709 1.31.2.2 christos op2 = 0; 2710 1.31.2.2 christos 2711 1.31.2.2 christos /* Fetch the value to be XOR'ed (op2). */ 2712 1.31.2.2 christos mem->data = (uint8_t *)&op2; 2713 1.31.2.2 christos mem->write = false; 2714 1.31.2.2 christos (*mach->cbs.mem)(mem); 2715 1.31.2.2 christos 2716 1.31.2.2 christos /* Perform the XOR. */ 2717 1.31.2.2 christos ret = exec_xor(*op1, op2, &fl, mem->size); 2718 1.31.2.2 christos 2719 1.31.2.2 christos if (write) { 2720 1.31.2.2 christos /* Write back the result. */ 2721 1.31.2.2 christos mem->data = (uint8_t *)&ret; 2722 1.31.2.2 christos mem->write = true; 2723 1.31.2.2 christos (*mach->cbs.mem)(mem); 2724 1.31.2.2 christos } else { 2725 1.31.2.2 christos /* Return data to the caller. */ 2726 1.31.2.2 christos *retval = ret; 2727 1.31.2.2 christos } 2728 1.31.2.2 christos 2729 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_XOR_MASK; 2730 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_XOR_MASK); 2731 1.31.2.2 christos } 2732 1.31.2.2 christos 2733 1.31.2.2 christos static void 2734 1.31.2.2 christos x86_func_cmp(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2735 1.31.2.2 christos { 2736 1.31.2.2 christos uint64_t *op1, *op2, fl; 2737 1.31.2.2 christos uint64_t tmp; 2738 1.31.2.2 christos bool memop1; 2739 1.31.2.2 christos 2740 1.31.2.2 christos memop1 = !mem->write; 2741 1.31.2.2 christos op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2742 1.31.2.2 christos op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2743 1.31.2.2 christos 2744 1.31.2.2 christos /* Fetch the value to be CMP'ed (op1 or op2). */ 2745 1.31.2.2 christos mem->data = (uint8_t *)&tmp; 2746 1.31.2.2 christos mem->write = false; 2747 1.31.2.2 christos (*mach->cbs.mem)(mem); 2748 1.31.2.2 christos 2749 1.31.2.2 christos /* Perform the CMP. */ 2750 1.31.2.2 christos exec_sub(*op1, *op2, &fl, mem->size); 2751 1.31.2.2 christos 2752 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2753 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2754 1.31.2.2 christos } 2755 1.31.2.2 christos 2756 1.31.2.2 christos static void 2757 1.31.2.2 christos x86_func_test(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2758 1.31.2.2 christos { 2759 1.31.2.2 christos uint64_t *op1, *op2, fl; 2760 1.31.2.2 christos uint64_t tmp; 2761 1.31.2.2 christos bool memop1; 2762 1.31.2.2 christos 2763 1.31.2.2 christos memop1 = !mem->write; 2764 1.31.2.2 christos op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2765 1.31.2.2 christos op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2766 1.31.2.2 christos 2767 1.31.2.2 christos /* Fetch the value to be TEST'ed (op1 or op2). */ 2768 1.31.2.2 christos mem->data = (uint8_t *)&tmp; 2769 1.31.2.2 christos mem->write = false; 2770 1.31.2.2 christos (*mach->cbs.mem)(mem); 2771 1.31.2.2 christos 2772 1.31.2.2 christos /* Perform the TEST. */ 2773 1.31.2.2 christos exec_and(*op1, *op2, &fl, mem->size); 2774 1.31.2.2 christos 2775 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2776 1.31.2.2 christos gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2777 1.31.2.2 christos } 2778 1.31.2.2 christos 2779 1.31.2.2 christos static void 2780 1.31.2.2 christos x86_func_mov(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2781 1.31.2.2 christos { 2782 1.31.2.2 christos /* 2783 1.31.2.2 christos * Nothing special, just move without emulation. 2784 1.31.2.2 christos */ 2785 1.31.2.2 christos (*mach->cbs.mem)(mem); 2786 1.31.2.2 christos } 2787 1.31.2.2 christos 2788 1.31.2.2 christos static void 2789 1.31.2.2 christos x86_func_stos(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2790 1.31.2.2 christos { 2791 1.31.2.2 christos /* 2792 1.31.2.2 christos * Just move, and update RDI. 2793 1.31.2.2 christos */ 2794 1.31.2.2 christos (*mach->cbs.mem)(mem); 2795 1.31.2.2 christos 2796 1.31.2.2 christos if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2797 1.31.2.2 christos gprs[NVMM_X64_GPR_RDI] -= mem->size; 2798 1.31.2.2 christos } else { 2799 1.31.2.2 christos gprs[NVMM_X64_GPR_RDI] += mem->size; 2800 1.31.2.2 christos } 2801 1.31.2.2 christos } 2802 1.31.2.2 christos 2803 1.31.2.2 christos static void 2804 1.31.2.2 christos x86_func_lods(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2805 1.31.2.2 christos { 2806 1.31.2.2 christos /* 2807 1.31.2.2 christos * Just move, and update RSI. 2808 1.31.2.2 christos */ 2809 1.31.2.2 christos (*mach->cbs.mem)(mem); 2810 1.31.2.2 christos 2811 1.31.2.2 christos if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2812 1.31.2.2 christos gprs[NVMM_X64_GPR_RSI] -= mem->size; 2813 1.31.2.2 christos } else { 2814 1.31.2.2 christos gprs[NVMM_X64_GPR_RSI] += mem->size; 2815 1.31.2.2 christos } 2816 1.31.2.2 christos } 2817 1.31.2.2 christos 2818 1.31.2.2 christos static void 2819 1.31.2.2 christos x86_func_movs(struct nvmm_machine *mach, struct nvmm_mem *mem, uint64_t *gprs) 2820 1.31.2.2 christos { 2821 1.31.2.2 christos /* 2822 1.31.2.2 christos * Special instruction: double memory operand. Don't call the cb, 2823 1.31.2.2 christos * because the storage has already been performed earlier. 2824 1.31.2.2 christos */ 2825 1.31.2.2 christos 2826 1.31.2.2 christos if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2827 1.31.2.2 christos gprs[NVMM_X64_GPR_RSI] -= mem->size; 2828 1.31.2.2 christos gprs[NVMM_X64_GPR_RDI] -= mem->size; 2829 1.31.2.2 christos } else { 2830 1.31.2.2 christos gprs[NVMM_X64_GPR_RSI] += mem->size; 2831 1.31.2.2 christos gprs[NVMM_X64_GPR_RDI] += mem->size; 2832 1.31.2.2 christos } 2833 1.31.2.2 christos } 2834 1.31.2.2 christos 2835 1.31.2.2 christos /* -------------------------------------------------------------------------- */ 2836 1.31.2.2 christos 2837 1.31.2.2 christos static inline uint64_t 2838 1.31.2.2 christos gpr_read_address(struct x86_instr *instr, struct nvmm_x64_state *state, int gpr) 2839 1.31.2.2 christos { 2840 1.31.2.2 christos uint64_t val; 2841 1.31.2.2 christos 2842 1.31.2.2 christos val = state->gprs[gpr]; 2843 1.31.2.2 christos val &= size_to_mask(instr->address_size); 2844 1.31.2.2 christos 2845 1.31.2.2 christos return val; 2846 1.31.2.2 christos } 2847 1.31.2.2 christos 2848 1.31.2.2 christos static int 2849 1.31.2.2 christos store_to_gva(struct nvmm_x64_state *state, struct x86_instr *instr, 2850 1.31.2.2 christos struct x86_store *store, gvaddr_t *gvap, size_t size) 2851 1.31.2.2 christos { 2852 1.31.2.2 christos struct x86_sib *sib; 2853 1.31.2.2 christos gvaddr_t gva = 0; 2854 1.31.2.2 christos uint64_t reg; 2855 1.31.2.2 christos int ret, seg; 2856 1.31.2.2 christos 2857 1.31.2.2 christos if (store->type == STORE_SIB) { 2858 1.31.2.2 christos sib = &store->u.sib; 2859 1.31.2.2 christos if (sib->bas != NULL) 2860 1.31.2.2 christos gva += gpr_read_address(instr, state, sib->bas->num); 2861 1.31.2.2 christos if (sib->idx != NULL) { 2862 1.31.2.2 christos reg = gpr_read_address(instr, state, sib->idx->num); 2863 1.31.2.2 christos gva += sib->scale * reg; 2864 1.31.2.2 christos } 2865 1.31.2.2 christos } else if (store->type == STORE_REG) { 2866 1.31.2.2 christos if (store->u.reg == NULL) { 2867 1.31.2.2 christos /* The base is null. Happens with disp32-only. */ 2868 1.31.2.2 christos } else { 2869 1.31.2.2 christos gva = gpr_read_address(instr, state, store->u.reg->num); 2870 1.31.2.2 christos } 2871 1.31.2.2 christos } else { 2872 1.31.2.2 christos gva = store->u.dmo; 2873 1.31.2.2 christos } 2874 1.31.2.2 christos 2875 1.31.2.2 christos if (store->disp.type != DISP_NONE) { 2876 1.31.2.2 christos gva += store->disp.data; 2877 1.31.2.2 christos } 2878 1.31.2.2 christos 2879 1.31.2.2 christos if (store->hardseg != -1) { 2880 1.31.2.2 christos seg = store->hardseg; 2881 1.31.2.2 christos } else { 2882 1.31.2.2 christos if (__predict_false(instr->legpref.seg != -1)) { 2883 1.31.2.2 christos seg = instr->legpref.seg; 2884 1.31.2.2 christos } else { 2885 1.31.2.2 christos seg = NVMM_X64_SEG_DS; 2886 1.31.2.2 christos } 2887 1.31.2.2 christos } 2888 1.31.2.2 christos 2889 1.31.2.2 christos if (__predict_true(is_long_mode(state))) { 2890 1.31.2.2 christos if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 2891 1.31.2.2 christos segment_apply(&state->segs[seg], &gva); 2892 1.31.2.2 christos } 2893 1.31.2.2 christos } else { 2894 1.31.2.2 christos ret = segment_check(&state->segs[seg], gva, size); 2895 1.31.2.2 christos if (ret == -1) 2896 1.31.2.2 christos return -1; 2897 1.31.2.2 christos segment_apply(&state->segs[seg], &gva); 2898 1.31.2.2 christos } 2899 1.31.2.2 christos 2900 1.31.2.2 christos *gvap = gva; 2901 1.31.2.2 christos return 0; 2902 1.31.2.2 christos } 2903 1.31.2.2 christos 2904 1.31.2.2 christos static int 2905 1.31.2.2 christos fetch_segment(struct nvmm_machine *mach, struct nvmm_x64_state *state) 2906 1.31.2.2 christos { 2907 1.31.2.2 christos uint8_t inst_bytes[5], byte; 2908 1.31.2.2 christos size_t i, fetchsize; 2909 1.31.2.2 christos gvaddr_t gva; 2910 1.31.2.2 christos int ret, seg; 2911 1.31.2.2 christos 2912 1.31.2.2 christos fetchsize = sizeof(inst_bytes); 2913 1.31.2.2 christos 2914 1.31.2.2 christos gva = state->gprs[NVMM_X64_GPR_RIP]; 2915 1.31.2.2 christos if (__predict_false(!is_long_mode(state))) { 2916 1.31.2.2 christos ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 2917 1.31.2.2 christos fetchsize); 2918 1.31.2.2 christos if (ret == -1) 2919 1.31.2.2 christos return -1; 2920 1.31.2.2 christos segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 2921 1.31.2.2 christos } 2922 1.31.2.2 christos 2923 1.31.2.2 christos ret = read_guest_memory(mach, state, gva, inst_bytes, fetchsize); 2924 1.31.2.2 christos if (ret == -1) 2925 1.31.2.2 christos return -1; 2926 1.31.2.2 christos 2927 1.31.2.2 christos seg = NVMM_X64_SEG_DS; 2928 1.31.2.2 christos for (i = 0; i < fetchsize; i++) { 2929 1.31.2.2 christos byte = inst_bytes[i]; 2930 1.31.2.2 christos 2931 1.31.2.2 christos if (byte == LEG_OVR_DS) { 2932 1.31.2.2 christos seg = NVMM_X64_SEG_DS; 2933 1.31.2.2 christos } else if (byte == LEG_OVR_ES) { 2934 1.31.2.2 christos seg = NVMM_X64_SEG_ES; 2935 1.31.2.2 christos } else if (byte == LEG_OVR_GS) { 2936 1.31.2.2 christos seg = NVMM_X64_SEG_GS; 2937 1.31.2.2 christos } else if (byte == LEG_OVR_FS) { 2938 1.31.2.2 christos seg = NVMM_X64_SEG_FS; 2939 1.31.2.2 christos } else if (byte == LEG_OVR_CS) { 2940 1.31.2.2 christos seg = NVMM_X64_SEG_CS; 2941 1.31.2.2 christos } else if (byte == LEG_OVR_SS) { 2942 1.31.2.2 christos seg = NVMM_X64_SEG_SS; 2943 1.31.2.2 christos } else if (byte == LEG_OPR_OVR) { 2944 1.31.2.2 christos /* nothing */ 2945 1.31.2.2 christos } else if (byte == LEG_ADR_OVR) { 2946 1.31.2.2 christos /* nothing */ 2947 1.31.2.2 christos } else if (byte == LEG_REP) { 2948 1.31.2.2 christos /* nothing */ 2949 1.31.2.2 christos } else if (byte == LEG_REPN) { 2950 1.31.2.2 christos /* nothing */ 2951 1.31.2.2 christos } else if (byte == LEG_LOCK) { 2952 1.31.2.2 christos /* nothing */ 2953 1.31.2.2 christos } else { 2954 1.31.2.2 christos return seg; 2955 1.31.2.2 christos } 2956 1.31.2.2 christos } 2957 1.31.2.2 christos 2958 1.31.2.2 christos return seg; 2959 1.31.2.2 christos } 2960 1.31.2.2 christos 2961 1.31.2.2 christos static int 2962 1.31.2.2 christos fetch_instruction(struct nvmm_machine *mach, struct nvmm_x64_state *state, 2963 1.31.2.2 christos struct nvmm_exit *exit) 2964 1.31.2.2 christos { 2965 1.31.2.2 christos size_t fetchsize; 2966 1.31.2.2 christos gvaddr_t gva; 2967 1.31.2.2 christos int ret; 2968 1.31.2.2 christos 2969 1.31.2.2 christos fetchsize = sizeof(exit->u.mem.inst_bytes); 2970 1.31.2.2 christos 2971 1.31.2.2 christos gva = state->gprs[NVMM_X64_GPR_RIP]; 2972 1.31.2.2 christos if (__predict_false(!is_long_mode(state))) { 2973 1.31.2.2 christos ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 2974 1.31.2.2 christos fetchsize); 2975 1.31.2.2 christos if (ret == -1) 2976 1.31.2.2 christos return -1; 2977 1.31.2.2 christos segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 2978 1.31.2.2 christos } 2979 1.31.2.2 christos 2980 1.31.2.2 christos ret = read_guest_memory(mach, state, gva, exit->u.mem.inst_bytes, 2981 1.31.2.2 christos fetchsize); 2982 1.31.2.2 christos if (ret == -1) 2983 1.31.2.2 christos return -1; 2984 1.31.2.2 christos 2985 1.31.2.2 christos exit->u.mem.inst_len = fetchsize; 2986 1.31.2.2 christos 2987 1.31.2.2 christos return 0; 2988 1.31.2.2 christos } 2989 1.31.2.2 christos 2990 1.31.2.2 christos static int 2991 1.31.2.2 christos assist_mem_double(struct nvmm_machine *mach, struct nvmm_x64_state *state, 2992 1.31.2.2 christos struct x86_instr *instr) 2993 1.31.2.2 christos { 2994 1.31.2.2 christos struct nvmm_mem mem; 2995 1.31.2.2 christos uint8_t data[8]; 2996 1.31.2.2 christos gvaddr_t gva; 2997 1.31.2.2 christos size_t size; 2998 1.31.2.2 christos int ret; 2999 1.31.2.2 christos 3000 1.31.2.2 christos size = instr->operand_size; 3001 1.31.2.2 christos 3002 1.31.2.2 christos /* Source. */ 3003 1.31.2.2 christos ret = store_to_gva(state, instr, &instr->src, &gva, size); 3004 1.31.2.2 christos if (ret == -1) 3005 1.31.2.2 christos return -1; 3006 1.31.2.2 christos ret = read_guest_memory(mach, state, gva, data, size); 3007 1.31.2.2 christos if (ret == -1) 3008 1.31.2.2 christos return -1; 3009 1.31.2.2 christos 3010 1.31.2.2 christos /* Destination. */ 3011 1.31.2.2 christos ret = store_to_gva(state, instr, &instr->dst, &gva, size); 3012 1.31.2.2 christos if (ret == -1) 3013 1.31.2.2 christos return -1; 3014 1.31.2.2 christos ret = write_guest_memory(mach, state, gva, data, size); 3015 1.31.2.2 christos if (ret == -1) 3016 1.31.2.2 christos return -1; 3017 1.31.2.2 christos 3018 1.31.2.2 christos mem.size = size; 3019 1.31.2.2 christos (*instr->emul->func)(mach, &mem, state->gprs); 3020 1.31.2.2 christos 3021 1.31.2.2 christos return 0; 3022 1.31.2.2 christos } 3023 1.31.2.2 christos 3024 1.31.2.2 christos #define DISASSEMBLER_BUG() \ 3025 1.31.2.2 christos do { \ 3026 1.31.2.2 christos errno = EINVAL; \ 3027 1.31.2.2 christos return -1; \ 3028 1.31.2.2 christos } while (0); 3029 1.31.2.2 christos 3030 1.31.2.2 christos static int 3031 1.31.2.2 christos assist_mem_single(struct nvmm_machine *mach, struct nvmm_x64_state *state, 3032 1.31.2.2 christos struct x86_instr *instr, struct nvmm_exit *exit) 3033 1.31.2.2 christos { 3034 1.31.2.2 christos struct nvmm_mem mem; 3035 1.31.2.2 christos uint8_t membuf[8]; 3036 1.31.2.2 christos uint64_t val; 3037 1.31.2.2 christos 3038 1.31.2.2 christos memset(membuf, 0, sizeof(membuf)); 3039 1.31.2.2 christos 3040 1.31.2.2 christos mem.gpa = exit->u.mem.gpa; 3041 1.31.2.2 christos mem.size = instr->operand_size; 3042 1.31.2.2 christos mem.data = membuf; 3043 1.31.2.2 christos 3044 1.31.2.2 christos /* Determine the direction. */ 3045 1.31.2.2 christos switch (instr->src.type) { 3046 1.31.2.2 christos case STORE_REG: 3047 1.31.2.2 christos if (instr->src.disp.type != DISP_NONE) { 3048 1.31.2.2 christos /* Indirect access. */ 3049 1.31.2.2 christos mem.write = false; 3050 1.31.2.2 christos } else { 3051 1.31.2.2 christos /* Direct access. */ 3052 1.31.2.2 christos mem.write = true; 3053 1.31.2.2 christos } 3054 1.31.2.2 christos break; 3055 1.31.2.2 christos case STORE_IMM: 3056 1.31.2.2 christos mem.write = true; 3057 1.31.2.2 christos break; 3058 1.31.2.2 christos case STORE_SIB: 3059 1.31.2.2 christos mem.write = false; 3060 1.31.2.2 christos break; 3061 1.31.2.2 christos case STORE_DMO: 3062 1.31.2.2 christos mem.write = false; 3063 1.31.2.2 christos break; 3064 1.31.2.2 christos default: 3065 1.31.2.2 christos DISASSEMBLER_BUG(); 3066 1.31.2.2 christos } 3067 1.31.2.2 christos 3068 1.31.2.2 christos if (mem.write) { 3069 1.31.2.2 christos switch (instr->src.type) { 3070 1.31.2.2 christos case STORE_REG: 3071 1.31.2.2 christos if (instr->src.disp.type != DISP_NONE) { 3072 1.31.2.2 christos DISASSEMBLER_BUG(); 3073 1.31.2.2 christos } 3074 1.31.2.2 christos val = state->gprs[instr->src.u.reg->num]; 3075 1.31.2.2 christos val = __SHIFTOUT(val, instr->src.u.reg->mask); 3076 1.31.2.2 christos memcpy(mem.data, &val, mem.size); 3077 1.31.2.2 christos break; 3078 1.31.2.2 christos case STORE_IMM: 3079 1.31.2.2 christos memcpy(mem.data, &instr->src.u.imm.data, mem.size); 3080 1.31.2.2 christos break; 3081 1.31.2.2 christos default: 3082 1.31.2.2 christos DISASSEMBLER_BUG(); 3083 1.31.2.2 christos } 3084 1.31.2.2 christos } else if (instr->emul->read) { 3085 1.31.2.2 christos if (instr->dst.type != STORE_REG) { 3086 1.31.2.2 christos DISASSEMBLER_BUG(); 3087 1.31.2.2 christos } 3088 1.31.2.2 christos if (instr->dst.disp.type != DISP_NONE) { 3089 1.31.2.2 christos DISASSEMBLER_BUG(); 3090 1.31.2.2 christos } 3091 1.31.2.2 christos val = state->gprs[instr->dst.u.reg->num]; 3092 1.31.2.2 christos val = __SHIFTOUT(val, instr->dst.u.reg->mask); 3093 1.31.2.2 christos memcpy(mem.data, &val, mem.size); 3094 1.31.2.2 christos } 3095 1.31.2.2 christos 3096 1.31.2.2 christos (*instr->emul->func)(mach, &mem, state->gprs); 3097 1.31.2.2 christos 3098 1.31.2.2 christos if (!instr->emul->notouch && !mem.write) { 3099 1.31.2.2 christos if (instr->dst.type != STORE_REG) { 3100 1.31.2.2 christos DISASSEMBLER_BUG(); 3101 1.31.2.2 christos } 3102 1.31.2.2 christos memcpy(&val, membuf, sizeof(uint64_t)); 3103 1.31.2.2 christos val = __SHIFTIN(val, instr->dst.u.reg->mask); 3104 1.31.2.2 christos state->gprs[instr->dst.u.reg->num] &= ~instr->dst.u.reg->mask; 3105 1.31.2.2 christos state->gprs[instr->dst.u.reg->num] |= val; 3106 1.31.2.2 christos state->gprs[instr->dst.u.reg->num] &= ~instr->zeroextend_mask; 3107 1.31.2.2 christos } 3108 1.31.2.2 christos 3109 1.31.2.2 christos return 0; 3110 1.31.2.2 christos } 3111 1.31.2.2 christos 3112 1.31.2.2 christos int 3113 1.31.2.2 christos nvmm_assist_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3114 1.31.2.2 christos { 3115 1.31.2.2 christos struct nvmm_x64_state *state = vcpu->state; 3116 1.31.2.2 christos struct nvmm_exit *exit = vcpu->exit; 3117 1.31.2.2 christos struct x86_instr instr; 3118 1.31.2.2 christos uint64_t cnt = 0; /* GCC */ 3119 1.31.2.2 christos int ret; 3120 1.31.2.2 christos 3121 1.31.2.2 christos if (__predict_false(exit->reason != NVMM_EXIT_MEMORY)) { 3122 1.31.2.2 christos errno = EINVAL; 3123 1.31.2.2 christos return -1; 3124 1.31.2.2 christos } 3125 1.31.2.2 christos 3126 1.31.2.2 christos ret = nvmm_vcpu_getstate(mach, vcpu, 3127 1.31.2.2 christos NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 3128 1.31.2.2 christos NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 3129 1.31.2.2 christos if (ret == -1) 3130 1.31.2.2 christos return -1; 3131 1.31.2.2 christos 3132 1.31.2.2 christos if (exit->u.mem.inst_len == 0) { 3133 1.31.2.2 christos /* 3134 1.31.2.2 christos * The instruction was not fetched from the kernel. Fetch 3135 1.31.2.2 christos * it ourselves. 3136 1.31.2.2 christos */ 3137 1.31.2.2 christos ret = fetch_instruction(mach, state, exit); 3138 1.31.2.2 christos if (ret == -1) 3139 1.31.2.2 christos return -1; 3140 1.31.2.2 christos } 3141 1.31.2.2 christos 3142 1.31.2.2 christos ret = x86_decode(exit->u.mem.inst_bytes, exit->u.mem.inst_len, 3143 1.31.2.2 christos &instr, state); 3144 1.31.2.2 christos if (ret == -1) { 3145 1.31.2.2 christos errno = ENODEV; 3146 1.31.2.2 christos return -1; 3147 1.31.2.2 christos } 3148 1.31.2.2 christos 3149 1.31.2.2 christos if (instr.legpref.rep || instr.legpref.repn) { 3150 1.31.2.2 christos cnt = rep_get_cnt(state, instr.address_size); 3151 1.31.2.2 christos if (__predict_false(cnt == 0)) { 3152 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3153 1.31.2.2 christos goto out; 3154 1.31.2.2 christos } 3155 1.31.2.2 christos } 3156 1.31.2.2 christos 3157 1.31.2.2 christos if (instr.opcode->movs) { 3158 1.31.2.2 christos ret = assist_mem_double(mach, state, &instr); 3159 1.31.2.2 christos } else { 3160 1.31.2.2 christos ret = assist_mem_single(mach, state, &instr, exit); 3161 1.31.2.2 christos } 3162 1.31.2.2 christos if (ret == -1) { 3163 1.31.2.2 christos errno = ENODEV; 3164 1.31.2.2 christos return -1; 3165 1.31.2.2 christos } 3166 1.31.2.2 christos 3167 1.31.2.2 christos if (instr.legpref.rep || instr.legpref.repn) { 3168 1.31.2.2 christos cnt -= 1; 3169 1.31.2.2 christos rep_set_cnt(state, instr.address_size, cnt); 3170 1.31.2.2 christos if (cnt == 0) { 3171 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3172 1.31.2.2 christos } else if (__predict_false(instr.legpref.repn)) { 3173 1.31.2.2 christos if (state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_Z) { 3174 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3175 1.31.2.2 christos } 3176 1.31.2.2 christos } 3177 1.31.2.2 christos } else { 3178 1.31.2.2 christos state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3179 1.31.2.2 christos } 3180 1.31.2.2 christos 3181 1.31.2.2 christos out: 3182 1.31.2.2 christos ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 3183 1.31.2.2 christos if (ret == -1) 3184 1.31.2.2 christos return -1; 3185 1.31.2.2 christos 3186 1.31.2.2 christos return 0; 3187 1.31.2.2 christos } 3188
Indexes created Sun Oct 19 02:09:48 GMT 2025