Lines Matching refs:cpudata
672 struct svm_cpudata *cpudata = vcpu->cpudata;
673 struct vmcb *vmcb = cpudata->vmcb;
677 cpudata->nmi_window_exit = true;
682 cpudata->int_window_exit = true;
691 struct svm_cpudata *cpudata = vcpu->cpudata;
692 struct vmcb *vmcb = cpudata->vmcb;
696 cpudata->nmi_window_exit = false;
701 cpudata->int_window_exit = false;
743 struct svm_cpudata *cpudata = vcpu->cpudata;
744 struct vmcb *vmcb = cpudata->vmcb;
786 cpudata->evt_pending = true;
849 svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx)
854 cpudata->vmcb->state.rax = descs[0];
855 cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1];
856 cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2];
857 cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3];
863 struct svm_cpudata *cpudata = vcpu->cpudata;
886 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
891 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
896 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
906 cpudata->vmcb->state.rax = svm_cpuid_max_basic;
909 cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax;
911 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID;
912 cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid,
915 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx;
916 cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ;
918 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx;
921 cr4 = cpudata->vmcb->state.cr4;
923 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~CPUID2_OSXSAVE;
931 cpudata->vmcb->state.rax = 0;
932 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
933 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
934 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
939 cpudata->vmcb->state.rax = 0;
940 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
941 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
942 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
945 cpudata->vmcb->state.rax = 0;
946 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
947 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
948 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
957 cpudata->vmcb->state.rax = 0;
958 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
959 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
960 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
968 cpudata->vmcb->state.rax = svm_xcr0_mask & 0xFFFFFFFF;
969 if (cpudata->gxcr0 & XCR0_SSE) {
970 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct fxsave);
972 cpudata->gprs[NVMM_X64_GPR_RBX] = sizeof(struct save87);
974 cpudata->gprs[NVMM_X64_GPR_RBX] += 64; /* XSAVE header */
975 cpudata->gprs[NVMM_X64_GPR_RCX] = sizeof(struct fxsave) + 64;
976 cpudata->gprs[NVMM_X64_GPR_RDX] = svm_xcr0_mask >> 32;
979 cpudata->vmcb->state.rax &=
982 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
983 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
984 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
987 cpudata->vmcb->state.rax = 0;
988 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
989 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
990 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
999 cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR;
1000 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1001 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1002 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1003 memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4);
1004 memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4);
1005 memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4);
1008 cpudata->gprs[NVMM_X64_GPR_RAX] = curcpu()->ci_data.cpu_cc_freq / 1000;
1010 cpudata->gprs[NVMM_X64_GPR_RBX] = lapic_per_second / 1000;
1012 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1013 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1014 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1021 cpudata->vmcb->state.rax = svm_cpuid_max_extended;
1024 cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax;
1025 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx;
1026 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx;
1027 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx;
1036 cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax;
1037 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx;
1038 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx;
1039 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx;
1042 cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax;
1043 cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx;
1044 cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx;
1045 cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx;
1063 cpudata->vmcb->state.rax = 0;
1064 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1065 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1066 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1073 cpudata->vmcb->state.rax = 0;
1074 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1075 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1076 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1082 cpudata->vmcb->state.rax = 0;
1083 cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
1084 cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
1085 cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
1104 struct svm_cpudata *cpudata = vcpu->cpudata;
1109 eax = cpudata->vmcb->state.rax;
1110 ecx = cpudata->gprs[NVMM_X64_GPR_RCX];
1111 svm_inkernel_exec_cpuid(cpudata, eax, ecx);
1115 if (!cpudata->cpuidpresent[i]) {
1118 cpuid = &cpudata->cpuid[i];
1124 svm_exit_insn(cpudata->vmcb, exit, NVMM_VCPU_EXIT_CPUID);
1130 cpudata->vmcb->state.rax &= ~cpuid->u.mask.del.eax;
1131 cpudata->gprs[NVMM_X64_GPR_RBX] &= ~cpuid->u.mask.del.ebx;
1132 cpudata->gprs[NVMM_X64_GPR_RCX] &= ~cpuid->u.mask.del.ecx;
1133 cpudata->gprs[NVMM_X64_GPR_RDX] &= ~cpuid->u.mask.del.edx;
1136 cpudata->vmcb->state.rax |= cpuid->u.mask.set.eax;
1137 cpudata->gprs[NVMM_X64_GPR_RBX] |= cpuid->u.mask.set.ebx;
1138 cpudata->gprs[NVMM_X64_GPR_RCX] |= cpuid->u.mask.set.ecx;
1139 cpudata->gprs[NVMM_X64_GPR_RDX] |= cpuid->u.mask.set.edx;
1144 svm_inkernel_advance(cpudata->vmcb);
1152 struct svm_cpudata *cpudata = vcpu->cpudata;
1153 struct vmcb *vmcb = cpudata->vmcb;
1155 if (cpudata->int_window_exit && (vmcb->state.rflags & PSL_I)) {
1159 svm_inkernel_advance(cpudata->vmcb);
1179 struct svm_cpudata *cpudata = vcpu->cpudata;
1180 uint64_t info = cpudata->vmcb->ctrl.exitinfo1;
1181 uint64_t nextpc = cpudata->vmcb->ctrl.exitinfo2;
1231 struct svm_cpudata *cpudata = vcpu->cpudata;
1232 struct vmcb *vmcb = cpudata->vmcb;
1240 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1246 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1254 cpudata->gprs[NVMM_X64_GPR_RDX] = (val >> 32);
1264 cpudata->gtlb_want_flush = true;
1271 cpudata->gtsc = exit->u.wrmsr.val;
1272 cpudata->gtsc_want_update = true;
1285 svm_inkernel_advance(cpudata->vmcb);
1297 struct svm_cpudata *cpudata = vcpu->cpudata;
1300 exit->u.rdmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF);
1301 exit->u.rdmsr.npc = cpudata->vmcb->ctrl.nrip;
1315 struct svm_cpudata *cpudata = vcpu->cpudata;
1318 rdx = cpudata->gprs[NVMM_X64_GPR_RDX];
1319 rax = cpudata->vmcb->state.rax;
1322 exit->u.wrmsr.msr = (cpudata->gprs[NVMM_X64_GPR_RCX] & 0xFFFFFFFF);
1324 exit->u.wrmsr.npc = cpudata->vmcb->ctrl.nrip;
1338 struct svm_cpudata *cpudata = vcpu->cpudata;
1339 uint64_t info = cpudata->vmcb->ctrl.exitinfo1;
1352 struct svm_cpudata *cpudata = vcpu->cpudata;
1353 gpaddr_t gpa = cpudata->vmcb->ctrl.exitinfo2;
1356 if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_W)
1358 else if (cpudata->vmcb->ctrl.exitinfo1 & PGEX_I)
1363 exit->u.mem.inst_len = cpudata->vmcb->ctrl.inst_len;
1364 memcpy(exit->u.mem.inst_bytes, cpudata->vmcb->ctrl.inst_bytes,
1376 struct svm_cpudata *cpudata = vcpu->cpudata;
1377 struct vmcb *vmcb = cpudata->vmcb;
1382 val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) |
1385 if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) {
1395 cpudata->gxcr0 = val;
1397 svm_inkernel_advance(cpudata->vmcb);
1416 struct svm_cpudata *cpudata = vcpu->cpudata;
1420 fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask, false);
1423 cpudata->hxcr0 = rdxcr(0);
1424 wrxcr(0, cpudata->gxcr0);
1431 struct svm_cpudata *cpudata = vcpu->cpudata;
1434 cpudata->gxcr0 = rdxcr(0);
1435 wrxcr(0, cpudata->hxcr0);
1439 fpu_area_save(&cpudata->gfpu, svm_xcr0_mask, false);
1446 struct svm_cpudata *cpudata = vcpu->cpudata;
1452 ldr0(cpudata->drs[NVMM_X64_DR_DR0]);
1453 ldr1(cpudata->drs[NVMM_X64_DR_DR1]);
1454 ldr2(cpudata->drs[NVMM_X64_DR_DR2]);
1455 ldr3(cpudata->drs[NVMM_X64_DR_DR3]);
1461 struct svm_cpudata *cpudata = vcpu->cpudata;
1463 cpudata->drs[NVMM_X64_DR_DR0] = rdr0();
1464 cpudata->drs[NVMM_X64_DR_DR1] = rdr1();
1465 cpudata->drs[NVMM_X64_DR_DR2] = rdr2();
1466 cpudata->drs[NVMM_X64_DR_DR3] = rdr3();
1474 struct svm_cpudata *cpudata = vcpu->cpudata;
1476 cpudata->fsbase = rdmsr(MSR_FSBASE);
1477 cpudata->kernelgsbase = rdmsr(MSR_KERNELGSBASE);
1483 struct svm_cpudata *cpudata = vcpu->cpudata;
1485 wrmsr(MSR_STAR, cpudata->star);
1486 wrmsr(MSR_LSTAR, cpudata->lstar);
1487 wrmsr(MSR_CSTAR, cpudata->cstar);
1488 wrmsr(MSR_SFMASK, cpudata->sfmask);
1489 wrmsr(MSR_FSBASE, cpudata->fsbase);
1490 wrmsr(MSR_KERNELGSBASE, cpudata->kernelgsbase);
1498 struct svm_cpudata *cpudata = vcpu->cpudata;
1500 if (vcpu->hcpu_last != hcpu || cpudata->shared_asid) {
1501 cpudata->gtlb_want_flush = true;
1517 svm_htlb_flush(struct svm_machdata *machdata, struct svm_cpudata *cpudata)
1519 struct vmcb *vmcb = cpudata->vmcb;
1523 if (__predict_true(machgen == cpudata->vcpu_htlb_gen)) {
1532 svm_htlb_flush_ack(struct svm_cpudata *cpudata, uint64_t machgen)
1534 struct vmcb *vmcb = cpudata->vmcb;
1537 cpudata->vcpu_htlb_gen = machgen;
1542 svm_exit_evt(struct svm_cpudata *cpudata, struct vmcb *vmcb)
1544 cpudata->evt_pending = false;
1548 cpudata->evt_pending = true;
1558 struct svm_cpudata *cpudata = vcpu->cpudata;
1559 struct vmcb *vmcb = cpudata->vmcb;
1578 cpudata->gtsc_want_update = true;
1585 if (cpudata->gtlb_want_flush) {
1591 if (__predict_false(cpudata->gtsc_want_update)) {
1592 vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc();
1598 machgen = svm_htlb_flush(machdata, cpudata);
1599 svm_vmrun(cpudata->vmcb_pa, cpudata->gprs);
1600 svm_htlb_flush_ack(cpudata, machgen);
1607 cpudata->gtlb_want_flush = false;
1608 cpudata->gtsc_want_update = false;
1611 svm_exit_evt(cpudata, vmcb);
1688 cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset;
1699 exit->exitstate.int_window_exiting = cpudata->int_window_exit;
1700 exit->exitstate.nmi_window_exiting = cpudata->nmi_window_exit;
1701 exit->exitstate.evt_pending = cpudata->evt_pending;
1869 struct svm_cpudata *cpudata = vcpu->cpudata;
1870 struct vmcb *vmcb = cpudata->vmcb;
1877 cpudata->gtlb_want_flush = true;
1905 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
1907 memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs));
1928 cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0];
1929 cpudata->gxcr0 &= svm_xcr0_mask;
1930 cpudata->gxcr0 |= XCR0_X87;
1934 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
1936 memcpy(cpudata->drs, state->drs, sizeof(state->drs));
1961 cpudata->gtsc = state->msrs[NVMM_X64_MSR_TSC];
1962 cpudata->gtsc_want_update = true;
1985 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
1987 memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu,
1990 fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave;
1996 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
2011 struct svm_cpudata *cpudata = vcpu->cpudata;
2012 struct vmcb *vmcb = cpudata->vmcb;
2042 CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
2044 memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs));
2059 state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0;
2062 CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
2064 memcpy(state->drs, cpudata->drs, sizeof(state->drs));
2085 state->msrs[NVMM_X64_MSR_TSC] = cpudata->gtsc;
2094 state->intr.int_window_exiting = cpudata->int_window_exit;
2095 state->intr.nmi_window_exiting = cpudata->nmi_window_exit;
2096 state->intr.evt_pending = cpudata->evt_pending;
2099 CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
2101 memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave,
2129 struct svm_cpudata *cpudata = vcpu->cpudata;
2130 struct vmcb *vmcb = cpudata->vmcb;
2153 cpudata->shared_asid = true;
2161 struct svm_cpudata *cpudata = vcpu->cpudata;
2162 struct vmcb *vmcb = cpudata->vmcb;
2165 if (cpudata->shared_asid) {
2180 struct svm_cpudata *cpudata = vcpu->cpudata;
2181 struct vmcb *vmcb = cpudata->vmcb;
2263 memset(cpudata->iobm, 0xFF, IOBM_SIZE);
2264 vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
2267 memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
2268 svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
2269 svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
2270 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
2271 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
2272 svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
2273 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
2274 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
2275 svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
2276 svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
2277 svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
2278 svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
2279 svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
2280 vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
2293 cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
2294 cpudata->gfpu.xsh_xcomp_bv = 0;
2297 cpudata->star = rdmsr(MSR_STAR);
2298 cpudata->lstar = rdmsr(MSR_LSTAR);
2299 cpudata->cstar = rdmsr(MSR_CSTAR);
2300 cpudata->sfmask = rdmsr(MSR_SFMASK);
2313 struct svm_cpudata *cpudata;
2316 /* Allocate the SVM cpudata. */
2317 cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
2318 roundup(sizeof(*cpudata), PAGE_SIZE), 0,
2320 vcpu->cpudata = cpudata;
2323 cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
2329 error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
2335 error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
2346 if (cpudata->vmcb_pa) {
2347 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
2350 if (cpudata->iobm_pa) {
2351 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
2354 if (cpudata->msrbm_pa) {
2355 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
2358 uvm_km_free(kernel_map, (vaddr_t)cpudata,
2359 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
2366 struct svm_cpudata *cpudata = vcpu->cpudata;
2370 svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
2371 svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
2372 svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
2374 uvm_km_free(kernel_map, (vaddr_t)cpudata,
2375 roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
2381 svm_vcpu_configure_cpuid(struct svm_cpudata *cpudata, void *data)
2400 if (!cpudata->cpuidpresent[i]) {
2403 if (cpudata->cpuid[i].leaf == cpuid->leaf) {
2404 cpudata->cpuidpresent[i] = false;
2412 if (!cpudata->cpuidpresent[i]) {
2415 if (cpudata->cpuid[i].leaf == cpuid->leaf) {
2416 memcpy(&cpudata->cpuid[i], cpuid,
2424 if (!cpudata->cpuidpresent[i]) {
2425 cpudata->cpuidpresent[i] = true;
2426 memcpy(&cpudata->cpuid[i], cpuid,
2438 struct svm_cpudata *cpudata = vcpu->cpudata;
2442 return svm_vcpu_configure_cpuid(cpudata, data);
Indexes created Sat Oct 11 19:10:01 GMT 2025