1 /* $NetBSD: hypervisor_machdep.c,v 1.46 2023/03/01 08:13:44 riastradh Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 2004 Christian Limpach. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /****************************************************************************** 30 * hypervisor.c 31 * 32 * Communication to/from hypervisor. 33 * 34 * Copyright (c) 2002-2004, K A Fraser 35 * 36 * Permission is hereby granted, free of charge, to any person obtaining a copy 37 * of this software and associated documentation files (the "Software"), to 38 * deal in the Software without restriction, including without limitation the 39 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 40 * sell copies of the Software, and to permit persons to whom the Software is 41 * furnished to do so, subject to the following conditions: 42 * 43 * The above copyright notice and this permission notice shall be included in 44 * all copies or substantial portions of the Software. 45 * 46 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 51 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 52 * DEALINGS IN THE SOFTWARE. 53 */ 54 55 56 #include <sys/cdefs.h> 57 __KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.46 2023/03/01 08:13:44 riastradh Exp $"); 58 59 #include <sys/param.h> 60 #include <sys/systm.h> 61 #include <sys/kmem.h> 62 #include <sys/cpu.h> 63 #include <sys/ksyms.h> 64 65 #include <uvm/uvm_extern.h> 66 67 #include <machine/vmparam.h> 68 #include <machine/pmap.h> 69 #include <machine/pmap_private.h> 70 71 #include <x86/machdep.h> 72 #include <x86/cpuvar.h> 73 74 #include <xen/xen.h> 75 #include <xen/intr.h> 76 #include <xen/hypervisor.h> 77 #include <xen/evtchn.h> 78 #include <xen/xenpmap.h> 79 80 #include "opt_xen.h" 81 #include "opt_modular.h" 82 #include "opt_ddb.h" 83 #include "isa.h" 84 #include "pci.h" 85 #include "ksyms.h" 86 87 #ifdef DDB 88 #include <machine/db_machdep.h> 89 #include <ddb/db_extern.h> 90 #include <ddb/db_output.h> 91 #include <ddb/db_interface.h> 92 #endif 93 94 #ifdef XENPV 95 /* 96 * arch-dependent p2m frame lists list (L3 and L2) 97 * used by Xen for save/restore mappings 98 */ 99 static unsigned long * l3_p2m_page; 100 static unsigned long * l2_p2m_page; 101 static int l2_p2m_page_size; /* size of L2 page, in pages */ 102 103 static void build_p2m_frame_list_list(void); 104 static void update_p2m_frame_list_list(void); 105 106 #endif 107 108 // #define PORT_DEBUG 4 109 // #define EARLY_DEBUG_EVENT 110 111 /* callback function type */ 112 typedef void (*iterate_func_t)(unsigned int, unsigned int, 113 unsigned int, void *); 114 115 static inline void 116 evt_iterate_bits(volatile unsigned long *pendingl1, 117 volatile unsigned long *pendingl2, 118 volatile unsigned long *mask, 119 iterate_func_t iterate_pending, void *iterate_args) 120 { 121 122 KASSERT(pendingl1 != NULL); 123 KASSERT(pendingl2 != NULL); 124 125 unsigned long l1, l2; 126 unsigned int l1i, l2i, port; 127 128 l1 = xen_atomic_xchg(pendingl1, 0); 129 while ((l1i = xen_ffs(l1)) != 0) { 130 l1i--; 131 l1 &= ~(1UL << l1i); 132 133 l2 = pendingl2[l1i] & (mask != NULL ? ~mask[l1i] : -1UL); 134 l2 &= curcpu()->ci_evtmask[l1i]; 135 136 if (mask != NULL) xen_atomic_setbits_l(&mask[l1i], l2); 137 xen_atomic_clearbits_l(&pendingl2[l1i], l2); 138 139 while ((l2i = xen_ffs(l2)) != 0) { 140 l2i--; 141 l2 &= ~(1UL << l2i); 142 143 port = (l1i << LONG_SHIFT) + l2i; 144 145 iterate_pending(port, l1i, l2i, iterate_args); 146 } 147 } 148 } 149 150 /* 151 * Set per-cpu "pending" information for outstanding events that 152 * cannot be processed now. 153 */ 154 155 static inline void 156 evt_set_pending(unsigned int port, unsigned int l1i, 157 unsigned int l2i, void *args) 158 { 159 160 KASSERT(args != NULL); 161 162 int *ret = args; 163 struct intrhand *ih; 164 165 if (evtsource[port]) { 166 hypervisor_set_ipending(evtsource[port]->ev_imask, l1i, l2i); 167 evtsource[port]->ev_evcnt.ev_count++; 168 ih = evtsource[port]->ev_handlers; 169 while (ih != NULL) { 170 ih->ih_pending++; 171 ih = ih->ih_evt_next; 172 } 173 174 if (*ret == 0 && curcpu()->ci_ilevel < 175 evtsource[port]->ev_maxlevel) 176 *ret = 1; 177 } 178 #ifdef DOM0OPS 179 else { 180 /* set pending event */ 181 xenevt_setipending(l1i, l2i); 182 } 183 #endif 184 } 185 186 int stipending(void); 187 int 188 stipending(void) 189 { 190 volatile shared_info_t *s = HYPERVISOR_shared_info; 191 struct cpu_info *ci; 192 volatile struct vcpu_info *vci; 193 int ret; 194 195 kpreempt_disable(); 196 197 ret = 0; 198 ci = curcpu(); 199 vci = ci->ci_vcpu; 200 201 #if 0 202 if (HYPERVISOR_shared_info->events) 203 printf("stipending events %08lx mask %08lx ilevel %d\n", 204 HYPERVISOR_shared_info->events, 205 HYPERVISOR_shared_info->events_mask, ci->ci_ilevel); 206 #endif 207 208 #ifdef EARLY_DEBUG_EVENT 209 if (xen_atomic_test_bit(&s->evtchn_pending[0], debug_port)) { 210 xen_debug_handler(NULL); 211 xen_atomic_clear_bit(&s->evtchn_pending[0], debug_port); 212 } 213 #endif 214 215 /* 216 * we're only called after STIC, so we know that we'll have to 217 * STI at the end 218 */ 219 220 while (vci->evtchn_upcall_pending) { 221 x86_disable_intr(); 222 223 vci->evtchn_upcall_pending = 0; 224 225 evt_iterate_bits(&vci->evtchn_pending_sel, 226 s->evtchn_pending, s->evtchn_mask, 227 evt_set_pending, &ret); 228 229 x86_enable_intr(); 230 } 231 232 kpreempt_enable(); 233 234 return (ret); 235 } 236 237 /* Iterate through pending events and call the event handler */ 238 239 static inline void 240 evt_do_hypervisor_callback(unsigned int port, unsigned int l1i, 241 unsigned int l2i, void *args) 242 { 243 KASSERT(args != NULL); 244 245 #ifdef DOM0OPS 246 struct cpu_info *ci = curcpu(); 247 #endif 248 struct intrframe *regs = args; 249 250 #ifdef PORT_DEBUG 251 if (port == PORT_DEBUG) 252 printf("do_hypervisor_callback event %d\n", port); 253 #endif 254 if (evtsource[port]) { 255 KASSERT(cpu_intr_p()); 256 evtchn_do_event(port, regs); 257 } 258 #ifdef DOM0OPS 259 else { 260 if (ci->ci_ilevel < IPL_HIGH) { 261 /* fast path */ 262 int oipl = ci->ci_ilevel; 263 ci->ci_ilevel = IPL_HIGH; 264 KASSERT(cpu_intr_p()); 265 xenevt_event(port); 266 ci->ci_ilevel = oipl; 267 } else { 268 /* set pending event */ 269 xenevt_setipending(l1i, l2i); 270 } 271 } 272 #endif 273 } 274 275 void 276 do_hypervisor_callback(struct intrframe *regs) 277 { 278 volatile shared_info_t *s = HYPERVISOR_shared_info; 279 struct cpu_info *ci; 280 volatile struct vcpu_info *vci; 281 uint64_t level __diagused; 282 283 ci = curcpu(); 284 vci = ci->ci_vcpu; 285 level = ci->ci_ilevel; 286 287 /* Save trapframe for clock handler */ 288 KASSERT(regs != NULL); 289 ci->ci_xen_clockf_usermode = USERMODE(regs->_INTRFRAME_CS); 290 ci->ci_xen_clockf_pc = regs->_INTRFRAME_IP; 291 292 // DDD printf("do_hypervisor_callback\n"); 293 294 #ifdef EARLY_DEBUG_EVENT 295 if (xen_atomic_test_bit(&s->evtchn_pending[0], debug_port)) { 296 xen_debug_handler(NULL); 297 xen_atomic_clear_bit(&s->evtchn_pending[0], debug_port); 298 } 299 #endif 300 301 while (vci->evtchn_upcall_pending) { 302 vci->evtchn_upcall_pending = 0; 303 304 evt_iterate_bits(&vci->evtchn_pending_sel, 305 s->evtchn_pending, s->evtchn_mask, 306 evt_do_hypervisor_callback, regs); 307 } 308 309 #ifdef DIAGNOSTIC 310 if (level != ci->ci_ilevel) 311 printf("hypervisor done %08x level %" PRIu64 "/%" PRIu64 " ipending %0" PRIx64 "\n", 312 (uint)vci->evtchn_pending_sel, 313 level, (uint64_t)ci->ci_ilevel, (uint64_t)ci->ci_ipending); 314 #endif 315 } 316 317 #if 0 318 void 319 hypervisor_send_event(struct cpu_info *ci, unsigned int ev) 320 { 321 KASSERT(ci != NULL); 322 323 volatile shared_info_t *s = HYPERVISOR_shared_info; 324 volatile struct vcpu_info *vci = ci->ci_vcpu; 325 326 #ifdef PORT_DEBUG 327 if (ev == PORT_DEBUG) 328 printf("hypervisor_send_event %d\n", ev); 329 #endif 330 331 xen_atomic_set_bit(&s->evtchn_pending[0], ev); 332 333 if (__predict_false(ci == curcpu())) { 334 xen_atomic_set_bit(&vci->evtchn_pending_sel, 335 ev >> LONG_SHIFT); 336 xen_atomic_set_bit(&vci->evtchn_upcall_pending, 0); 337 } 338 339 xen_atomic_clear_bit(&s->evtchn_mask[0], ev); 340 341 if (__predict_true(ci == curcpu())) { 342 hypervisor_force_callback(); 343 } else { 344 if (__predict_false(xen_send_ipi(ci, XEN_IPI_HVCB))) { 345 panic("xen_send_ipi(cpu%d id %d, XEN_IPI_HVCB) failed\n", 346 (int) ci->ci_cpuid, ci->ci_vcpuid); 347 } 348 } 349 } 350 #endif 351 352 void 353 hypervisor_unmask_event(unsigned int ev) 354 { 355 356 KASSERT(ev > 0 && ev < NR_EVENT_CHANNELS); 357 358 #ifdef PORT_DEBUG 359 if (ev == PORT_DEBUG) 360 printf("hypervisor_unmask_event %d\n", ev); 361 #endif 362 363 /* Xen unmasks the evtchn_mask[0]:ev bit for us. */ 364 evtchn_op_t op; 365 op.cmd = EVTCHNOP_unmask; 366 op.u.unmask.port = ev; 367 if (HYPERVISOR_event_channel_op(&op) != 0) 368 panic("Failed to unmask event %d\n", ev); 369 370 return; 371 } 372 373 void 374 hypervisor_mask_event(unsigned int ev) 375 { 376 volatile shared_info_t *s = HYPERVISOR_shared_info; 377 #ifdef PORT_DEBUG 378 if (ev == PORT_DEBUG) 379 printf("hypervisor_mask_event %d\n", ev); 380 #endif 381 382 xen_atomic_set_bit(&s->evtchn_mask[0], ev); 383 } 384 385 void 386 hypervisor_clear_event(unsigned int ev) 387 { 388 volatile shared_info_t *s = HYPERVISOR_shared_info; 389 #ifdef PORT_DEBUG 390 if (ev == PORT_DEBUG) 391 printf("hypervisor_clear_event %d\n", ev); 392 #endif 393 394 xen_atomic_clear_bit(&s->evtchn_pending[0], ev); 395 } 396 397 static inline void 398 evt_enable_event(unsigned int port, unsigned int l1i, 399 unsigned int l2i, void *args) 400 { 401 KASSERT(args == NULL); 402 hypervisor_unmask_event(port); 403 #if defined(XENPV) && (NPCI > 0 || NISA > 0) 404 hypervisor_ack_pirq_event(port); 405 #endif /* NPCI > 0 || NISA > 0 */ 406 } 407 408 void 409 hypervisor_enable_sir(unsigned int sir) 410 { 411 struct cpu_info *ci = curcpu(); 412 413 /* 414 * enable all events for ipl. As we only set an event in ipl_evt_mask 415 * for its lowest IPL, and pending IPLs are processed high to low, 416 * we know that all callback for this event have been processed. 417 */ 418 419 evt_iterate_bits(&ci->ci_isources[sir]->ipl_evt_mask1, 420 ci->ci_isources[sir]->ipl_evt_mask2, NULL, 421 evt_enable_event, NULL); 422 423 } 424 425 void 426 hypervisor_set_ipending(uint64_t imask, int l1, int l2) 427 { 428 429 /* This function is not re-entrant */ 430 KASSERT(x86_read_psl() != 0); 431 432 int sir; 433 struct cpu_info *ci = curcpu(); 434 435 /* set pending bit for the appropriate IPLs */ 436 ci->ci_ipending |= imask; 437 438 /* 439 * And set event pending bit for the lowest IPL. As IPL are handled 440 * from high to low, this ensure that all callbacks will have been 441 * called when we ack the event 442 */ 443 sir = ffs(imask); 444 KASSERT(sir > SIR_XENIPL_VM); 445 sir--; 446 KASSERT(sir <= SIR_XENIPL_HIGH); 447 KASSERT(ci->ci_isources[sir] != NULL); 448 ci->ci_isources[sir]->ipl_evt_mask1 |= 1UL << l1; 449 ci->ci_isources[sir]->ipl_evt_mask2[l1] |= 1UL << l2; 450 KASSERT(ci == curcpu()); 451 #if 0 452 if (__predict_false(ci != curcpu())) { 453 if (xen_send_ipi(ci, XEN_IPI_HVCB)) { 454 panic("hypervisor_set_ipending: " 455 "xen_send_ipi(cpu%d id %d, XEN_IPI_HVCB) failed\n", 456 (int) ci->ci_cpuid, ci->ci_vcpuid); 457 } 458 } 459 #endif 460 } 461 462 void 463 hypervisor_machdep_attach(void) 464 { 465 #ifdef XENPV 466 /* dom0 does not require the arch-dependent P2M translation table */ 467 if (!xendomain_is_dom0()) { 468 build_p2m_frame_list_list(); 469 sysctl_xen_suspend_setup(); 470 } 471 #endif 472 } 473 474 void 475 hypervisor_machdep_resume(void) 476 { 477 #ifdef XENPV 478 /* dom0 does not require the arch-dependent P2M translation table */ 479 if (!xendomain_is_dom0()) 480 update_p2m_frame_list_list(); 481 #endif 482 } 483 484 /* 485 * idle_block() 486 * 487 * Called from the idle loop when we have nothing to do but wait 488 * for an interrupt. 489 */ 490 static void 491 idle_block(void) 492 { 493 KASSERT(curcpu()->ci_ipending == 0); 494 HYPERVISOR_block(); 495 KASSERT(curcpu()->ci_ipending == 0); 496 } 497 498 void 499 x86_cpu_idle_xen(void) 500 { 501 struct cpu_info *ci = curcpu(); 502 503 KASSERT(ci->ci_ilevel == IPL_NONE); 504 505 x86_disable_intr(); 506 if (__predict_false(!ci->ci_want_resched)) { 507 idle_block(); 508 } else { 509 x86_enable_intr(); 510 } 511 } 512 513 #ifdef XENPV 514 /* 515 * Generate the p2m_frame_list_list table, 516 * needed for guest save/restore 517 */ 518 static void 519 build_p2m_frame_list_list(void) 520 { 521 int fpp; /* number of page (frame) pointer per page */ 522 unsigned long max_pfn; 523 /* 524 * The p2m list is composed of three levels of indirection, 525 * each layer containing MFNs pointing to lower level pages 526 * The indirection is used to convert a given PFN to its MFN 527 * Each N level page can point to @fpp (N-1) level pages 528 * For example, for x86 32bit, we have: 529 * - PAGE_SIZE: 4096 bytes 530 * - fpp: 1024 (one L3 page can address 1024 L2 pages) 531 * A L1 page contains the list of MFN we are looking for 532 */ 533 max_pfn = xen_start_info.nr_pages; 534 fpp = PAGE_SIZE / sizeof(xen_pfn_t); 535 536 /* we only need one L3 page */ 537 l3_p2m_page = (vaddr_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 538 PAGE_SIZE, UVM_KMF_WIRED | UVM_KMF_NOWAIT); 539 if (l3_p2m_page == NULL) 540 panic("could not allocate memory for l3_p2m_page"); 541 542 /* 543 * Determine how many L2 pages we need for the mapping 544 * Each L2 can map a total of @fpp L1 pages 545 */ 546 l2_p2m_page_size = howmany(max_pfn, fpp); 547 548 l2_p2m_page = (vaddr_t *)uvm_km_alloc(kernel_map, 549 l2_p2m_page_size * PAGE_SIZE, 550 PAGE_SIZE, UVM_KMF_WIRED | UVM_KMF_NOWAIT); 551 if (l2_p2m_page == NULL) 552 panic("could not allocate memory for l2_p2m_page"); 553 554 /* We now have L3 and L2 pages ready, update L1 mapping */ 555 update_p2m_frame_list_list(); 556 557 } 558 559 /* 560 * Update the L1 p2m_frame_list_list mapping (during guest boot or resume) 561 */ 562 static void 563 update_p2m_frame_list_list(void) 564 { 565 int i; 566 int fpp; /* number of page (frame) pointer per page */ 567 unsigned long max_pfn; 568 569 max_pfn = xen_start_info.nr_pages; 570 fpp = PAGE_SIZE / sizeof(xen_pfn_t); 571 572 for (i = 0; i < l2_p2m_page_size; i++) { 573 /* 574 * Each time we start a new L2 page, 575 * store its MFN in the L3 page 576 */ 577 if ((i % fpp) == 0) { 578 l3_p2m_page[i/fpp] = vtomfn( 579 (vaddr_t)&l2_p2m_page[i]); 580 } 581 /* 582 * we use a shortcut 583 * since @xpmap_phys_to_machine_mapping array 584 * already contains PFN to MFN mapping, we just 585 * set the l2_p2m_page MFN pointer to the MFN of the 586 * according frame of @xpmap_phys_to_machine_mapping 587 */ 588 l2_p2m_page[i] = vtomfn((vaddr_t) 589 &xpmap_phys_to_machine_mapping[i*fpp]); 590 } 591 592 HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = 593 vtomfn((vaddr_t)l3_p2m_page); 594 HYPERVISOR_shared_info->arch.max_pfn = max_pfn; 595 596 } 597 #endif /* XENPV */ 598 599 void 600 xen_init_ksyms(void) 601 { 602 #if NKSYMS || defined(DDB) || defined(MODULAR) 603 extern int end; 604 extern int *esym; 605 #ifdef DDB 606 db_machine_init(); 607 #endif 608 609 #ifdef XENPV 610 esym = xen_start_info.mod_start ? 611 (void *)xen_start_info.mod_start : 612 (void *)xen_start_info.mfn_list; 613 #endif /* XENPV */ 614 /* for PVH, esym is set in locore.S */ 615 ksyms_addsyms_elf(*(int *)(void *)&end, 616 ((int *)(void *)&end) + 1, esym); 617 #endif 618 } 619
Indexes created Sun Sep 21 20:09:37 GMT 2025