x86_xpmap.c revision 1.12.4.4
1 /* $NetBSD: x86_xpmap.c,v 1.12.4.4 2009/07/23 23:31:37 jym Exp $ */ 2 3 /* 4 * Copyright (c) 2006 Mathieu Ropert <mro (at) adviseo.fr> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * Copyright (c) 2006, 2007 Manuel Bouyer. 21 * 22 * Redistribution and use in source and binary forms, with or without 23 * modification, are permitted provided that the following conditions 24 * are met: 25 * 1. Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * 2. Redistributions in binary form must reproduce the above copyright 28 * notice, this list of conditions and the following disclaimer in the 29 * documentation and/or other materials provided with the distribution. 30 * 3. All advertising materials mentioning features or use of this software 31 * must display the following acknowledgement: 32 * This product includes software developed by Manuel Bouyer. 33 * 4. The name of the author may not be used to endorse or promote products 34 * derived from this software without specific prior written permission. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 37 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 38 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 39 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 40 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 45 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 */ 48 49 /* 50 * 51 * Copyright (c) 2004 Christian Limpach. 52 * All rights reserved. 53 * 54 * Redistribution and use in source and binary forms, with or without 55 * modification, are permitted provided that the following conditions 56 * are met: 57 * 1. Redistributions of source code must retain the above copyright 58 * notice, this list of conditions and the following disclaimer. 59 * 2. Redistributions in binary form must reproduce the above copyright 60 * notice, this list of conditions and the following disclaimer in the 61 * documentation and/or other materials provided with the distribution. 62 * 3. All advertising materials mentioning features or use of this software 63 * must display the following acknowledgement: 64 * This product includes software developed by Christian Limpach. 65 * 4. The name of the author may not be used to endorse or promote products 66 * derived from this software without specific prior written permission. 67 * 68 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 69 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 70 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 71 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 72 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 73 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 74 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 75 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 76 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 77 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 78 */ 79 80 81 #include <sys/cdefs.h> 82 __KERNEL_RCSID(0, "$NetBSD: x86_xpmap.c,v 1.12.4.4 2009/07/23 23:31:37 jym Exp $"); 83 84 #include "opt_xen.h" 85 #include "opt_ddb.h" 86 #include "ksyms.h" 87 88 #include <sys/param.h> 89 #include <sys/systm.h> 90 #include <sys/rwlock.h> 91 92 #include <uvm/uvm.h> 93 94 #include <machine/pmap.h> 95 #include <machine/gdt.h> 96 #include <xen/xenfunc.h> 97 98 #include <dev/isa/isareg.h> 99 #include <machine/isa_machdep.h> 100 101 #undef XENDEBUG 102 /* #define XENDEBUG_SYNC */ 103 /* #define XENDEBUG_LOW */ 104 105 #ifdef XENDEBUG 106 #define XENPRINTF(x) printf x 107 #define XENPRINTK(x) printk x 108 #define XENPRINTK2(x) /* printk x */ 109 110 static char XBUF[256]; 111 #else 112 #define XENPRINTF(x) 113 #define XENPRINTK(x) 114 #define XENPRINTK2(x) 115 #endif 116 #define PRINTF(x) printf x 117 #define PRINTK(x) printk x 118 119 /* on x86_64 kernel runs in ring 3 */ 120 #ifdef __x86_64__ 121 #define PG_k PG_u 122 #else 123 #define PG_k 0 124 #endif 125 126 volatile shared_info_t *HYPERVISOR_shared_info; 127 /* Xen requires the start_info struct to be page aligned */ 128 union start_info_union start_info_union __aligned(PAGE_SIZE); 129 unsigned long *xpmap_phys_to_machine_mapping; 130 131 /* 132 * We should avoid the domU to manipulate MFNs when it is suspending 133 * or migrating, as they could be invalid once domU resumes operations. 134 * 135 * We use a read/write lock for that: when a thread is expected to 136 * manipulate MFNs, it should first acquire a reader lock, then proceed 137 * to MFN's manipulation. Once it has finished with it, the reader lock is 138 * released. 139 * 140 * The thread responsible for the domU suspension will acquire an exclusive 141 * (writer) lock. 142 * 143 * XXX JYM the locking will need revisit - rwlock(9) is currently inadequate 144 */ 145 static krwlock_t xen_ptom_lock; 146 147 void 148 xen_init_ptom_lock(void) { 149 rw_init(&xen_ptom_lock); 150 } 151 152 void 153 xen_release_ptom_lock(void) { 154 /* rw_exit(&xen_ptom_lock); */ 155 } 156 157 void 158 xen_acquire_reader_ptom_lock(void) { 159 /* rw_enter(&xen_ptom_lock, RW_READER); */ 160 } 161 162 void 163 xen_acquire_writer_ptom_lock(void) { 164 /* rw_enter(&xen_ptom_lock, RW_WRITER); */ 165 } 166 167 void xen_failsafe_handler(void); 168 169 #ifdef XEN3 170 #define HYPERVISOR_mmu_update_self(req, count, success_count) \ 171 HYPERVISOR_mmu_update((req), (count), (success_count), DOMID_SELF) 172 #else 173 #define HYPERVISOR_mmu_update_self(req, count, success_count) \ 174 HYPERVISOR_mmu_update((req), (count), (success_count)) 175 #endif 176 177 void 178 xen_failsafe_handler(void) 179 { 180 181 panic("xen_failsafe_handler called!\n"); 182 } 183 184 185 void 186 xen_set_ldt(vaddr_t base, uint32_t entries) 187 { 188 vaddr_t va; 189 vaddr_t end; 190 pt_entry_t *ptp; 191 int s; 192 193 #ifdef __x86_64__ 194 end = base + (entries << 3); 195 #else 196 end = base + entries * sizeof(union descriptor); 197 #endif 198 199 #ifdef XEN3 200 xen_acquire_reader_ptom_lock(); 201 #endif 202 203 for (va = base; va < end; va += PAGE_SIZE) { 204 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 205 ptp = kvtopte(va); 206 XENPRINTF(("xen_set_ldt %p %d %p\n", (void *)base, 207 entries, ptp)); 208 pmap_pte_clearbits(ptp, PG_RW); 209 } 210 s = splvm(); 211 xpq_queue_set_ldt(base, entries); 212 xpq_flush_queue(); 213 214 #ifdef XEN3 215 xen_release_ptom_lock(); 216 #endif 217 218 splx(s); 219 } 220 221 #ifdef XENDEBUG 222 void xpq_debug_dump(void); 223 #endif 224 225 #define XPQUEUE_SIZE 2048 226 static mmu_update_t xpq_queue[XPQUEUE_SIZE]; 227 static int xpq_idx = 0; 228 229 void 230 xpq_flush_queue(void) 231 { 232 int i, ok; 233 234 XENPRINTK2(("flush queue %p entries %d\n", xpq_queue, xpq_idx)); 235 for (i = 0; i < xpq_idx; i++) 236 XENPRINTK2(("%d: %p %08" PRIx64 "\n", i, 237 (uint64_t)xpq_queue[i].ptr, (uint64_t)xpq_queue[i].val)); 238 if (xpq_idx != 0 && 239 HYPERVISOR_mmu_update_self(xpq_queue, xpq_idx, &ok) < 0) { 240 printf("xpq_flush_queue: %d entries \n", xpq_idx); 241 for (i = 0; i < xpq_idx; i++) 242 printf("0x%016" PRIx64 ": 0x%016" PRIx64 "\n", 243 (uint64_t)xpq_queue[i].ptr, 244 (uint64_t)xpq_queue[i].val); 245 panic("HYPERVISOR_mmu_update failed\n"); 246 } 247 xpq_idx = 0; 248 } 249 250 static inline void 251 xpq_increment_idx(void) 252 { 253 254 xpq_idx++; 255 if (__predict_false(xpq_idx == XPQUEUE_SIZE)) 256 xpq_flush_queue(); 257 } 258 259 void 260 xpq_queue_machphys_update(paddr_t ma, paddr_t pa) 261 { 262 XENPRINTK2(("xpq_queue_machphys_update ma=0x%" PRIx64 " pa=0x%" PRIx64 263 "\n", (int64_t)ma, (int64_t)pa)); 264 xpq_queue[xpq_idx].ptr = ma | MMU_MACHPHYS_UPDATE; 265 xpq_queue[xpq_idx].val = (pa - XPMAP_OFFSET) >> PAGE_SHIFT; 266 xpq_increment_idx(); 267 #ifdef XENDEBUG_SYNC 268 xpq_flush_queue(); 269 #endif 270 } 271 272 void 273 xpq_queue_pte_update(paddr_t ptr, pt_entry_t val) 274 { 275 276 KASSERT((ptr & 3) == 0); 277 xpq_queue[xpq_idx].ptr = (paddr_t)ptr | MMU_NORMAL_PT_UPDATE; 278 xpq_queue[xpq_idx].val = val; 279 xpq_increment_idx(); 280 #ifdef XENDEBUG_SYNC 281 xpq_flush_queue(); 282 #endif 283 } 284 285 #ifdef XEN3 286 void 287 xpq_queue_pt_switch(paddr_t pa) 288 { 289 struct mmuext_op op; 290 xpq_flush_queue(); 291 292 XENPRINTK2(("xpq_queue_pt_switch: 0x%" PRIx64 " 0x%" PRIx64 "\n", 293 (int64_t)pa, (int64_t)pa)); 294 op.cmd = MMUEXT_NEW_BASEPTR; 295 op.arg1.mfn = pa >> PAGE_SHIFT; 296 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 297 panic("xpq_queue_pt_switch"); 298 } 299 300 void 301 xpq_queue_pin_table(paddr_t pa) 302 { 303 struct mmuext_op op; 304 xpq_flush_queue(); 305 306 XENPRINTK2(("xpq_queue_pin_table: 0x%" PRIx64 " 0x%" PRIx64 "\n", 307 (int64_t)pa, (int64_t)pa)); 308 op.arg1.mfn = pa >> PAGE_SHIFT; 309 310 #if defined(__x86_64__) 311 op.cmd = MMUEXT_PIN_L4_TABLE; 312 #else 313 op.cmd = MMUEXT_PIN_L2_TABLE; 314 #endif 315 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 316 panic("xpq_queue_pin_table"); 317 } 318 319 #ifdef PAE 320 static void 321 xpq_queue_pin_l3_table(paddr_t pa) 322 { 323 struct mmuext_op op; 324 xpq_flush_queue(); 325 326 XENPRINTK2(("xpq_queue_pin_l2_table: 0x%" PRIx64 " 0x%" PRIx64 "\n", 327 (int64_t)pa, (int64_t)pa)); 328 op.arg1.mfn = pa >> PAGE_SHIFT; 329 330 op.cmd = MMUEXT_PIN_L3_TABLE; 331 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 332 panic("xpq_queue_pin_table"); 333 } 334 #endif 335 336 void 337 xpq_queue_unpin_table(paddr_t pa) 338 { 339 struct mmuext_op op; 340 xpq_flush_queue(); 341 342 XENPRINTK2(("xpq_queue_unpin_table: 0x%" PRIx64 " 0x%" PRIx64 "\n", 343 (int64_t)pa, (int64_t)pa)); 344 op.arg1.mfn = pa >> PAGE_SHIFT; 345 op.cmd = MMUEXT_UNPIN_TABLE; 346 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 347 panic("xpq_queue_unpin_table"); 348 } 349 350 void 351 xpq_queue_set_ldt(vaddr_t va, uint32_t entries) 352 { 353 struct mmuext_op op; 354 xpq_flush_queue(); 355 356 XENPRINTK2(("xpq_queue_set_ldt\n")); 357 KASSERT(va == (va & ~PAGE_MASK)); 358 op.cmd = MMUEXT_SET_LDT; 359 op.arg1.linear_addr = va; 360 op.arg2.nr_ents = entries; 361 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 362 panic("xpq_queue_set_ldt"); 363 } 364 365 void 366 xpq_queue_tlb_flush(void) 367 { 368 struct mmuext_op op; 369 xpq_flush_queue(); 370 371 XENPRINTK2(("xpq_queue_tlb_flush\n")); 372 op.cmd = MMUEXT_TLB_FLUSH_LOCAL; 373 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 374 panic("xpq_queue_tlb_flush"); 375 } 376 377 void 378 xpq_flush_cache(void) 379 { 380 struct mmuext_op op; 381 int s = splvm(); 382 xpq_flush_queue(); 383 384 XENPRINTK2(("xpq_queue_flush_cache\n")); 385 op.cmd = MMUEXT_FLUSH_CACHE; 386 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 387 panic("xpq_flush_cache"); 388 splx(s); 389 } 390 391 void 392 xpq_queue_invlpg(vaddr_t va) 393 { 394 struct mmuext_op op; 395 xpq_flush_queue(); 396 397 XENPRINTK2(("xpq_queue_invlpg %p\n", (void *)va)); 398 op.cmd = MMUEXT_INVLPG_LOCAL; 399 op.arg1.linear_addr = (va & ~PAGE_MASK); 400 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 401 panic("xpq_queue_invlpg"); 402 } 403 404 int 405 xpq_update_foreign(paddr_t ptr, pt_entry_t val, int dom) 406 { 407 mmu_update_t op; 408 int ok; 409 xpq_flush_queue(); 410 411 op.ptr = ptr; 412 op.val = val; 413 if (HYPERVISOR_mmu_update(&op, 1, &ok, dom) < 0) 414 return EFAULT; 415 return (0); 416 } 417 #else /* XEN3 */ 418 void 419 xpq_queue_pt_switch(paddr_t pa) 420 { 421 422 XENPRINTK2(("xpq_queue_pt_switch: %p %p\n", (void *)pa, (void *)pa)); 423 xpq_queue[xpq_idx].ptr = pa | MMU_EXTENDED_COMMAND; 424 xpq_queue[xpq_idx].val = MMUEXT_NEW_BASEPTR; 425 xpq_increment_idx(); 426 } 427 428 void 429 xpq_queue_pin_table(paddr_t pa) 430 { 431 432 XENPRINTK2(("xpq_queue_pin_table: %p %p\n", (void *)pa, (void *)pa)); 433 xpq_queue[xpq_idx].ptr = pa | MMU_EXTENDED_COMMAND; 434 xpq_queue[xpq_idx].val = MMUEXT_PIN_L2_TABLE; 435 xpq_increment_idx(); 436 } 437 438 void 439 xpq_queue_unpin_table(paddr_t pa) 440 { 441 442 XENPRINTK2(("xpq_queue_unpin_table: %p %p\n", (void *)pa, (void *)pa)); 443 xpq_queue[xpq_idx].ptr = pa | MMU_EXTENDED_COMMAND; 444 xpq_queue[xpq_idx].val = MMUEXT_UNPIN_TABLE; 445 xpq_increment_idx(); 446 } 447 448 void 449 xpq_queue_set_ldt(vaddr_t va, uint32_t entries) 450 { 451 452 XENPRINTK2(("xpq_queue_set_ldt\n")); 453 KASSERT(va == (va & ~PAGE_MASK)); 454 xpq_queue[xpq_idx].ptr = MMU_EXTENDED_COMMAND | va; 455 xpq_queue[xpq_idx].val = MMUEXT_SET_LDT | (entries << MMUEXT_CMD_SHIFT); 456 xpq_increment_idx(); 457 } 458 459 void 460 xpq_queue_tlb_flush(void) 461 { 462 463 XENPRINTK2(("xpq_queue_tlb_flush\n")); 464 xpq_queue[xpq_idx].ptr = MMU_EXTENDED_COMMAND; 465 xpq_queue[xpq_idx].val = MMUEXT_TLB_FLUSH; 466 xpq_increment_idx(); 467 } 468 469 void 470 xpq_flush_cache(void) 471 { 472 int s = splvm(); 473 474 XENPRINTK2(("xpq_queue_flush_cache\n")); 475 xpq_queue[xpq_idx].ptr = MMU_EXTENDED_COMMAND; 476 xpq_queue[xpq_idx].val = MMUEXT_FLUSH_CACHE; 477 xpq_increment_idx(); 478 xpq_flush_queue(); 479 splx(s); 480 } 481 482 void 483 xpq_queue_invlpg(vaddr_t va) 484 { 485 486 XENPRINTK2(("xpq_queue_invlpg %p\n", (void *)va)); 487 xpq_queue[xpq_idx].ptr = (va & ~PAGE_MASK) | MMU_EXTENDED_COMMAND; 488 xpq_queue[xpq_idx].val = MMUEXT_INVLPG; 489 xpq_increment_idx(); 490 } 491 492 int 493 xpq_update_foreign(paddr_t ptr, pt_entry_t val, int dom) 494 { 495 mmu_update_t xpq_up[3]; 496 497 xpq_up[0].ptr = MMU_EXTENDED_COMMAND; 498 xpq_up[0].val = MMUEXT_SET_FOREIGNDOM | (dom << 16); 499 xpq_up[1].ptr = ptr; 500 xpq_up[1].val = val; 501 if (HYPERVISOR_mmu_update_self(xpq_up, 2, NULL) < 0) 502 return EFAULT; 503 return (0); 504 } 505 #endif /* XEN3 */ 506 507 #ifdef XENDEBUG 508 void 509 xpq_debug_dump(void) 510 { 511 int i; 512 513 XENPRINTK2(("idx: %d\n", xpq_idx)); 514 for (i = 0; i < xpq_idx; i++) { 515 snprintf(XBUF, sizeof(XBUF), "%" PRIx64 " %08" PRIx64, 516 (uint64_t)xpq_queue[i].ptr, (uint64_t)xpq_queue[i].val); 517 if (++i < xpq_idx) 518 snprintf(XBUF + strlen(XBUF), 519 sizeof(XBUF) - strlen(XBUF), 520 "%" PRIx64 " %08" PRIx64, 521 (uint64_t)xpq_queue[i].ptr, 522 (uint64_t)xpq_queue[i].val); 523 if (++i < xpq_idx) 524 snprintf(XBUF + strlen(XBUF), 525 sizeof(XBUF) - strlen(XBUF), 526 "%" PRIx64 " %08" PRIx64, 527 (uint64_t)xpq_queue[i].ptr, 528 (uint64_t)xpq_queue[i].val); 529 if (++i < xpq_idx) 530 snprintf(XBUF + strlen(XBUF), 531 sizeof(XBUF) - strlen(XBUF), 532 "%" PRIx64 " %08" PRIx64, 533 (uint64_t)xpq_queue[i].ptr, 534 (uint64_t)xpq_queue[i].val); 535 XENPRINTK2(("%d: %s\n", xpq_idx, XBUF)); 536 } 537 } 538 #endif 539 540 541 extern volatile struct xencons_interface *xencons_interface; /* XXX */ 542 extern struct xenstore_domain_interface *xenstore_interface; /* XXX */ 543 544 static void xen_bt_set_readonly (vaddr_t); 545 static void xen_bootstrap_tables (vaddr_t, vaddr_t, int, int, int); 546 547 /* How many PDEs ? */ 548 #if L2_SLOT_KERNBASE > 0 549 #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1)) 550 #else 551 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 552 #endif 553 554 /* 555 * Construct and switch to new pagetables 556 * first_avail is the first vaddr we can use after 557 * we get rid of Xen pagetables 558 */ 559 560 vaddr_t xen_pmap_bootstrap (void); 561 562 /* 563 * Function to get rid of Xen bootstrap tables 564 */ 565 566 /* How many PDP do we need: */ 567 #ifdef PAE 568 /* 569 * For PAE, we consider a single contigous L2 "superpage" of 4 pages, 570 * all of them mapped by the L3 page. We also need a shadow page 571 * for L3[3]. 572 */ 573 static const int l2_4_count = 6; 574 #else 575 static const int l2_4_count = PTP_LEVELS - 1; 576 #endif 577 578 vaddr_t 579 xen_pmap_bootstrap(void) 580 { 581 int count, oldcount; 582 long mapsize; 583 vaddr_t bootstrap_tables, init_tables; 584 585 xpmap_phys_to_machine_mapping = 586 (unsigned long *)xen_start_info.mfn_list; 587 init_tables = xen_start_info.pt_base; 588 __PRINTK(("xen_arch_pmap_bootstrap init_tables=0x%lx\n", init_tables)); 589 590 /* Space after Xen boostrap tables should be free */ 591 bootstrap_tables = xen_start_info.pt_base + 592 (xen_start_info.nr_pt_frames * PAGE_SIZE); 593 594 /* 595 * Calculate how many space we need 596 * first everything mapped before the Xen bootstrap tables 597 */ 598 mapsize = init_tables - KERNTEXTOFF; 599 /* after the tables we'll have: 600 * - UAREA 601 * - dummy user PGD (x86_64) 602 * - HYPERVISOR_shared_info 603 * - ISA I/O mem (if needed) 604 */ 605 mapsize += UPAGES * NBPG; 606 #ifdef __x86_64__ 607 mapsize += NBPG; 608 #endif 609 mapsize += NBPG; 610 611 #ifdef DOM0OPS 612 if (xendomain_is_dom0()) { 613 /* space for ISA I/O mem */ 614 mapsize += IOM_SIZE; 615 } 616 #endif 617 /* at this point mapsize doens't include the table size */ 618 619 #ifdef __x86_64__ 620 count = TABLE_L2_ENTRIES; 621 #else 622 count = (mapsize + (NBPD_L2 -1)) >> L2_SHIFT; 623 #endif /* __x86_64__ */ 624 625 /* now compute how many L2 pages we need exactly */ 626 XENPRINTK(("bootstrap_final mapsize 0x%lx count %d\n", mapsize, count)); 627 while (mapsize + (count + l2_4_count) * PAGE_SIZE + KERNTEXTOFF > 628 ((long)count << L2_SHIFT) + KERNBASE) { 629 count++; 630 } 631 #ifndef __x86_64__ 632 /* 633 * one more L2 page: we'll alocate several pages after kva_start 634 * in pmap_bootstrap() before pmap_growkernel(), which have not been 635 * counted here. It's not a big issue to allocate one more L2 as 636 * pmap_growkernel() will be called anyway. 637 */ 638 count++; 639 nkptp[1] = count; 640 #endif 641 642 /* 643 * install bootstrap pages. We may need more L2 pages than will 644 * have the final table here, as it's installed after the final table 645 */ 646 oldcount = count; 647 648 bootstrap_again: 649 XENPRINTK(("bootstrap_again oldcount %d\n", oldcount)); 650 /* 651 * Xen space we'll reclaim may not be enough for our new page tables, 652 * move bootstrap tables if necessary 653 */ 654 if (bootstrap_tables < init_tables + ((count + l2_4_count) * PAGE_SIZE)) 655 bootstrap_tables = init_tables + 656 ((count + l2_4_count) * PAGE_SIZE); 657 /* make sure we have enough to map the bootstrap_tables */ 658 if (bootstrap_tables + ((oldcount + l2_4_count) * PAGE_SIZE) > 659 ((long)oldcount << L2_SHIFT) + KERNBASE) { 660 oldcount++; 661 goto bootstrap_again; 662 } 663 664 /* Create temporary tables */ 665 xen_bootstrap_tables(xen_start_info.pt_base, bootstrap_tables, 666 xen_start_info.nr_pt_frames, oldcount, 0); 667 668 /* Create final tables */ 669 xen_bootstrap_tables(bootstrap_tables, init_tables, 670 oldcount + l2_4_count, count, 1); 671 672 /* zero out free space after tables */ 673 memset((void *)(init_tables + ((count + l2_4_count) * PAGE_SIZE)), 0, 674 (UPAGES + 1) * NBPG); 675 return (init_tables + ((count + l2_4_count) * PAGE_SIZE)); 676 } 677 678 679 /* 680 * Build a new table and switch to it 681 * old_count is # of old tables (including PGD, PDTPE and PDE) 682 * new_count is # of new tables (PTE only) 683 * we assume areas don't overlap 684 */ 685 686 687 static void 688 xen_bootstrap_tables (vaddr_t old_pgd, vaddr_t new_pgd, 689 int old_count, int new_count, int final) 690 { 691 pd_entry_t *pdtpe, *pde, *pte; 692 pd_entry_t *cur_pgd, *bt_pgd; 693 paddr_t addr; 694 vaddr_t page, avail, text_end, map_end; 695 int i; 696 extern char __data_start; 697 698 __PRINTK(("xen_bootstrap_tables(0x%lx, 0x%lx, %d, %d)\n", 699 old_pgd, new_pgd, old_count, new_count)); 700 text_end = ((vaddr_t)&__data_start) & ~PAGE_MASK; 701 /* 702 * size of R/W area after kernel text: 703 * xencons_interface (if present) 704 * xenstore_interface (if present) 705 * table pages (new_count + l2_4_count entries) 706 * extra mappings (only when final is true): 707 * UAREA 708 * dummy user PGD (x86_64 only)/gdt page (i386 only) 709 * HYPERVISOR_shared_info 710 * ISA I/O mem (if needed) 711 */ 712 map_end = new_pgd + ((new_count + l2_4_count) * NBPG); 713 if (final) { 714 map_end += (UPAGES + 1) * NBPG; 715 HYPERVISOR_shared_info = (shared_info_t *)map_end; 716 map_end += NBPG; 717 } 718 /* 719 * we always set atdevbase, as it's used by init386 to find the first 720 * available VA. map_end is updated only if we are dom0, so 721 * atdevbase -> atdevbase + IOM_SIZE will be mapped only in 722 * this case. 723 */ 724 if (final) 725 atdevbase = map_end; 726 #ifdef DOM0OPS 727 if (final && xendomain_is_dom0()) { 728 /* ISA I/O mem */ 729 map_end += IOM_SIZE; 730 } 731 #endif /* DOM0OPS */ 732 733 __PRINTK(("xen_bootstrap_tables text_end 0x%lx map_end 0x%lx\n", 734 text_end, map_end)); 735 __PRINTK(("console 0x%lx ", xen_start_info.console.domU.mfn)); 736 __PRINTK(("xenstore 0x%lx\n", xen_start_info.store_mfn)); 737 738 /* 739 * Create bootstrap page tables 740 * What we need: 741 * - a PGD (level 4) 742 * - a PDTPE (level 3) 743 * - a PDE (level2) 744 * - some PTEs (level 1) 745 */ 746 747 cur_pgd = (pd_entry_t *) old_pgd; 748 bt_pgd = (pd_entry_t *) new_pgd; 749 memset (bt_pgd, 0, PAGE_SIZE); 750 avail = new_pgd + PAGE_SIZE; 751 #if PTP_LEVELS > 3 752 /* Install level 3 */ 753 pdtpe = (pd_entry_t *) avail; 754 memset (pdtpe, 0, PAGE_SIZE); 755 avail += PAGE_SIZE; 756 757 addr = ((u_long) pdtpe) - KERNBASE; 758 bt_pgd[pl4_pi(KERNTEXTOFF)] = 759 xpmap_ptom_masked(addr) | PG_k | PG_RW | PG_V; 760 761 __PRINTK(("L3 va 0x%lx pa 0x%" PRIx64 " entry 0x%" PRIx64 " -> L4[0x%x]\n", 762 pdtpe, (uint64_t)addr, (uint64_t)bt_pgd[pl4_pi(KERNTEXTOFF)], 763 pl4_pi(KERNTEXTOFF))); 764 #else 765 pdtpe = bt_pgd; 766 #endif /* PTP_LEVELS > 3 */ 767 768 #if PTP_LEVELS > 2 769 /* Level 2 */ 770 pde = (pd_entry_t *) avail; 771 memset(pde, 0, PAGE_SIZE); 772 avail += PAGE_SIZE; 773 774 addr = ((u_long) pde) - KERNBASE; 775 pdtpe[pl3_pi(KERNTEXTOFF)] = 776 xpmap_ptom_masked(addr) | PG_k | PG_V | PG_RW; 777 __PRINTK(("L2 va 0x%lx pa 0x%" PRIx64 " entry 0x%" PRIx64 " -> L3[0x%x]\n", 778 pde, (int64_t)addr, (int64_t)pdtpe[pl3_pi(KERNTEXTOFF)], 779 pl3_pi(KERNTEXTOFF))); 780 #elif defined(PAE) 781 /* our PAE-style level 2: 5 contigous pages (4 L2 + 1 shadow) */ 782 pde = (pd_entry_t *) avail; 783 memset(pde, 0, PAGE_SIZE * 5); 784 avail += PAGE_SIZE * 5; 785 addr = ((u_long) pde) - KERNBASE; 786 /* 787 * enter L2 pages in the L3. 788 * The real L2 kernel PD will be the last one (so that 789 * pde[L2_SLOT_KERN] always point to the shadow). 790 */ 791 for (i = 0; i < 3; i++, addr += PAGE_SIZE) { 792 /* 793 * Xen doens't want R/W mappings in L3 entries, it'll add it 794 * itself. 795 */ 796 pdtpe[i] = xpmap_ptom_masked(addr) | PG_k | PG_V; 797 __PRINTK(("L2 va 0x%lx pa 0x%" PRIx64 " entry 0x%" PRIx64 798 " -> L3[0x%x]\n", (vaddr_t)pde + PAGE_SIZE * i, 799 (int64_t)addr, (int64_t)pdtpe[i], i)); 800 } 801 addr += PAGE_SIZE; 802 pdtpe[3] = xpmap_ptom_masked(addr) | PG_k | PG_V; 803 __PRINTK(("L2 va 0x%lx pa 0x%" PRIx64 " entry 0x%" PRIx64 804 " -> L3[0x%x]\n", (vaddr_t)pde + PAGE_SIZE * 4, 805 (int64_t)addr, (int64_t)pdtpe[3], 3)); 806 807 #else /* PAE */ 808 pde = bt_pgd; 809 #endif /* PTP_LEVELS > 2 */ 810 811 /* Level 1 */ 812 page = KERNTEXTOFF; 813 for (i = 0; i < new_count; i ++) { 814 vaddr_t cur_page = page; 815 816 pte = (pd_entry_t *) avail; 817 avail += PAGE_SIZE; 818 819 memset(pte, 0, PAGE_SIZE); 820 while (pl2_pi(page) == pl2_pi (cur_page)) { 821 if (page >= map_end) { 822 /* not mapped at all */ 823 pte[pl1_pi(page)] = 0; 824 page += PAGE_SIZE; 825 continue; 826 } 827 pte[pl1_pi(page)] = xpmap_ptom_masked(page - KERNBASE); 828 if (page == (vaddr_t)HYPERVISOR_shared_info) { 829 pte[pl1_pi(page)] = xen_start_info.shared_info; 830 __PRINTK(("HYPERVISOR_shared_info " 831 "va 0x%lx pte 0x%" PRIx64 "\n", 832 HYPERVISOR_shared_info, (int64_t)pte[pl1_pi(page)])); 833 } 834 #ifdef XEN3 835 if ((xpmap_ptom_masked(page - KERNBASE) >> PAGE_SHIFT) 836 == xen_start_info.console.domU.mfn) { 837 xencons_interface = (void *)page; 838 pte[pl1_pi(page)] = xen_start_info.console.domU.mfn; 839 pte[pl1_pi(page)] <<= PAGE_SHIFT; 840 __PRINTK(("xencons_interface " 841 "va 0x%lx pte 0x%" PRIx64 "\n", 842 xencons_interface, (int64_t)pte[pl1_pi(page)])); 843 } 844 if ((xpmap_ptom_masked(page - KERNBASE) >> PAGE_SHIFT) 845 == xen_start_info.store_mfn) { 846 xenstore_interface = (void *)page; 847 pte[pl1_pi(page)] = xen_start_info.store_mfn; 848 pte[pl1_pi(page)] <<= PAGE_SHIFT; 849 __PRINTK(("xenstore_interface " 850 "va 0x%lx pte 0x%" PRIx64 "\n", 851 xenstore_interface, (int64_t)pte[pl1_pi(page)])); 852 } 853 #endif /* XEN3 */ 854 #ifdef DOM0OPS 855 if (page >= (vaddr_t)atdevbase && 856 page < (vaddr_t)atdevbase + IOM_SIZE) { 857 pte[pl1_pi(page)] = 858 IOM_BEGIN + (page - (vaddr_t)atdevbase); 859 } 860 #endif 861 pte[pl1_pi(page)] |= PG_k | PG_V; 862 if (page < text_end) { 863 /* map kernel text RO */ 864 pte[pl1_pi(page)] |= 0; 865 } else if (page >= old_pgd 866 && page < old_pgd + (old_count * PAGE_SIZE)) { 867 /* map old page tables RO */ 868 pte[pl1_pi(page)] |= 0; 869 } else if (page >= new_pgd && 870 page < new_pgd + ((new_count + l2_4_count) * PAGE_SIZE)) { 871 /* map new page tables RO */ 872 pte[pl1_pi(page)] |= 0; 873 } else { 874 /* map page RW */ 875 pte[pl1_pi(page)] |= PG_RW; 876 } 877 878 if ((page >= old_pgd && page < old_pgd + (old_count * PAGE_SIZE)) 879 || page >= new_pgd) { 880 __PRINTK(("va 0x%lx pa 0x%lx " 881 "entry 0x%" PRIx64 " -> L1[0x%x]\n", 882 page, page - KERNBASE, 883 (int64_t)pte[pl1_pi(page)], pl1_pi(page))); 884 } 885 page += PAGE_SIZE; 886 } 887 888 addr = ((u_long) pte) - KERNBASE; 889 pde[pl2_pi(cur_page)] = 890 xpmap_ptom_masked(addr) | PG_k | PG_RW | PG_V; 891 __PRINTK(("L1 va 0x%lx pa 0x%" PRIx64 " entry 0x%" PRIx64 892 " -> L2[0x%x]\n", pte, (int64_t)addr, 893 (int64_t)pde[pl2_pi(cur_page)], pl2_pi(cur_page))); 894 /* Mark readonly */ 895 xen_bt_set_readonly((vaddr_t) pte); 896 } 897 898 /* Install recursive page tables mapping */ 899 #ifdef PAE 900 /* 901 * we need a shadow page for the kernel's L2 page 902 * The real L2 kernel PD will be the last one (so that 903 * pde[L2_SLOT_KERN] always point to the shadow. 904 */ 905 memcpy(&pde[L2_SLOT_KERN + NPDPG], &pde[L2_SLOT_KERN], PAGE_SIZE); 906 pmap_kl2pd = &pde[L2_SLOT_KERN + NPDPG]; 907 pmap_kl2paddr = (u_long)pmap_kl2pd - KERNBASE; 908 909 /* 910 * We don't enter a recursive entry from the L3 PD. Instead, 911 * we enter the first 4 L2 pages, which includes the kernel's L2 912 * shadow. But we have to entrer the shadow after switching 913 * %cr3, or Xen will refcount some PTE with the wrong type. 914 */ 915 addr = (u_long)pde - KERNBASE; 916 for (i = 0; i < 3; i++, addr += PAGE_SIZE) { 917 pde[PDIR_SLOT_PTE + i] = xpmap_ptom_masked(addr) | PG_k | PG_V; 918 __PRINTK(("pde[%d] va 0x%lx pa 0x%lx entry 0x%" PRIx64 "\n", 919 (int)(PDIR_SLOT_PTE + i), pde + PAGE_SIZE * i, (long)addr, 920 (int64_t)pde[PDIR_SLOT_PTE + i])); 921 } 922 #if 0 923 addr += PAGE_SIZE; /* point to shadow L2 */ 924 pde[PDIR_SLOT_PTE + 3] = xpmap_ptom_masked(addr) | PG_k | PG_V; 925 __PRINTK(("pde[%d] va 0x%lx pa 0x%lx entry 0x%" PRIx64 "\n", 926 (int)(PDIR_SLOT_PTE + 3), pde + PAGE_SIZE * 4, (long)addr, 927 (int64_t)pde[PDIR_SLOT_PTE + 3])); 928 #endif 929 /* Mark tables RO, and pin the kernel's shadow as L2 */ 930 addr = (u_long)pde - KERNBASE; 931 for (i = 0; i < 5; i++, addr += PAGE_SIZE) { 932 xen_bt_set_readonly(((vaddr_t)pde) + PAGE_SIZE * i); 933 if (i == 2 || i == 3) 934 continue; 935 #if 0 936 __PRINTK(("pin L2 %d addr 0x%" PRIx64 "\n", i, (int64_t)addr)); 937 xpq_queue_pin_table(xpmap_ptom_masked(addr)); 938 #endif 939 } 940 if (final) { 941 addr = (u_long)pde - KERNBASE + 3 * PAGE_SIZE; 942 __PRINTK(("pin L2 %d addr 0x%" PRIx64 "\n", 2, (int64_t)addr)); 943 xpq_queue_pin_table(xpmap_ptom_masked(addr)); 944 } 945 #if 0 946 addr = (u_long)pde - KERNBASE + 2 * PAGE_SIZE; 947 __PRINTK(("pin L2 %d addr 0x%" PRIx64 "\n", 2, (int64_t)addr)); 948 xpq_queue_pin_table(xpmap_ptom_masked(addr)); 949 #endif 950 #else /* PAE */ 951 /* recursive entry in higher-level PD */ 952 bt_pgd[PDIR_SLOT_PTE] = 953 xpmap_ptom_masked(new_pgd - KERNBASE) | PG_k | PG_V; 954 __PRINTK(("bt_pgd[PDIR_SLOT_PTE] va 0x%lx pa 0x%" PRIx64 955 " entry 0x%" PRIx64 "\n", new_pgd, (int64_t)new_pgd - KERNBASE, 956 (int64_t)bt_pgd[PDIR_SLOT_PTE])); 957 /* Mark tables RO */ 958 xen_bt_set_readonly((vaddr_t) pde); 959 #endif 960 #if PTP_LEVELS > 2 || defined(PAE) 961 xen_bt_set_readonly((vaddr_t) pdtpe); 962 #endif 963 #if PTP_LEVELS > 3 964 xen_bt_set_readonly(new_pgd); 965 #endif 966 /* Pin the PGD */ 967 __PRINTK(("pin PGD\n")); 968 #ifdef PAE 969 xpq_queue_pin_l3_table(xpmap_ptom_masked(new_pgd - KERNBASE)); 970 #else 971 xpq_queue_pin_table(xpmap_ptom_masked(new_pgd - KERNBASE)); 972 #endif 973 #ifdef __i386__ 974 /* Save phys. addr of PDP, for libkvm. */ 975 PDPpaddr = (long)pde; 976 #ifdef PAE 977 /* also save the address of the L3 page */ 978 pmap_l3pd = pdtpe; 979 pmap_l3paddr = (new_pgd - KERNBASE); 980 #endif /* PAE */ 981 #endif /* i386 */ 982 /* Switch to new tables */ 983 __PRINTK(("switch to PGD\n")); 984 xpq_queue_pt_switch(xpmap_ptom_masked(new_pgd - KERNBASE)); 985 __PRINTK(("bt_pgd[PDIR_SLOT_PTE] now entry 0x%" PRIx64 "\n", 986 (int64_t)bt_pgd[PDIR_SLOT_PTE])); 987 #ifdef PAE 988 if (final) { 989 /* now enter kernel's PTE mappings */ 990 addr = (u_long)pde - KERNBASE + PAGE_SIZE * 3; 991 xpq_queue_pte_update( 992 xpmap_ptom(((vaddr_t)&pde[PDIR_SLOT_PTE + 3]) - KERNBASE), 993 xpmap_ptom_masked(addr) | PG_k | PG_V); 994 xpq_flush_queue(); 995 } 996 #endif 997 998 999 1000 /* Now we can safely reclaim space taken by old tables */ 1001 1002 __PRINTK(("unpin old PGD\n")); 1003 /* Unpin old PGD */ 1004 xpq_queue_unpin_table(xpmap_ptom_masked(old_pgd - KERNBASE)); 1005 /* Mark old tables RW */ 1006 page = old_pgd; 1007 addr = (paddr_t) pde[pl2_pi(page)] & PG_FRAME; 1008 addr = xpmap_mtop(addr); 1009 pte = (pd_entry_t *) ((u_long)addr + KERNBASE); 1010 pte += pl1_pi(page); 1011 __PRINTK(("*pde 0x%" PRIx64 " addr 0x%" PRIx64 " pte 0x%lx\n", 1012 (int64_t)pde[pl2_pi(page)], (int64_t)addr, (long)pte)); 1013 while (page < old_pgd + (old_count * PAGE_SIZE) && page < map_end) { 1014 addr = xpmap_ptom(((u_long) pte) - KERNBASE); 1015 XENPRINTK(("addr 0x%" PRIx64 " pte 0x%lx *pte 0x%" PRIx64 "\n", 1016 (int64_t)addr, (long)pte, (int64_t)*pte)); 1017 xpq_queue_pte_update(addr, *pte | PG_RW); 1018 page += PAGE_SIZE; 1019 /* 1020 * Our ptes are contiguous 1021 * so it's safe to just "++" here 1022 */ 1023 pte++; 1024 } 1025 xpq_flush_queue(); 1026 } 1027 1028 1029 /* 1030 * Bootstrap helper functions 1031 */ 1032 1033 /* 1034 * Mark a page readonly 1035 * XXX: assuming vaddr = paddr + KERNBASE 1036 */ 1037 1038 static void 1039 xen_bt_set_readonly (vaddr_t page) 1040 { 1041 pt_entry_t entry; 1042 1043 xen_acquire_reader_ptom_lock(); 1044 1045 entry = xpmap_ptom_masked(page - KERNBASE); 1046 entry |= PG_k | PG_V; 1047 1048 HYPERVISOR_update_va_mapping (page, entry, UVMF_INVLPG); 1049 1050 xen_release_ptom_lock(); 1051 } 1052 1053 #ifdef __x86_64__ 1054 void 1055 xen_set_user_pgd(paddr_t page) 1056 { 1057 struct mmuext_op op; 1058 int s = splvm(); 1059 1060 xpq_flush_queue(); 1061 op.cmd = MMUEXT_NEW_USER_BASEPTR; 1062 1063 xen_acquire_reader_ptom_lock(); 1064 1065 op.arg1.mfn = pfn_to_mfn(page >> PAGE_SHIFT); 1066 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) 1067 panic("xen_set_user_pgd: failed to install new user page" 1068 " directory %lx", page); 1069 1070 xen_release_ptom_lock(); 1071 1072 splx(s); 1073 } 1074 #endif /* __x86_64__ */ 1075
Indexes created Tue Oct 21 15:09:54 GMT 2025