pmap.c revision 1.191
1 /* $NetBSD: pmap.c,v 1.191 2008/11/19 06:24:04 matt Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependant vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of 0xffff0000, then 166 * MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* 183 * Special compilation symbols 184 * PMAP_DEBUG - Build in pmap_debug_level code 185 */ 186 187 /* Include header files */ 188 189 #include "opt_cpuoptions.h" 190 #include "opt_pmap_debug.h" 191 #include "opt_ddb.h" 192 #include "opt_lockdebug.h" 193 #include "opt_multiprocessor.h" 194 195 #include <sys/param.h> 196 #include <sys/types.h> 197 #include <sys/kernel.h> 198 #include <sys/systm.h> 199 #include <sys/proc.h> 200 #include <sys/malloc.h> 201 #include <sys/user.h> 202 #include <sys/pool.h> 203 #include <sys/cdefs.h> 204 #include <sys/cpu.h> 205 #include <sys/sysctl.h> 206 207 #include <uvm/uvm.h> 208 209 #include <machine/bus.h> 210 #include <machine/pmap.h> 211 #include <machine/pcb.h> 212 #include <machine/param.h> 213 #include <arm/arm32/katelib.h> 214 215 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.191 2008/11/19 06:24:04 matt Exp $"); 216 217 #ifdef PMAP_DEBUG 218 219 /* XXX need to get rid of all refs to this */ 220 int pmap_debug_level = 0; 221 222 /* 223 * for switching to potentially finer grained debugging 224 */ 225 #define PDB_FOLLOW 0x0001 226 #define PDB_INIT 0x0002 227 #define PDB_ENTER 0x0004 228 #define PDB_REMOVE 0x0008 229 #define PDB_CREATE 0x0010 230 #define PDB_PTPAGE 0x0020 231 #define PDB_GROWKERN 0x0040 232 #define PDB_BITS 0x0080 233 #define PDB_COLLECT 0x0100 234 #define PDB_PROTECT 0x0200 235 #define PDB_MAP_L1 0x0400 236 #define PDB_BOOTSTRAP 0x1000 237 #define PDB_PARANOIA 0x2000 238 #define PDB_WIRING 0x4000 239 #define PDB_PVDUMP 0x8000 240 #define PDB_VAC 0x10000 241 #define PDB_KENTER 0x20000 242 #define PDB_KREMOVE 0x40000 243 #define PDB_EXEC 0x80000 244 245 int debugmap = 1; 246 int pmapdebug = 0; 247 #define NPDEBUG(_lev_,_stat_) \ 248 if (pmapdebug & (_lev_)) \ 249 ((_stat_)) 250 251 #else /* PMAP_DEBUG */ 252 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 253 #endif /* PMAP_DEBUG */ 254 255 /* 256 * pmap_kernel() points here 257 */ 258 struct pmap kernel_pmap_store; 259 260 /* 261 * Which pmap is currently 'live' in the cache 262 * 263 * XXXSCW: Fix for SMP ... 264 */ 265 static pmap_t pmap_recent_user; 266 267 /* 268 * Pointer to last active lwp, or NULL if it exited. 269 */ 270 struct lwp *pmap_previous_active_lwp; 271 272 /* 273 * Pool and cache that pmap structures are allocated from. 274 * We use a cache to avoid clearing the pm_l2[] array (1KB) 275 * in pmap_create(). 276 */ 277 static struct pool_cache pmap_cache; 278 static LIST_HEAD(, pmap) pmap_pmaps; 279 280 /* 281 * Pool of PV structures 282 */ 283 static struct pool pmap_pv_pool; 284 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 285 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 286 static struct pool_allocator pmap_bootstrap_pv_allocator = { 287 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 288 }; 289 290 /* 291 * Pool and cache of l2_dtable structures. 292 * We use a cache to avoid clearing the structures when they're 293 * allocated. (196 bytes) 294 */ 295 static struct pool_cache pmap_l2dtable_cache; 296 static vaddr_t pmap_kernel_l2dtable_kva; 297 298 /* 299 * Pool and cache of L2 page descriptors. 300 * We use a cache to avoid clearing the descriptor table 301 * when they're allocated. (1KB) 302 */ 303 static struct pool_cache pmap_l2ptp_cache; 304 static vaddr_t pmap_kernel_l2ptp_kva; 305 static paddr_t pmap_kernel_l2ptp_phys; 306 307 #ifdef PMAPCOUNTERS 308 #define PMAP_EVCNT_INITIALIZER(name) \ 309 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 310 311 #ifdef PMAP_CACHE_VIPT 312 static struct evcnt pmap_ev_vac_color_new = 313 PMAP_EVCNT_INITIALIZER("new page color"); 314 static struct evcnt pmap_ev_vac_color_reuse = 315 PMAP_EVCNT_INITIALIZER("ok first page color"); 316 static struct evcnt pmap_ev_vac_color_ok = 317 PMAP_EVCNT_INITIALIZER("ok page color"); 318 static struct evcnt pmap_ev_vac_color_blind = 319 PMAP_EVCNT_INITIALIZER("blind page color"); 320 static struct evcnt pmap_ev_vac_color_change = 321 PMAP_EVCNT_INITIALIZER("change page color"); 322 static struct evcnt pmap_ev_vac_color_erase = 323 PMAP_EVCNT_INITIALIZER("erase page color"); 324 static struct evcnt pmap_ev_vac_color_none = 325 PMAP_EVCNT_INITIALIZER("no page color"); 326 static struct evcnt pmap_ev_vac_color_restore = 327 PMAP_EVCNT_INITIALIZER("restore page color"); 328 329 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 330 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 331 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 332 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 333 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 334 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 335 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 336 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 337 #endif 338 339 static struct evcnt pmap_ev_mappings = 340 PMAP_EVCNT_INITIALIZER("pages mapped"); 341 static struct evcnt pmap_ev_unmappings = 342 PMAP_EVCNT_INITIALIZER("pages unmapped"); 343 static struct evcnt pmap_ev_remappings = 344 PMAP_EVCNT_INITIALIZER("pages remapped"); 345 346 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 347 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 348 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 349 350 static struct evcnt pmap_ev_kernel_mappings = 351 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 352 static struct evcnt pmap_ev_kernel_unmappings = 353 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 354 static struct evcnt pmap_ev_kernel_remappings = 355 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 356 357 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 358 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 359 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 360 361 static struct evcnt pmap_ev_kenter_mappings = 362 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 363 static struct evcnt pmap_ev_kenter_unmappings = 364 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 365 static struct evcnt pmap_ev_kenter_remappings = 366 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 367 static struct evcnt pmap_ev_pt_mappings = 368 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 369 370 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 371 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 372 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 373 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 374 375 #ifdef PMAP_CACHE_VIPT 376 static struct evcnt pmap_ev_exec_mappings = 377 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 378 static struct evcnt pmap_ev_exec_cached = 379 PMAP_EVCNT_INITIALIZER("exec pages cached"); 380 381 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 382 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 383 384 static struct evcnt pmap_ev_exec_synced = 385 PMAP_EVCNT_INITIALIZER("exec pages synced"); 386 static struct evcnt pmap_ev_exec_synced_map = 387 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 388 static struct evcnt pmap_ev_exec_synced_unmap = 389 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 390 static struct evcnt pmap_ev_exec_synced_remap = 391 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 392 static struct evcnt pmap_ev_exec_synced_clearbit = 393 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 394 static struct evcnt pmap_ev_exec_synced_kremove = 395 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 396 397 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 398 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 399 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 400 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 401 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 402 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 403 404 static struct evcnt pmap_ev_exec_discarded_unmap = 405 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 406 static struct evcnt pmap_ev_exec_discarded_zero = 407 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 408 static struct evcnt pmap_ev_exec_discarded_copy = 409 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 410 static struct evcnt pmap_ev_exec_discarded_page_protect = 411 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 412 static struct evcnt pmap_ev_exec_discarded_clearbit = 413 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 414 static struct evcnt pmap_ev_exec_discarded_kremove = 415 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 416 417 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 418 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 419 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 420 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 421 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 422 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 423 #endif /* PMAP_CACHE_VIPT */ 424 425 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 426 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 427 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 428 429 EVCNT_ATTACH_STATIC(pmap_ev_updates); 430 EVCNT_ATTACH_STATIC(pmap_ev_collects); 431 EVCNT_ATTACH_STATIC(pmap_ev_activations); 432 433 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 434 #else 435 #define PMAPCOUNT(x) ((void)0) 436 #endif 437 438 /* 439 * pmap copy/zero page, and mem(5) hook point 440 */ 441 static pt_entry_t *csrc_pte, *cdst_pte; 442 static vaddr_t csrcp, cdstp; 443 vaddr_t memhook; /* used by mem.c */ 444 kmutex_t memlock; /* used by mem.c */ 445 void *zeropage; /* used by mem.c */ 446 extern void *msgbufaddr; 447 int pmap_kmpages; 448 /* 449 * Flag to indicate if pmap_init() has done its thing 450 */ 451 bool pmap_initialized; 452 453 /* 454 * Misc. locking data structures 455 */ 456 457 #if 0 /* defined(MULTIPROCESSOR) || defined(LOCKDEBUG) */ 458 static struct lock pmap_main_lock; 459 460 #define PMAP_MAP_TO_HEAD_LOCK() \ 461 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 462 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 463 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 464 #define PMAP_HEAD_TO_MAP_LOCK() \ 465 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 466 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 467 spinlockmgr(&pmap_main_lock, LK_RELEASE, (void *) 0) 468 #else 469 #define PMAP_MAP_TO_HEAD_LOCK() /* null */ 470 #define PMAP_MAP_TO_HEAD_UNLOCK() /* null */ 471 #define PMAP_HEAD_TO_MAP_LOCK() /* null */ 472 #define PMAP_HEAD_TO_MAP_UNLOCK() /* null */ 473 #endif 474 475 #define pmap_acquire_pmap_lock(pm) \ 476 do { \ 477 if ((pm) != pmap_kernel()) \ 478 mutex_enter(&(pm)->pm_lock); \ 479 } while (/*CONSTCOND*/0) 480 481 #define pmap_release_pmap_lock(pm) \ 482 do { \ 483 if ((pm) != pmap_kernel()) \ 484 mutex_exit(&(pm)->pm_lock); \ 485 } while (/*CONSTCOND*/0) 486 487 488 /* 489 * Metadata for L1 translation tables. 490 */ 491 struct l1_ttable { 492 /* Entry on the L1 Table list */ 493 SLIST_ENTRY(l1_ttable) l1_link; 494 495 /* Entry on the L1 Least Recently Used list */ 496 TAILQ_ENTRY(l1_ttable) l1_lru; 497 498 /* Track how many domains are allocated from this L1 */ 499 volatile u_int l1_domain_use_count; 500 501 /* 502 * A free-list of domain numbers for this L1. 503 * We avoid using ffs() and a bitmap to track domains since ffs() 504 * is slow on ARM. 505 */ 506 u_int8_t l1_domain_first; 507 u_int8_t l1_domain_free[PMAP_DOMAINS]; 508 509 /* Physical address of this L1 page table */ 510 paddr_t l1_physaddr; 511 512 /* KVA of this L1 page table */ 513 pd_entry_t *l1_kva; 514 }; 515 516 /* 517 * Convert a virtual address into its L1 table index. That is, the 518 * index used to locate the L2 descriptor table pointer in an L1 table. 519 * This is basically used to index l1->l1_kva[]. 520 * 521 * Each L2 descriptor table represents 1MB of VA space. 522 */ 523 #define L1_IDX(va) (((vaddr_t)(va)) >> L1_S_SHIFT) 524 525 /* 526 * L1 Page Tables are tracked using a Least Recently Used list. 527 * - New L1s are allocated from the HEAD. 528 * - Freed L1s are added to the TAIl. 529 * - Recently accessed L1s (where an 'access' is some change to one of 530 * the userland pmaps which owns this L1) are moved to the TAIL. 531 */ 532 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 533 static struct simplelock l1_lru_lock; 534 535 /* 536 * A list of all L1 tables 537 */ 538 static SLIST_HEAD(, l1_ttable) l1_list; 539 540 /* 541 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 542 * 543 * This is normally 16MB worth L2 page descriptors for any given pmap. 544 * Reference counts are maintained for L2 descriptors so they can be 545 * freed when empty. 546 */ 547 struct l2_dtable { 548 /* The number of L2 page descriptors allocated to this l2_dtable */ 549 u_int l2_occupancy; 550 551 /* List of L2 page descriptors */ 552 struct l2_bucket { 553 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 554 paddr_t l2b_phys; /* Physical address of same */ 555 u_short l2b_l1idx; /* This L2 table's L1 index */ 556 u_short l2b_occupancy; /* How many active descriptors */ 557 } l2_bucket[L2_BUCKET_SIZE]; 558 }; 559 560 /* 561 * Given an L1 table index, calculate the corresponding l2_dtable index 562 * and bucket index within the l2_dtable. 563 */ 564 #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 565 (L2_SIZE - 1)) 566 #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 567 568 /* 569 * Given a virtual address, this macro returns the 570 * virtual address required to drop into the next L2 bucket. 571 */ 572 #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 573 574 /* 575 * L2 allocation. 576 */ 577 #define pmap_alloc_l2_dtable() \ 578 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 579 #define pmap_free_l2_dtable(l2) \ 580 pool_cache_put(&pmap_l2dtable_cache, (l2)) 581 #define pmap_alloc_l2_ptp(pap) \ 582 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 583 PR_NOWAIT, (pap))) 584 585 /* 586 * We try to map the page tables write-through, if possible. However, not 587 * all CPUs have a write-through cache mode, so on those we have to sync 588 * the cache when we frob page tables. 589 * 590 * We try to evaluate this at compile time, if possible. However, it's 591 * not always possible to do that, hence this run-time var. 592 */ 593 int pmap_needs_pte_sync; 594 595 /* 596 * Real definition of pv_entry. 597 */ 598 struct pv_entry { 599 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 600 pmap_t pv_pmap; /* pmap where mapping lies */ 601 vaddr_t pv_va; /* virtual address for mapping */ 602 u_int pv_flags; /* flags */ 603 }; 604 605 /* 606 * Macro to determine if a mapping might be resident in the 607 * instruction cache and/or TLB 608 */ 609 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 610 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 611 612 /* 613 * Macro to determine if a mapping might be resident in the 614 * data cache and/or TLB 615 */ 616 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 617 618 /* 619 * Local prototypes 620 */ 621 static int pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t); 622 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 623 pt_entry_t **); 624 static bool pmap_is_current(pmap_t); 625 static bool pmap_is_cached(pmap_t); 626 static void pmap_enter_pv(struct vm_page *, struct pv_entry *, 627 pmap_t, vaddr_t, u_int); 628 static struct pv_entry *pmap_find_pv(struct vm_page *, pmap_t, vaddr_t); 629 static struct pv_entry *pmap_remove_pv(struct vm_page *, pmap_t, vaddr_t, int); 630 static u_int pmap_modify_pv(struct vm_page *, pmap_t, vaddr_t, 631 u_int, u_int); 632 633 static void pmap_pinit(pmap_t); 634 static int pmap_pmap_ctor(void *, void *, int); 635 636 static void pmap_alloc_l1(pmap_t); 637 static void pmap_free_l1(pmap_t); 638 static void pmap_use_l1(pmap_t); 639 640 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 641 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 642 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 643 static int pmap_l2ptp_ctor(void *, void *, int); 644 static int pmap_l2dtable_ctor(void *, void *, int); 645 646 static void pmap_vac_me_harder(struct vm_page *, pmap_t, vaddr_t); 647 #ifdef PMAP_CACHE_VIVT 648 static void pmap_vac_me_kpmap(struct vm_page *, pmap_t, vaddr_t); 649 static void pmap_vac_me_user(struct vm_page *, pmap_t, vaddr_t); 650 #endif 651 652 static void pmap_clearbit(struct vm_page *, u_int); 653 #ifdef PMAP_CACHE_VIVT 654 static int pmap_clean_page(struct pv_entry *, bool); 655 #endif 656 #ifdef PMAP_CACHE_VIPT 657 static void pmap_syncicache_page(struct vm_page *); 658 static void pmap_flush_page(struct vm_page *, bool); 659 #endif 660 static void pmap_page_remove(struct vm_page *); 661 662 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 663 static vaddr_t kernel_pt_lookup(paddr_t); 664 665 666 /* 667 * External function prototypes 668 */ 669 extern void bzero_page(vaddr_t); 670 extern void bcopy_page(vaddr_t, vaddr_t); 671 672 /* 673 * Misc variables 674 */ 675 vaddr_t virtual_avail; 676 vaddr_t virtual_end; 677 vaddr_t pmap_curmaxkvaddr; 678 679 vaddr_t avail_start; 680 vaddr_t avail_end; 681 682 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 683 pv_addr_t kernelpages; 684 pv_addr_t kernel_l1pt; 685 pv_addr_t systempage; 686 687 /* Function to set the debug level of the pmap code */ 688 689 #ifdef PMAP_DEBUG 690 void 691 pmap_debug(int level) 692 { 693 pmap_debug_level = level; 694 printf("pmap_debug: level=%d\n", pmap_debug_level); 695 } 696 #endif /* PMAP_DEBUG */ 697 698 /* 699 * A bunch of routines to conditionally flush the caches/TLB depending 700 * on whether the specified pmap actually needs to be flushed at any 701 * given time. 702 */ 703 static inline void 704 pmap_tlb_flushID_SE(pmap_t pm, vaddr_t va) 705 { 706 707 if (pm->pm_cstate.cs_tlb_id) 708 cpu_tlb_flushID_SE(va); 709 } 710 711 static inline void 712 pmap_tlb_flushD_SE(pmap_t pm, vaddr_t va) 713 { 714 715 if (pm->pm_cstate.cs_tlb_d) 716 cpu_tlb_flushD_SE(va); 717 } 718 719 static inline void 720 pmap_tlb_flushID(pmap_t pm) 721 { 722 723 if (pm->pm_cstate.cs_tlb_id) { 724 cpu_tlb_flushID(); 725 pm->pm_cstate.cs_tlb = 0; 726 } 727 } 728 729 static inline void 730 pmap_tlb_flushD(pmap_t pm) 731 { 732 733 if (pm->pm_cstate.cs_tlb_d) { 734 cpu_tlb_flushD(); 735 pm->pm_cstate.cs_tlb_d = 0; 736 } 737 } 738 739 #ifdef PMAP_CACHE_VIVT 740 static inline void 741 pmap_idcache_wbinv_range(pmap_t pm, vaddr_t va, vsize_t len) 742 { 743 if (pm->pm_cstate.cs_cache_id) { 744 cpu_idcache_wbinv_range(va, len); 745 } 746 } 747 748 static inline void 749 pmap_dcache_wb_range(pmap_t pm, vaddr_t va, vsize_t len, 750 bool do_inv, bool rd_only) 751 { 752 753 if (pm->pm_cstate.cs_cache_d) { 754 if (do_inv) { 755 if (rd_only) 756 cpu_dcache_inv_range(va, len); 757 else 758 cpu_dcache_wbinv_range(va, len); 759 } else 760 if (!rd_only) 761 cpu_dcache_wb_range(va, len); 762 } 763 } 764 765 static inline void 766 pmap_idcache_wbinv_all(pmap_t pm) 767 { 768 if (pm->pm_cstate.cs_cache_id) { 769 cpu_idcache_wbinv_all(); 770 pm->pm_cstate.cs_cache = 0; 771 } 772 } 773 774 static inline void 775 pmap_dcache_wbinv_all(pmap_t pm) 776 { 777 if (pm->pm_cstate.cs_cache_d) { 778 cpu_dcache_wbinv_all(); 779 pm->pm_cstate.cs_cache_d = 0; 780 } 781 } 782 #endif /* PMAP_CACHE_VIVT */ 783 784 static inline bool 785 pmap_is_current(pmap_t pm) 786 { 787 788 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 789 return true; 790 791 return false; 792 } 793 794 static inline bool 795 pmap_is_cached(pmap_t pm) 796 { 797 798 if (pm == pmap_kernel() || pmap_recent_user == NULL || 799 pmap_recent_user == pm) 800 return (true); 801 802 return false; 803 } 804 805 /* 806 * PTE_SYNC_CURRENT: 807 * 808 * Make sure the pte is written out to RAM. 809 * We need to do this for one of two cases: 810 * - We're dealing with the kernel pmap 811 * - There is no pmap active in the cache/tlb. 812 * - The specified pmap is 'active' in the cache/tlb. 813 */ 814 #ifdef PMAP_INCLUDE_PTE_SYNC 815 #define PTE_SYNC_CURRENT(pm, ptep) \ 816 do { \ 817 if (PMAP_NEEDS_PTE_SYNC && \ 818 pmap_is_cached(pm)) \ 819 PTE_SYNC(ptep); \ 820 } while (/*CONSTCOND*/0) 821 #else 822 #define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ 823 #endif 824 825 /* 826 * main pv_entry manipulation functions: 827 * pmap_enter_pv: enter a mapping onto a vm_page list 828 * pmap_remove_pv: remove a mappiing from a vm_page list 829 * 830 * NOTE: pmap_enter_pv expects to lock the pvh itself 831 * pmap_remove_pv expects te caller to lock the pvh before calling 832 */ 833 834 /* 835 * pmap_enter_pv: enter a mapping onto a vm_page lst 836 * 837 * => caller should hold the proper lock on pmap_main_lock 838 * => caller should have pmap locked 839 * => we will gain the lock on the vm_page and allocate the new pv_entry 840 * => caller should adjust ptp's wire_count before calling 841 * => caller should not adjust pmap's wire_count 842 */ 843 static void 844 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 845 vaddr_t va, u_int flags) 846 { 847 struct pv_entry **pvp; 848 849 NPDEBUG(PDB_PVDUMP, 850 printf("pmap_enter_pv: pm %p, pg %p, flags 0x%x\n", pm, pg, flags)); 851 852 pve->pv_pmap = pm; 853 pve->pv_va = va; 854 pve->pv_flags = flags; 855 856 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 857 pvp = &SLIST_FIRST(&pg->mdpage.pvh_list); 858 #ifdef PMAP_CACHE_VIPT 859 /* 860 * Insert unmanaged entries, writeable first, at the head of 861 * the pv list. 862 */ 863 if (__predict_true((flags & PVF_KENTRY) == 0)) { 864 while (*pvp != NULL && (*pvp)->pv_flags & PVF_KENTRY) 865 pvp = &SLIST_NEXT(*pvp, pv_link); 866 } else if ((flags & PVF_WRITE) == 0) { 867 while (*pvp != NULL && (*pvp)->pv_flags & PVF_WRITE) 868 pvp = &SLIST_NEXT(*pvp, pv_link); 869 } 870 #endif 871 SLIST_NEXT(pve, pv_link) = *pvp; /* add to ... */ 872 *pvp = pve; /* ... locked list */ 873 pg->mdpage.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 874 #ifdef PMAP_CACHE_VIPT 875 if ((pve->pv_flags & PVF_KWRITE) == PVF_KWRITE) 876 pg->mdpage.pvh_attrs |= PVF_KMOD; 877 if ((pg->mdpage.pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 878 pg->mdpage.pvh_attrs |= PVF_DIRTY; 879 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 880 #endif 881 if (pm == pmap_kernel()) { 882 PMAPCOUNT(kernel_mappings); 883 if (flags & PVF_WRITE) 884 pg->mdpage.krw_mappings++; 885 else 886 pg->mdpage.kro_mappings++; 887 } else 888 if (flags & PVF_WRITE) 889 pg->mdpage.urw_mappings++; 890 else 891 pg->mdpage.uro_mappings++; 892 893 #ifdef PMAP_CACHE_VIPT 894 /* 895 * If this is an exec mapping and its the first exec mapping 896 * for this page, make sure to sync the I-cache. 897 */ 898 if (PV_IS_EXEC_P(flags)) { 899 if (!PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) { 900 pmap_syncicache_page(pg); 901 PMAPCOUNT(exec_synced_map); 902 } 903 PMAPCOUNT(exec_mappings); 904 } 905 #endif 906 907 PMAPCOUNT(mappings); 908 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 909 910 if (pve->pv_flags & PVF_WIRED) 911 ++pm->pm_stats.wired_count; 912 } 913 914 /* 915 * 916 * pmap_find_pv: Find a pv entry 917 * 918 * => caller should hold lock on vm_page 919 */ 920 static inline struct pv_entry * 921 pmap_find_pv(struct vm_page *pg, pmap_t pm, vaddr_t va) 922 { 923 struct pv_entry *pv; 924 925 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 926 if (pm == pv->pv_pmap && va == pv->pv_va) 927 break; 928 } 929 930 return (pv); 931 } 932 933 /* 934 * pmap_remove_pv: try to remove a mapping from a pv_list 935 * 936 * => caller should hold proper lock on pmap_main_lock 937 * => pmap should be locked 938 * => caller should hold lock on vm_page [so that attrs can be adjusted] 939 * => caller should adjust ptp's wire_count and free PTP if needed 940 * => caller should NOT adjust pmap's wire_count 941 * => we return the removed pve 942 */ 943 static struct pv_entry * 944 pmap_remove_pv(struct vm_page *pg, pmap_t pm, vaddr_t va, int skip_wired) 945 { 946 struct pv_entry *pve, **prevptr; 947 948 NPDEBUG(PDB_PVDUMP, 949 printf("pmap_remove_pv: pm %p, pg %p, va 0x%08lx\n", pm, pg, va)); 950 951 prevptr = &SLIST_FIRST(&pg->mdpage.pvh_list); /* prev pv_entry ptr */ 952 pve = *prevptr; 953 954 while (pve) { 955 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 956 NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, pg " 957 "%p, flags 0x%x\n", pm, pg, pve->pv_flags)); 958 if (pve->pv_flags & PVF_WIRED) { 959 if (skip_wired) 960 return (NULL); 961 --pm->pm_stats.wired_count; 962 } 963 *prevptr = SLIST_NEXT(pve, pv_link); /* remove it! */ 964 if (pm == pmap_kernel()) { 965 PMAPCOUNT(kernel_unmappings); 966 if (pve->pv_flags & PVF_WRITE) 967 pg->mdpage.krw_mappings--; 968 else 969 pg->mdpage.kro_mappings--; 970 } else 971 if (pve->pv_flags & PVF_WRITE) 972 pg->mdpage.urw_mappings--; 973 else 974 pg->mdpage.uro_mappings--; 975 976 PMAPCOUNT(unmappings); 977 #ifdef PMAP_CACHE_VIPT 978 if (!(pve->pv_flags & PVF_WRITE)) 979 break; 980 /* 981 * If this page has had an exec mapping, then if 982 * this was the last mapping, discard the contents, 983 * otherwise sync the i-cache for this page. 984 */ 985 if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) { 986 if (SLIST_EMPTY(&pg->mdpage.pvh_list)) { 987 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 988 PMAPCOUNT(exec_discarded_unmap); 989 } else { 990 pmap_syncicache_page(pg); 991 PMAPCOUNT(exec_synced_unmap); 992 } 993 } 994 #endif /* PMAP_CACHE_VIPT */ 995 break; 996 } 997 prevptr = &SLIST_NEXT(pve, pv_link); /* previous pointer */ 998 pve = *prevptr; /* advance */ 999 } 1000 1001 #ifdef PMAP_CACHE_VIPT 1002 /* 1003 * If we no longer have a WRITEABLE KENTRY at the head of list, 1004 * clear the KMOD attribute from the page. 1005 */ 1006 if (SLIST_FIRST(&pg->mdpage.pvh_list) == NULL 1007 || (SLIST_FIRST(&pg->mdpage.pvh_list)->pv_flags & PVF_KWRITE) == PVF_KWRITE) 1008 pg->mdpage.pvh_attrs &= ~PVF_KMOD; 1009 1010 /* 1011 * If this was a writeable page and there are no more writeable 1012 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1013 * the contents to memory. 1014 */ 1015 if (pg->mdpage.krw_mappings + pg->mdpage.urw_mappings == 0) 1016 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 1017 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1018 #endif /* PMAP_CACHE_VIPT */ 1019 1020 return(pve); /* return removed pve */ 1021 } 1022 1023 /* 1024 * 1025 * pmap_modify_pv: Update pv flags 1026 * 1027 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1028 * => caller should NOT adjust pmap's wire_count 1029 * => caller must call pmap_vac_me_harder() if writable status of a page 1030 * may have changed. 1031 * => we return the old flags 1032 * 1033 * Modify a physical-virtual mapping in the pv table 1034 */ 1035 static u_int 1036 pmap_modify_pv(struct vm_page *pg, pmap_t pm, vaddr_t va, 1037 u_int clr_mask, u_int set_mask) 1038 { 1039 struct pv_entry *npv; 1040 u_int flags, oflags; 1041 1042 KASSERT((clr_mask & PVF_KENTRY) == 0); 1043 KASSERT((set_mask & PVF_KENTRY) == 0); 1044 1045 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1046 return (0); 1047 1048 NPDEBUG(PDB_PVDUMP, 1049 printf("pmap_modify_pv: pm %p, pg %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, pg, clr_mask, set_mask, npv->pv_flags)); 1050 1051 /* 1052 * There is at least one VA mapping this page. 1053 */ 1054 1055 if (clr_mask & (PVF_REF | PVF_MOD)) { 1056 pg->mdpage.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1057 #ifdef PMAP_CACHE_VIPT 1058 if ((pg->mdpage.pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1059 pg->mdpage.pvh_attrs |= PVF_DIRTY; 1060 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1061 #endif 1062 } 1063 1064 oflags = npv->pv_flags; 1065 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1066 1067 if ((flags ^ oflags) & PVF_WIRED) { 1068 if (flags & PVF_WIRED) 1069 ++pm->pm_stats.wired_count; 1070 else 1071 --pm->pm_stats.wired_count; 1072 } 1073 1074 if ((flags ^ oflags) & PVF_WRITE) { 1075 if (pm == pmap_kernel()) { 1076 if (flags & PVF_WRITE) { 1077 pg->mdpage.krw_mappings++; 1078 pg->mdpage.kro_mappings--; 1079 } else { 1080 pg->mdpage.kro_mappings++; 1081 pg->mdpage.krw_mappings--; 1082 } 1083 } else 1084 if (flags & PVF_WRITE) { 1085 pg->mdpage.urw_mappings++; 1086 pg->mdpage.uro_mappings--; 1087 } else { 1088 pg->mdpage.uro_mappings++; 1089 pg->mdpage.urw_mappings--; 1090 } 1091 } 1092 #ifdef PMAP_CACHE_VIPT 1093 if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) 1094 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 1095 /* 1096 * We have two cases here: the first is from enter_pv (new exec 1097 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1098 * Since in latter, pmap_enter_pv won't do anything, we just have 1099 * to do what pmap_remove_pv would do. 1100 */ 1101 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) 1102 || (PV_IS_EXEC_P(pg->mdpage.pvh_attrs) 1103 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1104 pmap_syncicache_page(pg); 1105 PMAPCOUNT(exec_synced_remap); 1106 } 1107 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1108 #endif 1109 1110 PMAPCOUNT(remappings); 1111 1112 return (oflags); 1113 } 1114 1115 /* 1116 * Allocate an L1 translation table for the specified pmap. 1117 * This is called at pmap creation time. 1118 */ 1119 static void 1120 pmap_alloc_l1(pmap_t pm) 1121 { 1122 struct l1_ttable *l1; 1123 u_int8_t domain; 1124 1125 /* 1126 * Remove the L1 at the head of the LRU list 1127 */ 1128 simple_lock(&l1_lru_lock); 1129 l1 = TAILQ_FIRST(&l1_lru_list); 1130 KDASSERT(l1 != NULL); 1131 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1132 1133 /* 1134 * Pick the first available domain number, and update 1135 * the link to the next number. 1136 */ 1137 domain = l1->l1_domain_first; 1138 l1->l1_domain_first = l1->l1_domain_free[domain]; 1139 1140 /* 1141 * If there are still free domain numbers in this L1, 1142 * put it back on the TAIL of the LRU list. 1143 */ 1144 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1145 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1146 1147 simple_unlock(&l1_lru_lock); 1148 1149 /* 1150 * Fix up the relevant bits in the pmap structure 1151 */ 1152 pm->pm_l1 = l1; 1153 pm->pm_domain = domain; 1154 } 1155 1156 /* 1157 * Free an L1 translation table. 1158 * This is called at pmap destruction time. 1159 */ 1160 static void 1161 pmap_free_l1(pmap_t pm) 1162 { 1163 struct l1_ttable *l1 = pm->pm_l1; 1164 1165 simple_lock(&l1_lru_lock); 1166 1167 /* 1168 * If this L1 is currently on the LRU list, remove it. 1169 */ 1170 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1171 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1172 1173 /* 1174 * Free up the domain number which was allocated to the pmap 1175 */ 1176 l1->l1_domain_free[pm->pm_domain] = l1->l1_domain_first; 1177 l1->l1_domain_first = pm->pm_domain; 1178 l1->l1_domain_use_count--; 1179 1180 /* 1181 * The L1 now must have at least 1 free domain, so add 1182 * it back to the LRU list. If the use count is zero, 1183 * put it at the head of the list, otherwise it goes 1184 * to the tail. 1185 */ 1186 if (l1->l1_domain_use_count == 0) 1187 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1188 else 1189 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1190 1191 simple_unlock(&l1_lru_lock); 1192 } 1193 1194 static inline void 1195 pmap_use_l1(pmap_t pm) 1196 { 1197 struct l1_ttable *l1; 1198 1199 /* 1200 * Do nothing if we're in interrupt context. 1201 * Access to an L1 by the kernel pmap must not affect 1202 * the LRU list. 1203 */ 1204 if (cpu_intr_p() || pm == pmap_kernel()) 1205 return; 1206 1207 l1 = pm->pm_l1; 1208 1209 /* 1210 * If the L1 is not currently on the LRU list, just return 1211 */ 1212 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1213 return; 1214 1215 simple_lock(&l1_lru_lock); 1216 1217 /* 1218 * Check the use count again, now that we've acquired the lock 1219 */ 1220 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1221 simple_unlock(&l1_lru_lock); 1222 return; 1223 } 1224 1225 /* 1226 * Move the L1 to the back of the LRU list 1227 */ 1228 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1229 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1230 1231 simple_unlock(&l1_lru_lock); 1232 } 1233 1234 /* 1235 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1236 * 1237 * Free an L2 descriptor table. 1238 */ 1239 static inline void 1240 #ifndef PMAP_INCLUDE_PTE_SYNC 1241 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1242 #else 1243 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1244 #endif 1245 { 1246 #ifdef PMAP_INCLUDE_PTE_SYNC 1247 #ifdef PMAP_CACHE_VIVT 1248 /* 1249 * Note: With a write-back cache, we may need to sync this 1250 * L2 table before re-using it. 1251 * This is because it may have belonged to a non-current 1252 * pmap, in which case the cache syncs would have been 1253 * skipped for the pages that were being unmapped. If the 1254 * L2 table were then to be immediately re-allocated to 1255 * the *current* pmap, it may well contain stale mappings 1256 * which have not yet been cleared by a cache write-back 1257 * and so would still be visible to the mmu. 1258 */ 1259 if (need_sync) 1260 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1261 #endif /* PMAP_CACHE_VIVT */ 1262 #endif /* PMAP_INCLUDE_PTE_SYNC */ 1263 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1264 } 1265 1266 /* 1267 * Returns a pointer to the L2 bucket associated with the specified pmap 1268 * and VA, or NULL if no L2 bucket exists for the address. 1269 */ 1270 static inline struct l2_bucket * 1271 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1272 { 1273 struct l2_dtable *l2; 1274 struct l2_bucket *l2b; 1275 u_short l1idx; 1276 1277 l1idx = L1_IDX(va); 1278 1279 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 1280 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 1281 return (NULL); 1282 1283 return (l2b); 1284 } 1285 1286 /* 1287 * Returns a pointer to the L2 bucket associated with the specified pmap 1288 * and VA. 1289 * 1290 * If no L2 bucket exists, perform the necessary allocations to put an L2 1291 * bucket/page table in place. 1292 * 1293 * Note that if a new L2 bucket/page was allocated, the caller *must* 1294 * increment the bucket occupancy counter appropriately *before* 1295 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1296 * the bucket/page in the meantime. 1297 */ 1298 static struct l2_bucket * 1299 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1300 { 1301 struct l2_dtable *l2; 1302 struct l2_bucket *l2b; 1303 u_short l1idx; 1304 1305 l1idx = L1_IDX(va); 1306 1307 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 1308 /* 1309 * No mapping at this address, as there is 1310 * no entry in the L1 table. 1311 * Need to allocate a new l2_dtable. 1312 */ 1313 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1314 return (NULL); 1315 1316 /* 1317 * Link it into the parent pmap 1318 */ 1319 pm->pm_l2[L2_IDX(l1idx)] = l2; 1320 } 1321 1322 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1323 1324 /* 1325 * Fetch pointer to the L2 page table associated with the address. 1326 */ 1327 if (l2b->l2b_kva == NULL) { 1328 pt_entry_t *ptep; 1329 1330 /* 1331 * No L2 page table has been allocated. Chances are, this 1332 * is because we just allocated the l2_dtable, above. 1333 */ 1334 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_phys)) == NULL) { 1335 /* 1336 * Oops, no more L2 page tables available at this 1337 * time. We may need to deallocate the l2_dtable 1338 * if we allocated a new one above. 1339 */ 1340 if (l2->l2_occupancy == 0) { 1341 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1342 pmap_free_l2_dtable(l2); 1343 } 1344 return (NULL); 1345 } 1346 1347 l2->l2_occupancy++; 1348 l2b->l2b_kva = ptep; 1349 l2b->l2b_l1idx = l1idx; 1350 } 1351 1352 return (l2b); 1353 } 1354 1355 /* 1356 * One or more mappings in the specified L2 descriptor table have just been 1357 * invalidated. 1358 * 1359 * Garbage collect the metadata and descriptor table itself if necessary. 1360 * 1361 * The pmap lock must be acquired when this is called (not necessary 1362 * for the kernel pmap). 1363 */ 1364 static void 1365 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1366 { 1367 struct l2_dtable *l2; 1368 pd_entry_t *pl1pd, l1pd; 1369 pt_entry_t *ptep; 1370 u_short l1idx; 1371 1372 KDASSERT(count <= l2b->l2b_occupancy); 1373 1374 /* 1375 * Update the bucket's reference count according to how many 1376 * PTEs the caller has just invalidated. 1377 */ 1378 l2b->l2b_occupancy -= count; 1379 1380 /* 1381 * Note: 1382 * 1383 * Level 2 page tables allocated to the kernel pmap are never freed 1384 * as that would require checking all Level 1 page tables and 1385 * removing any references to the Level 2 page table. See also the 1386 * comment elsewhere about never freeing bootstrap L2 descriptors. 1387 * 1388 * We make do with just invalidating the mapping in the L2 table. 1389 * 1390 * This isn't really a big deal in practice and, in fact, leads 1391 * to a performance win over time as we don't need to continually 1392 * alloc/free. 1393 */ 1394 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1395 return; 1396 1397 /* 1398 * There are no more valid mappings in this level 2 page table. 1399 * Go ahead and NULL-out the pointer in the bucket, then 1400 * free the page table. 1401 */ 1402 l1idx = l2b->l2b_l1idx; 1403 ptep = l2b->l2b_kva; 1404 l2b->l2b_kva = NULL; 1405 1406 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1407 1408 /* 1409 * If the L1 slot matches the pmap's domain 1410 * number, then invalidate it. 1411 */ 1412 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 1413 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 1414 *pl1pd = 0; 1415 PTE_SYNC(pl1pd); 1416 } 1417 1418 /* 1419 * Release the L2 descriptor table back to the pool cache. 1420 */ 1421 #ifndef PMAP_INCLUDE_PTE_SYNC 1422 pmap_free_l2_ptp(ptep, l2b->l2b_phys); 1423 #else 1424 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_phys); 1425 #endif 1426 1427 /* 1428 * Update the reference count in the associated l2_dtable 1429 */ 1430 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1431 if (--l2->l2_occupancy > 0) 1432 return; 1433 1434 /* 1435 * There are no more valid mappings in any of the Level 1 1436 * slots managed by this l2_dtable. Go ahead and NULL-out 1437 * the pointer in the parent pmap and free the l2_dtable. 1438 */ 1439 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1440 pmap_free_l2_dtable(l2); 1441 } 1442 1443 /* 1444 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1445 * structures. 1446 */ 1447 static int 1448 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1449 { 1450 #ifndef PMAP_INCLUDE_PTE_SYNC 1451 struct l2_bucket *l2b; 1452 pt_entry_t *ptep, pte; 1453 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1454 1455 /* 1456 * The mappings for these page tables were initially made using 1457 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1458 * mode will not be right for page table mappings. To avoid 1459 * polluting the pmap_kenter_pa() code with a special case for 1460 * page tables, we simply fix up the cache-mode here if it's not 1461 * correct. 1462 */ 1463 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1464 KDASSERT(l2b != NULL); 1465 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1466 pte = *ptep; 1467 1468 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1469 /* 1470 * Page tables must have the cache-mode set to Write-Thru. 1471 */ 1472 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1473 PTE_SYNC(ptep); 1474 cpu_tlb_flushD_SE(va); 1475 cpu_cpwait(); 1476 } 1477 #endif 1478 1479 memset(v, 0, L2_TABLE_SIZE_REAL); 1480 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1481 return (0); 1482 } 1483 1484 static int 1485 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1486 { 1487 1488 memset(v, 0, sizeof(struct l2_dtable)); 1489 return (0); 1490 } 1491 1492 static int 1493 pmap_pmap_ctor(void *arg, void *v, int flags) 1494 { 1495 1496 memset(v, 0, sizeof(struct pmap)); 1497 return (0); 1498 } 1499 1500 static void 1501 pmap_pinit(pmap_t pm) 1502 { 1503 struct l2_bucket *l2b; 1504 1505 if (vector_page < KERNEL_BASE) { 1506 /* 1507 * Map the vector page. 1508 */ 1509 pmap_enter(pm, vector_page, systempage.pv_pa, 1510 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1511 pmap_update(pm); 1512 1513 pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 1514 l2b = pmap_get_l2_bucket(pm, vector_page); 1515 pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO | 1516 L1_C_DOM(pm->pm_domain); 1517 } else 1518 pm->pm_pl1vec = NULL; 1519 } 1520 1521 #ifdef PMAP_CACHE_VIVT 1522 /* 1523 * Since we have a virtually indexed cache, we may need to inhibit caching if 1524 * there is more than one mapping and at least one of them is writable. 1525 * Since we purge the cache on every context switch, we only need to check for 1526 * other mappings within the same pmap, or kernel_pmap. 1527 * This function is also called when a page is unmapped, to possibly reenable 1528 * caching on any remaining mappings. 1529 * 1530 * The code implements the following logic, where: 1531 * 1532 * KW = # of kernel read/write pages 1533 * KR = # of kernel read only pages 1534 * UW = # of user read/write pages 1535 * UR = # of user read only pages 1536 * 1537 * KC = kernel mapping is cacheable 1538 * UC = user mapping is cacheable 1539 * 1540 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1541 * +--------------------------------------------- 1542 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1543 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1544 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1545 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1546 */ 1547 1548 static const int pmap_vac_flags[4][4] = { 1549 {-1, 0, 0, PVF_KNC}, 1550 {0, 0, PVF_NC, PVF_NC}, 1551 {0, PVF_NC, PVF_NC, PVF_NC}, 1552 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1553 }; 1554 1555 static inline int 1556 pmap_get_vac_flags(const struct vm_page *pg) 1557 { 1558 int kidx, uidx; 1559 1560 kidx = 0; 1561 if (pg->mdpage.kro_mappings || pg->mdpage.krw_mappings > 1) 1562 kidx |= 1; 1563 if (pg->mdpage.krw_mappings) 1564 kidx |= 2; 1565 1566 uidx = 0; 1567 if (pg->mdpage.uro_mappings || pg->mdpage.urw_mappings > 1) 1568 uidx |= 1; 1569 if (pg->mdpage.urw_mappings) 1570 uidx |= 2; 1571 1572 return (pmap_vac_flags[uidx][kidx]); 1573 } 1574 1575 static inline void 1576 pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vaddr_t va) 1577 { 1578 int nattr; 1579 1580 nattr = pmap_get_vac_flags(pg); 1581 1582 if (nattr < 0) { 1583 pg->mdpage.pvh_attrs &= ~PVF_NC; 1584 return; 1585 } 1586 1587 if (nattr == 0 && (pg->mdpage.pvh_attrs & PVF_NC) == 0) 1588 return; 1589 1590 if (pm == pmap_kernel()) 1591 pmap_vac_me_kpmap(pg, pm, va); 1592 else 1593 pmap_vac_me_user(pg, pm, va); 1594 1595 pg->mdpage.pvh_attrs = (pg->mdpage.pvh_attrs & ~PVF_NC) | nattr; 1596 } 1597 1598 static void 1599 pmap_vac_me_kpmap(struct vm_page *pg, pmap_t pm, vaddr_t va) 1600 { 1601 u_int u_cacheable, u_entries; 1602 struct pv_entry *pv; 1603 pmap_t last_pmap = pm; 1604 1605 /* 1606 * Pass one, see if there are both kernel and user pmaps for 1607 * this page. Calculate whether there are user-writable or 1608 * kernel-writable pages. 1609 */ 1610 u_cacheable = 0; 1611 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 1612 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1613 u_cacheable++; 1614 } 1615 1616 u_entries = pg->mdpage.urw_mappings + pg->mdpage.uro_mappings; 1617 1618 /* 1619 * We know we have just been updating a kernel entry, so if 1620 * all user pages are already cacheable, then there is nothing 1621 * further to do. 1622 */ 1623 if (pg->mdpage.k_mappings == 0 && u_cacheable == u_entries) 1624 return; 1625 1626 if (u_entries) { 1627 /* 1628 * Scan over the list again, for each entry, if it 1629 * might not be set correctly, call pmap_vac_me_user 1630 * to recalculate the settings. 1631 */ 1632 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 1633 /* 1634 * We know kernel mappings will get set 1635 * correctly in other calls. We also know 1636 * that if the pmap is the same as last_pmap 1637 * then we've just handled this entry. 1638 */ 1639 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1640 continue; 1641 1642 /* 1643 * If there are kernel entries and this page 1644 * is writable but non-cacheable, then we can 1645 * skip this entry also. 1646 */ 1647 if (pg->mdpage.k_mappings && 1648 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1649 (PVF_NC | PVF_WRITE)) 1650 continue; 1651 1652 /* 1653 * Similarly if there are no kernel-writable 1654 * entries and the page is already 1655 * read-only/cacheable. 1656 */ 1657 if (pg->mdpage.krw_mappings == 0 && 1658 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1659 continue; 1660 1661 /* 1662 * For some of the remaining cases, we know 1663 * that we must recalculate, but for others we 1664 * can't tell if they are correct or not, so 1665 * we recalculate anyway. 1666 */ 1667 pmap_vac_me_user(pg, (last_pmap = pv->pv_pmap), 0); 1668 } 1669 1670 if (pg->mdpage.k_mappings == 0) 1671 return; 1672 } 1673 1674 pmap_vac_me_user(pg, pm, va); 1675 } 1676 1677 static void 1678 pmap_vac_me_user(struct vm_page *pg, pmap_t pm, vaddr_t va) 1679 { 1680 pmap_t kpmap = pmap_kernel(); 1681 struct pv_entry *pv, *npv = NULL; 1682 struct l2_bucket *l2b; 1683 pt_entry_t *ptep, pte; 1684 u_int entries = 0; 1685 u_int writable = 0; 1686 u_int cacheable_entries = 0; 1687 u_int kern_cacheable = 0; 1688 u_int other_writable = 0; 1689 1690 /* 1691 * Count mappings and writable mappings in this pmap. 1692 * Include kernel mappings as part of our own. 1693 * Keep a pointer to the first one. 1694 */ 1695 npv = NULL; 1696 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 1697 /* Count mappings in the same pmap */ 1698 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1699 if (entries++ == 0) 1700 npv = pv; 1701 1702 /* Cacheable mappings */ 1703 if ((pv->pv_flags & PVF_NC) == 0) { 1704 cacheable_entries++; 1705 if (kpmap == pv->pv_pmap) 1706 kern_cacheable++; 1707 } 1708 1709 /* Writable mappings */ 1710 if (pv->pv_flags & PVF_WRITE) 1711 ++writable; 1712 } else 1713 if (pv->pv_flags & PVF_WRITE) 1714 other_writable = 1; 1715 } 1716 1717 /* 1718 * Enable or disable caching as necessary. 1719 * Note: the first entry might be part of the kernel pmap, 1720 * so we can't assume this is indicative of the state of the 1721 * other (maybe non-kpmap) entries. 1722 */ 1723 if ((entries > 1 && writable) || 1724 (entries > 0 && pm == kpmap && other_writable)) { 1725 if (cacheable_entries == 0) 1726 return; 1727 1728 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1729 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1730 (pv->pv_flags & PVF_NC)) 1731 continue; 1732 1733 pv->pv_flags |= PVF_NC; 1734 1735 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1736 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1737 pte = *ptep & ~L2_S_CACHE_MASK; 1738 1739 if ((va != pv->pv_va || pm != pv->pv_pmap) && 1740 l2pte_valid(pte)) { 1741 if (PV_BEEN_EXECD(pv->pv_flags)) { 1742 #ifdef PMAP_CACHE_VIVT 1743 pmap_idcache_wbinv_range(pv->pv_pmap, 1744 pv->pv_va, PAGE_SIZE); 1745 #endif 1746 pmap_tlb_flushID_SE(pv->pv_pmap, 1747 pv->pv_va); 1748 } else 1749 if (PV_BEEN_REFD(pv->pv_flags)) { 1750 #ifdef PMAP_CACHE_VIVT 1751 pmap_dcache_wb_range(pv->pv_pmap, 1752 pv->pv_va, PAGE_SIZE, true, 1753 (pv->pv_flags & PVF_WRITE) == 0); 1754 #endif 1755 pmap_tlb_flushD_SE(pv->pv_pmap, 1756 pv->pv_va); 1757 } 1758 } 1759 1760 *ptep = pte; 1761 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1762 } 1763 cpu_cpwait(); 1764 } else 1765 if (entries > cacheable_entries) { 1766 /* 1767 * Turn cacheing back on for some pages. If it is a kernel 1768 * page, only do so if there are no other writable pages. 1769 */ 1770 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1771 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1772 (kpmap != pv->pv_pmap || other_writable))) 1773 continue; 1774 1775 pv->pv_flags &= ~PVF_NC; 1776 1777 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1778 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1779 pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; 1780 1781 if (l2pte_valid(pte)) { 1782 if (PV_BEEN_EXECD(pv->pv_flags)) { 1783 pmap_tlb_flushID_SE(pv->pv_pmap, 1784 pv->pv_va); 1785 } else 1786 if (PV_BEEN_REFD(pv->pv_flags)) { 1787 pmap_tlb_flushD_SE(pv->pv_pmap, 1788 pv->pv_va); 1789 } 1790 } 1791 1792 *ptep = pte; 1793 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1794 } 1795 } 1796 } 1797 #endif 1798 1799 #ifdef PMAP_CACHE_VIPT 1800 /* 1801 * For virtually indexed / physically tagged caches, what we have to worry 1802 * about is illegal cache aliases. To prevent this, we must ensure that 1803 * virtual addresses that map the physical page use the same bits for those 1804 * bits masked by "arm_cache_prefer_mask" (bits 12+). If there is a conflict, 1805 * all mappings of the page must be non-cached. 1806 */ 1807 #if 0 1808 static inline vaddr_t 1809 pmap_check_sets(paddr_t pa) 1810 { 1811 extern int arm_dcache_l2_nsets; 1812 int set, way; 1813 vaddr_t mask = 0; 1814 int v; 1815 pa |= 1; 1816 for (set = 0; set < (1 << arm_dcache_l2_nsets); set++) { 1817 for (way = 0; way < 4; way++) { 1818 v = (way << 30) | (set << 5); 1819 asm("mcr p15, 3, %0, c15, c2, 0" :: "r"(v)); 1820 asm("mrc p15, 3, %0, c15, c0, 0" : "=r"(v)); 1821 1822 if ((v & (1 | ~(PAGE_SIZE-1))) == pa) { 1823 mask |= 1 << (set >> 7); 1824 } 1825 } 1826 } 1827 return mask; 1828 } 1829 #endif 1830 static void 1831 pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vaddr_t va) 1832 { 1833 struct pv_entry *pv; 1834 vaddr_t tst_mask; 1835 bool bad_alias; 1836 struct l2_bucket *l2b; 1837 pt_entry_t *ptep, pte, opte; 1838 const u_int 1839 rw_mappings = pg->mdpage.urw_mappings + pg->mdpage.krw_mappings, 1840 ro_mappings = pg->mdpage.uro_mappings + pg->mdpage.kro_mappings; 1841 1842 /* do we need to do anything? */ 1843 if (arm_cache_prefer_mask == 0) 1844 return; 1845 1846 NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: pg=%p, pmap=%p va=%08lx\n", 1847 pg, pm, va)); 1848 1849 #define popc4(x) \ 1850 (((0x94 >> ((x & 3) << 1)) & 3) + ((0x94 >> ((x & 12) >> 1)) & 3)) 1851 #if 0 1852 tst_mask = pmap_check_sets(pg->phys_addr); 1853 KASSERT(popc4(tst_mask) < 2); 1854 #endif 1855 1856 KASSERT(!va || pm); 1857 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1858 1859 /* Already a conflict? */ 1860 if (__predict_false(pg->mdpage.pvh_attrs & PVF_NC)) { 1861 /* just an add, things are already non-cached */ 1862 KASSERT(!(pg->mdpage.pvh_attrs & PVF_DIRTY)); 1863 bad_alias = false; 1864 if (va) { 1865 PMAPCOUNT(vac_color_none); 1866 bad_alias = true; 1867 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 1868 goto fixup; 1869 } 1870 pv = SLIST_FIRST(&pg->mdpage.pvh_list); 1871 /* the list can't be empty because it would be cachable */ 1872 if (pg->mdpage.pvh_attrs & PVF_KMPAGE) { 1873 tst_mask = pg->mdpage.pvh_attrs; 1874 } else { 1875 KASSERT(pv); 1876 tst_mask = pv->pv_va; 1877 pv = SLIST_NEXT(pv, pv_link); 1878 } 1879 /* 1880 * Only check for a bad alias if we have writable mappings. 1881 */ 1882 tst_mask &= arm_cache_prefer_mask; 1883 if (rw_mappings > 0 && arm_cache_prefer_mask) { 1884 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 1885 /* if there's a bad alias, stop checking. */ 1886 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 1887 bad_alias = true; 1888 } 1889 pg->mdpage.pvh_attrs |= PVF_WRITE; 1890 if (!bad_alias) 1891 pg->mdpage.pvh_attrs |= PVF_DIRTY; 1892 } else { 1893 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 1894 } 1895 /* If no conflicting colors, set everything back to cached */ 1896 if (!bad_alias) { 1897 #ifdef DEBUG 1898 if ((pg->mdpage.pvh_attrs & PVF_WRITE) 1899 || ro_mappings < 2) { 1900 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) 1901 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 1902 } 1903 1904 #endif 1905 pg->mdpage.pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 1906 pg->mdpage.pvh_attrs |= tst_mask | PVF_COLORED; 1907 /* 1908 * Restore DIRTY bit if page is modified 1909 */ 1910 if (pg->mdpage.pvh_attrs & PVF_DMOD) 1911 pg->mdpage.pvh_attrs |= PVF_DIRTY; 1912 PMAPCOUNT(vac_color_restore); 1913 } else { 1914 KASSERT(SLIST_FIRST(&pg->mdpage.pvh_list) != NULL); 1915 KASSERT(SLIST_NEXT(SLIST_FIRST(&pg->mdpage.pvh_list), pv_link) != NULL); 1916 } 1917 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1918 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 1919 } else if (!va) { 1920 KASSERT(pmap_is_page_colored_p(pg)); 1921 KASSERT(!(pg->mdpage.pvh_attrs & PVF_WRITE) 1922 || (pg->mdpage.pvh_attrs & PVF_DIRTY)); 1923 if (rw_mappings == 0) 1924 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 1925 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1926 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 1927 return; 1928 } else if (!pmap_is_page_colored_p(pg)) { 1929 /* not colored so we just use its color */ 1930 KASSERT(pg->mdpage.pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 1931 PMAPCOUNT(vac_color_new); 1932 pg->mdpage.pvh_attrs &= PAGE_SIZE - 1; 1933 pg->mdpage.pvh_attrs |= PVF_COLORED 1934 | (va & arm_cache_prefer_mask) 1935 | (rw_mappings > 0 ? PVF_WRITE : 0); 1936 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1937 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 1938 return; 1939 } else if (((pg->mdpage.pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 1940 bad_alias = false; 1941 if (rw_mappings > 0) { 1942 /* 1943 * We now have writeable mappings and more than one 1944 * readonly mapping, verify the colors don't clash 1945 * and mark the page as writeable. 1946 */ 1947 if (ro_mappings > 1 1948 && (pg->mdpage.pvh_attrs & PVF_WRITE) == 0 1949 && arm_cache_prefer_mask) { 1950 tst_mask = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 1951 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 1952 /* if there's a bad alias, stop checking. */ 1953 if (((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0) { 1954 bad_alias = true; 1955 break; 1956 } 1957 } 1958 } 1959 pg->mdpage.pvh_attrs |= PVF_WRITE; 1960 } 1961 /* If no conflicting colors, set everything back to cached */ 1962 if (!bad_alias) { 1963 #ifdef DEBUG 1964 if (rw_mappings > 0 1965 || (pg->mdpage.pvh_attrs & PMAP_KMPAGE)) { 1966 tst_mask = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 1967 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) 1968 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 1969 } 1970 #endif 1971 if (SLIST_EMPTY(&pg->mdpage.pvh_list)) 1972 PMAPCOUNT(vac_color_reuse); 1973 else 1974 PMAPCOUNT(vac_color_ok); 1975 1976 /* matching color, just return */ 1977 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1978 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 1979 return; 1980 } 1981 KASSERT(SLIST_FIRST(&pg->mdpage.pvh_list) != NULL); 1982 KASSERT(SLIST_NEXT(SLIST_FIRST(&pg->mdpage.pvh_list), pv_link) != NULL); 1983 1984 /* color conflict. evict from cache. */ 1985 1986 pmap_flush_page(pg, true); 1987 pg->mdpage.pvh_attrs &= ~PVF_COLORED; 1988 pg->mdpage.pvh_attrs |= PVF_NC; 1989 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 1990 PMAPCOUNT(vac_color_erase); 1991 } else if (rw_mappings == 0 1992 && (pg->mdpage.pvh_attrs & PVF_KMPAGE) == 0) { 1993 KASSERT((pg->mdpage.pvh_attrs & PVF_WRITE) == 0); 1994 1995 /* 1996 * If the page has dirty cache lines, clean it. 1997 */ 1998 if (pg->mdpage.pvh_attrs & PVF_DIRTY) 1999 pmap_flush_page(pg, false); 2000 2001 /* 2002 * If this is the first remapping (we know that there are no 2003 * writeable mappings), then this is a simple color change. 2004 * Otherwise this is a seconary r/o mapping, which means 2005 * we don't have to do anything. 2006 */ 2007 if (ro_mappings == 1) { 2008 KASSERT(((pg->mdpage.pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2009 pg->mdpage.pvh_attrs &= PAGE_SIZE - 1; 2010 pg->mdpage.pvh_attrs |= (va & arm_cache_prefer_mask); 2011 PMAPCOUNT(vac_color_change); 2012 } else { 2013 PMAPCOUNT(vac_color_blind); 2014 } 2015 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 2016 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 2017 return; 2018 } else { 2019 if (rw_mappings > 0) 2020 pg->mdpage.pvh_attrs |= PVF_WRITE; 2021 2022 /* color conflict. evict from cache. */ 2023 pmap_flush_page(pg, true); 2024 2025 /* the list can't be empty because this was a enter/modify */ 2026 pv = SLIST_FIRST(&pg->mdpage.pvh_list); 2027 if ((pg->mdpage.pvh_attrs & PVF_KMPAGE) == 0) { 2028 KASSERT(pv); 2029 /* 2030 * If there's only one mapped page, change color to the 2031 * page's new color and return. Restore the DIRTY bit 2032 * that was erased by pmap_flush_page. 2033 */ 2034 if (SLIST_NEXT(pv, pv_link) == NULL) { 2035 pg->mdpage.pvh_attrs &= PAGE_SIZE - 1; 2036 pg->mdpage.pvh_attrs |= (va & arm_cache_prefer_mask); 2037 if (pg->mdpage.pvh_attrs & PVF_DMOD) 2038 pg->mdpage.pvh_attrs |= PVF_DIRTY; 2039 PMAPCOUNT(vac_color_change); 2040 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 2041 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 2042 return; 2043 } 2044 } 2045 bad_alias = true; 2046 pg->mdpage.pvh_attrs &= ~PVF_COLORED; 2047 pg->mdpage.pvh_attrs |= PVF_NC; 2048 PMAPCOUNT(vac_color_erase); 2049 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 2050 } 2051 2052 fixup: 2053 KASSERT((rw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 2054 2055 /* 2056 * Turn cacheing on/off for all pages. 2057 */ 2058 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 2059 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2060 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2061 opte = *ptep; 2062 pte = opte & ~L2_S_CACHE_MASK; 2063 if (bad_alias) { 2064 pv->pv_flags |= PVF_NC; 2065 } else { 2066 pv->pv_flags &= ~PVF_NC; 2067 pte |= pte_l2_s_cache_mode; 2068 } 2069 2070 if (opte == pte) /* only update is there's a change */ 2071 continue; 2072 2073 if (l2pte_valid(pte)) { 2074 if (PV_BEEN_EXECD(pv->pv_flags)) { 2075 pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); 2076 } else if (PV_BEEN_REFD(pv->pv_flags)) { 2077 pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); 2078 } 2079 } 2080 2081 *ptep = pte; 2082 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2083 } 2084 } 2085 #endif /* PMAP_CACHE_VIPT */ 2086 2087 2088 /* 2089 * Modify pte bits for all ptes corresponding to the given physical address. 2090 * We use `maskbits' rather than `clearbits' because we're always passing 2091 * constants and the latter would require an extra inversion at run-time. 2092 */ 2093 static void 2094 pmap_clearbit(struct vm_page *pg, u_int maskbits) 2095 { 2096 struct l2_bucket *l2b; 2097 struct pv_entry *pv; 2098 pt_entry_t *ptep, npte, opte; 2099 pmap_t pm; 2100 vaddr_t va; 2101 u_int oflags; 2102 #ifdef PMAP_CACHE_VIPT 2103 const bool want_syncicache = PV_IS_EXEC_P(pg->mdpage.pvh_attrs); 2104 bool need_syncicache = false; 2105 bool did_syncicache = false; 2106 bool need_vac_me_harder = false; 2107 #endif 2108 2109 NPDEBUG(PDB_BITS, 2110 printf("pmap_clearbit: pg %p (0x%08lx) mask 0x%x\n", 2111 pg, VM_PAGE_TO_PHYS(pg), maskbits)); 2112 2113 PMAP_HEAD_TO_MAP_LOCK(); 2114 simple_lock(&pg->mdpage.pvh_slock); 2115 2116 #ifdef PMAP_CACHE_VIPT 2117 /* 2118 * If we might want to sync the I-cache and we've modified it, 2119 * then we know we definitely need to sync or discard it. 2120 */ 2121 if (want_syncicache) 2122 need_syncicache = pg->mdpage.pvh_attrs & PVF_MOD; 2123 #endif 2124 /* 2125 * Clear saved attributes (modify, reference) 2126 */ 2127 pg->mdpage.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2128 2129 if (SLIST_EMPTY(&pg->mdpage.pvh_list)) { 2130 #ifdef PMAP_CACHE_VIPT 2131 if (need_syncicache) { 2132 /* 2133 * No one has it mapped, so just discard it. The next 2134 * exec remapping will cause it to be synced. 2135 */ 2136 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 2137 PMAPCOUNT(exec_discarded_clearbit); 2138 } 2139 #endif 2140 simple_unlock(&pg->mdpage.pvh_slock); 2141 PMAP_HEAD_TO_MAP_UNLOCK(); 2142 return; 2143 } 2144 2145 /* 2146 * Loop over all current mappings setting/clearing as appropos 2147 */ 2148 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 2149 va = pv->pv_va; 2150 pm = pv->pv_pmap; 2151 oflags = pv->pv_flags; 2152 /* 2153 * Kernel entries are unmanaged and as such not to be changed. 2154 */ 2155 if (oflags & PVF_KENTRY) 2156 continue; 2157 pv->pv_flags &= ~maskbits; 2158 2159 pmap_acquire_pmap_lock(pm); 2160 2161 l2b = pmap_get_l2_bucket(pm, va); 2162 KDASSERT(l2b != NULL); 2163 2164 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2165 npte = opte = *ptep; 2166 2167 NPDEBUG(PDB_BITS, 2168 printf( 2169 "pmap_clearbit: pv %p, pm %p, va 0x%08lx, flag 0x%x\n", 2170 pv, pv->pv_pmap, pv->pv_va, oflags)); 2171 2172 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2173 #ifdef PMAP_CACHE_VIVT 2174 if ((pv->pv_flags & PVF_NC)) { 2175 /* 2176 * Entry is not cacheable: 2177 * 2178 * Don't turn caching on again if this is a 2179 * modified emulation. This would be 2180 * inconsitent with the settings created by 2181 * pmap_vac_me_harder(). Otherwise, it's safe 2182 * to re-enable cacheing. 2183 * 2184 * There's no need to call pmap_vac_me_harder() 2185 * here: all pages are losing their write 2186 * permission. 2187 */ 2188 if (maskbits & PVF_WRITE) { 2189 npte |= pte_l2_s_cache_mode; 2190 pv->pv_flags &= ~PVF_NC; 2191 } 2192 } else 2193 if (opte & L2_S_PROT_W) { 2194 /* 2195 * Entry is writable/cacheable: check if pmap 2196 * is current if it is flush it, otherwise it 2197 * won't be in the cache 2198 */ 2199 if (PV_BEEN_EXECD(oflags)) 2200 pmap_idcache_wbinv_range(pm, pv->pv_va, 2201 PAGE_SIZE); 2202 else 2203 if (PV_BEEN_REFD(oflags)) 2204 pmap_dcache_wb_range(pm, pv->pv_va, 2205 PAGE_SIZE, 2206 (maskbits & PVF_REF) != 0, false); 2207 } 2208 #endif 2209 2210 /* make the pte read only */ 2211 npte &= ~L2_S_PROT_W; 2212 2213 if (maskbits & oflags & PVF_WRITE) { 2214 /* 2215 * Keep alias accounting up to date 2216 */ 2217 if (pv->pv_pmap == pmap_kernel()) { 2218 pg->mdpage.krw_mappings--; 2219 pg->mdpage.kro_mappings++; 2220 } else { 2221 pg->mdpage.urw_mappings--; 2222 pg->mdpage.uro_mappings++; 2223 } 2224 #ifdef PMAP_CACHE_VIPT 2225 if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) 2226 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 2227 if (want_syncicache) 2228 need_syncicache = true; 2229 need_vac_me_harder = true; 2230 #endif 2231 } 2232 } 2233 2234 if (maskbits & PVF_REF) { 2235 if ((pv->pv_flags & PVF_NC) == 0 && 2236 (maskbits & (PVF_WRITE|PVF_MOD)) == 0 && 2237 l2pte_valid(npte)) { 2238 #ifdef PMAP_CACHE_VIVT 2239 /* 2240 * Check npte here; we may have already 2241 * done the wbinv above, and the validity 2242 * of the PTE is the same for opte and 2243 * npte. 2244 */ 2245 /* XXXJRT need idcache_inv_range */ 2246 if (PV_BEEN_EXECD(oflags)) 2247 pmap_idcache_wbinv_range(pm, 2248 pv->pv_va, PAGE_SIZE); 2249 else 2250 if (PV_BEEN_REFD(oflags)) 2251 pmap_dcache_wb_range(pm, 2252 pv->pv_va, PAGE_SIZE, 2253 true, true); 2254 #endif 2255 } 2256 2257 /* 2258 * Make the PTE invalid so that we will take a 2259 * page fault the next time the mapping is 2260 * referenced. 2261 */ 2262 npte &= ~L2_TYPE_MASK; 2263 npte |= L2_TYPE_INV; 2264 } 2265 2266 if (npte != opte) { 2267 *ptep = npte; 2268 PTE_SYNC(ptep); 2269 /* Flush the TLB entry if a current pmap. */ 2270 if (PV_BEEN_EXECD(oflags)) 2271 pmap_tlb_flushID_SE(pm, pv->pv_va); 2272 else 2273 if (PV_BEEN_REFD(oflags)) 2274 pmap_tlb_flushD_SE(pm, pv->pv_va); 2275 } 2276 2277 pmap_release_pmap_lock(pm); 2278 2279 NPDEBUG(PDB_BITS, 2280 printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n", 2281 pm, va, opte, npte)); 2282 } 2283 2284 #ifdef PMAP_CACHE_VIPT 2285 /* 2286 * If we need to sync the I-cache and we haven't done it yet, do it. 2287 */ 2288 if (need_syncicache && !did_syncicache) { 2289 pmap_syncicache_page(pg); 2290 PMAPCOUNT(exec_synced_clearbit); 2291 } 2292 /* 2293 * If we are changing this to read-only, we need to call vac_me_harder 2294 * so we can change all the read-only pages to cacheable. We pretend 2295 * this as a page deletion. 2296 */ 2297 if (need_vac_me_harder) { 2298 if (pg->mdpage.pvh_attrs & PVF_NC) 2299 pmap_vac_me_harder(pg, NULL, 0); 2300 } 2301 #endif 2302 2303 simple_unlock(&pg->mdpage.pvh_slock); 2304 PMAP_HEAD_TO_MAP_UNLOCK(); 2305 } 2306 2307 /* 2308 * pmap_clean_page() 2309 * 2310 * This is a local function used to work out the best strategy to clean 2311 * a single page referenced by its entry in the PV table. It's used by 2312 * pmap_copy_page, pmap_zero page and maybe some others later on. 2313 * 2314 * Its policy is effectively: 2315 * o If there are no mappings, we don't bother doing anything with the cache. 2316 * o If there is one mapping, we clean just that page. 2317 * o If there are multiple mappings, we clean the entire cache. 2318 * 2319 * So that some functions can be further optimised, it returns 0 if it didn't 2320 * clean the entire cache, or 1 if it did. 2321 * 2322 * XXX One bug in this routine is that if the pv_entry has a single page 2323 * mapped at 0x00000000 a whole cache clean will be performed rather than 2324 * just the 1 page. Since this should not occur in everyday use and if it does 2325 * it will just result in not the most efficient clean for the page. 2326 */ 2327 #ifdef PMAP_CACHE_VIVT 2328 static int 2329 pmap_clean_page(struct pv_entry *pv, bool is_src) 2330 { 2331 pmap_t pm, pm_to_clean = NULL; 2332 struct pv_entry *npv; 2333 u_int cache_needs_cleaning = 0; 2334 u_int flags = 0; 2335 vaddr_t page_to_clean = 0; 2336 2337 if (pv == NULL) { 2338 /* nothing mapped in so nothing to flush */ 2339 return (0); 2340 } 2341 2342 /* 2343 * Since we flush the cache each time we change to a different 2344 * user vmspace, we only need to flush the page if it is in the 2345 * current pmap. 2346 */ 2347 pm = curproc->p_vmspace->vm_map.pmap; 2348 2349 for (npv = pv; npv; npv = SLIST_NEXT(npv, pv_link)) { 2350 if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) { 2351 flags |= npv->pv_flags; 2352 /* 2353 * The page is mapped non-cacheable in 2354 * this map. No need to flush the cache. 2355 */ 2356 if (npv->pv_flags & PVF_NC) { 2357 #ifdef DIAGNOSTIC 2358 if (cache_needs_cleaning) 2359 panic("pmap_clean_page: " 2360 "cache inconsistency"); 2361 #endif 2362 break; 2363 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 2364 continue; 2365 if (cache_needs_cleaning) { 2366 page_to_clean = 0; 2367 break; 2368 } else { 2369 page_to_clean = npv->pv_va; 2370 pm_to_clean = npv->pv_pmap; 2371 } 2372 cache_needs_cleaning = 1; 2373 } 2374 } 2375 2376 if (page_to_clean) { 2377 if (PV_BEEN_EXECD(flags)) 2378 pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, 2379 PAGE_SIZE); 2380 else 2381 pmap_dcache_wb_range(pm_to_clean, page_to_clean, 2382 PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); 2383 } else if (cache_needs_cleaning) { 2384 if (PV_BEEN_EXECD(flags)) 2385 pmap_idcache_wbinv_all(pm); 2386 else 2387 pmap_dcache_wbinv_all(pm); 2388 return (1); 2389 } 2390 return (0); 2391 } 2392 #endif 2393 2394 #ifdef PMAP_CACHE_VIPT 2395 /* 2396 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2397 * right cache alias to make sure we flush the right stuff. 2398 */ 2399 void 2400 pmap_syncicache_page(struct vm_page *pg) 2401 { 2402 const vsize_t va_offset = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 2403 pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT]; 2404 2405 NPDEBUG(PDB_EXEC, printf("pmap_syncicache_page: pg=%p (attrs=%#x)\n", 2406 pg, pg->mdpage.pvh_attrs)); 2407 /* 2408 * No need to clean the page if it's non-cached. 2409 */ 2410 if (pg->mdpage.pvh_attrs & PVF_NC) 2411 return; 2412 KASSERT(pg->mdpage.pvh_attrs & PVF_COLORED); 2413 2414 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2415 /* 2416 * Set up a PTE with the right coloring to flush existing cache lines. 2417 */ 2418 *ptep = L2_S_PROTO | 2419 VM_PAGE_TO_PHYS(pg) 2420 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2421 | pte_l2_s_cache_mode; 2422 PTE_SYNC(ptep); 2423 2424 /* 2425 * Flush it. 2426 */ 2427 cpu_icache_sync_range(cdstp + va_offset, PAGE_SIZE); 2428 /* 2429 * Unmap the page. 2430 */ 2431 *ptep = 0; 2432 PTE_SYNC(ptep); 2433 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2434 2435 pg->mdpage.pvh_attrs |= PVF_EXEC; 2436 PMAPCOUNT(exec_synced); 2437 } 2438 2439 void 2440 pmap_flush_page(struct vm_page *pg, bool flush) 2441 { 2442 const vsize_t va_offset = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 2443 const size_t pte_offset = va_offset >> PGSHIFT; 2444 pt_entry_t * const ptep = &cdst_pte[pte_offset]; 2445 const pt_entry_t oldpte = *ptep; 2446 #if 0 2447 vaddr_t mask; 2448 #endif 2449 2450 KASSERT(!(pg->mdpage.pvh_attrs & PVF_NC)); 2451 #if 0 2452 mask = pmap_check_sets(pg->phys_addr); 2453 KASSERT(popc4(mask) < 2); 2454 #endif 2455 2456 NPDEBUG(PDB_VAC, printf("pmap_flush_page: pg=%p (attrs=%#x)\n", 2457 pg, pg->mdpage.pvh_attrs)); 2458 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2459 /* 2460 * Set up a PTE with the right coloring to flush existing cache entries. 2461 */ 2462 *ptep = L2_S_PROTO 2463 | VM_PAGE_TO_PHYS(pg) 2464 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2465 | pte_l2_s_cache_mode; 2466 PTE_SYNC(ptep); 2467 2468 /* 2469 * Flush it. 2470 */ 2471 if (flush) { 2472 cpu_idcache_wbinv_range(cdstp + va_offset, PAGE_SIZE); 2473 pg->mdpage.pvh_attrs &= ~PVF_DIRTY; 2474 } else { 2475 cpu_dcache_wb_range(cdstp + va_offset, PAGE_SIZE); 2476 /* 2477 * Mark that the page is no longer dirty. 2478 */ 2479 if ((pg->mdpage.pvh_attrs & PVF_DMOD) == 0) 2480 pg->mdpage.pvh_attrs &= ~PVF_DIRTY; 2481 } 2482 2483 /* 2484 * Restore the page table entry since we might have interrupted 2485 * pmap_zero_page or pmap_copy_page which was already using this pte. 2486 */ 2487 *ptep = oldpte; 2488 PTE_SYNC(ptep); 2489 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2490 #if 0 2491 mask = pmap_check_sets(pg->phys_addr); 2492 KASSERT(mask == 0); 2493 #endif 2494 } 2495 #endif /* PMAP_CACHE_VIPT */ 2496 2497 /* 2498 * Routine: pmap_page_remove 2499 * Function: 2500 * Removes this physical page from 2501 * all physical maps in which it resides. 2502 * Reflects back modify bits to the pager. 2503 */ 2504 static void 2505 pmap_page_remove(struct vm_page *pg) 2506 { 2507 struct l2_bucket *l2b; 2508 struct pv_entry *pv, *npv, **pvp; 2509 pmap_t pm, curpm; 2510 pt_entry_t *ptep, pte; 2511 bool flush; 2512 u_int flags; 2513 2514 NPDEBUG(PDB_FOLLOW, 2515 printf("pmap_page_remove: pg %p (0x%08lx)\n", pg, 2516 VM_PAGE_TO_PHYS(pg))); 2517 2518 PMAP_HEAD_TO_MAP_LOCK(); 2519 simple_lock(&pg->mdpage.pvh_slock); 2520 2521 pv = SLIST_FIRST(&pg->mdpage.pvh_list); 2522 if (pv == NULL) { 2523 #ifdef PMAP_CACHE_VIPT 2524 /* 2525 * We *know* the page contents are about to be replaced. 2526 * Discard the exec contents 2527 */ 2528 if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) 2529 PMAPCOUNT(exec_discarded_page_protect); 2530 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 2531 KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 2532 #endif 2533 simple_unlock(&pg->mdpage.pvh_slock); 2534 PMAP_HEAD_TO_MAP_UNLOCK(); 2535 return; 2536 } 2537 #ifdef PMAP_CACHE_VIPT 2538 KASSERT(pmap_is_page_colored_p(pg)); 2539 #endif 2540 2541 /* 2542 * Clear alias counts 2543 */ 2544 #ifdef PMAP_CACHE_VIVT 2545 pg->mdpage.k_mappings = 0; 2546 #endif 2547 pg->mdpage.urw_mappings = pg->mdpage.uro_mappings = 0; 2548 2549 flush = false; 2550 flags = 0; 2551 curpm = curproc->p_vmspace->vm_map.pmap; 2552 2553 #ifdef PMAP_CACHE_VIVT 2554 pmap_clean_page(pv, false); 2555 #endif 2556 2557 pvp = &SLIST_FIRST(&pg->mdpage.pvh_list); 2558 while (pv) { 2559 pm = pv->pv_pmap; 2560 npv = SLIST_NEXT(pv, pv_link); 2561 if (flush == false && (pm == curpm || pm == pmap_kernel())) 2562 flush = true; 2563 2564 if (pm == pmap_kernel()) { 2565 #ifdef PMAP_CACHE_VIPT 2566 /* 2567 * If this was unmanaged mapping, it must be preserved. 2568 * Move it back on the list and advance the end-of-list 2569 * pointer. 2570 */ 2571 if (pv->pv_flags & PVF_KENTRY) { 2572 *pvp = pv; 2573 pvp = &SLIST_NEXT(pv, pv_link); 2574 pv = npv; 2575 continue; 2576 } 2577 if (pv->pv_flags & PVF_WRITE) 2578 pg->mdpage.krw_mappings--; 2579 else 2580 pg->mdpage.kro_mappings--; 2581 #endif 2582 PMAPCOUNT(kernel_unmappings); 2583 } 2584 PMAPCOUNT(unmappings); 2585 2586 pmap_acquire_pmap_lock(pm); 2587 2588 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2589 KDASSERT(l2b != NULL); 2590 2591 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2592 pte = *ptep; 2593 2594 /* 2595 * Update statistics 2596 */ 2597 --pm->pm_stats.resident_count; 2598 2599 /* Wired bit */ 2600 if (pv->pv_flags & PVF_WIRED) 2601 --pm->pm_stats.wired_count; 2602 2603 flags |= pv->pv_flags; 2604 2605 /* 2606 * Invalidate the PTEs. 2607 */ 2608 *ptep = 0; 2609 PTE_SYNC_CURRENT(pm, ptep); 2610 pmap_free_l2_bucket(pm, l2b, 1); 2611 2612 pool_put(&pmap_pv_pool, pv); 2613 pv = npv; 2614 /* 2615 * if we reach the end of the list and there are still 2616 * mappings, they might be able to be cached now. 2617 */ 2618 if (pv == NULL) { 2619 *pvp = NULL; 2620 if (!SLIST_EMPTY(&pg->mdpage.pvh_list)) 2621 pmap_vac_me_harder(pg, pm, 0); 2622 } 2623 pmap_release_pmap_lock(pm); 2624 } 2625 #ifdef PMAP_CACHE_VIPT 2626 /* 2627 * Its EXEC cache is now gone. 2628 */ 2629 if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) 2630 PMAPCOUNT(exec_discarded_page_protect); 2631 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 2632 KASSERT(pg->mdpage.urw_mappings == 0); 2633 KASSERT(pg->mdpage.uro_mappings == 0); 2634 if (pg->mdpage.krw_mappings == 0) 2635 pg->mdpage.pvh_attrs &= ~PVF_WRITE; 2636 KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE)); 2637 #endif 2638 simple_unlock(&pg->mdpage.pvh_slock); 2639 PMAP_HEAD_TO_MAP_UNLOCK(); 2640 2641 if (flush) { 2642 /* 2643 * Note: We can't use pmap_tlb_flush{I,}D() here since that 2644 * would need a subsequent call to pmap_update() to ensure 2645 * curpm->pm_cstate.cs_all is reset. Our callers are not 2646 * required to do that (see pmap(9)), so we can't modify 2647 * the current pmap's state. 2648 */ 2649 if (PV_BEEN_EXECD(flags)) 2650 cpu_tlb_flushID(); 2651 else 2652 cpu_tlb_flushD(); 2653 } 2654 cpu_cpwait(); 2655 } 2656 2657 /* 2658 * pmap_t pmap_create(void) 2659 * 2660 * Create a new pmap structure from scratch. 2661 */ 2662 pmap_t 2663 pmap_create(void) 2664 { 2665 pmap_t pm; 2666 2667 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 2668 2669 UVM_OBJ_INIT(&pm->pm_obj, NULL, 1); 2670 pm->pm_stats.wired_count = 0; 2671 pm->pm_stats.resident_count = 1; 2672 pm->pm_cstate.cs_all = 0; 2673 pmap_alloc_l1(pm); 2674 2675 /* 2676 * Note: The pool cache ensures that the pm_l2[] array is already 2677 * initialised to zero. 2678 */ 2679 2680 pmap_pinit(pm); 2681 2682 LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list); 2683 2684 return (pm); 2685 } 2686 2687 /* 2688 * void pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 2689 * int flags) 2690 * 2691 * Insert the given physical page (p) at 2692 * the specified virtual address (v) in the 2693 * target physical map with the protection requested. 2694 * 2695 * NB: This is the only routine which MAY NOT lazy-evaluate 2696 * or lose information. That is, this routine must actually 2697 * insert this page into the given map NOW. 2698 */ 2699 int 2700 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 2701 { 2702 struct l2_bucket *l2b; 2703 struct vm_page *pg, *opg; 2704 struct pv_entry *pve; 2705 pt_entry_t *ptep, npte, opte; 2706 u_int nflags; 2707 u_int oflags; 2708 2709 NPDEBUG(PDB_ENTER, printf("pmap_enter: pm %p va 0x%lx pa 0x%lx prot %x flag %x\n", pm, va, pa, prot, flags)); 2710 2711 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2712 KDASSERT(((va | pa) & PGOFSET) == 0); 2713 2714 /* 2715 * Get a pointer to the page. Later on in this function, we 2716 * test for a managed page by checking pg != NULL. 2717 */ 2718 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2719 2720 nflags = 0; 2721 if (prot & VM_PROT_WRITE) 2722 nflags |= PVF_WRITE; 2723 if (prot & VM_PROT_EXECUTE) 2724 nflags |= PVF_EXEC; 2725 if (flags & PMAP_WIRED) 2726 nflags |= PVF_WIRED; 2727 2728 PMAP_MAP_TO_HEAD_LOCK(); 2729 pmap_acquire_pmap_lock(pm); 2730 2731 /* 2732 * Fetch the L2 bucket which maps this page, allocating one if 2733 * necessary for user pmaps. 2734 */ 2735 if (pm == pmap_kernel()) 2736 l2b = pmap_get_l2_bucket(pm, va); 2737 else 2738 l2b = pmap_alloc_l2_bucket(pm, va); 2739 if (l2b == NULL) { 2740 if (flags & PMAP_CANFAIL) { 2741 pmap_release_pmap_lock(pm); 2742 PMAP_MAP_TO_HEAD_UNLOCK(); 2743 return (ENOMEM); 2744 } 2745 panic("pmap_enter: failed to allocate L2 bucket"); 2746 } 2747 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2748 opte = *ptep; 2749 npte = pa; 2750 oflags = 0; 2751 2752 if (opte) { 2753 /* 2754 * There is already a mapping at this address. 2755 * If the physical address is different, lookup the 2756 * vm_page. 2757 */ 2758 if (l2pte_pa(opte) != pa) 2759 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 2760 else 2761 opg = pg; 2762 } else 2763 opg = NULL; 2764 2765 if (pg) { 2766 /* 2767 * This is to be a managed mapping. 2768 */ 2769 if ((flags & VM_PROT_ALL) || 2770 (pg->mdpage.pvh_attrs & PVF_REF)) { 2771 /* 2772 * - The access type indicates that we don't need 2773 * to do referenced emulation. 2774 * OR 2775 * - The physical page has already been referenced 2776 * so no need to re-do referenced emulation here. 2777 */ 2778 npte |= L2_S_PROTO; 2779 2780 nflags |= PVF_REF; 2781 2782 if ((prot & VM_PROT_WRITE) != 0 && 2783 ((flags & VM_PROT_WRITE) != 0 || 2784 (pg->mdpage.pvh_attrs & PVF_MOD) != 0)) { 2785 /* 2786 * This is a writable mapping, and the 2787 * page's mod state indicates it has 2788 * already been modified. Make it 2789 * writable from the outset. 2790 */ 2791 npte |= L2_S_PROT_W; 2792 nflags |= PVF_MOD; 2793 } 2794 } else { 2795 /* 2796 * Need to do page referenced emulation. 2797 */ 2798 npte |= L2_TYPE_INV; 2799 } 2800 2801 npte |= pte_l2_s_cache_mode; 2802 2803 if (pg == opg) { 2804 /* 2805 * We're changing the attrs of an existing mapping. 2806 */ 2807 simple_lock(&pg->mdpage.pvh_slock); 2808 oflags = pmap_modify_pv(pg, pm, va, 2809 PVF_WRITE | PVF_EXEC | PVF_WIRED | 2810 PVF_MOD | PVF_REF, nflags); 2811 simple_unlock(&pg->mdpage.pvh_slock); 2812 2813 #ifdef PMAP_CACHE_VIVT 2814 /* 2815 * We may need to flush the cache if we're 2816 * doing rw-ro... 2817 */ 2818 if (pm->pm_cstate.cs_cache_d && 2819 (oflags & PVF_NC) == 0 && 2820 (opte & L2_S_PROT_W) != 0 && 2821 (prot & VM_PROT_WRITE) == 0) 2822 cpu_dcache_wb_range(va, PAGE_SIZE); 2823 #endif 2824 } else { 2825 /* 2826 * New mapping, or changing the backing page 2827 * of an existing mapping. 2828 */ 2829 if (opg) { 2830 /* 2831 * Replacing an existing mapping with a new one. 2832 * It is part of our managed memory so we 2833 * must remove it from the PV list 2834 */ 2835 simple_lock(&opg->mdpage.pvh_slock); 2836 pve = pmap_remove_pv(opg, pm, va, 0); 2837 pmap_vac_me_harder(opg, pm, 0); 2838 simple_unlock(&opg->mdpage.pvh_slock); 2839 oflags = pve->pv_flags; 2840 2841 #ifdef PMAP_CACHE_VIVT 2842 /* 2843 * If the old mapping was valid (ref/mod 2844 * emulation creates 'invalid' mappings 2845 * initially) then make sure to frob 2846 * the cache. 2847 */ 2848 if ((oflags & PVF_NC) == 0 && 2849 l2pte_valid(opte)) { 2850 if (PV_BEEN_EXECD(oflags)) { 2851 pmap_idcache_wbinv_range(pm, va, 2852 PAGE_SIZE); 2853 } else 2854 if (PV_BEEN_REFD(oflags)) { 2855 pmap_dcache_wb_range(pm, va, 2856 PAGE_SIZE, true, 2857 (oflags & PVF_WRITE) == 0); 2858 } 2859 } 2860 #endif 2861 } else 2862 if ((pve = pool_get(&pmap_pv_pool, PR_NOWAIT)) == NULL){ 2863 if ((flags & PMAP_CANFAIL) == 0) 2864 panic("pmap_enter: no pv entries"); 2865 2866 if (pm != pmap_kernel()) 2867 pmap_free_l2_bucket(pm, l2b, 0); 2868 pmap_release_pmap_lock(pm); 2869 PMAP_MAP_TO_HEAD_UNLOCK(); 2870 NPDEBUG(PDB_ENTER, 2871 printf("pmap_enter: ENOMEM\n")); 2872 return (ENOMEM); 2873 } 2874 2875 pmap_enter_pv(pg, pve, pm, va, nflags); 2876 } 2877 } else { 2878 /* 2879 * We're mapping an unmanaged page. 2880 * These are always readable, and possibly writable, from 2881 * the get go as we don't need to track ref/mod status. 2882 */ 2883 npte |= L2_S_PROTO; 2884 if (prot & VM_PROT_WRITE) 2885 npte |= L2_S_PROT_W; 2886 2887 /* 2888 * Make sure the vector table is mapped cacheable 2889 */ 2890 if (pm != pmap_kernel() && va == vector_page) 2891 npte |= pte_l2_s_cache_mode; 2892 2893 if (opg) { 2894 /* 2895 * Looks like there's an existing 'managed' mapping 2896 * at this address. 2897 */ 2898 simple_lock(&opg->mdpage.pvh_slock); 2899 pve = pmap_remove_pv(opg, pm, va, 0); 2900 pmap_vac_me_harder(opg, pm, 0); 2901 simple_unlock(&opg->mdpage.pvh_slock); 2902 oflags = pve->pv_flags; 2903 2904 #ifdef PMAP_CACHE_VIVT 2905 if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { 2906 if (PV_BEEN_EXECD(oflags)) 2907 pmap_idcache_wbinv_range(pm, va, 2908 PAGE_SIZE); 2909 else 2910 if (PV_BEEN_REFD(oflags)) 2911 pmap_dcache_wb_range(pm, va, PAGE_SIZE, 2912 true, (oflags & PVF_WRITE) == 0); 2913 } 2914 #endif 2915 pool_put(&pmap_pv_pool, pve); 2916 } 2917 } 2918 2919 /* 2920 * Make sure userland mappings get the right permissions 2921 */ 2922 if (pm != pmap_kernel() && va != vector_page) 2923 npte |= L2_S_PROT_U; 2924 2925 /* 2926 * Keep the stats up to date 2927 */ 2928 if (opte == 0) { 2929 l2b->l2b_occupancy++; 2930 pm->pm_stats.resident_count++; 2931 } 2932 2933 NPDEBUG(PDB_ENTER, 2934 printf("pmap_enter: opte 0x%08x npte 0x%08x\n", opte, npte)); 2935 2936 /* 2937 * If this is just a wiring change, the two PTEs will be 2938 * identical, so there's no need to update the page table. 2939 */ 2940 if (npte != opte) { 2941 bool is_cached = pmap_is_cached(pm); 2942 2943 *ptep = npte; 2944 if (is_cached) { 2945 /* 2946 * We only need to frob the cache/tlb if this pmap 2947 * is current 2948 */ 2949 PTE_SYNC(ptep); 2950 if (va != vector_page && l2pte_valid(npte)) { 2951 /* 2952 * This mapping is likely to be accessed as 2953 * soon as we return to userland. Fix up the 2954 * L1 entry to avoid taking another 2955 * page/domain fault. 2956 */ 2957 pd_entry_t *pl1pd, l1pd; 2958 2959 pl1pd = &pm->pm_l1->l1_kva[L1_IDX(va)]; 2960 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | 2961 L1_C_PROTO; 2962 if (*pl1pd != l1pd) { 2963 *pl1pd = l1pd; 2964 PTE_SYNC(pl1pd); 2965 } 2966 } 2967 } 2968 2969 if (PV_BEEN_EXECD(oflags)) 2970 pmap_tlb_flushID_SE(pm, va); 2971 else 2972 if (PV_BEEN_REFD(oflags)) 2973 pmap_tlb_flushD_SE(pm, va); 2974 2975 NPDEBUG(PDB_ENTER, 2976 printf("pmap_enter: is_cached %d cs 0x%08x\n", 2977 is_cached, pm->pm_cstate.cs_all)); 2978 2979 if (pg != NULL) { 2980 simple_lock(&pg->mdpage.pvh_slock); 2981 pmap_vac_me_harder(pg, pm, va); 2982 simple_unlock(&pg->mdpage.pvh_slock); 2983 } 2984 } 2985 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 2986 if (pg) { 2987 simple_lock(&pg->mdpage.pvh_slock); 2988 KASSERT((pg->mdpage.pvh_attrs & PVF_DMOD) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 2989 KASSERT(((pg->mdpage.pvh_attrs & PVF_WRITE) == 0) == (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0)); 2990 simple_unlock(&pg->mdpage.pvh_slock); 2991 } 2992 #endif 2993 2994 pmap_release_pmap_lock(pm); 2995 PMAP_MAP_TO_HEAD_UNLOCK(); 2996 2997 return (0); 2998 } 2999 3000 /* 3001 * pmap_remove() 3002 * 3003 * pmap_remove is responsible for nuking a number of mappings for a range 3004 * of virtual address space in the current pmap. To do this efficiently 3005 * is interesting, because in a number of cases a wide virtual address 3006 * range may be supplied that contains few actual mappings. So, the 3007 * optimisations are: 3008 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3009 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3010 * maybe do just a partial cache clean. This path of execution is 3011 * complicated by the fact that the cache must be flushed _before_ 3012 * the PTE is nuked, being a VAC :-) 3013 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3014 * all invalidations until pmap_update(), since pmap_remove_all() has 3015 * already flushed the cache. 3016 * 4. Maybe later fast-case a single page, but I don't think this is 3017 * going to make _that_ much difference overall. 3018 */ 3019 3020 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3021 3022 void 3023 pmap_do_remove(pmap_t pm, vaddr_t sva, vaddr_t eva, int skip_wired) 3024 { 3025 struct l2_bucket *l2b; 3026 vaddr_t next_bucket; 3027 pt_entry_t *ptep; 3028 u_int cleanlist_idx, total, cnt; 3029 struct { 3030 vaddr_t va; 3031 pt_entry_t *ptep; 3032 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3033 u_int mappings, is_exec, is_refd; 3034 3035 NPDEBUG(PDB_REMOVE, printf("pmap_do_remove: pmap=%p sva=%08lx " 3036 "eva=%08lx\n", pm, sva, eva)); 3037 3038 /* 3039 * we lock in the pmap => pv_head direction 3040 */ 3041 PMAP_MAP_TO_HEAD_LOCK(); 3042 pmap_acquire_pmap_lock(pm); 3043 3044 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3045 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3046 if (pm->pm_cstate.cs_tlb == 0) 3047 pm->pm_remove_all = true; 3048 } else 3049 cleanlist_idx = 0; 3050 3051 total = 0; 3052 3053 while (sva < eva) { 3054 /* 3055 * Do one L2 bucket's worth at a time. 3056 */ 3057 next_bucket = L2_NEXT_BUCKET(sva); 3058 if (next_bucket > eva) 3059 next_bucket = eva; 3060 3061 l2b = pmap_get_l2_bucket(pm, sva); 3062 if (l2b == NULL) { 3063 sva = next_bucket; 3064 continue; 3065 } 3066 3067 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3068 3069 for (mappings = 0; sva < next_bucket; sva += PAGE_SIZE, ptep++){ 3070 struct vm_page *pg; 3071 pt_entry_t pte; 3072 paddr_t pa; 3073 3074 pte = *ptep; 3075 3076 if (pte == 0) { 3077 /* Nothing here, move along */ 3078 continue; 3079 } 3080 3081 pa = l2pte_pa(pte); 3082 is_exec = 0; 3083 is_refd = 1; 3084 3085 /* 3086 * Update flags. In a number of circumstances, 3087 * we could cluster a lot of these and do a 3088 * number of sequential pages in one go. 3089 */ 3090 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3091 struct pv_entry *pve; 3092 simple_lock(&pg->mdpage.pvh_slock); 3093 pve = pmap_remove_pv(pg, pm, sva, skip_wired); 3094 pmap_vac_me_harder(pg, pm, 0); 3095 simple_unlock(&pg->mdpage.pvh_slock); 3096 if (pve != NULL) { 3097 if (pm->pm_remove_all == false) { 3098 is_exec = 3099 PV_BEEN_EXECD(pve->pv_flags); 3100 is_refd = 3101 PV_BEEN_REFD(pve->pv_flags); 3102 } 3103 pool_put(&pmap_pv_pool, pve); 3104 } else 3105 if (skip_wired) { 3106 /* The mapping is wired. Skip it */ 3107 continue; 3108 } 3109 } else 3110 if (skip_wired) { 3111 /* Unmanaged pages are always wired. */ 3112 continue; 3113 } 3114 3115 mappings++; 3116 3117 if (!l2pte_valid(pte)) { 3118 /* 3119 * Ref/Mod emulation is still active for this 3120 * mapping, therefore it is has not yet been 3121 * accessed. No need to frob the cache/tlb. 3122 */ 3123 *ptep = 0; 3124 PTE_SYNC_CURRENT(pm, ptep); 3125 continue; 3126 } 3127 3128 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3129 /* Add to the clean list. */ 3130 cleanlist[cleanlist_idx].ptep = ptep; 3131 cleanlist[cleanlist_idx].va = 3132 sva | (is_exec & 1); 3133 cleanlist_idx++; 3134 } else 3135 if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3136 /* Nuke everything if needed. */ 3137 #ifdef PMAP_CACHE_VIVT 3138 pmap_idcache_wbinv_all(pm); 3139 #endif 3140 pmap_tlb_flushID(pm); 3141 3142 /* 3143 * Roll back the previous PTE list, 3144 * and zero out the current PTE. 3145 */ 3146 for (cnt = 0; 3147 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3148 *cleanlist[cnt].ptep = 0; 3149 PTE_SYNC(cleanlist[cnt].ptep); 3150 } 3151 *ptep = 0; 3152 PTE_SYNC(ptep); 3153 cleanlist_idx++; 3154 pm->pm_remove_all = true; 3155 } else { 3156 *ptep = 0; 3157 PTE_SYNC(ptep); 3158 if (pm->pm_remove_all == false) { 3159 if (is_exec) 3160 pmap_tlb_flushID_SE(pm, sva); 3161 else 3162 if (is_refd) 3163 pmap_tlb_flushD_SE(pm, sva); 3164 } 3165 } 3166 } 3167 3168 /* 3169 * Deal with any left overs 3170 */ 3171 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3172 total += cleanlist_idx; 3173 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3174 if (pm->pm_cstate.cs_all != 0) { 3175 vaddr_t clva = cleanlist[cnt].va & ~1; 3176 if (cleanlist[cnt].va & 1) { 3177 #ifdef PMAP_CACHE_VIVT 3178 pmap_idcache_wbinv_range(pm, 3179 clva, PAGE_SIZE); 3180 #endif 3181 pmap_tlb_flushID_SE(pm, clva); 3182 } else { 3183 #ifdef PMAP_CACHE_VIVT 3184 pmap_dcache_wb_range(pm, 3185 clva, PAGE_SIZE, true, 3186 false); 3187 #endif 3188 pmap_tlb_flushD_SE(pm, clva); 3189 } 3190 } 3191 *cleanlist[cnt].ptep = 0; 3192 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3193 } 3194 3195 /* 3196 * If it looks like we're removing a whole bunch 3197 * of mappings, it's faster to just write-back 3198 * the whole cache now and defer TLB flushes until 3199 * pmap_update() is called. 3200 */ 3201 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3202 cleanlist_idx = 0; 3203 else { 3204 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3205 #ifdef PMAP_CACHE_VIVT 3206 pmap_idcache_wbinv_all(pm); 3207 #endif 3208 pm->pm_remove_all = true; 3209 } 3210 } 3211 3212 pmap_free_l2_bucket(pm, l2b, mappings); 3213 pm->pm_stats.resident_count -= mappings; 3214 } 3215 3216 pmap_release_pmap_lock(pm); 3217 PMAP_MAP_TO_HEAD_UNLOCK(); 3218 } 3219 3220 #ifdef PMAP_CACHE_VIPT 3221 static struct pv_entry * 3222 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3223 { 3224 struct pv_entry *pv; 3225 3226 simple_lock(&pg->mdpage.pvh_slock); 3227 KASSERT(pg->mdpage.pvh_attrs & (PVF_COLORED|PVF_NC)); 3228 KASSERT((pg->mdpage.pvh_attrs & PVF_KMPAGE) == 0); 3229 3230 pv = pmap_remove_pv(pg, pmap_kernel(), va, false); 3231 KASSERT(pv); 3232 KASSERT(pv->pv_flags & PVF_KENTRY); 3233 3234 /* 3235 * If we are removing a writeable mapping to a cached exec page, 3236 * if it's the last mapping then clear it execness other sync 3237 * the page to the icache. 3238 */ 3239 if ((pg->mdpage.pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3240 && (pv->pv_flags & PVF_WRITE) != 0) { 3241 if (SLIST_EMPTY(&pg->mdpage.pvh_list)) { 3242 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 3243 PMAPCOUNT(exec_discarded_kremove); 3244 } else { 3245 pmap_syncicache_page(pg); 3246 PMAPCOUNT(exec_synced_kremove); 3247 } 3248 } 3249 pmap_vac_me_harder(pg, pmap_kernel(), 0); 3250 simple_unlock(&pg->mdpage.pvh_slock); 3251 3252 return pv; 3253 } 3254 #endif /* PMAP_CACHE_VIPT */ 3255 3256 /* 3257 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3258 * 3259 * We assume there is already sufficient KVM space available 3260 * to do this, as we can't allocate L2 descriptor tables/metadata 3261 * from here. 3262 */ 3263 void 3264 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 3265 { 3266 struct l2_bucket *l2b; 3267 pt_entry_t *ptep, opte; 3268 #ifdef PMAP_CACHE_VIVT 3269 struct vm_page *pg = (prot & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3270 #endif 3271 #ifdef PMAP_CACHE_VIPT 3272 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3273 struct vm_page *opg; 3274 struct pv_entry *pv = NULL; 3275 #endif 3276 3277 NPDEBUG(PDB_KENTER, 3278 printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n", 3279 va, pa, prot)); 3280 3281 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 3282 KDASSERT(l2b != NULL); 3283 3284 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3285 opte = *ptep; 3286 3287 if (opte == 0) { 3288 PMAPCOUNT(kenter_mappings); 3289 l2b->l2b_occupancy++; 3290 } else { 3291 PMAPCOUNT(kenter_remappings); 3292 #ifdef PMAP_CACHE_VIPT 3293 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3294 if (opg) { 3295 KASSERT(opg != pg); 3296 KASSERT((opg->mdpage.pvh_attrs & PVF_KMPAGE) == 0); 3297 KASSERT((prot & PMAP_KMPAGE) == 0); 3298 simple_lock(&opg->mdpage.pvh_slock); 3299 pv = pmap_kremove_pg(opg, va); 3300 simple_unlock(&opg->mdpage.pvh_slock); 3301 } 3302 #endif 3303 if (l2pte_valid(opte)) { 3304 #ifdef PMAP_CACHE_VIVT 3305 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3306 #endif 3307 cpu_tlb_flushD_SE(va); 3308 cpu_cpwait(); 3309 } 3310 } 3311 3312 *ptep = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | 3313 pte_l2_s_cache_mode; 3314 PTE_SYNC(ptep); 3315 3316 if (pg) { 3317 if (prot & PMAP_KMPAGE) { 3318 simple_lock(&pg->mdpage.pvh_slock); 3319 KASSERT(pg->mdpage.urw_mappings == 0); 3320 KASSERT(pg->mdpage.uro_mappings == 0); 3321 KASSERT(pg->mdpage.krw_mappings == 0); 3322 KASSERT(pg->mdpage.kro_mappings == 0); 3323 #ifdef PMAP_CACHE_VIPT 3324 KASSERT(pv == NULL); 3325 KASSERT((va & PVF_COLORED) == 0); 3326 KASSERT((pg->mdpage.pvh_attrs & PVF_NC) == 0); 3327 /* if there is a color conflict, evict from cache. */ 3328 if (pmap_is_page_colored_p(pg) 3329 && ((va ^ pg->mdpage.pvh_attrs) & arm_cache_prefer_mask)) { 3330 PMAPCOUNT(vac_color_change); 3331 pmap_flush_page(pg, true); 3332 } 3333 pg->mdpage.pvh_attrs &= PAGE_SIZE - 1; 3334 pg->mdpage.pvh_attrs |= PVF_KMPAGE 3335 | PVF_COLORED | PVF_DIRTY 3336 | (va & arm_cache_prefer_mask); 3337 #endif 3338 #ifdef PMAP_CACHE_VIVT 3339 pg->mdpage.pvh_attrs |= PVF_KMPAGE; 3340 #endif 3341 pmap_kmpages++; 3342 simple_unlock(&pg->mdpage.pvh_slock); 3343 #ifdef PMAP_CACHE_VIPT 3344 } else { 3345 if (pv == NULL) { 3346 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3347 KASSERT(pv != NULL); 3348 } 3349 pmap_enter_pv(pg, pv, pmap_kernel(), va, 3350 PVF_WIRED | PVF_KENTRY 3351 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3352 if ((prot & VM_PROT_WRITE) 3353 && !(pg->mdpage.pvh_attrs & PVF_NC)) 3354 pg->mdpage.pvh_attrs |= PVF_DIRTY; 3355 KASSERT((prot & VM_PROT_WRITE) == 0 || (pg->mdpage.pvh_attrs & (PVF_DIRTY|PVF_NC))); 3356 simple_lock(&pg->mdpage.pvh_slock); 3357 pmap_vac_me_harder(pg, pmap_kernel(), va); 3358 simple_unlock(&pg->mdpage.pvh_slock); 3359 #endif 3360 } 3361 #ifdef PMAP_CACHE_VIPT 3362 } else { 3363 if (pv != NULL) 3364 pool_put(&pmap_pv_pool, pv); 3365 #endif 3366 } 3367 } 3368 3369 void 3370 pmap_kremove(vaddr_t va, vsize_t len) 3371 { 3372 struct l2_bucket *l2b; 3373 pt_entry_t *ptep, *sptep, opte; 3374 vaddr_t next_bucket, eva; 3375 u_int mappings; 3376 struct vm_page *opg; 3377 3378 PMAPCOUNT(kenter_unmappings); 3379 3380 NPDEBUG(PDB_KREMOVE, printf("pmap_kremove: va 0x%08lx, len 0x%08lx\n", 3381 va, len)); 3382 3383 eva = va + len; 3384 3385 while (va < eva) { 3386 next_bucket = L2_NEXT_BUCKET(va); 3387 if (next_bucket > eva) 3388 next_bucket = eva; 3389 3390 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 3391 KDASSERT(l2b != NULL); 3392 3393 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 3394 mappings = 0; 3395 3396 while (va < next_bucket) { 3397 opte = *ptep; 3398 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3399 if (opg) { 3400 if (opg->mdpage.pvh_attrs & PVF_KMPAGE) { 3401 simple_lock(&opg->mdpage.pvh_slock); 3402 KASSERT(opg->mdpage.urw_mappings == 0); 3403 KASSERT(opg->mdpage.uro_mappings == 0); 3404 KASSERT(opg->mdpage.krw_mappings == 0); 3405 KASSERT(opg->mdpage.kro_mappings == 0); 3406 opg->mdpage.pvh_attrs &= ~PVF_KMPAGE; 3407 #ifdef PMAP_CACHE_VIPT 3408 opg->mdpage.pvh_attrs &= ~PVF_WRITE; 3409 #endif 3410 pmap_kmpages--; 3411 simple_unlock(&opg->mdpage.pvh_slock); 3412 #ifdef PMAP_CACHE_VIPT 3413 } else { 3414 pool_put(&pmap_pv_pool, 3415 pmap_kremove_pg(opg, va)); 3416 #endif 3417 } 3418 } 3419 if (l2pte_valid(opte)) { 3420 #ifdef PMAP_CACHE_VIVT 3421 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3422 #endif 3423 cpu_tlb_flushD_SE(va); 3424 } 3425 if (opte) { 3426 *ptep = 0; 3427 mappings++; 3428 } 3429 va += PAGE_SIZE; 3430 ptep++; 3431 } 3432 KDASSERT(mappings <= l2b->l2b_occupancy); 3433 l2b->l2b_occupancy -= mappings; 3434 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3435 } 3436 cpu_cpwait(); 3437 } 3438 3439 bool 3440 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3441 { 3442 struct l2_dtable *l2; 3443 pd_entry_t *pl1pd, l1pd; 3444 pt_entry_t *ptep, pte; 3445 paddr_t pa; 3446 u_int l1idx; 3447 3448 pmap_acquire_pmap_lock(pm); 3449 3450 l1idx = L1_IDX(va); 3451 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3452 l1pd = *pl1pd; 3453 3454 if (l1pte_section_p(l1pd)) { 3455 /* 3456 * These should only happen for pmap_kernel() 3457 */ 3458 KDASSERT(pm == pmap_kernel()); 3459 pmap_release_pmap_lock(pm); 3460 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3461 } else { 3462 /* 3463 * Note that we can't rely on the validity of the L1 3464 * descriptor as an indication that a mapping exists. 3465 * We have to look it up in the L2 dtable. 3466 */ 3467 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3468 3469 if (l2 == NULL || 3470 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3471 pmap_release_pmap_lock(pm); 3472 return false; 3473 } 3474 3475 ptep = &ptep[l2pte_index(va)]; 3476 pte = *ptep; 3477 pmap_release_pmap_lock(pm); 3478 3479 if (pte == 0) 3480 return false; 3481 3482 switch (pte & L2_TYPE_MASK) { 3483 case L2_TYPE_L: 3484 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3485 break; 3486 3487 default: 3488 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3489 break; 3490 } 3491 } 3492 3493 if (pap != NULL) 3494 *pap = pa; 3495 3496 return true; 3497 } 3498 3499 void 3500 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 3501 { 3502 struct l2_bucket *l2b; 3503 pt_entry_t *ptep, pte; 3504 vaddr_t next_bucket; 3505 u_int flags; 3506 u_int clr_mask; 3507 int flush; 3508 3509 NPDEBUG(PDB_PROTECT, 3510 printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n", 3511 pm, sva, eva, prot)); 3512 3513 if ((prot & VM_PROT_READ) == 0) { 3514 pmap_remove(pm, sva, eva); 3515 return; 3516 } 3517 3518 if (prot & VM_PROT_WRITE) { 3519 /* 3520 * If this is a read->write transition, just ignore it and let 3521 * uvm_fault() take care of it later. 3522 */ 3523 return; 3524 } 3525 3526 PMAP_MAP_TO_HEAD_LOCK(); 3527 pmap_acquire_pmap_lock(pm); 3528 3529 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 3530 flags = 0; 3531 clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 3532 3533 while (sva < eva) { 3534 next_bucket = L2_NEXT_BUCKET(sva); 3535 if (next_bucket > eva) 3536 next_bucket = eva; 3537 3538 l2b = pmap_get_l2_bucket(pm, sva); 3539 if (l2b == NULL) { 3540 sva = next_bucket; 3541 continue; 3542 } 3543 3544 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3545 3546 while (sva < next_bucket) { 3547 pte = *ptep; 3548 if (l2pte_valid(pte) != 0 && (pte & L2_S_PROT_W) != 0) { 3549 struct vm_page *pg; 3550 u_int f; 3551 3552 #ifdef PMAP_CACHE_VIVT 3553 /* 3554 * OK, at this point, we know we're doing 3555 * write-protect operation. If the pmap is 3556 * active, write-back the page. 3557 */ 3558 pmap_dcache_wb_range(pm, sva, PAGE_SIZE, 3559 false, false); 3560 #endif 3561 3562 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3563 pte &= ~L2_S_PROT_W; 3564 *ptep = pte; 3565 PTE_SYNC(ptep); 3566 3567 if (pg != NULL) { 3568 simple_lock(&pg->mdpage.pvh_slock); 3569 f = pmap_modify_pv(pg, pm, sva, 3570 clr_mask, 0); 3571 pmap_vac_me_harder(pg, pm, sva); 3572 simple_unlock(&pg->mdpage.pvh_slock); 3573 } else 3574 f = PVF_REF | PVF_EXEC; 3575 3576 if (flush >= 0) { 3577 flush++; 3578 flags |= f; 3579 } else 3580 if (PV_BEEN_EXECD(f)) 3581 pmap_tlb_flushID_SE(pm, sva); 3582 else 3583 if (PV_BEEN_REFD(f)) 3584 pmap_tlb_flushD_SE(pm, sva); 3585 } 3586 3587 sva += PAGE_SIZE; 3588 ptep++; 3589 } 3590 } 3591 3592 pmap_release_pmap_lock(pm); 3593 PMAP_MAP_TO_HEAD_UNLOCK(); 3594 3595 if (flush) { 3596 if (PV_BEEN_EXECD(flags)) 3597 pmap_tlb_flushID(pm); 3598 else 3599 if (PV_BEEN_REFD(flags)) 3600 pmap_tlb_flushD(pm); 3601 } 3602 } 3603 3604 void 3605 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 3606 { 3607 struct l2_bucket *l2b; 3608 pt_entry_t *ptep; 3609 vaddr_t next_bucket; 3610 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 3611 3612 NPDEBUG(PDB_EXEC, 3613 printf("pmap_icache_sync_range: pm %p sva 0x%lx eva 0x%lx\n", 3614 pm, sva, eva)); 3615 3616 PMAP_MAP_TO_HEAD_LOCK(); 3617 pmap_acquire_pmap_lock(pm); 3618 3619 while (sva < eva) { 3620 next_bucket = L2_NEXT_BUCKET(sva); 3621 if (next_bucket > eva) 3622 next_bucket = eva; 3623 3624 l2b = pmap_get_l2_bucket(pm, sva); 3625 if (l2b == NULL) { 3626 sva = next_bucket; 3627 continue; 3628 } 3629 3630 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3631 sva < next_bucket; 3632 sva += page_size, ptep++, page_size = PAGE_SIZE) { 3633 if (l2pte_valid(*ptep)) { 3634 cpu_icache_sync_range(sva, 3635 min(page_size, eva - sva)); 3636 } 3637 } 3638 } 3639 3640 pmap_release_pmap_lock(pm); 3641 PMAP_MAP_TO_HEAD_UNLOCK(); 3642 } 3643 3644 void 3645 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 3646 { 3647 3648 NPDEBUG(PDB_PROTECT, 3649 printf("pmap_page_protect: pg %p (0x%08lx), prot 0x%x\n", 3650 pg, VM_PAGE_TO_PHYS(pg), prot)); 3651 3652 switch(prot) { 3653 return; 3654 case VM_PROT_READ|VM_PROT_WRITE: 3655 #if defined(PMAP_CHECK_VIPT) && defined(PMAP_APX) 3656 pmap_clearbit(pg, PVF_EXEC); 3657 break; 3658 #endif 3659 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 3660 break; 3661 3662 case VM_PROT_READ: 3663 #if defined(PMAP_CHECK_VIPT) && defined(PMAP_APX) 3664 pmap_clearbit(pg, PVF_WRITE|PVF_EXEC); 3665 break; 3666 #endif 3667 case VM_PROT_READ|VM_PROT_EXECUTE: 3668 pmap_clearbit(pg, PVF_WRITE); 3669 break; 3670 3671 default: 3672 pmap_page_remove(pg); 3673 break; 3674 } 3675 } 3676 3677 /* 3678 * pmap_clear_modify: 3679 * 3680 * Clear the "modified" attribute for a page. 3681 */ 3682 bool 3683 pmap_clear_modify(struct vm_page *pg) 3684 { 3685 bool rv; 3686 3687 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3688 rv = true; 3689 pmap_clearbit(pg, PVF_MOD); 3690 } else 3691 rv = false; 3692 3693 return (rv); 3694 } 3695 3696 /* 3697 * pmap_clear_reference: 3698 * 3699 * Clear the "referenced" attribute for a page. 3700 */ 3701 bool 3702 pmap_clear_reference(struct vm_page *pg) 3703 { 3704 bool rv; 3705 3706 if (pg->mdpage.pvh_attrs & PVF_REF) { 3707 rv = true; 3708 pmap_clearbit(pg, PVF_REF); 3709 } else 3710 rv = false; 3711 3712 return (rv); 3713 } 3714 3715 /* 3716 * pmap_is_modified: 3717 * 3718 * Test if a page has the "modified" attribute. 3719 */ 3720 /* See <arm/arm32/pmap.h> */ 3721 3722 /* 3723 * pmap_is_referenced: 3724 * 3725 * Test if a page has the "referenced" attribute. 3726 */ 3727 /* See <arm/arm32/pmap.h> */ 3728 3729 int 3730 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 3731 { 3732 struct l2_dtable *l2; 3733 struct l2_bucket *l2b; 3734 pd_entry_t *pl1pd, l1pd; 3735 pt_entry_t *ptep, pte; 3736 paddr_t pa; 3737 u_int l1idx; 3738 int rv = 0; 3739 3740 PMAP_MAP_TO_HEAD_LOCK(); 3741 pmap_acquire_pmap_lock(pm); 3742 3743 l1idx = L1_IDX(va); 3744 3745 /* 3746 * If there is no l2_dtable for this address, then the process 3747 * has no business accessing it. 3748 * 3749 * Note: This will catch userland processes trying to access 3750 * kernel addresses. 3751 */ 3752 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3753 if (l2 == NULL) 3754 goto out; 3755 3756 /* 3757 * Likewise if there is no L2 descriptor table 3758 */ 3759 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 3760 if (l2b->l2b_kva == NULL) 3761 goto out; 3762 3763 /* 3764 * Check the PTE itself. 3765 */ 3766 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3767 pte = *ptep; 3768 if (pte == 0) 3769 goto out; 3770 3771 /* 3772 * Catch a userland access to the vector page mapped at 0x0 3773 */ 3774 if (user && (pte & L2_S_PROT_U) == 0) 3775 goto out; 3776 3777 pa = l2pte_pa(pte); 3778 3779 if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { 3780 /* 3781 * This looks like a good candidate for "page modified" 3782 * emulation... 3783 */ 3784 struct pv_entry *pv; 3785 struct vm_page *pg; 3786 3787 /* Extract the physical address of the page */ 3788 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3789 goto out; 3790 3791 /* Get the current flags for this page. */ 3792 simple_lock(&pg->mdpage.pvh_slock); 3793 3794 pv = pmap_find_pv(pg, pm, va); 3795 if (pv == NULL) { 3796 simple_unlock(&pg->mdpage.pvh_slock); 3797 goto out; 3798 } 3799 3800 /* 3801 * Do the flags say this page is writable? If not then it 3802 * is a genuine write fault. If yes then the write fault is 3803 * our fault as we did not reflect the write access in the 3804 * PTE. Now we know a write has occurred we can correct this 3805 * and also set the modified bit 3806 */ 3807 if ((pv->pv_flags & PVF_WRITE) == 0) { 3808 simple_unlock(&pg->mdpage.pvh_slock); 3809 goto out; 3810 } 3811 3812 NPDEBUG(PDB_FOLLOW, 3813 printf("pmap_fault_fixup: mod emul. pm %p, va 0x%08lx, pa 0x%08lx\n", 3814 pm, va, VM_PAGE_TO_PHYS(pg))); 3815 3816 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3817 pv->pv_flags |= PVF_REF | PVF_MOD; 3818 #ifdef PMAP_CACHE_VIPT 3819 /* 3820 * If there are cacheable mappings for this page, mark it dirty. 3821 */ 3822 if ((pg->mdpage.pvh_attrs & PVF_NC) == 0) 3823 pg->mdpage.pvh_attrs |= PVF_DIRTY; 3824 #endif 3825 simple_unlock(&pg->mdpage.pvh_slock); 3826 3827 /* 3828 * Re-enable write permissions for the page. No need to call 3829 * pmap_vac_me_harder(), since this is just a 3830 * modified-emulation fault, and the PVF_WRITE bit isn't 3831 * changing. We've already set the cacheable bits based on 3832 * the assumption that we can write to this page. 3833 */ 3834 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3835 PTE_SYNC(ptep); 3836 rv = 1; 3837 } else 3838 if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 3839 /* 3840 * This looks like a good candidate for "page referenced" 3841 * emulation. 3842 */ 3843 struct pv_entry *pv; 3844 struct vm_page *pg; 3845 3846 /* Extract the physical address of the page */ 3847 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3848 goto out; 3849 3850 /* Get the current flags for this page. */ 3851 simple_lock(&pg->mdpage.pvh_slock); 3852 3853 pv = pmap_find_pv(pg, pm, va); 3854 if (pv == NULL) { 3855 simple_unlock(&pg->mdpage.pvh_slock); 3856 goto out; 3857 } 3858 3859 pg->mdpage.pvh_attrs |= PVF_REF; 3860 pv->pv_flags |= PVF_REF; 3861 simple_unlock(&pg->mdpage.pvh_slock); 3862 3863 NPDEBUG(PDB_FOLLOW, 3864 printf("pmap_fault_fixup: ref emul. pm %p, va 0x%08lx, pa 0x%08lx\n", 3865 pm, va, VM_PAGE_TO_PHYS(pg))); 3866 3867 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 3868 PTE_SYNC(ptep); 3869 rv = 1; 3870 } 3871 3872 /* 3873 * We know there is a valid mapping here, so simply 3874 * fix up the L1 if necessary. 3875 */ 3876 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3877 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 3878 if (*pl1pd != l1pd) { 3879 *pl1pd = l1pd; 3880 PTE_SYNC(pl1pd); 3881 rv = 1; 3882 } 3883 3884 #ifdef CPU_SA110 3885 /* 3886 * There are bugs in the rev K SA110. This is a check for one 3887 * of them. 3888 */ 3889 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 3890 curcpu()->ci_arm_cpurev < 3) { 3891 /* Always current pmap */ 3892 if (l2pte_valid(pte)) { 3893 extern int kernel_debug; 3894 if (kernel_debug & 1) { 3895 struct proc *p = curlwp->l_proc; 3896 printf("prefetch_abort: page is already " 3897 "mapped - pte=%p *pte=%08x\n", ptep, pte); 3898 printf("prefetch_abort: pc=%08lx proc=%p " 3899 "process=%s\n", va, p, p->p_comm); 3900 printf("prefetch_abort: far=%08x fs=%x\n", 3901 cpu_faultaddress(), cpu_faultstatus()); 3902 } 3903 #ifdef DDB 3904 if (kernel_debug & 2) 3905 Debugger(); 3906 #endif 3907 rv = 1; 3908 } 3909 } 3910 #endif /* CPU_SA110 */ 3911 3912 #ifdef DEBUG 3913 /* 3914 * If 'rv == 0' at this point, it generally indicates that there is a 3915 * stale TLB entry for the faulting address. This happens when two or 3916 * more processes are sharing an L1. Since we don't flush the TLB on 3917 * a context switch between such processes, we can take domain faults 3918 * for mappings which exist at the same VA in both processes. EVEN IF 3919 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 3920 * example. 3921 * 3922 * This is extremely likely to happen if pmap_enter() updated the L1 3923 * entry for a recently entered mapping. In this case, the TLB is 3924 * flushed for the new mapping, but there may still be TLB entries for 3925 * other mappings belonging to other processes in the 1MB range 3926 * covered by the L1 entry. 3927 * 3928 * Since 'rv == 0', we know that the L1 already contains the correct 3929 * value, so the fault must be due to a stale TLB entry. 3930 * 3931 * Since we always need to flush the TLB anyway in the case where we 3932 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 3933 * stale TLB entries dynamically. 3934 * 3935 * However, the above condition can ONLY happen if the current L1 is 3936 * being shared. If it happens when the L1 is unshared, it indicates 3937 * that other parts of the pmap are not doing their job WRT managing 3938 * the TLB. 3939 */ 3940 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 3941 extern int last_fault_code; 3942 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 3943 pm, va, ftype); 3944 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 3945 l2, l2b, ptep, pl1pd); 3946 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 3947 pte, l1pd, last_fault_code); 3948 #ifdef DDB 3949 Debugger(); 3950 #endif 3951 } 3952 #endif 3953 3954 cpu_tlb_flushID_SE(va); 3955 cpu_cpwait(); 3956 3957 rv = 1; 3958 3959 out: 3960 pmap_release_pmap_lock(pm); 3961 PMAP_MAP_TO_HEAD_UNLOCK(); 3962 3963 return (rv); 3964 } 3965 3966 /* 3967 * pmap_collect: free resources held by a pmap 3968 * 3969 * => optional function. 3970 * => called when a process is swapped out to free memory. 3971 */ 3972 void 3973 pmap_collect(pmap_t pm) 3974 { 3975 3976 #ifdef PMAP_CACHE_VIVT 3977 pmap_idcache_wbinv_all(pm); 3978 #endif 3979 pm->pm_remove_all = true; 3980 pmap_do_remove(pm, VM_MIN_ADDRESS, VM_MAX_ADDRESS, 1); 3981 pmap_update(pm); 3982 PMAPCOUNT(collects); 3983 } 3984 3985 /* 3986 * Routine: pmap_procwr 3987 * 3988 * Function: 3989 * Synchronize caches corresponding to [addr, addr+len) in p. 3990 * 3991 */ 3992 void 3993 pmap_procwr(struct proc *p, vaddr_t va, int len) 3994 { 3995 /* We only need to do anything if it is the current process. */ 3996 if (p == curproc) 3997 cpu_icache_sync_range(va, len); 3998 } 3999 4000 /* 4001 * Routine: pmap_unwire 4002 * Function: Clear the wired attribute for a map/virtual-address pair. 4003 * 4004 * In/out conditions: 4005 * The mapping must already exist in the pmap. 4006 */ 4007 void 4008 pmap_unwire(pmap_t pm, vaddr_t va) 4009 { 4010 struct l2_bucket *l2b; 4011 pt_entry_t *ptep, pte; 4012 struct vm_page *pg; 4013 paddr_t pa; 4014 4015 NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va)); 4016 4017 PMAP_MAP_TO_HEAD_LOCK(); 4018 pmap_acquire_pmap_lock(pm); 4019 4020 l2b = pmap_get_l2_bucket(pm, va); 4021 KDASSERT(l2b != NULL); 4022 4023 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4024 pte = *ptep; 4025 4026 /* Extract the physical address of the page */ 4027 pa = l2pte_pa(pte); 4028 4029 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4030 /* Update the wired bit in the pv entry for this page. */ 4031 simple_lock(&pg->mdpage.pvh_slock); 4032 (void) pmap_modify_pv(pg, pm, va, PVF_WIRED, 0); 4033 simple_unlock(&pg->mdpage.pvh_slock); 4034 } 4035 4036 pmap_release_pmap_lock(pm); 4037 PMAP_MAP_TO_HEAD_UNLOCK(); 4038 } 4039 4040 void 4041 pmap_activate(struct lwp *l) 4042 { 4043 extern int block_userspace_access; 4044 pmap_t opm, npm, rpm; 4045 uint32_t odacr, ndacr; 4046 int oldirqstate; 4047 4048 /* 4049 * If activating a non-current lwp or the current lwp is 4050 * already active, just return. 4051 */ 4052 if (l != curlwp || 4053 l->l_proc->p_vmspace->vm_map.pmap->pm_activated == true) 4054 return; 4055 4056 npm = l->l_proc->p_vmspace->vm_map.pmap; 4057 ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 4058 (DOMAIN_CLIENT << (npm->pm_domain * 2)); 4059 4060 /* 4061 * If TTB and DACR are unchanged, short-circuit all the 4062 * TLB/cache management stuff. 4063 */ 4064 if (pmap_previous_active_lwp != NULL) { 4065 opm = pmap_previous_active_lwp->l_proc->p_vmspace->vm_map.pmap; 4066 odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 4067 (DOMAIN_CLIENT << (opm->pm_domain * 2)); 4068 4069 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4070 goto all_done; 4071 } else 4072 opm = NULL; 4073 4074 PMAPCOUNT(activations); 4075 block_userspace_access = 1; 4076 4077 /* 4078 * If switching to a user vmspace which is different to the 4079 * most recent one, and the most recent one is potentially 4080 * live in the cache, we must write-back and invalidate the 4081 * entire cache. 4082 */ 4083 rpm = pmap_recent_user; 4084 if (npm != pmap_kernel() && rpm && npm != rpm && 4085 rpm->pm_cstate.cs_cache) { 4086 rpm->pm_cstate.cs_cache = 0; 4087 #ifdef PMAP_CACHE_VIVT 4088 cpu_idcache_wbinv_all(); 4089 #endif 4090 } 4091 4092 /* No interrupts while we frob the TTB/DACR */ 4093 oldirqstate = disable_interrupts(IF32_bits); 4094 4095 /* 4096 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 4097 * entry corresponding to 'vector_page' in the incoming L1 table 4098 * before switching to it otherwise subsequent interrupts/exceptions 4099 * (including domain faults!) will jump into hyperspace. 4100 */ 4101 if (npm->pm_pl1vec != NULL) { 4102 cpu_tlb_flushID_SE((u_int)vector_page); 4103 cpu_cpwait(); 4104 *npm->pm_pl1vec = npm->pm_l1vec; 4105 PTE_SYNC(npm->pm_pl1vec); 4106 } 4107 4108 cpu_domains(ndacr); 4109 4110 if (npm == pmap_kernel() || npm == rpm) { 4111 /* 4112 * Switching to a kernel thread, or back to the 4113 * same user vmspace as before... Simply update 4114 * the TTB (no TLB flush required) 4115 */ 4116 __asm volatile("mcr p15, 0, %0, c2, c0, 0" :: 4117 "r"(npm->pm_l1->l1_physaddr)); 4118 cpu_cpwait(); 4119 } else { 4120 /* 4121 * Otherwise, update TTB and flush TLB 4122 */ 4123 cpu_context_switch(npm->pm_l1->l1_physaddr); 4124 if (rpm != NULL) 4125 rpm->pm_cstate.cs_tlb = 0; 4126 } 4127 4128 restore_interrupts(oldirqstate); 4129 4130 block_userspace_access = 0; 4131 4132 all_done: 4133 /* 4134 * The new pmap is resident. Make sure it's marked 4135 * as resident in the cache/TLB. 4136 */ 4137 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4138 if (npm != pmap_kernel()) 4139 pmap_recent_user = npm; 4140 4141 /* The old pmap is not longer active */ 4142 if (opm != NULL) 4143 opm->pm_activated = false; 4144 4145 /* But the new one is */ 4146 npm->pm_activated = true; 4147 } 4148 4149 void 4150 pmap_deactivate(struct lwp *l) 4151 { 4152 4153 /* 4154 * If the process is exiting, make sure pmap_activate() does 4155 * a full MMU context-switch and cache flush, which we might 4156 * otherwise skip. See PR port-arm/38950. 4157 */ 4158 if (l->l_proc->p_sflag & PS_WEXIT) 4159 pmap_previous_active_lwp = NULL; 4160 4161 l->l_proc->p_vmspace->vm_map.pmap->pm_activated = false; 4162 } 4163 4164 void 4165 pmap_update(pmap_t pm) 4166 { 4167 4168 if (pm->pm_remove_all) { 4169 /* 4170 * Finish up the pmap_remove_all() optimisation by flushing 4171 * the TLB. 4172 */ 4173 pmap_tlb_flushID(pm); 4174 pm->pm_remove_all = false; 4175 } 4176 4177 if (pmap_is_current(pm)) { 4178 /* 4179 * If we're dealing with a current userland pmap, move its L1 4180 * to the end of the LRU. 4181 */ 4182 if (pm != pmap_kernel()) 4183 pmap_use_l1(pm); 4184 4185 /* 4186 * We can assume we're done with frobbing the cache/tlb for 4187 * now. Make sure any future pmap ops don't skip cache/tlb 4188 * flushes. 4189 */ 4190 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4191 } 4192 4193 PMAPCOUNT(updates); 4194 4195 /* 4196 * make sure TLB/cache operations have completed. 4197 */ 4198 cpu_cpwait(); 4199 } 4200 4201 void 4202 pmap_remove_all(pmap_t pm) 4203 { 4204 4205 /* 4206 * The vmspace described by this pmap is about to be torn down. 4207 * Until pmap_update() is called, UVM will only make calls 4208 * to pmap_remove(). We can make life much simpler by flushing 4209 * the cache now, and deferring TLB invalidation to pmap_update(). 4210 */ 4211 #ifdef PMAP_CACHE_VIVT 4212 pmap_idcache_wbinv_all(pm); 4213 #endif 4214 pm->pm_remove_all = true; 4215 } 4216 4217 /* 4218 * Retire the given physical map from service. 4219 * Should only be called if the map contains no valid mappings. 4220 */ 4221 void 4222 pmap_destroy(pmap_t pm) 4223 { 4224 u_int count; 4225 4226 if (pm == NULL) 4227 return; 4228 4229 if (pm->pm_remove_all) { 4230 pmap_tlb_flushID(pm); 4231 pm->pm_remove_all = false; 4232 } 4233 4234 /* 4235 * Drop reference count 4236 */ 4237 mutex_enter(&pm->pm_lock); 4238 count = --pm->pm_obj.uo_refs; 4239 mutex_exit(&pm->pm_lock); 4240 if (count > 0) { 4241 if (pmap_is_current(pm)) { 4242 if (pm != pmap_kernel()) 4243 pmap_use_l1(pm); 4244 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4245 } 4246 return; 4247 } 4248 4249 /* 4250 * reference count is zero, free pmap resources and then free pmap. 4251 */ 4252 4253 if (vector_page < KERNEL_BASE) { 4254 KDASSERT(!pmap_is_current(pm)); 4255 4256 /* Remove the vector page mapping */ 4257 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 4258 pmap_update(pm); 4259 } 4260 4261 LIST_REMOVE(pm, pm_list); 4262 4263 pmap_free_l1(pm); 4264 4265 if (pmap_recent_user == pm) 4266 pmap_recent_user = NULL; 4267 4268 UVM_OBJ_DESTROY(&pm->pm_obj); 4269 4270 /* return the pmap to the pool */ 4271 pool_cache_put(&pmap_cache, pm); 4272 } 4273 4274 4275 /* 4276 * void pmap_reference(pmap_t pm) 4277 * 4278 * Add a reference to the specified pmap. 4279 */ 4280 void 4281 pmap_reference(pmap_t pm) 4282 { 4283 4284 if (pm == NULL) 4285 return; 4286 4287 pmap_use_l1(pm); 4288 4289 mutex_enter(&pm->pm_lock); 4290 pm->pm_obj.uo_refs++; 4291 mutex_exit(&pm->pm_lock); 4292 } 4293 4294 #if ARM_MMU_V6 > 0 4295 4296 static struct evcnt pmap_prefer_nochange_ev = 4297 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 4298 static struct evcnt pmap_prefer_change_ev = 4299 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 4300 4301 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 4302 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 4303 4304 void 4305 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 4306 { 4307 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 4308 vaddr_t va = *vap; 4309 vaddr_t diff = (hint - va) & mask; 4310 if (diff == 0) { 4311 pmap_prefer_nochange_ev.ev_count++; 4312 } else { 4313 pmap_prefer_change_ev.ev_count++; 4314 if (__predict_false(td)) 4315 va -= mask + 1; 4316 *vap = va + diff; 4317 } 4318 } 4319 #endif /* ARM_MMU_V6 */ 4320 4321 /* 4322 * pmap_zero_page() 4323 * 4324 * Zero a given physical page by mapping it at a page hook point. 4325 * In doing the zero page op, the page we zero is mapped cachable, as with 4326 * StrongARM accesses to non-cached pages are non-burst making writing 4327 * _any_ bulk data very slow. 4328 */ 4329 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 4330 void 4331 pmap_zero_page_generic(paddr_t phys) 4332 { 4333 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4334 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4335 #endif 4336 #ifdef PMAP_CACHE_VIPT 4337 /* Choose the last page color it had, if any */ 4338 const vsize_t va_offset = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 4339 #else 4340 const vsize_t va_offset = 0; 4341 #endif 4342 pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT]; 4343 4344 #ifdef DEBUG 4345 if (!SLIST_EMPTY(&pg->mdpage.pvh_list)) 4346 panic("pmap_zero_page: page has mappings"); 4347 #endif 4348 4349 KDASSERT((phys & PGOFSET) == 0); 4350 4351 /* 4352 * Hook in the page, zero it, and purge the cache for that 4353 * zeroed page. Invalidate the TLB as needed. 4354 */ 4355 *ptep = L2_S_PROTO | phys | 4356 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4357 PTE_SYNC(ptep); 4358 cpu_tlb_flushD_SE(cdstp + va_offset); 4359 cpu_cpwait(); 4360 bzero_page(cdstp + va_offset); 4361 /* 4362 * Unmap the page. 4363 */ 4364 *ptep = 0; 4365 PTE_SYNC(ptep); 4366 cpu_tlb_flushD_SE(cdstp + va_offset); 4367 #ifdef PMAP_CACHE_VIVT 4368 cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE); 4369 #endif 4370 #ifdef PMAP_CACHE_VIPT 4371 /* 4372 * This page is now cache resident so it now has a page color. 4373 * Any contents have been obliterated so clear the EXEC flag. 4374 */ 4375 if (!pmap_is_page_colored_p(pg)) { 4376 PMAPCOUNT(vac_color_new); 4377 pg->mdpage.pvh_attrs |= PVF_COLORED; 4378 } 4379 if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) { 4380 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 4381 PMAPCOUNT(exec_discarded_zero); 4382 } 4383 pg->mdpage.pvh_attrs |= PVF_DIRTY; 4384 #endif 4385 } 4386 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 4387 4388 #if ARM_MMU_XSCALE == 1 4389 void 4390 pmap_zero_page_xscale(paddr_t phys) 4391 { 4392 #ifdef DEBUG 4393 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4394 4395 if (!SLIST_EMPTY(&pg->mdpage.pvh_list)) 4396 panic("pmap_zero_page: page has mappings"); 4397 #endif 4398 4399 KDASSERT((phys & PGOFSET) == 0); 4400 4401 /* 4402 * Hook in the page, zero it, and purge the cache for that 4403 * zeroed page. Invalidate the TLB as needed. 4404 */ 4405 *cdst_pte = L2_S_PROTO | phys | 4406 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4407 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4408 PTE_SYNC(cdst_pte); 4409 cpu_tlb_flushD_SE(cdstp); 4410 cpu_cpwait(); 4411 bzero_page(cdstp); 4412 xscale_cache_clean_minidata(); 4413 } 4414 #endif /* ARM_MMU_XSCALE == 1 */ 4415 4416 /* pmap_pageidlezero() 4417 * 4418 * The same as above, except that we assume that the page is not 4419 * mapped. This means we never have to flush the cache first. Called 4420 * from the idle loop. 4421 */ 4422 bool 4423 pmap_pageidlezero(paddr_t phys) 4424 { 4425 unsigned int i; 4426 int *ptr; 4427 bool rv = true; 4428 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4429 struct vm_page * const pg = PHYS_TO_VM_PAGE(phys); 4430 #endif 4431 #ifdef PMAP_CACHE_VIPT 4432 /* Choose the last page color it had, if any */ 4433 const vsize_t va_offset = pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 4434 #else 4435 const vsize_t va_offset = 0; 4436 #endif 4437 pt_entry_t * const ptep = &csrc_pte[va_offset >> PGSHIFT]; 4438 4439 4440 #ifdef DEBUG 4441 if (!SLIST_EMPTY(&pg->mdpage.pvh_list)) 4442 panic("pmap_pageidlezero: page has mappings"); 4443 #endif 4444 4445 KDASSERT((phys & PGOFSET) == 0); 4446 4447 /* 4448 * Hook in the page, zero it, and purge the cache for that 4449 * zeroed page. Invalidate the TLB as needed. 4450 */ 4451 *ptep = L2_S_PROTO | phys | 4452 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4453 PTE_SYNC(ptep); 4454 cpu_tlb_flushD_SE(cdstp + va_offset); 4455 cpu_cpwait(); 4456 4457 for (i = 0, ptr = (int *)(cdstp + va_offset); 4458 i < (PAGE_SIZE / sizeof(int)); i++) { 4459 if (sched_curcpu_runnable_p() != 0) { 4460 /* 4461 * A process has become ready. Abort now, 4462 * so we don't keep it waiting while we 4463 * do slow memory access to finish this 4464 * page. 4465 */ 4466 rv = false; 4467 break; 4468 } 4469 *ptr++ = 0; 4470 } 4471 4472 #ifdef PMAP_CACHE_VIVT 4473 if (rv) 4474 /* 4475 * if we aborted we'll rezero this page again later so don't 4476 * purge it unless we finished it 4477 */ 4478 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4479 #elif defined(PMAP_CACHE_VIPT) 4480 /* 4481 * This page is now cache resident so it now has a page color. 4482 * Any contents have been obliterated so clear the EXEC flag. 4483 */ 4484 if (!pmap_is_page_colored_p(pg)) { 4485 PMAPCOUNT(vac_color_new); 4486 pg->mdpage.pvh_attrs |= PVF_COLORED; 4487 } 4488 if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)) { 4489 pg->mdpage.pvh_attrs &= ~PVF_EXEC; 4490 PMAPCOUNT(exec_discarded_zero); 4491 } 4492 #endif 4493 /* 4494 * Unmap the page. 4495 */ 4496 *ptep = 0; 4497 PTE_SYNC(ptep); 4498 cpu_tlb_flushD_SE(cdstp + va_offset); 4499 4500 return (rv); 4501 } 4502 4503 /* 4504 * pmap_copy_page() 4505 * 4506 * Copy one physical page into another, by mapping the pages into 4507 * hook points. The same comment regarding cachability as in 4508 * pmap_zero_page also applies here. 4509 */ 4510 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 4511 void 4512 pmap_copy_page_generic(paddr_t src, paddr_t dst) 4513 { 4514 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 4515 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4516 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 4517 #endif 4518 #ifdef PMAP_CACHE_VIPT 4519 const vsize_t src_va_offset = src_pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 4520 const vsize_t dst_va_offset = dst_pg->mdpage.pvh_attrs & arm_cache_prefer_mask; 4521 #else 4522 const vsize_t src_va_offset = 0; 4523 const vsize_t dst_va_offset = 0; 4524 #endif 4525 pt_entry_t * const src_ptep = &csrc_pte[src_va_offset >> PGSHIFT]; 4526 pt_entry_t * const dst_ptep = &cdst_pte[dst_va_offset >> PGSHIFT]; 4527 4528 #ifdef DEBUG 4529 if (!SLIST_EMPTY(&dst_pg->mdpage.pvh_list)) 4530 panic("pmap_copy_page: dst page has mappings"); 4531 #endif 4532 4533 #ifdef PMAP_CACHE_VIPT 4534 KASSERT(src_pg->mdpage.pvh_attrs & (PVF_COLORED|PVF_NC)); 4535 #endif 4536 KDASSERT((src & PGOFSET) == 0); 4537 KDASSERT((dst & PGOFSET) == 0); 4538 4539 /* 4540 * Clean the source page. Hold the source page's lock for 4541 * the duration of the copy so that no other mappings can 4542 * be created while we have a potentially aliased mapping. 4543 */ 4544 simple_lock(&src_pg->mdpage.pvh_slock); 4545 #ifdef PMAP_CACHE_VIVT 4546 (void) pmap_clean_page(SLIST_FIRST(&src_pg->mdpage.pvh_list), true); 4547 #endif 4548 4549 /* 4550 * Map the pages into the page hook points, copy them, and purge 4551 * the cache for the appropriate page. Invalidate the TLB 4552 * as required. 4553 */ 4554 *src_ptep = L2_S_PROTO 4555 | src 4556 #ifdef PMAP_CACHE_VIPT 4557 | ((src_pg->mdpage.pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 4558 #endif 4559 #ifdef PMAP_CACHE_VIVT 4560 | pte_l2_s_cache_mode 4561 #endif 4562 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 4563 *dst_ptep = L2_S_PROTO | dst | 4564 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4565 PTE_SYNC(src_ptep); 4566 PTE_SYNC(dst_ptep); 4567 cpu_tlb_flushD_SE(csrcp + src_va_offset); 4568 cpu_tlb_flushD_SE(cdstp + dst_va_offset); 4569 cpu_cpwait(); 4570 bcopy_page(csrcp + src_va_offset, cdstp + dst_va_offset); 4571 #ifdef PMAP_CACHE_VIVT 4572 cpu_dcache_inv_range(csrcp + src_va_offset, PAGE_SIZE); 4573 #endif 4574 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 4575 #ifdef PMAP_CACHE_VIVT 4576 cpu_dcache_wbinv_range(cdstp + dst_va_offset, PAGE_SIZE); 4577 #endif 4578 /* 4579 * Unmap the pages. 4580 */ 4581 *src_ptep = 0; 4582 *dst_ptep = 0; 4583 PTE_SYNC(src_ptep); 4584 PTE_SYNC(dst_ptep); 4585 cpu_tlb_flushD_SE(csrcp + src_va_offset); 4586 cpu_tlb_flushD_SE(cdstp + dst_va_offset); 4587 #ifdef PMAP_CACHE_VIPT 4588 /* 4589 * Now that the destination page is in the cache, mark it as colored. 4590 * If this was an exec page, discard it. 4591 */ 4592 if (!pmap_is_page_colored_p(dst_pg)) { 4593 PMAPCOUNT(vac_color_new); 4594 dst_pg->mdpage.pvh_attrs |= PVF_COLORED; 4595 } 4596 if (PV_IS_EXEC_P(dst_pg->mdpage.pvh_attrs)) { 4597 dst_pg->mdpage.pvh_attrs &= ~PVF_EXEC; 4598 PMAPCOUNT(exec_discarded_copy); 4599 } 4600 dst_pg->mdpage.pvh_attrs |= PVF_DIRTY; 4601 #endif 4602 } 4603 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 4604 4605 #if ARM_MMU_XSCALE == 1 4606 void 4607 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 4608 { 4609 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4610 #ifdef DEBUG 4611 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4612 4613 if (!SLIST_EMPTY(&dst_pg->mdpage.pvh_list)) 4614 panic("pmap_copy_page: dst page has mappings"); 4615 #endif 4616 4617 KDASSERT((src & PGOFSET) == 0); 4618 KDASSERT((dst & PGOFSET) == 0); 4619 4620 /* 4621 * Clean the source page. Hold the source page's lock for 4622 * the duration of the copy so that no other mappings can 4623 * be created while we have a potentially aliased mapping. 4624 */ 4625 simple_lock(&src_pg->mdpage.pvh_slock); 4626 #ifdef PMAP_CACHE_VIVT 4627 (void) pmap_clean_page(SLIST_FIRST(&src_pg->mdpage.pvh_list), true); 4628 #endif 4629 4630 /* 4631 * Map the pages into the page hook points, copy them, and purge 4632 * the cache for the appropriate page. Invalidate the TLB 4633 * as required. 4634 */ 4635 *csrc_pte = L2_S_PROTO | src | 4636 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4637 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4638 PTE_SYNC(csrc_pte); 4639 *cdst_pte = L2_S_PROTO | dst | 4640 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4641 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4642 PTE_SYNC(cdst_pte); 4643 cpu_tlb_flushD_SE(csrcp); 4644 cpu_tlb_flushD_SE(cdstp); 4645 cpu_cpwait(); 4646 bcopy_page(csrcp, cdstp); 4647 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 4648 xscale_cache_clean_minidata(); 4649 } 4650 #endif /* ARM_MMU_XSCALE == 1 */ 4651 4652 /* 4653 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 4654 * 4655 * Return the start and end addresses of the kernel's virtual space. 4656 * These values are setup in pmap_bootstrap and are updated as pages 4657 * are allocated. 4658 */ 4659 void 4660 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 4661 { 4662 *start = virtual_avail; 4663 *end = virtual_end; 4664 } 4665 4666 /* 4667 * Helper function for pmap_grow_l2_bucket() 4668 */ 4669 static inline int 4670 pmap_grow_map(vaddr_t va, pt_entry_t cache_mode, paddr_t *pap) 4671 { 4672 struct l2_bucket *l2b; 4673 pt_entry_t *ptep; 4674 paddr_t pa; 4675 4676 if (uvm.page_init_done == false) { 4677 #ifdef PMAP_STEAL_MEMORY 4678 pv_addr_t pv; 4679 pmap_boot_pagealloc(PAGE_SIZE, 4680 #ifdef PMAP_CACHE_VIPT 4681 arm_cache_prefer_mask, 4682 va & arm_cache_prefer_mask, 4683 #else 4684 0, 0, 4685 #endif 4686 &pv); 4687 pa = pv.pv_pa; 4688 #else 4689 if (uvm_page_physget(&pa) == false) 4690 return (1); 4691 #endif /* PMAP_STEAL_MEMORY */ 4692 } else { 4693 struct vm_page *pg; 4694 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 4695 if (pg == NULL) 4696 return (1); 4697 pa = VM_PAGE_TO_PHYS(pg); 4698 #ifdef PMAP_CACHE_VIPT 4699 /* 4700 * This new page must not have any mappings. Enter it via 4701 * pmap_kenter_pa and let that routine do the hard work. 4702 */ 4703 KASSERT(SLIST_EMPTY(&pg->mdpage.pvh_list)); 4704 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE|PMAP_KMPAGE); 4705 #endif 4706 } 4707 4708 if (pap) 4709 *pap = pa; 4710 4711 PMAPCOUNT(pt_mappings); 4712 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 4713 KDASSERT(l2b != NULL); 4714 4715 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4716 *ptep = L2_S_PROTO | pa | cache_mode | 4717 L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); 4718 PTE_SYNC(ptep); 4719 memset((void *)va, 0, PAGE_SIZE); 4720 return (0); 4721 } 4722 4723 /* 4724 * This is the same as pmap_alloc_l2_bucket(), except that it is only 4725 * used by pmap_growkernel(). 4726 */ 4727 static inline struct l2_bucket * 4728 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 4729 { 4730 struct l2_dtable *l2; 4731 struct l2_bucket *l2b; 4732 u_short l1idx; 4733 vaddr_t nva; 4734 4735 l1idx = L1_IDX(va); 4736 4737 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 4738 /* 4739 * No mapping at this address, as there is 4740 * no entry in the L1 table. 4741 * Need to allocate a new l2_dtable. 4742 */ 4743 nva = pmap_kernel_l2dtable_kva; 4744 if ((nva & PGOFSET) == 0) { 4745 /* 4746 * Need to allocate a backing page 4747 */ 4748 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 4749 return (NULL); 4750 } 4751 4752 l2 = (struct l2_dtable *)nva; 4753 nva += sizeof(struct l2_dtable); 4754 4755 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 4756 /* 4757 * The new l2_dtable straddles a page boundary. 4758 * Map in another page to cover it. 4759 */ 4760 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 4761 return (NULL); 4762 } 4763 4764 pmap_kernel_l2dtable_kva = nva; 4765 4766 /* 4767 * Link it into the parent pmap 4768 */ 4769 pm->pm_l2[L2_IDX(l1idx)] = l2; 4770 } 4771 4772 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 4773 4774 /* 4775 * Fetch pointer to the L2 page table associated with the address. 4776 */ 4777 if (l2b->l2b_kva == NULL) { 4778 pt_entry_t *ptep; 4779 4780 /* 4781 * No L2 page table has been allocated. Chances are, this 4782 * is because we just allocated the l2_dtable, above. 4783 */ 4784 nva = pmap_kernel_l2ptp_kva; 4785 ptep = (pt_entry_t *)nva; 4786 if ((nva & PGOFSET) == 0) { 4787 /* 4788 * Need to allocate a backing page 4789 */ 4790 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 4791 &pmap_kernel_l2ptp_phys)) 4792 return (NULL); 4793 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 4794 } 4795 4796 l2->l2_occupancy++; 4797 l2b->l2b_kva = ptep; 4798 l2b->l2b_l1idx = l1idx; 4799 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 4800 4801 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 4802 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 4803 } 4804 4805 return (l2b); 4806 } 4807 4808 vaddr_t 4809 pmap_growkernel(vaddr_t maxkvaddr) 4810 { 4811 pmap_t kpm = pmap_kernel(); 4812 struct l1_ttable *l1; 4813 struct l2_bucket *l2b; 4814 pd_entry_t *pl1pd; 4815 int s; 4816 4817 if (maxkvaddr <= pmap_curmaxkvaddr) 4818 goto out; /* we are OK */ 4819 4820 NPDEBUG(PDB_GROWKERN, 4821 printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n", 4822 pmap_curmaxkvaddr, maxkvaddr)); 4823 4824 KDASSERT(maxkvaddr <= virtual_end); 4825 4826 /* 4827 * whoops! we need to add kernel PTPs 4828 */ 4829 4830 s = splhigh(); /* to be safe */ 4831 mutex_enter(&kpm->pm_lock); 4832 4833 /* Map 1MB at a time */ 4834 for (; pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE) { 4835 4836 l2b = pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 4837 KDASSERT(l2b != NULL); 4838 4839 /* Distribute new L1 entry to all other L1s */ 4840 SLIST_FOREACH(l1, &l1_list, l1_link) { 4841 pl1pd = &l1->l1_kva[L1_IDX(pmap_curmaxkvaddr)]; 4842 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 4843 L1_C_PROTO; 4844 PTE_SYNC(pl1pd); 4845 } 4846 } 4847 4848 /* 4849 * flush out the cache, expensive but growkernel will happen so 4850 * rarely 4851 */ 4852 cpu_dcache_wbinv_all(); 4853 cpu_tlb_flushD(); 4854 cpu_cpwait(); 4855 4856 mutex_exit(&kpm->pm_lock); 4857 splx(s); 4858 4859 out: 4860 return (pmap_curmaxkvaddr); 4861 } 4862 4863 /************************ Utility routines ****************************/ 4864 4865 /* 4866 * vector_page_setprot: 4867 * 4868 * Manipulate the protection of the vector page. 4869 */ 4870 void 4871 vector_page_setprot(int prot) 4872 { 4873 struct l2_bucket *l2b; 4874 pt_entry_t *ptep; 4875 4876 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 4877 KDASSERT(l2b != NULL); 4878 4879 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 4880 4881 *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 4882 PTE_SYNC(ptep); 4883 cpu_tlb_flushD_SE(vector_page); 4884 cpu_cpwait(); 4885 } 4886 4887 /* 4888 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 4889 * Returns true if the mapping exists, else false. 4890 * 4891 * NOTE: This function is only used by a couple of arm-specific modules. 4892 * It is not safe to take any pmap locks here, since we could be right 4893 * in the middle of debugging the pmap anyway... 4894 * 4895 * It is possible for this routine to return false even though a valid 4896 * mapping does exist. This is because we don't lock, so the metadata 4897 * state may be inconsistent. 4898 * 4899 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 4900 * a "section" mapping. 4901 */ 4902 bool 4903 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 4904 { 4905 struct l2_dtable *l2; 4906 pd_entry_t *pl1pd, l1pd; 4907 pt_entry_t *ptep; 4908 u_short l1idx; 4909 4910 if (pm->pm_l1 == NULL) 4911 return false; 4912 4913 l1idx = L1_IDX(va); 4914 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 4915 l1pd = *pl1pd; 4916 4917 if (l1pte_section_p(l1pd)) { 4918 *ptp = NULL; 4919 return true; 4920 } 4921 4922 if (pm->pm_l2 == NULL) 4923 return false; 4924 4925 l2 = pm->pm_l2[L2_IDX(l1idx)]; 4926 4927 if (l2 == NULL || 4928 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 4929 return false; 4930 } 4931 4932 *ptp = &ptep[l2pte_index(va)]; 4933 return true; 4934 } 4935 4936 bool 4937 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 4938 { 4939 u_short l1idx; 4940 4941 if (pm->pm_l1 == NULL) 4942 return false; 4943 4944 l1idx = L1_IDX(va); 4945 *pdp = &pm->pm_l1->l1_kva[l1idx]; 4946 4947 return true; 4948 } 4949 4950 /************************ Bootstrapping routines ****************************/ 4951 4952 static void 4953 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 4954 { 4955 int i; 4956 4957 l1->l1_kva = l1pt; 4958 l1->l1_domain_use_count = 0; 4959 l1->l1_domain_first = 0; 4960 4961 for (i = 0; i < PMAP_DOMAINS; i++) 4962 l1->l1_domain_free[i] = i + 1; 4963 4964 /* 4965 * Copy the kernel's L1 entries to each new L1. 4966 */ 4967 if (pmap_initialized) 4968 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 4969 4970 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 4971 &l1->l1_physaddr) == false) 4972 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 4973 4974 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 4975 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 4976 } 4977 4978 /* 4979 * pmap_bootstrap() is called from the board-specific initarm() routine 4980 * once the kernel L1/L2 descriptors tables have been set up. 4981 * 4982 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 4983 * spread over a number of disparate files/functions. 4984 * 4985 * We are passed the following parameters 4986 * - kernel_l1pt 4987 * This is a pointer to the base of the kernel's L1 translation table. 4988 * - vstart 4989 * 1MB-aligned start of managed kernel virtual memory. 4990 * - vend 4991 * 1MB-aligned end of managed kernel virtual memory. 4992 * 4993 * We use the first parameter to build the metadata (struct l1_ttable and 4994 * struct l2_dtable) necessary to track kernel mappings. 4995 */ 4996 #define PMAP_STATIC_L2_SIZE 16 4997 void 4998 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 4999 { 5000 static struct l1_ttable static_l1; 5001 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 5002 struct l1_ttable *l1 = &static_l1; 5003 struct l2_dtable *l2; 5004 struct l2_bucket *l2b; 5005 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 5006 pmap_t pm = pmap_kernel(); 5007 pd_entry_t pde; 5008 pt_entry_t *ptep; 5009 paddr_t pa; 5010 vaddr_t va; 5011 vsize_t size; 5012 int nptes, l1idx, l2idx, l2next = 0; 5013 5014 /* 5015 * Initialise the kernel pmap object 5016 */ 5017 pm->pm_l1 = l1; 5018 pm->pm_domain = PMAP_DOMAIN_KERNEL; 5019 pm->pm_activated = true; 5020 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5021 UVM_OBJ_INIT(&pm->pm_obj, NULL, 1); 5022 5023 /* 5024 * Scan the L1 translation table created by initarm() and create 5025 * the required metadata for all valid mappings found in it. 5026 */ 5027 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 5028 pde = l1pt[l1idx]; 5029 5030 /* 5031 * We're only interested in Coarse mappings. 5032 * pmap_extract() can deal with section mappings without 5033 * recourse to checking L2 metadata. 5034 */ 5035 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 5036 continue; 5037 5038 /* 5039 * Lookup the KVA of this L2 descriptor table 5040 */ 5041 pa = (paddr_t)(pde & L1_C_ADDR_MASK); 5042 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 5043 if (ptep == NULL) { 5044 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 5045 (u_int)l1idx << L1_S_SHIFT, pa); 5046 } 5047 5048 /* 5049 * Fetch the associated L2 metadata structure. 5050 * Allocate a new one if necessary. 5051 */ 5052 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 5053 if (l2next == PMAP_STATIC_L2_SIZE) 5054 panic("pmap_bootstrap: out of static L2s"); 5055 pm->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++]; 5056 } 5057 5058 /* 5059 * One more L1 slot tracked... 5060 */ 5061 l2->l2_occupancy++; 5062 5063 /* 5064 * Fill in the details of the L2 descriptor in the 5065 * appropriate bucket. 5066 */ 5067 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 5068 l2b->l2b_kva = ptep; 5069 l2b->l2b_phys = pa; 5070 l2b->l2b_l1idx = l1idx; 5071 5072 /* 5073 * Establish an initial occupancy count for this descriptor 5074 */ 5075 for (l2idx = 0; 5076 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 5077 l2idx++) { 5078 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 5079 l2b->l2b_occupancy++; 5080 } 5081 } 5082 5083 /* 5084 * Make sure the descriptor itself has the correct cache mode. 5085 * If not, fix it, but whine about the problem. Port-meisters 5086 * should consider this a clue to fix up their initarm() 5087 * function. :) 5088 */ 5089 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep)) { 5090 printf("pmap_bootstrap: WARNING! wrong cache mode for " 5091 "L2 pte @ %p\n", ptep); 5092 } 5093 } 5094 5095 /* 5096 * Ensure the primary (kernel) L1 has the correct cache mode for 5097 * a page table. Bitch if it is not correctly set. 5098 */ 5099 for (va = (vaddr_t)l1pt; 5100 va < ((vaddr_t)l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 5101 if (pmap_set_pt_cache_mode(l1pt, va)) 5102 printf("pmap_bootstrap: WARNING! wrong cache mode for " 5103 "primary L1 @ 0x%lx\n", va); 5104 } 5105 5106 cpu_dcache_wbinv_all(); 5107 cpu_tlb_flushID(); 5108 cpu_cpwait(); 5109 5110 /* 5111 * now we allocate the "special" VAs which are used for tmp mappings 5112 * by the pmap (and other modules). we allocate the VAs by advancing 5113 * virtual_avail (note that there are no pages mapped at these VAs). 5114 * 5115 * Managed KVM space start from wherever initarm() tells us. 5116 */ 5117 virtual_avail = vstart; 5118 virtual_end = vend; 5119 5120 #ifdef PMAP_CACHE_VIPT 5121 /* 5122 * If we have a VIPT cache, we need one page/pte per possible alias 5123 * page so we won't violate cache aliasing rules. 5124 */ 5125 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 5126 nptes = (arm_cache_prefer_mask >> PGSHIFT) + 1; 5127 #else 5128 nptes = 1; 5129 #endif 5130 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 5131 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte); 5132 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 5133 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte); 5134 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 5135 pmap_alloc_specials(&virtual_avail, round_page(MSGBUFSIZE) / PAGE_SIZE, 5136 (void *)&msgbufaddr, NULL); 5137 5138 /* 5139 * Allocate a range of kernel virtual address space to be used 5140 * for L2 descriptor tables and metadata allocation in 5141 * pmap_growkernel(). 5142 */ 5143 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 5144 pmap_alloc_specials(&virtual_avail, 5145 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 5146 &pmap_kernel_l2ptp_kva, NULL); 5147 5148 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 5149 pmap_alloc_specials(&virtual_avail, 5150 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 5151 &pmap_kernel_l2dtable_kva, NULL); 5152 5153 /* 5154 * init the static-global locks and global pmap list. 5155 */ 5156 /* spinlockinit(&pmap_main_lock, "pmaplk", 0); */ 5157 5158 /* 5159 * We can now initialise the first L1's metadata. 5160 */ 5161 SLIST_INIT(&l1_list); 5162 TAILQ_INIT(&l1_lru_list); 5163 simple_lock_init(&l1_lru_lock); 5164 pmap_init_l1(l1, l1pt); 5165 5166 /* Set up vector page L1 details, if necessary */ 5167 if (vector_page < KERNEL_BASE) { 5168 pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 5169 l2b = pmap_get_l2_bucket(pm, vector_page); 5170 pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO | 5171 L1_C_DOM(pm->pm_domain); 5172 } else 5173 pm->pm_pl1vec = NULL; 5174 5175 /* 5176 * Initialize the pmap cache 5177 */ 5178 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 5179 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 5180 LIST_INIT(&pmap_pmaps); 5181 LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list); 5182 5183 /* 5184 * Initialize the pv pool. 5185 */ 5186 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 5187 &pmap_bootstrap_pv_allocator, IPL_NONE); 5188 5189 /* 5190 * Initialize the L2 dtable pool and cache. 5191 */ 5192 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 5193 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 5194 5195 /* 5196 * Initialise the L2 descriptor table pool and cache 5197 */ 5198 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 0, 5199 L2_TABLE_SIZE_REAL, 0, "l2ptppl", NULL, IPL_NONE, 5200 pmap_l2ptp_ctor, NULL, NULL); 5201 5202 cpu_dcache_wbinv_all(); 5203 } 5204 5205 static int 5206 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va) 5207 { 5208 pd_entry_t *pdep, pde; 5209 pt_entry_t *ptep, pte; 5210 vaddr_t pa; 5211 int rv = 0; 5212 5213 /* 5214 * Make sure the descriptor itself has the correct cache mode 5215 */ 5216 pdep = &kl1[L1_IDX(va)]; 5217 pde = *pdep; 5218 5219 if (l1pte_section_p(pde)) { 5220 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 5221 *pdep = (pde & ~L1_S_CACHE_MASK) | 5222 pte_l1_s_cache_mode_pt; 5223 PTE_SYNC(pdep); 5224 cpu_dcache_wbinv_range((vaddr_t)pdep, sizeof(*pdep)); 5225 rv = 1; 5226 } 5227 } else { 5228 pa = (paddr_t)(pde & L1_C_ADDR_MASK); 5229 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 5230 if (ptep == NULL) 5231 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 5232 5233 ptep = &ptep[l2pte_index(va)]; 5234 pte = *ptep; 5235 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 5236 *ptep = (pte & ~L2_S_CACHE_MASK) | 5237 pte_l2_s_cache_mode_pt; 5238 PTE_SYNC(ptep); 5239 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 5240 rv = 1; 5241 } 5242 } 5243 5244 return (rv); 5245 } 5246 5247 static void 5248 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 5249 { 5250 vaddr_t va = *availp; 5251 struct l2_bucket *l2b; 5252 5253 if (ptep) { 5254 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5255 if (l2b == NULL) 5256 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 5257 5258 if (ptep) 5259 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 5260 } 5261 5262 *vap = va; 5263 *availp = va + (PAGE_SIZE * pages); 5264 } 5265 5266 void 5267 pmap_init(void) 5268 { 5269 extern int physmem; 5270 5271 /* 5272 * Set the available memory vars - These do not map to real memory 5273 * addresses and cannot as the physical memory is fragmented. 5274 * They are used by ps for %mem calculations. 5275 * One could argue whether this should be the entire memory or just 5276 * the memory that is useable in a user process. 5277 */ 5278 avail_start = 0; 5279 avail_end = physmem * PAGE_SIZE; 5280 5281 /* 5282 * Now we need to free enough pv_entry structures to allow us to get 5283 * the kmem_map/kmem_object allocated and inited (done after this 5284 * function is finished). to do this we allocate one bootstrap page out 5285 * of kernel_map and use it to provide an initial pool of pv_entry 5286 * structures. we never free this page. 5287 */ 5288 pool_setlowat(&pmap_pv_pool, 5289 (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 5290 5291 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 5292 zeropage = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 5293 UVM_KMF_WIRED|UVM_KMF_ZERO); 5294 5295 pmap_initialized = true; 5296 } 5297 5298 static vaddr_t last_bootstrap_page = 0; 5299 static void *free_bootstrap_pages = NULL; 5300 5301 static void * 5302 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 5303 { 5304 extern void *pool_page_alloc(struct pool *, int); 5305 vaddr_t new_page; 5306 void *rv; 5307 5308 if (pmap_initialized) 5309 return (pool_page_alloc(pp, flags)); 5310 5311 if (free_bootstrap_pages) { 5312 rv = free_bootstrap_pages; 5313 free_bootstrap_pages = *((void **)rv); 5314 return (rv); 5315 } 5316 5317 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 5318 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 5319 5320 KASSERT(new_page > last_bootstrap_page); 5321 last_bootstrap_page = new_page; 5322 return ((void *)new_page); 5323 } 5324 5325 static void 5326 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 5327 { 5328 extern void pool_page_free(struct pool *, void *); 5329 5330 if ((vaddr_t)v <= last_bootstrap_page) { 5331 *((void **)v) = free_bootstrap_pages; 5332 free_bootstrap_pages = v; 5333 return; 5334 } 5335 5336 if (pmap_initialized) { 5337 pool_page_free(pp, v); 5338 return; 5339 } 5340 } 5341 5342 /* 5343 * pmap_postinit() 5344 * 5345 * This routine is called after the vm and kmem subsystems have been 5346 * initialised. This allows the pmap code to perform any initialisation 5347 * that can only be done one the memory allocation is in place. 5348 */ 5349 void 5350 pmap_postinit(void) 5351 { 5352 extern paddr_t physical_start, physical_end; 5353 struct l2_bucket *l2b; 5354 struct l1_ttable *l1; 5355 struct pglist plist; 5356 struct vm_page *m; 5357 pd_entry_t *pl1pt; 5358 pt_entry_t *ptep, pte; 5359 vaddr_t va, eva; 5360 u_int loop, needed; 5361 int error; 5362 5363 pool_cache_setlowat(&pmap_l2ptp_cache, 5364 (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 5365 pool_cache_setlowat(&pmap_l2dtable_cache, 5366 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 5367 5368 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 5369 needed -= 1; 5370 5371 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 5372 5373 for (loop = 0; loop < needed; loop++, l1++) { 5374 /* Allocate a L1 page table */ 5375 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 5376 if (va == 0) 5377 panic("Cannot allocate L1 KVM"); 5378 5379 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 5380 physical_end, L1_TABLE_SIZE, 0, &plist, 1, M_WAITOK); 5381 if (error) 5382 panic("Cannot allocate L1 physical pages"); 5383 5384 m = TAILQ_FIRST(&plist); 5385 eva = va + L1_TABLE_SIZE; 5386 pl1pt = (pd_entry_t *)va; 5387 5388 while (m && va < eva) { 5389 paddr_t pa = VM_PAGE_TO_PHYS(m); 5390 5391 pmap_kenter_pa(va, pa, 5392 VM_PROT_READ|VM_PROT_WRITE|PMAP_KMPAGE); 5393 5394 /* 5395 * Make sure the L1 descriptor table is mapped 5396 * with the cache-mode set to write-through. 5397 */ 5398 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5399 ptep = &l2b->l2b_kva[l2pte_index(va)]; 5400 pte = *ptep; 5401 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 5402 *ptep = pte; 5403 PTE_SYNC(ptep); 5404 cpu_tlb_flushD_SE(va); 5405 5406 va += PAGE_SIZE; 5407 m = TAILQ_NEXT(m, pageq.queue); 5408 } 5409 5410 #ifdef DIAGNOSTIC 5411 if (m) 5412 panic("pmap_alloc_l1pt: pglist not empty"); 5413 #endif /* DIAGNOSTIC */ 5414 5415 pmap_init_l1(l1, pl1pt); 5416 } 5417 5418 #ifdef DEBUG 5419 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 5420 needed); 5421 #endif 5422 } 5423 5424 /* 5425 * Note that the following routines are used by board-specific initialisation 5426 * code to configure the initial kernel page tables. 5427 * 5428 * If ARM32_NEW_VM_LAYOUT is *not* defined, they operate on the assumption that 5429 * L2 page-table pages are 4KB in size and use 4 L1 slots. This mimics the 5430 * behaviour of the old pmap, and provides an easy migration path for 5431 * initial bring-up of the new pmap on existing ports. Fortunately, 5432 * pmap_bootstrap() compensates for this hackery. This is only a stop-gap and 5433 * will be deprecated. 5434 * 5435 * If ARM32_NEW_VM_LAYOUT *is* defined, these functions deal with 1KB L2 page 5436 * tables. 5437 */ 5438 5439 /* 5440 * This list exists for the benefit of pmap_map_chunk(). It keeps track 5441 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 5442 * find them as necessary. 5443 * 5444 * Note that the data on this list MUST remain valid after initarm() returns, 5445 * as pmap_bootstrap() uses it to contruct L2 table metadata. 5446 */ 5447 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 5448 5449 static vaddr_t 5450 kernel_pt_lookup(paddr_t pa) 5451 { 5452 pv_addr_t *pv; 5453 5454 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 5455 #ifndef ARM32_NEW_VM_LAYOUT 5456 if (pv->pv_pa == (pa & ~PGOFSET)) 5457 return (pv->pv_va | (pa & PGOFSET)); 5458 #else 5459 if (pv->pv_pa == pa) 5460 return (pv->pv_va); 5461 #endif 5462 } 5463 return (0); 5464 } 5465 5466 /* 5467 * pmap_map_section: 5468 * 5469 * Create a single section mapping. 5470 */ 5471 void 5472 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 5473 { 5474 pd_entry_t *pde = (pd_entry_t *) l1pt; 5475 pd_entry_t fl; 5476 5477 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 5478 5479 switch (cache) { 5480 case PTE_NOCACHE: 5481 default: 5482 fl = 0; 5483 break; 5484 5485 case PTE_CACHE: 5486 fl = pte_l1_s_cache_mode; 5487 break; 5488 5489 case PTE_PAGETABLE: 5490 fl = pte_l1_s_cache_mode_pt; 5491 break; 5492 } 5493 5494 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 5495 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 5496 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 5497 } 5498 5499 /* 5500 * pmap_map_entry: 5501 * 5502 * Create a single page mapping. 5503 */ 5504 void 5505 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 5506 { 5507 pd_entry_t *pde = (pd_entry_t *) l1pt; 5508 pt_entry_t fl; 5509 pt_entry_t *pte; 5510 5511 KASSERT(((va | pa) & PGOFSET) == 0); 5512 5513 switch (cache) { 5514 case PTE_NOCACHE: 5515 default: 5516 fl = 0; 5517 break; 5518 5519 case PTE_CACHE: 5520 fl = pte_l2_s_cache_mode; 5521 break; 5522 5523 case PTE_PAGETABLE: 5524 fl = pte_l2_s_cache_mode_pt; 5525 break; 5526 } 5527 5528 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 5529 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 5530 5531 #ifndef ARM32_NEW_VM_LAYOUT 5532 pte = (pt_entry_t *) 5533 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 5534 #else 5535 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 5536 #endif 5537 if (pte == NULL) 5538 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 5539 5540 fl |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 5541 #ifndef ARM32_NEW_VM_LAYOUT 5542 pte += (va >> PGSHIFT) & 0x3ff; 5543 #else 5544 pte += l2pte_index(va); 5545 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 5546 #endif 5547 *pte = fl; 5548 PTE_SYNC(pte); 5549 } 5550 5551 /* 5552 * pmap_link_l2pt: 5553 * 5554 * Link the L2 page table specified by "l2pv" into the L1 5555 * page table at the slot for "va". 5556 */ 5557 void 5558 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 5559 { 5560 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 5561 u_int slot = va >> L1_S_SHIFT; 5562 5563 #ifndef ARM32_NEW_VM_LAYOUT 5564 KASSERT((va & ((L1_S_SIZE * 4) - 1)) == 0); 5565 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 5566 #endif 5567 5568 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 5569 5570 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 5571 #ifdef ARM32_NEW_VM_LAYOUT 5572 PTE_SYNC(&pde[slot]); 5573 #else 5574 pde[slot + 1] = proto | (l2pv->pv_pa + 0x400); 5575 pde[slot + 2] = proto | (l2pv->pv_pa + 0x800); 5576 pde[slot + 3] = proto | (l2pv->pv_pa + 0xc00); 5577 PTE_SYNC_RANGE(&pde[slot + 0], 4); 5578 #endif 5579 5580 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 5581 } 5582 5583 /* 5584 * pmap_map_chunk: 5585 * 5586 * Map a chunk of memory using the most efficient mappings 5587 * possible (section, large page, small page) into the 5588 * provided L1 and L2 tables at the specified virtual address. 5589 */ 5590 vsize_t 5591 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 5592 int prot, int cache) 5593 { 5594 pd_entry_t *pde = (pd_entry_t *) l1pt; 5595 pt_entry_t *pte, f1, f2s, f2l; 5596 vsize_t resid; 5597 int i; 5598 5599 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 5600 5601 if (l1pt == 0) 5602 panic("pmap_map_chunk: no L1 table provided"); 5603 5604 #ifdef VERBOSE_INIT_ARM 5605 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 5606 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 5607 #endif 5608 5609 switch (cache) { 5610 case PTE_NOCACHE: 5611 default: 5612 f1 = 0; 5613 f2l = 0; 5614 f2s = 0; 5615 break; 5616 5617 case PTE_CACHE: 5618 f1 = pte_l1_s_cache_mode; 5619 f2l = pte_l2_l_cache_mode; 5620 f2s = pte_l2_s_cache_mode; 5621 break; 5622 5623 case PTE_PAGETABLE: 5624 f1 = pte_l1_s_cache_mode_pt; 5625 f2l = pte_l2_l_cache_mode_pt; 5626 f2s = pte_l2_s_cache_mode_pt; 5627 break; 5628 } 5629 5630 size = resid; 5631 5632 while (resid > 0) { 5633 /* See if we can use a section mapping. */ 5634 if (L1_S_MAPPABLE_P(va, pa, resid)) { 5635 #ifdef VERBOSE_INIT_ARM 5636 printf("S"); 5637 #endif 5638 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 5639 L1_S_PROT(PTE_KERNEL, prot) | f1 | 5640 L1_S_DOM(PMAP_DOMAIN_KERNEL); 5641 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 5642 va += L1_S_SIZE; 5643 pa += L1_S_SIZE; 5644 resid -= L1_S_SIZE; 5645 continue; 5646 } 5647 5648 /* 5649 * Ok, we're going to use an L2 table. Make sure 5650 * one is actually in the corresponding L1 slot 5651 * for the current VA. 5652 */ 5653 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 5654 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 5655 5656 #ifndef ARM32_NEW_VM_LAYOUT 5657 pte = (pt_entry_t *) 5658 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 5659 #else 5660 pte = (pt_entry_t *) kernel_pt_lookup( 5661 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 5662 #endif 5663 if (pte == NULL) 5664 panic("pmap_map_chunk: can't find L2 table for VA" 5665 "0x%08lx", va); 5666 5667 /* See if we can use a L2 large page mapping. */ 5668 if (L2_L_MAPPABLE_P(va, pa, resid)) { 5669 #ifdef VERBOSE_INIT_ARM 5670 printf("L"); 5671 #endif 5672 for (i = 0; i < 16; i++) { 5673 #ifndef ARM32_NEW_VM_LAYOUT 5674 pte[((va >> PGSHIFT) & 0x3f0) + i] = 5675 L2_L_PROTO | pa | 5676 L2_L_PROT(PTE_KERNEL, prot) | f2l; 5677 PTE_SYNC(&pte[((va >> PGSHIFT) & 0x3f0) + i]); 5678 #else 5679 pte[l2pte_index(va) + i] = 5680 L2_L_PROTO | pa | 5681 L2_L_PROT(PTE_KERNEL, prot) | f2l; 5682 PTE_SYNC(&pte[l2pte_index(va) + i]); 5683 #endif 5684 } 5685 va += L2_L_SIZE; 5686 pa += L2_L_SIZE; 5687 resid -= L2_L_SIZE; 5688 continue; 5689 } 5690 5691 /* Use a small page mapping. */ 5692 #ifdef VERBOSE_INIT_ARM 5693 printf("P"); 5694 #endif 5695 #ifndef ARM32_NEW_VM_LAYOUT 5696 pte[(va >> PGSHIFT) & 0x3ff] = 5697 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 5698 PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]); 5699 #else 5700 pte[l2pte_index(va)] = 5701 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 5702 PTE_SYNC(&pte[l2pte_index(va)]); 5703 #endif 5704 va += PAGE_SIZE; 5705 pa += PAGE_SIZE; 5706 resid -= PAGE_SIZE; 5707 } 5708 #ifdef VERBOSE_INIT_ARM 5709 printf("\n"); 5710 #endif 5711 return (size); 5712 } 5713 5714 /********************** Static device map routines ***************************/ 5715 5716 static const struct pmap_devmap *pmap_devmap_table; 5717 5718 /* 5719 * Register the devmap table. This is provided in case early console 5720 * initialization needs to register mappings created by bootstrap code 5721 * before pmap_devmap_bootstrap() is called. 5722 */ 5723 void 5724 pmap_devmap_register(const struct pmap_devmap *table) 5725 { 5726 5727 pmap_devmap_table = table; 5728 } 5729 5730 /* 5731 * Map all of the static regions in the devmap table, and remember 5732 * the devmap table so other parts of the kernel can look up entries 5733 * later. 5734 */ 5735 void 5736 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 5737 { 5738 int i; 5739 5740 pmap_devmap_table = table; 5741 5742 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 5743 #ifdef VERBOSE_INIT_ARM 5744 printf("devmap: %08lx -> %08lx @ %08lx\n", 5745 pmap_devmap_table[i].pd_pa, 5746 pmap_devmap_table[i].pd_pa + 5747 pmap_devmap_table[i].pd_size - 1, 5748 pmap_devmap_table[i].pd_va); 5749 #endif 5750 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 5751 pmap_devmap_table[i].pd_pa, 5752 pmap_devmap_table[i].pd_size, 5753 pmap_devmap_table[i].pd_prot, 5754 pmap_devmap_table[i].pd_cache); 5755 } 5756 } 5757 5758 const struct pmap_devmap * 5759 pmap_devmap_find_pa(paddr_t pa, psize_t size) 5760 { 5761 uint64_t endpa; 5762 int i; 5763 5764 if (pmap_devmap_table == NULL) 5765 return (NULL); 5766 5767 endpa = (uint64_t)pa + (uint64_t)(size - 1); 5768 5769 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 5770 if (pa >= pmap_devmap_table[i].pd_pa && 5771 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 5772 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 5773 return (&pmap_devmap_table[i]); 5774 } 5775 5776 return (NULL); 5777 } 5778 5779 const struct pmap_devmap * 5780 pmap_devmap_find_va(vaddr_t va, vsize_t size) 5781 { 5782 int i; 5783 5784 if (pmap_devmap_table == NULL) 5785 return (NULL); 5786 5787 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 5788 if (va >= pmap_devmap_table[i].pd_va && 5789 va + size - 1 <= pmap_devmap_table[i].pd_va + 5790 pmap_devmap_table[i].pd_size - 1) 5791 return (&pmap_devmap_table[i]); 5792 } 5793 5794 return (NULL); 5795 } 5796 5797 /********************** PTE initialization routines **************************/ 5798 5799 /* 5800 * These routines are called when the CPU type is identified to set up 5801 * the PTE prototypes, cache modes, etc. 5802 * 5803 * The variables are always here, just in case modules need to reference 5804 * them (though, they shouldn't). 5805 */ 5806 5807 pt_entry_t pte_l1_s_cache_mode; 5808 pt_entry_t pte_l1_s_cache_mode_pt; 5809 pt_entry_t pte_l1_s_cache_mask; 5810 5811 pt_entry_t pte_l2_l_cache_mode; 5812 pt_entry_t pte_l2_l_cache_mode_pt; 5813 pt_entry_t pte_l2_l_cache_mask; 5814 5815 pt_entry_t pte_l2_s_cache_mode; 5816 pt_entry_t pte_l2_s_cache_mode_pt; 5817 pt_entry_t pte_l2_s_cache_mask; 5818 5819 pt_entry_t pte_l2_s_prot_u; 5820 pt_entry_t pte_l2_s_prot_w; 5821 pt_entry_t pte_l2_s_prot_mask; 5822 5823 pt_entry_t pte_l1_s_proto; 5824 pt_entry_t pte_l1_c_proto; 5825 pt_entry_t pte_l2_s_proto; 5826 5827 void (*pmap_copy_page_func)(paddr_t, paddr_t); 5828 void (*pmap_zero_page_func)(paddr_t); 5829 5830 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 5831 void 5832 pmap_pte_init_generic(void) 5833 { 5834 5835 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 5836 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 5837 5838 pte_l2_l_cache_mode = L2_B|L2_C; 5839 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 5840 5841 pte_l2_s_cache_mode = L2_B|L2_C; 5842 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 5843 5844 /* 5845 * If we have a write-through cache, set B and C. If 5846 * we have a write-back cache, then we assume setting 5847 * only C will make those pages write-through. 5848 */ 5849 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { 5850 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 5851 pte_l2_l_cache_mode_pt = L2_B|L2_C; 5852 pte_l2_s_cache_mode_pt = L2_B|L2_C; 5853 } else { 5854 #if ARM_MMU_V6 > 1 5855 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; /* arm116 errata 399234 */ 5856 pte_l2_l_cache_mode_pt = L2_B|L2_C; /* arm116 errata 399234 */ 5857 pte_l2_s_cache_mode_pt = L2_B|L2_C; /* arm116 errata 399234 */ 5858 #else 5859 pte_l1_s_cache_mode_pt = L1_S_C; 5860 pte_l2_l_cache_mode_pt = L2_C; 5861 pte_l2_s_cache_mode_pt = L2_C; 5862 #endif 5863 } 5864 5865 pte_l2_s_prot_u = L2_S_PROT_U_generic; 5866 pte_l2_s_prot_w = L2_S_PROT_W_generic; 5867 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 5868 5869 pte_l1_s_proto = L1_S_PROTO_generic; 5870 pte_l1_c_proto = L1_C_PROTO_generic; 5871 pte_l2_s_proto = L2_S_PROTO_generic; 5872 5873 pmap_copy_page_func = pmap_copy_page_generic; 5874 pmap_zero_page_func = pmap_zero_page_generic; 5875 } 5876 5877 #if defined(CPU_ARM8) 5878 void 5879 pmap_pte_init_arm8(void) 5880 { 5881 5882 /* 5883 * ARM8 is compatible with generic, but we need to use 5884 * the page tables uncached. 5885 */ 5886 pmap_pte_init_generic(); 5887 5888 pte_l1_s_cache_mode_pt = 0; 5889 pte_l2_l_cache_mode_pt = 0; 5890 pte_l2_s_cache_mode_pt = 0; 5891 } 5892 #endif /* CPU_ARM8 */ 5893 5894 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 5895 void 5896 pmap_pte_init_arm9(void) 5897 { 5898 5899 /* 5900 * ARM9 is compatible with generic, but we want to use 5901 * write-through caching for now. 5902 */ 5903 pmap_pte_init_generic(); 5904 5905 pte_l1_s_cache_mode = L1_S_C; 5906 pte_l2_l_cache_mode = L2_C; 5907 pte_l2_s_cache_mode = L2_C; 5908 5909 pte_l1_s_cache_mode_pt = L1_S_C; 5910 pte_l2_l_cache_mode_pt = L2_C; 5911 pte_l2_s_cache_mode_pt = L2_C; 5912 } 5913 #endif /* CPU_ARM9 */ 5914 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5915 5916 #if defined(CPU_ARM10) 5917 void 5918 pmap_pte_init_arm10(void) 5919 { 5920 5921 /* 5922 * ARM10 is compatible with generic, but we want to use 5923 * write-through caching for now. 5924 */ 5925 pmap_pte_init_generic(); 5926 5927 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 5928 pte_l2_l_cache_mode = L2_B | L2_C; 5929 pte_l2_s_cache_mode = L2_B | L2_C; 5930 5931 pte_l1_s_cache_mode_pt = L1_S_C; 5932 pte_l2_l_cache_mode_pt = L2_C; 5933 pte_l2_s_cache_mode_pt = L2_C; 5934 5935 } 5936 #endif /* CPU_ARM10 */ 5937 5938 #if ARM_MMU_SA1 == 1 5939 void 5940 pmap_pte_init_sa1(void) 5941 { 5942 5943 /* 5944 * The StrongARM SA-1 cache does not have a write-through 5945 * mode. So, do the generic initialization, then reset 5946 * the page table cache mode to B=1,C=1, and note that 5947 * the PTEs need to be sync'd. 5948 */ 5949 pmap_pte_init_generic(); 5950 5951 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 5952 pte_l2_l_cache_mode_pt = L2_B|L2_C; 5953 pte_l2_s_cache_mode_pt = L2_B|L2_C; 5954 5955 pmap_needs_pte_sync = 1; 5956 } 5957 #endif /* ARM_MMU_SA1 == 1*/ 5958 5959 #if ARM_MMU_XSCALE == 1 5960 #if (ARM_NMMUS > 1) 5961 static u_int xscale_use_minidata; 5962 #endif 5963 5964 void 5965 pmap_pte_init_xscale(void) 5966 { 5967 uint32_t auxctl; 5968 int write_through = 0; 5969 5970 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 5971 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 5972 5973 pte_l2_l_cache_mode = L2_B|L2_C; 5974 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 5975 5976 pte_l2_s_cache_mode = L2_B|L2_C; 5977 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 5978 5979 pte_l1_s_cache_mode_pt = L1_S_C; 5980 pte_l2_l_cache_mode_pt = L2_C; 5981 pte_l2_s_cache_mode_pt = L2_C; 5982 5983 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 5984 /* 5985 * The XScale core has an enhanced mode where writes that 5986 * miss the cache cause a cache line to be allocated. This 5987 * is significantly faster than the traditional, write-through 5988 * behavior of this case. 5989 */ 5990 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 5991 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 5992 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 5993 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 5994 5995 #ifdef XSCALE_CACHE_WRITE_THROUGH 5996 /* 5997 * Some versions of the XScale core have various bugs in 5998 * their cache units, the work-around for which is to run 5999 * the cache in write-through mode. Unfortunately, this 6000 * has a major (negative) impact on performance. So, we 6001 * go ahead and run fast-and-loose, in the hopes that we 6002 * don't line up the planets in a way that will trip the 6003 * bugs. 6004 * 6005 * However, we give you the option to be slow-but-correct. 6006 */ 6007 write_through = 1; 6008 #elif defined(XSCALE_CACHE_WRITE_BACK) 6009 /* force write back cache mode */ 6010 write_through = 0; 6011 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 6012 /* 6013 * Intel PXA2[15]0 processors are known to have a bug in 6014 * write-back cache on revision 4 and earlier (stepping 6015 * A[01] and B[012]). Fixed for C0 and later. 6016 */ 6017 { 6018 uint32_t id, type; 6019 6020 id = cpufunc_id(); 6021 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 6022 6023 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 6024 if ((id & CPU_ID_REVISION_MASK) < 5) { 6025 /* write through for stepping A0-1 and B0-2 */ 6026 write_through = 1; 6027 } 6028 } 6029 } 6030 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 6031 6032 if (write_through) { 6033 pte_l1_s_cache_mode = L1_S_C; 6034 pte_l2_l_cache_mode = L2_C; 6035 pte_l2_s_cache_mode = L2_C; 6036 } 6037 6038 #if (ARM_NMMUS > 1) 6039 xscale_use_minidata = 1; 6040 #endif 6041 6042 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 6043 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 6044 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 6045 6046 pte_l1_s_proto = L1_S_PROTO_xscale; 6047 pte_l1_c_proto = L1_C_PROTO_xscale; 6048 pte_l2_s_proto = L2_S_PROTO_xscale; 6049 6050 pmap_copy_page_func = pmap_copy_page_xscale; 6051 pmap_zero_page_func = pmap_zero_page_xscale; 6052 6053 /* 6054 * Disable ECC protection of page table access, for now. 6055 */ 6056 __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 6057 auxctl &= ~XSCALE_AUXCTL_P; 6058 __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 6059 } 6060 6061 /* 6062 * xscale_setup_minidata: 6063 * 6064 * Set up the mini-data cache clean area. We require the 6065 * caller to allocate the right amount of physically and 6066 * virtually contiguous space. 6067 */ 6068 void 6069 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 6070 { 6071 extern vaddr_t xscale_minidata_clean_addr; 6072 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 6073 pd_entry_t *pde = (pd_entry_t *) l1pt; 6074 pt_entry_t *pte; 6075 vsize_t size; 6076 uint32_t auxctl; 6077 6078 xscale_minidata_clean_addr = va; 6079 6080 /* Round it to page size. */ 6081 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 6082 6083 for (; size != 0; 6084 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 6085 #ifndef ARM32_NEW_VM_LAYOUT 6086 pte = (pt_entry_t *) 6087 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 6088 #else 6089 pte = (pt_entry_t *) kernel_pt_lookup( 6090 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 6091 #endif 6092 if (pte == NULL) 6093 panic("xscale_setup_minidata: can't find L2 table for " 6094 "VA 0x%08lx", va); 6095 #ifndef ARM32_NEW_VM_LAYOUT 6096 pte[(va >> PGSHIFT) & 0x3ff] = 6097 #else 6098 pte[l2pte_index(va)] = 6099 #endif 6100 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 6101 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); 6102 } 6103 6104 /* 6105 * Configure the mini-data cache for write-back with 6106 * read/write-allocate. 6107 * 6108 * NOTE: In order to reconfigure the mini-data cache, we must 6109 * make sure it contains no valid data! In order to do that, 6110 * we must issue a global data cache invalidate command! 6111 * 6112 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 6113 * THIS IS VERY IMPORTANT! 6114 */ 6115 6116 /* Invalidate data and mini-data. */ 6117 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 6118 __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 6119 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 6120 __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 6121 } 6122 6123 /* 6124 * Change the PTEs for the specified kernel mappings such that they 6125 * will use the mini data cache instead of the main data cache. 6126 */ 6127 void 6128 pmap_uarea(vaddr_t va) 6129 { 6130 struct l2_bucket *l2b; 6131 pt_entry_t *ptep, *sptep, pte; 6132 vaddr_t next_bucket, eva; 6133 6134 #if (ARM_NMMUS > 1) 6135 if (xscale_use_minidata == 0) 6136 return; 6137 #endif 6138 6139 eva = va + USPACE; 6140 6141 while (va < eva) { 6142 next_bucket = L2_NEXT_BUCKET(va); 6143 if (next_bucket > eva) 6144 next_bucket = eva; 6145 6146 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6147 KDASSERT(l2b != NULL); 6148 6149 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 6150 6151 while (va < next_bucket) { 6152 pte = *ptep; 6153 if (!l2pte_minidata(pte)) { 6154 cpu_dcache_wbinv_range(va, PAGE_SIZE); 6155 cpu_tlb_flushD_SE(va); 6156 *ptep = pte & ~L2_B; 6157 } 6158 ptep++; 6159 va += PAGE_SIZE; 6160 } 6161 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 6162 } 6163 cpu_cpwait(); 6164 } 6165 #endif /* ARM_MMU_XSCALE == 1 */ 6166 6167 /* 6168 * return the PA of the current L1 table, for use when handling a crash dump 6169 */ 6170 uint32_t pmap_kernel_L1_addr() 6171 { 6172 return pmap_kernel()->pm_l1->l1_physaddr; 6173 } 6174 6175 #if defined(DDB) 6176 /* 6177 * A couple of ddb-callable functions for dumping pmaps 6178 */ 6179 void pmap_dump_all(void); 6180 void pmap_dump(pmap_t); 6181 6182 void 6183 pmap_dump_all(void) 6184 { 6185 pmap_t pm; 6186 6187 LIST_FOREACH(pm, &pmap_pmaps, pm_list) { 6188 if (pm == pmap_kernel()) 6189 continue; 6190 pmap_dump(pm); 6191 printf("\n"); 6192 } 6193 } 6194 6195 static pt_entry_t ncptes[64]; 6196 static void pmap_dump_ncpg(pmap_t); 6197 6198 void 6199 pmap_dump(pmap_t pm) 6200 { 6201 struct l2_dtable *l2; 6202 struct l2_bucket *l2b; 6203 pt_entry_t *ptep, pte; 6204 vaddr_t l2_va, l2b_va, va; 6205 int i, j, k, occ, rows = 0; 6206 6207 if (pm == pmap_kernel()) 6208 printf("pmap_kernel (%p): ", pm); 6209 else 6210 printf("user pmap (%p): ", pm); 6211 6212 printf("domain %d, l1 at %p\n", pm->pm_domain, pm->pm_l1->l1_kva); 6213 6214 l2_va = 0; 6215 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 6216 l2 = pm->pm_l2[i]; 6217 6218 if (l2 == NULL || l2->l2_occupancy == 0) 6219 continue; 6220 6221 l2b_va = l2_va; 6222 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 6223 l2b = &l2->l2_bucket[j]; 6224 6225 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 6226 continue; 6227 6228 ptep = l2b->l2b_kva; 6229 6230 for (k = 0; k < 256 && ptep[k] == 0; k++) 6231 ; 6232 6233 k &= ~63; 6234 occ = l2b->l2b_occupancy; 6235 va = l2b_va + (k * 4096); 6236 for (; k < 256; k++, va += 0x1000) { 6237 char ch = ' '; 6238 if ((k % 64) == 0) { 6239 if ((rows % 8) == 0) { 6240 printf( 6241 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 6242 } 6243 printf("%08lx: ", va); 6244 } 6245 6246 ncptes[k & 63] = 0; 6247 pte = ptep[k]; 6248 if (pte == 0) { 6249 ch = '.'; 6250 } else { 6251 occ--; 6252 switch (pte & 0x0c) { 6253 case 0x00: 6254 ch = 'D'; /* No cache No buff */ 6255 break; 6256 case 0x04: 6257 ch = 'B'; /* No cache buff */ 6258 break; 6259 case 0x08: 6260 if (pte & 0x40) 6261 ch = 'm'; 6262 else 6263 ch = 'C'; /* Cache No buff */ 6264 break; 6265 case 0x0c: 6266 ch = 'F'; /* Cache Buff */ 6267 break; 6268 } 6269 6270 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 6271 ch += 0x20; 6272 6273 if ((pte & 0xc) == 0) 6274 ncptes[k & 63] = pte; 6275 } 6276 6277 if ((k % 64) == 63) { 6278 rows++; 6279 printf("%c\n", ch); 6280 pmap_dump_ncpg(pm); 6281 if (occ == 0) 6282 break; 6283 } else 6284 printf("%c", ch); 6285 } 6286 } 6287 } 6288 } 6289 6290 static void 6291 pmap_dump_ncpg(pmap_t pm) 6292 { 6293 struct vm_page *pg; 6294 struct pv_entry *pv; 6295 int i; 6296 6297 for (i = 0; i < 63; i++) { 6298 if (ncptes[i] == 0) 6299 continue; 6300 6301 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 6302 if (pg == NULL) 6303 continue; 6304 6305 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 6306 VM_PAGE_TO_PHYS(pg), 6307 pg->mdpage.krw_mappings, pg->mdpage.kro_mappings, 6308 pg->mdpage.urw_mappings, pg->mdpage.uro_mappings); 6309 6310 SLIST_FOREACH(pv, &pg->mdpage.pvh_list, pv_link) { 6311 printf(" %c va 0x%08lx, flags 0x%x\n", 6312 (pm == pv->pv_pmap) ? '*' : ' ', 6313 pv->pv_va, pv->pv_flags); 6314 } 6315 } 6316 } 6317 #endif 6318 6319 #ifdef PMAP_STEAL_MEMORY 6320 void 6321 pmap_boot_pageadd(pv_addr_t *newpv) 6322 { 6323 pv_addr_t *pv, *npv; 6324 6325 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 6326 if (newpv->pv_pa < pv->pv_va) { 6327 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 6328 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 6329 newpv->pv_size += pv->pv_size; 6330 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 6331 } 6332 pv = NULL; 6333 } else { 6334 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 6335 pv = npv) { 6336 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 6337 KASSERT(pv->pv_pa < newpv->pv_pa); 6338 if (newpv->pv_pa > npv->pv_pa) 6339 continue; 6340 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 6341 pv->pv_size += newpv->pv_size; 6342 return; 6343 } 6344 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 6345 break; 6346 newpv->pv_size += npv->pv_size; 6347 SLIST_INSERT_AFTER(pv, newpv, pv_list); 6348 SLIST_REMOVE_AFTER(newpv, pv_list); 6349 return; 6350 } 6351 } 6352 } 6353 6354 if (pv) { 6355 SLIST_INSERT_AFTER(pv, newpv, pv_list); 6356 } else { 6357 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 6358 } 6359 } 6360 6361 void 6362 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 6363 pv_addr_t *rpv) 6364 { 6365 pv_addr_t *pv, **pvp; 6366 struct vm_physseg *ps; 6367 size_t i; 6368 6369 KASSERT(amount & PGOFSET); 6370 KASSERT((mask & PGOFSET) == 0); 6371 KASSERT((match & PGOFSET) == 0); 6372 KASSERT(amount != 0); 6373 6374 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 6375 (pv = *pvp) != NULL; 6376 pvp = &SLIST_NEXT(pv, pv_list)) { 6377 pv_addr_t *newpv; 6378 psize_t off; 6379 /* 6380 * If this entry is too small to satify the request... 6381 */ 6382 KASSERT(pv->pv_size > 0); 6383 if (pv->pv_size < amount) 6384 continue; 6385 6386 for (off = 0; off <= mask; off += PAGE_SIZE) { 6387 if (((pv->pv_pa + off) & mask) == match 6388 && off + amount <= pv->pv_size) 6389 break; 6390 } 6391 if (off > mask) 6392 continue; 6393 6394 rpv->pv_va = pv->pv_va + off; 6395 rpv->pv_pa = pv->pv_pa + off; 6396 rpv->pv_size = amount; 6397 pv->pv_size -= amount; 6398 if (pv->pv_size == 0) { 6399 KASSERT(off == 0); 6400 KASSERT((vaddr_t) pv == rpv->pv_va); 6401 *pvp = SLIST_NEXT(pv, pv_list); 6402 } else if (off == 0) { 6403 KASSERT((vaddr_t) pv == rpv->pv_va); 6404 newpv = (pv_addr_t *) (rpv->pv_va + amount); 6405 *newpv = *pv; 6406 newpv->pv_pa += amount; 6407 newpv->pv_va += amount; 6408 *pvp = newpv; 6409 } else if (off < pv->pv_size) { 6410 newpv = (pv_addr_t *) (rpv->pv_va + amount); 6411 *newpv = *pv; 6412 newpv->pv_size -= off; 6413 newpv->pv_pa += off + amount; 6414 newpv->pv_va += off + amount; 6415 6416 SLIST_NEXT(pv, pv_list) = newpv; 6417 pv->pv_size = off; 6418 } else { 6419 KASSERT((vaddr_t) pv != rpv->pv_va); 6420 } 6421 memset((void *)rpv->pv_va, 0, amount); 6422 return; 6423 } 6424 6425 if (vm_nphysseg == 0) 6426 panic("pmap_boot_pagealloc: couldn't allocate memory"); 6427 6428 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 6429 (pv = *pvp) != NULL; 6430 pvp = &SLIST_NEXT(pv, pv_list)) { 6431 if (SLIST_NEXT(pv, pv_list) == NULL) 6432 break; 6433 } 6434 KASSERT(mask == 0); 6435 for (ps = vm_physmem, i = 0; i < vm_nphysseg; ps++, i++) { 6436 if (ps->avail_start == atop(pv->pv_pa + pv->pv_size) 6437 && pv->pv_va + pv->pv_size <= ptoa(ps->avail_end)) { 6438 rpv->pv_va = pv->pv_va; 6439 rpv->pv_pa = pv->pv_pa; 6440 rpv->pv_size = amount; 6441 *pvp = NULL; 6442 pmap_map_chunk(kernel_l1pt.pv_va, 6443 ptoa(ps->avail_start) + (pv->pv_va - pv->pv_pa), 6444 ptoa(ps->avail_start), 6445 amount - pv->pv_size, 6446 VM_PROT_READ|VM_PROT_WRITE, 6447 PTE_CACHE); 6448 ps->avail_start += atop(amount - pv->pv_size); 6449 /* 6450 * If we consumed the entire physseg, remove it. 6451 */ 6452 if (ps->avail_start == ps->avail_end) { 6453 for (--vm_nphysseg; i < vm_nphysseg; i++, ps++) 6454 ps[0] = ps[1]; 6455 } 6456 memset((void *)rpv->pv_va, 0, rpv->pv_size); 6457 return; 6458 } 6459 } 6460 6461 panic("pmap_boot_pagealloc: couldn't allocate memory"); 6462 } 6463 6464 vaddr_t 6465 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 6466 { 6467 pv_addr_t pv; 6468 6469 pmap_boot_pagealloc(size, 0, 0, &pv); 6470 6471 return pv.pv_va; 6472 } 6473 #endif /* PMAP_STEAL_MEMORY */ 6474 6475 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 6476 { 6477 sysctl_createv(clog, 0, NULL, NULL, 6478 CTLFLAG_PERMANENT, 6479 CTLTYPE_NODE, "machdep", NULL, 6480 NULL, 0, NULL, 0, 6481 CTL_MACHDEP, CTL_EOL); 6482 6483 sysctl_createv(clog, 0, NULL, NULL, 6484 CTLFLAG_PERMANENT, 6485 CTLTYPE_INT, "kmpages", 6486 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 6487 NULL, 0, &pmap_kmpages, 0, 6488 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 6489 } 6490
Indexes created Sat Sep 27 14:09:57 GMT 2025