pmap.c revision 1.252
1 /* $NetBSD: pmap.c,v 1.252 2013/02/04 13:37:30 macallan Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependent vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of 0xffff0000, then 166 * MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* 183 * Special compilation symbols 184 * PMAP_DEBUG - Build in pmap_debug_level code 185 */ 186 187 /* Include header files */ 188 189 #include "opt_cpuoptions.h" 190 #include "opt_pmap_debug.h" 191 #include "opt_ddb.h" 192 #include "opt_lockdebug.h" 193 #include "opt_multiprocessor.h" 194 195 #include <sys/param.h> 196 #include <sys/types.h> 197 #include <sys/kernel.h> 198 #include <sys/systm.h> 199 #include <sys/proc.h> 200 #include <sys/pool.h> 201 #include <sys/kmem.h> 202 #include <sys/cdefs.h> 203 #include <sys/cpu.h> 204 #include <sys/sysctl.h> 205 206 #include <uvm/uvm.h> 207 208 #include <sys/bus.h> 209 #include <machine/pmap.h> 210 #include <machine/pcb.h> 211 #include <machine/param.h> 212 #include <arm/cpuconf.h> 213 #include <arm/arm32/katelib.h> 214 215 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.252 2013/02/04 13:37:30 macallan Exp $"); 216 217 #ifdef PMAP_DEBUG 218 219 /* XXX need to get rid of all refs to this */ 220 int pmap_debug_level = 0; 221 222 /* 223 * for switching to potentially finer grained debugging 224 */ 225 #define PDB_FOLLOW 0x0001 226 #define PDB_INIT 0x0002 227 #define PDB_ENTER 0x0004 228 #define PDB_REMOVE 0x0008 229 #define PDB_CREATE 0x0010 230 #define PDB_PTPAGE 0x0020 231 #define PDB_GROWKERN 0x0040 232 #define PDB_BITS 0x0080 233 #define PDB_COLLECT 0x0100 234 #define PDB_PROTECT 0x0200 235 #define PDB_MAP_L1 0x0400 236 #define PDB_BOOTSTRAP 0x1000 237 #define PDB_PARANOIA 0x2000 238 #define PDB_WIRING 0x4000 239 #define PDB_PVDUMP 0x8000 240 #define PDB_VAC 0x10000 241 #define PDB_KENTER 0x20000 242 #define PDB_KREMOVE 0x40000 243 #define PDB_EXEC 0x80000 244 245 int debugmap = 1; 246 int pmapdebug = 0; 247 #define NPDEBUG(_lev_,_stat_) \ 248 if (pmapdebug & (_lev_)) \ 249 ((_stat_)) 250 251 #else /* PMAP_DEBUG */ 252 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 253 #endif /* PMAP_DEBUG */ 254 255 /* 256 * pmap_kernel() points here 257 */ 258 static struct pmap kernel_pmap_store; 259 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; 260 #ifdef PMAP_NEED_ALLOC_POOLPAGE 261 int arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT; 262 #endif 263 264 /* 265 * Which pmap is currently 'live' in the cache 266 * 267 * XXXSCW: Fix for SMP ... 268 */ 269 static pmap_t pmap_recent_user; 270 271 /* 272 * Pointer to last active lwp, or NULL if it exited. 273 */ 274 struct lwp *pmap_previous_active_lwp; 275 276 /* 277 * Pool and cache that pmap structures are allocated from. 278 * We use a cache to avoid clearing the pm_l2[] array (1KB) 279 * in pmap_create(). 280 */ 281 static struct pool_cache pmap_cache; 282 static LIST_HEAD(, pmap) pmap_pmaps; 283 284 /* 285 * Pool of PV structures 286 */ 287 static struct pool pmap_pv_pool; 288 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 289 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 290 static struct pool_allocator pmap_bootstrap_pv_allocator = { 291 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 292 }; 293 294 /* 295 * Pool and cache of l2_dtable structures. 296 * We use a cache to avoid clearing the structures when they're 297 * allocated. (196 bytes) 298 */ 299 static struct pool_cache pmap_l2dtable_cache; 300 static vaddr_t pmap_kernel_l2dtable_kva; 301 302 /* 303 * Pool and cache of L2 page descriptors. 304 * We use a cache to avoid clearing the descriptor table 305 * when they're allocated. (1KB) 306 */ 307 static struct pool_cache pmap_l2ptp_cache; 308 static vaddr_t pmap_kernel_l2ptp_kva; 309 static paddr_t pmap_kernel_l2ptp_phys; 310 311 #ifdef PMAPCOUNTERS 312 #define PMAP_EVCNT_INITIALIZER(name) \ 313 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 314 315 #ifdef PMAP_CACHE_VIPT 316 static struct evcnt pmap_ev_vac_clean_one = 317 PMAP_EVCNT_INITIALIZER("clean page (1 color)"); 318 static struct evcnt pmap_ev_vac_flush_one = 319 PMAP_EVCNT_INITIALIZER("flush page (1 color)"); 320 static struct evcnt pmap_ev_vac_flush_lots = 321 PMAP_EVCNT_INITIALIZER("flush page (2+ colors)"); 322 static struct evcnt pmap_ev_vac_flush_lots2 = 323 PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)"); 324 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one); 325 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one); 326 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots); 327 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2); 328 329 static struct evcnt pmap_ev_vac_color_new = 330 PMAP_EVCNT_INITIALIZER("new page color"); 331 static struct evcnt pmap_ev_vac_color_reuse = 332 PMAP_EVCNT_INITIALIZER("ok first page color"); 333 static struct evcnt pmap_ev_vac_color_ok = 334 PMAP_EVCNT_INITIALIZER("ok page color"); 335 static struct evcnt pmap_ev_vac_color_blind = 336 PMAP_EVCNT_INITIALIZER("blind page color"); 337 static struct evcnt pmap_ev_vac_color_change = 338 PMAP_EVCNT_INITIALIZER("change page color"); 339 static struct evcnt pmap_ev_vac_color_erase = 340 PMAP_EVCNT_INITIALIZER("erase page color"); 341 static struct evcnt pmap_ev_vac_color_none = 342 PMAP_EVCNT_INITIALIZER("no page color"); 343 static struct evcnt pmap_ev_vac_color_restore = 344 PMAP_EVCNT_INITIALIZER("restore page color"); 345 346 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 347 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 348 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 349 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 350 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 351 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 352 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 353 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 354 #endif 355 356 static struct evcnt pmap_ev_mappings = 357 PMAP_EVCNT_INITIALIZER("pages mapped"); 358 static struct evcnt pmap_ev_unmappings = 359 PMAP_EVCNT_INITIALIZER("pages unmapped"); 360 static struct evcnt pmap_ev_remappings = 361 PMAP_EVCNT_INITIALIZER("pages remapped"); 362 363 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 364 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 365 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 366 367 static struct evcnt pmap_ev_kernel_mappings = 368 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 369 static struct evcnt pmap_ev_kernel_unmappings = 370 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 371 static struct evcnt pmap_ev_kernel_remappings = 372 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 373 374 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 375 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 376 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 377 378 static struct evcnt pmap_ev_kenter_mappings = 379 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 380 static struct evcnt pmap_ev_kenter_unmappings = 381 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 382 static struct evcnt pmap_ev_kenter_remappings = 383 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 384 static struct evcnt pmap_ev_pt_mappings = 385 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 386 387 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 388 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 389 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 390 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 391 392 #ifdef PMAP_CACHE_VIPT 393 static struct evcnt pmap_ev_exec_mappings = 394 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 395 static struct evcnt pmap_ev_exec_cached = 396 PMAP_EVCNT_INITIALIZER("exec pages cached"); 397 398 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 399 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 400 401 static struct evcnt pmap_ev_exec_synced = 402 PMAP_EVCNT_INITIALIZER("exec pages synced"); 403 static struct evcnt pmap_ev_exec_synced_map = 404 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 405 static struct evcnt pmap_ev_exec_synced_unmap = 406 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 407 static struct evcnt pmap_ev_exec_synced_remap = 408 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 409 static struct evcnt pmap_ev_exec_synced_clearbit = 410 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 411 static struct evcnt pmap_ev_exec_synced_kremove = 412 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 413 414 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 415 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 416 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 417 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 418 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 419 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 420 421 static struct evcnt pmap_ev_exec_discarded_unmap = 422 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 423 static struct evcnt pmap_ev_exec_discarded_zero = 424 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 425 static struct evcnt pmap_ev_exec_discarded_copy = 426 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 427 static struct evcnt pmap_ev_exec_discarded_page_protect = 428 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 429 static struct evcnt pmap_ev_exec_discarded_clearbit = 430 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 431 static struct evcnt pmap_ev_exec_discarded_kremove = 432 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 433 434 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 435 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 436 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 437 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 438 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 439 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 440 #endif /* PMAP_CACHE_VIPT */ 441 442 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 443 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 444 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 445 446 EVCNT_ATTACH_STATIC(pmap_ev_updates); 447 EVCNT_ATTACH_STATIC(pmap_ev_collects); 448 EVCNT_ATTACH_STATIC(pmap_ev_activations); 449 450 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 451 #else 452 #define PMAPCOUNT(x) ((void)0) 453 #endif 454 455 /* 456 * pmap copy/zero page, and mem(5) hook point 457 */ 458 static pt_entry_t *csrc_pte, *cdst_pte; 459 static vaddr_t csrcp, cdstp; 460 vaddr_t memhook; /* used by mem.c */ 461 kmutex_t memlock; /* used by mem.c */ 462 void *zeropage; /* used by mem.c */ 463 extern void *msgbufaddr; 464 int pmap_kmpages; 465 /* 466 * Flag to indicate if pmap_init() has done its thing 467 */ 468 bool pmap_initialized; 469 470 /* 471 * Misc. locking data structures 472 */ 473 474 #define pmap_acquire_pmap_lock(pm) \ 475 do { \ 476 if ((pm) != pmap_kernel()) \ 477 mutex_enter((pm)->pm_lock); \ 478 } while (/*CONSTCOND*/0) 479 480 #define pmap_release_pmap_lock(pm) \ 481 do { \ 482 if ((pm) != pmap_kernel()) \ 483 mutex_exit((pm)->pm_lock); \ 484 } while (/*CONSTCOND*/0) 485 486 487 /* 488 * Metadata for L1 translation tables. 489 */ 490 struct l1_ttable { 491 /* Entry on the L1 Table list */ 492 SLIST_ENTRY(l1_ttable) l1_link; 493 494 /* Entry on the L1 Least Recently Used list */ 495 TAILQ_ENTRY(l1_ttable) l1_lru; 496 497 /* Track how many domains are allocated from this L1 */ 498 volatile u_int l1_domain_use_count; 499 500 /* 501 * A free-list of domain numbers for this L1. 502 * We avoid using ffs() and a bitmap to track domains since ffs() 503 * is slow on ARM. 504 */ 505 uint8_t l1_domain_first; 506 uint8_t l1_domain_free[PMAP_DOMAINS]; 507 508 /* Physical address of this L1 page table */ 509 paddr_t l1_physaddr; 510 511 /* KVA of this L1 page table */ 512 pd_entry_t *l1_kva; 513 }; 514 515 /* 516 * Convert a virtual address into its L1 table index. That is, the 517 * index used to locate the L2 descriptor table pointer in an L1 table. 518 * This is basically used to index l1->l1_kva[]. 519 * 520 * Each L2 descriptor table represents 1MB of VA space. 521 */ 522 #define L1_IDX(va) (((vaddr_t)(va)) >> L1_S_SHIFT) 523 524 /* 525 * L1 Page Tables are tracked using a Least Recently Used list. 526 * - New L1s are allocated from the HEAD. 527 * - Freed L1s are added to the TAIl. 528 * - Recently accessed L1s (where an 'access' is some change to one of 529 * the userland pmaps which owns this L1) are moved to the TAIL. 530 */ 531 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 532 static kmutex_t l1_lru_lock __cacheline_aligned; 533 534 /* 535 * A list of all L1 tables 536 */ 537 static SLIST_HEAD(, l1_ttable) l1_list; 538 539 /* 540 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 541 * 542 * This is normally 16MB worth L2 page descriptors for any given pmap. 543 * Reference counts are maintained for L2 descriptors so they can be 544 * freed when empty. 545 */ 546 struct l2_dtable { 547 /* The number of L2 page descriptors allocated to this l2_dtable */ 548 u_int l2_occupancy; 549 550 /* List of L2 page descriptors */ 551 struct l2_bucket { 552 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 553 paddr_t l2b_phys; /* Physical address of same */ 554 u_short l2b_l1idx; /* This L2 table's L1 index */ 555 u_short l2b_occupancy; /* How many active descriptors */ 556 } l2_bucket[L2_BUCKET_SIZE]; 557 }; 558 559 /* 560 * Given an L1 table index, calculate the corresponding l2_dtable index 561 * and bucket index within the l2_dtable. 562 */ 563 #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 564 (L2_SIZE - 1)) 565 #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 566 567 /* 568 * Given a virtual address, this macro returns the 569 * virtual address required to drop into the next L2 bucket. 570 */ 571 #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 572 573 /* 574 * L2 allocation. 575 */ 576 #define pmap_alloc_l2_dtable() \ 577 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 578 #define pmap_free_l2_dtable(l2) \ 579 pool_cache_put(&pmap_l2dtable_cache, (l2)) 580 #define pmap_alloc_l2_ptp(pap) \ 581 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 582 PR_NOWAIT, (pap))) 583 584 /* 585 * We try to map the page tables write-through, if possible. However, not 586 * all CPUs have a write-through cache mode, so on those we have to sync 587 * the cache when we frob page tables. 588 * 589 * We try to evaluate this at compile time, if possible. However, it's 590 * not always possible to do that, hence this run-time var. 591 */ 592 int pmap_needs_pte_sync; 593 594 /* 595 * Real definition of pv_entry. 596 */ 597 struct pv_entry { 598 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 599 pmap_t pv_pmap; /* pmap where mapping lies */ 600 vaddr_t pv_va; /* virtual address for mapping */ 601 u_int pv_flags; /* flags */ 602 }; 603 604 /* 605 * Macro to determine if a mapping might be resident in the 606 * instruction cache and/or TLB 607 */ 608 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 609 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 610 611 /* 612 * Macro to determine if a mapping might be resident in the 613 * data cache and/or TLB 614 */ 615 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 616 617 /* 618 * Local prototypes 619 */ 620 static int pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t); 621 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 622 pt_entry_t **); 623 static bool pmap_is_current(pmap_t); 624 static bool pmap_is_cached(pmap_t); 625 static void pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *, 626 pmap_t, vaddr_t, u_int); 627 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t); 628 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 629 static u_int pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t, 630 u_int, u_int); 631 632 static void pmap_pinit(pmap_t); 633 static int pmap_pmap_ctor(void *, void *, int); 634 635 static void pmap_alloc_l1(pmap_t); 636 static void pmap_free_l1(pmap_t); 637 static void pmap_use_l1(pmap_t); 638 639 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 640 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 641 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 642 static int pmap_l2ptp_ctor(void *, void *, int); 643 static int pmap_l2dtable_ctor(void *, void *, int); 644 645 static void pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 646 #ifdef PMAP_CACHE_VIVT 647 static void pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 648 static void pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 649 #endif 650 651 static void pmap_clearbit(struct vm_page_md *, paddr_t, u_int); 652 #ifdef PMAP_CACHE_VIVT 653 static int pmap_clean_page(struct pv_entry *, bool); 654 #endif 655 #ifdef PMAP_CACHE_VIPT 656 static void pmap_syncicache_page(struct vm_page_md *, paddr_t); 657 enum pmap_flush_op { 658 PMAP_FLUSH_PRIMARY, 659 PMAP_FLUSH_SECONDARY, 660 PMAP_CLEAN_PRIMARY 661 }; 662 static void pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op); 663 #endif 664 static void pmap_page_remove(struct vm_page_md *, paddr_t); 665 666 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 667 static vaddr_t kernel_pt_lookup(paddr_t); 668 669 670 /* 671 * Misc variables 672 */ 673 vaddr_t virtual_avail; 674 vaddr_t virtual_end; 675 vaddr_t pmap_curmaxkvaddr; 676 677 paddr_t avail_start; 678 paddr_t avail_end; 679 680 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 681 pv_addr_t kernelpages; 682 pv_addr_t kernel_l1pt; 683 pv_addr_t systempage; 684 685 /* Function to set the debug level of the pmap code */ 686 687 #ifdef PMAP_DEBUG 688 void 689 pmap_debug(int level) 690 { 691 pmap_debug_level = level; 692 printf("pmap_debug: level=%d\n", pmap_debug_level); 693 } 694 #endif /* PMAP_DEBUG */ 695 696 #ifdef PMAP_CACHE_VIPT 697 #define PMAP_VALIDATE_MD_PAGE(md) \ 698 KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \ 699 "(md) %p: attrs=%#x urw=%u krw=%u", (md), \ 700 (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings); 701 #endif /* PMAP_CACHE_VIPT */ 702 /* 703 * A bunch of routines to conditionally flush the caches/TLB depending 704 * on whether the specified pmap actually needs to be flushed at any 705 * given time. 706 */ 707 static inline void 708 pmap_tlb_flushID_SE(pmap_t pm, vaddr_t va) 709 { 710 711 if (pm->pm_cstate.cs_tlb_id) 712 cpu_tlb_flushID_SE(va); 713 } 714 715 static inline void 716 pmap_tlb_flushD_SE(pmap_t pm, vaddr_t va) 717 { 718 719 if (pm->pm_cstate.cs_tlb_d) 720 cpu_tlb_flushD_SE(va); 721 } 722 723 static inline void 724 pmap_tlb_flushID(pmap_t pm) 725 { 726 727 if (pm->pm_cstate.cs_tlb_id) { 728 cpu_tlb_flushID(); 729 pm->pm_cstate.cs_tlb = 0; 730 } 731 } 732 733 static inline void 734 pmap_tlb_flushD(pmap_t pm) 735 { 736 737 if (pm->pm_cstate.cs_tlb_d) { 738 cpu_tlb_flushD(); 739 pm->pm_cstate.cs_tlb_d = 0; 740 } 741 } 742 743 #ifdef PMAP_CACHE_VIVT 744 static inline void 745 pmap_idcache_wbinv_range(pmap_t pm, vaddr_t va, vsize_t len) 746 { 747 if (pm->pm_cstate.cs_cache_id) { 748 cpu_idcache_wbinv_range(va, len); 749 } 750 } 751 752 static inline void 753 pmap_dcache_wb_range(pmap_t pm, vaddr_t va, vsize_t len, 754 bool do_inv, bool rd_only) 755 { 756 757 if (pm->pm_cstate.cs_cache_d) { 758 if (do_inv) { 759 if (rd_only) 760 cpu_dcache_inv_range(va, len); 761 else 762 cpu_dcache_wbinv_range(va, len); 763 } else 764 if (!rd_only) 765 cpu_dcache_wb_range(va, len); 766 } 767 } 768 769 static inline void 770 pmap_idcache_wbinv_all(pmap_t pm) 771 { 772 if (pm->pm_cstate.cs_cache_id) { 773 cpu_idcache_wbinv_all(); 774 pm->pm_cstate.cs_cache = 0; 775 } 776 } 777 778 static inline void 779 pmap_dcache_wbinv_all(pmap_t pm) 780 { 781 if (pm->pm_cstate.cs_cache_d) { 782 cpu_dcache_wbinv_all(); 783 pm->pm_cstate.cs_cache_d = 0; 784 } 785 } 786 #endif /* PMAP_CACHE_VIVT */ 787 788 static inline bool 789 pmap_is_current(pmap_t pm) 790 { 791 792 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 793 return true; 794 795 return false; 796 } 797 798 static inline bool 799 pmap_is_cached(pmap_t pm) 800 { 801 802 if (pm == pmap_kernel() || pmap_recent_user == NULL || 803 pmap_recent_user == pm) 804 return (true); 805 806 return false; 807 } 808 809 /* 810 * PTE_SYNC_CURRENT: 811 * 812 * Make sure the pte is written out to RAM. 813 * We need to do this for one of two cases: 814 * - We're dealing with the kernel pmap 815 * - There is no pmap active in the cache/tlb. 816 * - The specified pmap is 'active' in the cache/tlb. 817 */ 818 #ifdef PMAP_INCLUDE_PTE_SYNC 819 #define PTE_SYNC_CURRENT(pm, ptep) \ 820 do { \ 821 if (PMAP_NEEDS_PTE_SYNC && \ 822 pmap_is_cached(pm)) \ 823 PTE_SYNC(ptep); \ 824 } while (/*CONSTCOND*/0) 825 #else 826 #define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ 827 #endif 828 829 /* 830 * main pv_entry manipulation functions: 831 * pmap_enter_pv: enter a mapping onto a vm_page list 832 * pmap_remove_pv: remove a mapping from a vm_page list 833 * 834 * NOTE: pmap_enter_pv expects to lock the pvh itself 835 * pmap_remove_pv expects the caller to lock the pvh before calling 836 */ 837 838 /* 839 * pmap_enter_pv: enter a mapping onto a vm_page lst 840 * 841 * => caller should hold the proper lock on pmap_main_lock 842 * => caller should have pmap locked 843 * => we will gain the lock on the vm_page and allocate the new pv_entry 844 * => caller should adjust ptp's wire_count before calling 845 * => caller should not adjust pmap's wire_count 846 */ 847 static void 848 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm, 849 vaddr_t va, u_int flags) 850 { 851 struct pv_entry **pvp; 852 853 NPDEBUG(PDB_PVDUMP, 854 printf("pmap_enter_pv: pm %p, md %p, flags 0x%x\n", pm, md, flags)); 855 856 pv->pv_pmap = pm; 857 pv->pv_va = va; 858 pv->pv_flags = flags; 859 860 pvp = &SLIST_FIRST(&md->pvh_list); 861 #ifdef PMAP_CACHE_VIPT 862 /* 863 * Insert unmanaged entries, writeable first, at the head of 864 * the pv list. 865 */ 866 if (__predict_true((flags & PVF_KENTRY) == 0)) { 867 while (*pvp != NULL && (*pvp)->pv_flags & PVF_KENTRY) 868 pvp = &SLIST_NEXT(*pvp, pv_link); 869 } else if ((flags & PVF_WRITE) == 0) { 870 while (*pvp != NULL && (*pvp)->pv_flags & PVF_WRITE) 871 pvp = &SLIST_NEXT(*pvp, pv_link); 872 } 873 #endif 874 SLIST_NEXT(pv, pv_link) = *pvp; /* add to ... */ 875 *pvp = pv; /* ... locked list */ 876 md->pvh_attrs |= flags & (PVF_REF | PVF_MOD); 877 #ifdef PMAP_CACHE_VIPT 878 if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE) 879 md->pvh_attrs |= PVF_KMOD; 880 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 881 md->pvh_attrs |= PVF_DIRTY; 882 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 883 #endif 884 if (pm == pmap_kernel()) { 885 PMAPCOUNT(kernel_mappings); 886 if (flags & PVF_WRITE) 887 md->krw_mappings++; 888 else 889 md->kro_mappings++; 890 } else { 891 if (flags & PVF_WRITE) 892 md->urw_mappings++; 893 else 894 md->uro_mappings++; 895 } 896 897 #ifdef PMAP_CACHE_VIPT 898 /* 899 * Even though pmap_vac_me_harder will set PVF_WRITE for us, 900 * do it here as well to keep the mappings & KVF_WRITE consistent. 901 */ 902 if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) { 903 md->pvh_attrs |= PVF_WRITE; 904 } 905 /* 906 * If this is an exec mapping and its the first exec mapping 907 * for this page, make sure to sync the I-cache. 908 */ 909 if (PV_IS_EXEC_P(flags)) { 910 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 911 pmap_syncicache_page(md, pa); 912 PMAPCOUNT(exec_synced_map); 913 } 914 PMAPCOUNT(exec_mappings); 915 } 916 #endif 917 918 PMAPCOUNT(mappings); 919 920 if (pv->pv_flags & PVF_WIRED) 921 ++pm->pm_stats.wired_count; 922 } 923 924 /* 925 * 926 * pmap_find_pv: Find a pv entry 927 * 928 * => caller should hold lock on vm_page 929 */ 930 static inline struct pv_entry * 931 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va) 932 { 933 struct pv_entry *pv; 934 935 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 936 if (pm == pv->pv_pmap && va == pv->pv_va) 937 break; 938 } 939 940 return (pv); 941 } 942 943 /* 944 * pmap_remove_pv: try to remove a mapping from a pv_list 945 * 946 * => caller should hold proper lock on pmap_main_lock 947 * => pmap should be locked 948 * => caller should hold lock on vm_page [so that attrs can be adjusted] 949 * => caller should adjust ptp's wire_count and free PTP if needed 950 * => caller should NOT adjust pmap's wire_count 951 * => we return the removed pv 952 */ 953 static struct pv_entry * 954 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 955 { 956 struct pv_entry *pv, **prevptr; 957 958 NPDEBUG(PDB_PVDUMP, 959 printf("pmap_remove_pv: pm %p, md %p, va 0x%08lx\n", pm, md, va)); 960 961 prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */ 962 pv = *prevptr; 963 964 while (pv) { 965 if (pv->pv_pmap == pm && pv->pv_va == va) { /* match? */ 966 NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, md " 967 "%p, flags 0x%x\n", pm, md, pv->pv_flags)); 968 if (pv->pv_flags & PVF_WIRED) { 969 --pm->pm_stats.wired_count; 970 } 971 *prevptr = SLIST_NEXT(pv, pv_link); /* remove it! */ 972 if (pm == pmap_kernel()) { 973 PMAPCOUNT(kernel_unmappings); 974 if (pv->pv_flags & PVF_WRITE) 975 md->krw_mappings--; 976 else 977 md->kro_mappings--; 978 } else { 979 if (pv->pv_flags & PVF_WRITE) 980 md->urw_mappings--; 981 else 982 md->uro_mappings--; 983 } 984 985 PMAPCOUNT(unmappings); 986 #ifdef PMAP_CACHE_VIPT 987 if (!(pv->pv_flags & PVF_WRITE)) 988 break; 989 /* 990 * If this page has had an exec mapping, then if 991 * this was the last mapping, discard the contents, 992 * otherwise sync the i-cache for this page. 993 */ 994 if (PV_IS_EXEC_P(md->pvh_attrs)) { 995 if (SLIST_EMPTY(&md->pvh_list)) { 996 md->pvh_attrs &= ~PVF_EXEC; 997 PMAPCOUNT(exec_discarded_unmap); 998 } else { 999 pmap_syncicache_page(md, pa); 1000 PMAPCOUNT(exec_synced_unmap); 1001 } 1002 } 1003 #endif /* PMAP_CACHE_VIPT */ 1004 break; 1005 } 1006 prevptr = &SLIST_NEXT(pv, pv_link); /* previous pointer */ 1007 pv = *prevptr; /* advance */ 1008 } 1009 1010 #ifdef PMAP_CACHE_VIPT 1011 /* 1012 * If we no longer have a WRITEABLE KENTRY at the head of list, 1013 * clear the KMOD attribute from the page. 1014 */ 1015 if (SLIST_FIRST(&md->pvh_list) == NULL 1016 || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE) 1017 md->pvh_attrs &= ~PVF_KMOD; 1018 1019 /* 1020 * If this was a writeable page and there are no more writeable 1021 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1022 * the contents to memory. 1023 */ 1024 if (arm_cache_prefer_mask != 0) { 1025 if (md->krw_mappings + md->urw_mappings == 0) 1026 md->pvh_attrs &= ~PVF_WRITE; 1027 PMAP_VALIDATE_MD_PAGE(md); 1028 } 1029 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1030 #endif /* PMAP_CACHE_VIPT */ 1031 1032 return(pv); /* return removed pv */ 1033 } 1034 1035 /* 1036 * 1037 * pmap_modify_pv: Update pv flags 1038 * 1039 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1040 * => caller should NOT adjust pmap's wire_count 1041 * => caller must call pmap_vac_me_harder() if writable status of a page 1042 * may have changed. 1043 * => we return the old flags 1044 * 1045 * Modify a physical-virtual mapping in the pv table 1046 */ 1047 static u_int 1048 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va, 1049 u_int clr_mask, u_int set_mask) 1050 { 1051 struct pv_entry *npv; 1052 u_int flags, oflags; 1053 1054 KASSERT((clr_mask & PVF_KENTRY) == 0); 1055 KASSERT((set_mask & PVF_KENTRY) == 0); 1056 1057 if ((npv = pmap_find_pv(md, pm, va)) == NULL) 1058 return (0); 1059 1060 NPDEBUG(PDB_PVDUMP, 1061 printf("pmap_modify_pv: pm %p, md %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, md, clr_mask, set_mask, npv->pv_flags)); 1062 1063 /* 1064 * There is at least one VA mapping this page. 1065 */ 1066 1067 if (clr_mask & (PVF_REF | PVF_MOD)) { 1068 md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1069 #ifdef PMAP_CACHE_VIPT 1070 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1071 md->pvh_attrs |= PVF_DIRTY; 1072 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1073 #endif 1074 } 1075 1076 oflags = npv->pv_flags; 1077 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1078 1079 if ((flags ^ oflags) & PVF_WIRED) { 1080 if (flags & PVF_WIRED) 1081 ++pm->pm_stats.wired_count; 1082 else 1083 --pm->pm_stats.wired_count; 1084 } 1085 1086 if ((flags ^ oflags) & PVF_WRITE) { 1087 if (pm == pmap_kernel()) { 1088 if (flags & PVF_WRITE) { 1089 md->krw_mappings++; 1090 md->kro_mappings--; 1091 } else { 1092 md->kro_mappings++; 1093 md->krw_mappings--; 1094 } 1095 } else { 1096 if (flags & PVF_WRITE) { 1097 md->urw_mappings++; 1098 md->uro_mappings--; 1099 } else { 1100 md->uro_mappings++; 1101 md->urw_mappings--; 1102 } 1103 } 1104 } 1105 #ifdef PMAP_CACHE_VIPT 1106 if (arm_cache_prefer_mask != 0) { 1107 if (md->urw_mappings + md->krw_mappings == 0) { 1108 md->pvh_attrs &= ~PVF_WRITE; 1109 } else { 1110 md->pvh_attrs |= PVF_WRITE; 1111 } 1112 } 1113 /* 1114 * We have two cases here: the first is from enter_pv (new exec 1115 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1116 * Since in latter, pmap_enter_pv won't do anything, we just have 1117 * to do what pmap_remove_pv would do. 1118 */ 1119 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs)) 1120 || (PV_IS_EXEC_P(md->pvh_attrs) 1121 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1122 pmap_syncicache_page(md, pa); 1123 PMAPCOUNT(exec_synced_remap); 1124 } 1125 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1126 #endif 1127 1128 PMAPCOUNT(remappings); 1129 1130 return (oflags); 1131 } 1132 1133 /* 1134 * Allocate an L1 translation table for the specified pmap. 1135 * This is called at pmap creation time. 1136 */ 1137 static void 1138 pmap_alloc_l1(pmap_t pm) 1139 { 1140 struct l1_ttable *l1; 1141 uint8_t domain; 1142 1143 /* 1144 * Remove the L1 at the head of the LRU list 1145 */ 1146 mutex_spin_enter(&l1_lru_lock); 1147 l1 = TAILQ_FIRST(&l1_lru_list); 1148 KDASSERT(l1 != NULL); 1149 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1150 1151 /* 1152 * Pick the first available domain number, and update 1153 * the link to the next number. 1154 */ 1155 domain = l1->l1_domain_first; 1156 l1->l1_domain_first = l1->l1_domain_free[domain]; 1157 1158 /* 1159 * If there are still free domain numbers in this L1, 1160 * put it back on the TAIL of the LRU list. 1161 */ 1162 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1163 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1164 1165 mutex_spin_exit(&l1_lru_lock); 1166 1167 /* 1168 * Fix up the relevant bits in the pmap structure 1169 */ 1170 pm->pm_l1 = l1; 1171 pm->pm_domain = domain + 1; 1172 } 1173 1174 /* 1175 * Free an L1 translation table. 1176 * This is called at pmap destruction time. 1177 */ 1178 static void 1179 pmap_free_l1(pmap_t pm) 1180 { 1181 struct l1_ttable *l1 = pm->pm_l1; 1182 1183 mutex_spin_enter(&l1_lru_lock); 1184 1185 /* 1186 * If this L1 is currently on the LRU list, remove it. 1187 */ 1188 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1189 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1190 1191 /* 1192 * Free up the domain number which was allocated to the pmap 1193 */ 1194 l1->l1_domain_free[pm->pm_domain - 1] = l1->l1_domain_first; 1195 l1->l1_domain_first = pm->pm_domain - 1; 1196 l1->l1_domain_use_count--; 1197 1198 /* 1199 * The L1 now must have at least 1 free domain, so add 1200 * it back to the LRU list. If the use count is zero, 1201 * put it at the head of the list, otherwise it goes 1202 * to the tail. 1203 */ 1204 if (l1->l1_domain_use_count == 0) 1205 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1206 else 1207 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1208 1209 mutex_spin_exit(&l1_lru_lock); 1210 } 1211 1212 static inline void 1213 pmap_use_l1(pmap_t pm) 1214 { 1215 struct l1_ttable *l1; 1216 1217 /* 1218 * Do nothing if we're in interrupt context. 1219 * Access to an L1 by the kernel pmap must not affect 1220 * the LRU list. 1221 */ 1222 if (cpu_intr_p() || pm == pmap_kernel()) 1223 return; 1224 1225 l1 = pm->pm_l1; 1226 1227 /* 1228 * If the L1 is not currently on the LRU list, just return 1229 */ 1230 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1231 return; 1232 1233 mutex_spin_enter(&l1_lru_lock); 1234 1235 /* 1236 * Check the use count again, now that we've acquired the lock 1237 */ 1238 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1239 mutex_spin_exit(&l1_lru_lock); 1240 return; 1241 } 1242 1243 /* 1244 * Move the L1 to the back of the LRU list 1245 */ 1246 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1247 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1248 1249 mutex_spin_exit(&l1_lru_lock); 1250 } 1251 1252 /* 1253 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1254 * 1255 * Free an L2 descriptor table. 1256 */ 1257 static inline void 1258 #ifndef PMAP_INCLUDE_PTE_SYNC 1259 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1260 #else 1261 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1262 #endif 1263 { 1264 #ifdef PMAP_INCLUDE_PTE_SYNC 1265 #ifdef PMAP_CACHE_VIVT 1266 /* 1267 * Note: With a write-back cache, we may need to sync this 1268 * L2 table before re-using it. 1269 * This is because it may have belonged to a non-current 1270 * pmap, in which case the cache syncs would have been 1271 * skipped for the pages that were being unmapped. If the 1272 * L2 table were then to be immediately re-allocated to 1273 * the *current* pmap, it may well contain stale mappings 1274 * which have not yet been cleared by a cache write-back 1275 * and so would still be visible to the mmu. 1276 */ 1277 if (need_sync) 1278 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1279 #endif /* PMAP_CACHE_VIVT */ 1280 #endif /* PMAP_INCLUDE_PTE_SYNC */ 1281 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1282 } 1283 1284 /* 1285 * Returns a pointer to the L2 bucket associated with the specified pmap 1286 * and VA, or NULL if no L2 bucket exists for the address. 1287 */ 1288 static inline struct l2_bucket * 1289 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1290 { 1291 struct l2_dtable *l2; 1292 struct l2_bucket *l2b; 1293 u_short l1idx; 1294 1295 l1idx = L1_IDX(va); 1296 1297 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 1298 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 1299 return (NULL); 1300 1301 return (l2b); 1302 } 1303 1304 /* 1305 * Returns a pointer to the L2 bucket associated with the specified pmap 1306 * and VA. 1307 * 1308 * If no L2 bucket exists, perform the necessary allocations to put an L2 1309 * bucket/page table in place. 1310 * 1311 * Note that if a new L2 bucket/page was allocated, the caller *must* 1312 * increment the bucket occupancy counter appropriately *before* 1313 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1314 * the bucket/page in the meantime. 1315 */ 1316 static struct l2_bucket * 1317 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1318 { 1319 struct l2_dtable *l2; 1320 struct l2_bucket *l2b; 1321 u_short l1idx; 1322 1323 l1idx = L1_IDX(va); 1324 1325 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 1326 /* 1327 * No mapping at this address, as there is 1328 * no entry in the L1 table. 1329 * Need to allocate a new l2_dtable. 1330 */ 1331 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1332 return (NULL); 1333 1334 /* 1335 * Link it into the parent pmap 1336 */ 1337 pm->pm_l2[L2_IDX(l1idx)] = l2; 1338 } 1339 1340 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 1341 1342 /* 1343 * Fetch pointer to the L2 page table associated with the address. 1344 */ 1345 if (l2b->l2b_kva == NULL) { 1346 pt_entry_t *ptep; 1347 1348 /* 1349 * No L2 page table has been allocated. Chances are, this 1350 * is because we just allocated the l2_dtable, above. 1351 */ 1352 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_phys)) == NULL) { 1353 /* 1354 * Oops, no more L2 page tables available at this 1355 * time. We may need to deallocate the l2_dtable 1356 * if we allocated a new one above. 1357 */ 1358 if (l2->l2_occupancy == 0) { 1359 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1360 pmap_free_l2_dtable(l2); 1361 } 1362 return (NULL); 1363 } 1364 1365 l2->l2_occupancy++; 1366 l2b->l2b_kva = ptep; 1367 l2b->l2b_l1idx = l1idx; 1368 } 1369 1370 return (l2b); 1371 } 1372 1373 /* 1374 * One or more mappings in the specified L2 descriptor table have just been 1375 * invalidated. 1376 * 1377 * Garbage collect the metadata and descriptor table itself if necessary. 1378 * 1379 * The pmap lock must be acquired when this is called (not necessary 1380 * for the kernel pmap). 1381 */ 1382 static void 1383 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1384 { 1385 struct l2_dtable *l2; 1386 pd_entry_t *pl1pd, l1pd; 1387 pt_entry_t *ptep; 1388 u_short l1idx; 1389 1390 KDASSERT(count <= l2b->l2b_occupancy); 1391 1392 /* 1393 * Update the bucket's reference count according to how many 1394 * PTEs the caller has just invalidated. 1395 */ 1396 l2b->l2b_occupancy -= count; 1397 1398 /* 1399 * Note: 1400 * 1401 * Level 2 page tables allocated to the kernel pmap are never freed 1402 * as that would require checking all Level 1 page tables and 1403 * removing any references to the Level 2 page table. See also the 1404 * comment elsewhere about never freeing bootstrap L2 descriptors. 1405 * 1406 * We make do with just invalidating the mapping in the L2 table. 1407 * 1408 * This isn't really a big deal in practice and, in fact, leads 1409 * to a performance win over time as we don't need to continually 1410 * alloc/free. 1411 */ 1412 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1413 return; 1414 1415 /* 1416 * There are no more valid mappings in this level 2 page table. 1417 * Go ahead and NULL-out the pointer in the bucket, then 1418 * free the page table. 1419 */ 1420 l1idx = l2b->l2b_l1idx; 1421 ptep = l2b->l2b_kva; 1422 l2b->l2b_kva = NULL; 1423 1424 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1425 1426 /* 1427 * If the L1 slot matches the pmap's domain 1428 * number, then invalidate it. 1429 */ 1430 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 1431 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 1432 *pl1pd = 0; 1433 PTE_SYNC(pl1pd); 1434 } 1435 1436 /* 1437 * Release the L2 descriptor table back to the pool cache. 1438 */ 1439 #ifndef PMAP_INCLUDE_PTE_SYNC 1440 pmap_free_l2_ptp(ptep, l2b->l2b_phys); 1441 #else 1442 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_phys); 1443 #endif 1444 1445 /* 1446 * Update the reference count in the associated l2_dtable 1447 */ 1448 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1449 if (--l2->l2_occupancy > 0) 1450 return; 1451 1452 /* 1453 * There are no more valid mappings in any of the Level 1 1454 * slots managed by this l2_dtable. Go ahead and NULL-out 1455 * the pointer in the parent pmap and free the l2_dtable. 1456 */ 1457 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1458 pmap_free_l2_dtable(l2); 1459 } 1460 1461 /* 1462 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1463 * structures. 1464 */ 1465 static int 1466 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1467 { 1468 #ifndef PMAP_INCLUDE_PTE_SYNC 1469 struct l2_bucket *l2b; 1470 pt_entry_t *ptep, pte; 1471 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1472 1473 /* 1474 * The mappings for these page tables were initially made using 1475 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1476 * mode will not be right for page table mappings. To avoid 1477 * polluting the pmap_kenter_pa() code with a special case for 1478 * page tables, we simply fix up the cache-mode here if it's not 1479 * correct. 1480 */ 1481 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1482 KDASSERT(l2b != NULL); 1483 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1484 pte = *ptep; 1485 1486 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1487 /* 1488 * Page tables must have the cache-mode set to Write-Thru. 1489 */ 1490 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1491 PTE_SYNC(ptep); 1492 cpu_tlb_flushD_SE(va); 1493 cpu_cpwait(); 1494 } 1495 #endif 1496 1497 memset(v, 0, L2_TABLE_SIZE_REAL); 1498 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1499 return (0); 1500 } 1501 1502 static int 1503 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1504 { 1505 1506 memset(v, 0, sizeof(struct l2_dtable)); 1507 return (0); 1508 } 1509 1510 static int 1511 pmap_pmap_ctor(void *arg, void *v, int flags) 1512 { 1513 1514 memset(v, 0, sizeof(struct pmap)); 1515 return (0); 1516 } 1517 1518 static void 1519 pmap_pinit(pmap_t pm) 1520 { 1521 struct l2_bucket *l2b; 1522 1523 if (vector_page < KERNEL_BASE) { 1524 /* 1525 * Map the vector page. 1526 */ 1527 pmap_enter(pm, vector_page, systempage.pv_pa, 1528 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1529 pmap_update(pm); 1530 1531 pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 1532 l2b = pmap_get_l2_bucket(pm, vector_page); 1533 KDASSERT(l2b != NULL); 1534 pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO | 1535 L1_C_DOM(pm->pm_domain); 1536 } else 1537 pm->pm_pl1vec = NULL; 1538 } 1539 1540 #ifdef PMAP_CACHE_VIVT 1541 /* 1542 * Since we have a virtually indexed cache, we may need to inhibit caching if 1543 * there is more than one mapping and at least one of them is writable. 1544 * Since we purge the cache on every context switch, we only need to check for 1545 * other mappings within the same pmap, or kernel_pmap. 1546 * This function is also called when a page is unmapped, to possibly reenable 1547 * caching on any remaining mappings. 1548 * 1549 * The code implements the following logic, where: 1550 * 1551 * KW = # of kernel read/write pages 1552 * KR = # of kernel read only pages 1553 * UW = # of user read/write pages 1554 * UR = # of user read only pages 1555 * 1556 * KC = kernel mapping is cacheable 1557 * UC = user mapping is cacheable 1558 * 1559 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1560 * +--------------------------------------------- 1561 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1562 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1563 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1564 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1565 */ 1566 1567 static const int pmap_vac_flags[4][4] = { 1568 {-1, 0, 0, PVF_KNC}, 1569 {0, 0, PVF_NC, PVF_NC}, 1570 {0, PVF_NC, PVF_NC, PVF_NC}, 1571 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1572 }; 1573 1574 static inline int 1575 pmap_get_vac_flags(const struct vm_page_md *md) 1576 { 1577 int kidx, uidx; 1578 1579 kidx = 0; 1580 if (md->kro_mappings || md->krw_mappings > 1) 1581 kidx |= 1; 1582 if (md->krw_mappings) 1583 kidx |= 2; 1584 1585 uidx = 0; 1586 if (md->uro_mappings || md->urw_mappings > 1) 1587 uidx |= 1; 1588 if (md->urw_mappings) 1589 uidx |= 2; 1590 1591 return (pmap_vac_flags[uidx][kidx]); 1592 } 1593 1594 static inline void 1595 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1596 { 1597 int nattr; 1598 1599 nattr = pmap_get_vac_flags(md); 1600 1601 if (nattr < 0) { 1602 md->pvh_attrs &= ~PVF_NC; 1603 return; 1604 } 1605 1606 if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0) 1607 return; 1608 1609 if (pm == pmap_kernel()) 1610 pmap_vac_me_kpmap(md, pa, pm, va); 1611 else 1612 pmap_vac_me_user(md, pa, pm, va); 1613 1614 md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr; 1615 } 1616 1617 static void 1618 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1619 { 1620 u_int u_cacheable, u_entries; 1621 struct pv_entry *pv; 1622 pmap_t last_pmap = pm; 1623 1624 /* 1625 * Pass one, see if there are both kernel and user pmaps for 1626 * this page. Calculate whether there are user-writable or 1627 * kernel-writable pages. 1628 */ 1629 u_cacheable = 0; 1630 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1631 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1632 u_cacheable++; 1633 } 1634 1635 u_entries = md->urw_mappings + md->uro_mappings; 1636 1637 /* 1638 * We know we have just been updating a kernel entry, so if 1639 * all user pages are already cacheable, then there is nothing 1640 * further to do. 1641 */ 1642 if (md->k_mappings == 0 && u_cacheable == u_entries) 1643 return; 1644 1645 if (u_entries) { 1646 /* 1647 * Scan over the list again, for each entry, if it 1648 * might not be set correctly, call pmap_vac_me_user 1649 * to recalculate the settings. 1650 */ 1651 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1652 /* 1653 * We know kernel mappings will get set 1654 * correctly in other calls. We also know 1655 * that if the pmap is the same as last_pmap 1656 * then we've just handled this entry. 1657 */ 1658 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1659 continue; 1660 1661 /* 1662 * If there are kernel entries and this page 1663 * is writable but non-cacheable, then we can 1664 * skip this entry also. 1665 */ 1666 if (md->k_mappings && 1667 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1668 (PVF_NC | PVF_WRITE)) 1669 continue; 1670 1671 /* 1672 * Similarly if there are no kernel-writable 1673 * entries and the page is already 1674 * read-only/cacheable. 1675 */ 1676 if (md->krw_mappings == 0 && 1677 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1678 continue; 1679 1680 /* 1681 * For some of the remaining cases, we know 1682 * that we must recalculate, but for others we 1683 * can't tell if they are correct or not, so 1684 * we recalculate anyway. 1685 */ 1686 pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0); 1687 } 1688 1689 if (md->k_mappings == 0) 1690 return; 1691 } 1692 1693 pmap_vac_me_user(md, pa, pm, va); 1694 } 1695 1696 static void 1697 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1698 { 1699 pmap_t kpmap = pmap_kernel(); 1700 struct pv_entry *pv, *npv = NULL; 1701 struct l2_bucket *l2b; 1702 pt_entry_t *ptep, pte; 1703 u_int entries = 0; 1704 u_int writable = 0; 1705 u_int cacheable_entries = 0; 1706 u_int kern_cacheable = 0; 1707 u_int other_writable = 0; 1708 1709 /* 1710 * Count mappings and writable mappings in this pmap. 1711 * Include kernel mappings as part of our own. 1712 * Keep a pointer to the first one. 1713 */ 1714 npv = NULL; 1715 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1716 /* Count mappings in the same pmap */ 1717 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1718 if (entries++ == 0) 1719 npv = pv; 1720 1721 /* Cacheable mappings */ 1722 if ((pv->pv_flags & PVF_NC) == 0) { 1723 cacheable_entries++; 1724 if (kpmap == pv->pv_pmap) 1725 kern_cacheable++; 1726 } 1727 1728 /* Writable mappings */ 1729 if (pv->pv_flags & PVF_WRITE) 1730 ++writable; 1731 } else 1732 if (pv->pv_flags & PVF_WRITE) 1733 other_writable = 1; 1734 } 1735 1736 /* 1737 * Enable or disable caching as necessary. 1738 * Note: the first entry might be part of the kernel pmap, 1739 * so we can't assume this is indicative of the state of the 1740 * other (maybe non-kpmap) entries. 1741 */ 1742 if ((entries > 1 && writable) || 1743 (entries > 0 && pm == kpmap && other_writable)) { 1744 if (cacheable_entries == 0) 1745 return; 1746 1747 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1748 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1749 (pv->pv_flags & PVF_NC)) 1750 continue; 1751 1752 pv->pv_flags |= PVF_NC; 1753 1754 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1755 KDASSERT(l2b != NULL); 1756 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1757 pte = *ptep & ~L2_S_CACHE_MASK; 1758 1759 if ((va != pv->pv_va || pm != pv->pv_pmap) && 1760 l2pte_valid(pte)) { 1761 if (PV_BEEN_EXECD(pv->pv_flags)) { 1762 #ifdef PMAP_CACHE_VIVT 1763 pmap_idcache_wbinv_range(pv->pv_pmap, 1764 pv->pv_va, PAGE_SIZE); 1765 #endif 1766 pmap_tlb_flushID_SE(pv->pv_pmap, 1767 pv->pv_va); 1768 } else 1769 if (PV_BEEN_REFD(pv->pv_flags)) { 1770 #ifdef PMAP_CACHE_VIVT 1771 pmap_dcache_wb_range(pv->pv_pmap, 1772 pv->pv_va, PAGE_SIZE, true, 1773 (pv->pv_flags & PVF_WRITE) == 0); 1774 #endif 1775 pmap_tlb_flushD_SE(pv->pv_pmap, 1776 pv->pv_va); 1777 } 1778 } 1779 1780 *ptep = pte; 1781 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1782 } 1783 cpu_cpwait(); 1784 } else 1785 if (entries > cacheable_entries) { 1786 /* 1787 * Turn cacheing back on for some pages. If it is a kernel 1788 * page, only do so if there are no other writable pages. 1789 */ 1790 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1791 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1792 (kpmap != pv->pv_pmap || other_writable))) 1793 continue; 1794 1795 pv->pv_flags &= ~PVF_NC; 1796 1797 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1798 KDASSERT(l2b != NULL); 1799 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1800 pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; 1801 1802 if (l2pte_valid(pte)) { 1803 if (PV_BEEN_EXECD(pv->pv_flags)) { 1804 pmap_tlb_flushID_SE(pv->pv_pmap, 1805 pv->pv_va); 1806 } else 1807 if (PV_BEEN_REFD(pv->pv_flags)) { 1808 pmap_tlb_flushD_SE(pv->pv_pmap, 1809 pv->pv_va); 1810 } 1811 } 1812 1813 *ptep = pte; 1814 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1815 } 1816 } 1817 } 1818 #endif 1819 1820 #ifdef PMAP_CACHE_VIPT 1821 static void 1822 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1823 { 1824 struct pv_entry *pv; 1825 vaddr_t tst_mask; 1826 bool bad_alias; 1827 struct l2_bucket *l2b; 1828 pt_entry_t *ptep, pte, opte; 1829 const u_int 1830 rw_mappings = md->urw_mappings + md->krw_mappings, 1831 ro_mappings = md->uro_mappings + md->kro_mappings; 1832 1833 /* do we need to do anything? */ 1834 if (arm_cache_prefer_mask == 0) 1835 return; 1836 1837 NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: md=%p, pmap=%p va=%08lx\n", 1838 md, pm, va)); 1839 1840 KASSERT(!va || pm); 1841 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1842 1843 /* Already a conflict? */ 1844 if (__predict_false(md->pvh_attrs & PVF_NC)) { 1845 /* just an add, things are already non-cached */ 1846 KASSERT(!(md->pvh_attrs & PVF_DIRTY)); 1847 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 1848 bad_alias = false; 1849 if (va) { 1850 PMAPCOUNT(vac_color_none); 1851 bad_alias = true; 1852 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 1853 goto fixup; 1854 } 1855 pv = SLIST_FIRST(&md->pvh_list); 1856 /* the list can't be empty because it would be cachable */ 1857 if (md->pvh_attrs & PVF_KMPAGE) { 1858 tst_mask = md->pvh_attrs; 1859 } else { 1860 KASSERT(pv); 1861 tst_mask = pv->pv_va; 1862 pv = SLIST_NEXT(pv, pv_link); 1863 } 1864 /* 1865 * Only check for a bad alias if we have writable mappings. 1866 */ 1867 tst_mask &= arm_cache_prefer_mask; 1868 if (rw_mappings > 0) { 1869 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 1870 /* if there's a bad alias, stop checking. */ 1871 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 1872 bad_alias = true; 1873 } 1874 md->pvh_attrs |= PVF_WRITE; 1875 if (!bad_alias) 1876 md->pvh_attrs |= PVF_DIRTY; 1877 } else { 1878 /* 1879 * We have only read-only mappings. Let's see if there 1880 * are multiple colors in use or if we mapped a KMPAGE. 1881 * If the latter, we have a bad alias. If the former, 1882 * we need to remember that. 1883 */ 1884 for (; pv; pv = SLIST_NEXT(pv, pv_link)) { 1885 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) { 1886 if (md->pvh_attrs & PVF_KMPAGE) 1887 bad_alias = true; 1888 break; 1889 } 1890 } 1891 md->pvh_attrs &= ~PVF_WRITE; 1892 /* 1893 * No KMPAGE and we exited early, so we must have 1894 * multiple color mappings. 1895 */ 1896 if (!bad_alias && pv != NULL) 1897 md->pvh_attrs |= PVF_MULTCLR; 1898 } 1899 1900 /* If no conflicting colors, set everything back to cached */ 1901 if (!bad_alias) { 1902 #ifdef DEBUG 1903 if ((md->pvh_attrs & PVF_WRITE) 1904 || ro_mappings < 2) { 1905 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 1906 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 1907 } 1908 #endif 1909 md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 1910 md->pvh_attrs |= tst_mask | PVF_COLORED; 1911 /* 1912 * Restore DIRTY bit if page is modified 1913 */ 1914 if (md->pvh_attrs & PVF_DMOD) 1915 md->pvh_attrs |= PVF_DIRTY; 1916 PMAPCOUNT(vac_color_restore); 1917 } else { 1918 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 1919 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 1920 } 1921 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1922 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 1923 } else if (!va) { 1924 KASSERT(pmap_is_page_colored_p(md)); 1925 KASSERT(!(md->pvh_attrs & PVF_WRITE) 1926 || (md->pvh_attrs & PVF_DIRTY)); 1927 if (rw_mappings == 0) { 1928 md->pvh_attrs &= ~PVF_WRITE; 1929 if (ro_mappings == 1 1930 && (md->pvh_attrs & PVF_MULTCLR)) { 1931 /* 1932 * If this is the last readonly mapping 1933 * but it doesn't match the current color 1934 * for the page, change the current color 1935 * to match this last readonly mapping. 1936 */ 1937 pv = SLIST_FIRST(&md->pvh_list); 1938 tst_mask = (md->pvh_attrs ^ pv->pv_va) 1939 & arm_cache_prefer_mask; 1940 if (tst_mask) { 1941 md->pvh_attrs ^= tst_mask; 1942 PMAPCOUNT(vac_color_change); 1943 } 1944 } 1945 } 1946 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1947 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 1948 return; 1949 } else if (!pmap_is_page_colored_p(md)) { 1950 /* not colored so we just use its color */ 1951 KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 1952 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 1953 PMAPCOUNT(vac_color_new); 1954 md->pvh_attrs &= PAGE_SIZE - 1; 1955 md->pvh_attrs |= PVF_COLORED 1956 | (va & arm_cache_prefer_mask) 1957 | (rw_mappings > 0 ? PVF_WRITE : 0); 1958 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1959 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 1960 return; 1961 } else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 1962 bad_alias = false; 1963 if (rw_mappings > 0) { 1964 /* 1965 * We now have writeable mappings and if we have 1966 * readonly mappings in more than once color, we have 1967 * an aliasing problem. Regardless mark the page as 1968 * writeable. 1969 */ 1970 if (md->pvh_attrs & PVF_MULTCLR) { 1971 if (ro_mappings < 2) { 1972 /* 1973 * If we only have less than two 1974 * read-only mappings, just flush the 1975 * non-primary colors from the cache. 1976 */ 1977 pmap_flush_page(md, pa, 1978 PMAP_FLUSH_SECONDARY); 1979 } else { 1980 bad_alias = true; 1981 } 1982 } 1983 md->pvh_attrs |= PVF_WRITE; 1984 } 1985 /* If no conflicting colors, set everything back to cached */ 1986 if (!bad_alias) { 1987 #ifdef DEBUG 1988 if (rw_mappings > 0 1989 || (md->pvh_attrs & PMAP_KMPAGE)) { 1990 tst_mask = md->pvh_attrs & arm_cache_prefer_mask; 1991 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 1992 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 1993 } 1994 #endif 1995 if (SLIST_EMPTY(&md->pvh_list)) 1996 PMAPCOUNT(vac_color_reuse); 1997 else 1998 PMAPCOUNT(vac_color_ok); 1999 2000 /* matching color, just return */ 2001 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2002 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2003 return; 2004 } 2005 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2006 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2007 2008 /* color conflict. evict from cache. */ 2009 2010 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2011 md->pvh_attrs &= ~PVF_COLORED; 2012 md->pvh_attrs |= PVF_NC; 2013 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2014 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2015 PMAPCOUNT(vac_color_erase); 2016 } else if (rw_mappings == 0 2017 && (md->pvh_attrs & PVF_KMPAGE) == 0) { 2018 KASSERT((md->pvh_attrs & PVF_WRITE) == 0); 2019 2020 /* 2021 * If the page has dirty cache lines, clean it. 2022 */ 2023 if (md->pvh_attrs & PVF_DIRTY) 2024 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 2025 2026 /* 2027 * If this is the first remapping (we know that there are no 2028 * writeable mappings), then this is a simple color change. 2029 * Otherwise this is a seconary r/o mapping, which means 2030 * we don't have to do anything. 2031 */ 2032 if (ro_mappings == 1) { 2033 KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2034 md->pvh_attrs &= PAGE_SIZE - 1; 2035 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2036 PMAPCOUNT(vac_color_change); 2037 } else { 2038 PMAPCOUNT(vac_color_blind); 2039 } 2040 md->pvh_attrs |= PVF_MULTCLR; 2041 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2042 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2043 return; 2044 } else { 2045 if (rw_mappings > 0) 2046 md->pvh_attrs |= PVF_WRITE; 2047 2048 /* color conflict. evict from cache. */ 2049 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2050 2051 /* the list can't be empty because this was a enter/modify */ 2052 pv = SLIST_FIRST(&md->pvh_list); 2053 if ((md->pvh_attrs & PVF_KMPAGE) == 0) { 2054 KASSERT(pv); 2055 /* 2056 * If there's only one mapped page, change color to the 2057 * page's new color and return. Restore the DIRTY bit 2058 * that was erased by pmap_flush_page. 2059 */ 2060 if (SLIST_NEXT(pv, pv_link) == NULL) { 2061 md->pvh_attrs &= PAGE_SIZE - 1; 2062 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2063 if (md->pvh_attrs & PVF_DMOD) 2064 md->pvh_attrs |= PVF_DIRTY; 2065 PMAPCOUNT(vac_color_change); 2066 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2067 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2068 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2069 return; 2070 } 2071 } 2072 bad_alias = true; 2073 md->pvh_attrs &= ~PVF_COLORED; 2074 md->pvh_attrs |= PVF_NC; 2075 PMAPCOUNT(vac_color_erase); 2076 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2077 } 2078 2079 fixup: 2080 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2081 2082 /* 2083 * Turn cacheing on/off for all pages. 2084 */ 2085 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2086 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2087 KDASSERT(l2b != NULL); 2088 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2089 opte = *ptep; 2090 pte = opte & ~L2_S_CACHE_MASK; 2091 if (bad_alias) { 2092 pv->pv_flags |= PVF_NC; 2093 } else { 2094 pv->pv_flags &= ~PVF_NC; 2095 pte |= pte_l2_s_cache_mode; 2096 } 2097 2098 if (opte == pte) /* only update is there's a change */ 2099 continue; 2100 2101 if (l2pte_valid(pte)) { 2102 if (PV_BEEN_EXECD(pv->pv_flags)) { 2103 pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); 2104 } else if (PV_BEEN_REFD(pv->pv_flags)) { 2105 pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); 2106 } 2107 } 2108 2109 *ptep = pte; 2110 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2111 } 2112 } 2113 #endif /* PMAP_CACHE_VIPT */ 2114 2115 2116 /* 2117 * Modify pte bits for all ptes corresponding to the given physical address. 2118 * We use `maskbits' rather than `clearbits' because we're always passing 2119 * constants and the latter would require an extra inversion at run-time. 2120 */ 2121 static void 2122 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits) 2123 { 2124 struct l2_bucket *l2b; 2125 struct pv_entry *pv; 2126 pt_entry_t *ptep, npte, opte; 2127 pmap_t pm; 2128 vaddr_t va; 2129 u_int oflags; 2130 #ifdef PMAP_CACHE_VIPT 2131 const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs); 2132 bool need_syncicache = false; 2133 bool did_syncicache = false; 2134 bool need_vac_me_harder = false; 2135 #endif 2136 2137 NPDEBUG(PDB_BITS, 2138 printf("pmap_clearbit: md %p mask 0x%x\n", 2139 md, maskbits)); 2140 2141 #ifdef PMAP_CACHE_VIPT 2142 /* 2143 * If we might want to sync the I-cache and we've modified it, 2144 * then we know we definitely need to sync or discard it. 2145 */ 2146 if (want_syncicache) 2147 need_syncicache = md->pvh_attrs & PVF_MOD; 2148 #endif 2149 /* 2150 * Clear saved attributes (modify, reference) 2151 */ 2152 md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2153 2154 if (SLIST_EMPTY(&md->pvh_list)) { 2155 #ifdef PMAP_CACHE_VIPT 2156 if (need_syncicache) { 2157 /* 2158 * No one has it mapped, so just discard it. The next 2159 * exec remapping will cause it to be synced. 2160 */ 2161 md->pvh_attrs &= ~PVF_EXEC; 2162 PMAPCOUNT(exec_discarded_clearbit); 2163 } 2164 #endif 2165 return; 2166 } 2167 2168 /* 2169 * Loop over all current mappings setting/clearing as appropos 2170 */ 2171 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2172 va = pv->pv_va; 2173 pm = pv->pv_pmap; 2174 oflags = pv->pv_flags; 2175 /* 2176 * Kernel entries are unmanaged and as such not to be changed. 2177 */ 2178 if (oflags & PVF_KENTRY) 2179 continue; 2180 pv->pv_flags &= ~maskbits; 2181 2182 pmap_acquire_pmap_lock(pm); 2183 2184 l2b = pmap_get_l2_bucket(pm, va); 2185 KDASSERT(l2b != NULL); 2186 2187 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2188 npte = opte = *ptep; 2189 2190 NPDEBUG(PDB_BITS, 2191 printf( 2192 "pmap_clearbit: pv %p, pm %p, va 0x%08lx, flag 0x%x\n", 2193 pv, pv->pv_pmap, pv->pv_va, oflags)); 2194 2195 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2196 #ifdef PMAP_CACHE_VIVT 2197 if ((pv->pv_flags & PVF_NC)) { 2198 /* 2199 * Entry is not cacheable: 2200 * 2201 * Don't turn caching on again if this is a 2202 * modified emulation. This would be 2203 * inconsitent with the settings created by 2204 * pmap_vac_me_harder(). Otherwise, it's safe 2205 * to re-enable cacheing. 2206 * 2207 * There's no need to call pmap_vac_me_harder() 2208 * here: all pages are losing their write 2209 * permission. 2210 */ 2211 if (maskbits & PVF_WRITE) { 2212 npte |= pte_l2_s_cache_mode; 2213 pv->pv_flags &= ~PVF_NC; 2214 } 2215 } else 2216 if (l2pte_writable_p(opte)) { 2217 /* 2218 * Entry is writable/cacheable: check if pmap 2219 * is current if it is flush it, otherwise it 2220 * won't be in the cache 2221 */ 2222 if (PV_BEEN_EXECD(oflags)) 2223 pmap_idcache_wbinv_range(pm, pv->pv_va, 2224 PAGE_SIZE); 2225 else 2226 if (PV_BEEN_REFD(oflags)) 2227 pmap_dcache_wb_range(pm, pv->pv_va, 2228 PAGE_SIZE, 2229 (maskbits & PVF_REF) != 0, false); 2230 } 2231 #endif 2232 2233 /* make the pte read only */ 2234 npte = l2pte_set_readonly(npte); 2235 2236 if (maskbits & oflags & PVF_WRITE) { 2237 /* 2238 * Keep alias accounting up to date 2239 */ 2240 if (pv->pv_pmap == pmap_kernel()) { 2241 md->krw_mappings--; 2242 md->kro_mappings++; 2243 } else { 2244 md->urw_mappings--; 2245 md->uro_mappings++; 2246 } 2247 #ifdef PMAP_CACHE_VIPT 2248 if (arm_cache_prefer_mask != 0) { 2249 if (md->urw_mappings + md->krw_mappings == 0) { 2250 md->pvh_attrs &= ~PVF_WRITE; 2251 } else { 2252 PMAP_VALIDATE_MD_PAGE(md); 2253 } 2254 } 2255 if (want_syncicache) 2256 need_syncicache = true; 2257 need_vac_me_harder = true; 2258 #endif 2259 } 2260 } 2261 2262 if (maskbits & PVF_REF) { 2263 if ((pv->pv_flags & PVF_NC) == 0 && 2264 (maskbits & (PVF_WRITE|PVF_MOD)) == 0 && 2265 l2pte_valid(npte)) { 2266 #ifdef PMAP_CACHE_VIVT 2267 /* 2268 * Check npte here; we may have already 2269 * done the wbinv above, and the validity 2270 * of the PTE is the same for opte and 2271 * npte. 2272 */ 2273 /* XXXJRT need idcache_inv_range */ 2274 if (PV_BEEN_EXECD(oflags)) 2275 pmap_idcache_wbinv_range(pm, 2276 pv->pv_va, PAGE_SIZE); 2277 else 2278 if (PV_BEEN_REFD(oflags)) 2279 pmap_dcache_wb_range(pm, 2280 pv->pv_va, PAGE_SIZE, 2281 true, true); 2282 #endif 2283 } 2284 2285 /* 2286 * Make the PTE invalid so that we will take a 2287 * page fault the next time the mapping is 2288 * referenced. 2289 */ 2290 npte &= ~L2_TYPE_MASK; 2291 npte |= L2_TYPE_INV; 2292 } 2293 2294 if (npte != opte) { 2295 *ptep = npte; 2296 PTE_SYNC(ptep); 2297 /* Flush the TLB entry if a current pmap. */ 2298 if (PV_BEEN_EXECD(oflags)) 2299 pmap_tlb_flushID_SE(pm, pv->pv_va); 2300 else 2301 if (PV_BEEN_REFD(oflags)) 2302 pmap_tlb_flushD_SE(pm, pv->pv_va); 2303 } 2304 2305 pmap_release_pmap_lock(pm); 2306 2307 NPDEBUG(PDB_BITS, 2308 printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n", 2309 pm, va, opte, npte)); 2310 } 2311 2312 #ifdef PMAP_CACHE_VIPT 2313 /* 2314 * If we need to sync the I-cache and we haven't done it yet, do it. 2315 */ 2316 if (need_syncicache && !did_syncicache) { 2317 pmap_syncicache_page(md, pa); 2318 PMAPCOUNT(exec_synced_clearbit); 2319 } 2320 /* 2321 * If we are changing this to read-only, we need to call vac_me_harder 2322 * so we can change all the read-only pages to cacheable. We pretend 2323 * this as a page deletion. 2324 */ 2325 if (need_vac_me_harder) { 2326 if (md->pvh_attrs & PVF_NC) 2327 pmap_vac_me_harder(md, pa, NULL, 0); 2328 } 2329 #endif 2330 } 2331 2332 /* 2333 * pmap_clean_page() 2334 * 2335 * This is a local function used to work out the best strategy to clean 2336 * a single page referenced by its entry in the PV table. It's used by 2337 * pmap_copy_page, pmap_zero page and maybe some others later on. 2338 * 2339 * Its policy is effectively: 2340 * o If there are no mappings, we don't bother doing anything with the cache. 2341 * o If there is one mapping, we clean just that page. 2342 * o If there are multiple mappings, we clean the entire cache. 2343 * 2344 * So that some functions can be further optimised, it returns 0 if it didn't 2345 * clean the entire cache, or 1 if it did. 2346 * 2347 * XXX One bug in this routine is that if the pv_entry has a single page 2348 * mapped at 0x00000000 a whole cache clean will be performed rather than 2349 * just the 1 page. Since this should not occur in everyday use and if it does 2350 * it will just result in not the most efficient clean for the page. 2351 */ 2352 #ifdef PMAP_CACHE_VIVT 2353 static int 2354 pmap_clean_page(struct pv_entry *pv, bool is_src) 2355 { 2356 pmap_t pm_to_clean = NULL; 2357 struct pv_entry *npv; 2358 u_int cache_needs_cleaning = 0; 2359 u_int flags = 0; 2360 vaddr_t page_to_clean = 0; 2361 2362 if (pv == NULL) { 2363 /* nothing mapped in so nothing to flush */ 2364 return (0); 2365 } 2366 2367 /* 2368 * Since we flush the cache each time we change to a different 2369 * user vmspace, we only need to flush the page if it is in the 2370 * current pmap. 2371 */ 2372 2373 for (npv = pv; npv; npv = SLIST_NEXT(npv, pv_link)) { 2374 if (pmap_is_current(npv->pv_pmap)) { 2375 flags |= npv->pv_flags; 2376 /* 2377 * The page is mapped non-cacheable in 2378 * this map. No need to flush the cache. 2379 */ 2380 if (npv->pv_flags & PVF_NC) { 2381 #ifdef DIAGNOSTIC 2382 if (cache_needs_cleaning) 2383 panic("pmap_clean_page: " 2384 "cache inconsistency"); 2385 #endif 2386 break; 2387 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 2388 continue; 2389 if (cache_needs_cleaning) { 2390 page_to_clean = 0; 2391 break; 2392 } else { 2393 page_to_clean = npv->pv_va; 2394 pm_to_clean = npv->pv_pmap; 2395 } 2396 cache_needs_cleaning = 1; 2397 } 2398 } 2399 2400 if (page_to_clean) { 2401 if (PV_BEEN_EXECD(flags)) 2402 pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, 2403 PAGE_SIZE); 2404 else 2405 pmap_dcache_wb_range(pm_to_clean, page_to_clean, 2406 PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); 2407 } else if (cache_needs_cleaning) { 2408 pmap_t const pm = curproc->p_vmspace->vm_map.pmap; 2409 2410 if (PV_BEEN_EXECD(flags)) 2411 pmap_idcache_wbinv_all(pm); 2412 else 2413 pmap_dcache_wbinv_all(pm); 2414 return (1); 2415 } 2416 return (0); 2417 } 2418 #endif 2419 2420 #ifdef PMAP_CACHE_VIPT 2421 /* 2422 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2423 * right cache alias to make sure we flush the right stuff. 2424 */ 2425 void 2426 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa) 2427 { 2428 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2429 pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT]; 2430 2431 NPDEBUG(PDB_EXEC, printf("pmap_syncicache_page: md=%p (attrs=%#x)\n", 2432 md, md->pvh_attrs)); 2433 /* 2434 * No need to clean the page if it's non-cached. 2435 */ 2436 if (md->pvh_attrs & PVF_NC) 2437 return; 2438 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED); 2439 2440 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2441 /* 2442 * Set up a PTE with the right coloring to flush existing cache lines. 2443 */ 2444 *ptep = L2_S_PROTO | 2445 pa 2446 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2447 | pte_l2_s_cache_mode; 2448 PTE_SYNC(ptep); 2449 2450 /* 2451 * Flush it. 2452 */ 2453 cpu_icache_sync_range(cdstp + va_offset, PAGE_SIZE); 2454 /* 2455 * Unmap the page. 2456 */ 2457 *ptep = 0; 2458 PTE_SYNC(ptep); 2459 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2460 2461 md->pvh_attrs |= PVF_EXEC; 2462 PMAPCOUNT(exec_synced); 2463 } 2464 2465 void 2466 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush) 2467 { 2468 vsize_t va_offset, end_va; 2469 void (*cf)(vaddr_t, vsize_t); 2470 2471 if (arm_cache_prefer_mask == 0) 2472 return; 2473 2474 switch (flush) { 2475 case PMAP_FLUSH_PRIMARY: 2476 if (md->pvh_attrs & PVF_MULTCLR) { 2477 va_offset = 0; 2478 end_va = arm_cache_prefer_mask; 2479 md->pvh_attrs &= ~PVF_MULTCLR; 2480 PMAPCOUNT(vac_flush_lots); 2481 } else { 2482 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2483 end_va = va_offset; 2484 PMAPCOUNT(vac_flush_one); 2485 } 2486 /* 2487 * Mark that the page is no longer dirty. 2488 */ 2489 md->pvh_attrs &= ~PVF_DIRTY; 2490 cf = cpufuncs.cf_idcache_wbinv_range; 2491 break; 2492 case PMAP_FLUSH_SECONDARY: 2493 va_offset = 0; 2494 end_va = arm_cache_prefer_mask; 2495 cf = cpufuncs.cf_idcache_wbinv_range; 2496 md->pvh_attrs &= ~PVF_MULTCLR; 2497 PMAPCOUNT(vac_flush_lots); 2498 break; 2499 case PMAP_CLEAN_PRIMARY: 2500 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2501 end_va = va_offset; 2502 cf = cpufuncs.cf_dcache_wb_range; 2503 /* 2504 * Mark that the page is no longer dirty. 2505 */ 2506 if ((md->pvh_attrs & PVF_DMOD) == 0) 2507 md->pvh_attrs &= ~PVF_DIRTY; 2508 PMAPCOUNT(vac_clean_one); 2509 break; 2510 default: 2511 return; 2512 } 2513 2514 KASSERT(!(md->pvh_attrs & PVF_NC)); 2515 2516 NPDEBUG(PDB_VAC, printf("pmap_flush_page: md=%p (attrs=%#x)\n", 2517 md, md->pvh_attrs)); 2518 2519 for (; va_offset <= end_va; va_offset += PAGE_SIZE) { 2520 const size_t pte_offset = va_offset >> PGSHIFT; 2521 pt_entry_t * const ptep = &cdst_pte[pte_offset]; 2522 const pt_entry_t oldpte = *ptep; 2523 2524 if (flush == PMAP_FLUSH_SECONDARY 2525 && va_offset == (md->pvh_attrs & arm_cache_prefer_mask)) 2526 continue; 2527 2528 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2529 /* 2530 * Set up a PTE with the right coloring to flush 2531 * existing cache entries. 2532 */ 2533 *ptep = L2_S_PROTO 2534 | pa 2535 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2536 | pte_l2_s_cache_mode; 2537 PTE_SYNC(ptep); 2538 2539 /* 2540 * Flush it. 2541 */ 2542 (*cf)(cdstp + va_offset, PAGE_SIZE); 2543 2544 /* 2545 * Restore the page table entry since we might have interrupted 2546 * pmap_zero_page or pmap_copy_page which was already using 2547 * this pte. 2548 */ 2549 *ptep = oldpte; 2550 PTE_SYNC(ptep); 2551 pmap_tlb_flushID_SE(pmap_kernel(), cdstp + va_offset); 2552 } 2553 } 2554 #endif /* PMAP_CACHE_VIPT */ 2555 2556 /* 2557 * Routine: pmap_page_remove 2558 * Function: 2559 * Removes this physical page from 2560 * all physical maps in which it resides. 2561 * Reflects back modify bits to the pager. 2562 */ 2563 static void 2564 pmap_page_remove(struct vm_page_md *md, paddr_t pa) 2565 { 2566 struct l2_bucket *l2b; 2567 struct pv_entry *pv, *npv, **pvp; 2568 pmap_t pm; 2569 pt_entry_t *ptep; 2570 bool flush; 2571 u_int flags; 2572 2573 NPDEBUG(PDB_FOLLOW, 2574 printf("pmap_page_remove: md %p (0x%08lx)\n", md, 2575 pa)); 2576 2577 pv = SLIST_FIRST(&md->pvh_list); 2578 if (pv == NULL) { 2579 #ifdef PMAP_CACHE_VIPT 2580 /* 2581 * We *know* the page contents are about to be replaced. 2582 * Discard the exec contents 2583 */ 2584 if (PV_IS_EXEC_P(md->pvh_attrs)) 2585 PMAPCOUNT(exec_discarded_page_protect); 2586 md->pvh_attrs &= ~PVF_EXEC; 2587 PMAP_VALIDATE_MD_PAGE(md); 2588 #endif 2589 return; 2590 } 2591 #ifdef PMAP_CACHE_VIPT 2592 KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md)); 2593 #endif 2594 2595 /* 2596 * Clear alias counts 2597 */ 2598 #ifdef PMAP_CACHE_VIVT 2599 md->k_mappings = 0; 2600 #endif 2601 md->urw_mappings = md->uro_mappings = 0; 2602 2603 flush = false; 2604 flags = 0; 2605 2606 #ifdef PMAP_CACHE_VIVT 2607 pmap_clean_page(pv, false); 2608 #endif 2609 2610 pvp = &SLIST_FIRST(&md->pvh_list); 2611 while (pv) { 2612 pm = pv->pv_pmap; 2613 npv = SLIST_NEXT(pv, pv_link); 2614 if (flush == false && pmap_is_current(pm)) 2615 flush = true; 2616 2617 if (pm == pmap_kernel()) { 2618 #ifdef PMAP_CACHE_VIPT 2619 /* 2620 * If this was unmanaged mapping, it must be preserved. 2621 * Move it back on the list and advance the end-of-list 2622 * pointer. 2623 */ 2624 if (pv->pv_flags & PVF_KENTRY) { 2625 *pvp = pv; 2626 pvp = &SLIST_NEXT(pv, pv_link); 2627 pv = npv; 2628 continue; 2629 } 2630 if (pv->pv_flags & PVF_WRITE) 2631 md->krw_mappings--; 2632 else 2633 md->kro_mappings--; 2634 #endif 2635 PMAPCOUNT(kernel_unmappings); 2636 } 2637 PMAPCOUNT(unmappings); 2638 2639 pmap_acquire_pmap_lock(pm); 2640 2641 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2642 KDASSERT(l2b != NULL); 2643 2644 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2645 2646 /* 2647 * Update statistics 2648 */ 2649 --pm->pm_stats.resident_count; 2650 2651 /* Wired bit */ 2652 if (pv->pv_flags & PVF_WIRED) 2653 --pm->pm_stats.wired_count; 2654 2655 flags |= pv->pv_flags; 2656 2657 /* 2658 * Invalidate the PTEs. 2659 */ 2660 *ptep = 0; 2661 PTE_SYNC_CURRENT(pm, ptep); 2662 pmap_free_l2_bucket(pm, l2b, 1); 2663 2664 pool_put(&pmap_pv_pool, pv); 2665 pv = npv; 2666 /* 2667 * if we reach the end of the list and there are still 2668 * mappings, they might be able to be cached now. 2669 */ 2670 if (pv == NULL) { 2671 *pvp = NULL; 2672 if (!SLIST_EMPTY(&md->pvh_list)) 2673 pmap_vac_me_harder(md, pa, pm, 0); 2674 } 2675 pmap_release_pmap_lock(pm); 2676 } 2677 #ifdef PMAP_CACHE_VIPT 2678 /* 2679 * Its EXEC cache is now gone. 2680 */ 2681 if (PV_IS_EXEC_P(md->pvh_attrs)) 2682 PMAPCOUNT(exec_discarded_page_protect); 2683 md->pvh_attrs &= ~PVF_EXEC; 2684 KASSERT(md->urw_mappings == 0); 2685 KASSERT(md->uro_mappings == 0); 2686 if (arm_cache_prefer_mask != 0) { 2687 if (md->krw_mappings == 0) 2688 md->pvh_attrs &= ~PVF_WRITE; 2689 PMAP_VALIDATE_MD_PAGE(md); 2690 } 2691 #endif 2692 2693 if (flush) { 2694 /* 2695 * Note: We can't use pmap_tlb_flush{I,D}() here since that 2696 * would need a subsequent call to pmap_update() to ensure 2697 * curpm->pm_cstate.cs_all is reset. Our callers are not 2698 * required to do that (see pmap(9)), so we can't modify 2699 * the current pmap's state. 2700 */ 2701 if (PV_BEEN_EXECD(flags)) 2702 cpu_tlb_flushID(); 2703 else 2704 cpu_tlb_flushD(); 2705 } 2706 cpu_cpwait(); 2707 } 2708 2709 /* 2710 * pmap_t pmap_create(void) 2711 * 2712 * Create a new pmap structure from scratch. 2713 */ 2714 pmap_t 2715 pmap_create(void) 2716 { 2717 pmap_t pm; 2718 2719 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 2720 2721 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE); 2722 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 2723 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 2724 2725 pm->pm_stats.wired_count = 0; 2726 pm->pm_stats.resident_count = 1; 2727 pm->pm_cstate.cs_all = 0; 2728 pmap_alloc_l1(pm); 2729 2730 /* 2731 * Note: The pool cache ensures that the pm_l2[] array is already 2732 * initialised to zero. 2733 */ 2734 2735 pmap_pinit(pm); 2736 2737 LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list); 2738 2739 return (pm); 2740 } 2741 2742 u_int 2743 arm32_mmap_flags(paddr_t pa) 2744 { 2745 /* 2746 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff 2747 * and we're using the upper bits in page numbers to pass flags around 2748 * so we might as well use the same bits 2749 */ 2750 return (u_int)pa & PMAP_MD_MASK; 2751 } 2752 /* 2753 * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 2754 * u_int flags) 2755 * 2756 * Insert the given physical page (p) at 2757 * the specified virtual address (v) in the 2758 * target physical map with the protection requested. 2759 * 2760 * NB: This is the only routine which MAY NOT lazy-evaluate 2761 * or lose information. That is, this routine must actually 2762 * insert this page into the given map NOW. 2763 */ 2764 int 2765 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2766 { 2767 struct l2_bucket *l2b; 2768 struct vm_page *pg, *opg; 2769 struct pv_entry *pv; 2770 pt_entry_t *ptep, npte, opte; 2771 u_int nflags; 2772 u_int oflags; 2773 2774 NPDEBUG(PDB_ENTER, printf("pmap_enter: pm %p va 0x%lx pa 0x%lx prot %x flag %x\n", pm, va, pa, prot, flags)); 2775 2776 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 2777 KDASSERT(((va | pa) & PGOFSET) == 0); 2778 2779 /* 2780 * Get a pointer to the page. Later on in this function, we 2781 * test for a managed page by checking pg != NULL. 2782 */ 2783 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2784 2785 nflags = 0; 2786 if (prot & VM_PROT_WRITE) 2787 nflags |= PVF_WRITE; 2788 if (prot & VM_PROT_EXECUTE) 2789 nflags |= PVF_EXEC; 2790 if (flags & PMAP_WIRED) 2791 nflags |= PVF_WIRED; 2792 2793 pmap_acquire_pmap_lock(pm); 2794 2795 /* 2796 * Fetch the L2 bucket which maps this page, allocating one if 2797 * necessary for user pmaps. 2798 */ 2799 if (pm == pmap_kernel()) 2800 l2b = pmap_get_l2_bucket(pm, va); 2801 else 2802 l2b = pmap_alloc_l2_bucket(pm, va); 2803 if (l2b == NULL) { 2804 if (flags & PMAP_CANFAIL) { 2805 pmap_release_pmap_lock(pm); 2806 return (ENOMEM); 2807 } 2808 panic("pmap_enter: failed to allocate L2 bucket"); 2809 } 2810 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2811 opte = *ptep; 2812 npte = pa; 2813 oflags = 0; 2814 2815 if (opte) { 2816 /* 2817 * There is already a mapping at this address. 2818 * If the physical address is different, lookup the 2819 * vm_page. 2820 */ 2821 if (l2pte_pa(opte) != pa) 2822 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 2823 else 2824 opg = pg; 2825 } else 2826 opg = NULL; 2827 2828 if (pg) { 2829 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 2830 2831 /* 2832 * This is to be a managed mapping. 2833 */ 2834 if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) { 2835 /* 2836 * - The access type indicates that we don't need 2837 * to do referenced emulation. 2838 * OR 2839 * - The physical page has already been referenced 2840 * so no need to re-do referenced emulation here. 2841 */ 2842 npte |= l2pte_set_readonly(L2_S_PROTO); 2843 2844 nflags |= PVF_REF; 2845 2846 if ((prot & VM_PROT_WRITE) != 0 && 2847 ((flags & VM_PROT_WRITE) != 0 || 2848 (md->pvh_attrs & PVF_MOD) != 0)) { 2849 /* 2850 * This is a writable mapping, and the 2851 * page's mod state indicates it has 2852 * already been modified. Make it 2853 * writable from the outset. 2854 */ 2855 npte = l2pte_set_writable(npte); 2856 nflags |= PVF_MOD; 2857 } 2858 } else { 2859 /* 2860 * Need to do page referenced emulation. 2861 */ 2862 npte |= L2_TYPE_INV; 2863 } 2864 2865 if (flags & ARM32_MMAP_WRITECOMBINE) { 2866 npte |= pte_l2_s_wc_mode; 2867 } else 2868 npte |= pte_l2_s_cache_mode; 2869 2870 if (pg == opg) { 2871 /* 2872 * We're changing the attrs of an existing mapping. 2873 */ 2874 #ifdef MULTIPROCESSOR 2875 KASSERT(uvm_page_locked_p(pg)); 2876 #endif 2877 oflags = pmap_modify_pv(md, pa, pm, va, 2878 PVF_WRITE | PVF_EXEC | PVF_WIRED | 2879 PVF_MOD | PVF_REF, nflags); 2880 2881 #ifdef PMAP_CACHE_VIVT 2882 /* 2883 * We may need to flush the cache if we're 2884 * doing rw-ro... 2885 */ 2886 if (pm->pm_cstate.cs_cache_d && 2887 (oflags & PVF_NC) == 0 && 2888 l2pte_writable_p(opte) && 2889 (prot & VM_PROT_WRITE) == 0) 2890 cpu_dcache_wb_range(va, PAGE_SIZE); 2891 #endif 2892 } else { 2893 /* 2894 * New mapping, or changing the backing page 2895 * of an existing mapping. 2896 */ 2897 if (opg) { 2898 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 2899 paddr_t opa = VM_PAGE_TO_PHYS(opg); 2900 2901 /* 2902 * Replacing an existing mapping with a new one. 2903 * It is part of our managed memory so we 2904 * must remove it from the PV list 2905 */ 2906 #ifdef MULTIPROCESSOR 2907 KASSERT(uvm_page_locked_p(opg)); 2908 #endif 2909 pv = pmap_remove_pv(omd, opa, pm, va); 2910 pmap_vac_me_harder(omd, opa, pm, 0); 2911 oflags = pv->pv_flags; 2912 2913 #ifdef PMAP_CACHE_VIVT 2914 /* 2915 * If the old mapping was valid (ref/mod 2916 * emulation creates 'invalid' mappings 2917 * initially) then make sure to frob 2918 * the cache. 2919 */ 2920 if ((oflags & PVF_NC) == 0 && 2921 l2pte_valid(opte)) { 2922 if (PV_BEEN_EXECD(oflags)) { 2923 pmap_idcache_wbinv_range(pm, va, 2924 PAGE_SIZE); 2925 } else 2926 if (PV_BEEN_REFD(oflags)) { 2927 pmap_dcache_wb_range(pm, va, 2928 PAGE_SIZE, true, 2929 (oflags & PVF_WRITE) == 0); 2930 } 2931 } 2932 #endif 2933 } else 2934 if ((pv = pool_get(&pmap_pv_pool, PR_NOWAIT)) == NULL){ 2935 if ((flags & PMAP_CANFAIL) == 0) 2936 panic("pmap_enter: no pv entries"); 2937 2938 if (pm != pmap_kernel()) 2939 pmap_free_l2_bucket(pm, l2b, 0); 2940 pmap_release_pmap_lock(pm); 2941 NPDEBUG(PDB_ENTER, 2942 printf("pmap_enter: ENOMEM\n")); 2943 return (ENOMEM); 2944 } 2945 2946 #ifdef MULTIPROCESSOR 2947 KASSERT(uvm_page_locked_p(pg)); 2948 #endif 2949 pmap_enter_pv(md, pa, pv, pm, va, nflags); 2950 } 2951 } else { 2952 /* 2953 * We're mapping an unmanaged page. 2954 * These are always readable, and possibly writable, from 2955 * the get go as we don't need to track ref/mod status. 2956 */ 2957 npte |= l2pte_set_readonly(L2_S_PROTO); 2958 if (prot & VM_PROT_WRITE) 2959 npte = l2pte_set_writable(npte); 2960 2961 /* 2962 * Make sure the vector table is mapped cacheable 2963 */ 2964 if ((pm != pmap_kernel() && va == vector_page) || 2965 (flags & ARM32_MMAP_CACHEABLE)) { 2966 npte |= pte_l2_s_cache_mode; 2967 } else if (flags & ARM32_MMAP_WRITECOMBINE) { 2968 npte |= pte_l2_s_wc_mode; 2969 } 2970 if (opg) { 2971 /* 2972 * Looks like there's an existing 'managed' mapping 2973 * at this address. 2974 */ 2975 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 2976 paddr_t opa = VM_PAGE_TO_PHYS(opg); 2977 2978 #ifdef MULTIPROCESSOR 2979 KASSERT(uvm_page_locked_p(opg)); 2980 #endif 2981 pv = pmap_remove_pv(omd, opa, pm, va); 2982 pmap_vac_me_harder(omd, opa, pm, 0); 2983 oflags = pv->pv_flags; 2984 2985 #ifdef PMAP_CACHE_VIVT 2986 if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { 2987 if (PV_BEEN_EXECD(oflags)) 2988 pmap_idcache_wbinv_range(pm, va, 2989 PAGE_SIZE); 2990 else 2991 if (PV_BEEN_REFD(oflags)) 2992 pmap_dcache_wb_range(pm, va, PAGE_SIZE, 2993 true, (oflags & PVF_WRITE) == 0); 2994 } 2995 #endif 2996 pool_put(&pmap_pv_pool, pv); 2997 } 2998 } 2999 3000 /* 3001 * Make sure userland mappings get the right permissions 3002 */ 3003 if (pm != pmap_kernel() && va != vector_page) 3004 npte |= L2_S_PROT_U; 3005 3006 /* 3007 * Keep the stats up to date 3008 */ 3009 if (opte == 0) { 3010 l2b->l2b_occupancy++; 3011 pm->pm_stats.resident_count++; 3012 } 3013 3014 NPDEBUG(PDB_ENTER, 3015 printf("pmap_enter: opte 0x%08x npte 0x%08x\n", opte, npte)); 3016 3017 /* 3018 * If this is just a wiring change, the two PTEs will be 3019 * identical, so there's no need to update the page table. 3020 */ 3021 if (npte != opte) { 3022 bool is_cached = pmap_is_cached(pm); 3023 3024 *ptep = npte; 3025 PTE_SYNC(ptep); 3026 if (is_cached) { 3027 /* 3028 * We only need to frob the cache/tlb if this pmap 3029 * is current 3030 */ 3031 if (va != vector_page && l2pte_valid(npte)) { 3032 /* 3033 * This mapping is likely to be accessed as 3034 * soon as we return to userland. Fix up the 3035 * L1 entry to avoid taking another 3036 * page/domain fault. 3037 */ 3038 pd_entry_t *pl1pd, l1pd; 3039 3040 pl1pd = &pm->pm_l1->l1_kva[L1_IDX(va)]; 3041 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | 3042 L1_C_PROTO; 3043 if (*pl1pd != l1pd) { 3044 *pl1pd = l1pd; 3045 PTE_SYNC(pl1pd); 3046 } 3047 } 3048 } 3049 3050 if (PV_BEEN_EXECD(oflags)) 3051 pmap_tlb_flushID_SE(pm, va); 3052 else 3053 if (PV_BEEN_REFD(oflags)) 3054 pmap_tlb_flushD_SE(pm, va); 3055 3056 NPDEBUG(PDB_ENTER, 3057 printf("pmap_enter: is_cached %d cs 0x%08x\n", 3058 is_cached, pm->pm_cstate.cs_all)); 3059 3060 if (pg != NULL) { 3061 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3062 3063 #ifdef MULTIPROCESSOR 3064 KASSERT(uvm_page_locked_p(pg)); 3065 #endif 3066 pmap_vac_me_harder(md, pa, pm, va); 3067 } 3068 } 3069 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 3070 if (pg) { 3071 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3072 3073 #ifdef MULTIPROCESSOR 3074 KASSERT(uvm_page_locked_p(pg)); 3075 #endif 3076 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3077 PMAP_VALIDATE_MD_PAGE(md); 3078 } 3079 #endif 3080 3081 pmap_release_pmap_lock(pm); 3082 3083 return (0); 3084 } 3085 3086 /* 3087 * pmap_remove() 3088 * 3089 * pmap_remove is responsible for nuking a number of mappings for a range 3090 * of virtual address space in the current pmap. To do this efficiently 3091 * is interesting, because in a number of cases a wide virtual address 3092 * range may be supplied that contains few actual mappings. So, the 3093 * optimisations are: 3094 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3095 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3096 * maybe do just a partial cache clean. This path of execution is 3097 * complicated by the fact that the cache must be flushed _before_ 3098 * the PTE is nuked, being a VAC :-) 3099 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3100 * all invalidations until pmap_update(), since pmap_remove_all() has 3101 * already flushed the cache. 3102 * 4. Maybe later fast-case a single page, but I don't think this is 3103 * going to make _that_ much difference overall. 3104 */ 3105 3106 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3107 3108 void 3109 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 3110 { 3111 struct l2_bucket *l2b; 3112 vaddr_t next_bucket; 3113 pt_entry_t *ptep; 3114 u_int cleanlist_idx, total, cnt; 3115 struct { 3116 vaddr_t va; 3117 pt_entry_t *ptep; 3118 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3119 u_int mappings, is_exec, is_refd; 3120 3121 NPDEBUG(PDB_REMOVE, printf("pmap_do_remove: pmap=%p sva=%08lx " 3122 "eva=%08lx\n", pm, sva, eva)); 3123 3124 /* 3125 * we lock in the pmap => pv_head direction 3126 */ 3127 pmap_acquire_pmap_lock(pm); 3128 3129 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3130 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3131 if (pm->pm_cstate.cs_tlb == 0) 3132 pm->pm_remove_all = true; 3133 } else 3134 cleanlist_idx = 0; 3135 3136 total = 0; 3137 3138 while (sva < eva) { 3139 /* 3140 * Do one L2 bucket's worth at a time. 3141 */ 3142 next_bucket = L2_NEXT_BUCKET(sva); 3143 if (next_bucket > eva) 3144 next_bucket = eva; 3145 3146 l2b = pmap_get_l2_bucket(pm, sva); 3147 if (l2b == NULL) { 3148 sva = next_bucket; 3149 continue; 3150 } 3151 3152 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3153 3154 for (mappings = 0; sva < next_bucket; sva += PAGE_SIZE, ptep++){ 3155 struct vm_page *pg; 3156 pt_entry_t pte; 3157 paddr_t pa; 3158 3159 pte = *ptep; 3160 3161 if (pte == 0) { 3162 /* Nothing here, move along */ 3163 continue; 3164 } 3165 3166 pa = l2pte_pa(pte); 3167 is_exec = 0; 3168 is_refd = 1; 3169 3170 /* 3171 * Update flags. In a number of circumstances, 3172 * we could cluster a lot of these and do a 3173 * number of sequential pages in one go. 3174 */ 3175 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3176 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3177 struct pv_entry *pv; 3178 3179 #ifdef MULTIPROCESSOR 3180 KASSERT(uvm_page_locked_p(pg)); 3181 #endif 3182 pv = pmap_remove_pv(md, pa, pm, sva); 3183 pmap_vac_me_harder(md, pa, pm, 0); 3184 if (pv != NULL) { 3185 if (pm->pm_remove_all == false) { 3186 is_exec = 3187 PV_BEEN_EXECD(pv->pv_flags); 3188 is_refd = 3189 PV_BEEN_REFD(pv->pv_flags); 3190 } 3191 pool_put(&pmap_pv_pool, pv); 3192 } 3193 } 3194 mappings++; 3195 3196 if (!l2pte_valid(pte)) { 3197 /* 3198 * Ref/Mod emulation is still active for this 3199 * mapping, therefore it is has not yet been 3200 * accessed. No need to frob the cache/tlb. 3201 */ 3202 *ptep = 0; 3203 PTE_SYNC_CURRENT(pm, ptep); 3204 continue; 3205 } 3206 3207 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3208 /* Add to the clean list. */ 3209 cleanlist[cleanlist_idx].ptep = ptep; 3210 cleanlist[cleanlist_idx].va = 3211 sva | (is_exec & 1); 3212 cleanlist_idx++; 3213 } else 3214 if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3215 /* Nuke everything if needed. */ 3216 #ifdef PMAP_CACHE_VIVT 3217 pmap_idcache_wbinv_all(pm); 3218 #endif 3219 pmap_tlb_flushID(pm); 3220 3221 /* 3222 * Roll back the previous PTE list, 3223 * and zero out the current PTE. 3224 */ 3225 for (cnt = 0; 3226 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3227 *cleanlist[cnt].ptep = 0; 3228 PTE_SYNC(cleanlist[cnt].ptep); 3229 } 3230 *ptep = 0; 3231 PTE_SYNC(ptep); 3232 cleanlist_idx++; 3233 pm->pm_remove_all = true; 3234 } else { 3235 *ptep = 0; 3236 PTE_SYNC(ptep); 3237 if (pm->pm_remove_all == false) { 3238 if (is_exec) 3239 pmap_tlb_flushID_SE(pm, sva); 3240 else 3241 if (is_refd) 3242 pmap_tlb_flushD_SE(pm, sva); 3243 } 3244 } 3245 } 3246 3247 /* 3248 * Deal with any left overs 3249 */ 3250 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3251 total += cleanlist_idx; 3252 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3253 if (pm->pm_cstate.cs_all != 0) { 3254 vaddr_t clva = cleanlist[cnt].va & ~1; 3255 if (cleanlist[cnt].va & 1) { 3256 #ifdef PMAP_CACHE_VIVT 3257 pmap_idcache_wbinv_range(pm, 3258 clva, PAGE_SIZE); 3259 #endif 3260 pmap_tlb_flushID_SE(pm, clva); 3261 } else { 3262 #ifdef PMAP_CACHE_VIVT 3263 pmap_dcache_wb_range(pm, 3264 clva, PAGE_SIZE, true, 3265 false); 3266 #endif 3267 pmap_tlb_flushD_SE(pm, clva); 3268 } 3269 } 3270 *cleanlist[cnt].ptep = 0; 3271 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3272 } 3273 3274 /* 3275 * If it looks like we're removing a whole bunch 3276 * of mappings, it's faster to just write-back 3277 * the whole cache now and defer TLB flushes until 3278 * pmap_update() is called. 3279 */ 3280 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3281 cleanlist_idx = 0; 3282 else { 3283 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3284 #ifdef PMAP_CACHE_VIVT 3285 pmap_idcache_wbinv_all(pm); 3286 #endif 3287 pm->pm_remove_all = true; 3288 } 3289 } 3290 3291 pmap_free_l2_bucket(pm, l2b, mappings); 3292 pm->pm_stats.resident_count -= mappings; 3293 } 3294 3295 pmap_release_pmap_lock(pm); 3296 } 3297 3298 #ifdef PMAP_CACHE_VIPT 3299 static struct pv_entry * 3300 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3301 { 3302 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3303 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3304 struct pv_entry *pv; 3305 3306 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC)); 3307 KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0); 3308 3309 pv = pmap_remove_pv(md, pa, pmap_kernel(), va); 3310 KASSERT(pv); 3311 KASSERT(pv->pv_flags & PVF_KENTRY); 3312 3313 /* 3314 * If we are removing a writeable mapping to a cached exec page, 3315 * if it's the last mapping then clear it execness other sync 3316 * the page to the icache. 3317 */ 3318 if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3319 && (pv->pv_flags & PVF_WRITE) != 0) { 3320 if (SLIST_EMPTY(&md->pvh_list)) { 3321 md->pvh_attrs &= ~PVF_EXEC; 3322 PMAPCOUNT(exec_discarded_kremove); 3323 } else { 3324 pmap_syncicache_page(md, pa); 3325 PMAPCOUNT(exec_synced_kremove); 3326 } 3327 } 3328 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 3329 3330 return pv; 3331 } 3332 #endif /* PMAP_CACHE_VIPT */ 3333 3334 /* 3335 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3336 * 3337 * We assume there is already sufficient KVM space available 3338 * to do this, as we can't allocate L2 descriptor tables/metadata 3339 * from here. 3340 */ 3341 void 3342 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3343 { 3344 struct l2_bucket *l2b; 3345 pt_entry_t *ptep, opte; 3346 #ifdef PMAP_CACHE_VIVT 3347 struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3348 #endif 3349 #ifdef PMAP_CACHE_VIPT 3350 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3351 struct vm_page *opg; 3352 struct pv_entry *pv = NULL; 3353 #endif 3354 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3355 3356 NPDEBUG(PDB_KENTER, 3357 printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n", 3358 va, pa, prot)); 3359 3360 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 3361 KDASSERT(l2b != NULL); 3362 3363 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3364 opte = *ptep; 3365 3366 if (opte == 0) { 3367 PMAPCOUNT(kenter_mappings); 3368 l2b->l2b_occupancy++; 3369 } else { 3370 PMAPCOUNT(kenter_remappings); 3371 #ifdef PMAP_CACHE_VIPT 3372 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3373 #ifdef DIAGNOSTIC 3374 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3375 #endif 3376 if (opg) { 3377 KASSERT(opg != pg); 3378 KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0); 3379 KASSERT((flags & PMAP_KMPAGE) == 0); 3380 pv = pmap_kremove_pg(opg, va); 3381 } 3382 #endif 3383 if (l2pte_valid(opte)) { 3384 #ifdef PMAP_CACHE_VIVT 3385 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3386 #endif 3387 cpu_tlb_flushD_SE(va); 3388 cpu_cpwait(); 3389 } 3390 } 3391 3392 *ptep = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) 3393 | ((flags & PMAP_NOCACHE) ? 0 : pte_l2_s_cache_mode); 3394 PTE_SYNC(ptep); 3395 3396 if (pg) { 3397 #ifdef MULTIPROCESSOR 3398 KASSERT(uvm_page_locked_p(pg)); 3399 #endif 3400 if (flags & PMAP_KMPAGE) { 3401 KASSERT(md->urw_mappings == 0); 3402 KASSERT(md->uro_mappings == 0); 3403 KASSERT(md->krw_mappings == 0); 3404 KASSERT(md->kro_mappings == 0); 3405 #ifdef PMAP_CACHE_VIPT 3406 KASSERT(pv == NULL); 3407 KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0); 3408 KASSERT((md->pvh_attrs & PVF_NC) == 0); 3409 /* if there is a color conflict, evict from cache. */ 3410 if (pmap_is_page_colored_p(md) 3411 && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) { 3412 PMAPCOUNT(vac_color_change); 3413 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 3414 } else if (md->pvh_attrs & PVF_MULTCLR) { 3415 /* 3416 * If this page has multiple colors, expunge 3417 * them. 3418 */ 3419 PMAPCOUNT(vac_flush_lots2); 3420 pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY); 3421 } 3422 md->pvh_attrs &= PAGE_SIZE - 1; 3423 md->pvh_attrs |= PVF_KMPAGE 3424 | PVF_COLORED | PVF_DIRTY 3425 | (va & arm_cache_prefer_mask); 3426 #endif 3427 #ifdef PMAP_CACHE_VIVT 3428 md->pvh_attrs |= PVF_KMPAGE; 3429 #endif 3430 pmap_kmpages++; 3431 #ifdef PMAP_CACHE_VIPT 3432 } else { 3433 if (pv == NULL) { 3434 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3435 KASSERT(pv != NULL); 3436 } 3437 pmap_enter_pv(md, pa, pv, pmap_kernel(), va, 3438 PVF_WIRED | PVF_KENTRY 3439 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3440 if ((prot & VM_PROT_WRITE) 3441 && !(md->pvh_attrs & PVF_NC)) 3442 md->pvh_attrs |= PVF_DIRTY; 3443 KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3444 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 3445 #endif 3446 } 3447 #ifdef PMAP_CACHE_VIPT 3448 } else { 3449 if (pv != NULL) 3450 pool_put(&pmap_pv_pool, pv); 3451 #endif 3452 } 3453 } 3454 3455 void 3456 pmap_kremove(vaddr_t va, vsize_t len) 3457 { 3458 struct l2_bucket *l2b; 3459 pt_entry_t *ptep, *sptep, opte; 3460 vaddr_t next_bucket, eva; 3461 u_int mappings; 3462 struct vm_page *opg; 3463 3464 PMAPCOUNT(kenter_unmappings); 3465 3466 NPDEBUG(PDB_KREMOVE, printf("pmap_kremove: va 0x%08lx, len 0x%08lx\n", 3467 va, len)); 3468 3469 eva = va + len; 3470 3471 while (va < eva) { 3472 next_bucket = L2_NEXT_BUCKET(va); 3473 if (next_bucket > eva) 3474 next_bucket = eva; 3475 3476 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 3477 KDASSERT(l2b != NULL); 3478 3479 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 3480 mappings = 0; 3481 3482 while (va < next_bucket) { 3483 opte = *ptep; 3484 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3485 if (opg) { 3486 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3487 3488 if (omd->pvh_attrs & PVF_KMPAGE) { 3489 KASSERT(omd->urw_mappings == 0); 3490 KASSERT(omd->uro_mappings == 0); 3491 KASSERT(omd->krw_mappings == 0); 3492 KASSERT(omd->kro_mappings == 0); 3493 omd->pvh_attrs &= ~PVF_KMPAGE; 3494 #ifdef PMAP_CACHE_VIPT 3495 if (arm_cache_prefer_mask != 0) { 3496 omd->pvh_attrs &= ~PVF_WRITE; 3497 } 3498 #endif 3499 pmap_kmpages--; 3500 #ifdef PMAP_CACHE_VIPT 3501 } else { 3502 pool_put(&pmap_pv_pool, 3503 pmap_kremove_pg(opg, va)); 3504 #endif 3505 } 3506 } 3507 if (l2pte_valid(opte)) { 3508 #ifdef PMAP_CACHE_VIVT 3509 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3510 #endif 3511 cpu_tlb_flushD_SE(va); 3512 } 3513 if (opte) { 3514 *ptep = 0; 3515 mappings++; 3516 } 3517 va += PAGE_SIZE; 3518 ptep++; 3519 } 3520 KDASSERT(mappings <= l2b->l2b_occupancy); 3521 l2b->l2b_occupancy -= mappings; 3522 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3523 } 3524 cpu_cpwait(); 3525 } 3526 3527 bool 3528 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3529 { 3530 struct l2_dtable *l2; 3531 pd_entry_t *pl1pd, l1pd; 3532 pt_entry_t *ptep, pte; 3533 paddr_t pa; 3534 u_int l1idx; 3535 3536 pmap_acquire_pmap_lock(pm); 3537 3538 l1idx = L1_IDX(va); 3539 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3540 l1pd = *pl1pd; 3541 3542 if (l1pte_section_p(l1pd)) { 3543 /* 3544 * These should only happen for pmap_kernel() 3545 */ 3546 KDASSERT(pm == pmap_kernel()); 3547 pmap_release_pmap_lock(pm); 3548 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 3549 if (l1pte_supersection_p(l1pd)) { 3550 pa = (l1pd & L1_SS_FRAME) | (va & L1_SS_OFFSET); 3551 } else 3552 #endif 3553 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3554 } else { 3555 /* 3556 * Note that we can't rely on the validity of the L1 3557 * descriptor as an indication that a mapping exists. 3558 * We have to look it up in the L2 dtable. 3559 */ 3560 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3561 3562 if (l2 == NULL || 3563 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3564 pmap_release_pmap_lock(pm); 3565 return false; 3566 } 3567 3568 ptep = &ptep[l2pte_index(va)]; 3569 pte = *ptep; 3570 pmap_release_pmap_lock(pm); 3571 3572 if (pte == 0) 3573 return false; 3574 3575 switch (pte & L2_TYPE_MASK) { 3576 case L2_TYPE_L: 3577 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3578 break; 3579 3580 default: 3581 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3582 break; 3583 } 3584 } 3585 3586 if (pap != NULL) 3587 *pap = pa; 3588 3589 return true; 3590 } 3591 3592 void 3593 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 3594 { 3595 struct l2_bucket *l2b; 3596 pt_entry_t *ptep, pte; 3597 vaddr_t next_bucket; 3598 u_int flags; 3599 u_int clr_mask; 3600 int flush; 3601 3602 NPDEBUG(PDB_PROTECT, 3603 printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n", 3604 pm, sva, eva, prot)); 3605 3606 if ((prot & VM_PROT_READ) == 0) { 3607 pmap_remove(pm, sva, eva); 3608 return; 3609 } 3610 3611 if (prot & VM_PROT_WRITE) { 3612 /* 3613 * If this is a read->write transition, just ignore it and let 3614 * uvm_fault() take care of it later. 3615 */ 3616 return; 3617 } 3618 3619 pmap_acquire_pmap_lock(pm); 3620 3621 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 3622 flags = 0; 3623 clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 3624 3625 while (sva < eva) { 3626 next_bucket = L2_NEXT_BUCKET(sva); 3627 if (next_bucket > eva) 3628 next_bucket = eva; 3629 3630 l2b = pmap_get_l2_bucket(pm, sva); 3631 if (l2b == NULL) { 3632 sva = next_bucket; 3633 continue; 3634 } 3635 3636 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3637 3638 while (sva < next_bucket) { 3639 pte = *ptep; 3640 if (l2pte_valid(pte) != 0 && l2pte_writable_p(pte)) { 3641 struct vm_page *pg; 3642 u_int f; 3643 3644 #ifdef PMAP_CACHE_VIVT 3645 /* 3646 * OK, at this point, we know we're doing 3647 * write-protect operation. If the pmap is 3648 * active, write-back the page. 3649 */ 3650 pmap_dcache_wb_range(pm, sva, PAGE_SIZE, 3651 false, false); 3652 #endif 3653 3654 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3655 pte = l2pte_set_readonly(pte); 3656 *ptep = pte; 3657 PTE_SYNC(ptep); 3658 3659 if (pg != NULL) { 3660 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3661 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3662 3663 #ifdef MULTIPROCESSOR 3664 KASSERT(uvm_page_locked_p(pg)); 3665 #endif 3666 f = pmap_modify_pv(md, pa, pm, sva, 3667 clr_mask, 0); 3668 pmap_vac_me_harder(md, pa, pm, sva); 3669 } else { 3670 f = PVF_REF | PVF_EXEC; 3671 } 3672 3673 if (flush >= 0) { 3674 flush++; 3675 flags |= f; 3676 } else 3677 if (PV_BEEN_EXECD(f)) 3678 pmap_tlb_flushID_SE(pm, sva); 3679 else 3680 if (PV_BEEN_REFD(f)) 3681 pmap_tlb_flushD_SE(pm, sva); 3682 } 3683 3684 sva += PAGE_SIZE; 3685 ptep++; 3686 } 3687 } 3688 3689 pmap_release_pmap_lock(pm); 3690 3691 if (flush) { 3692 if (PV_BEEN_EXECD(flags)) 3693 pmap_tlb_flushID(pm); 3694 else 3695 if (PV_BEEN_REFD(flags)) 3696 pmap_tlb_flushD(pm); 3697 } 3698 } 3699 3700 void 3701 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 3702 { 3703 struct l2_bucket *l2b; 3704 pt_entry_t *ptep; 3705 vaddr_t next_bucket; 3706 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 3707 3708 NPDEBUG(PDB_EXEC, 3709 printf("pmap_icache_sync_range: pm %p sva 0x%lx eva 0x%lx\n", 3710 pm, sva, eva)); 3711 3712 pmap_acquire_pmap_lock(pm); 3713 3714 while (sva < eva) { 3715 next_bucket = L2_NEXT_BUCKET(sva); 3716 if (next_bucket > eva) 3717 next_bucket = eva; 3718 3719 l2b = pmap_get_l2_bucket(pm, sva); 3720 if (l2b == NULL) { 3721 sva = next_bucket; 3722 continue; 3723 } 3724 3725 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3726 sva < next_bucket; 3727 sva += page_size, ptep++, page_size = PAGE_SIZE) { 3728 if (l2pte_valid(*ptep)) { 3729 cpu_icache_sync_range(sva, 3730 min(page_size, eva - sva)); 3731 } 3732 } 3733 } 3734 3735 pmap_release_pmap_lock(pm); 3736 } 3737 3738 void 3739 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 3740 { 3741 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3742 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3743 3744 NPDEBUG(PDB_PROTECT, 3745 printf("pmap_page_protect: md %p (0x%08lx), prot 0x%x\n", 3746 md, pa, prot)); 3747 3748 #ifdef MULTIPROCESSOR 3749 KASSERT(uvm_page_locked_p(pg)); 3750 #endif 3751 3752 switch(prot) { 3753 case VM_PROT_READ|VM_PROT_WRITE: 3754 #if defined(PMAP_CHECK_VIPT) && defined(PMAP_APX) 3755 pmap_clearbit(md, pa, PVF_EXEC); 3756 break; 3757 #endif 3758 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 3759 break; 3760 3761 case VM_PROT_READ: 3762 #if defined(PMAP_CHECK_VIPT) && defined(PMAP_APX) 3763 pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC); 3764 break; 3765 #endif 3766 case VM_PROT_READ|VM_PROT_EXECUTE: 3767 pmap_clearbit(md, pa, PVF_WRITE); 3768 break; 3769 3770 default: 3771 pmap_page_remove(md, pa); 3772 break; 3773 } 3774 } 3775 3776 /* 3777 * pmap_clear_modify: 3778 * 3779 * Clear the "modified" attribute for a page. 3780 */ 3781 bool 3782 pmap_clear_modify(struct vm_page *pg) 3783 { 3784 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3785 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3786 bool rv; 3787 3788 #ifdef MULTIPROCESSOR 3789 KASSERT(uvm_page_locked_p(pg)); 3790 #endif 3791 3792 if (md->pvh_attrs & PVF_MOD) { 3793 rv = true; 3794 #ifdef PMAP_CACHE_VIPT 3795 /* 3796 * If we are going to clear the modified bit and there are 3797 * no other modified bits set, flush the page to memory and 3798 * mark it clean. 3799 */ 3800 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD) 3801 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 3802 #endif 3803 pmap_clearbit(md, pa, PVF_MOD); 3804 } else 3805 rv = false; 3806 3807 return (rv); 3808 } 3809 3810 /* 3811 * pmap_clear_reference: 3812 * 3813 * Clear the "referenced" attribute for a page. 3814 */ 3815 bool 3816 pmap_clear_reference(struct vm_page *pg) 3817 { 3818 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3819 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3820 bool rv; 3821 3822 #ifdef MULTIPROCESSOR 3823 KASSERT(uvm_page_locked_p(pg)); 3824 #endif 3825 3826 if (md->pvh_attrs & PVF_REF) { 3827 rv = true; 3828 pmap_clearbit(md, pa, PVF_REF); 3829 } else 3830 rv = false; 3831 3832 return (rv); 3833 } 3834 3835 /* 3836 * pmap_is_modified: 3837 * 3838 * Test if a page has the "modified" attribute. 3839 */ 3840 /* See <arm/arm32/pmap.h> */ 3841 3842 /* 3843 * pmap_is_referenced: 3844 * 3845 * Test if a page has the "referenced" attribute. 3846 */ 3847 /* See <arm/arm32/pmap.h> */ 3848 3849 int 3850 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 3851 { 3852 struct l2_dtable *l2; 3853 struct l2_bucket *l2b; 3854 pd_entry_t *pl1pd, l1pd; 3855 pt_entry_t *ptep, pte; 3856 paddr_t pa; 3857 u_int l1idx; 3858 int rv = 0; 3859 3860 pmap_acquire_pmap_lock(pm); 3861 3862 l1idx = L1_IDX(va); 3863 3864 /* 3865 * If there is no l2_dtable for this address, then the process 3866 * has no business accessing it. 3867 * 3868 * Note: This will catch userland processes trying to access 3869 * kernel addresses. 3870 */ 3871 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3872 if (l2 == NULL) 3873 goto out; 3874 3875 /* 3876 * Likewise if there is no L2 descriptor table 3877 */ 3878 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 3879 if (l2b->l2b_kva == NULL) 3880 goto out; 3881 3882 /* 3883 * Check the PTE itself. 3884 */ 3885 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3886 pte = *ptep; 3887 if (pte == 0) 3888 goto out; 3889 3890 /* 3891 * Catch a userland access to the vector page mapped at 0x0 3892 */ 3893 if (user && (pte & L2_S_PROT_U) == 0) 3894 goto out; 3895 3896 pa = l2pte_pa(pte); 3897 3898 if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(pte)) { 3899 /* 3900 * This looks like a good candidate for "page modified" 3901 * emulation... 3902 */ 3903 struct pv_entry *pv; 3904 struct vm_page *pg; 3905 3906 /* Extract the physical address of the page */ 3907 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3908 goto out; 3909 3910 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3911 3912 /* Get the current flags for this page. */ 3913 #ifdef MULTIPROCESSOR 3914 KASSERT(uvm_page_locked_p(pg)); 3915 #endif 3916 3917 pv = pmap_find_pv(md, pm, va); 3918 if (pv == NULL) { 3919 goto out; 3920 } 3921 3922 /* 3923 * Do the flags say this page is writable? If not then it 3924 * is a genuine write fault. If yes then the write fault is 3925 * our fault as we did not reflect the write access in the 3926 * PTE. Now we know a write has occurred we can correct this 3927 * and also set the modified bit 3928 */ 3929 if ((pv->pv_flags & PVF_WRITE) == 0) { 3930 goto out; 3931 } 3932 3933 NPDEBUG(PDB_FOLLOW, 3934 printf("pmap_fault_fixup: mod emul. pm %p, va 0x%08lx, pa 0x%08lx\n", 3935 pm, va, pa)); 3936 3937 md->pvh_attrs |= PVF_REF | PVF_MOD; 3938 pv->pv_flags |= PVF_REF | PVF_MOD; 3939 #ifdef PMAP_CACHE_VIPT 3940 /* 3941 * If there are cacheable mappings for this page, mark it dirty. 3942 */ 3943 if ((md->pvh_attrs & PVF_NC) == 0) 3944 md->pvh_attrs |= PVF_DIRTY; 3945 #endif 3946 3947 /* 3948 * Re-enable write permissions for the page. No need to call 3949 * pmap_vac_me_harder(), since this is just a 3950 * modified-emulation fault, and the PVF_WRITE bit isn't 3951 * changing. We've already set the cacheable bits based on 3952 * the assumption that we can write to this page. 3953 */ 3954 *ptep = l2pte_set_writable((pte & ~L2_TYPE_MASK) | L2_S_PROTO); 3955 PTE_SYNC(ptep); 3956 rv = 1; 3957 } else 3958 if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 3959 /* 3960 * This looks like a good candidate for "page referenced" 3961 * emulation. 3962 */ 3963 struct pv_entry *pv; 3964 struct vm_page *pg; 3965 3966 /* Extract the physical address of the page */ 3967 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3968 goto out; 3969 3970 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3971 3972 /* Get the current flags for this page. */ 3973 #ifdef MULTIPROCESSOR 3974 KASSERT(uvm_page_locked_p(pg)); 3975 #endif 3976 3977 pv = pmap_find_pv(md, pm, va); 3978 if (pv == NULL) { 3979 goto out; 3980 } 3981 3982 md->pvh_attrs |= PVF_REF; 3983 pv->pv_flags |= PVF_REF; 3984 3985 NPDEBUG(PDB_FOLLOW, 3986 printf("pmap_fault_fixup: ref emul. pm %p, va 0x%08lx, pa 0x%08lx\n", 3987 pm, va, pa)); 3988 3989 *ptep = l2pte_set_readonly((pte & ~L2_TYPE_MASK) | L2_S_PROTO); 3990 PTE_SYNC(ptep); 3991 rv = 1; 3992 } 3993 3994 /* 3995 * We know there is a valid mapping here, so simply 3996 * fix up the L1 if necessary. 3997 */ 3998 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3999 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 4000 if (*pl1pd != l1pd) { 4001 *pl1pd = l1pd; 4002 PTE_SYNC(pl1pd); 4003 rv = 1; 4004 } 4005 4006 #ifdef CPU_SA110 4007 /* 4008 * There are bugs in the rev K SA110. This is a check for one 4009 * of them. 4010 */ 4011 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 4012 curcpu()->ci_arm_cpurev < 3) { 4013 /* Always current pmap */ 4014 if (l2pte_valid(pte)) { 4015 extern int kernel_debug; 4016 if (kernel_debug & 1) { 4017 struct proc *p = curlwp->l_proc; 4018 printf("prefetch_abort: page is already " 4019 "mapped - pte=%p *pte=%08x\n", ptep, pte); 4020 printf("prefetch_abort: pc=%08lx proc=%p " 4021 "process=%s\n", va, p, p->p_comm); 4022 printf("prefetch_abort: far=%08x fs=%x\n", 4023 cpu_faultaddress(), cpu_faultstatus()); 4024 } 4025 #ifdef DDB 4026 if (kernel_debug & 2) 4027 Debugger(); 4028 #endif 4029 rv = 1; 4030 } 4031 } 4032 #endif /* CPU_SA110 */ 4033 4034 /* 4035 * If 'rv == 0' at this point, it generally indicates that there is a 4036 * stale TLB entry for the faulting address. That might be due to a 4037 * wrong setting of pmap_needs_pte_sync. So set it and retry. 4038 */ 4039 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1 4040 && pmap_needs_pte_sync == 0) { 4041 pmap_needs_pte_sync = 1; 4042 PTE_SYNC(ptep); 4043 rv = 1; 4044 } 4045 4046 #ifdef DEBUG 4047 /* 4048 * If 'rv == 0' at this point, it generally indicates that there is a 4049 * stale TLB entry for the faulting address. This happens when two or 4050 * more processes are sharing an L1. Since we don't flush the TLB on 4051 * a context switch between such processes, we can take domain faults 4052 * for mappings which exist at the same VA in both processes. EVEN IF 4053 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 4054 * example. 4055 * 4056 * This is extremely likely to happen if pmap_enter() updated the L1 4057 * entry for a recently entered mapping. In this case, the TLB is 4058 * flushed for the new mapping, but there may still be TLB entries for 4059 * other mappings belonging to other processes in the 1MB range 4060 * covered by the L1 entry. 4061 * 4062 * Since 'rv == 0', we know that the L1 already contains the correct 4063 * value, so the fault must be due to a stale TLB entry. 4064 * 4065 * Since we always need to flush the TLB anyway in the case where we 4066 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 4067 * stale TLB entries dynamically. 4068 * 4069 * However, the above condition can ONLY happen if the current L1 is 4070 * being shared. If it happens when the L1 is unshared, it indicates 4071 * that other parts of the pmap are not doing their job WRT managing 4072 * the TLB. 4073 */ 4074 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 4075 extern int last_fault_code; 4076 extern int kernel_debug; 4077 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 4078 pm, va, ftype); 4079 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 4080 l2, l2b, ptep, pl1pd); 4081 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 4082 pte, l1pd, last_fault_code); 4083 #ifdef DDB 4084 if (kernel_debug & 2) 4085 Debugger(); 4086 #endif 4087 } 4088 #endif 4089 4090 cpu_tlb_flushID_SE(va); 4091 cpu_cpwait(); 4092 4093 rv = 1; 4094 4095 out: 4096 pmap_release_pmap_lock(pm); 4097 4098 return (rv); 4099 } 4100 4101 /* 4102 * Routine: pmap_procwr 4103 * 4104 * Function: 4105 * Synchronize caches corresponding to [addr, addr+len) in p. 4106 * 4107 */ 4108 void 4109 pmap_procwr(struct proc *p, vaddr_t va, int len) 4110 { 4111 /* We only need to do anything if it is the current process. */ 4112 if (p == curproc) 4113 cpu_icache_sync_range(va, len); 4114 } 4115 4116 /* 4117 * Routine: pmap_unwire 4118 * Function: Clear the wired attribute for a map/virtual-address pair. 4119 * 4120 * In/out conditions: 4121 * The mapping must already exist in the pmap. 4122 */ 4123 void 4124 pmap_unwire(pmap_t pm, vaddr_t va) 4125 { 4126 struct l2_bucket *l2b; 4127 pt_entry_t *ptep, pte; 4128 struct vm_page *pg; 4129 paddr_t pa; 4130 4131 NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va)); 4132 4133 pmap_acquire_pmap_lock(pm); 4134 4135 l2b = pmap_get_l2_bucket(pm, va); 4136 KDASSERT(l2b != NULL); 4137 4138 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4139 pte = *ptep; 4140 4141 /* Extract the physical address of the page */ 4142 pa = l2pte_pa(pte); 4143 4144 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4145 /* Update the wired bit in the pv entry for this page. */ 4146 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4147 4148 #ifdef MULTIPROCESSOR 4149 KASSERT(uvm_page_locked_p(pg)); 4150 #endif 4151 (void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0); 4152 } 4153 4154 pmap_release_pmap_lock(pm); 4155 } 4156 4157 void 4158 pmap_activate(struct lwp *l) 4159 { 4160 extern int block_userspace_access; 4161 pmap_t opm, npm, rpm; 4162 uint32_t odacr, ndacr; 4163 int oldirqstate; 4164 4165 /* 4166 * If activating a non-current lwp or the current lwp is 4167 * already active, just return. 4168 */ 4169 if (l != curlwp || 4170 l->l_proc->p_vmspace->vm_map.pmap->pm_activated == true) 4171 return; 4172 4173 npm = l->l_proc->p_vmspace->vm_map.pmap; 4174 ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 4175 (DOMAIN_CLIENT << (npm->pm_domain * 2)); 4176 4177 /* 4178 * If TTB and DACR are unchanged, short-circuit all the 4179 * TLB/cache management stuff. 4180 */ 4181 if (pmap_previous_active_lwp != NULL) { 4182 opm = pmap_previous_active_lwp->l_proc->p_vmspace->vm_map.pmap; 4183 odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 4184 (DOMAIN_CLIENT << (opm->pm_domain * 2)); 4185 4186 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4187 goto all_done; 4188 } else 4189 opm = NULL; 4190 4191 PMAPCOUNT(activations); 4192 block_userspace_access = 1; 4193 4194 /* 4195 * If switching to a user vmspace which is different to the 4196 * most recent one, and the most recent one is potentially 4197 * live in the cache, we must write-back and invalidate the 4198 * entire cache. 4199 */ 4200 rpm = pmap_recent_user; 4201 4202 /* 4203 * XXXSCW: There's a corner case here which can leave turds in the cache as 4204 * reported in kern/41058. They're probably left over during tear-down and 4205 * switching away from an exiting process. Until the root cause is identified 4206 * and fixed, zap the cache when switching pmaps. This will result in a few 4207 * unnecessary cache flushes, but that's better than silently corrupting data. 4208 */ 4209 #if 0 4210 if (npm != pmap_kernel() && rpm && npm != rpm && 4211 rpm->pm_cstate.cs_cache) { 4212 rpm->pm_cstate.cs_cache = 0; 4213 #ifdef PMAP_CACHE_VIVT 4214 cpu_idcache_wbinv_all(); 4215 #endif 4216 } 4217 #else 4218 if (rpm) { 4219 rpm->pm_cstate.cs_cache = 0; 4220 if (npm == pmap_kernel()) 4221 pmap_recent_user = NULL; 4222 #ifdef PMAP_CACHE_VIVT 4223 cpu_idcache_wbinv_all(); 4224 #endif 4225 } 4226 #endif 4227 4228 /* No interrupts while we frob the TTB/DACR */ 4229 oldirqstate = disable_interrupts(IF32_bits); 4230 4231 /* 4232 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 4233 * entry corresponding to 'vector_page' in the incoming L1 table 4234 * before switching to it otherwise subsequent interrupts/exceptions 4235 * (including domain faults!) will jump into hyperspace. 4236 */ 4237 if (npm->pm_pl1vec != NULL) { 4238 cpu_tlb_flushID_SE((u_int)vector_page); 4239 cpu_cpwait(); 4240 *npm->pm_pl1vec = npm->pm_l1vec; 4241 PTE_SYNC(npm->pm_pl1vec); 4242 } 4243 4244 cpu_domains(ndacr); 4245 4246 if (npm == pmap_kernel() || npm == rpm) { 4247 /* 4248 * Switching to a kernel thread, or back to the 4249 * same user vmspace as before... Simply update 4250 * the TTB (no TLB flush required) 4251 */ 4252 cpu_setttb(npm->pm_l1->l1_physaddr, false); 4253 cpu_cpwait(); 4254 } else { 4255 /* 4256 * Otherwise, update TTB and flush TLB 4257 */ 4258 cpu_context_switch(npm->pm_l1->l1_physaddr); 4259 if (rpm != NULL) 4260 rpm->pm_cstate.cs_tlb = 0; 4261 } 4262 4263 restore_interrupts(oldirqstate); 4264 4265 block_userspace_access = 0; 4266 4267 all_done: 4268 /* 4269 * The new pmap is resident. Make sure it's marked 4270 * as resident in the cache/TLB. 4271 */ 4272 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4273 if (npm != pmap_kernel()) 4274 pmap_recent_user = npm; 4275 4276 /* The old pmap is not longer active */ 4277 if (opm != NULL) 4278 opm->pm_activated = false; 4279 4280 /* But the new one is */ 4281 npm->pm_activated = true; 4282 } 4283 4284 void 4285 pmap_deactivate(struct lwp *l) 4286 { 4287 4288 /* 4289 * If the process is exiting, make sure pmap_activate() does 4290 * a full MMU context-switch and cache flush, which we might 4291 * otherwise skip. See PR port-arm/38950. 4292 */ 4293 if (l->l_proc->p_sflag & PS_WEXIT) 4294 pmap_previous_active_lwp = NULL; 4295 4296 l->l_proc->p_vmspace->vm_map.pmap->pm_activated = false; 4297 } 4298 4299 void 4300 pmap_update(pmap_t pm) 4301 { 4302 4303 if (pm->pm_remove_all) { 4304 /* 4305 * Finish up the pmap_remove_all() optimisation by flushing 4306 * the TLB. 4307 */ 4308 pmap_tlb_flushID(pm); 4309 pm->pm_remove_all = false; 4310 } 4311 4312 if (pmap_is_current(pm)) { 4313 /* 4314 * If we're dealing with a current userland pmap, move its L1 4315 * to the end of the LRU. 4316 */ 4317 if (pm != pmap_kernel()) 4318 pmap_use_l1(pm); 4319 4320 /* 4321 * We can assume we're done with frobbing the cache/tlb for 4322 * now. Make sure any future pmap ops don't skip cache/tlb 4323 * flushes. 4324 */ 4325 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4326 } 4327 4328 PMAPCOUNT(updates); 4329 4330 /* 4331 * make sure TLB/cache operations have completed. 4332 */ 4333 cpu_cpwait(); 4334 } 4335 4336 void 4337 pmap_remove_all(pmap_t pm) 4338 { 4339 4340 /* 4341 * The vmspace described by this pmap is about to be torn down. 4342 * Until pmap_update() is called, UVM will only make calls 4343 * to pmap_remove(). We can make life much simpler by flushing 4344 * the cache now, and deferring TLB invalidation to pmap_update(). 4345 */ 4346 #ifdef PMAP_CACHE_VIVT 4347 pmap_idcache_wbinv_all(pm); 4348 #endif 4349 pm->pm_remove_all = true; 4350 } 4351 4352 /* 4353 * Retire the given physical map from service. 4354 * Should only be called if the map contains no valid mappings. 4355 */ 4356 void 4357 pmap_destroy(pmap_t pm) 4358 { 4359 u_int count; 4360 4361 if (pm == NULL) 4362 return; 4363 4364 if (pm->pm_remove_all) { 4365 pmap_tlb_flushID(pm); 4366 pm->pm_remove_all = false; 4367 } 4368 4369 /* 4370 * Drop reference count 4371 */ 4372 mutex_enter(pm->pm_lock); 4373 count = --pm->pm_obj.uo_refs; 4374 mutex_exit(pm->pm_lock); 4375 if (count > 0) { 4376 if (pmap_is_current(pm)) { 4377 if (pm != pmap_kernel()) 4378 pmap_use_l1(pm); 4379 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 4380 } 4381 return; 4382 } 4383 4384 /* 4385 * reference count is zero, free pmap resources and then free pmap. 4386 */ 4387 4388 if (vector_page < KERNEL_BASE) { 4389 KDASSERT(!pmap_is_current(pm)); 4390 4391 /* Remove the vector page mapping */ 4392 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 4393 pmap_update(pm); 4394 } 4395 4396 LIST_REMOVE(pm, pm_list); 4397 4398 pmap_free_l1(pm); 4399 4400 if (pmap_recent_user == pm) 4401 pmap_recent_user = NULL; 4402 4403 uvm_obj_destroy(&pm->pm_obj, false); 4404 mutex_destroy(&pm->pm_obj_lock); 4405 pool_cache_put(&pmap_cache, pm); 4406 } 4407 4408 4409 /* 4410 * void pmap_reference(pmap_t pm) 4411 * 4412 * Add a reference to the specified pmap. 4413 */ 4414 void 4415 pmap_reference(pmap_t pm) 4416 { 4417 4418 if (pm == NULL) 4419 return; 4420 4421 pmap_use_l1(pm); 4422 4423 mutex_enter(pm->pm_lock); 4424 pm->pm_obj.uo_refs++; 4425 mutex_exit(pm->pm_lock); 4426 } 4427 4428 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 4429 4430 static struct evcnt pmap_prefer_nochange_ev = 4431 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 4432 static struct evcnt pmap_prefer_change_ev = 4433 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 4434 4435 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 4436 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 4437 4438 void 4439 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 4440 { 4441 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 4442 vaddr_t va = *vap; 4443 vaddr_t diff = (hint - va) & mask; 4444 if (diff == 0) { 4445 pmap_prefer_nochange_ev.ev_count++; 4446 } else { 4447 pmap_prefer_change_ev.ev_count++; 4448 if (__predict_false(td)) 4449 va -= mask + 1; 4450 *vap = va + diff; 4451 } 4452 } 4453 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */ 4454 4455 /* 4456 * pmap_zero_page() 4457 * 4458 * Zero a given physical page by mapping it at a page hook point. 4459 * In doing the zero page op, the page we zero is mapped cachable, as with 4460 * StrongARM accesses to non-cached pages are non-burst making writing 4461 * _any_ bulk data very slow. 4462 */ 4463 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 4464 void 4465 pmap_zero_page_generic(paddr_t phys) 4466 { 4467 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4468 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4469 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4470 #endif 4471 #if defined(PMAP_CACHE_VIPT) 4472 /* Choose the last page color it had, if any */ 4473 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 4474 #else 4475 const vsize_t va_offset = 0; 4476 #endif 4477 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 4478 /* 4479 * Is this page mapped at its natural color? 4480 * If we have all of memory mapped, then just convert PA to VA. 4481 */ 4482 const bool okcolor = va_offset == (phys & arm_cache_prefer_mask); 4483 const vaddr_t vdstp = KERNEL_BASE + (phys - physical_start); 4484 #else 4485 const bool okcolor = false; 4486 const vaddr_t vdstp = cdstp + va_offset; 4487 #endif 4488 pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT]; 4489 4490 4491 #ifdef DEBUG 4492 if (!SLIST_EMPTY(&md->pvh_list)) 4493 panic("pmap_zero_page: page has mappings"); 4494 #endif 4495 4496 KDASSERT((phys & PGOFSET) == 0); 4497 4498 if (!okcolor) { 4499 /* 4500 * Hook in the page, zero it, and purge the cache for that 4501 * zeroed page. Invalidate the TLB as needed. 4502 */ 4503 *ptep = L2_S_PROTO | phys | 4504 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4505 PTE_SYNC(ptep); 4506 cpu_tlb_flushD_SE(cdstp + va_offset); 4507 cpu_cpwait(); 4508 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 4509 /* 4510 * If we are direct-mapped and our color isn't ok, then before 4511 * we bzero the page invalidate its contents from the cache and 4512 * reset the color to its natural color. 4513 */ 4514 cpu_dcache_inv_range(cdstp + va_offset, PAGE_SIZE); 4515 md->pvh_attrs &= ~arm_cache_prefer_mask; 4516 md->pvh_attrs |= (phys & arm_cache_prefer_mask); 4517 #endif 4518 } 4519 bzero_page(vdstp); 4520 if (!okcolor) { 4521 /* 4522 * Unmap the page. 4523 */ 4524 *ptep = 0; 4525 PTE_SYNC(ptep); 4526 cpu_tlb_flushD_SE(cdstp + va_offset); 4527 #ifdef PMAP_CACHE_VIVT 4528 cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE); 4529 #endif 4530 } 4531 #ifdef PMAP_CACHE_VIPT 4532 /* 4533 * This page is now cache resident so it now has a page color. 4534 * Any contents have been obliterated so clear the EXEC flag. 4535 */ 4536 if (!pmap_is_page_colored_p(md)) { 4537 PMAPCOUNT(vac_color_new); 4538 md->pvh_attrs |= PVF_COLORED; 4539 } 4540 if (PV_IS_EXEC_P(md->pvh_attrs)) { 4541 md->pvh_attrs &= ~PVF_EXEC; 4542 PMAPCOUNT(exec_discarded_zero); 4543 } 4544 md->pvh_attrs |= PVF_DIRTY; 4545 #endif 4546 } 4547 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 4548 4549 #if ARM_MMU_XSCALE == 1 4550 void 4551 pmap_zero_page_xscale(paddr_t phys) 4552 { 4553 #ifdef DEBUG 4554 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4555 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4556 4557 if (!SLIST_EMPTY(&md->pvh_list)) 4558 panic("pmap_zero_page: page has mappings"); 4559 #endif 4560 4561 KDASSERT((phys & PGOFSET) == 0); 4562 4563 /* 4564 * Hook in the page, zero it, and purge the cache for that 4565 * zeroed page. Invalidate the TLB as needed. 4566 */ 4567 *cdst_pte = L2_S_PROTO | phys | 4568 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4569 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4570 PTE_SYNC(cdst_pte); 4571 cpu_tlb_flushD_SE(cdstp); 4572 cpu_cpwait(); 4573 bzero_page(cdstp); 4574 xscale_cache_clean_minidata(); 4575 } 4576 #endif /* ARM_MMU_XSCALE == 1 */ 4577 4578 /* pmap_pageidlezero() 4579 * 4580 * The same as above, except that we assume that the page is not 4581 * mapped. This means we never have to flush the cache first. Called 4582 * from the idle loop. 4583 */ 4584 bool 4585 pmap_pageidlezero(paddr_t phys) 4586 { 4587 unsigned int i; 4588 int *ptr; 4589 bool rv = true; 4590 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4591 struct vm_page * const pg = PHYS_TO_VM_PAGE(phys); 4592 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4593 #endif 4594 #ifdef PMAP_CACHE_VIPT 4595 /* Choose the last page color it had, if any */ 4596 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 4597 #else 4598 const vsize_t va_offset = 0; 4599 #endif 4600 pt_entry_t * const ptep = &csrc_pte[va_offset >> PGSHIFT]; 4601 4602 4603 #ifdef DEBUG 4604 if (!SLIST_EMPTY(&md->pvh_list)) 4605 panic("pmap_pageidlezero: page has mappings"); 4606 #endif 4607 4608 KDASSERT((phys & PGOFSET) == 0); 4609 4610 /* 4611 * Hook in the page, zero it, and purge the cache for that 4612 * zeroed page. Invalidate the TLB as needed. 4613 */ 4614 *ptep = L2_S_PROTO | phys | 4615 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4616 PTE_SYNC(ptep); 4617 cpu_tlb_flushD_SE(cdstp + va_offset); 4618 cpu_cpwait(); 4619 4620 for (i = 0, ptr = (int *)(cdstp + va_offset); 4621 i < (PAGE_SIZE / sizeof(int)); i++) { 4622 if (sched_curcpu_runnable_p() != 0) { 4623 /* 4624 * A process has become ready. Abort now, 4625 * so we don't keep it waiting while we 4626 * do slow memory access to finish this 4627 * page. 4628 */ 4629 rv = false; 4630 break; 4631 } 4632 *ptr++ = 0; 4633 } 4634 4635 #ifdef PMAP_CACHE_VIVT 4636 if (rv) 4637 /* 4638 * if we aborted we'll rezero this page again later so don't 4639 * purge it unless we finished it 4640 */ 4641 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4642 #elif defined(PMAP_CACHE_VIPT) 4643 /* 4644 * This page is now cache resident so it now has a page color. 4645 * Any contents have been obliterated so clear the EXEC flag. 4646 */ 4647 if (!pmap_is_page_colored_p(md)) { 4648 PMAPCOUNT(vac_color_new); 4649 md->pvh_attrs |= PVF_COLORED; 4650 } 4651 if (PV_IS_EXEC_P(md->pvh_attrs)) { 4652 md->pvh_attrs &= ~PVF_EXEC; 4653 PMAPCOUNT(exec_discarded_zero); 4654 } 4655 #endif 4656 /* 4657 * Unmap the page. 4658 */ 4659 *ptep = 0; 4660 PTE_SYNC(ptep); 4661 cpu_tlb_flushD_SE(cdstp + va_offset); 4662 4663 return (rv); 4664 } 4665 4666 /* 4667 * pmap_copy_page() 4668 * 4669 * Copy one physical page into another, by mapping the pages into 4670 * hook points. The same comment regarding cachability as in 4671 * pmap_zero_page also applies here. 4672 */ 4673 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 4674 void 4675 pmap_copy_page_generic(paddr_t src, paddr_t dst) 4676 { 4677 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 4678 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 4679 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 4680 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 4681 struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg); 4682 #endif 4683 #ifdef PMAP_CACHE_VIPT 4684 const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask; 4685 const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask; 4686 #else 4687 const vsize_t src_va_offset = 0; 4688 const vsize_t dst_va_offset = 0; 4689 #endif 4690 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 4691 /* 4692 * Is this page mapped at its natural color? 4693 * If we have all of memory mapped, then just convert PA to VA. 4694 */ 4695 const bool src_okcolor = src_va_offset == (src & arm_cache_prefer_mask); 4696 const bool dst_okcolor = dst_va_offset == (dst & arm_cache_prefer_mask); 4697 const vaddr_t vsrcp = src_okcolor 4698 ? KERNEL_BASE + (src - physical_start) 4699 : csrcp + src_va_offset; 4700 const vaddr_t vdstp = KERNEL_BASE + (dst - physical_start); 4701 #else 4702 const bool src_okcolor = false; 4703 const bool dst_okcolor = false; 4704 const vaddr_t vsrcp = csrcp + src_va_offset; 4705 const vaddr_t vdstp = cdstp + dst_va_offset; 4706 #endif 4707 pt_entry_t * const src_ptep = &csrc_pte[src_va_offset >> PGSHIFT]; 4708 pt_entry_t * const dst_ptep = &cdst_pte[dst_va_offset >> PGSHIFT]; 4709 4710 #ifdef DEBUG 4711 if (!SLIST_EMPTY(&dst_md->pvh_list)) 4712 panic("pmap_copy_page: dst page has mappings"); 4713 #endif 4714 4715 #ifdef PMAP_CACHE_VIPT 4716 KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC)); 4717 #endif 4718 KDASSERT((src & PGOFSET) == 0); 4719 KDASSERT((dst & PGOFSET) == 0); 4720 4721 /* 4722 * Clean the source page. Hold the source page's lock for 4723 * the duration of the copy so that no other mappings can 4724 * be created while we have a potentially aliased mapping. 4725 */ 4726 #ifdef MULTIPROCESSOR 4727 KASSERT(uvm_page_locked_p(src_pg)); 4728 #endif 4729 #ifdef PMAP_CACHE_VIVT 4730 (void) pmap_clean_page(SLIST_FIRST(&src_md->pvh_list), true); 4731 #endif 4732 4733 /* 4734 * Map the pages into the page hook points, copy them, and purge 4735 * the cache for the appropriate page. Invalidate the TLB 4736 * as required. 4737 */ 4738 if (!src_okcolor) { 4739 *src_ptep = L2_S_PROTO 4740 | src 4741 #ifdef PMAP_CACHE_VIPT 4742 | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 4743 #endif 4744 #ifdef PMAP_CACHE_VIVT 4745 | pte_l2_s_cache_mode 4746 #endif 4747 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 4748 PTE_SYNC(src_ptep); 4749 cpu_tlb_flushD_SE(csrcp + src_va_offset); 4750 cpu_cpwait(); 4751 } 4752 if (!dst_okcolor) { 4753 *dst_ptep = L2_S_PROTO | dst | 4754 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4755 PTE_SYNC(dst_ptep); 4756 cpu_tlb_flushD_SE(cdstp + dst_va_offset); 4757 cpu_cpwait(); 4758 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 4759 /* 4760 * If we are direct-mapped and our color isn't ok, then before 4761 * we bcopy to the new page invalidate its contents from the 4762 * cache and reset its color to its natural color. 4763 */ 4764 cpu_dcache_inv_range(cdstp + dst_va_offset, PAGE_SIZE); 4765 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 4766 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 4767 #endif 4768 } 4769 bcopy_page(vsrcp, vdstp); 4770 #ifdef PMAP_CACHE_VIVT 4771 cpu_dcache_inv_range(vsrcp, PAGE_SIZE); 4772 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 4773 #endif 4774 /* 4775 * Unmap the pages. 4776 */ 4777 if (!src_okcolor) { 4778 *src_ptep = 0; 4779 PTE_SYNC(src_ptep); 4780 cpu_tlb_flushD_SE(csrcp + src_va_offset); 4781 cpu_cpwait(); 4782 } 4783 if (!dst_okcolor) { 4784 *dst_ptep = 0; 4785 PTE_SYNC(dst_ptep); 4786 cpu_tlb_flushD_SE(cdstp + dst_va_offset); 4787 cpu_cpwait(); 4788 } 4789 #ifdef PMAP_CACHE_VIPT 4790 /* 4791 * Now that the destination page is in the cache, mark it as colored. 4792 * If this was an exec page, discard it. 4793 */ 4794 if (!pmap_is_page_colored_p(dst_md)) { 4795 PMAPCOUNT(vac_color_new); 4796 dst_md->pvh_attrs |= PVF_COLORED; 4797 } 4798 if (PV_IS_EXEC_P(dst_md->pvh_attrs)) { 4799 dst_md->pvh_attrs &= ~PVF_EXEC; 4800 PMAPCOUNT(exec_discarded_copy); 4801 } 4802 dst_md->pvh_attrs |= PVF_DIRTY; 4803 #endif 4804 } 4805 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 4806 4807 #if ARM_MMU_XSCALE == 1 4808 void 4809 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 4810 { 4811 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4812 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 4813 #ifdef DEBUG 4814 struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)); 4815 4816 if (!SLIST_EMPTY(&dst_md->pvh_list)) 4817 panic("pmap_copy_page: dst page has mappings"); 4818 #endif 4819 4820 KDASSERT((src & PGOFSET) == 0); 4821 KDASSERT((dst & PGOFSET) == 0); 4822 4823 /* 4824 * Clean the source page. Hold the source page's lock for 4825 * the duration of the copy so that no other mappings can 4826 * be created while we have a potentially aliased mapping. 4827 */ 4828 #ifdef MULTIPROCESSOR 4829 KASSERT(uvm_page_locked_p(src_pg)); 4830 #endif 4831 #ifdef PMAP_CACHE_VIVT 4832 (void) pmap_clean_page(SLIST_FIRST(&src_md->pvh_list), true); 4833 #endif 4834 4835 /* 4836 * Map the pages into the page hook points, copy them, and purge 4837 * the cache for the appropriate page. Invalidate the TLB 4838 * as required. 4839 */ 4840 *csrc_pte = L2_S_PROTO | src | 4841 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4842 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4843 PTE_SYNC(csrc_pte); 4844 *cdst_pte = L2_S_PROTO | dst | 4845 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4846 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 4847 PTE_SYNC(cdst_pte); 4848 cpu_tlb_flushD_SE(csrcp); 4849 cpu_tlb_flushD_SE(cdstp); 4850 cpu_cpwait(); 4851 bcopy_page(csrcp, cdstp); 4852 xscale_cache_clean_minidata(); 4853 } 4854 #endif /* ARM_MMU_XSCALE == 1 */ 4855 4856 /* 4857 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 4858 * 4859 * Return the start and end addresses of the kernel's virtual space. 4860 * These values are setup in pmap_bootstrap and are updated as pages 4861 * are allocated. 4862 */ 4863 void 4864 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 4865 { 4866 *start = virtual_avail; 4867 *end = virtual_end; 4868 } 4869 4870 /* 4871 * Helper function for pmap_grow_l2_bucket() 4872 */ 4873 static inline int 4874 pmap_grow_map(vaddr_t va, pt_entry_t cache_mode, paddr_t *pap) 4875 { 4876 struct l2_bucket *l2b; 4877 pt_entry_t *ptep; 4878 paddr_t pa; 4879 4880 if (uvm.page_init_done == false) { 4881 #ifdef PMAP_STEAL_MEMORY 4882 pv_addr_t pv; 4883 pmap_boot_pagealloc(PAGE_SIZE, 4884 #ifdef PMAP_CACHE_VIPT 4885 arm_cache_prefer_mask, 4886 va & arm_cache_prefer_mask, 4887 #else 4888 0, 0, 4889 #endif 4890 &pv); 4891 pa = pv.pv_pa; 4892 #else 4893 if (uvm_page_physget(&pa) == false) 4894 return (1); 4895 #endif /* PMAP_STEAL_MEMORY */ 4896 } else { 4897 struct vm_page *pg; 4898 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 4899 if (pg == NULL) 4900 return (1); 4901 pa = VM_PAGE_TO_PHYS(pg); 4902 #ifdef PMAP_CACHE_VIPT 4903 #ifdef DIAGNOSTIC 4904 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4905 #endif 4906 /* 4907 * This new page must not have any mappings. Enter it via 4908 * pmap_kenter_pa and let that routine do the hard work. 4909 */ 4910 KASSERT(SLIST_EMPTY(&md->pvh_list)); 4911 pmap_kenter_pa(va, pa, 4912 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE); 4913 #endif 4914 } 4915 4916 if (pap) 4917 *pap = pa; 4918 4919 PMAPCOUNT(pt_mappings); 4920 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 4921 KDASSERT(l2b != NULL); 4922 4923 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4924 *ptep = L2_S_PROTO | pa | cache_mode | 4925 L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); 4926 PTE_SYNC(ptep); 4927 memset((void *)va, 0, PAGE_SIZE); 4928 return (0); 4929 } 4930 4931 /* 4932 * This is the same as pmap_alloc_l2_bucket(), except that it is only 4933 * used by pmap_growkernel(). 4934 */ 4935 static inline struct l2_bucket * 4936 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 4937 { 4938 struct l2_dtable *l2; 4939 struct l2_bucket *l2b; 4940 u_short l1idx; 4941 vaddr_t nva; 4942 4943 l1idx = L1_IDX(va); 4944 4945 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 4946 /* 4947 * No mapping at this address, as there is 4948 * no entry in the L1 table. 4949 * Need to allocate a new l2_dtable. 4950 */ 4951 nva = pmap_kernel_l2dtable_kva; 4952 if ((nva & PGOFSET) == 0) { 4953 /* 4954 * Need to allocate a backing page 4955 */ 4956 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 4957 return (NULL); 4958 } 4959 4960 l2 = (struct l2_dtable *)nva; 4961 nva += sizeof(struct l2_dtable); 4962 4963 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 4964 /* 4965 * The new l2_dtable straddles a page boundary. 4966 * Map in another page to cover it. 4967 */ 4968 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 4969 return (NULL); 4970 } 4971 4972 pmap_kernel_l2dtable_kva = nva; 4973 4974 /* 4975 * Link it into the parent pmap 4976 */ 4977 pm->pm_l2[L2_IDX(l1idx)] = l2; 4978 } 4979 4980 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 4981 4982 /* 4983 * Fetch pointer to the L2 page table associated with the address. 4984 */ 4985 if (l2b->l2b_kva == NULL) { 4986 pt_entry_t *ptep; 4987 4988 /* 4989 * No L2 page table has been allocated. Chances are, this 4990 * is because we just allocated the l2_dtable, above. 4991 */ 4992 nva = pmap_kernel_l2ptp_kva; 4993 ptep = (pt_entry_t *)nva; 4994 if ((nva & PGOFSET) == 0) { 4995 /* 4996 * Need to allocate a backing page 4997 */ 4998 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 4999 &pmap_kernel_l2ptp_phys)) 5000 return (NULL); 5001 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 5002 } 5003 5004 l2->l2_occupancy++; 5005 l2b->l2b_kva = ptep; 5006 l2b->l2b_l1idx = l1idx; 5007 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 5008 5009 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 5010 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 5011 } 5012 5013 return (l2b); 5014 } 5015 5016 vaddr_t 5017 pmap_growkernel(vaddr_t maxkvaddr) 5018 { 5019 pmap_t kpm = pmap_kernel(); 5020 struct l1_ttable *l1; 5021 struct l2_bucket *l2b; 5022 pd_entry_t *pl1pd; 5023 int s; 5024 5025 if (maxkvaddr <= pmap_curmaxkvaddr) 5026 goto out; /* we are OK */ 5027 5028 NPDEBUG(PDB_GROWKERN, 5029 printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n", 5030 pmap_curmaxkvaddr, maxkvaddr)); 5031 5032 KDASSERT(maxkvaddr <= virtual_end); 5033 5034 /* 5035 * whoops! we need to add kernel PTPs 5036 */ 5037 5038 s = splhigh(); /* to be safe */ 5039 mutex_enter(kpm->pm_lock); 5040 5041 /* Map 1MB at a time */ 5042 for (; pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE) { 5043 5044 l2b = pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 5045 KDASSERT(l2b != NULL); 5046 5047 /* Distribute new L1 entry to all other L1s */ 5048 SLIST_FOREACH(l1, &l1_list, l1_link) { 5049 pl1pd = &l1->l1_kva[L1_IDX(pmap_curmaxkvaddr)]; 5050 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 5051 L1_C_PROTO; 5052 PTE_SYNC(pl1pd); 5053 } 5054 } 5055 5056 /* 5057 * flush out the cache, expensive but growkernel will happen so 5058 * rarely 5059 */ 5060 cpu_dcache_wbinv_all(); 5061 cpu_tlb_flushD(); 5062 cpu_cpwait(); 5063 5064 mutex_exit(kpm->pm_lock); 5065 splx(s); 5066 5067 out: 5068 return (pmap_curmaxkvaddr); 5069 } 5070 5071 /************************ Utility routines ****************************/ 5072 5073 /* 5074 * vector_page_setprot: 5075 * 5076 * Manipulate the protection of the vector page. 5077 */ 5078 void 5079 vector_page_setprot(int prot) 5080 { 5081 struct l2_bucket *l2b; 5082 pt_entry_t *ptep; 5083 5084 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 5085 KDASSERT(l2b != NULL); 5086 5087 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 5088 5089 *ptep = (*ptep & ~L2_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 5090 PTE_SYNC(ptep); 5091 cpu_tlb_flushD_SE(vector_page); 5092 cpu_cpwait(); 5093 } 5094 5095 /* 5096 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 5097 * Returns true if the mapping exists, else false. 5098 * 5099 * NOTE: This function is only used by a couple of arm-specific modules. 5100 * It is not safe to take any pmap locks here, since we could be right 5101 * in the middle of debugging the pmap anyway... 5102 * 5103 * It is possible for this routine to return false even though a valid 5104 * mapping does exist. This is because we don't lock, so the metadata 5105 * state may be inconsistent. 5106 * 5107 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 5108 * a "section" mapping. 5109 */ 5110 bool 5111 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 5112 { 5113 struct l2_dtable *l2; 5114 pd_entry_t *pl1pd, l1pd; 5115 pt_entry_t *ptep; 5116 u_short l1idx; 5117 5118 if (pm->pm_l1 == NULL) 5119 return false; 5120 5121 l1idx = L1_IDX(va); 5122 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 5123 l1pd = *pl1pd; 5124 5125 if (l1pte_section_p(l1pd)) { 5126 *ptp = NULL; 5127 return true; 5128 } 5129 5130 if (pm->pm_l2 == NULL) 5131 return false; 5132 5133 l2 = pm->pm_l2[L2_IDX(l1idx)]; 5134 5135 if (l2 == NULL || 5136 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 5137 return false; 5138 } 5139 5140 *ptp = &ptep[l2pte_index(va)]; 5141 return true; 5142 } 5143 5144 bool 5145 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 5146 { 5147 u_short l1idx; 5148 5149 if (pm->pm_l1 == NULL) 5150 return false; 5151 5152 l1idx = L1_IDX(va); 5153 *pdp = &pm->pm_l1->l1_kva[l1idx]; 5154 5155 return true; 5156 } 5157 5158 /************************ Bootstrapping routines ****************************/ 5159 5160 static void 5161 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 5162 { 5163 int i; 5164 5165 l1->l1_kva = l1pt; 5166 l1->l1_domain_use_count = 0; 5167 l1->l1_domain_first = 0; 5168 5169 for (i = 0; i < PMAP_DOMAINS; i++) 5170 l1->l1_domain_free[i] = i + 1; 5171 5172 /* 5173 * Copy the kernel's L1 entries to each new L1. 5174 */ 5175 if (pmap_initialized) 5176 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 5177 5178 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 5179 &l1->l1_physaddr) == false) 5180 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 5181 5182 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 5183 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 5184 } 5185 5186 /* 5187 * pmap_bootstrap() is called from the board-specific initarm() routine 5188 * once the kernel L1/L2 descriptors tables have been set up. 5189 * 5190 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 5191 * spread over a number of disparate files/functions. 5192 * 5193 * We are passed the following parameters 5194 * - kernel_l1pt 5195 * This is a pointer to the base of the kernel's L1 translation table. 5196 * - vstart 5197 * 1MB-aligned start of managed kernel virtual memory. 5198 * - vend 5199 * 1MB-aligned end of managed kernel virtual memory. 5200 * 5201 * We use the first parameter to build the metadata (struct l1_ttable and 5202 * struct l2_dtable) necessary to track kernel mappings. 5203 */ 5204 #define PMAP_STATIC_L2_SIZE 16 5205 void 5206 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 5207 { 5208 static struct l1_ttable static_l1; 5209 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 5210 struct l1_ttable *l1 = &static_l1; 5211 struct l2_dtable *l2; 5212 struct l2_bucket *l2b; 5213 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 5214 pmap_t pm = pmap_kernel(); 5215 pd_entry_t pde; 5216 pt_entry_t *ptep; 5217 paddr_t pa; 5218 vaddr_t va; 5219 vsize_t size; 5220 int nptes, l1idx, l2idx, l2next = 0; 5221 5222 /* 5223 * Initialise the kernel pmap object 5224 */ 5225 pm->pm_l1 = l1; 5226 pm->pm_domain = PMAP_DOMAIN_KERNEL; 5227 pm->pm_activated = true; 5228 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5229 5230 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE); 5231 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 5232 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 5233 5234 /* 5235 * Scan the L1 translation table created by initarm() and create 5236 * the required metadata for all valid mappings found in it. 5237 */ 5238 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 5239 pde = l1pt[l1idx]; 5240 5241 /* 5242 * We're only interested in Coarse mappings. 5243 * pmap_extract() can deal with section mappings without 5244 * recourse to checking L2 metadata. 5245 */ 5246 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 5247 continue; 5248 5249 /* 5250 * Lookup the KVA of this L2 descriptor table 5251 */ 5252 pa = (paddr_t)(pde & L1_C_ADDR_MASK); 5253 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 5254 if (ptep == NULL) { 5255 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 5256 (u_int)l1idx << L1_S_SHIFT, pa); 5257 } 5258 5259 /* 5260 * Fetch the associated L2 metadata structure. 5261 * Allocate a new one if necessary. 5262 */ 5263 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 5264 if (l2next == PMAP_STATIC_L2_SIZE) 5265 panic("pmap_bootstrap: out of static L2s"); 5266 pm->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++]; 5267 } 5268 5269 /* 5270 * One more L1 slot tracked... 5271 */ 5272 l2->l2_occupancy++; 5273 5274 /* 5275 * Fill in the details of the L2 descriptor in the 5276 * appropriate bucket. 5277 */ 5278 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 5279 l2b->l2b_kva = ptep; 5280 l2b->l2b_phys = pa; 5281 l2b->l2b_l1idx = l1idx; 5282 5283 /* 5284 * Establish an initial occupancy count for this descriptor 5285 */ 5286 for (l2idx = 0; 5287 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 5288 l2idx++) { 5289 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 5290 l2b->l2b_occupancy++; 5291 } 5292 } 5293 5294 /* 5295 * Make sure the descriptor itself has the correct cache mode. 5296 * If not, fix it, but whine about the problem. Port-meisters 5297 * should consider this a clue to fix up their initarm() 5298 * function. :) 5299 */ 5300 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep)) { 5301 printf("pmap_bootstrap: WARNING! wrong cache mode for " 5302 "L2 pte @ %p\n", ptep); 5303 } 5304 } 5305 5306 /* 5307 * Ensure the primary (kernel) L1 has the correct cache mode for 5308 * a page table. Bitch if it is not correctly set. 5309 */ 5310 for (va = (vaddr_t)l1pt; 5311 va < ((vaddr_t)l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 5312 if (pmap_set_pt_cache_mode(l1pt, va)) 5313 printf("pmap_bootstrap: WARNING! wrong cache mode for " 5314 "primary L1 @ 0x%lx\n", va); 5315 } 5316 5317 cpu_dcache_wbinv_all(); 5318 cpu_tlb_flushID(); 5319 cpu_cpwait(); 5320 5321 /* 5322 * now we allocate the "special" VAs which are used for tmp mappings 5323 * by the pmap (and other modules). we allocate the VAs by advancing 5324 * virtual_avail (note that there are no pages mapped at these VAs). 5325 * 5326 * Managed KVM space start from wherever initarm() tells us. 5327 */ 5328 virtual_avail = vstart; 5329 virtual_end = vend; 5330 5331 #ifdef PMAP_CACHE_VIPT 5332 /* 5333 * If we have a VIPT cache, we need one page/pte per possible alias 5334 * page so we won't violate cache aliasing rules. 5335 */ 5336 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 5337 nptes = (arm_cache_prefer_mask >> PGSHIFT) + 1; 5338 #else 5339 nptes = 1; 5340 #endif 5341 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 5342 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte); 5343 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 5344 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte); 5345 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 5346 pmap_alloc_specials(&virtual_avail, round_page(MSGBUFSIZE) / PAGE_SIZE, 5347 (void *)&msgbufaddr, NULL); 5348 5349 /* 5350 * Allocate a range of kernel virtual address space to be used 5351 * for L2 descriptor tables and metadata allocation in 5352 * pmap_growkernel(). 5353 */ 5354 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 5355 pmap_alloc_specials(&virtual_avail, 5356 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 5357 &pmap_kernel_l2ptp_kva, NULL); 5358 5359 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 5360 pmap_alloc_specials(&virtual_avail, 5361 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 5362 &pmap_kernel_l2dtable_kva, NULL); 5363 5364 /* 5365 * init the static-global locks and global pmap list. 5366 */ 5367 mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM); 5368 5369 /* 5370 * We can now initialise the first L1's metadata. 5371 */ 5372 SLIST_INIT(&l1_list); 5373 TAILQ_INIT(&l1_lru_list); 5374 pmap_init_l1(l1, l1pt); 5375 5376 /* Set up vector page L1 details, if necessary */ 5377 if (vector_page < KERNEL_BASE) { 5378 pm->pm_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 5379 l2b = pmap_get_l2_bucket(pm, vector_page); 5380 KDASSERT(l2b != NULL); 5381 pm->pm_l1vec = l2b->l2b_phys | L1_C_PROTO | 5382 L1_C_DOM(pm->pm_domain); 5383 } else 5384 pm->pm_pl1vec = NULL; 5385 5386 /* 5387 * Initialize the pmap cache 5388 */ 5389 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 5390 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 5391 LIST_INIT(&pmap_pmaps); 5392 LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list); 5393 5394 /* 5395 * Initialize the pv pool. 5396 */ 5397 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 5398 &pmap_bootstrap_pv_allocator, IPL_NONE); 5399 5400 /* 5401 * Initialize the L2 dtable pool and cache. 5402 */ 5403 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 5404 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 5405 5406 /* 5407 * Initialise the L2 descriptor table pool and cache 5408 */ 5409 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 0, 5410 L2_TABLE_SIZE_REAL, 0, "l2ptppl", NULL, IPL_NONE, 5411 pmap_l2ptp_ctor, NULL, NULL); 5412 5413 cpu_dcache_wbinv_all(); 5414 } 5415 5416 static int 5417 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va) 5418 { 5419 pd_entry_t *pdep, pde; 5420 pt_entry_t *ptep, pte; 5421 vaddr_t pa; 5422 int rv = 0; 5423 5424 /* 5425 * Make sure the descriptor itself has the correct cache mode 5426 */ 5427 pdep = &kl1[L1_IDX(va)]; 5428 pde = *pdep; 5429 5430 if (l1pte_section_p(pde)) { 5431 __CTASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0); 5432 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 5433 *pdep = (pde & ~L1_S_CACHE_MASK) | 5434 pte_l1_s_cache_mode_pt; 5435 PTE_SYNC(pdep); 5436 cpu_dcache_wbinv_range((vaddr_t)pdep, sizeof(*pdep)); 5437 rv = 1; 5438 } 5439 } else { 5440 pa = (paddr_t)(pde & L1_C_ADDR_MASK); 5441 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 5442 if (ptep == NULL) 5443 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 5444 5445 ptep = &ptep[l2pte_index(va)]; 5446 pte = *ptep; 5447 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 5448 *ptep = (pte & ~L2_S_CACHE_MASK) | 5449 pte_l2_s_cache_mode_pt; 5450 PTE_SYNC(ptep); 5451 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 5452 rv = 1; 5453 } 5454 } 5455 5456 return (rv); 5457 } 5458 5459 static void 5460 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 5461 { 5462 vaddr_t va = *availp; 5463 struct l2_bucket *l2b; 5464 5465 if (ptep) { 5466 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5467 if (l2b == NULL) 5468 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 5469 5470 if (ptep) 5471 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 5472 } 5473 5474 *vap = va; 5475 *availp = va + (PAGE_SIZE * pages); 5476 } 5477 5478 void 5479 pmap_init(void) 5480 { 5481 5482 /* 5483 * Set the available memory vars - These do not map to real memory 5484 * addresses and cannot as the physical memory is fragmented. 5485 * They are used by ps for %mem calculations. 5486 * One could argue whether this should be the entire memory or just 5487 * the memory that is useable in a user process. 5488 */ 5489 avail_start = ptoa(VM_PHYSMEM_PTR(0)->start); 5490 avail_end = ptoa(VM_PHYSMEM_PTR(vm_nphysseg - 1)->end); 5491 5492 /* 5493 * Now we need to free enough pv_entry structures to allow us to get 5494 * the kmem_map/kmem_object allocated and inited (done after this 5495 * function is finished). to do this we allocate one bootstrap page out 5496 * of kernel_map and use it to provide an initial pool of pv_entry 5497 * structures. we never free this page. 5498 */ 5499 pool_setlowat(&pmap_pv_pool, 5500 (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 5501 5502 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 5503 zeropage = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 5504 UVM_KMF_WIRED|UVM_KMF_ZERO); 5505 5506 pmap_initialized = true; 5507 } 5508 5509 static vaddr_t last_bootstrap_page = 0; 5510 static void *free_bootstrap_pages = NULL; 5511 5512 static void * 5513 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 5514 { 5515 extern void *pool_page_alloc(struct pool *, int); 5516 vaddr_t new_page; 5517 void *rv; 5518 5519 if (pmap_initialized) 5520 return (pool_page_alloc(pp, flags)); 5521 5522 if (free_bootstrap_pages) { 5523 rv = free_bootstrap_pages; 5524 free_bootstrap_pages = *((void **)rv); 5525 return (rv); 5526 } 5527 5528 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 5529 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 5530 5531 KASSERT(new_page > last_bootstrap_page); 5532 last_bootstrap_page = new_page; 5533 return ((void *)new_page); 5534 } 5535 5536 static void 5537 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 5538 { 5539 extern void pool_page_free(struct pool *, void *); 5540 5541 if ((vaddr_t)v <= last_bootstrap_page) { 5542 *((void **)v) = free_bootstrap_pages; 5543 free_bootstrap_pages = v; 5544 return; 5545 } 5546 5547 if (pmap_initialized) { 5548 pool_page_free(pp, v); 5549 return; 5550 } 5551 } 5552 5553 /* 5554 * pmap_postinit() 5555 * 5556 * This routine is called after the vm and kmem subsystems have been 5557 * initialised. This allows the pmap code to perform any initialisation 5558 * that can only be done one the memory allocation is in place. 5559 */ 5560 void 5561 pmap_postinit(void) 5562 { 5563 extern paddr_t physical_start, physical_end; 5564 struct l2_bucket *l2b; 5565 struct l1_ttable *l1; 5566 struct pglist plist; 5567 struct vm_page *m; 5568 pd_entry_t *pl1pt; 5569 pt_entry_t *ptep, pte; 5570 vaddr_t va, eva; 5571 u_int loop, needed; 5572 int error; 5573 5574 pool_cache_setlowat(&pmap_l2ptp_cache, 5575 (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 5576 pool_cache_setlowat(&pmap_l2dtable_cache, 5577 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 5578 5579 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 5580 needed -= 1; 5581 5582 l1 = kmem_alloc(sizeof(*l1) * needed, KM_SLEEP); 5583 5584 for (loop = 0; loop < needed; loop++, l1++) { 5585 /* Allocate a L1 page table */ 5586 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 5587 if (va == 0) 5588 panic("Cannot allocate L1 KVM"); 5589 5590 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 5591 physical_end, L1_TABLE_SIZE, 0, &plist, 1, 1); 5592 if (error) 5593 panic("Cannot allocate L1 physical pages"); 5594 5595 m = TAILQ_FIRST(&plist); 5596 eva = va + L1_TABLE_SIZE; 5597 pl1pt = (pd_entry_t *)va; 5598 5599 while (m && va < eva) { 5600 paddr_t pa = VM_PAGE_TO_PHYS(m); 5601 5602 pmap_kenter_pa(va, pa, 5603 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE); 5604 5605 /* 5606 * Make sure the L1 descriptor table is mapped 5607 * with the cache-mode set to write-through. 5608 */ 5609 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5610 KDASSERT(l2b != NULL); 5611 ptep = &l2b->l2b_kva[l2pte_index(va)]; 5612 pte = *ptep; 5613 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 5614 *ptep = pte; 5615 PTE_SYNC(ptep); 5616 cpu_tlb_flushD_SE(va); 5617 5618 va += PAGE_SIZE; 5619 m = TAILQ_NEXT(m, pageq.queue); 5620 } 5621 5622 #ifdef DIAGNOSTIC 5623 if (m) 5624 panic("pmap_alloc_l1pt: pglist not empty"); 5625 #endif /* DIAGNOSTIC */ 5626 5627 pmap_init_l1(l1, pl1pt); 5628 } 5629 5630 #ifdef DEBUG 5631 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 5632 needed); 5633 #endif 5634 } 5635 5636 /* 5637 * Note that the following routines are used by board-specific initialisation 5638 * code to configure the initial kernel page tables. 5639 * 5640 * If ARM32_NEW_VM_LAYOUT is *not* defined, they operate on the assumption that 5641 * L2 page-table pages are 4KB in size and use 4 L1 slots. This mimics the 5642 * behaviour of the old pmap, and provides an easy migration path for 5643 * initial bring-up of the new pmap on existing ports. Fortunately, 5644 * pmap_bootstrap() compensates for this hackery. This is only a stop-gap and 5645 * will be deprecated. 5646 * 5647 * If ARM32_NEW_VM_LAYOUT *is* defined, these functions deal with 1KB L2 page 5648 * tables. 5649 */ 5650 5651 /* 5652 * This list exists for the benefit of pmap_map_chunk(). It keeps track 5653 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 5654 * find them as necessary. 5655 * 5656 * Note that the data on this list MUST remain valid after initarm() returns, 5657 * as pmap_bootstrap() uses it to contruct L2 table metadata. 5658 */ 5659 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 5660 5661 static vaddr_t 5662 kernel_pt_lookup(paddr_t pa) 5663 { 5664 pv_addr_t *pv; 5665 5666 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 5667 #ifndef ARM32_NEW_VM_LAYOUT 5668 if (pv->pv_pa == (pa & ~PGOFSET)) 5669 return (pv->pv_va | (pa & PGOFSET)); 5670 #else 5671 if (pv->pv_pa == pa) 5672 return (pv->pv_va); 5673 #endif 5674 } 5675 return (0); 5676 } 5677 5678 /* 5679 * pmap_map_section: 5680 * 5681 * Create a single section mapping. 5682 */ 5683 void 5684 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 5685 { 5686 pd_entry_t *pde = (pd_entry_t *) l1pt; 5687 pd_entry_t fl; 5688 5689 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 5690 5691 switch (cache) { 5692 case PTE_NOCACHE: 5693 default: 5694 fl = 0; 5695 break; 5696 5697 case PTE_CACHE: 5698 fl = pte_l1_s_cache_mode; 5699 break; 5700 5701 case PTE_PAGETABLE: 5702 fl = pte_l1_s_cache_mode_pt; 5703 break; 5704 } 5705 5706 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 5707 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 5708 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 5709 } 5710 5711 /* 5712 * pmap_map_entry: 5713 * 5714 * Create a single page mapping. 5715 */ 5716 void 5717 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 5718 { 5719 pd_entry_t *pde = (pd_entry_t *) l1pt; 5720 pt_entry_t fl; 5721 pt_entry_t *pte; 5722 5723 KASSERT(((va | pa) & PGOFSET) == 0); 5724 5725 switch (cache) { 5726 case PTE_NOCACHE: 5727 default: 5728 fl = 0; 5729 break; 5730 5731 case PTE_CACHE: 5732 fl = pte_l2_s_cache_mode; 5733 break; 5734 5735 case PTE_PAGETABLE: 5736 fl = pte_l2_s_cache_mode_pt; 5737 break; 5738 } 5739 5740 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 5741 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 5742 5743 #ifndef ARM32_NEW_VM_LAYOUT 5744 pte = (pt_entry_t *) 5745 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 5746 #else 5747 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 5748 #endif 5749 if (pte == NULL) 5750 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 5751 5752 fl |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 5753 #ifndef ARM32_NEW_VM_LAYOUT 5754 pte += (va >> PGSHIFT) & 0x3ff; 5755 #else 5756 pte += l2pte_index(va); 5757 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 5758 #endif 5759 *pte = fl; 5760 PTE_SYNC(pte); 5761 } 5762 5763 /* 5764 * pmap_link_l2pt: 5765 * 5766 * Link the L2 page table specified by "l2pv" into the L1 5767 * page table at the slot for "va". 5768 */ 5769 void 5770 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 5771 { 5772 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 5773 u_int slot = va >> L1_S_SHIFT; 5774 5775 #ifndef ARM32_NEW_VM_LAYOUT 5776 KASSERT((va & ((L1_S_SIZE * 4) - 1)) == 0); 5777 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 5778 #endif 5779 5780 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 5781 5782 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 5783 #ifdef ARM32_NEW_VM_LAYOUT 5784 PTE_SYNC(&pde[slot]); 5785 #else 5786 pde[slot + 1] = proto | (l2pv->pv_pa + 0x400); 5787 pde[slot + 2] = proto | (l2pv->pv_pa + 0x800); 5788 pde[slot + 3] = proto | (l2pv->pv_pa + 0xc00); 5789 PTE_SYNC_RANGE(&pde[slot + 0], 4); 5790 #endif 5791 5792 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 5793 } 5794 5795 /* 5796 * pmap_map_chunk: 5797 * 5798 * Map a chunk of memory using the most efficient mappings 5799 * possible (section, large page, small page) into the 5800 * provided L1 and L2 tables at the specified virtual address. 5801 */ 5802 vsize_t 5803 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 5804 int prot, int cache) 5805 { 5806 pd_entry_t *pdep = (pd_entry_t *) l1pt; 5807 pt_entry_t *pte, f1, f2s, f2l; 5808 vsize_t resid; 5809 int i; 5810 5811 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 5812 5813 if (l1pt == 0) 5814 panic("pmap_map_chunk: no L1 table provided"); 5815 5816 #ifdef VERBOSE_INIT_ARM 5817 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 5818 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 5819 #endif 5820 5821 switch (cache) { 5822 case PTE_NOCACHE: 5823 default: 5824 f1 = 0; 5825 f2l = 0; 5826 f2s = 0; 5827 break; 5828 5829 case PTE_CACHE: 5830 f1 = pte_l1_s_cache_mode; 5831 f2l = pte_l2_l_cache_mode; 5832 f2s = pte_l2_s_cache_mode; 5833 break; 5834 5835 case PTE_PAGETABLE: 5836 f1 = pte_l1_s_cache_mode_pt; 5837 f2l = pte_l2_l_cache_mode_pt; 5838 f2s = pte_l2_s_cache_mode_pt; 5839 break; 5840 } 5841 5842 size = resid; 5843 5844 while (resid > 0) { 5845 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 5846 /* See if we can use a supersection mapping. */ 5847 if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) { 5848 /* Supersection are always domain 0 */ 5849 pd_entry_t pde = L1_SS_PROTO | pa | 5850 L1_S_PROT(PTE_KERNEL, prot) | f1; 5851 #ifdef VERBOSE_INIT_ARM 5852 printf("sS"); 5853 #endif 5854 for (size_t s = va >> L1_S_SHIFT, 5855 e = s + L1_SS_SIZE / L1_S_SIZE; 5856 s < e; 5857 s++) { 5858 pdep[s] = pde; 5859 PTE_SYNC(&pdep[s]); 5860 } 5861 va += L1_SS_SIZE; 5862 pa += L1_SS_SIZE; 5863 resid -= L1_SS_SIZE; 5864 continue; 5865 } 5866 #endif 5867 /* See if we can use a section mapping. */ 5868 if (L1_S_MAPPABLE_P(va, pa, resid)) { 5869 #ifdef VERBOSE_INIT_ARM 5870 printf("S"); 5871 #endif 5872 pdep[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 5873 L1_S_PROT(PTE_KERNEL, prot) | f1 | 5874 L1_S_DOM(PMAP_DOMAIN_KERNEL); 5875 PTE_SYNC(&pdep[va >> L1_S_SHIFT]); 5876 va += L1_S_SIZE; 5877 pa += L1_S_SIZE; 5878 resid -= L1_S_SIZE; 5879 continue; 5880 } 5881 5882 /* 5883 * Ok, we're going to use an L2 table. Make sure 5884 * one is actually in the corresponding L1 slot 5885 * for the current VA. 5886 */ 5887 if ((pdep[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 5888 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 5889 5890 #ifndef ARM32_NEW_VM_LAYOUT 5891 pte = (pt_entry_t *) 5892 kernel_pt_lookup(pdep[va >> L1_S_SHIFT] & L2_S_FRAME); 5893 #else 5894 pte = (pt_entry_t *) kernel_pt_lookup( 5895 pdep[L1_IDX(va)] & L1_C_ADDR_MASK); 5896 #endif 5897 if (pte == NULL) 5898 panic("pmap_map_chunk: can't find L2 table for VA" 5899 "0x%08lx", va); 5900 5901 /* See if we can use a L2 large page mapping. */ 5902 if (L2_L_MAPPABLE_P(va, pa, resid)) { 5903 #ifdef VERBOSE_INIT_ARM 5904 printf("L"); 5905 #endif 5906 for (i = 0; i < 16; i++) { 5907 #ifndef ARM32_NEW_VM_LAYOUT 5908 pte[((va >> PGSHIFT) & 0x3f0) + i] = 5909 L2_L_PROTO | pa | 5910 L2_L_PROT(PTE_KERNEL, prot) | f2l; 5911 PTE_SYNC(&pte[((va >> PGSHIFT) & 0x3f0) + i]); 5912 #else 5913 pte[l2pte_index(va) + i] = 5914 L2_L_PROTO | pa | 5915 L2_L_PROT(PTE_KERNEL, prot) | f2l; 5916 PTE_SYNC(&pte[l2pte_index(va) + i]); 5917 #endif 5918 } 5919 va += L2_L_SIZE; 5920 pa += L2_L_SIZE; 5921 resid -= L2_L_SIZE; 5922 continue; 5923 } 5924 5925 /* Use a small page mapping. */ 5926 #ifdef VERBOSE_INIT_ARM 5927 printf("P"); 5928 #endif 5929 #ifndef ARM32_NEW_VM_LAYOUT 5930 pte[(va >> PGSHIFT) & 0x3ff] = 5931 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 5932 PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]); 5933 #else 5934 pte[l2pte_index(va)] = 5935 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 5936 PTE_SYNC(&pte[l2pte_index(va)]); 5937 #endif 5938 va += PAGE_SIZE; 5939 pa += PAGE_SIZE; 5940 resid -= PAGE_SIZE; 5941 } 5942 #ifdef VERBOSE_INIT_ARM 5943 printf("\n"); 5944 #endif 5945 return (size); 5946 } 5947 5948 /********************** Static device map routines ***************************/ 5949 5950 static const struct pmap_devmap *pmap_devmap_table; 5951 5952 /* 5953 * Register the devmap table. This is provided in case early console 5954 * initialization needs to register mappings created by bootstrap code 5955 * before pmap_devmap_bootstrap() is called. 5956 */ 5957 void 5958 pmap_devmap_register(const struct pmap_devmap *table) 5959 { 5960 5961 pmap_devmap_table = table; 5962 } 5963 5964 /* 5965 * Map all of the static regions in the devmap table, and remember 5966 * the devmap table so other parts of the kernel can look up entries 5967 * later. 5968 */ 5969 void 5970 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 5971 { 5972 int i; 5973 5974 pmap_devmap_table = table; 5975 5976 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 5977 #ifdef VERBOSE_INIT_ARM 5978 printf("devmap: %08lx -> %08lx @ %08lx\n", 5979 pmap_devmap_table[i].pd_pa, 5980 pmap_devmap_table[i].pd_pa + 5981 pmap_devmap_table[i].pd_size - 1, 5982 pmap_devmap_table[i].pd_va); 5983 #endif 5984 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 5985 pmap_devmap_table[i].pd_pa, 5986 pmap_devmap_table[i].pd_size, 5987 pmap_devmap_table[i].pd_prot, 5988 pmap_devmap_table[i].pd_cache); 5989 } 5990 } 5991 5992 const struct pmap_devmap * 5993 pmap_devmap_find_pa(paddr_t pa, psize_t size) 5994 { 5995 uint64_t endpa; 5996 int i; 5997 5998 if (pmap_devmap_table == NULL) 5999 return (NULL); 6000 6001 endpa = (uint64_t)pa + (uint64_t)(size - 1); 6002 6003 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6004 if (pa >= pmap_devmap_table[i].pd_pa && 6005 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 6006 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 6007 return (&pmap_devmap_table[i]); 6008 } 6009 6010 return (NULL); 6011 } 6012 6013 const struct pmap_devmap * 6014 pmap_devmap_find_va(vaddr_t va, vsize_t size) 6015 { 6016 int i; 6017 6018 if (pmap_devmap_table == NULL) 6019 return (NULL); 6020 6021 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6022 if (va >= pmap_devmap_table[i].pd_va && 6023 va + size - 1 <= pmap_devmap_table[i].pd_va + 6024 pmap_devmap_table[i].pd_size - 1) 6025 return (&pmap_devmap_table[i]); 6026 } 6027 6028 return (NULL); 6029 } 6030 6031 /********************** PTE initialization routines **************************/ 6032 6033 /* 6034 * These routines are called when the CPU type is identified to set up 6035 * the PTE prototypes, cache modes, etc. 6036 * 6037 * The variables are always here, just in case modules need to reference 6038 * them (though, they shouldn't). 6039 */ 6040 6041 pt_entry_t pte_l1_s_cache_mode; 6042 pt_entry_t pte_l1_s_wc_mode; 6043 pt_entry_t pte_l1_s_cache_mode_pt; 6044 pt_entry_t pte_l1_s_cache_mask; 6045 6046 pt_entry_t pte_l2_l_cache_mode; 6047 pt_entry_t pte_l2_l_wc_mode; 6048 pt_entry_t pte_l2_l_cache_mode_pt; 6049 pt_entry_t pte_l2_l_cache_mask; 6050 6051 pt_entry_t pte_l2_s_cache_mode; 6052 pt_entry_t pte_l2_s_wc_mode; 6053 pt_entry_t pte_l2_s_cache_mode_pt; 6054 pt_entry_t pte_l2_s_cache_mask; 6055 6056 pt_entry_t pte_l1_s_prot_u; 6057 pt_entry_t pte_l1_s_prot_w; 6058 pt_entry_t pte_l1_s_prot_ro; 6059 pt_entry_t pte_l1_s_prot_mask; 6060 6061 pt_entry_t pte_l2_s_prot_u; 6062 pt_entry_t pte_l2_s_prot_w; 6063 pt_entry_t pte_l2_s_prot_ro; 6064 pt_entry_t pte_l2_s_prot_mask; 6065 6066 pt_entry_t pte_l2_l_prot_u; 6067 pt_entry_t pte_l2_l_prot_w; 6068 pt_entry_t pte_l2_l_prot_ro; 6069 pt_entry_t pte_l2_l_prot_mask; 6070 6071 pt_entry_t pte_l1_ss_proto; 6072 pt_entry_t pte_l1_s_proto; 6073 pt_entry_t pte_l1_c_proto; 6074 pt_entry_t pte_l2_s_proto; 6075 6076 void (*pmap_copy_page_func)(paddr_t, paddr_t); 6077 void (*pmap_zero_page_func)(paddr_t); 6078 6079 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 6080 void 6081 pmap_pte_init_generic(void) 6082 { 6083 6084 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 6085 pte_l1_s_wc_mode = L1_S_B; 6086 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 6087 6088 pte_l2_l_cache_mode = L2_B|L2_C; 6089 pte_l2_l_wc_mode = L2_B; 6090 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 6091 6092 pte_l2_s_cache_mode = L2_B|L2_C; 6093 pte_l2_s_wc_mode = L2_B; 6094 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 6095 6096 /* 6097 * If we have a write-through cache, set B and C. If 6098 * we have a write-back cache, then we assume setting 6099 * only C will make those pages write-through (except for those 6100 * Cortex CPUs which can read the L1 caches). 6101 */ 6102 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop 6103 #if ARM_MMU_V7 > 0 6104 || CPU_ID_CORTEX_P(curcpu()->ci_arm_cpuid) 6105 #endif 6106 #if ARM_MMU_V6 > 0 6107 || CPU_ID_ARM11_P(curcpu()->ci_arm_cpuid) /* arm116 errata 399234 */ 6108 #endif 6109 || false) { 6110 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 6111 pte_l2_l_cache_mode_pt = L2_B|L2_C; 6112 pte_l2_s_cache_mode_pt = L2_B|L2_C; 6113 } else { 6114 pte_l1_s_cache_mode_pt = L1_S_C; /* write through */ 6115 pte_l2_l_cache_mode_pt = L2_C; /* write through */ 6116 pte_l2_s_cache_mode_pt = L2_C; /* write through */ 6117 } 6118 6119 pte_l1_s_prot_u = L1_S_PROT_U_generic; 6120 pte_l1_s_prot_w = L1_S_PROT_W_generic; 6121 pte_l1_s_prot_ro = L1_S_PROT_RO_generic; 6122 pte_l1_s_prot_mask = L1_S_PROT_MASK_generic; 6123 6124 pte_l2_s_prot_u = L2_S_PROT_U_generic; 6125 pte_l2_s_prot_w = L2_S_PROT_W_generic; 6126 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 6127 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 6128 6129 pte_l2_l_prot_u = L2_L_PROT_U_generic; 6130 pte_l2_l_prot_w = L2_L_PROT_W_generic; 6131 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 6132 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 6133 6134 pte_l1_ss_proto = L1_SS_PROTO_generic; 6135 pte_l1_s_proto = L1_S_PROTO_generic; 6136 pte_l1_c_proto = L1_C_PROTO_generic; 6137 pte_l2_s_proto = L2_S_PROTO_generic; 6138 6139 pmap_copy_page_func = pmap_copy_page_generic; 6140 pmap_zero_page_func = pmap_zero_page_generic; 6141 } 6142 6143 #if defined(CPU_ARM8) 6144 void 6145 pmap_pte_init_arm8(void) 6146 { 6147 6148 /* 6149 * ARM8 is compatible with generic, but we need to use 6150 * the page tables uncached. 6151 */ 6152 pmap_pte_init_generic(); 6153 6154 pte_l1_s_cache_mode_pt = 0; 6155 pte_l2_l_cache_mode_pt = 0; 6156 pte_l2_s_cache_mode_pt = 0; 6157 } 6158 #endif /* CPU_ARM8 */ 6159 6160 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 6161 void 6162 pmap_pte_init_arm9(void) 6163 { 6164 6165 /* 6166 * ARM9 is compatible with generic, but we want to use 6167 * write-through caching for now. 6168 */ 6169 pmap_pte_init_generic(); 6170 6171 pte_l1_s_cache_mode = L1_S_C; 6172 pte_l2_l_cache_mode = L2_C; 6173 pte_l2_s_cache_mode = L2_C; 6174 6175 pte_l1_s_wc_mode = L1_S_B; 6176 pte_l2_l_wc_mode = L2_B; 6177 pte_l2_s_wc_mode = L2_B; 6178 6179 pte_l1_s_cache_mode_pt = L1_S_C; 6180 pte_l2_l_cache_mode_pt = L2_C; 6181 pte_l2_s_cache_mode_pt = L2_C; 6182 } 6183 #endif /* CPU_ARM9 && ARM9_CACHE_WRITE_THROUGH */ 6184 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 6185 6186 #if defined(CPU_ARM10) 6187 void 6188 pmap_pte_init_arm10(void) 6189 { 6190 6191 /* 6192 * ARM10 is compatible with generic, but we want to use 6193 * write-through caching for now. 6194 */ 6195 pmap_pte_init_generic(); 6196 6197 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 6198 pte_l2_l_cache_mode = L2_B | L2_C; 6199 pte_l2_s_cache_mode = L2_B | L2_C; 6200 6201 pte_l1_s_cache_mode = L1_S_B; 6202 pte_l2_l_cache_mode = L2_B; 6203 pte_l2_s_cache_mode = L2_B; 6204 6205 pte_l1_s_cache_mode_pt = L1_S_C; 6206 pte_l2_l_cache_mode_pt = L2_C; 6207 pte_l2_s_cache_mode_pt = L2_C; 6208 6209 } 6210 #endif /* CPU_ARM10 */ 6211 6212 #if defined(CPU_ARM11) && defined(ARM11_CACHE_WRITE_THROUGH) 6213 void 6214 pmap_pte_init_arm11(void) 6215 { 6216 6217 /* 6218 * ARM11 is compatible with generic, but we want to use 6219 * write-through caching for now. 6220 */ 6221 pmap_pte_init_generic(); 6222 6223 pte_l1_s_cache_mode = L1_S_C; 6224 pte_l2_l_cache_mode = L2_C; 6225 pte_l2_s_cache_mode = L2_C; 6226 6227 pte_l1_s_wc_mode = L1_S_B; 6228 pte_l2_l_wc_mode = L2_B; 6229 pte_l2_s_wc_mode = L2_B; 6230 6231 pte_l1_s_cache_mode_pt = L1_S_C; 6232 pte_l2_l_cache_mode_pt = L2_C; 6233 pte_l2_s_cache_mode_pt = L2_C; 6234 } 6235 #endif /* CPU_ARM11 && ARM11_CACHE_WRITE_THROUGH */ 6236 6237 #if ARM_MMU_SA1 == 1 6238 void 6239 pmap_pte_init_sa1(void) 6240 { 6241 6242 /* 6243 * The StrongARM SA-1 cache does not have a write-through 6244 * mode. So, do the generic initialization, then reset 6245 * the page table cache mode to B=1,C=1, and note that 6246 * the PTEs need to be sync'd. 6247 */ 6248 pmap_pte_init_generic(); 6249 6250 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 6251 pte_l2_l_cache_mode_pt = L2_B|L2_C; 6252 pte_l2_s_cache_mode_pt = L2_B|L2_C; 6253 6254 pmap_needs_pte_sync = 1; 6255 } 6256 #endif /* ARM_MMU_SA1 == 1*/ 6257 6258 #if ARM_MMU_XSCALE == 1 6259 #if (ARM_NMMUS > 1) 6260 static u_int xscale_use_minidata; 6261 #endif 6262 6263 void 6264 pmap_pte_init_xscale(void) 6265 { 6266 uint32_t auxctl; 6267 int write_through = 0; 6268 6269 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 6270 pte_l1_s_wc_mode = L1_S_B; 6271 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 6272 6273 pte_l2_l_cache_mode = L2_B|L2_C; 6274 pte_l2_l_wc_mode = L2_B; 6275 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 6276 6277 pte_l2_s_cache_mode = L2_B|L2_C; 6278 pte_l2_s_wc_mode = L2_B; 6279 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 6280 6281 pte_l1_s_cache_mode_pt = L1_S_C; 6282 pte_l2_l_cache_mode_pt = L2_C; 6283 pte_l2_s_cache_mode_pt = L2_C; 6284 6285 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 6286 /* 6287 * The XScale core has an enhanced mode where writes that 6288 * miss the cache cause a cache line to be allocated. This 6289 * is significantly faster than the traditional, write-through 6290 * behavior of this case. 6291 */ 6292 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 6293 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 6294 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 6295 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 6296 6297 #ifdef XSCALE_CACHE_WRITE_THROUGH 6298 /* 6299 * Some versions of the XScale core have various bugs in 6300 * their cache units, the work-around for which is to run 6301 * the cache in write-through mode. Unfortunately, this 6302 * has a major (negative) impact on performance. So, we 6303 * go ahead and run fast-and-loose, in the hopes that we 6304 * don't line up the planets in a way that will trip the 6305 * bugs. 6306 * 6307 * However, we give you the option to be slow-but-correct. 6308 */ 6309 write_through = 1; 6310 #elif defined(XSCALE_CACHE_WRITE_BACK) 6311 /* force write back cache mode */ 6312 write_through = 0; 6313 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 6314 /* 6315 * Intel PXA2[15]0 processors are known to have a bug in 6316 * write-back cache on revision 4 and earlier (stepping 6317 * A[01] and B[012]). Fixed for C0 and later. 6318 */ 6319 { 6320 uint32_t id, type; 6321 6322 id = cpufunc_id(); 6323 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 6324 6325 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 6326 if ((id & CPU_ID_REVISION_MASK) < 5) { 6327 /* write through for stepping A0-1 and B0-2 */ 6328 write_through = 1; 6329 } 6330 } 6331 } 6332 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 6333 6334 if (write_through) { 6335 pte_l1_s_cache_mode = L1_S_C; 6336 pte_l2_l_cache_mode = L2_C; 6337 pte_l2_s_cache_mode = L2_C; 6338 } 6339 6340 #if (ARM_NMMUS > 1) 6341 xscale_use_minidata = 1; 6342 #endif 6343 6344 pte_l1_s_prot_u = L1_S_PROT_U_xscale; 6345 pte_l1_s_prot_w = L1_S_PROT_W_xscale; 6346 pte_l1_s_prot_ro = L1_S_PROT_RO_xscale; 6347 pte_l1_s_prot_mask = L1_S_PROT_MASK_xscale; 6348 6349 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 6350 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 6351 pte_l2_s_prot_ro = L2_S_PROT_RO_xscale; 6352 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 6353 6354 pte_l2_l_prot_u = L2_L_PROT_U_xscale; 6355 pte_l2_l_prot_w = L2_L_PROT_W_xscale; 6356 pte_l2_l_prot_ro = L2_L_PROT_RO_xscale; 6357 pte_l2_l_prot_mask = L2_L_PROT_MASK_xscale; 6358 6359 pte_l1_ss_proto = L1_SS_PROTO_xscale; 6360 pte_l1_s_proto = L1_S_PROTO_xscale; 6361 pte_l1_c_proto = L1_C_PROTO_xscale; 6362 pte_l2_s_proto = L2_S_PROTO_xscale; 6363 6364 pmap_copy_page_func = pmap_copy_page_xscale; 6365 pmap_zero_page_func = pmap_zero_page_xscale; 6366 6367 /* 6368 * Disable ECC protection of page table access, for now. 6369 */ 6370 __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 6371 auxctl &= ~XSCALE_AUXCTL_P; 6372 __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 6373 } 6374 6375 /* 6376 * xscale_setup_minidata: 6377 * 6378 * Set up the mini-data cache clean area. We require the 6379 * caller to allocate the right amount of physically and 6380 * virtually contiguous space. 6381 */ 6382 void 6383 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 6384 { 6385 extern vaddr_t xscale_minidata_clean_addr; 6386 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 6387 pd_entry_t *pde = (pd_entry_t *) l1pt; 6388 pt_entry_t *pte; 6389 vsize_t size; 6390 uint32_t auxctl; 6391 6392 xscale_minidata_clean_addr = va; 6393 6394 /* Round it to page size. */ 6395 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 6396 6397 for (; size != 0; 6398 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 6399 #ifndef ARM32_NEW_VM_LAYOUT 6400 pte = (pt_entry_t *) 6401 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 6402 #else 6403 pte = (pt_entry_t *) kernel_pt_lookup( 6404 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 6405 #endif 6406 if (pte == NULL) 6407 panic("xscale_setup_minidata: can't find L2 table for " 6408 "VA 0x%08lx", va); 6409 #ifndef ARM32_NEW_VM_LAYOUT 6410 pte[(va >> PGSHIFT) & 0x3ff] = 6411 #else 6412 pte[l2pte_index(va)] = 6413 #endif 6414 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 6415 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); 6416 } 6417 6418 /* 6419 * Configure the mini-data cache for write-back with 6420 * read/write-allocate. 6421 * 6422 * NOTE: In order to reconfigure the mini-data cache, we must 6423 * make sure it contains no valid data! In order to do that, 6424 * we must issue a global data cache invalidate command! 6425 * 6426 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 6427 * THIS IS VERY IMPORTANT! 6428 */ 6429 6430 /* Invalidate data and mini-data. */ 6431 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 6432 __asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 6433 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 6434 __asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 6435 } 6436 6437 /* 6438 * Change the PTEs for the specified kernel mappings such that they 6439 * will use the mini data cache instead of the main data cache. 6440 */ 6441 void 6442 pmap_uarea(vaddr_t va) 6443 { 6444 struct l2_bucket *l2b; 6445 pt_entry_t *ptep, *sptep, pte; 6446 vaddr_t next_bucket, eva; 6447 6448 #if (ARM_NMMUS > 1) 6449 if (xscale_use_minidata == 0) 6450 return; 6451 #endif 6452 6453 eva = va + USPACE; 6454 6455 while (va < eva) { 6456 next_bucket = L2_NEXT_BUCKET(va); 6457 if (next_bucket > eva) 6458 next_bucket = eva; 6459 6460 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6461 KDASSERT(l2b != NULL); 6462 6463 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 6464 6465 while (va < next_bucket) { 6466 pte = *ptep; 6467 if (!l2pte_minidata(pte)) { 6468 cpu_dcache_wbinv_range(va, PAGE_SIZE); 6469 cpu_tlb_flushD_SE(va); 6470 *ptep = pte & ~L2_B; 6471 } 6472 ptep++; 6473 va += PAGE_SIZE; 6474 } 6475 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 6476 } 6477 cpu_cpwait(); 6478 } 6479 #endif /* ARM_MMU_XSCALE == 1 */ 6480 6481 6482 #if defined(CPU_ARM11MPCORE) 6483 6484 void 6485 pmap_pte_init_arm11mpcore(void) 6486 { 6487 6488 /* cache mode is controlled by 5 bits (B, C, TEX) */ 6489 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6; 6490 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6; 6491 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 6492 /* use extended small page (without APn, with TEX) */ 6493 pte_l2_s_cache_mask = L2_XS_CACHE_MASK_armv6; 6494 #else 6495 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6c; 6496 #endif 6497 6498 /* write-back, write-allocate */ 6499 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 6500 pte_l2_l_cache_mode = L2_C | L2_B | L2_V6_L_TEX(0x01); 6501 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 6502 pte_l2_s_cache_mode = L2_C | L2_B | L2_V6_XS_TEX(0x01); 6503 #else 6504 /* no TEX. read-allocate */ 6505 pte_l2_s_cache_mode = L2_C | L2_B; 6506 #endif 6507 /* 6508 * write-back, write-allocate for page tables. 6509 */ 6510 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 6511 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_V6_L_TEX(0x01); 6512 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 6513 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_V6_XS_TEX(0x01); 6514 #else 6515 pte_l2_s_cache_mode_pt = L2_C | L2_B; 6516 #endif 6517 6518 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 6519 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 6520 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 6521 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 6522 6523 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 6524 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 6525 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 6526 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 6527 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 6528 6529 #else 6530 /* with AP[0..3] */ 6531 pte_l2_s_prot_u = L2_S_PROT_U_generic; 6532 pte_l2_s_prot_w = L2_S_PROT_W_generic; 6533 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 6534 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 6535 #endif 6536 6537 #ifdef ARM11MPCORE_COMPAT_MMU 6538 /* with AP[0..3] */ 6539 pte_l2_l_prot_u = L2_L_PROT_U_generic; 6540 pte_l2_l_prot_w = L2_L_PROT_W_generic; 6541 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 6542 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 6543 6544 pte_l1_ss_proto = L1_SS_PROTO_armv6; 6545 pte_l1_s_proto = L1_S_PROTO_armv6; 6546 pte_l1_c_proto = L1_C_PROTO_armv6; 6547 pte_l2_s_proto = L2_S_PROTO_armv6c; 6548 #else 6549 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 6550 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 6551 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 6552 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 6553 6554 pte_l1_ss_proto = L1_SS_PROTO_armv6; 6555 pte_l1_s_proto = L1_S_PROTO_armv6; 6556 pte_l1_c_proto = L1_C_PROTO_armv6; 6557 pte_l2_s_proto = L2_S_PROTO_armv6n; 6558 #endif 6559 6560 pmap_copy_page_func = pmap_copy_page_generic; 6561 pmap_zero_page_func = pmap_zero_page_generic; 6562 pmap_needs_pte_sync = 1; 6563 } 6564 #endif /* CPU_ARM11MPCORE */ 6565 6566 6567 #if ARM_MMU_V7 == 1 6568 void 6569 pmap_pte_init_armv7(void) 6570 { 6571 /* 6572 * The ARMv7-A MMU is mostly compatible with generic. If the 6573 * AP field is zero, that now means "no access" rather than 6574 * read-only. The prototypes are a little different because of 6575 * the XN bit. 6576 */ 6577 pmap_pte_init_generic(); 6578 6579 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv7; 6580 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv7; 6581 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv7; 6582 6583 if (CPU_ID_CORTEX_A9_P(curcpu()->ci_arm_cpuid)) { 6584 /* 6585 * write-back, no write-allocate, shareable for normal pages. 6586 */ 6587 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_S; 6588 pte_l2_l_cache_mode = L2_C | L2_B | L2_XS_S; 6589 pte_l2_s_cache_mode = L2_C | L2_B | L2_XS_S; 6590 6591 /* 6592 * write-back, no write-allocate, shareable for page tables. 6593 */ 6594 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_S; 6595 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_XS_S; 6596 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_XS_S; 6597 } 6598 6599 pte_l1_s_prot_u = L1_S_PROT_U_armv7; 6600 pte_l1_s_prot_w = L1_S_PROT_W_armv7; 6601 pte_l1_s_prot_ro = L1_S_PROT_RO_armv7; 6602 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv7; 6603 6604 pte_l2_s_prot_u = L2_S_PROT_U_armv7; 6605 pte_l2_s_prot_w = L2_S_PROT_W_armv7; 6606 pte_l2_s_prot_ro = L2_S_PROT_RO_armv7; 6607 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv7; 6608 6609 pte_l2_l_prot_u = L2_L_PROT_U_armv7; 6610 pte_l2_l_prot_w = L2_L_PROT_W_armv7; 6611 pte_l2_l_prot_ro = L2_L_PROT_RO_armv7; 6612 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv7; 6613 6614 pte_l1_ss_proto = L1_SS_PROTO_armv7; 6615 pte_l1_s_proto = L1_S_PROTO_armv7; 6616 pte_l1_c_proto = L1_C_PROTO_armv7; 6617 pte_l2_s_proto = L2_S_PROTO_armv7; 6618 6619 pmap_needs_pte_sync = 1; 6620 } 6621 #endif /* ARM_MMU_V7 */ 6622 6623 /* 6624 * return the PA of the current L1 table, for use when handling a crash dump 6625 */ 6626 uint32_t pmap_kernel_L1_addr(void) 6627 { 6628 return pmap_kernel()->pm_l1->l1_physaddr; 6629 } 6630 6631 #if defined(DDB) 6632 /* 6633 * A couple of ddb-callable functions for dumping pmaps 6634 */ 6635 void pmap_dump_all(void); 6636 void pmap_dump(pmap_t); 6637 6638 void 6639 pmap_dump_all(void) 6640 { 6641 pmap_t pm; 6642 6643 LIST_FOREACH(pm, &pmap_pmaps, pm_list) { 6644 if (pm == pmap_kernel()) 6645 continue; 6646 pmap_dump(pm); 6647 printf("\n"); 6648 } 6649 } 6650 6651 static pt_entry_t ncptes[64]; 6652 static void pmap_dump_ncpg(pmap_t); 6653 6654 void 6655 pmap_dump(pmap_t pm) 6656 { 6657 struct l2_dtable *l2; 6658 struct l2_bucket *l2b; 6659 pt_entry_t *ptep, pte; 6660 vaddr_t l2_va, l2b_va, va; 6661 int i, j, k, occ, rows = 0; 6662 6663 if (pm == pmap_kernel()) 6664 printf("pmap_kernel (%p): ", pm); 6665 else 6666 printf("user pmap (%p): ", pm); 6667 6668 printf("domain %d, l1 at %p\n", pm->pm_domain, pm->pm_l1->l1_kva); 6669 6670 l2_va = 0; 6671 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 6672 l2 = pm->pm_l2[i]; 6673 6674 if (l2 == NULL || l2->l2_occupancy == 0) 6675 continue; 6676 6677 l2b_va = l2_va; 6678 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 6679 l2b = &l2->l2_bucket[j]; 6680 6681 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 6682 continue; 6683 6684 ptep = l2b->l2b_kva; 6685 6686 for (k = 0; k < 256 && ptep[k] == 0; k++) 6687 ; 6688 6689 k &= ~63; 6690 occ = l2b->l2b_occupancy; 6691 va = l2b_va + (k * 4096); 6692 for (; k < 256; k++, va += 0x1000) { 6693 char ch = ' '; 6694 if ((k % 64) == 0) { 6695 if ((rows % 8) == 0) { 6696 printf( 6697 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 6698 } 6699 printf("%08lx: ", va); 6700 } 6701 6702 ncptes[k & 63] = 0; 6703 pte = ptep[k]; 6704 if (pte == 0) { 6705 ch = '.'; 6706 } else { 6707 occ--; 6708 switch (pte & 0x0c) { 6709 case 0x00: 6710 ch = 'D'; /* No cache No buff */ 6711 break; 6712 case 0x04: 6713 ch = 'B'; /* No cache buff */ 6714 break; 6715 case 0x08: 6716 if (pte & 0x40) 6717 ch = 'm'; 6718 else 6719 ch = 'C'; /* Cache No buff */ 6720 break; 6721 case 0x0c: 6722 ch = 'F'; /* Cache Buff */ 6723 break; 6724 } 6725 6726 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 6727 ch += 0x20; 6728 6729 if ((pte & 0xc) == 0) 6730 ncptes[k & 63] = pte; 6731 } 6732 6733 if ((k % 64) == 63) { 6734 rows++; 6735 printf("%c\n", ch); 6736 pmap_dump_ncpg(pm); 6737 if (occ == 0) 6738 break; 6739 } else 6740 printf("%c", ch); 6741 } 6742 } 6743 } 6744 } 6745 6746 static void 6747 pmap_dump_ncpg(pmap_t pm) 6748 { 6749 struct vm_page *pg; 6750 struct vm_page_md *md; 6751 struct pv_entry *pv; 6752 int i; 6753 6754 for (i = 0; i < 63; i++) { 6755 if (ncptes[i] == 0) 6756 continue; 6757 6758 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 6759 if (pg == NULL) 6760 continue; 6761 md = VM_PAGE_TO_MD(pg); 6762 6763 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 6764 VM_PAGE_TO_PHYS(pg), 6765 md->krw_mappings, md->kro_mappings, 6766 md->urw_mappings, md->uro_mappings); 6767 6768 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 6769 printf(" %c va 0x%08lx, flags 0x%x\n", 6770 (pm == pv->pv_pmap) ? '*' : ' ', 6771 pv->pv_va, pv->pv_flags); 6772 } 6773 } 6774 } 6775 #endif 6776 6777 #ifdef PMAP_STEAL_MEMORY 6778 void 6779 pmap_boot_pageadd(pv_addr_t *newpv) 6780 { 6781 pv_addr_t *pv, *npv; 6782 6783 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 6784 if (newpv->pv_pa < pv->pv_va) { 6785 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 6786 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 6787 newpv->pv_size += pv->pv_size; 6788 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 6789 } 6790 pv = NULL; 6791 } else { 6792 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 6793 pv = npv) { 6794 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 6795 KASSERT(pv->pv_pa < newpv->pv_pa); 6796 if (newpv->pv_pa > npv->pv_pa) 6797 continue; 6798 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 6799 pv->pv_size += newpv->pv_size; 6800 return; 6801 } 6802 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 6803 break; 6804 newpv->pv_size += npv->pv_size; 6805 SLIST_INSERT_AFTER(pv, newpv, pv_list); 6806 SLIST_REMOVE_AFTER(newpv, pv_list); 6807 return; 6808 } 6809 } 6810 } 6811 6812 if (pv) { 6813 SLIST_INSERT_AFTER(pv, newpv, pv_list); 6814 } else { 6815 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 6816 } 6817 } 6818 6819 void 6820 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 6821 pv_addr_t *rpv) 6822 { 6823 pv_addr_t *pv, **pvp; 6824 struct vm_physseg *ps; 6825 size_t i; 6826 6827 KASSERT(amount & PGOFSET); 6828 KASSERT((mask & PGOFSET) == 0); 6829 KASSERT((match & PGOFSET) == 0); 6830 KASSERT(amount != 0); 6831 6832 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 6833 (pv = *pvp) != NULL; 6834 pvp = &SLIST_NEXT(pv, pv_list)) { 6835 pv_addr_t *newpv; 6836 psize_t off; 6837 /* 6838 * If this entry is too small to satify the request... 6839 */ 6840 KASSERT(pv->pv_size > 0); 6841 if (pv->pv_size < amount) 6842 continue; 6843 6844 for (off = 0; off <= mask; off += PAGE_SIZE) { 6845 if (((pv->pv_pa + off) & mask) == match 6846 && off + amount <= pv->pv_size) 6847 break; 6848 } 6849 if (off > mask) 6850 continue; 6851 6852 rpv->pv_va = pv->pv_va + off; 6853 rpv->pv_pa = pv->pv_pa + off; 6854 rpv->pv_size = amount; 6855 pv->pv_size -= amount; 6856 if (pv->pv_size == 0) { 6857 KASSERT(off == 0); 6858 KASSERT((vaddr_t) pv == rpv->pv_va); 6859 *pvp = SLIST_NEXT(pv, pv_list); 6860 } else if (off == 0) { 6861 KASSERT((vaddr_t) pv == rpv->pv_va); 6862 newpv = (pv_addr_t *) (rpv->pv_va + amount); 6863 *newpv = *pv; 6864 newpv->pv_pa += amount; 6865 newpv->pv_va += amount; 6866 *pvp = newpv; 6867 } else if (off < pv->pv_size) { 6868 newpv = (pv_addr_t *) (rpv->pv_va + amount); 6869 *newpv = *pv; 6870 newpv->pv_size -= off; 6871 newpv->pv_pa += off + amount; 6872 newpv->pv_va += off + amount; 6873 6874 SLIST_NEXT(pv, pv_list) = newpv; 6875 pv->pv_size = off; 6876 } else { 6877 KASSERT((vaddr_t) pv != rpv->pv_va); 6878 } 6879 memset((void *)rpv->pv_va, 0, amount); 6880 return; 6881 } 6882 6883 if (vm_nphysseg == 0) 6884 panic("pmap_boot_pagealloc: couldn't allocate memory"); 6885 6886 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 6887 (pv = *pvp) != NULL; 6888 pvp = &SLIST_NEXT(pv, pv_list)) { 6889 if (SLIST_NEXT(pv, pv_list) == NULL) 6890 break; 6891 } 6892 KASSERT(mask == 0); 6893 for (i = 0; i < vm_nphysseg; i++) { 6894 ps = VM_PHYSMEM_PTR(i); 6895 if (ps->avail_start == atop(pv->pv_pa + pv->pv_size) 6896 && pv->pv_va + pv->pv_size <= ptoa(ps->avail_end)) { 6897 rpv->pv_va = pv->pv_va; 6898 rpv->pv_pa = pv->pv_pa; 6899 rpv->pv_size = amount; 6900 *pvp = NULL; 6901 pmap_map_chunk(kernel_l1pt.pv_va, 6902 ptoa(ps->avail_start) + (pv->pv_va - pv->pv_pa), 6903 ptoa(ps->avail_start), 6904 amount - pv->pv_size, 6905 VM_PROT_READ|VM_PROT_WRITE, 6906 PTE_CACHE); 6907 ps->avail_start += atop(amount - pv->pv_size); 6908 /* 6909 * If we consumed the entire physseg, remove it. 6910 */ 6911 if (ps->avail_start == ps->avail_end) { 6912 for (--vm_nphysseg; i < vm_nphysseg; i++) 6913 VM_PHYSMEM_PTR_SWAP(i, i + 1); 6914 } 6915 memset((void *)rpv->pv_va, 0, rpv->pv_size); 6916 return; 6917 } 6918 } 6919 6920 panic("pmap_boot_pagealloc: couldn't allocate memory"); 6921 } 6922 6923 vaddr_t 6924 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 6925 { 6926 pv_addr_t pv; 6927 6928 pmap_boot_pagealloc(size, 0, 0, &pv); 6929 6930 return pv.pv_va; 6931 } 6932 #endif /* PMAP_STEAL_MEMORY */ 6933 6934 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 6935 { 6936 sysctl_createv(clog, 0, NULL, NULL, 6937 CTLFLAG_PERMANENT, 6938 CTLTYPE_NODE, "machdep", NULL, 6939 NULL, 0, NULL, 0, 6940 CTL_MACHDEP, CTL_EOL); 6941 6942 sysctl_createv(clog, 0, NULL, NULL, 6943 CTLFLAG_PERMANENT, 6944 CTLTYPE_INT, "kmpages", 6945 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 6946 NULL, 0, &pmap_kmpages, 0, 6947 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 6948 } 6949 6950 #ifdef PMAP_NEED_ALLOC_POOLPAGE 6951 struct vm_page * 6952 arm_pmap_alloc_poolpage(int flags) 6953 { 6954 /* 6955 * On some systems, only some pages may be "coherent" for dma and we 6956 * want to prefer those for pool pages (think mbufs) but fallback to 6957 * any page if none is available. 6958 */ 6959 if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) { 6960 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 6961 UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist); 6962 } 6963 6964 return uvm_pagealloc(NULL, 0, NULL, flags); 6965 } 6966 #endif 6967
Indexes created Thu Oct 23 22:10:10 GMT 2025