pmap.c revision 1.408
1 /* $NetBSD: pmap.c,v 1.408 2020/04/18 10:46:32 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999, 2020 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependent vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of in kernel VA space, 166 * then MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* Include header files */ 183 184 #include "opt_arm_debug.h" 185 #include "opt_cpuoptions.h" 186 #include "opt_ddb.h" 187 #include "opt_lockdebug.h" 188 #include "opt_multiprocessor.h" 189 190 #ifdef MULTIPROCESSOR 191 #define _INTR_PRIVATE 192 #endif 193 194 #include <sys/cdefs.h> 195 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.408 2020/04/18 10:46:32 skrll Exp $"); 196 197 #include <sys/atomic.h> 198 #include <sys/param.h> 199 #include <sys/types.h> 200 #include <sys/atomic.h> 201 #include <sys/bus.h> 202 #include <sys/cpu.h> 203 #include <sys/intr.h> 204 #include <sys/kernel.h> 205 #include <sys/kernhist.h> 206 #include <sys/kmem.h> 207 #include <sys/pool.h> 208 #include <sys/proc.h> 209 #include <sys/sysctl.h> 210 #include <sys/systm.h> 211 212 #include <uvm/uvm.h> 213 #include <uvm/pmap/pmap_pvt.h> 214 215 #include <arm/locore.h> 216 217 #ifdef DDB 218 #include <arm/db_machdep.h> 219 #endif 220 221 #ifdef VERBOSE_INIT_ARM 222 #define VPRINTF(...) printf(__VA_ARGS__) 223 #else 224 #define VPRINTF(...) __nothing 225 #endif 226 227 /* 228 * pmap_kernel() points here 229 */ 230 static struct pmap kernel_pmap_store = { 231 #ifndef ARM_MMU_EXTENDED 232 .pm_activated = true, 233 .pm_domain = PMAP_DOMAIN_KERNEL, 234 .pm_cstate.cs_all = PMAP_CACHE_STATE_ALL, 235 #endif 236 }; 237 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 238 #undef pmap_kernel 239 #define pmap_kernel() (&kernel_pmap_store) 240 #ifdef PMAP_NEED_ALLOC_POOLPAGE 241 int arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT; 242 #endif 243 244 /* 245 * Pool and cache that pmap structures are allocated from. 246 * We use a cache to avoid clearing the pm_l2[] array (1KB) 247 * in pmap_create(). 248 */ 249 static struct pool_cache pmap_cache; 250 251 /* 252 * Pool of PV structures 253 */ 254 static struct pool pmap_pv_pool; 255 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 256 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 257 static struct pool_allocator pmap_bootstrap_pv_allocator = { 258 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 259 }; 260 261 /* 262 * Pool and cache of l2_dtable structures. 263 * We use a cache to avoid clearing the structures when they're 264 * allocated. (196 bytes) 265 */ 266 static struct pool_cache pmap_l2dtable_cache; 267 static vaddr_t pmap_kernel_l2dtable_kva; 268 269 /* 270 * Pool and cache of L2 page descriptors. 271 * We use a cache to avoid clearing the descriptor table 272 * when they're allocated. (1KB) 273 */ 274 static struct pool_cache pmap_l2ptp_cache; 275 static vaddr_t pmap_kernel_l2ptp_kva; 276 static paddr_t pmap_kernel_l2ptp_phys; 277 278 #ifdef PMAPCOUNTERS 279 #define PMAP_EVCNT_INITIALIZER(name) \ 280 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 281 282 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 283 static struct evcnt pmap_ev_vac_clean_one = 284 PMAP_EVCNT_INITIALIZER("clean page (1 color)"); 285 static struct evcnt pmap_ev_vac_flush_one = 286 PMAP_EVCNT_INITIALIZER("flush page (1 color)"); 287 static struct evcnt pmap_ev_vac_flush_lots = 288 PMAP_EVCNT_INITIALIZER("flush page (2+ colors)"); 289 static struct evcnt pmap_ev_vac_flush_lots2 = 290 PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)"); 291 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one); 292 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one); 293 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots); 294 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2); 295 296 static struct evcnt pmap_ev_vac_color_new = 297 PMAP_EVCNT_INITIALIZER("new page color"); 298 static struct evcnt pmap_ev_vac_color_reuse = 299 PMAP_EVCNT_INITIALIZER("ok first page color"); 300 static struct evcnt pmap_ev_vac_color_ok = 301 PMAP_EVCNT_INITIALIZER("ok page color"); 302 static struct evcnt pmap_ev_vac_color_blind = 303 PMAP_EVCNT_INITIALIZER("blind page color"); 304 static struct evcnt pmap_ev_vac_color_change = 305 PMAP_EVCNT_INITIALIZER("change page color"); 306 static struct evcnt pmap_ev_vac_color_erase = 307 PMAP_EVCNT_INITIALIZER("erase page color"); 308 static struct evcnt pmap_ev_vac_color_none = 309 PMAP_EVCNT_INITIALIZER("no page color"); 310 static struct evcnt pmap_ev_vac_color_restore = 311 PMAP_EVCNT_INITIALIZER("restore page color"); 312 313 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 314 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 315 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 316 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 317 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 318 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 319 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 320 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 321 #endif 322 323 static struct evcnt pmap_ev_mappings = 324 PMAP_EVCNT_INITIALIZER("pages mapped"); 325 static struct evcnt pmap_ev_unmappings = 326 PMAP_EVCNT_INITIALIZER("pages unmapped"); 327 static struct evcnt pmap_ev_remappings = 328 PMAP_EVCNT_INITIALIZER("pages remapped"); 329 330 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 331 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 332 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 333 334 static struct evcnt pmap_ev_kernel_mappings = 335 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 336 static struct evcnt pmap_ev_kernel_unmappings = 337 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 338 static struct evcnt pmap_ev_kernel_remappings = 339 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 340 341 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 342 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 343 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 344 345 static struct evcnt pmap_ev_kenter_mappings = 346 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 347 static struct evcnt pmap_ev_kenter_unmappings = 348 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 349 static struct evcnt pmap_ev_kenter_remappings = 350 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 351 static struct evcnt pmap_ev_pt_mappings = 352 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 353 354 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 355 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 356 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 357 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 358 359 static struct evcnt pmap_ev_fixup_mod = 360 PMAP_EVCNT_INITIALIZER("page modification emulations"); 361 static struct evcnt pmap_ev_fixup_ref = 362 PMAP_EVCNT_INITIALIZER("page reference emulations"); 363 static struct evcnt pmap_ev_fixup_exec = 364 PMAP_EVCNT_INITIALIZER("exec pages fixed up"); 365 static struct evcnt pmap_ev_fixup_pdes = 366 PMAP_EVCNT_INITIALIZER("pdes fixed up"); 367 #ifndef ARM_MMU_EXTENDED 368 static struct evcnt pmap_ev_fixup_ptesync = 369 PMAP_EVCNT_INITIALIZER("ptesync fixed"); 370 #endif 371 372 EVCNT_ATTACH_STATIC(pmap_ev_fixup_mod); 373 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ref); 374 EVCNT_ATTACH_STATIC(pmap_ev_fixup_exec); 375 EVCNT_ATTACH_STATIC(pmap_ev_fixup_pdes); 376 #ifndef ARM_MMU_EXTENDED 377 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ptesync); 378 #endif 379 380 #ifdef PMAP_CACHE_VIPT 381 static struct evcnt pmap_ev_exec_mappings = 382 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 383 static struct evcnt pmap_ev_exec_cached = 384 PMAP_EVCNT_INITIALIZER("exec pages cached"); 385 386 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 387 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 388 389 static struct evcnt pmap_ev_exec_synced = 390 PMAP_EVCNT_INITIALIZER("exec pages synced"); 391 static struct evcnt pmap_ev_exec_synced_map = 392 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 393 static struct evcnt pmap_ev_exec_synced_unmap = 394 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 395 static struct evcnt pmap_ev_exec_synced_remap = 396 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 397 static struct evcnt pmap_ev_exec_synced_clearbit = 398 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 399 #ifndef ARM_MMU_EXTENDED 400 static struct evcnt pmap_ev_exec_synced_kremove = 401 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 402 #endif 403 404 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 405 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 406 #ifndef ARM_MMU_EXTENDED 407 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 408 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 409 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 410 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 411 #endif 412 413 static struct evcnt pmap_ev_exec_discarded_unmap = 414 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 415 static struct evcnt pmap_ev_exec_discarded_zero = 416 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 417 static struct evcnt pmap_ev_exec_discarded_copy = 418 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 419 static struct evcnt pmap_ev_exec_discarded_page_protect = 420 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 421 static struct evcnt pmap_ev_exec_discarded_clearbit = 422 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 423 static struct evcnt pmap_ev_exec_discarded_kremove = 424 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 425 #ifdef ARM_MMU_EXTENDED 426 static struct evcnt pmap_ev_exec_discarded_modfixup = 427 PMAP_EVCNT_INITIALIZER("exec pages discarded (MF)"); 428 #endif 429 430 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 431 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 432 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 433 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 434 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 435 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 436 #ifdef ARM_MMU_EXTENDED 437 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_modfixup); 438 #endif 439 #endif /* PMAP_CACHE_VIPT */ 440 441 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 442 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 443 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 444 445 EVCNT_ATTACH_STATIC(pmap_ev_updates); 446 EVCNT_ATTACH_STATIC(pmap_ev_collects); 447 EVCNT_ATTACH_STATIC(pmap_ev_activations); 448 449 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 450 #else 451 #define PMAPCOUNT(x) ((void)0) 452 #endif 453 454 #ifdef ARM_MMU_EXTENDED 455 void pmap_md_pdetab_activate(pmap_t, struct lwp *); 456 void pmap_md_pdetab_deactivate(pmap_t pm); 457 #endif 458 459 /* 460 * pmap copy/zero page, and mem(5) hook point 461 */ 462 static pt_entry_t *csrc_pte, *cdst_pte; 463 static vaddr_t csrcp, cdstp; 464 #ifdef MULTIPROCESSOR 465 static size_t cnptes; 466 #define cpu_csrc_pte(o) (csrc_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 467 #define cpu_cdst_pte(o) (cdst_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 468 #define cpu_csrcp(o) (csrcp + L2_S_SIZE * cnptes * cpu_number() + (o)) 469 #define cpu_cdstp(o) (cdstp + L2_S_SIZE * cnptes * cpu_number() + (o)) 470 #else 471 #define cpu_csrc_pte(o) (csrc_pte + ((o) >> L2_S_SHIFT)) 472 #define cpu_cdst_pte(o) (cdst_pte + ((o) >> L2_S_SHIFT)) 473 #define cpu_csrcp(o) (csrcp + (o)) 474 #define cpu_cdstp(o) (cdstp + (o)) 475 #endif 476 vaddr_t memhook; /* used by mem.c & others */ 477 kmutex_t memlock __cacheline_aligned; /* used by mem.c & others */ 478 kmutex_t pmap_lock __cacheline_aligned; 479 kmutex_t kpm_lock __cacheline_aligned; 480 extern void *msgbufaddr; 481 int pmap_kmpages; 482 /* 483 * Flag to indicate if pmap_init() has done its thing 484 */ 485 bool pmap_initialized; 486 487 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 488 /* 489 * Virtual end of direct-mapped memory 490 */ 491 vaddr_t pmap_directlimit; 492 #endif 493 494 /* 495 * Misc. locking data structures 496 */ 497 498 static inline void 499 pmap_acquire_pmap_lock(pmap_t pm) 500 { 501 #if defined(MULTIPROCESSOR) && defined(DDB) 502 if (__predict_false(db_onproc != NULL)) 503 return; 504 #endif 505 506 mutex_enter(&pm->pm_lock); 507 } 508 509 static inline void 510 pmap_release_pmap_lock(pmap_t pm) 511 { 512 #if defined(MULTIPROCESSOR) && defined(DDB) 513 if (__predict_false(db_onproc != NULL)) 514 return; 515 #endif 516 mutex_exit(&pm->pm_lock); 517 } 518 519 static inline void 520 pmap_acquire_page_lock(struct vm_page_md *md) 521 { 522 mutex_enter(&pmap_lock); 523 } 524 525 static inline void 526 pmap_release_page_lock(struct vm_page_md *md) 527 { 528 mutex_exit(&pmap_lock); 529 } 530 531 #ifdef DIAGNOSTIC 532 static inline int 533 pmap_page_locked_p(struct vm_page_md *md) 534 { 535 return mutex_owned(&pmap_lock); 536 } 537 #endif 538 539 540 /* 541 * Metadata for L1 translation tables. 542 */ 543 #ifndef ARM_MMU_EXTENDED 544 struct l1_ttable { 545 /* Entry on the L1 Table list */ 546 SLIST_ENTRY(l1_ttable) l1_link; 547 548 /* Entry on the L1 Least Recently Used list */ 549 TAILQ_ENTRY(l1_ttable) l1_lru; 550 551 /* Track how many domains are allocated from this L1 */ 552 volatile u_int l1_domain_use_count; 553 554 /* 555 * A free-list of domain numbers for this L1. 556 * We avoid using ffs() and a bitmap to track domains since ffs() 557 * is slow on ARM. 558 */ 559 uint8_t l1_domain_first; 560 uint8_t l1_domain_free[PMAP_DOMAINS]; 561 562 /* Physical address of this L1 page table */ 563 paddr_t l1_physaddr; 564 565 /* KVA of this L1 page table */ 566 pd_entry_t *l1_kva; 567 }; 568 569 /* 570 * L1 Page Tables are tracked using a Least Recently Used list. 571 * - New L1s are allocated from the HEAD. 572 * - Freed L1s are added to the TAIL. 573 * - Recently accessed L1s (where an 'access' is some change to one of 574 * the userland pmaps which owns this L1) are moved to the TAIL. 575 */ 576 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 577 static kmutex_t l1_lru_lock __cacheline_aligned; 578 579 /* 580 * A list of all L1 tables 581 */ 582 static SLIST_HEAD(, l1_ttable) l1_list; 583 #endif /* ARM_MMU_EXTENDED */ 584 585 /* 586 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 587 * 588 * This is normally 16MB worth L2 page descriptors for any given pmap. 589 * Reference counts are maintained for L2 descriptors so they can be 590 * freed when empty. 591 */ 592 struct l2_bucket { 593 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 594 paddr_t l2b_pa; /* Physical address of same */ 595 u_short l2b_l1slot; /* This L2 table's L1 index */ 596 u_short l2b_occupancy; /* How many active descriptors */ 597 }; 598 599 struct l2_dtable { 600 /* The number of L2 page descriptors allocated to this l2_dtable */ 601 u_int l2_occupancy; 602 603 /* List of L2 page descriptors */ 604 struct l2_bucket l2_bucket[L2_BUCKET_SIZE]; 605 }; 606 607 /* 608 * Given an L1 table index, calculate the corresponding l2_dtable index 609 * and bucket index within the l2_dtable. 610 */ 611 #define L2_BUCKET_XSHIFT (L2_BUCKET_XLOG2 - L1_S_SHIFT) 612 #define L2_BUCKET_XFRAME (~(vaddr_t)0 << L2_BUCKET_XLOG2) 613 #define L2_BUCKET_IDX(l1slot) ((l1slot) >> L2_BUCKET_XSHIFT) 614 #define L2_IDX(l1slot) (L2_BUCKET_IDX(l1slot) >> L2_BUCKET_LOG2) 615 #define L2_BUCKET(l1slot) (L2_BUCKET_IDX(l1slot) & (L2_BUCKET_SIZE - 1)) 616 617 __CTASSERT(0x100000000ULL == ((uint64_t)L2_SIZE * L2_BUCKET_SIZE * L1_S_SIZE)); 618 __CTASSERT(L2_BUCKET_XFRAME == ~(L2_BUCKET_XSIZE-1)); 619 620 /* 621 * Given a virtual address, this macro returns the 622 * virtual address required to drop into the next L2 bucket. 623 */ 624 #define L2_NEXT_BUCKET_VA(va) (((va) & L2_BUCKET_XFRAME) + L2_BUCKET_XSIZE) 625 626 /* 627 * L2 allocation. 628 */ 629 #define pmap_alloc_l2_dtable() \ 630 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 631 #define pmap_free_l2_dtable(l2) \ 632 pool_cache_put(&pmap_l2dtable_cache, (l2)) 633 #define pmap_alloc_l2_ptp(pap) \ 634 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 635 PR_NOWAIT, (pap))) 636 637 /* 638 * We try to map the page tables write-through, if possible. However, not 639 * all CPUs have a write-through cache mode, so on those we have to sync 640 * the cache when we frob page tables. 641 * 642 * We try to evaluate this at compile time, if possible. However, it's 643 * not always possible to do that, hence this run-time var. 644 */ 645 int pmap_needs_pte_sync; 646 647 /* 648 * Real definition of pv_entry. 649 */ 650 struct pv_entry { 651 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 652 pmap_t pv_pmap; /* pmap where mapping lies */ 653 vaddr_t pv_va; /* virtual address for mapping */ 654 u_int pv_flags; /* flags */ 655 }; 656 657 /* 658 * Macros to determine if a mapping might be resident in the 659 * instruction/data cache and/or TLB 660 */ 661 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED) 662 /* 663 * Speculative loads by Cortex cores can cause TLB entries to be filled even if 664 * there are no explicit accesses, so there may be always be TLB entries to 665 * flush. If we used ASIDs then this would not be a problem. 666 */ 667 #define PV_BEEN_EXECD(f) (((f) & PVF_EXEC) == PVF_EXEC) 668 #define PV_BEEN_REFD(f) (true) 669 #else 670 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 671 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 672 #endif 673 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 674 #define PV_IS_KENTRY_P(f) (((f) & PVF_KENTRY) != 0) 675 #define PV_IS_WRITE_P(f) (((f) & PVF_WRITE) != 0) 676 677 /* 678 * Local prototypes 679 */ 680 static bool pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t, size_t); 681 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 682 pt_entry_t **); 683 static bool pmap_is_current(pmap_t) __unused; 684 static bool pmap_is_cached(pmap_t); 685 static void pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *, 686 pmap_t, vaddr_t, u_int); 687 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t); 688 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 689 static u_int pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t, 690 u_int, u_int); 691 692 static void pmap_pinit(pmap_t); 693 static int pmap_pmap_ctor(void *, void *, int); 694 695 static void pmap_alloc_l1(pmap_t); 696 static void pmap_free_l1(pmap_t); 697 #ifndef ARM_MMU_EXTENDED 698 static void pmap_use_l1(pmap_t); 699 #endif 700 701 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 702 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 703 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 704 static int pmap_l2ptp_ctor(void *, void *, int); 705 static int pmap_l2dtable_ctor(void *, void *, int); 706 707 static void pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 708 #ifdef PMAP_CACHE_VIVT 709 static void pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 710 static void pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 711 #endif 712 713 static void pmap_clearbit(struct vm_page_md *, paddr_t, u_int); 714 #ifdef PMAP_CACHE_VIVT 715 static bool pmap_clean_page(struct vm_page_md *, bool); 716 #endif 717 #ifdef PMAP_CACHE_VIPT 718 static void pmap_syncicache_page(struct vm_page_md *, paddr_t); 719 enum pmap_flush_op { 720 PMAP_FLUSH_PRIMARY, 721 PMAP_FLUSH_SECONDARY, 722 PMAP_CLEAN_PRIMARY 723 }; 724 #ifndef ARM_MMU_EXTENDED 725 static void pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op); 726 #endif 727 #endif 728 static void pmap_page_remove(struct vm_page_md *, paddr_t); 729 static void pmap_pv_remove(paddr_t); 730 731 #ifndef ARM_MMU_EXTENDED 732 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 733 #endif 734 static vaddr_t kernel_pt_lookup(paddr_t); 735 736 #ifdef ARM_MMU_EXTENDED 737 static struct pool_cache pmap_l1tt_cache; 738 739 static int pmap_l1tt_ctor(void *, void *, int); 740 static void * pmap_l1tt_alloc(struct pool *, int); 741 static void pmap_l1tt_free(struct pool *, void *); 742 743 static struct pool_allocator pmap_l1tt_allocator = { 744 .pa_alloc = pmap_l1tt_alloc, 745 .pa_free = pmap_l1tt_free, 746 .pa_pagesz = L1TT_SIZE, 747 }; 748 #endif 749 750 /* 751 * Misc variables 752 */ 753 vaddr_t virtual_avail; 754 vaddr_t virtual_end; 755 vaddr_t pmap_curmaxkvaddr; 756 757 paddr_t avail_start; 758 paddr_t avail_end; 759 760 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 761 pv_addr_t kernelpages; 762 pv_addr_t kernel_l1pt; 763 pv_addr_t systempage; 764 765 #ifdef PMAP_CACHE_VIPT 766 #define PMAP_VALIDATE_MD_PAGE(md) \ 767 KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \ 768 "(md) %p: attrs=%#x urw=%u krw=%u", (md), \ 769 (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings); 770 #endif /* PMAP_CACHE_VIPT */ 771 /* 772 * A bunch of routines to conditionally flush the caches/TLB depending 773 * on whether the specified pmap actually needs to be flushed at any 774 * given time. 775 */ 776 static inline void 777 pmap_tlb_flush_SE(pmap_t pm, vaddr_t va, u_int flags) 778 { 779 #ifdef ARM_MMU_EXTENDED 780 pmap_tlb_invalidate_addr(pm, va); 781 #else 782 if (pm->pm_cstate.cs_tlb_id != 0) { 783 if (PV_BEEN_EXECD(flags)) { 784 cpu_tlb_flushID_SE(va); 785 } else if (PV_BEEN_REFD(flags)) { 786 cpu_tlb_flushD_SE(va); 787 } 788 } 789 #endif /* ARM_MMU_EXTENDED */ 790 } 791 792 #ifndef ARM_MMU_EXTENDED 793 static inline void 794 pmap_tlb_flushID(pmap_t pm) 795 { 796 if (pm->pm_cstate.cs_tlb_id) { 797 cpu_tlb_flushID(); 798 #if ARM_MMU_V7 == 0 799 /* 800 * Speculative loads by Cortex cores can cause TLB entries to 801 * be filled even if there are no explicit accesses, so there 802 * may be always be TLB entries to flush. If we used ASIDs 803 * then it would not be a problem. 804 * This is not true for other CPUs. 805 */ 806 pm->pm_cstate.cs_tlb = 0; 807 #endif /* ARM_MMU_V7 */ 808 } 809 } 810 811 static inline void 812 pmap_tlb_flushD(pmap_t pm) 813 { 814 if (pm->pm_cstate.cs_tlb_d) { 815 cpu_tlb_flushD(); 816 #if ARM_MMU_V7 == 0 817 /* 818 * Speculative loads by Cortex cores can cause TLB entries to 819 * be filled even if there are no explicit accesses, so there 820 * may be always be TLB entries to flush. If we used ASIDs 821 * then it would not be a problem. 822 * This is not true for other CPUs. 823 */ 824 pm->pm_cstate.cs_tlb_d = 0; 825 #endif /* ARM_MMU_V7 */ 826 } 827 } 828 #endif /* ARM_MMU_EXTENDED */ 829 830 #ifdef PMAP_CACHE_VIVT 831 static inline void 832 pmap_cache_wbinv_page(pmap_t pm, vaddr_t va, bool do_inv, u_int flags) 833 { 834 if (PV_BEEN_EXECD(flags) && pm->pm_cstate.cs_cache_id) { 835 cpu_idcache_wbinv_range(va, PAGE_SIZE); 836 } else if (PV_BEEN_REFD(flags) && pm->pm_cstate.cs_cache_d) { 837 if (do_inv) { 838 if (flags & PVF_WRITE) 839 cpu_dcache_wbinv_range(va, PAGE_SIZE); 840 else 841 cpu_dcache_inv_range(va, PAGE_SIZE); 842 } else if (flags & PVF_WRITE) { 843 cpu_dcache_wb_range(va, PAGE_SIZE); 844 } 845 } 846 } 847 848 static inline void 849 pmap_cache_wbinv_all(pmap_t pm, u_int flags) 850 { 851 if (PV_BEEN_EXECD(flags)) { 852 if (pm->pm_cstate.cs_cache_id) { 853 cpu_idcache_wbinv_all(); 854 pm->pm_cstate.cs_cache = 0; 855 } 856 } else if (pm->pm_cstate.cs_cache_d) { 857 cpu_dcache_wbinv_all(); 858 pm->pm_cstate.cs_cache_d = 0; 859 } 860 } 861 #endif /* PMAP_CACHE_VIVT */ 862 863 static inline uint8_t 864 pmap_domain(pmap_t pm) 865 { 866 #ifdef ARM_MMU_EXTENDED 867 return pm == pmap_kernel() ? PMAP_DOMAIN_KERNEL : PMAP_DOMAIN_USER; 868 #else 869 return pm->pm_domain; 870 #endif 871 } 872 873 static inline pd_entry_t * 874 pmap_l1_kva(pmap_t pm) 875 { 876 #ifdef ARM_MMU_EXTENDED 877 return pm->pm_l1; 878 #else 879 return pm->pm_l1->l1_kva; 880 #endif 881 } 882 883 static inline bool 884 pmap_is_current(pmap_t pm) 885 { 886 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 887 return true; 888 889 return false; 890 } 891 892 static inline bool 893 pmap_is_cached(pmap_t pm) 894 { 895 #ifdef ARM_MMU_EXTENDED 896 if (pm == pmap_kernel()) 897 return true; 898 #ifdef MULTIPROCESSOR 899 // Is this pmap active on any CPU? 900 if (!kcpuset_iszero(pm->pm_active)) 901 return true; 902 #else 903 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 904 // Is this pmap active? 905 if (PMAP_PAI_ASIDVALID_P(PMAP_PAI(pm, ti), ti)) 906 return true; 907 #endif 908 #else 909 struct cpu_info * const ci = curcpu(); 910 if (pm == pmap_kernel() || ci->ci_pmap_lastuser == NULL 911 || ci->ci_pmap_lastuser == pm) 912 return true; 913 #endif /* ARM_MMU_EXTENDED */ 914 915 return false; 916 } 917 918 /* 919 * PTE_SYNC_CURRENT: 920 * 921 * Make sure the pte is written out to RAM. 922 * We need to do this for one of two cases: 923 * - We're dealing with the kernel pmap 924 * - There is no pmap active in the cache/tlb. 925 * - The specified pmap is 'active' in the cache/tlb. 926 */ 927 928 #ifdef PMAP_INCLUDE_PTE_SYNC 929 static inline void 930 pmap_pte_sync_current(pmap_t pm, pt_entry_t *ptep) 931 { 932 if (PMAP_NEEDS_PTE_SYNC && pmap_is_cached(pm)) 933 PTE_SYNC(ptep); 934 arm_dsb(); 935 } 936 937 # define PTE_SYNC_CURRENT(pm, ptep) pmap_pte_sync_current(pm, ptep) 938 #else 939 # define PTE_SYNC_CURRENT(pm, ptep) __nothing 940 #endif 941 942 /* 943 * main pv_entry manipulation functions: 944 * pmap_enter_pv: enter a mapping onto a vm_page list 945 * pmap_remove_pv: remove a mapping from a vm_page list 946 * 947 * NOTE: pmap_enter_pv expects to lock the pvh itself 948 * pmap_remove_pv expects the caller to lock the pvh before calling 949 */ 950 951 /* 952 * pmap_enter_pv: enter a mapping onto a vm_page lst 953 * 954 * => caller should hold the proper lock on pmap_main_lock 955 * => caller should have pmap locked 956 * => we will gain the lock on the vm_page and allocate the new pv_entry 957 * => caller should adjust ptp's wire_count before calling 958 * => caller should not adjust pmap's wire_count 959 */ 960 static void 961 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm, 962 vaddr_t va, u_int flags) 963 { 964 UVMHIST_FUNC(__func__); 965 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 966 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 967 UVMHIST_LOG(maphist, "...pv %#jx flags %#jx", 968 (uintptr_t)pv, flags, 0, 0); 969 970 struct pv_entry **pvp; 971 972 pv->pv_pmap = pm; 973 pv->pv_va = va; 974 pv->pv_flags = flags; 975 976 pvp = &SLIST_FIRST(&md->pvh_list); 977 #ifdef PMAP_CACHE_VIPT 978 /* 979 * Insert unmanaged entries, writeable first, at the head of 980 * the pv list. 981 */ 982 if (__predict_true(!PV_IS_KENTRY_P(flags))) { 983 while (*pvp != NULL && PV_IS_KENTRY_P((*pvp)->pv_flags)) 984 pvp = &SLIST_NEXT(*pvp, pv_link); 985 } 986 if (!PV_IS_WRITE_P(flags)) { 987 while (*pvp != NULL && PV_IS_WRITE_P((*pvp)->pv_flags)) 988 pvp = &SLIST_NEXT(*pvp, pv_link); 989 } 990 #endif 991 SLIST_NEXT(pv, pv_link) = *pvp; /* add to ... */ 992 *pvp = pv; /* ... locked list */ 993 md->pvh_attrs |= flags & (PVF_REF | PVF_MOD); 994 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 995 if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE) 996 md->pvh_attrs |= PVF_KMOD; 997 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 998 md->pvh_attrs |= PVF_DIRTY; 999 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1000 #endif 1001 if (pm == pmap_kernel()) { 1002 PMAPCOUNT(kernel_mappings); 1003 if (flags & PVF_WRITE) 1004 md->krw_mappings++; 1005 else 1006 md->kro_mappings++; 1007 } else { 1008 if (flags & PVF_WRITE) 1009 md->urw_mappings++; 1010 else 1011 md->uro_mappings++; 1012 } 1013 1014 #ifdef PMAP_CACHE_VIPT 1015 #ifndef ARM_MMU_EXTENDED 1016 /* 1017 * Even though pmap_vac_me_harder will set PVF_WRITE for us, 1018 * do it here as well to keep the mappings & KVF_WRITE consistent. 1019 */ 1020 if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) { 1021 md->pvh_attrs |= PVF_WRITE; 1022 } 1023 #endif 1024 /* 1025 * If this is an exec mapping and its the first exec mapping 1026 * for this page, make sure to sync the I-cache. 1027 */ 1028 if (PV_IS_EXEC_P(flags)) { 1029 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 1030 pmap_syncicache_page(md, pa); 1031 PMAPCOUNT(exec_synced_map); 1032 } 1033 PMAPCOUNT(exec_mappings); 1034 } 1035 #endif 1036 1037 PMAPCOUNT(mappings); 1038 1039 if (pv->pv_flags & PVF_WIRED) 1040 ++pm->pm_stats.wired_count; 1041 } 1042 1043 /* 1044 * 1045 * pmap_find_pv: Find a pv entry 1046 * 1047 * => caller should hold lock on vm_page 1048 */ 1049 static inline struct pv_entry * 1050 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va) 1051 { 1052 struct pv_entry *pv; 1053 1054 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1055 if (pm == pv->pv_pmap && va == pv->pv_va) 1056 break; 1057 } 1058 1059 return pv; 1060 } 1061 1062 /* 1063 * pmap_remove_pv: try to remove a mapping from a pv_list 1064 * 1065 * => caller should hold proper lock on pmap_main_lock 1066 * => pmap should be locked 1067 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1068 * => caller should adjust ptp's wire_count and free PTP if needed 1069 * => caller should NOT adjust pmap's wire_count 1070 * => we return the removed pv 1071 */ 1072 static struct pv_entry * 1073 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1074 { 1075 UVMHIST_FUNC(__func__); 1076 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 1077 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 1078 1079 struct pv_entry *pv, **prevptr; 1080 1081 prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */ 1082 pv = *prevptr; 1083 1084 while (pv) { 1085 if (pv->pv_pmap == pm && pv->pv_va == va) { /* match? */ 1086 UVMHIST_LOG(maphist, "pm %#jx md %#jx flags %#jx", 1087 (uintptr_t)pm, (uintptr_t)md, pv->pv_flags, 0); 1088 if (pv->pv_flags & PVF_WIRED) { 1089 --pm->pm_stats.wired_count; 1090 } 1091 *prevptr = SLIST_NEXT(pv, pv_link); /* remove it! */ 1092 if (pm == pmap_kernel()) { 1093 PMAPCOUNT(kernel_unmappings); 1094 if (pv->pv_flags & PVF_WRITE) 1095 md->krw_mappings--; 1096 else 1097 md->kro_mappings--; 1098 } else { 1099 if (pv->pv_flags & PVF_WRITE) 1100 md->urw_mappings--; 1101 else 1102 md->uro_mappings--; 1103 } 1104 1105 PMAPCOUNT(unmappings); 1106 #ifdef PMAP_CACHE_VIPT 1107 /* 1108 * If this page has had an exec mapping, then if 1109 * this was the last mapping, discard the contents, 1110 * otherwise sync the i-cache for this page. 1111 */ 1112 if (PV_IS_EXEC_P(md->pvh_attrs)) { 1113 if (SLIST_EMPTY(&md->pvh_list)) { 1114 md->pvh_attrs &= ~PVF_EXEC; 1115 PMAPCOUNT(exec_discarded_unmap); 1116 } else if (pv->pv_flags & PVF_WRITE) { 1117 pmap_syncicache_page(md, pa); 1118 PMAPCOUNT(exec_synced_unmap); 1119 } 1120 } 1121 #endif /* PMAP_CACHE_VIPT */ 1122 break; 1123 } 1124 prevptr = &SLIST_NEXT(pv, pv_link); /* previous pointer */ 1125 pv = *prevptr; /* advance */ 1126 } 1127 1128 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1129 /* 1130 * If we no longer have a WRITEABLE KENTRY at the head of list, 1131 * clear the KMOD attribute from the page. 1132 */ 1133 if (SLIST_FIRST(&md->pvh_list) == NULL 1134 || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE) 1135 md->pvh_attrs &= ~PVF_KMOD; 1136 1137 /* 1138 * If this was a writeable page and there are no more writeable 1139 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1140 * the contents to memory. 1141 */ 1142 if (arm_cache_prefer_mask != 0) { 1143 if (md->krw_mappings + md->urw_mappings == 0) 1144 md->pvh_attrs &= ~PVF_WRITE; 1145 PMAP_VALIDATE_MD_PAGE(md); 1146 } 1147 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1148 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1149 1150 /* return removed pv */ 1151 return pv; 1152 } 1153 1154 /* 1155 * 1156 * pmap_modify_pv: Update pv flags 1157 * 1158 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1159 * => caller should NOT adjust pmap's wire_count 1160 * => caller must call pmap_vac_me_harder() if writable status of a page 1161 * may have changed. 1162 * => we return the old flags 1163 * 1164 * Modify a physical-virtual mapping in the pv table 1165 */ 1166 static u_int 1167 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va, 1168 u_int clr_mask, u_int set_mask) 1169 { 1170 struct pv_entry *npv; 1171 u_int flags, oflags; 1172 UVMHIST_FUNC(__func__); 1173 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 1174 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 1175 UVMHIST_LOG(maphist, "... clr %#jx set %#jx", clr_mask, set_mask, 0, 0); 1176 1177 KASSERT(!PV_IS_KENTRY_P(clr_mask)); 1178 KASSERT(!PV_IS_KENTRY_P(set_mask)); 1179 1180 if ((npv = pmap_find_pv(md, pm, va)) == NULL) { 1181 UVMHIST_LOG(maphist, "<--- done (not found)", 0, 0, 0, 0); 1182 return 0; 1183 } 1184 1185 /* 1186 * There is at least one VA mapping this page. 1187 */ 1188 1189 if (clr_mask & (PVF_REF | PVF_MOD)) { 1190 md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1191 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1192 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1193 md->pvh_attrs |= PVF_DIRTY; 1194 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1195 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1196 } 1197 1198 oflags = npv->pv_flags; 1199 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1200 1201 if ((flags ^ oflags) & PVF_WIRED) { 1202 if (flags & PVF_WIRED) 1203 ++pm->pm_stats.wired_count; 1204 else 1205 --pm->pm_stats.wired_count; 1206 } 1207 1208 if ((flags ^ oflags) & PVF_WRITE) { 1209 if (pm == pmap_kernel()) { 1210 if (flags & PVF_WRITE) { 1211 md->krw_mappings++; 1212 md->kro_mappings--; 1213 } else { 1214 md->kro_mappings++; 1215 md->krw_mappings--; 1216 } 1217 } else { 1218 if (flags & PVF_WRITE) { 1219 md->urw_mappings++; 1220 md->uro_mappings--; 1221 } else { 1222 md->uro_mappings++; 1223 md->urw_mappings--; 1224 } 1225 } 1226 } 1227 #ifdef PMAP_CACHE_VIPT 1228 if (arm_cache_prefer_mask != 0) { 1229 if (md->urw_mappings + md->krw_mappings == 0) { 1230 md->pvh_attrs &= ~PVF_WRITE; 1231 } else { 1232 md->pvh_attrs |= PVF_WRITE; 1233 } 1234 } 1235 /* 1236 * We have two cases here: the first is from enter_pv (new exec 1237 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1238 * Since in latter, pmap_enter_pv won't do anything, we just have 1239 * to do what pmap_remove_pv would do. 1240 */ 1241 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs)) 1242 || (PV_IS_EXEC_P(md->pvh_attrs) 1243 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1244 pmap_syncicache_page(md, pa); 1245 PMAPCOUNT(exec_synced_remap); 1246 } 1247 #ifndef ARM_MMU_EXTENDED 1248 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1249 #endif /* !ARM_MMU_EXTENDED */ 1250 #endif /* PMAP_CACHE_VIPT */ 1251 1252 PMAPCOUNT(remappings); 1253 1254 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1255 1256 return oflags; 1257 } 1258 1259 1260 #if defined(ARM_MMU_EXTENDED) 1261 int 1262 pmap_maxproc_set(int nmaxproc) 1263 { 1264 static const char pmap_l1ttpool_warnmsg[] = 1265 "WARNING: l1ttpool limit reached; increase kern.maxproc"; 1266 1267 pool_cache_prime(&pmap_l1tt_cache, nmaxproc); 1268 1269 /* 1270 * Set the hard limit on the pmap_l1tt_cache to the number 1271 * of processes the kernel is to support. Log the limit 1272 * reached message max once a minute. 1273 */ 1274 pool_cache_sethardlimit(&pmap_l1tt_cache, nmaxproc, 1275 pmap_l1ttpool_warnmsg, 60); 1276 1277 return 0; 1278 } 1279 1280 #endif 1281 1282 /* 1283 * Allocate an L1 translation table for the specified pmap. 1284 * This is called at pmap creation time. 1285 */ 1286 static void 1287 pmap_alloc_l1(pmap_t pm) 1288 { 1289 #ifdef ARM_MMU_EXTENDED 1290 vaddr_t va = (vaddr_t)pool_cache_get_paddr(&pmap_l1tt_cache, PR_WAITOK, 1291 &pm->pm_l1_pa); 1292 1293 pm->pm_l1 = (pd_entry_t *)va; 1294 PTE_SYNC_RANGE(pm->pm_l1, L1TT_SIZE / sizeof(pt_entry_t)); 1295 #else 1296 struct l1_ttable *l1; 1297 uint8_t domain; 1298 1299 /* 1300 * Remove the L1 at the head of the LRU list 1301 */ 1302 mutex_spin_enter(&l1_lru_lock); 1303 l1 = TAILQ_FIRST(&l1_lru_list); 1304 KDASSERT(l1 != NULL); 1305 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1306 1307 /* 1308 * Pick the first available domain number, and update 1309 * the link to the next number. 1310 */ 1311 domain = l1->l1_domain_first; 1312 l1->l1_domain_first = l1->l1_domain_free[domain]; 1313 1314 /* 1315 * If there are still free domain numbers in this L1, 1316 * put it back on the TAIL of the LRU list. 1317 */ 1318 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1319 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1320 1321 mutex_spin_exit(&l1_lru_lock); 1322 1323 /* 1324 * Fix up the relevant bits in the pmap structure 1325 */ 1326 pm->pm_l1 = l1; 1327 pm->pm_domain = domain + 1; 1328 #endif 1329 } 1330 1331 /* 1332 * Free an L1 translation table. 1333 * This is called at pmap destruction time. 1334 */ 1335 static void 1336 pmap_free_l1(pmap_t pm) 1337 { 1338 #ifdef ARM_MMU_EXTENDED 1339 pool_cache_put_paddr(&pmap_l1tt_cache, (void *)pm->pm_l1, pm->pm_l1_pa); 1340 1341 pm->pm_l1 = NULL; 1342 pm->pm_l1_pa = 0; 1343 #else 1344 struct l1_ttable *l1 = pm->pm_l1; 1345 1346 mutex_spin_enter(&l1_lru_lock); 1347 1348 /* 1349 * If this L1 is currently on the LRU list, remove it. 1350 */ 1351 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1352 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1353 1354 /* 1355 * Free up the domain number which was allocated to the pmap 1356 */ 1357 l1->l1_domain_free[pmap_domain(pm) - 1] = l1->l1_domain_first; 1358 l1->l1_domain_first = pmap_domain(pm) - 1; 1359 l1->l1_domain_use_count--; 1360 1361 /* 1362 * The L1 now must have at least 1 free domain, so add 1363 * it back to the LRU list. If the use count is zero, 1364 * put it at the head of the list, otherwise it goes 1365 * to the tail. 1366 */ 1367 if (l1->l1_domain_use_count == 0) 1368 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1369 else 1370 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1371 1372 mutex_spin_exit(&l1_lru_lock); 1373 #endif /* ARM_MMU_EXTENDED */ 1374 } 1375 1376 #ifndef ARM_MMU_EXTENDED 1377 static inline void 1378 pmap_use_l1(pmap_t pm) 1379 { 1380 struct l1_ttable *l1; 1381 1382 /* 1383 * Do nothing if we're in interrupt context. 1384 * Access to an L1 by the kernel pmap must not affect 1385 * the LRU list. 1386 */ 1387 if (cpu_intr_p() || pm == pmap_kernel()) 1388 return; 1389 1390 l1 = pm->pm_l1; 1391 1392 /* 1393 * If the L1 is not currently on the LRU list, just return 1394 */ 1395 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1396 return; 1397 1398 mutex_spin_enter(&l1_lru_lock); 1399 1400 /* 1401 * Check the use count again, now that we've acquired the lock 1402 */ 1403 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1404 mutex_spin_exit(&l1_lru_lock); 1405 return; 1406 } 1407 1408 /* 1409 * Move the L1 to the back of the LRU list 1410 */ 1411 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1412 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1413 1414 mutex_spin_exit(&l1_lru_lock); 1415 } 1416 #endif /* !ARM_MMU_EXTENDED */ 1417 1418 /* 1419 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1420 * 1421 * Free an L2 descriptor table. 1422 */ 1423 static inline void 1424 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1425 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1426 #else 1427 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1428 #endif 1429 { 1430 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1431 /* 1432 * Note: With a write-back cache, we may need to sync this 1433 * L2 table before re-using it. 1434 * This is because it may have belonged to a non-current 1435 * pmap, in which case the cache syncs would have been 1436 * skipped for the pages that were being unmapped. If the 1437 * L2 table were then to be immediately re-allocated to 1438 * the *current* pmap, it may well contain stale mappings 1439 * which have not yet been cleared by a cache write-back 1440 * and so would still be visible to the mmu. 1441 */ 1442 if (need_sync) 1443 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1444 #endif /* PMAP_INCLUDE_PTE_SYNC && PMAP_CACHE_VIVT */ 1445 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1446 } 1447 1448 /* 1449 * Returns a pointer to the L2 bucket associated with the specified pmap 1450 * and VA, or NULL if no L2 bucket exists for the address. 1451 */ 1452 static inline struct l2_bucket * 1453 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1454 { 1455 const size_t l1slot = l1pte_index(va); 1456 struct l2_dtable *l2; 1457 struct l2_bucket *l2b; 1458 1459 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL || 1460 (l2b = &l2->l2_bucket[L2_BUCKET(l1slot)])->l2b_kva == NULL) 1461 return NULL; 1462 1463 return l2b; 1464 } 1465 1466 /* 1467 * Returns a pointer to the L2 bucket associated with the specified pmap 1468 * and VA. 1469 * 1470 * If no L2 bucket exists, perform the necessary allocations to put an L2 1471 * bucket/page table in place. 1472 * 1473 * Note that if a new L2 bucket/page was allocated, the caller *must* 1474 * increment the bucket occupancy counter appropriately *before* 1475 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1476 * the bucket/page in the meantime. 1477 */ 1478 static struct l2_bucket * 1479 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1480 { 1481 const size_t l1slot = l1pte_index(va); 1482 struct l2_dtable *l2; 1483 1484 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 1485 /* 1486 * No mapping at this address, as there is 1487 * no entry in the L1 table. 1488 * Need to allocate a new l2_dtable. 1489 */ 1490 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1491 return NULL; 1492 1493 /* 1494 * Link it into the parent pmap 1495 */ 1496 pm->pm_l2[L2_IDX(l1slot)] = l2; 1497 } 1498 1499 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 1500 1501 /* 1502 * Fetch pointer to the L2 page table associated with the address. 1503 */ 1504 if (l2b->l2b_kva == NULL) { 1505 pt_entry_t *ptep; 1506 1507 /* 1508 * No L2 page table has been allocated. Chances are, this 1509 * is because we just allocated the l2_dtable, above. 1510 */ 1511 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_pa)) == NULL) { 1512 /* 1513 * Oops, no more L2 page tables available at this 1514 * time. We may need to deallocate the l2_dtable 1515 * if we allocated a new one above. 1516 */ 1517 if (l2->l2_occupancy == 0) { 1518 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1519 pmap_free_l2_dtable(l2); 1520 } 1521 return NULL; 1522 } 1523 1524 l2->l2_occupancy++; 1525 l2b->l2b_kva = ptep; 1526 l2b->l2b_l1slot = l1slot; 1527 1528 #ifdef ARM_MMU_EXTENDED 1529 /* 1530 * We know there will be a mapping here, so simply 1531 * enter this PTP into the L1 now. 1532 */ 1533 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1534 pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 1535 | L1_C_DOM(pmap_domain(pm)); 1536 KASSERT(*pdep == 0); 1537 l1pte_setone(pdep, npde); 1538 PDE_SYNC(pdep); 1539 #endif 1540 } 1541 1542 return l2b; 1543 } 1544 1545 /* 1546 * One or more mappings in the specified L2 descriptor table have just been 1547 * invalidated. 1548 * 1549 * Garbage collect the metadata and descriptor table itself if necessary. 1550 * 1551 * The pmap lock must be acquired when this is called (not necessary 1552 * for the kernel pmap). 1553 */ 1554 static void 1555 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1556 { 1557 KDASSERT(count <= l2b->l2b_occupancy); 1558 1559 /* 1560 * Update the bucket's reference count according to how many 1561 * PTEs the caller has just invalidated. 1562 */ 1563 l2b->l2b_occupancy -= count; 1564 1565 /* 1566 * Note: 1567 * 1568 * Level 2 page tables allocated to the kernel pmap are never freed 1569 * as that would require checking all Level 1 page tables and 1570 * removing any references to the Level 2 page table. See also the 1571 * comment elsewhere about never freeing bootstrap L2 descriptors. 1572 * 1573 * We make do with just invalidating the mapping in the L2 table. 1574 * 1575 * This isn't really a big deal in practice and, in fact, leads 1576 * to a performance win over time as we don't need to continually 1577 * alloc/free. 1578 */ 1579 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1580 return; 1581 1582 /* 1583 * There are no more valid mappings in this level 2 page table. 1584 * Go ahead and NULL-out the pointer in the bucket, then 1585 * free the page table. 1586 */ 1587 const size_t l1slot = l2b->l2b_l1slot; 1588 pt_entry_t * const ptep = l2b->l2b_kva; 1589 l2b->l2b_kva = NULL; 1590 1591 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1592 pd_entry_t pde __diagused = *pdep; 1593 1594 #ifdef ARM_MMU_EXTENDED 1595 /* 1596 * Invalidate the L1 slot. 1597 */ 1598 KASSERT((pde & L1_TYPE_MASK) == L1_TYPE_C); 1599 #else 1600 /* 1601 * If the L1 slot matches the pmap's domain number, then invalidate it. 1602 */ 1603 if ((pde & (L1_C_DOM_MASK|L1_TYPE_MASK)) 1604 == (L1_C_DOM(pmap_domain(pm))|L1_TYPE_C)) { 1605 #endif 1606 l1pte_setone(pdep, 0); 1607 PDE_SYNC(pdep); 1608 #ifndef ARM_MMU_EXTENDED 1609 } 1610 #endif 1611 1612 /* 1613 * Release the L2 descriptor table back to the pool cache. 1614 */ 1615 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1616 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_pa); 1617 #else 1618 pmap_free_l2_ptp(ptep, l2b->l2b_pa); 1619 #endif 1620 1621 /* 1622 * Update the reference count in the associated l2_dtable 1623 */ 1624 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 1625 if (--l2->l2_occupancy > 0) 1626 return; 1627 1628 /* 1629 * There are no more valid mappings in any of the Level 1 1630 * slots managed by this l2_dtable. Go ahead and NULL-out 1631 * the pointer in the parent pmap and free the l2_dtable. 1632 */ 1633 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1634 pmap_free_l2_dtable(l2); 1635 } 1636 1637 #if defined(ARM_MMU_EXTENDED) 1638 /* 1639 * Pool cache constructors for L1 translation tables 1640 */ 1641 1642 static int 1643 pmap_l1tt_ctor(void *arg, void *v, int flags) 1644 { 1645 #ifndef PMAP_INCLUDE_PTE_SYNC 1646 #error not supported 1647 #endif 1648 1649 memset(v, 0, L1TT_SIZE); 1650 PTE_SYNC_RANGE(v, L1TT_SIZE / sizeof(pt_entry_t)); 1651 return 0; 1652 } 1653 #endif 1654 1655 /* 1656 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1657 * structures. 1658 */ 1659 static int 1660 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1661 { 1662 #ifndef PMAP_INCLUDE_PTE_SYNC 1663 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1664 1665 /* 1666 * The mappings for these page tables were initially made using 1667 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1668 * mode will not be right for page table mappings. To avoid 1669 * polluting the pmap_kenter_pa() code with a special case for 1670 * page tables, we simply fix up the cache-mode here if it's not 1671 * correct. 1672 */ 1673 if (pte_l2_s_cache_mode != pte_l2_s_cache_mode_pt) { 1674 const struct l2_bucket * const l2b = 1675 pmap_get_l2_bucket(pmap_kernel(), va); 1676 KASSERTMSG(l2b != NULL, "%#lx", va); 1677 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 1678 const pt_entry_t opte = *ptep; 1679 1680 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1681 /* 1682 * Page tables must have the cache-mode set correctly. 1683 */ 1684 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1685 | pte_l2_s_cache_mode_pt; 1686 l2pte_set(ptep, npte, opte); 1687 PTE_SYNC(ptep); 1688 cpu_tlb_flushD_SE(va); 1689 cpu_cpwait(); 1690 } 1691 } 1692 #endif 1693 1694 memset(v, 0, L2_TABLE_SIZE_REAL); 1695 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1696 return 0; 1697 } 1698 1699 static int 1700 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1701 { 1702 1703 memset(v, 0, sizeof(struct l2_dtable)); 1704 return 0; 1705 } 1706 1707 static int 1708 pmap_pmap_ctor(void *arg, void *v, int flags) 1709 { 1710 1711 memset(v, 0, sizeof(struct pmap)); 1712 return 0; 1713 } 1714 1715 static void 1716 pmap_pinit(pmap_t pm) 1717 { 1718 #ifndef ARM_HAS_VBAR 1719 struct l2_bucket *l2b; 1720 1721 if (vector_page < KERNEL_BASE) { 1722 /* 1723 * Map the vector page. 1724 */ 1725 pmap_enter(pm, vector_page, systempage.pv_pa, 1726 VM_PROT_READ | VM_PROT_EXECUTE, 1727 VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED); 1728 pmap_update(pm); 1729 1730 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 1731 l2b = pmap_get_l2_bucket(pm, vector_page); 1732 KASSERTMSG(l2b != NULL, "%#lx", vector_page); 1733 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 1734 L1_C_DOM(pmap_domain(pm)); 1735 } else 1736 pm->pm_pl1vec = NULL; 1737 #endif 1738 } 1739 1740 #ifdef PMAP_CACHE_VIVT 1741 /* 1742 * Since we have a virtually indexed cache, we may need to inhibit caching if 1743 * there is more than one mapping and at least one of them is writable. 1744 * Since we purge the cache on every context switch, we only need to check for 1745 * other mappings within the same pmap, or kernel_pmap. 1746 * This function is also called when a page is unmapped, to possibly reenable 1747 * caching on any remaining mappings. 1748 * 1749 * The code implements the following logic, where: 1750 * 1751 * KW = # of kernel read/write pages 1752 * KR = # of kernel read only pages 1753 * UW = # of user read/write pages 1754 * UR = # of user read only pages 1755 * 1756 * KC = kernel mapping is cacheable 1757 * UC = user mapping is cacheable 1758 * 1759 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1760 * +--------------------------------------------- 1761 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1762 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1763 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1764 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1765 */ 1766 1767 static const int pmap_vac_flags[4][4] = { 1768 {-1, 0, 0, PVF_KNC}, 1769 {0, 0, PVF_NC, PVF_NC}, 1770 {0, PVF_NC, PVF_NC, PVF_NC}, 1771 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1772 }; 1773 1774 static inline int 1775 pmap_get_vac_flags(const struct vm_page_md *md) 1776 { 1777 int kidx, uidx; 1778 1779 kidx = 0; 1780 if (md->kro_mappings || md->krw_mappings > 1) 1781 kidx |= 1; 1782 if (md->krw_mappings) 1783 kidx |= 2; 1784 1785 uidx = 0; 1786 if (md->uro_mappings || md->urw_mappings > 1) 1787 uidx |= 1; 1788 if (md->urw_mappings) 1789 uidx |= 2; 1790 1791 return pmap_vac_flags[uidx][kidx]; 1792 } 1793 1794 static inline void 1795 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1796 { 1797 int nattr; 1798 1799 nattr = pmap_get_vac_flags(md); 1800 1801 if (nattr < 0) { 1802 md->pvh_attrs &= ~PVF_NC; 1803 return; 1804 } 1805 1806 if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0) 1807 return; 1808 1809 if (pm == pmap_kernel()) 1810 pmap_vac_me_kpmap(md, pa, pm, va); 1811 else 1812 pmap_vac_me_user(md, pa, pm, va); 1813 1814 md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr; 1815 } 1816 1817 static void 1818 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1819 { 1820 u_int u_cacheable, u_entries; 1821 struct pv_entry *pv; 1822 pmap_t last_pmap = pm; 1823 1824 /* 1825 * Pass one, see if there are both kernel and user pmaps for 1826 * this page. Calculate whether there are user-writable or 1827 * kernel-writable pages. 1828 */ 1829 u_cacheable = 0; 1830 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1831 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1832 u_cacheable++; 1833 } 1834 1835 u_entries = md->urw_mappings + md->uro_mappings; 1836 1837 /* 1838 * We know we have just been updating a kernel entry, so if 1839 * all user pages are already cacheable, then there is nothing 1840 * further to do. 1841 */ 1842 if (md->k_mappings == 0 && u_cacheable == u_entries) 1843 return; 1844 1845 if (u_entries) { 1846 /* 1847 * Scan over the list again, for each entry, if it 1848 * might not be set correctly, call pmap_vac_me_user 1849 * to recalculate the settings. 1850 */ 1851 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1852 /* 1853 * We know kernel mappings will get set 1854 * correctly in other calls. We also know 1855 * that if the pmap is the same as last_pmap 1856 * then we've just handled this entry. 1857 */ 1858 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1859 continue; 1860 1861 /* 1862 * If there are kernel entries and this page 1863 * is writable but non-cacheable, then we can 1864 * skip this entry also. 1865 */ 1866 if (md->k_mappings && 1867 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1868 (PVF_NC | PVF_WRITE)) 1869 continue; 1870 1871 /* 1872 * Similarly if there are no kernel-writable 1873 * entries and the page is already 1874 * read-only/cacheable. 1875 */ 1876 if (md->krw_mappings == 0 && 1877 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1878 continue; 1879 1880 /* 1881 * For some of the remaining cases, we know 1882 * that we must recalculate, but for others we 1883 * can't tell if they are correct or not, so 1884 * we recalculate anyway. 1885 */ 1886 pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0); 1887 } 1888 1889 if (md->k_mappings == 0) 1890 return; 1891 } 1892 1893 pmap_vac_me_user(md, pa, pm, va); 1894 } 1895 1896 static void 1897 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1898 { 1899 pmap_t kpmap = pmap_kernel(); 1900 struct pv_entry *pv, *npv = NULL; 1901 u_int entries = 0; 1902 u_int writable = 0; 1903 u_int cacheable_entries = 0; 1904 u_int kern_cacheable = 0; 1905 u_int other_writable = 0; 1906 1907 /* 1908 * Count mappings and writable mappings in this pmap. 1909 * Include kernel mappings as part of our own. 1910 * Keep a pointer to the first one. 1911 */ 1912 npv = NULL; 1913 KASSERT(pmap_page_locked_p(md)); 1914 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1915 /* Count mappings in the same pmap */ 1916 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1917 if (entries++ == 0) 1918 npv = pv; 1919 1920 /* Cacheable mappings */ 1921 if ((pv->pv_flags & PVF_NC) == 0) { 1922 cacheable_entries++; 1923 if (kpmap == pv->pv_pmap) 1924 kern_cacheable++; 1925 } 1926 1927 /* Writable mappings */ 1928 if (pv->pv_flags & PVF_WRITE) 1929 ++writable; 1930 } else if (pv->pv_flags & PVF_WRITE) 1931 other_writable = 1; 1932 } 1933 1934 /* 1935 * Enable or disable caching as necessary. 1936 * Note: the first entry might be part of the kernel pmap, 1937 * so we can't assume this is indicative of the state of the 1938 * other (maybe non-kpmap) entries. 1939 */ 1940 if ((entries > 1 && writable) || 1941 (entries > 0 && pm == kpmap && other_writable)) { 1942 if (cacheable_entries == 0) { 1943 return; 1944 } 1945 1946 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1947 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1948 (pv->pv_flags & PVF_NC)) 1949 continue; 1950 1951 pv->pv_flags |= PVF_NC; 1952 1953 struct l2_bucket * const l2b 1954 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1955 KASSERTMSG(l2b != NULL, "%#lx", va); 1956 pt_entry_t * const ptep 1957 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1958 const pt_entry_t opte = *ptep; 1959 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 1960 1961 if ((va != pv->pv_va || pm != pv->pv_pmap) 1962 && l2pte_valid_p(opte)) { 1963 pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va, 1964 true, pv->pv_flags); 1965 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1966 pv->pv_flags); 1967 } 1968 1969 l2pte_set(ptep, npte, opte); 1970 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1971 } 1972 cpu_cpwait(); 1973 } else if (entries > cacheable_entries) { 1974 /* 1975 * Turn cacheing back on for some pages. If it is a kernel 1976 * page, only do so if there are no other writable pages. 1977 */ 1978 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1979 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1980 (kpmap != pv->pv_pmap || other_writable))) 1981 continue; 1982 1983 pv->pv_flags &= ~PVF_NC; 1984 1985 struct l2_bucket * const l2b 1986 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1987 KASSERTMSG(l2b != NULL, "%#lx", va); 1988 pt_entry_t * const ptep 1989 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1990 const pt_entry_t opte = *ptep; 1991 pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1992 | pte_l2_s_cache_mode; 1993 1994 if (l2pte_valid_p(opte)) { 1995 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1996 pv->pv_flags); 1997 } 1998 1999 l2pte_set(ptep, npte, opte); 2000 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2001 } 2002 } 2003 } 2004 #endif 2005 2006 #ifdef PMAP_CACHE_VIPT 2007 static void 2008 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 2009 { 2010 2011 #ifndef ARM_MMU_EXTENDED 2012 struct pv_entry *pv; 2013 vaddr_t tst_mask; 2014 bool bad_alias; 2015 const u_int 2016 rw_mappings = md->urw_mappings + md->krw_mappings, 2017 ro_mappings = md->uro_mappings + md->kro_mappings; 2018 2019 /* do we need to do anything? */ 2020 if (arm_cache_prefer_mask == 0) 2021 return; 2022 2023 UVMHIST_FUNC(__func__); 2024 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 2025 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 2026 2027 KASSERT(!va || pm); 2028 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2029 2030 /* Already a conflict? */ 2031 if (__predict_false(md->pvh_attrs & PVF_NC)) { 2032 /* just an add, things are already non-cached */ 2033 KASSERT(!(md->pvh_attrs & PVF_DIRTY)); 2034 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2035 bad_alias = false; 2036 if (va) { 2037 PMAPCOUNT(vac_color_none); 2038 bad_alias = true; 2039 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2040 goto fixup; 2041 } 2042 pv = SLIST_FIRST(&md->pvh_list); 2043 /* the list can't be empty because it would be cachable */ 2044 if (md->pvh_attrs & PVF_KMPAGE) { 2045 tst_mask = md->pvh_attrs; 2046 } else { 2047 KASSERT(pv); 2048 tst_mask = pv->pv_va; 2049 pv = SLIST_NEXT(pv, pv_link); 2050 } 2051 /* 2052 * Only check for a bad alias if we have writable mappings. 2053 */ 2054 tst_mask &= arm_cache_prefer_mask; 2055 if (rw_mappings > 0) { 2056 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 2057 /* if there's a bad alias, stop checking. */ 2058 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 2059 bad_alias = true; 2060 } 2061 md->pvh_attrs |= PVF_WRITE; 2062 if (!bad_alias) 2063 md->pvh_attrs |= PVF_DIRTY; 2064 } else { 2065 /* 2066 * We have only read-only mappings. Let's see if there 2067 * are multiple colors in use or if we mapped a KMPAGE. 2068 * If the latter, we have a bad alias. If the former, 2069 * we need to remember that. 2070 */ 2071 for (; pv; pv = SLIST_NEXT(pv, pv_link)) { 2072 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) { 2073 if (md->pvh_attrs & PVF_KMPAGE) 2074 bad_alias = true; 2075 break; 2076 } 2077 } 2078 md->pvh_attrs &= ~PVF_WRITE; 2079 /* 2080 * No KMPAGE and we exited early, so we must have 2081 * multiple color mappings. 2082 */ 2083 if (!bad_alias && pv != NULL) 2084 md->pvh_attrs |= PVF_MULTCLR; 2085 } 2086 2087 /* If no conflicting colors, set everything back to cached */ 2088 if (!bad_alias) { 2089 #ifdef DEBUG 2090 if ((md->pvh_attrs & PVF_WRITE) 2091 || ro_mappings < 2) { 2092 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2093 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2094 } 2095 #endif 2096 md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 2097 md->pvh_attrs |= tst_mask | PVF_COLORED; 2098 /* 2099 * Restore DIRTY bit if page is modified 2100 */ 2101 if (md->pvh_attrs & PVF_DMOD) 2102 md->pvh_attrs |= PVF_DIRTY; 2103 PMAPCOUNT(vac_color_restore); 2104 } else { 2105 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2106 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2107 } 2108 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2109 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2110 } else if (!va) { 2111 KASSERT(pmap_is_page_colored_p(md)); 2112 KASSERT(!(md->pvh_attrs & PVF_WRITE) 2113 || (md->pvh_attrs & PVF_DIRTY)); 2114 if (rw_mappings == 0) { 2115 md->pvh_attrs &= ~PVF_WRITE; 2116 if (ro_mappings == 1 2117 && (md->pvh_attrs & PVF_MULTCLR)) { 2118 /* 2119 * If this is the last readonly mapping 2120 * but it doesn't match the current color 2121 * for the page, change the current color 2122 * to match this last readonly mapping. 2123 */ 2124 pv = SLIST_FIRST(&md->pvh_list); 2125 tst_mask = (md->pvh_attrs ^ pv->pv_va) 2126 & arm_cache_prefer_mask; 2127 if (tst_mask) { 2128 md->pvh_attrs ^= tst_mask; 2129 PMAPCOUNT(vac_color_change); 2130 } 2131 } 2132 } 2133 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2134 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2135 return; 2136 } else if (!pmap_is_page_colored_p(md)) { 2137 /* not colored so we just use its color */ 2138 KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 2139 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2140 PMAPCOUNT(vac_color_new); 2141 md->pvh_attrs &= PAGE_SIZE - 1; 2142 md->pvh_attrs |= PVF_COLORED 2143 | (va & arm_cache_prefer_mask) 2144 | (rw_mappings > 0 ? PVF_WRITE : 0); 2145 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2146 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2147 return; 2148 } else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 2149 bad_alias = false; 2150 if (rw_mappings > 0) { 2151 /* 2152 * We now have writeable mappings and if we have 2153 * readonly mappings in more than once color, we have 2154 * an aliasing problem. Regardless mark the page as 2155 * writeable. 2156 */ 2157 if (md->pvh_attrs & PVF_MULTCLR) { 2158 if (ro_mappings < 2) { 2159 /* 2160 * If we only have less than two 2161 * read-only mappings, just flush the 2162 * non-primary colors from the cache. 2163 */ 2164 pmap_flush_page(md, pa, 2165 PMAP_FLUSH_SECONDARY); 2166 } else { 2167 bad_alias = true; 2168 } 2169 } 2170 md->pvh_attrs |= PVF_WRITE; 2171 } 2172 /* If no conflicting colors, set everything back to cached */ 2173 if (!bad_alias) { 2174 #ifdef DEBUG 2175 if (rw_mappings > 0 2176 || (md->pvh_attrs & PMAP_KMPAGE)) { 2177 tst_mask = md->pvh_attrs & arm_cache_prefer_mask; 2178 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2179 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2180 } 2181 #endif 2182 if (SLIST_EMPTY(&md->pvh_list)) 2183 PMAPCOUNT(vac_color_reuse); 2184 else 2185 PMAPCOUNT(vac_color_ok); 2186 2187 /* matching color, just return */ 2188 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2189 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2190 return; 2191 } 2192 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2193 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2194 2195 /* color conflict. evict from cache. */ 2196 2197 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2198 md->pvh_attrs &= ~PVF_COLORED; 2199 md->pvh_attrs |= PVF_NC; 2200 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2201 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2202 PMAPCOUNT(vac_color_erase); 2203 } else if (rw_mappings == 0 2204 && (md->pvh_attrs & PVF_KMPAGE) == 0) { 2205 KASSERT((md->pvh_attrs & PVF_WRITE) == 0); 2206 2207 /* 2208 * If the page has dirty cache lines, clean it. 2209 */ 2210 if (md->pvh_attrs & PVF_DIRTY) 2211 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 2212 2213 /* 2214 * If this is the first remapping (we know that there are no 2215 * writeable mappings), then this is a simple color change. 2216 * Otherwise this is a seconary r/o mapping, which means 2217 * we don't have to do anything. 2218 */ 2219 if (ro_mappings == 1) { 2220 KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2221 md->pvh_attrs &= PAGE_SIZE - 1; 2222 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2223 PMAPCOUNT(vac_color_change); 2224 } else { 2225 PMAPCOUNT(vac_color_blind); 2226 } 2227 md->pvh_attrs |= PVF_MULTCLR; 2228 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2229 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2230 return; 2231 } else { 2232 if (rw_mappings > 0) 2233 md->pvh_attrs |= PVF_WRITE; 2234 2235 /* color conflict. evict from cache. */ 2236 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2237 2238 /* the list can't be empty because this was a enter/modify */ 2239 pv = SLIST_FIRST(&md->pvh_list); 2240 if ((md->pvh_attrs & PVF_KMPAGE) == 0) { 2241 KASSERT(pv); 2242 /* 2243 * If there's only one mapped page, change color to the 2244 * page's new color and return. Restore the DIRTY bit 2245 * that was erased by pmap_flush_page. 2246 */ 2247 if (SLIST_NEXT(pv, pv_link) == NULL) { 2248 md->pvh_attrs &= PAGE_SIZE - 1; 2249 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2250 if (md->pvh_attrs & PVF_DMOD) 2251 md->pvh_attrs |= PVF_DIRTY; 2252 PMAPCOUNT(vac_color_change); 2253 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2254 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2255 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2256 return; 2257 } 2258 } 2259 bad_alias = true; 2260 md->pvh_attrs &= ~PVF_COLORED; 2261 md->pvh_attrs |= PVF_NC; 2262 PMAPCOUNT(vac_color_erase); 2263 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2264 } 2265 2266 fixup: 2267 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2268 2269 /* 2270 * Turn cacheing on/off for all pages. 2271 */ 2272 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2273 struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap, 2274 pv->pv_va); 2275 KASSERTMSG(l2b != NULL, "%#lx", va); 2276 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2277 const pt_entry_t opte = *ptep; 2278 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 2279 if (bad_alias) { 2280 pv->pv_flags |= PVF_NC; 2281 } else { 2282 pv->pv_flags &= ~PVF_NC; 2283 npte |= pte_l2_s_cache_mode; 2284 } 2285 2286 if (opte == npte) /* only update is there's a change */ 2287 continue; 2288 2289 if (l2pte_valid_p(opte)) { 2290 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags); 2291 } 2292 2293 l2pte_set(ptep, npte, opte); 2294 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2295 } 2296 #endif /* !ARM_MMU_EXTENDED */ 2297 } 2298 #endif /* PMAP_CACHE_VIPT */ 2299 2300 2301 /* 2302 * Modify pte bits for all ptes corresponding to the given physical address. 2303 * We use `maskbits' rather than `clearbits' because we're always passing 2304 * constants and the latter would require an extra inversion at run-time. 2305 */ 2306 static void 2307 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits) 2308 { 2309 struct pv_entry *pv; 2310 #ifdef PMAP_CACHE_VIPT 2311 const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs); 2312 bool need_syncicache = false; 2313 #ifdef ARM_MMU_EXTENDED 2314 const u_int execbits = (maskbits & PVF_EXEC) ? L2_XS_XN : 0; 2315 #else 2316 const u_int execbits = 0; 2317 bool need_vac_me_harder = false; 2318 #endif 2319 #else 2320 const u_int execbits = 0; 2321 #endif 2322 2323 UVMHIST_FUNC(__func__); 2324 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx maskbits %#jx", 2325 (uintptr_t)md, pa, maskbits, 0); 2326 2327 #ifdef PMAP_CACHE_VIPT 2328 /* 2329 * If we might want to sync the I-cache and we've modified it, 2330 * then we know we definitely need to sync or discard it. 2331 */ 2332 if (want_syncicache) { 2333 if (md->pvh_attrs & PVF_MOD) { 2334 need_syncicache = true; 2335 } 2336 } 2337 #endif 2338 KASSERT(pmap_page_locked_p(md)); 2339 2340 /* 2341 * Clear saved attributes (modify, reference) 2342 */ 2343 md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2344 2345 if (SLIST_EMPTY(&md->pvh_list)) { 2346 #if defined(PMAP_CACHE_VIPT) 2347 if (need_syncicache) { 2348 /* 2349 * No one has it mapped, so just discard it. The next 2350 * exec remapping will cause it to be synced. 2351 */ 2352 md->pvh_attrs &= ~PVF_EXEC; 2353 PMAPCOUNT(exec_discarded_clearbit); 2354 } 2355 #endif 2356 return; 2357 } 2358 2359 /* 2360 * Loop over all current mappings setting/clearing as appropos 2361 */ 2362 for (pv = SLIST_FIRST(&md->pvh_list); pv != NULL;) { 2363 pmap_t pm = pv->pv_pmap; 2364 const vaddr_t va = pv->pv_va; 2365 const u_int oflags = pv->pv_flags; 2366 #ifndef ARM_MMU_EXTENDED 2367 /* 2368 * Kernel entries are unmanaged and as such not to be changed. 2369 */ 2370 if (PV_IS_KENTRY_P(oflags)) { 2371 pv = SLIST_NEXT(pv, pv_link); 2372 continue; 2373 } 2374 #endif 2375 2376 /* 2377 * Anything to do? 2378 */ 2379 if ((oflags & maskbits) == 0) { 2380 pv = SLIST_NEXT(pv, pv_link); 2381 continue; 2382 } 2383 2384 /* 2385 * Try to get a hold on the pmap's lock. We must do this 2386 * while still holding the page locked, to know that the 2387 * page is still associated with the pmap and the mapping is 2388 * in place. If a hold can't be had, unlock and wait for 2389 * the pmap's lock to become available and retry. The pmap 2390 * must be ref'd over this dance to stop it disappearing 2391 * behind us. 2392 */ 2393 if (!mutex_tryenter(&pm->pm_lock)) { 2394 pmap_reference(pm); 2395 pmap_release_page_lock(md); 2396 pmap_acquire_pmap_lock(pm); 2397 /* nothing, just wait for it */ 2398 pmap_release_pmap_lock(pm); 2399 pmap_destroy(pm); 2400 /* Restart from the beginning. */ 2401 pmap_acquire_page_lock(md); 2402 pv = SLIST_FIRST(&md->pvh_list); 2403 continue; 2404 } 2405 pv->pv_flags &= ~maskbits; 2406 2407 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va); 2408 KASSERTMSG(l2b != NULL, "%#lx", va); 2409 2410 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 2411 const pt_entry_t opte = *ptep; 2412 pt_entry_t npte = opte | execbits; 2413 2414 #ifdef ARM_MMU_EXTENDED 2415 KASSERT((opte & L2_XS_nG) == (pm == pmap_kernel() ? 0 : L2_XS_nG)); 2416 #endif 2417 2418 UVMHIST_LOG(maphist, "pv %#jx pm %#jx va %#jx flag %#jx", 2419 (uintptr_t)pv, (uintptr_t)pm, va, oflags); 2420 2421 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2422 #ifdef PMAP_CACHE_VIVT 2423 if ((oflags & PVF_NC)) { 2424 /* 2425 * Entry is not cacheable: 2426 * 2427 * Don't turn caching on again if this is a 2428 * modified emulation. This would be 2429 * inconsitent with the settings created by 2430 * pmap_vac_me_harder(). Otherwise, it's safe 2431 * to re-enable cacheing. 2432 * 2433 * There's no need to call pmap_vac_me_harder() 2434 * here: all pages are losing their write 2435 * permission. 2436 */ 2437 if (maskbits & PVF_WRITE) { 2438 npte |= pte_l2_s_cache_mode; 2439 pv->pv_flags &= ~PVF_NC; 2440 } 2441 } else if (l2pte_writable_p(opte)) { 2442 /* 2443 * Entry is writable/cacheable: check if pmap 2444 * is current if it is flush it, otherwise it 2445 * won't be in the cache 2446 */ 2447 pmap_cache_wbinv_page(pm, va, 2448 (maskbits & PVF_REF) != 0, 2449 oflags|PVF_WRITE); 2450 } 2451 #endif 2452 2453 /* make the pte read only */ 2454 npte = l2pte_set_readonly(npte); 2455 2456 if ((maskbits & oflags & PVF_WRITE)) { 2457 /* 2458 * Keep alias accounting up to date 2459 */ 2460 if (pm == pmap_kernel()) { 2461 md->krw_mappings--; 2462 md->kro_mappings++; 2463 } else { 2464 md->urw_mappings--; 2465 md->uro_mappings++; 2466 } 2467 #ifdef PMAP_CACHE_VIPT 2468 if (arm_cache_prefer_mask != 0) { 2469 if (md->urw_mappings + md->krw_mappings == 0) { 2470 md->pvh_attrs &= ~PVF_WRITE; 2471 } else { 2472 PMAP_VALIDATE_MD_PAGE(md); 2473 } 2474 } 2475 if (want_syncicache) 2476 need_syncicache = true; 2477 #ifndef ARM_MMU_EXTENDED 2478 need_vac_me_harder = true; 2479 #endif 2480 #endif /* PMAP_CACHE_VIPT */ 2481 } 2482 } 2483 2484 if (maskbits & PVF_REF) { 2485 if (true 2486 #ifndef ARM_MMU_EXTENDED 2487 && (oflags & PVF_NC) == 0 2488 #endif 2489 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0 2490 && l2pte_valid_p(npte)) { 2491 #ifdef PMAP_CACHE_VIVT 2492 /* 2493 * Check npte here; we may have already 2494 * done the wbinv above, and the validity 2495 * of the PTE is the same for opte and 2496 * npte. 2497 */ 2498 pmap_cache_wbinv_page(pm, va, true, oflags); 2499 #endif 2500 } 2501 2502 /* 2503 * Make the PTE invalid so that we will take a 2504 * page fault the next time the mapping is 2505 * referenced. 2506 */ 2507 npte &= ~L2_TYPE_MASK; 2508 npte |= L2_TYPE_INV; 2509 } 2510 2511 if (npte != opte) { 2512 l2pte_reset(ptep); 2513 PTE_SYNC(ptep); 2514 2515 /* Flush the TLB entry if a current pmap. */ 2516 pmap_tlb_flush_SE(pm, va, oflags); 2517 2518 l2pte_set(ptep, npte, 0); 2519 PTE_SYNC(ptep); 2520 } 2521 2522 pmap_release_pmap_lock(pm); 2523 2524 UVMHIST_LOG(maphist, "pm %#jx va %#jx opte %#jx npte %#jx", 2525 (uintptr_t)pm, va, opte, npte); 2526 2527 /* Move to next entry. */ 2528 pv = SLIST_NEXT(pv, pv_link); 2529 } 2530 2531 #if defined(PMAP_CACHE_VIPT) 2532 /* 2533 * If we need to sync the I-cache and we haven't done it yet, do it. 2534 */ 2535 if (need_syncicache) { 2536 pmap_syncicache_page(md, pa); 2537 PMAPCOUNT(exec_synced_clearbit); 2538 } 2539 #ifndef ARM_MMU_EXTENDED 2540 /* 2541 * If we are changing this to read-only, we need to call vac_me_harder 2542 * so we can change all the read-only pages to cacheable. We pretend 2543 * this as a page deletion. 2544 */ 2545 if (need_vac_me_harder) { 2546 if (md->pvh_attrs & PVF_NC) 2547 pmap_vac_me_harder(md, pa, NULL, 0); 2548 } 2549 #endif /* !ARM_MMU_EXTENDED */ 2550 #endif /* PMAP_CACHE_VIPT */ 2551 } 2552 2553 /* 2554 * pmap_clean_page() 2555 * 2556 * This is a local function used to work out the best strategy to clean 2557 * a single page referenced by its entry in the PV table. It's used by 2558 * pmap_copy_page, pmap_zero_page and maybe some others later on. 2559 * 2560 * Its policy is effectively: 2561 * o If there are no mappings, we don't bother doing anything with the cache. 2562 * o If there is one mapping, we clean just that page. 2563 * o If there are multiple mappings, we clean the entire cache. 2564 * 2565 * So that some functions can be further optimised, it returns 0 if it didn't 2566 * clean the entire cache, or 1 if it did. 2567 * 2568 * XXX One bug in this routine is that if the pv_entry has a single page 2569 * mapped at 0x00000000 a whole cache clean will be performed rather than 2570 * just the 1 page. Since this should not occur in everyday use and if it does 2571 * it will just result in not the most efficient clean for the page. 2572 */ 2573 #ifdef PMAP_CACHE_VIVT 2574 static bool 2575 pmap_clean_page(struct vm_page_md *md, bool is_src) 2576 { 2577 struct pv_entry *pv; 2578 pmap_t pm_to_clean = NULL; 2579 bool cache_needs_cleaning = false; 2580 vaddr_t page_to_clean = 0; 2581 u_int flags = 0; 2582 2583 /* 2584 * Since we flush the cache each time we change to a different 2585 * user vmspace, we only need to flush the page if it is in the 2586 * current pmap. 2587 */ 2588 KASSERT(pmap_page_locked_p(md)); 2589 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2590 if (pmap_is_current(pv->pv_pmap)) { 2591 flags |= pv->pv_flags; 2592 /* 2593 * The page is mapped non-cacheable in 2594 * this map. No need to flush the cache. 2595 */ 2596 if (pv->pv_flags & PVF_NC) { 2597 #ifdef DIAGNOSTIC 2598 KASSERT(!cache_needs_cleaning); 2599 #endif 2600 break; 2601 } else if (is_src && (pv->pv_flags & PVF_WRITE) == 0) 2602 continue; 2603 if (cache_needs_cleaning) { 2604 page_to_clean = 0; 2605 break; 2606 } else { 2607 page_to_clean = pv->pv_va; 2608 pm_to_clean = pv->pv_pmap; 2609 } 2610 cache_needs_cleaning = true; 2611 } 2612 } 2613 2614 if (page_to_clean) { 2615 pmap_cache_wbinv_page(pm_to_clean, page_to_clean, 2616 !is_src, flags | PVF_REF); 2617 } else if (cache_needs_cleaning) { 2618 pmap_t const pm = curproc->p_vmspace->vm_map.pmap; 2619 2620 pmap_cache_wbinv_all(pm, flags); 2621 return true; 2622 } 2623 return false; 2624 } 2625 #endif 2626 2627 #ifdef PMAP_CACHE_VIPT 2628 /* 2629 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2630 * right cache alias to make sure we flush the right stuff. 2631 */ 2632 void 2633 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa) 2634 { 2635 pmap_t kpm = pmap_kernel(); 2636 const size_t way_size = arm_pcache.icache_type == CACHE_TYPE_PIPT 2637 ? PAGE_SIZE 2638 : arm_pcache.icache_way_size; 2639 2640 UVMHIST_FUNC(__func__); 2641 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx (attrs=%#jx)", 2642 (uintptr_t)md, pa, md->pvh_attrs, 0); 2643 2644 /* 2645 * No need to clean the page if it's non-cached. 2646 */ 2647 #ifndef ARM_MMU_EXTENDED 2648 if (md->pvh_attrs & PVF_NC) 2649 return; 2650 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED); 2651 #endif 2652 2653 pt_entry_t * const ptep = cpu_cdst_pte(0); 2654 const vaddr_t dstp = cpu_cdstp(0); 2655 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 2656 if (way_size <= PAGE_SIZE) { 2657 bool ok = false; 2658 vaddr_t vdstp = pmap_direct_mapped_phys(pa, &ok, dstp); 2659 if (ok) { 2660 cpu_icache_sync_range(vdstp, way_size); 2661 return; 2662 } 2663 } 2664 #endif 2665 2666 /* 2667 * We don't worry about the color of the exec page, we map the 2668 * same page to pages in the way and then do the icache_sync on 2669 * the entire way making sure we are cleaned. 2670 */ 2671 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 2672 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 2673 2674 for (size_t i = 0, j = 0; i < way_size; 2675 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2676 l2pte_reset(ptep + j); 2677 PTE_SYNC(ptep + j); 2678 2679 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2680 /* 2681 * Set up a PTE with to flush these cache lines. 2682 */ 2683 l2pte_set(ptep + j, npte, 0); 2684 } 2685 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2686 2687 /* 2688 * Flush it. 2689 */ 2690 cpu_icache_sync_range(dstp, way_size); 2691 2692 for (size_t i = 0, j = 0; i < way_size; 2693 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2694 /* 2695 * Unmap the page(s). 2696 */ 2697 l2pte_reset(ptep + j); 2698 PTE_SYNC(ptep + j); 2699 2700 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2701 } 2702 2703 md->pvh_attrs |= PVF_EXEC; 2704 PMAPCOUNT(exec_synced); 2705 } 2706 2707 #ifndef ARM_MMU_EXTENDED 2708 void 2709 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush) 2710 { 2711 vsize_t va_offset, end_va; 2712 bool wbinv_p; 2713 2714 if (arm_cache_prefer_mask == 0) 2715 return; 2716 2717 UVMHIST_FUNC(__func__); 2718 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx op %#jx", 2719 (uintptr_t)md, pa, op, 0); 2720 2721 switch (flush) { 2722 case PMAP_FLUSH_PRIMARY: 2723 if (md->pvh_attrs & PVF_MULTCLR) { 2724 va_offset = 0; 2725 end_va = arm_cache_prefer_mask; 2726 md->pvh_attrs &= ~PVF_MULTCLR; 2727 PMAPCOUNT(vac_flush_lots); 2728 } else { 2729 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2730 end_va = va_offset; 2731 PMAPCOUNT(vac_flush_one); 2732 } 2733 /* 2734 * Mark that the page is no longer dirty. 2735 */ 2736 md->pvh_attrs &= ~PVF_DIRTY; 2737 wbinv_p = true; 2738 break; 2739 case PMAP_FLUSH_SECONDARY: 2740 va_offset = 0; 2741 end_va = arm_cache_prefer_mask; 2742 wbinv_p = true; 2743 md->pvh_attrs &= ~PVF_MULTCLR; 2744 PMAPCOUNT(vac_flush_lots); 2745 break; 2746 case PMAP_CLEAN_PRIMARY: 2747 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2748 end_va = va_offset; 2749 wbinv_p = false; 2750 /* 2751 * Mark that the page is no longer dirty. 2752 */ 2753 if ((md->pvh_attrs & PVF_DMOD) == 0) 2754 md->pvh_attrs &= ~PVF_DIRTY; 2755 PMAPCOUNT(vac_clean_one); 2756 break; 2757 default: 2758 return; 2759 } 2760 2761 KASSERT(!(md->pvh_attrs & PVF_NC)); 2762 2763 UVMHIST_LOg(maphist, "md %#jx (attrs=%#jx)",(uintptr_t)md, 2764 md->pvh_attrs); 2765 2766 const size_t scache_line_size = arm_scache.dcache_line_size; 2767 2768 for (; va_offset <= end_va; va_offset += PAGE_SIZE) { 2769 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 2770 const vaddr_t dstp = cpu_cdstp(va_offset); 2771 const pt_entry_t opte = *ptep; 2772 2773 if (flush == PMAP_FLUSH_SECONDARY 2774 && va_offset == (md->pvh_attrs & arm_cache_prefer_mask)) 2775 continue; 2776 2777 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2778 /* 2779 * Set up a PTE with the right coloring to flush 2780 * existing cache entries. 2781 */ 2782 const pt_entry_t npte = L2_S_PROTO 2783 | pa 2784 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2785 | pte_l2_s_cache_mode; 2786 l2pte_set(ptep, npte, opte); 2787 PTE_SYNC(ptep); 2788 2789 /* 2790 * Flush it. Make sure to flush secondary cache too since 2791 * bus_dma will ignore uncached pages. 2792 */ 2793 if (scache_line_size != 0) { 2794 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2795 if (wbinv_p) { 2796 cpu_sdcache_wbinv_range(dstp, pa, PAGE_SIZE); 2797 cpu_dcache_inv_range(dstp, PAGE_SIZE); 2798 } else { 2799 cpu_sdcache_wb_range(dstp, pa, PAGE_SIZE); 2800 } 2801 } else { 2802 if (wbinv_p) { 2803 cpu_dcache_wbinv_range(dstp, PAGE_SIZE); 2804 } else { 2805 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2806 } 2807 } 2808 2809 /* 2810 * Restore the page table entry since we might have interrupted 2811 * pmap_zero_page or pmap_copy_page which was already using 2812 * this pte. 2813 */ 2814 if (opte) { 2815 l2pte_set(ptep, opte, npte); 2816 } else { 2817 l2pte_reset(ptep); 2818 } 2819 PTE_SYNC(ptep); 2820 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2821 } 2822 } 2823 #endif /* ARM_MMU_EXTENDED */ 2824 #endif /* PMAP_CACHE_VIPT */ 2825 2826 /* 2827 * Routine: pmap_page_remove 2828 * Function: 2829 * Removes this physical page from 2830 * all physical maps in which it resides. 2831 * Reflects back modify bits to the pager. 2832 */ 2833 static void 2834 pmap_page_remove(struct vm_page_md *md, paddr_t pa) 2835 { 2836 struct l2_bucket *l2b; 2837 struct pv_entry *pv; 2838 pt_entry_t *ptep; 2839 #ifndef ARM_MMU_EXTENDED 2840 bool flush = false; 2841 #endif 2842 u_int flags = 0; 2843 2844 UVMHIST_FUNC(__func__); 2845 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx", (uintptr_t)md, pa, 0, 0); 2846 2847 struct pv_entry **pvp = &SLIST_FIRST(&md->pvh_list); 2848 pmap_acquire_page_lock(md); 2849 if (*pvp == NULL) { 2850 #ifdef PMAP_CACHE_VIPT 2851 /* 2852 * We *know* the page contents are about to be replaced. 2853 * Discard the exec contents 2854 */ 2855 if (PV_IS_EXEC_P(md->pvh_attrs)) 2856 PMAPCOUNT(exec_discarded_page_protect); 2857 md->pvh_attrs &= ~PVF_EXEC; 2858 PMAP_VALIDATE_MD_PAGE(md); 2859 #endif 2860 pmap_release_page_lock(md); 2861 return; 2862 } 2863 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 2864 KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md)); 2865 #endif 2866 2867 /* 2868 * Clear alias counts 2869 */ 2870 #ifdef PMAP_CACHE_VIVT 2871 md->k_mappings = 0; 2872 #endif 2873 md->urw_mappings = md->uro_mappings = 0; 2874 2875 #ifdef PMAP_CACHE_VIVT 2876 pmap_clean_page(md, false); 2877 #endif 2878 2879 for (pv = *pvp; pv != NULL;) { 2880 pmap_t pm = pv->pv_pmap; 2881 #ifndef ARM_MMU_EXTENDED 2882 if (flush == false && pmap_is_current(pm)) 2883 flush = true; 2884 #endif 2885 2886 #ifdef PMAP_CACHE_VIPT 2887 if (pm == pmap_kernel() && PV_IS_KENTRY_P(pv->pv_flags)) { 2888 /* If this was unmanaged mapping, it must be ignored. */ 2889 pvp = &SLIST_NEXT(pv, pv_link); 2890 pv = *pvp; 2891 continue; 2892 } 2893 #endif 2894 2895 /* 2896 * Try to get a hold on the pmap's lock. We must do this 2897 * while still holding the page locked, to know that the 2898 * page is still associated with the pmap and the mapping is 2899 * in place. If a hold can't be had, unlock and wait for 2900 * the pmap's lock to become available and retry. The pmap 2901 * must be ref'd over this dance to stop it disappearing 2902 * behind us. 2903 */ 2904 if (!mutex_tryenter(&pm->pm_lock)) { 2905 pmap_reference(pm); 2906 pmap_release_page_lock(md); 2907 pmap_acquire_pmap_lock(pm); 2908 /* nothing, just wait for it */ 2909 pmap_release_pmap_lock(pm); 2910 pmap_destroy(pm); 2911 /* Restart from the beginning. */ 2912 pmap_acquire_page_lock(md); 2913 pvp = &SLIST_FIRST(&md->pvh_list); 2914 pv = *pvp; 2915 continue; 2916 } 2917 2918 if (pm == pmap_kernel()) { 2919 #ifdef PMAP_CACHE_VIPT 2920 if (pv->pv_flags & PVF_WRITE) 2921 md->krw_mappings--; 2922 else 2923 md->kro_mappings--; 2924 #endif 2925 PMAPCOUNT(kernel_unmappings); 2926 } 2927 *pvp = SLIST_NEXT(pv, pv_link); /* remove from list */ 2928 PMAPCOUNT(unmappings); 2929 2930 pmap_release_page_lock(md); 2931 2932 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2933 KASSERTMSG(l2b != NULL, "%#lx", pv->pv_va); 2934 2935 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2936 2937 /* 2938 * Update statistics 2939 */ 2940 --pm->pm_stats.resident_count; 2941 2942 /* Wired bit */ 2943 if (pv->pv_flags & PVF_WIRED) 2944 --pm->pm_stats.wired_count; 2945 2946 flags |= pv->pv_flags; 2947 2948 /* 2949 * Invalidate the PTEs. 2950 */ 2951 l2pte_reset(ptep); 2952 PTE_SYNC_CURRENT(pm, ptep); 2953 2954 #ifdef ARM_MMU_EXTENDED 2955 pmap_tlb_invalidate_addr(pm, pv->pv_va); 2956 #endif 2957 2958 pmap_free_l2_bucket(pm, l2b, PAGE_SIZE / L2_S_SIZE); 2959 2960 pmap_release_pmap_lock(pm); 2961 2962 pool_put(&pmap_pv_pool, pv); 2963 pmap_acquire_page_lock(md); 2964 2965 /* 2966 * Restart at the beginning of the list. 2967 */ 2968 pvp = &SLIST_FIRST(&md->pvh_list); 2969 pv = *pvp; 2970 } 2971 /* 2972 * if we reach the end of the list and there are still mappings, they 2973 * might be able to be cached now. And they must be kernel mappings. 2974 */ 2975 if (!SLIST_EMPTY(&md->pvh_list)) { 2976 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 2977 } 2978 2979 #ifdef PMAP_CACHE_VIPT 2980 /* 2981 * Its EXEC cache is now gone. 2982 */ 2983 if (PV_IS_EXEC_P(md->pvh_attrs)) 2984 PMAPCOUNT(exec_discarded_page_protect); 2985 md->pvh_attrs &= ~PVF_EXEC; 2986 KASSERT(md->urw_mappings == 0); 2987 KASSERT(md->uro_mappings == 0); 2988 #ifndef ARM_MMU_EXTENDED 2989 if (arm_cache_prefer_mask != 0) { 2990 if (md->krw_mappings == 0) 2991 md->pvh_attrs &= ~PVF_WRITE; 2992 PMAP_VALIDATE_MD_PAGE(md); 2993 } 2994 #endif /* ARM_MMU_EXTENDED */ 2995 #endif /* PMAP_CACHE_VIPT */ 2996 pmap_release_page_lock(md); 2997 2998 #ifndef ARM_MMU_EXTENDED 2999 if (flush) { 3000 /* 3001 * Note: We can't use pmap_tlb_flush{I,D}() here since that 3002 * would need a subsequent call to pmap_update() to ensure 3003 * curpm->pm_cstate.cs_all is reset. Our callers are not 3004 * required to do that (see pmap(9)), so we can't modify 3005 * the current pmap's state. 3006 */ 3007 if (PV_BEEN_EXECD(flags)) 3008 cpu_tlb_flushID(); 3009 else 3010 cpu_tlb_flushD(); 3011 } 3012 cpu_cpwait(); 3013 #endif /* ARM_MMU_EXTENDED */ 3014 } 3015 3016 /* 3017 * pmap_t pmap_create(void) 3018 * 3019 * Create a new pmap structure from scratch. 3020 */ 3021 pmap_t 3022 pmap_create(void) 3023 { 3024 pmap_t pm; 3025 3026 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 3027 3028 mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_NONE); 3029 3030 pm->pm_refs = 1; 3031 pm->pm_stats.wired_count = 0; 3032 pm->pm_stats.resident_count = 1; 3033 #ifdef ARM_MMU_EXTENDED 3034 #ifdef MULTIPROCESSOR 3035 kcpuset_create(&pm->pm_active, true); 3036 kcpuset_create(&pm->pm_onproc, true); 3037 #endif 3038 #else 3039 pm->pm_cstate.cs_all = 0; 3040 #endif 3041 pmap_alloc_l1(pm); 3042 3043 /* 3044 * Note: The pool cache ensures that the pm_l2[] array is already 3045 * initialised to zero. 3046 */ 3047 3048 pmap_pinit(pm); 3049 3050 return pm; 3051 } 3052 3053 u_int 3054 arm32_mmap_flags(paddr_t pa) 3055 { 3056 /* 3057 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff 3058 * and we're using the upper bits in page numbers to pass flags around 3059 * so we might as well use the same bits 3060 */ 3061 return (u_int)pa & PMAP_MD_MASK; 3062 } 3063 /* 3064 * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 3065 * u_int flags) 3066 * 3067 * Insert the given physical page (p) at 3068 * the specified virtual address (v) in the 3069 * target physical map with the protection requested. 3070 * 3071 * NB: This is the only routine which MAY NOT lazy-evaluate 3072 * or lose information. That is, this routine must actually 3073 * insert this page into the given map NOW. 3074 */ 3075 int 3076 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3077 { 3078 struct l2_bucket *l2b; 3079 struct vm_page *pg, *opg; 3080 u_int nflags; 3081 u_int oflags; 3082 const bool kpm_p = (pm == pmap_kernel()); 3083 #ifdef ARM_HAS_VBAR 3084 const bool vector_page_p = false; 3085 #else 3086 const bool vector_page_p = (va == vector_page); 3087 #endif 3088 struct pmap_page *pp = pmap_pv_tracked(pa); 3089 struct pv_entry *new_pv = NULL; 3090 struct pv_entry *old_pv = NULL; 3091 int error = 0; 3092 3093 UVMHIST_FUNC(__func__); 3094 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pa %#jx prot %#jx", 3095 (uintptr_t)pm, va, pa, prot); 3096 UVMHIST_LOG(maphist, " flag %#jx", flags, 0, 0, 0); 3097 3098 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 3099 KDASSERT(((va | pa) & PGOFSET) == 0); 3100 3101 /* 3102 * Get a pointer to the page. Later on in this function, we 3103 * test for a managed page by checking pg != NULL. 3104 */ 3105 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 3106 /* 3107 * if we may need a new pv entry allocate if now, as we can't do it 3108 * with the kernel_pmap locked 3109 */ 3110 if (pg || pp) 3111 new_pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3112 3113 nflags = 0; 3114 if (prot & VM_PROT_WRITE) 3115 nflags |= PVF_WRITE; 3116 if (prot & VM_PROT_EXECUTE) 3117 nflags |= PVF_EXEC; 3118 if (flags & PMAP_WIRED) 3119 nflags |= PVF_WIRED; 3120 3121 pmap_acquire_pmap_lock(pm); 3122 3123 /* 3124 * Fetch the L2 bucket which maps this page, allocating one if 3125 * necessary for user pmaps. 3126 */ 3127 if (kpm_p) { 3128 l2b = pmap_get_l2_bucket(pm, va); 3129 } else { 3130 l2b = pmap_alloc_l2_bucket(pm, va); 3131 } 3132 if (l2b == NULL) { 3133 if (flags & PMAP_CANFAIL) { 3134 pmap_release_pmap_lock(pm); 3135 error = ENOMEM; 3136 goto free_pv; 3137 } 3138 panic("pmap_enter: failed to allocate L2 bucket"); 3139 } 3140 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)]; 3141 const pt_entry_t opte = *ptep; 3142 pt_entry_t npte = pa; 3143 oflags = 0; 3144 3145 if (opte) { 3146 /* 3147 * There is already a mapping at this address. 3148 * If the physical address is different, lookup the 3149 * vm_page. 3150 */ 3151 if (l2pte_pa(opte) != pa) { 3152 KASSERT(!pmap_pv_tracked(pa)); 3153 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3154 } else 3155 opg = pg; 3156 } else 3157 opg = NULL; 3158 3159 if (pg || pp) { 3160 KASSERT((pg != NULL) != (pp != NULL)); 3161 struct vm_page_md *md = (pg != NULL) ? VM_PAGE_TO_MD(pg) : 3162 PMAP_PAGE_TO_MD(pp); 3163 3164 /* 3165 * This is to be a managed mapping. 3166 */ 3167 pmap_acquire_page_lock(md); 3168 if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) { 3169 /* 3170 * - The access type indicates that we don't need 3171 * to do referenced emulation. 3172 * OR 3173 * - The physical page has already been referenced 3174 * so no need to re-do referenced emulation here. 3175 */ 3176 npte |= l2pte_set_readonly(L2_S_PROTO); 3177 3178 nflags |= PVF_REF; 3179 3180 if ((prot & VM_PROT_WRITE) != 0 && 3181 ((flags & VM_PROT_WRITE) != 0 || 3182 (md->pvh_attrs & PVF_MOD) != 0)) { 3183 /* 3184 * This is a writable mapping, and the 3185 * page's mod state indicates it has 3186 * already been modified. Make it 3187 * writable from the outset. 3188 */ 3189 npte = l2pte_set_writable(npte); 3190 nflags |= PVF_MOD; 3191 } 3192 3193 #ifdef ARM_MMU_EXTENDED 3194 /* 3195 * If the page has been cleaned, then the pvh_attrs 3196 * will have PVF_EXEC set, so mark it execute so we 3197 * don't get an access fault when trying to execute 3198 * from it. 3199 */ 3200 if (md->pvh_attrs & nflags & PVF_EXEC) { 3201 npte &= ~L2_XS_XN; 3202 } 3203 #endif 3204 } else { 3205 /* 3206 * Need to do page referenced emulation. 3207 */ 3208 npte |= L2_TYPE_INV; 3209 } 3210 3211 if (flags & ARM32_MMAP_WRITECOMBINE) { 3212 npte |= pte_l2_s_wc_mode; 3213 } else 3214 npte |= pte_l2_s_cache_mode; 3215 3216 if (pg != NULL && pg == opg) { 3217 /* 3218 * We're changing the attrs of an existing mapping. 3219 */ 3220 oflags = pmap_modify_pv(md, pa, pm, va, 3221 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3222 PVF_MOD | PVF_REF, nflags); 3223 3224 #ifdef PMAP_CACHE_VIVT 3225 /* 3226 * We may need to flush the cache if we're 3227 * doing rw-ro... 3228 */ 3229 if (pm->pm_cstate.cs_cache_d && 3230 (oflags & PVF_NC) == 0 && 3231 l2pte_writable_p(opte) && 3232 (prot & VM_PROT_WRITE) == 0) 3233 cpu_dcache_wb_range(va, PAGE_SIZE); 3234 #endif 3235 } else { 3236 struct pv_entry *pv; 3237 /* 3238 * New mapping, or changing the backing page 3239 * of an existing mapping. 3240 */ 3241 if (opg) { 3242 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3243 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3244 3245 /* 3246 * Replacing an existing mapping with a new one. 3247 * It is part of our managed memory so we 3248 * must remove it from the PV list 3249 */ 3250 pv = pmap_remove_pv(omd, opa, pm, va); 3251 pmap_vac_me_harder(omd, opa, pm, 0); 3252 oflags = pv->pv_flags; 3253 3254 #ifdef PMAP_CACHE_VIVT 3255 /* 3256 * If the old mapping was valid (ref/mod 3257 * emulation creates 'invalid' mappings 3258 * initially) then make sure to frob 3259 * the cache. 3260 */ 3261 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3262 pmap_cache_wbinv_page(pm, va, true, 3263 oflags); 3264 } 3265 #endif 3266 } else { 3267 pv = new_pv; 3268 new_pv = NULL; 3269 if (pv == NULL) { 3270 pmap_release_page_lock(md); 3271 pmap_release_pmap_lock(pm); 3272 if ((flags & PMAP_CANFAIL) == 0) 3273 panic("pmap_enter: " 3274 "no pv entries"); 3275 3276 pmap_free_l2_bucket(pm, l2b, 0); 3277 UVMHIST_LOG(maphist, " <-- done (ENOMEM)", 3278 0, 0, 0, 0); 3279 return ENOMEM; 3280 } 3281 } 3282 3283 pmap_enter_pv(md, pa, pv, pm, va, nflags); 3284 } 3285 pmap_release_page_lock(md); 3286 } else { 3287 /* 3288 * We're mapping an unmanaged page. 3289 * These are always readable, and possibly writable, from 3290 * the get go as we don't need to track ref/mod status. 3291 */ 3292 npte |= l2pte_set_readonly(L2_S_PROTO); 3293 if (prot & VM_PROT_WRITE) 3294 npte = l2pte_set_writable(npte); 3295 3296 /* 3297 * Make sure the vector table is mapped cacheable 3298 */ 3299 if ((vector_page_p && !kpm_p) 3300 || (flags & ARM32_MMAP_CACHEABLE)) { 3301 npte |= pte_l2_s_cache_mode; 3302 #ifdef ARM_MMU_EXTENDED 3303 npte &= ~L2_XS_XN; /* and executable */ 3304 #endif 3305 } else if (flags & ARM32_MMAP_WRITECOMBINE) { 3306 npte |= pte_l2_s_wc_mode; 3307 } 3308 if (opg) { 3309 /* 3310 * Looks like there's an existing 'managed' mapping 3311 * at this address. 3312 */ 3313 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3314 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3315 3316 pmap_acquire_page_lock(omd); 3317 old_pv = pmap_remove_pv(omd, opa, pm, va); 3318 pmap_vac_me_harder(omd, opa, pm, 0); 3319 oflags = old_pv->pv_flags; 3320 pmap_release_page_lock(omd); 3321 3322 #ifdef PMAP_CACHE_VIVT 3323 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3324 pmap_cache_wbinv_page(pm, va, true, oflags); 3325 } 3326 #endif 3327 } 3328 } 3329 3330 /* 3331 * Make sure userland mappings get the right permissions 3332 */ 3333 if (!vector_page_p && !kpm_p) { 3334 npte |= L2_S_PROT_U; 3335 #ifdef ARM_MMU_EXTENDED 3336 npte |= L2_XS_nG; /* user pages are not global */ 3337 #endif 3338 } 3339 3340 /* 3341 * Keep the stats up to date 3342 */ 3343 if (opte == 0) { 3344 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3345 pm->pm_stats.resident_count++; 3346 } 3347 3348 UVMHIST_LOG(maphist, " opte %#jx npte %#jx", opte, npte, 0, 0); 3349 3350 #if defined(ARM_MMU_EXTENDED) 3351 /* 3352 * If exec protection was requested but the page hasn't been synced, 3353 * sync it now and allow execution from it. 3354 */ 3355 if ((nflags & PVF_EXEC) && (npte & L2_XS_XN)) { 3356 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3357 npte &= ~L2_XS_XN; 3358 pmap_syncicache_page(md, pa); 3359 PMAPCOUNT(exec_synced_map); 3360 } 3361 #endif 3362 /* 3363 * If this is just a wiring change, the two PTEs will be 3364 * identical, so there's no need to update the page table. 3365 */ 3366 if (npte != opte) { 3367 l2pte_reset(ptep); 3368 PTE_SYNC(ptep); 3369 if (l2pte_valid_p(opte)) { 3370 pmap_tlb_flush_SE(pm, va, oflags); 3371 } 3372 l2pte_set(ptep, npte, 0); 3373 PTE_SYNC(ptep); 3374 #ifndef ARM_MMU_EXTENDED 3375 bool is_cached = pmap_is_cached(pm); 3376 if (is_cached) { 3377 /* 3378 * We only need to frob the cache/tlb if this pmap 3379 * is current 3380 */ 3381 if (!vector_page_p && l2pte_valid_p(npte)) { 3382 /* 3383 * This mapping is likely to be accessed as 3384 * soon as we return to userland. Fix up the 3385 * L1 entry to avoid taking another 3386 * page/domain fault. 3387 */ 3388 pd_entry_t *pdep = pmap_l1_kva(pm) 3389 + l1pte_index(va); 3390 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa 3391 | L1_C_DOM(pmap_domain(pm)); 3392 if (*pdep != pde) { 3393 l1pte_setone(pdep, pde); 3394 PDE_SYNC(pdep); 3395 } 3396 } 3397 } 3398 3399 UVMHIST_LOG(maphist, " is_cached %jd cs 0x%08jx", 3400 is_cached, pm->pm_cstate.cs_all, 0, 0); 3401 3402 if (pg != NULL) { 3403 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3404 3405 pmap_acquire_page_lock(md); 3406 pmap_vac_me_harder(md, pa, pm, va); 3407 pmap_release_page_lock(md); 3408 } 3409 #endif 3410 } 3411 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 3412 if (pg) { 3413 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3414 3415 pmap_acquire_page_lock(md); 3416 #ifndef ARM_MMU_EXTENDED 3417 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3418 #endif 3419 PMAP_VALIDATE_MD_PAGE(md); 3420 pmap_release_page_lock(md); 3421 } 3422 #endif 3423 3424 pmap_release_pmap_lock(pm); 3425 3426 3427 if (old_pv) 3428 pool_put(&pmap_pv_pool, old_pv); 3429 free_pv: 3430 if (new_pv) 3431 pool_put(&pmap_pv_pool, new_pv); 3432 return error; 3433 } 3434 3435 /* 3436 * pmap_remove() 3437 * 3438 * pmap_remove is responsible for nuking a number of mappings for a range 3439 * of virtual address space in the current pmap. To do this efficiently 3440 * is interesting, because in a number of cases a wide virtual address 3441 * range may be supplied that contains few actual mappings. So, the 3442 * optimisations are: 3443 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3444 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3445 * maybe do just a partial cache clean. This path of execution is 3446 * complicated by the fact that the cache must be flushed _before_ 3447 * the PTE is nuked, being a VAC :-) 3448 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3449 * all invalidations until pmap_update(), since pmap_remove_all() has 3450 * already flushed the cache. 3451 * 4. Maybe later fast-case a single page, but I don't think this is 3452 * going to make _that_ much difference overall. 3453 */ 3454 3455 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3456 3457 void 3458 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 3459 { 3460 SLIST_HEAD(,pv_entry) opv_list; 3461 struct pv_entry *pv, *npv; 3462 UVMHIST_FUNC(__func__); 3463 UVMHIST_CALLARGS(maphist, " (pm=%#jx, sva=%#jx, eva=%#jx)", 3464 (uintptr_t)pm, sva, eva, 0); 3465 3466 #ifdef PMAP_FAULTINFO 3467 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 3468 curpcb->pcb_faultinfo.pfi_repeats = 0; 3469 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 3470 #endif 3471 3472 SLIST_INIT(&opv_list); 3473 /* 3474 * we lock in the pmap => pv_head direction 3475 */ 3476 pmap_acquire_pmap_lock(pm); 3477 3478 #ifndef ARM_MMU_EXTENDED 3479 u_int cleanlist_idx, total, cnt; 3480 struct { 3481 vaddr_t va; 3482 pt_entry_t *ptep; 3483 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3484 3485 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3486 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3487 if (pm->pm_cstate.cs_tlb == 0) 3488 pm->pm_remove_all = true; 3489 } else 3490 cleanlist_idx = 0; 3491 total = 0; 3492 #endif 3493 3494 while (sva < eva) { 3495 /* 3496 * Do one L2 bucket's worth at a time. 3497 */ 3498 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(sva); 3499 if (next_bucket > eva) 3500 next_bucket = eva; 3501 3502 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva); 3503 if (l2b == NULL) { 3504 sva = next_bucket; 3505 continue; 3506 } 3507 3508 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3509 u_int mappings = 0; 3510 3511 for (;sva < next_bucket; 3512 sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) { 3513 pt_entry_t opte = *ptep; 3514 3515 if (opte == 0) { 3516 /* Nothing here, move along */ 3517 continue; 3518 } 3519 3520 u_int flags = PVF_REF; 3521 paddr_t pa = l2pte_pa(opte); 3522 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 3523 3524 /* 3525 * Update flags. In a number of circumstances, 3526 * we could cluster a lot of these and do a 3527 * number of sequential pages in one go. 3528 */ 3529 if (pg != NULL) { 3530 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3531 3532 pmap_acquire_page_lock(md); 3533 pv = pmap_remove_pv(md, pa, pm, sva); 3534 pmap_vac_me_harder(md, pa, pm, 0); 3535 pmap_release_page_lock(md); 3536 if (pv != NULL) { 3537 if (pm->pm_remove_all == false) { 3538 flags = pv->pv_flags; 3539 } 3540 SLIST_INSERT_HEAD(&opv_list, 3541 pv, pv_link); 3542 } 3543 } 3544 mappings += PAGE_SIZE / L2_S_SIZE; 3545 3546 if (!l2pte_valid_p(opte)) { 3547 /* 3548 * Ref/Mod emulation is still active for this 3549 * mapping, therefore it is has not yet been 3550 * accessed. No need to frob the cache/tlb. 3551 */ 3552 l2pte_reset(ptep); 3553 PTE_SYNC_CURRENT(pm, ptep); 3554 continue; 3555 } 3556 3557 #ifdef ARM_MMU_EXTENDED 3558 l2pte_reset(ptep); 3559 PTE_SYNC(ptep); 3560 if (__predict_false(pm->pm_remove_all == false)) { 3561 pmap_tlb_flush_SE(pm, sva, flags); 3562 } 3563 #else 3564 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3565 /* Add to the clean list. */ 3566 cleanlist[cleanlist_idx].ptep = ptep; 3567 cleanlist[cleanlist_idx].va = 3568 sva | (flags & PVF_EXEC); 3569 cleanlist_idx++; 3570 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3571 /* Nuke everything if needed. */ 3572 #ifdef PMAP_CACHE_VIVT 3573 pmap_cache_wbinv_all(pm, PVF_EXEC); 3574 #endif 3575 /* 3576 * Roll back the previous PTE list, 3577 * and zero out the current PTE. 3578 */ 3579 for (cnt = 0; 3580 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3581 l2pte_reset(cleanlist[cnt].ptep); 3582 PTE_SYNC(cleanlist[cnt].ptep); 3583 } 3584 l2pte_reset(ptep); 3585 PTE_SYNC(ptep); 3586 cleanlist_idx++; 3587 pm->pm_remove_all = true; 3588 } else { 3589 l2pte_reset(ptep); 3590 PTE_SYNC(ptep); 3591 if (pm->pm_remove_all == false) { 3592 pmap_tlb_flush_SE(pm, sva, flags); 3593 } 3594 } 3595 #endif 3596 } 3597 3598 #ifndef ARM_MMU_EXTENDED 3599 /* 3600 * Deal with any left overs 3601 */ 3602 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3603 total += cleanlist_idx; 3604 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3605 l2pte_reset(cleanlist[cnt].ptep); 3606 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3607 vaddr_t va = cleanlist[cnt].va; 3608 if (pm->pm_cstate.cs_all != 0) { 3609 vaddr_t clva = va & ~PAGE_MASK; 3610 u_int flags = va & PVF_EXEC; 3611 #ifdef PMAP_CACHE_VIVT 3612 pmap_cache_wbinv_page(pm, clva, true, 3613 PVF_REF | PVF_WRITE | flags); 3614 #endif 3615 pmap_tlb_flush_SE(pm, clva, 3616 PVF_REF | flags); 3617 } 3618 } 3619 3620 /* 3621 * If it looks like we're removing a whole bunch 3622 * of mappings, it's faster to just write-back 3623 * the whole cache now and defer TLB flushes until 3624 * pmap_update() is called. 3625 */ 3626 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3627 cleanlist_idx = 0; 3628 else { 3629 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3630 #ifdef PMAP_CACHE_VIVT 3631 pmap_cache_wbinv_all(pm, PVF_EXEC); 3632 #endif 3633 pm->pm_remove_all = true; 3634 } 3635 } 3636 #endif /* ARM_MMU_EXTENDED */ 3637 3638 pmap_free_l2_bucket(pm, l2b, mappings); 3639 pm->pm_stats.resident_count -= mappings / (PAGE_SIZE/L2_S_SIZE); 3640 } 3641 3642 pmap_release_pmap_lock(pm); 3643 SLIST_FOREACH_SAFE(pv, &opv_list, pv_link, npv) { 3644 pool_put(&pmap_pv_pool, pv); 3645 } 3646 } 3647 3648 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3649 static struct pv_entry * 3650 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3651 { 3652 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3653 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3654 struct pv_entry *pv; 3655 3656 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC)); 3657 KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0); 3658 KASSERT(pmap_page_locked_p(md)); 3659 3660 pv = pmap_remove_pv(md, pa, pmap_kernel(), va); 3661 KASSERTMSG(pv, "pg %p (pa #%lx) va %#lx", pg, pa, va); 3662 KASSERT(PV_IS_KENTRY_P(pv->pv_flags)); 3663 3664 /* 3665 * We are removing a writeable mapping to a cached exec page, if 3666 * it's the last mapping then clear its execness otherwise sync 3667 * the page to the icache. 3668 */ 3669 if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3670 && (pv->pv_flags & PVF_WRITE) != 0) { 3671 if (SLIST_EMPTY(&md->pvh_list)) { 3672 md->pvh_attrs &= ~PVF_EXEC; 3673 PMAPCOUNT(exec_discarded_kremove); 3674 } else { 3675 pmap_syncicache_page(md, pa); 3676 PMAPCOUNT(exec_synced_kremove); 3677 } 3678 } 3679 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 3680 3681 return pv; 3682 } 3683 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 3684 3685 /* 3686 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3687 * 3688 * We assume there is already sufficient KVM space available 3689 * to do this, as we can't allocate L2 descriptor tables/metadata 3690 * from here. 3691 */ 3692 void 3693 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3694 { 3695 #ifdef PMAP_CACHE_VIVT 3696 struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3697 #endif 3698 #ifdef PMAP_CACHE_VIPT 3699 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3700 struct vm_page *opg; 3701 #ifndef ARM_MMU_EXTENDED 3702 struct pv_entry *pv = NULL; 3703 #endif 3704 #endif 3705 struct vm_page_md *md = pg != NULL ? VM_PAGE_TO_MD(pg) : NULL; 3706 3707 UVMHIST_FUNC(__func__); 3708 3709 if (pmap_initialized) { 3710 UVMHIST_CALLARGS(maphist, 3711 "va=%#jx, pa=%#jx, prot=%#jx, flags=%#jx", va, pa, prot, 3712 flags); 3713 } 3714 3715 pmap_t kpm = pmap_kernel(); 3716 pmap_acquire_pmap_lock(kpm); 3717 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3718 const size_t l1slot __diagused = l1pte_index(va); 3719 KASSERTMSG(l2b != NULL, 3720 "va %#lx pa %#lx prot %d maxkvaddr %#lx: l2 %p l2b %p kva %p", 3721 va, pa, prot, pmap_curmaxkvaddr, kpm->pm_l2[L2_IDX(l1slot)], 3722 kpm->pm_l2[L2_IDX(l1slot)] 3723 ? &kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)] 3724 : NULL, 3725 kpm->pm_l2[L2_IDX(l1slot)] 3726 ? kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)].l2b_kva 3727 : NULL); 3728 KASSERT(l2b->l2b_kva != NULL); 3729 3730 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 3731 const pt_entry_t opte = *ptep; 3732 3733 if (opte == 0) { 3734 PMAPCOUNT(kenter_mappings); 3735 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3736 } else { 3737 PMAPCOUNT(kenter_remappings); 3738 #ifdef PMAP_CACHE_VIPT 3739 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3740 #if !defined(ARM_MMU_EXTENDED) || defined(DIAGNOSTIC) 3741 struct vm_page_md *omd __diagused = VM_PAGE_TO_MD(opg); 3742 #endif 3743 if (opg && arm_cache_prefer_mask != 0) { 3744 KASSERT(opg != pg); 3745 KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0); 3746 KASSERT((flags & PMAP_KMPAGE) == 0); 3747 #ifndef ARM_MMU_EXTENDED 3748 pmap_acquire_page_lock(omd); 3749 pv = pmap_kremove_pg(opg, va); 3750 pmap_release_page_lock(omd); 3751 #endif 3752 } 3753 #endif 3754 if (l2pte_valid_p(opte)) { 3755 l2pte_reset(ptep); 3756 PTE_SYNC(ptep); 3757 #ifdef PMAP_CACHE_VIVT 3758 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3759 #endif 3760 cpu_tlb_flushD_SE(va); 3761 cpu_cpwait(); 3762 } 3763 } 3764 pmap_release_pmap_lock(kpm); 3765 pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 3766 3767 if (flags & PMAP_PTE) { 3768 KASSERT((flags & PMAP_CACHE_MASK) == 0); 3769 if (!(flags & PMAP_NOCACHE)) 3770 npte |= pte_l2_s_cache_mode_pt; 3771 } else { 3772 switch (flags & (PMAP_CACHE_MASK | PMAP_DEV_MASK)) { 3773 case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK: 3774 break; 3775 case PMAP_NOCACHE: 3776 npte |= pte_l2_s_nocache_mode; 3777 break; 3778 case PMAP_WRITE_COMBINE: 3779 npte |= pte_l2_s_wc_mode; 3780 break; 3781 default: 3782 npte |= pte_l2_s_cache_mode; 3783 break; 3784 } 3785 } 3786 #ifdef ARM_MMU_EXTENDED 3787 if (prot & VM_PROT_EXECUTE) 3788 npte &= ~L2_XS_XN; 3789 #endif 3790 l2pte_set(ptep, npte, 0); 3791 PTE_SYNC(ptep); 3792 3793 if (pg) { 3794 if (flags & PMAP_KMPAGE) { 3795 KASSERT(md->urw_mappings == 0); 3796 KASSERT(md->uro_mappings == 0); 3797 KASSERT(md->krw_mappings == 0); 3798 KASSERT(md->kro_mappings == 0); 3799 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3800 KASSERT(pv == NULL); 3801 KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0); 3802 KASSERT((md->pvh_attrs & PVF_NC) == 0); 3803 /* if there is a color conflict, evict from cache. */ 3804 if (pmap_is_page_colored_p(md) 3805 && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) { 3806 PMAPCOUNT(vac_color_change); 3807 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 3808 } else if (md->pvh_attrs & PVF_MULTCLR) { 3809 /* 3810 * If this page has multiple colors, expunge 3811 * them. 3812 */ 3813 PMAPCOUNT(vac_flush_lots2); 3814 pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY); 3815 } 3816 /* 3817 * Since this is a KMPAGE, there can be no contention 3818 * for this page so don't lock it. 3819 */ 3820 md->pvh_attrs &= PAGE_SIZE - 1; 3821 md->pvh_attrs |= PVF_KMPAGE | PVF_COLORED | PVF_DIRTY 3822 | (va & arm_cache_prefer_mask); 3823 #else /* !PMAP_CACHE_VIPT || ARM_MMU_EXTENDED */ 3824 md->pvh_attrs |= PVF_KMPAGE; 3825 #endif 3826 atomic_inc_32(&pmap_kmpages); 3827 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3828 } else if (arm_cache_prefer_mask != 0) { 3829 if (pv == NULL) { 3830 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3831 KASSERT(pv != NULL); 3832 } 3833 pmap_acquire_page_lock(md); 3834 pmap_enter_pv(md, pa, pv, pmap_kernel(), va, 3835 PVF_WIRED | PVF_KENTRY 3836 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3837 if ((prot & VM_PROT_WRITE) 3838 && !(md->pvh_attrs & PVF_NC)) 3839 md->pvh_attrs |= PVF_DIRTY; 3840 KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3841 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 3842 pmap_release_page_lock(md); 3843 #endif 3844 } 3845 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3846 } else { 3847 if (pv != NULL) 3848 pool_put(&pmap_pv_pool, pv); 3849 #endif 3850 } 3851 if (pmap_initialized) { 3852 UVMHIST_LOG(maphist, " <-- done (ptep %#jx: %#jx -> %#jx)", 3853 (uintptr_t)ptep, opte, npte, 0); 3854 } 3855 3856 } 3857 3858 void 3859 pmap_kremove(vaddr_t va, vsize_t len) 3860 { 3861 #ifdef UVMHIST 3862 u_int total_mappings = 0; 3863 #endif 3864 3865 PMAPCOUNT(kenter_unmappings); 3866 3867 UVMHIST_FUNC(__func__); 3868 UVMHIST_CALLARGS(maphist, " (va=%#jx, len=%#jx)", va, len, 0, 0); 3869 3870 const vaddr_t eva = va + len; 3871 pmap_t kpm = pmap_kernel(); 3872 3873 pmap_acquire_pmap_lock(kpm); 3874 3875 while (va < eva) { 3876 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(va); 3877 if (next_bucket > eva) 3878 next_bucket = eva; 3879 3880 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3881 KDASSERT(l2b != NULL); 3882 3883 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 3884 pt_entry_t *ptep = sptep; 3885 u_int mappings = 0; 3886 3887 while (va < next_bucket) { 3888 const pt_entry_t opte = *ptep; 3889 struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3890 if (opg != NULL) { 3891 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3892 3893 if (omd->pvh_attrs & PVF_KMPAGE) { 3894 KASSERT(omd->urw_mappings == 0); 3895 KASSERT(omd->uro_mappings == 0); 3896 KASSERT(omd->krw_mappings == 0); 3897 KASSERT(omd->kro_mappings == 0); 3898 omd->pvh_attrs &= ~PVF_KMPAGE; 3899 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3900 if (arm_cache_prefer_mask != 0) { 3901 omd->pvh_attrs &= ~PVF_WRITE; 3902 } 3903 #endif 3904 atomic_dec_32(&pmap_kmpages); 3905 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3906 } else if (arm_cache_prefer_mask != 0) { 3907 pmap_acquire_page_lock(omd); 3908 pool_put(&pmap_pv_pool, 3909 pmap_kremove_pg(opg, va)); 3910 pmap_release_page_lock(omd); 3911 #endif 3912 } 3913 } 3914 if (l2pte_valid_p(opte)) { 3915 l2pte_reset(ptep); 3916 PTE_SYNC(ptep); 3917 #ifdef PMAP_CACHE_VIVT 3918 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3919 #endif 3920 cpu_tlb_flushD_SE(va); 3921 3922 mappings += PAGE_SIZE / L2_S_SIZE; 3923 } 3924 va += PAGE_SIZE; 3925 ptep += PAGE_SIZE / L2_S_SIZE; 3926 } 3927 KDASSERTMSG(mappings <= l2b->l2b_occupancy, "%u %u", 3928 mappings, l2b->l2b_occupancy); 3929 l2b->l2b_occupancy -= mappings; 3930 //PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3931 #ifdef UVMHIST 3932 total_mappings += mappings; 3933 #endif 3934 } 3935 pmap_release_pmap_lock(kpm); 3936 cpu_cpwait(); 3937 UVMHIST_LOG(maphist, " <--- done (%ju mappings removed)", 3938 total_mappings, 0, 0, 0); 3939 } 3940 3941 bool 3942 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3943 { 3944 3945 return pmap_extract_coherency(pm, va, pap, NULL); 3946 } 3947 3948 bool 3949 pmap_extract_coherency(pmap_t pm, vaddr_t va, paddr_t *pap, bool *coherentp) 3950 { 3951 struct l2_dtable *l2; 3952 pd_entry_t *pdep, pde; 3953 pt_entry_t *ptep, pte; 3954 paddr_t pa; 3955 u_int l1slot; 3956 bool coherent; 3957 3958 pmap_acquire_pmap_lock(pm); 3959 3960 l1slot = l1pte_index(va); 3961 pdep = pmap_l1_kva(pm) + l1slot; 3962 pde = *pdep; 3963 3964 if (l1pte_section_p(pde)) { 3965 /* 3966 * These should only happen for pmap_kernel() 3967 */ 3968 KDASSERT(pm == pmap_kernel()); 3969 pmap_release_pmap_lock(pm); 3970 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 3971 if (l1pte_supersection_p(pde)) { 3972 pa = (pde & L1_SS_FRAME) | (va & L1_SS_OFFSET); 3973 } else 3974 #endif 3975 pa = (pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3976 coherent = (pde & L1_S_CACHE_MASK) == 0; 3977 } else { 3978 /* 3979 * Note that we can't rely on the validity of the L1 3980 * descriptor as an indication that a mapping exists. 3981 * We have to look it up in the L2 dtable. 3982 */ 3983 l2 = pm->pm_l2[L2_IDX(l1slot)]; 3984 3985 if (l2 == NULL || 3986 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 3987 pmap_release_pmap_lock(pm); 3988 return false; 3989 } 3990 3991 pte = ptep[l2pte_index(va)]; 3992 pmap_release_pmap_lock(pm); 3993 3994 if (pte == 0) 3995 return false; 3996 3997 switch (pte & L2_TYPE_MASK) { 3998 case L2_TYPE_L: 3999 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 4000 coherent = (pte & L2_L_CACHE_MASK) == 0; 4001 break; 4002 4003 default: 4004 pa = (pte & ~PAGE_MASK) | (va & PAGE_MASK); 4005 coherent = (pte & L2_S_CACHE_MASK) == 0; 4006 break; 4007 } 4008 } 4009 4010 if (pap != NULL) 4011 *pap = pa; 4012 4013 if (coherentp != NULL) 4014 *coherentp = (pm == pmap_kernel() && coherent); 4015 4016 return true; 4017 } 4018 4019 /* 4020 * pmap_pv_remove: remove an unmanaged pv-tracked page from all pmaps 4021 * that map it 4022 */ 4023 4024 static void 4025 pmap_pv_remove(paddr_t pa) 4026 { 4027 struct pmap_page *pp; 4028 4029 pp = pmap_pv_tracked(pa); 4030 if (pp == NULL) 4031 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 4032 pa); 4033 4034 struct vm_page_md *md = PMAP_PAGE_TO_MD(pp); 4035 pmap_page_remove(md, pa); 4036 } 4037 4038 void 4039 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 4040 { 4041 4042 /* the only case is remove at the moment */ 4043 KASSERT(prot == VM_PROT_NONE); 4044 pmap_pv_remove(pa); 4045 } 4046 4047 void 4048 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 4049 { 4050 struct l2_bucket *l2b; 4051 vaddr_t next_bucket; 4052 4053 UVMHIST_FUNC(__func__); 4054 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx...#%jx prot %#jx", 4055 (uintptr_t)pm, sva, eva, prot); 4056 4057 if ((prot & VM_PROT_READ) == 0) { 4058 pmap_remove(pm, sva, eva); 4059 return; 4060 } 4061 4062 if (prot & VM_PROT_WRITE) { 4063 /* 4064 * If this is a read->write transition, just ignore it and let 4065 * uvm_fault() take care of it later. 4066 */ 4067 return; 4068 } 4069 4070 pmap_acquire_pmap_lock(pm); 4071 4072 #ifndef ARM_MMU_EXTENDED 4073 const bool flush = eva - sva >= PAGE_SIZE * 4; 4074 u_int flags = 0; 4075 #endif 4076 u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 4077 4078 while (sva < eva) { 4079 next_bucket = L2_NEXT_BUCKET_VA(sva); 4080 if (next_bucket > eva) 4081 next_bucket = eva; 4082 4083 l2b = pmap_get_l2_bucket(pm, sva); 4084 if (l2b == NULL) { 4085 sva = next_bucket; 4086 continue; 4087 } 4088 4089 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4090 4091 while (sva < next_bucket) { 4092 const pt_entry_t opte = *ptep; 4093 if (l2pte_valid_p(opte) && l2pte_writable_p(opte)) { 4094 struct vm_page *pg; 4095 #ifndef ARM_MMU_EXTENDED 4096 u_int f; 4097 #endif 4098 4099 #ifdef PMAP_CACHE_VIVT 4100 /* 4101 * OK, at this point, we know we're doing 4102 * write-protect operation. If the pmap is 4103 * active, write-back the page. 4104 */ 4105 pmap_cache_wbinv_page(pm, sva, false, 4106 PVF_REF | PVF_WRITE); 4107 #endif 4108 4109 pg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 4110 pt_entry_t npte = l2pte_set_readonly(opte); 4111 l2pte_reset(ptep); 4112 PTE_SYNC(ptep); 4113 #ifdef ARM_MMU_EXTENDED 4114 pmap_tlb_flush_SE(pm, sva, PVF_REF); 4115 #endif 4116 l2pte_set(ptep, npte, 0); 4117 PTE_SYNC(ptep); 4118 4119 if (pg != NULL) { 4120 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4121 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4122 4123 pmap_acquire_page_lock(md); 4124 #ifndef ARM_MMU_EXTENDED 4125 f = 4126 #endif 4127 pmap_modify_pv(md, pa, pm, sva, 4128 clr_mask, 0); 4129 pmap_vac_me_harder(md, pa, pm, sva); 4130 pmap_release_page_lock(md); 4131 #ifndef ARM_MMU_EXTENDED 4132 } else { 4133 f = PVF_REF | PVF_EXEC; 4134 } 4135 4136 if (flush) { 4137 flags |= f; 4138 } else { 4139 pmap_tlb_flush_SE(pm, sva, f); 4140 #endif 4141 } 4142 } 4143 4144 sva += PAGE_SIZE; 4145 ptep += PAGE_SIZE / L2_S_SIZE; 4146 } 4147 } 4148 4149 #ifndef ARM_MMU_EXTENDED 4150 if (flush) { 4151 if (PV_BEEN_EXECD(flags)) { 4152 pmap_tlb_flushID(pm); 4153 } else if (PV_BEEN_REFD(flags)) { 4154 pmap_tlb_flushD(pm); 4155 } 4156 } 4157 #endif 4158 4159 pmap_release_pmap_lock(pm); 4160 } 4161 4162 void 4163 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 4164 { 4165 struct l2_bucket *l2b; 4166 pt_entry_t *ptep; 4167 vaddr_t next_bucket; 4168 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 4169 4170 UVMHIST_FUNC(__func__); 4171 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx...#%jx", 4172 (uintptr_t)pm, sva, eva, 0); 4173 4174 pmap_acquire_pmap_lock(pm); 4175 4176 while (sva < eva) { 4177 next_bucket = L2_NEXT_BUCKET_VA(sva); 4178 if (next_bucket > eva) 4179 next_bucket = eva; 4180 4181 l2b = pmap_get_l2_bucket(pm, sva); 4182 if (l2b == NULL) { 4183 sva = next_bucket; 4184 continue; 4185 } 4186 4187 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4188 sva < next_bucket; 4189 sva += page_size, 4190 ptep += PAGE_SIZE / L2_S_SIZE, 4191 page_size = PAGE_SIZE) { 4192 if (l2pte_valid_p(*ptep)) { 4193 cpu_icache_sync_range(sva, 4194 uimin(page_size, eva - sva)); 4195 } 4196 } 4197 } 4198 4199 pmap_release_pmap_lock(pm); 4200 } 4201 4202 void 4203 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 4204 { 4205 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4206 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4207 4208 UVMHIST_FUNC(__func__); 4209 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx prot %#jx", 4210 (uintptr_t)md, pa, prot, 0); 4211 4212 switch(prot) { 4213 case VM_PROT_READ|VM_PROT_WRITE: 4214 #if defined(ARM_MMU_EXTENDED) 4215 pmap_acquire_page_lock(md); 4216 pmap_clearbit(md, pa, PVF_EXEC); 4217 pmap_release_page_lock(md); 4218 break; 4219 #endif 4220 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 4221 break; 4222 4223 case VM_PROT_READ: 4224 #if defined(ARM_MMU_EXTENDED) 4225 pmap_acquire_page_lock(md); 4226 pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC); 4227 pmap_release_page_lock(md); 4228 break; 4229 #endif 4230 case VM_PROT_READ|VM_PROT_EXECUTE: 4231 pmap_acquire_page_lock(md); 4232 pmap_clearbit(md, pa, PVF_WRITE); 4233 pmap_release_page_lock(md); 4234 break; 4235 4236 default: 4237 pmap_page_remove(md, pa); 4238 break; 4239 } 4240 } 4241 4242 /* 4243 * pmap_clear_modify: 4244 * 4245 * Clear the "modified" attribute for a page. 4246 */ 4247 bool 4248 pmap_clear_modify(struct vm_page *pg) 4249 { 4250 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4251 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4252 bool rv; 4253 4254 pmap_acquire_page_lock(md); 4255 4256 if (md->pvh_attrs & PVF_MOD) { 4257 rv = true; 4258 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4259 /* 4260 * If we are going to clear the modified bit and there are 4261 * no other modified bits set, flush the page to memory and 4262 * mark it clean. 4263 */ 4264 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD) 4265 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 4266 #endif 4267 pmap_clearbit(md, pa, PVF_MOD); 4268 } else { 4269 rv = false; 4270 } 4271 pmap_release_page_lock(md); 4272 4273 return rv; 4274 } 4275 4276 /* 4277 * pmap_clear_reference: 4278 * 4279 * Clear the "referenced" attribute for a page. 4280 */ 4281 bool 4282 pmap_clear_reference(struct vm_page *pg) 4283 { 4284 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4285 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4286 bool rv; 4287 4288 pmap_acquire_page_lock(md); 4289 4290 if (md->pvh_attrs & PVF_REF) { 4291 rv = true; 4292 pmap_clearbit(md, pa, PVF_REF); 4293 } else { 4294 rv = false; 4295 } 4296 pmap_release_page_lock(md); 4297 4298 return rv; 4299 } 4300 4301 /* 4302 * pmap_is_modified: 4303 * 4304 * Test if a page has the "modified" attribute. 4305 */ 4306 /* See <arm/arm32/pmap.h> */ 4307 4308 /* 4309 * pmap_is_referenced: 4310 * 4311 * Test if a page has the "referenced" attribute. 4312 */ 4313 /* See <arm/arm32/pmap.h> */ 4314 4315 #if defined(ARM_MMU_EXTENDED) && 0 4316 int 4317 pmap_prefetchabt_fixup(void *v) 4318 { 4319 struct trapframe * const tf = v; 4320 vaddr_t va = trunc_page(tf->tf_pc); 4321 int rv = ABORT_FIXUP_FAILED; 4322 4323 if (!TRAP_USERMODE(tf) && va < VM_MAXUSER_ADDRESS) 4324 return rv; 4325 4326 kpreempt_disable(); 4327 pmap_t pm = curcpu()->ci_pmap_cur; 4328 const size_t l1slot = l1pte_index(va); 4329 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 4330 if (l2 == NULL) 4331 goto out; 4332 4333 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4334 if (l2b->l2b_kva == NULL) 4335 goto out; 4336 4337 /* 4338 * Check the PTE itself. 4339 */ 4340 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4341 const pt_entry_t opte = *ptep; 4342 if ((opte & L2_S_PROT_U) == 0 || (opte & L2_XS_XN) == 0) 4343 goto out; 4344 4345 paddr_t pa = l2pte_pa(opte); 4346 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4347 KASSERT(pg != NULL); 4348 4349 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4350 4351 pmap_acquire_page_lock(md); 4352 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4353 KASSERT(pv != NULL); 4354 4355 if (PV_IS_EXEC_P(pv->pv_flags)) { 4356 l2pte_reset(ptep); 4357 PTE_SYNC(ptep); 4358 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4359 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 4360 pmap_syncicache_page(md, pa); 4361 } 4362 rv = ABORT_FIXUP_RETURN; 4363 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4364 PTE_SYNC(ptep); 4365 } 4366 pmap_release_page_lock(md); 4367 4368 out: 4369 kpreempt_enable(); 4370 return rv; 4371 } 4372 #endif 4373 4374 int 4375 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 4376 { 4377 struct l2_dtable *l2; 4378 struct l2_bucket *l2b; 4379 paddr_t pa; 4380 const size_t l1slot = l1pte_index(va); 4381 int rv = 0; 4382 4383 UVMHIST_FUNC(__func__); 4384 UVMHIST_CALLARGS(maphist, "pm=%#jx, va=%#jx, ftype=%#jx, user=%jd", 4385 (uintptr_t)pm, va, ftype, user); 4386 4387 va = trunc_page(va); 4388 4389 KASSERT(!user || (pm != pmap_kernel())); 4390 4391 #ifdef ARM_MMU_EXTENDED 4392 UVMHIST_LOG(maphist, " ti=%#jx pai=%#jx asid=%#jx", 4393 (uintptr_t)cpu_tlb_info(curcpu()), 4394 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu())), 4395 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu()))->pai_asid, 0); 4396 #endif 4397 4398 pmap_acquire_pmap_lock(pm); 4399 4400 /* 4401 * If there is no l2_dtable for this address, then the process 4402 * has no business accessing it. 4403 * 4404 * Note: This will catch userland processes trying to access 4405 * kernel addresses. 4406 */ 4407 l2 = pm->pm_l2[L2_IDX(l1slot)]; 4408 if (l2 == NULL) { 4409 UVMHIST_LOG(maphist, " no l2 for l1slot %#jx", l1slot, 0, 0, 0); 4410 goto out; 4411 } 4412 4413 /* 4414 * Likewise if there is no L2 descriptor table 4415 */ 4416 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4417 if (l2b->l2b_kva == NULL) { 4418 UVMHIST_LOG(maphist, " <-- done (no ptep for l1slot %#jx)", 4419 l1slot, 0, 0, 0); 4420 goto out; 4421 } 4422 4423 /* 4424 * Check the PTE itself. 4425 */ 4426 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4427 pt_entry_t const opte = *ptep; 4428 if (opte == 0 || (opte & L2_TYPE_MASK) == L2_TYPE_L) { 4429 UVMHIST_LOG(maphist, " <-- done (empty pde for l1slot %#jx)", 4430 l1slot, 0, 0, 0); 4431 goto out; 4432 } 4433 4434 #ifndef ARM_HAS_VBAR 4435 /* 4436 * Catch a userland access to the vector page mapped at 0x0 4437 */ 4438 if (user && (opte & L2_S_PROT_U) == 0) { 4439 UVMHIST_LOG(maphist, " <-- done (vector_page)", 0, 0, 0, 0); 4440 goto out; 4441 } 4442 #endif 4443 4444 pa = l2pte_pa(opte); 4445 4446 if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(opte)) { 4447 /* 4448 * This looks like a good candidate for "page modified" 4449 * emulation... 4450 */ 4451 struct pv_entry *pv; 4452 struct vm_page *pg; 4453 4454 /* Extract the physical address of the page */ 4455 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4456 UVMHIST_LOG(maphist, " <-- done (mod/ref unmanaged page)", 0, 0, 0, 0); 4457 goto out; 4458 } 4459 4460 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4461 4462 /* Get the current flags for this page. */ 4463 pmap_acquire_page_lock(md); 4464 pv = pmap_find_pv(md, pm, va); 4465 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4466 pmap_release_page_lock(md); 4467 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: no PV)", 0, 0, 0, 0); 4468 goto out; 4469 } 4470 4471 /* 4472 * Do the flags say this page is writable? If not then it 4473 * is a genuine write fault. If yes then the write fault is 4474 * our fault as we did not reflect the write access in the 4475 * PTE. Now we know a write has occurred we can correct this 4476 * and also set the modified bit 4477 */ 4478 if ((pv->pv_flags & PVF_WRITE) == 0) { 4479 pmap_release_page_lock(md); 4480 goto out; 4481 } 4482 4483 md->pvh_attrs |= PVF_REF | PVF_MOD; 4484 pv->pv_flags |= PVF_REF | PVF_MOD; 4485 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4486 /* 4487 * If there are cacheable mappings for this page, mark it dirty. 4488 */ 4489 if ((md->pvh_attrs & PVF_NC) == 0) 4490 md->pvh_attrs |= PVF_DIRTY; 4491 #endif 4492 #ifdef ARM_MMU_EXTENDED 4493 if (md->pvh_attrs & PVF_EXEC) { 4494 md->pvh_attrs &= ~PVF_EXEC; 4495 PMAPCOUNT(exec_discarded_modfixup); 4496 } 4497 #endif 4498 pmap_release_page_lock(md); 4499 4500 /* 4501 * Re-enable write permissions for the page. No need to call 4502 * pmap_vac_me_harder(), since this is just a 4503 * modified-emulation fault, and the PVF_WRITE bit isn't 4504 * changing. We've already set the cacheable bits based on 4505 * the assumption that we can write to this page. 4506 */ 4507 const pt_entry_t npte = 4508 l2pte_set_writable((opte & ~L2_TYPE_MASK) | L2_S_PROTO) 4509 #ifdef ARM_MMU_EXTENDED 4510 | (pm != pmap_kernel() ? L2_XS_nG : 0) 4511 #endif 4512 | 0; 4513 l2pte_reset(ptep); 4514 PTE_SYNC(ptep); 4515 pmap_tlb_flush_SE(pm, va, 4516 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4517 l2pte_set(ptep, npte, 0); 4518 PTE_SYNC(ptep); 4519 PMAPCOUNT(fixup_mod); 4520 rv = 1; 4521 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: changed pte " 4522 "from %#jx to %#jx)", opte, npte, 0, 0); 4523 } else if ((opte & L2_TYPE_MASK) == L2_TYPE_INV) { 4524 /* 4525 * This looks like a good candidate for "page referenced" 4526 * emulation. 4527 */ 4528 struct vm_page *pg; 4529 4530 /* Extract the physical address of the page */ 4531 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4532 UVMHIST_LOG(maphist, " <-- done (ref emul: unmanaged page)", 0, 0, 0, 0); 4533 goto out; 4534 } 4535 4536 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4537 4538 /* Get the current flags for this page. */ 4539 pmap_acquire_page_lock(md); 4540 struct pv_entry *pv = pmap_find_pv(md, pm, va); 4541 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4542 pmap_release_page_lock(md); 4543 UVMHIST_LOG(maphist, " <-- done (ref emul no PV)", 0, 0, 0, 0); 4544 goto out; 4545 } 4546 4547 md->pvh_attrs |= PVF_REF; 4548 pv->pv_flags |= PVF_REF; 4549 4550 pt_entry_t npte = 4551 l2pte_set_readonly((opte & ~L2_TYPE_MASK) | L2_S_PROTO); 4552 #ifdef ARM_MMU_EXTENDED 4553 if (pm != pmap_kernel()) { 4554 npte |= L2_XS_nG; 4555 } 4556 /* 4557 * If we got called from prefetch abort, then ftype will have 4558 * VM_PROT_EXECUTE set. Now see if we have no-execute set in 4559 * the PTE. 4560 */ 4561 if (user && (ftype & VM_PROT_EXECUTE) && (npte & L2_XS_XN)) { 4562 /* 4563 * Is this a mapping of an executable page? 4564 */ 4565 if ((pv->pv_flags & PVF_EXEC) == 0) { 4566 pmap_release_page_lock(md); 4567 UVMHIST_LOG(maphist, " <-- done (ref emul: no exec)", 4568 0, 0, 0, 0); 4569 goto out; 4570 } 4571 /* 4572 * If we haven't synced the page, do so now. 4573 */ 4574 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4575 UVMHIST_LOG(maphist, " ref emul: syncicache " 4576 "page #%#jx", pa, 0, 0, 0); 4577 pmap_syncicache_page(md, pa); 4578 PMAPCOUNT(fixup_exec); 4579 } 4580 npte &= ~L2_XS_XN; 4581 } 4582 #endif /* ARM_MMU_EXTENDED */ 4583 pmap_release_page_lock(md); 4584 l2pte_reset(ptep); 4585 PTE_SYNC(ptep); 4586 pmap_tlb_flush_SE(pm, va, 4587 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4588 l2pte_set(ptep, npte, 0); 4589 PTE_SYNC(ptep); 4590 PMAPCOUNT(fixup_ref); 4591 rv = 1; 4592 UVMHIST_LOG(maphist, " <-- done (ref emul: changed pte from " 4593 "%#jx to %#jx)", opte, npte, 0, 0); 4594 #ifdef ARM_MMU_EXTENDED 4595 } else if (user && (ftype & VM_PROT_EXECUTE) && (opte & L2_XS_XN)) { 4596 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4597 if (pg == NULL) { 4598 UVMHIST_LOG(maphist, " <-- done (unmanaged page)", 0, 0, 0, 0); 4599 goto out; 4600 } 4601 4602 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4603 4604 /* Get the current flags for this page. */ 4605 pmap_acquire_page_lock(md); 4606 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4607 if (pv == NULL || (pv->pv_flags & PVF_EXEC) == 0) { 4608 pmap_release_page_lock(md); 4609 UVMHIST_LOG(maphist, " <-- done (no PV or not EXEC)", 0, 0, 0, 0); 4610 goto out; 4611 } 4612 4613 /* 4614 * If we haven't synced the page, do so now. 4615 */ 4616 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4617 UVMHIST_LOG(maphist, "syncicache page #%#jx", 4618 pa, 0, 0, 0); 4619 pmap_syncicache_page(md, pa); 4620 } 4621 pmap_release_page_lock(md); 4622 /* 4623 * Turn off no-execute. 4624 */ 4625 KASSERT(opte & L2_XS_nG); 4626 l2pte_reset(ptep); 4627 PTE_SYNC(ptep); 4628 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4629 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4630 PTE_SYNC(ptep); 4631 rv = 1; 4632 PMAPCOUNT(fixup_exec); 4633 UVMHIST_LOG(maphist, "exec: changed pte from %#jx to %#jx", 4634 opte, opte & ~L2_XS_XN, 0, 0); 4635 #endif 4636 } 4637 4638 #ifndef ARM_MMU_EXTENDED 4639 /* 4640 * We know there is a valid mapping here, so simply 4641 * fix up the L1 if necessary. 4642 */ 4643 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 4644 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa | L1_C_DOM(pmap_domain(pm)); 4645 if (*pdep != pde) { 4646 l1pte_setone(pdep, pde); 4647 PDE_SYNC(pdep); 4648 rv = 1; 4649 PMAPCOUNT(fixup_pdes); 4650 } 4651 #endif 4652 4653 #ifdef CPU_SA110 4654 /* 4655 * There are bugs in the rev K SA110. This is a check for one 4656 * of them. 4657 */ 4658 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 4659 curcpu()->ci_arm_cpurev < 3) { 4660 /* Always current pmap */ 4661 if (l2pte_valid_p(opte)) { 4662 extern int kernel_debug; 4663 if (kernel_debug & 1) { 4664 struct proc *p = curlwp->l_proc; 4665 printf("prefetch_abort: page is already " 4666 "mapped - pte=%p *pte=%08x\n", ptep, opte); 4667 printf("prefetch_abort: pc=%08lx proc=%p " 4668 "process=%s\n", va, p, p->p_comm); 4669 printf("prefetch_abort: far=%08x fs=%x\n", 4670 cpu_faultaddress(), cpu_faultstatus()); 4671 } 4672 #ifdef DDB 4673 if (kernel_debug & 2) 4674 Debugger(); 4675 #endif 4676 rv = 1; 4677 } 4678 } 4679 #endif /* CPU_SA110 */ 4680 4681 #ifndef ARM_MMU_EXTENDED 4682 /* 4683 * If 'rv == 0' at this point, it generally indicates that there is a 4684 * stale TLB entry for the faulting address. That might be due to a 4685 * wrong setting of pmap_needs_pte_sync. So set it and retry. 4686 */ 4687 if (rv == 0 4688 && pm->pm_l1->l1_domain_use_count == 1 4689 && pmap_needs_pte_sync == 0) { 4690 pmap_needs_pte_sync = 1; 4691 PTE_SYNC(ptep); 4692 PMAPCOUNT(fixup_ptesync); 4693 rv = 1; 4694 } 4695 #endif 4696 4697 #ifndef MULTIPROCESSOR 4698 #if defined(DEBUG) || 1 4699 /* 4700 * If 'rv == 0' at this point, it generally indicates that there is a 4701 * stale TLB entry for the faulting address. This happens when two or 4702 * more processes are sharing an L1. Since we don't flush the TLB on 4703 * a context switch between such processes, we can take domain faults 4704 * for mappings which exist at the same VA in both processes. EVEN IF 4705 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 4706 * example. 4707 * 4708 * This is extremely likely to happen if pmap_enter() updated the L1 4709 * entry for a recently entered mapping. In this case, the TLB is 4710 * flushed for the new mapping, but there may still be TLB entries for 4711 * other mappings belonging to other processes in the 1MB range 4712 * covered by the L1 entry. 4713 * 4714 * Since 'rv == 0', we know that the L1 already contains the correct 4715 * value, so the fault must be due to a stale TLB entry. 4716 * 4717 * Since we always need to flush the TLB anyway in the case where we 4718 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 4719 * stale TLB entries dynamically. 4720 * 4721 * However, the above condition can ONLY happen if the current L1 is 4722 * being shared. If it happens when the L1 is unshared, it indicates 4723 * that other parts of the pmap are not doing their job WRT managing 4724 * the TLB. 4725 */ 4726 if (rv == 0 4727 #ifndef ARM_MMU_EXTENDED 4728 && pm->pm_l1->l1_domain_use_count == 1 4729 #endif 4730 && true) { 4731 #ifdef DEBUG 4732 extern int last_fault_code; 4733 #else 4734 int last_fault_code = ftype & VM_PROT_EXECUTE 4735 ? armreg_ifsr_read() 4736 : armreg_dfsr_read(); 4737 #endif 4738 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 4739 pm, va, ftype); 4740 printf("fixup: l2 %p, l2b %p, ptep %p, pte %#x\n", 4741 l2, l2b, ptep, opte); 4742 4743 #ifndef ARM_MMU_EXTENDED 4744 printf("fixup: pdep %p, pde %#x, fsr %#x\n", 4745 pdep, pde, last_fault_code); 4746 #else 4747 printf("fixup: pdep %p, pde %#x, ttbcr %#x\n", 4748 &pmap_l1_kva(pm)[l1slot], pmap_l1_kva(pm)[l1slot], 4749 armreg_ttbcr_read()); 4750 printf("fixup: fsr %#x cpm %p casid %#x contextidr %#x dacr %#x\n", 4751 last_fault_code, curcpu()->ci_pmap_cur, 4752 curcpu()->ci_pmap_asid_cur, 4753 armreg_contextidr_read(), armreg_dacr_read()); 4754 #ifdef _ARM_ARCH_7 4755 if (ftype & VM_PROT_WRITE) 4756 armreg_ats1cuw_write(va); 4757 else 4758 armreg_ats1cur_write(va); 4759 arm_isb(); 4760 printf("fixup: par %#x\n", armreg_par_read()); 4761 #endif 4762 #endif 4763 #ifdef DDB 4764 extern int kernel_debug; 4765 4766 if (kernel_debug & 2) { 4767 pmap_release_pmap_lock(pm); 4768 #ifdef UVMHIST 4769 KERNHIST_DUMP(maphist); 4770 #endif 4771 cpu_Debugger(); 4772 pmap_acquire_pmap_lock(pm); 4773 } 4774 #endif 4775 } 4776 #endif 4777 #endif 4778 4779 #ifndef ARM_MMU_EXTENDED 4780 /* Flush the TLB in the shared L1 case - see comment above */ 4781 pmap_tlb_flush_SE(pm, va, 4782 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4783 #endif 4784 4785 rv = 1; 4786 4787 out: 4788 pmap_release_pmap_lock(pm); 4789 4790 return rv; 4791 } 4792 4793 /* 4794 * Routine: pmap_procwr 4795 * 4796 * Function: 4797 * Synchronize caches corresponding to [addr, addr+len) in p. 4798 * 4799 */ 4800 void 4801 pmap_procwr(struct proc *p, vaddr_t va, int len) 4802 { 4803 #ifndef ARM_MMU_EXTENDED 4804 4805 /* We only need to do anything if it is the current process. */ 4806 if (p == curproc) 4807 cpu_icache_sync_range(va, len); 4808 #endif 4809 } 4810 4811 /* 4812 * Routine: pmap_unwire 4813 * Function: Clear the wired attribute for a map/virtual-address pair. 4814 * 4815 * In/out conditions: 4816 * The mapping must already exist in the pmap. 4817 */ 4818 void 4819 pmap_unwire(pmap_t pm, vaddr_t va) 4820 { 4821 struct l2_bucket *l2b; 4822 pt_entry_t *ptep, pte; 4823 struct vm_page *pg; 4824 paddr_t pa; 4825 4826 UVMHIST_FUNC(__func__); 4827 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx", (uintptr_t)pm, va, 0, 0); 4828 4829 pmap_acquire_pmap_lock(pm); 4830 4831 l2b = pmap_get_l2_bucket(pm, va); 4832 KDASSERT(l2b != NULL); 4833 4834 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4835 pte = *ptep; 4836 4837 /* Extract the physical address of the page */ 4838 pa = l2pte_pa(pte); 4839 4840 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4841 /* Update the wired bit in the pv entry for this page. */ 4842 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4843 4844 pmap_acquire_page_lock(md); 4845 (void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0); 4846 pmap_release_page_lock(md); 4847 } 4848 4849 pmap_release_pmap_lock(pm); 4850 } 4851 4852 #ifdef ARM_MMU_EXTENDED 4853 void 4854 pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) 4855 { 4856 UVMHIST_FUNC(__func__); 4857 struct cpu_info * const ci = curcpu(); 4858 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); 4859 4860 UVMHIST_CALLARGS(maphist, "pm %#jx (pm->pm_l1_pa %08jx asid %ju)", 4861 (uintptr_t)pm, pm->pm_l1_pa, pai->pai_asid, 0); 4862 4863 /* 4864 * Assume that TTBR1 has only global mappings and TTBR0 only 4865 * has non-global mappings. To prevent speculation from doing 4866 * evil things we disable translation table walks using TTBR0 4867 * before setting the CONTEXTIDR (ASID) or new TTBR0 value. 4868 * Once both are set, table walks are reenabled. 4869 */ 4870 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4871 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4872 arm_isb(); 4873 4874 pmap_tlb_asid_acquire(pm, l); 4875 4876 cpu_setttb(pm->pm_l1_pa, pai->pai_asid); 4877 /* 4878 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 4879 * have been updated. 4880 */ 4881 arm_isb(); 4882 4883 if (pm != pmap_kernel()) { 4884 armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); 4885 } 4886 cpu_cpwait(); 4887 4888 KASSERTMSG(ci->ci_pmap_asid_cur == pai->pai_asid, "%u vs %u", 4889 ci->ci_pmap_asid_cur, pai->pai_asid); 4890 ci->ci_pmap_cur = pm; 4891 } 4892 4893 void 4894 pmap_md_pdetab_deactivate(pmap_t pm) 4895 { 4896 4897 UVMHIST_FUNC(__func__); 4898 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 4899 4900 kpreempt_disable(); 4901 struct cpu_info * const ci = curcpu(); 4902 /* 4903 * Disable translation table walks from TTBR0 while no pmap has been 4904 * activated. 4905 */ 4906 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4907 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4908 arm_isb(); 4909 pmap_tlb_asid_deactivate(pm); 4910 cpu_setttb(pmap_kernel()->pm_l1_pa, KERNEL_PID); 4911 arm_isb(); 4912 4913 ci->ci_pmap_cur = pmap_kernel(); 4914 KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", 4915 ci->ci_pmap_asid_cur); 4916 kpreempt_enable(); 4917 } 4918 #endif 4919 4920 void 4921 pmap_activate(struct lwp *l) 4922 { 4923 extern int block_userspace_access; 4924 pmap_t npm = l->l_proc->p_vmspace->vm_map.pmap; 4925 4926 UVMHIST_FUNC(__func__); 4927 UVMHIST_CALLARGS(maphist, "l=%#jx pm=%#jx", (uintptr_t)l, 4928 (uintptr_t)npm, 0, 0); 4929 4930 struct cpu_info * const ci = curcpu(); 4931 4932 /* 4933 * If activating a non-current lwp or the current lwp is 4934 * already active, just return. 4935 */ 4936 if (false 4937 || l != curlwp 4938 #ifdef ARM_MMU_EXTENDED 4939 || (ci->ci_pmap_cur == npm && 4940 (npm == pmap_kernel() 4941 /* || PMAP_PAI_ASIDVALID_P(pai, cpu_tlb_info(ci)) */)) 4942 #else 4943 || npm->pm_activated == true 4944 #endif 4945 || false) { 4946 UVMHIST_LOG(maphist, " <-- (same pmap)", (uintptr_t)curlwp, 4947 (uintptr_t)l, 0, 0); 4948 return; 4949 } 4950 4951 #ifndef ARM_MMU_EXTENDED 4952 const uint32_t ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4953 | (DOMAIN_CLIENT << (pmap_domain(npm) * 2)); 4954 4955 /* 4956 * If TTB and DACR are unchanged, short-circuit all the 4957 * TLB/cache management stuff. 4958 */ 4959 pmap_t opm = ci->ci_lastlwp 4960 ? ci->ci_lastlwp->l_proc->p_vmspace->vm_map.pmap 4961 : NULL; 4962 if (opm != NULL) { 4963 uint32_t odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4964 | (DOMAIN_CLIENT << (pmap_domain(opm) * 2)); 4965 4966 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4967 goto all_done; 4968 } 4969 #endif /* !ARM_MMU_EXTENDED */ 4970 4971 PMAPCOUNT(activations); 4972 block_userspace_access = 1; 4973 4974 #ifndef ARM_MMU_EXTENDED 4975 /* 4976 * If switching to a user vmspace which is different to the 4977 * most recent one, and the most recent one is potentially 4978 * live in the cache, we must write-back and invalidate the 4979 * entire cache. 4980 */ 4981 pmap_t rpm = ci->ci_pmap_lastuser; 4982 4983 /* 4984 * XXXSCW: There's a corner case here which can leave turds in the 4985 * cache as reported in kern/41058. They're probably left over during 4986 * tear-down and switching away from an exiting process. Until the root 4987 * cause is identified and fixed, zap the cache when switching pmaps. 4988 * This will result in a few unnecessary cache flushes, but that's 4989 * better than silently corrupting data. 4990 */ 4991 #if 0 4992 if (npm != pmap_kernel() && rpm && npm != rpm && 4993 rpm->pm_cstate.cs_cache) { 4994 rpm->pm_cstate.cs_cache = 0; 4995 #ifdef PMAP_CACHE_VIVT 4996 cpu_idcache_wbinv_all(); 4997 #endif 4998 } 4999 #else 5000 if (rpm) { 5001 rpm->pm_cstate.cs_cache = 0; 5002 if (npm == pmap_kernel()) 5003 ci->ci_pmap_lastuser = NULL; 5004 #ifdef PMAP_CACHE_VIVT 5005 cpu_idcache_wbinv_all(); 5006 #endif 5007 } 5008 #endif 5009 5010 /* No interrupts while we frob the TTB/DACR */ 5011 uint32_t oldirqstate = disable_interrupts(IF32_bits); 5012 #endif /* !ARM_MMU_EXTENDED */ 5013 5014 #ifndef ARM_HAS_VBAR 5015 /* 5016 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 5017 * entry corresponding to 'vector_page' in the incoming L1 table 5018 * before switching to it otherwise subsequent interrupts/exceptions 5019 * (including domain faults!) will jump into hyperspace. 5020 */ 5021 if (npm->pm_pl1vec != NULL) { 5022 cpu_tlb_flushID_SE((u_int)vector_page); 5023 cpu_cpwait(); 5024 *npm->pm_pl1vec = npm->pm_l1vec; 5025 PTE_SYNC(npm->pm_pl1vec); 5026 } 5027 #endif 5028 5029 #ifdef ARM_MMU_EXTENDED 5030 pmap_md_pdetab_activate(npm, l); 5031 #else 5032 cpu_domains(ndacr); 5033 if (npm == pmap_kernel() || npm == rpm) { 5034 /* 5035 * Switching to a kernel thread, or back to the 5036 * same user vmspace as before... Simply update 5037 * the TTB (no TLB flush required) 5038 */ 5039 cpu_setttb(npm->pm_l1->l1_physaddr, false); 5040 cpu_cpwait(); 5041 } else { 5042 /* 5043 * Otherwise, update TTB and flush TLB 5044 */ 5045 cpu_context_switch(npm->pm_l1->l1_physaddr); 5046 if (rpm != NULL) 5047 rpm->pm_cstate.cs_tlb = 0; 5048 } 5049 5050 restore_interrupts(oldirqstate); 5051 #endif /* ARM_MMU_EXTENDED */ 5052 5053 block_userspace_access = 0; 5054 5055 #ifndef ARM_MMU_EXTENDED 5056 all_done: 5057 /* 5058 * The new pmap is resident. Make sure it's marked 5059 * as resident in the cache/TLB. 5060 */ 5061 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5062 if (npm != pmap_kernel()) 5063 ci->ci_pmap_lastuser = npm; 5064 5065 /* The old pmap is not longer active */ 5066 if (opm != npm) { 5067 if (opm != NULL) 5068 opm->pm_activated = false; 5069 5070 /* But the new one is */ 5071 npm->pm_activated = true; 5072 } 5073 ci->ci_pmap_cur = npm; 5074 #endif 5075 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5076 } 5077 5078 void 5079 pmap_deactivate(struct lwp *l) 5080 { 5081 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 5082 5083 UVMHIST_FUNC(__func__); 5084 UVMHIST_CALLARGS(maphist, "l=%#jx (pm=%#jx)", (uintptr_t)l, 5085 (uintptr_t)pm, 0, 0); 5086 5087 #ifdef ARM_MMU_EXTENDED 5088 pmap_md_pdetab_deactivate(pm); 5089 #else 5090 /* 5091 * If the process is exiting, make sure pmap_activate() does 5092 * a full MMU context-switch and cache flush, which we might 5093 * otherwise skip. See PR port-arm/38950. 5094 */ 5095 if (l->l_proc->p_sflag & PS_WEXIT) 5096 curcpu()->ci_lastlwp = NULL; 5097 5098 pm->pm_activated = false; 5099 #endif 5100 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5101 } 5102 5103 void 5104 pmap_update(pmap_t pm) 5105 { 5106 5107 UVMHIST_FUNC(__func__); 5108 UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5109 pm->pm_remove_all, 0, 0); 5110 5111 #ifndef ARM_MMU_EXTENDED 5112 if (pm->pm_remove_all) { 5113 /* 5114 * Finish up the pmap_remove_all() optimisation by flushing 5115 * the TLB. 5116 */ 5117 pmap_tlb_flushID(pm); 5118 pm->pm_remove_all = false; 5119 } 5120 5121 if (pmap_is_current(pm)) { 5122 /* 5123 * If we're dealing with a current userland pmap, move its L1 5124 * to the end of the LRU. 5125 */ 5126 if (pm != pmap_kernel()) 5127 pmap_use_l1(pm); 5128 5129 /* 5130 * We can assume we're done with frobbing the cache/tlb for 5131 * now. Make sure any future pmap ops don't skip cache/tlb 5132 * flushes. 5133 */ 5134 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5135 } 5136 #else 5137 5138 kpreempt_disable(); 5139 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 5140 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 5141 if (pending && pmap_tlb_shootdown_bystanders(pmap)) { 5142 PMAP_COUNT(shootdown_ipis); 5143 } 5144 #endif 5145 5146 /* 5147 * If pmap_remove_all was called, we deactivated ourselves and released 5148 * our ASID. Now we have to reactivate ourselves. 5149 */ 5150 if (__predict_false(pm->pm_remove_all)) { 5151 pm->pm_remove_all = false; 5152 5153 KASSERT(pm != pmap_kernel()); 5154 pmap_md_pdetab_activate(pm, curlwp); 5155 } 5156 5157 if (arm_has_mpext_p) 5158 armreg_bpiallis_write(0); 5159 else 5160 armreg_bpiall_write(0); 5161 5162 kpreempt_enable(); 5163 5164 KASSERTMSG(pm == pmap_kernel() 5165 || curcpu()->ci_pmap_cur != pm 5166 || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, 5167 "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, 5168 pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, 5169 curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur); 5170 #endif 5171 5172 PMAPCOUNT(updates); 5173 5174 /* 5175 * make sure TLB/cache operations have completed. 5176 */ 5177 cpu_cpwait(); 5178 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5179 } 5180 5181 bool 5182 pmap_remove_all(pmap_t pm) 5183 { 5184 5185 /* 5186 * The vmspace described by this pmap is about to be torn down. 5187 * Until pmap_update() is called, UVM will only make calls 5188 * to pmap_remove(). We can make life much simpler by flushing 5189 * the cache now, and deferring TLB invalidation to pmap_update(). 5190 */ 5191 #ifdef PMAP_CACHE_VIVT 5192 pmap_cache_wbinv_all(pm, PVF_EXEC); 5193 #endif 5194 #ifdef ARM_MMU_EXTENDED 5195 #ifdef MULTIPROCESSOR 5196 struct cpu_info * const ci = curcpu(); 5197 // This should be the last CPU with this pmap onproc 5198 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 5199 if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) 5200 #endif 5201 pmap_tlb_asid_deactivate(pm); 5202 #ifdef MULTIPROCESSOR 5203 KASSERT(kcpuset_iszero(pm->pm_onproc)); 5204 #endif 5205 5206 pmap_tlb_asid_release_all(pm); 5207 #endif 5208 pm->pm_remove_all = true; 5209 return false; 5210 } 5211 5212 /* 5213 * Retire the given physical map from service. 5214 * Should only be called if the map contains no valid mappings. 5215 */ 5216 void 5217 pmap_destroy(pmap_t pm) 5218 { 5219 UVMHIST_FUNC(__func__); 5220 UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5221 pm ? pm->pm_remove_all : 0, 0, 0); 5222 5223 if (pm == NULL) 5224 return; 5225 5226 if (pm->pm_remove_all) { 5227 #ifdef ARM_MMU_EXTENDED 5228 pmap_tlb_asid_release_all(pm); 5229 #else 5230 pmap_tlb_flushID(pm); 5231 #endif 5232 pm->pm_remove_all = false; 5233 } 5234 5235 /* 5236 * Drop reference count 5237 */ 5238 if (atomic_dec_uint_nv(&pm->pm_refs) > 0) { 5239 #ifndef ARM_MMU_EXTENDED 5240 if (pmap_is_current(pm)) { 5241 if (pm != pmap_kernel()) 5242 pmap_use_l1(pm); 5243 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5244 } 5245 #endif 5246 return; 5247 } 5248 5249 /* 5250 * reference count is zero, free pmap resources and then free pmap. 5251 */ 5252 5253 #ifndef ARM_HAS_VBAR 5254 if (vector_page < KERNEL_BASE) { 5255 KDASSERT(!pmap_is_current(pm)); 5256 5257 /* Remove the vector page mapping */ 5258 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 5259 pmap_update(pm); 5260 } 5261 #endif 5262 5263 pmap_free_l1(pm); 5264 5265 #ifdef ARM_MMU_EXTENDED 5266 #ifdef MULTIPROCESSOR 5267 kcpuset_destroy(pm->pm_active); 5268 kcpuset_destroy(pm->pm_onproc); 5269 #endif 5270 #else 5271 struct cpu_info * const ci = curcpu(); 5272 if (ci->ci_pmap_lastuser == pm) 5273 ci->ci_pmap_lastuser = NULL; 5274 #endif 5275 5276 mutex_destroy(&pm->pm_lock); 5277 pool_cache_put(&pmap_cache, pm); 5278 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5279 } 5280 5281 5282 /* 5283 * void pmap_reference(pmap_t pm) 5284 * 5285 * Add a reference to the specified pmap. 5286 */ 5287 void 5288 pmap_reference(pmap_t pm) 5289 { 5290 5291 if (pm == NULL) 5292 return; 5293 5294 #ifndef ARM_MMU_EXTENDED 5295 pmap_use_l1(pm); 5296 #endif 5297 5298 atomic_inc_uint(&pm->pm_refs); 5299 } 5300 5301 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 5302 5303 static struct evcnt pmap_prefer_nochange_ev = 5304 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 5305 static struct evcnt pmap_prefer_change_ev = 5306 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 5307 5308 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 5309 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 5310 5311 void 5312 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 5313 { 5314 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 5315 vaddr_t va = *vap; 5316 vaddr_t diff = (hint - va) & mask; 5317 if (diff == 0) { 5318 pmap_prefer_nochange_ev.ev_count++; 5319 } else { 5320 pmap_prefer_change_ev.ev_count++; 5321 if (__predict_false(td)) 5322 va -= mask + 1; 5323 *vap = va + diff; 5324 } 5325 } 5326 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */ 5327 5328 /* 5329 * pmap_zero_page() 5330 * 5331 * Zero a given physical page by mapping it at a page hook point. 5332 * In doing the zero page op, the page we zero is mapped cachable, as with 5333 * StrongARM accesses to non-cached pages are non-burst making writing 5334 * _any_ bulk data very slow. 5335 */ 5336 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5337 void 5338 pmap_zero_page_generic(paddr_t pa) 5339 { 5340 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5341 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5342 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5343 #endif 5344 #if defined(PMAP_CACHE_VIPT) 5345 /* Choose the last page color it had, if any */ 5346 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5347 #else 5348 const vsize_t va_offset = 0; 5349 #endif 5350 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5351 /* 5352 * Is this page mapped at its natural color? 5353 * If we have all of memory mapped, then just convert PA to VA. 5354 */ 5355 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5356 || va_offset == (pa & arm_cache_prefer_mask); 5357 const vaddr_t vdstp = okcolor 5358 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5359 : cpu_cdstp(va_offset); 5360 #else 5361 const bool okcolor = false; 5362 const vaddr_t vdstp = cpu_cdstp(va_offset); 5363 #endif 5364 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5365 5366 5367 #ifdef DEBUG 5368 if (!SLIST_EMPTY(&md->pvh_list)) 5369 panic("pmap_zero_page: page has mappings"); 5370 #endif 5371 5372 KDASSERT((pa & PGOFSET) == 0); 5373 5374 if (!okcolor) { 5375 /* 5376 * Hook in the page, zero it, and purge the cache for that 5377 * zeroed page. Invalidate the TLB as needed. 5378 */ 5379 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 5380 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); 5381 l2pte_set(ptep, npte, 0); 5382 PTE_SYNC(ptep); 5383 cpu_tlb_flushD_SE(vdstp); 5384 cpu_cpwait(); 5385 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) \ 5386 && !defined(ARM_MMU_EXTENDED) 5387 /* 5388 * If we are direct-mapped and our color isn't ok, then before 5389 * we bzero the page invalidate its contents from the cache and 5390 * reset the color to its natural color. 5391 */ 5392 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5393 md->pvh_attrs &= ~arm_cache_prefer_mask; 5394 md->pvh_attrs |= (pa & arm_cache_prefer_mask); 5395 #endif 5396 } 5397 bzero_page(vdstp); 5398 if (!okcolor) { 5399 /* 5400 * Unmap the page. 5401 */ 5402 l2pte_reset(ptep); 5403 PTE_SYNC(ptep); 5404 cpu_tlb_flushD_SE(vdstp); 5405 #ifdef PMAP_CACHE_VIVT 5406 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5407 #endif 5408 } 5409 #ifdef PMAP_CACHE_VIPT 5410 /* 5411 * This page is now cache resident so it now has a page color. 5412 * Any contents have been obliterated so clear the EXEC flag. 5413 */ 5414 #ifndef ARM_MMU_EXTENDED 5415 if (!pmap_is_page_colored_p(md)) { 5416 PMAPCOUNT(vac_color_new); 5417 md->pvh_attrs |= PVF_COLORED; 5418 } 5419 md->pvh_attrs |= PVF_DIRTY; 5420 #endif 5421 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5422 md->pvh_attrs &= ~PVF_EXEC; 5423 PMAPCOUNT(exec_discarded_zero); 5424 } 5425 #endif 5426 } 5427 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5428 5429 #if ARM_MMU_XSCALE == 1 5430 void 5431 pmap_zero_page_xscale(paddr_t pa) 5432 { 5433 #ifdef DEBUG 5434 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5435 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5436 5437 if (!SLIST_EMPTY(&md->pvh_list)) 5438 panic("pmap_zero_page: page has mappings"); 5439 #endif 5440 5441 KDASSERT((pa & PGOFSET) == 0); 5442 5443 /* 5444 * Hook in the page, zero it, and purge the cache for that 5445 * zeroed page. Invalidate the TLB as needed. 5446 */ 5447 5448 pt_entry_t npte = L2_S_PROTO | pa | 5449 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 5450 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5451 l2pte_set(cdst_pte, npte, 0); 5452 PTE_SYNC(cdst_pte); 5453 cpu_tlb_flushD_SE(cdstp); 5454 cpu_cpwait(); 5455 bzero_page(cdstp); 5456 xscale_cache_clean_minidata(); 5457 l2pte_reset(cdst_pte); 5458 PTE_SYNC(cdst_pte); 5459 } 5460 #endif /* ARM_MMU_XSCALE == 1 */ 5461 5462 /* pmap_pageidlezero() 5463 * 5464 * The same as above, except that we assume that the page is not 5465 * mapped. This means we never have to flush the cache first. Called 5466 * from the idle loop. 5467 */ 5468 bool 5469 pmap_pageidlezero(paddr_t pa) 5470 { 5471 bool rv = true; 5472 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5473 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 5474 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5475 #endif 5476 #ifdef PMAP_CACHE_VIPT 5477 /* Choose the last page color it had, if any */ 5478 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5479 #else 5480 const vsize_t va_offset = 0; 5481 #endif 5482 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 5483 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5484 || va_offset == (pa & arm_cache_prefer_mask); 5485 const vaddr_t vdstp = okcolor 5486 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5487 : cpu_cdstp(va_offset); 5488 #else 5489 const bool okcolor = false; 5490 const vaddr_t vdstp = cpu_cdstp(va_offset); 5491 #endif 5492 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5493 5494 5495 #ifdef DEBUG 5496 if (!SLIST_EMPTY(&md->pvh_list)) 5497 panic("pmap_pageidlezero: page has mappings"); 5498 #endif 5499 5500 KDASSERT((pa & PGOFSET) == 0); 5501 5502 if (!okcolor) { 5503 /* 5504 * Hook in the page, zero it, and purge the cache for that 5505 * zeroed page. Invalidate the TLB as needed. 5506 */ 5507 const pt_entry_t npte = L2_S_PROTO | pa | 5508 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5509 l2pte_set(ptep, npte, 0); 5510 PTE_SYNC(ptep); 5511 cpu_tlb_flushD_SE(vdstp); 5512 cpu_cpwait(); 5513 } 5514 5515 uint64_t *ptr = (uint64_t *)vdstp; 5516 for (size_t i = 0; i < PAGE_SIZE / sizeof(*ptr); i++) { 5517 if (sched_curcpu_runnable_p() != 0) { 5518 /* 5519 * A process has become ready. Abort now, 5520 * so we don't keep it waiting while we 5521 * do slow memory access to finish this 5522 * page. 5523 */ 5524 rv = false; 5525 break; 5526 } 5527 *ptr++ = 0; 5528 } 5529 5530 #ifdef PMAP_CACHE_VIVT 5531 if (rv) 5532 /* 5533 * if we aborted we'll rezero this page again later so don't 5534 * purge it unless we finished it 5535 */ 5536 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5537 #elif defined(PMAP_CACHE_VIPT) 5538 /* 5539 * This page is now cache resident so it now has a page color. 5540 * Any contents have been obliterated so clear the EXEC flag. 5541 */ 5542 #ifndef ARM_MMU_EXTENDED 5543 if (!pmap_is_page_colored_p(md)) { 5544 PMAPCOUNT(vac_color_new); 5545 md->pvh_attrs |= PVF_COLORED; 5546 } 5547 #endif 5548 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5549 md->pvh_attrs &= ~PVF_EXEC; 5550 PMAPCOUNT(exec_discarded_zero); 5551 } 5552 #endif 5553 /* 5554 * Unmap the page. 5555 */ 5556 if (!okcolor) { 5557 l2pte_reset(ptep); 5558 PTE_SYNC(ptep); 5559 cpu_tlb_flushD_SE(vdstp); 5560 } 5561 5562 return rv; 5563 } 5564 5565 /* 5566 * pmap_copy_page() 5567 * 5568 * Copy one physical page into another, by mapping the pages into 5569 * hook points. The same comment regarding cachability as in 5570 * pmap_zero_page also applies here. 5571 */ 5572 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5573 void 5574 pmap_copy_page_generic(paddr_t src, paddr_t dst) 5575 { 5576 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 5577 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5578 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5579 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 5580 struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg); 5581 #endif 5582 #ifdef PMAP_CACHE_VIPT 5583 const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask; 5584 const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask; 5585 #else 5586 const vsize_t src_va_offset = 0; 5587 const vsize_t dst_va_offset = 0; 5588 #endif 5589 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5590 /* 5591 * Is this page mapped at its natural color? 5592 * If we have all of memory mapped, then just convert PA to VA. 5593 */ 5594 bool src_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5595 || src_va_offset == (src & arm_cache_prefer_mask); 5596 bool dst_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5597 || dst_va_offset == (dst & arm_cache_prefer_mask); 5598 const vaddr_t vsrcp = src_okcolor 5599 ? pmap_direct_mapped_phys(src, &src_okcolor, 5600 cpu_csrcp(src_va_offset)) 5601 : cpu_csrcp(src_va_offset); 5602 const vaddr_t vdstp = pmap_direct_mapped_phys(dst, &dst_okcolor, 5603 cpu_cdstp(dst_va_offset)); 5604 #else 5605 const bool src_okcolor = false; 5606 const bool dst_okcolor = false; 5607 const vaddr_t vsrcp = cpu_csrcp(src_va_offset); 5608 const vaddr_t vdstp = cpu_cdstp(dst_va_offset); 5609 #endif 5610 pt_entry_t * const src_ptep = cpu_csrc_pte(src_va_offset); 5611 pt_entry_t * const dst_ptep = cpu_cdst_pte(dst_va_offset); 5612 5613 #ifdef DEBUG 5614 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5615 panic("pmap_copy_page: dst page has mappings"); 5616 #endif 5617 5618 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5619 KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC)); 5620 #endif 5621 KDASSERT((src & PGOFSET) == 0); 5622 KDASSERT((dst & PGOFSET) == 0); 5623 5624 /* 5625 * Clean the source page. Hold the source page's lock for 5626 * the duration of the copy so that no other mappings can 5627 * be created while we have a potentially aliased mapping. 5628 */ 5629 #ifdef PMAP_CACHE_VIVT 5630 pmap_acquire_page_lock(src_md); 5631 (void) pmap_clean_page(src_md, true); 5632 pmap_release_page_lock(src_md); 5633 #endif 5634 5635 /* 5636 * Map the pages into the page hook points, copy them, and purge 5637 * the cache for the appropriate page. Invalidate the TLB 5638 * as required. 5639 */ 5640 if (!src_okcolor) { 5641 const pt_entry_t nsrc_pte = L2_S_PROTO 5642 | src 5643 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5644 | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 5645 #else // defined(PMAP_CACHE_VIVT) || defined(ARM_MMU_EXTENDED) 5646 | pte_l2_s_cache_mode 5647 #endif 5648 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 5649 l2pte_set(src_ptep, nsrc_pte, 0); 5650 PTE_SYNC(src_ptep); 5651 cpu_tlb_flushD_SE(vsrcp); 5652 cpu_cpwait(); 5653 } 5654 if (!dst_okcolor) { 5655 const pt_entry_t ndst_pte = L2_S_PROTO | dst | 5656 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5657 l2pte_set(dst_ptep, ndst_pte, 0); 5658 PTE_SYNC(dst_ptep); 5659 cpu_tlb_flushD_SE(vdstp); 5660 cpu_cpwait(); 5661 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 5662 /* 5663 * If we are direct-mapped and our color isn't ok, then before 5664 * we bcopy to the new page invalidate its contents from the 5665 * cache and reset its color to its natural color. 5666 */ 5667 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5668 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5669 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5670 #endif 5671 } 5672 bcopy_page(vsrcp, vdstp); 5673 #ifdef PMAP_CACHE_VIVT 5674 cpu_dcache_inv_range(vsrcp, PAGE_SIZE); 5675 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5676 #endif 5677 /* 5678 * Unmap the pages. 5679 */ 5680 if (!src_okcolor) { 5681 l2pte_reset(src_ptep); 5682 PTE_SYNC(src_ptep); 5683 cpu_tlb_flushD_SE(vsrcp); 5684 cpu_cpwait(); 5685 } 5686 if (!dst_okcolor) { 5687 l2pte_reset(dst_ptep); 5688 PTE_SYNC(dst_ptep); 5689 cpu_tlb_flushD_SE(vdstp); 5690 cpu_cpwait(); 5691 } 5692 #ifdef PMAP_CACHE_VIPT 5693 /* 5694 * Now that the destination page is in the cache, mark it as colored. 5695 * If this was an exec page, discard it. 5696 */ 5697 pmap_acquire_page_lock(dst_md); 5698 #ifndef ARM_MMU_EXTENDED 5699 if (arm_pcache.cache_type == CACHE_TYPE_PIPT) { 5700 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5701 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5702 } 5703 if (!pmap_is_page_colored_p(dst_md)) { 5704 PMAPCOUNT(vac_color_new); 5705 dst_md->pvh_attrs |= PVF_COLORED; 5706 } 5707 dst_md->pvh_attrs |= PVF_DIRTY; 5708 #endif 5709 if (PV_IS_EXEC_P(dst_md->pvh_attrs)) { 5710 dst_md->pvh_attrs &= ~PVF_EXEC; 5711 PMAPCOUNT(exec_discarded_copy); 5712 } 5713 pmap_release_page_lock(dst_md); 5714 #endif 5715 } 5716 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5717 5718 #if ARM_MMU_XSCALE == 1 5719 void 5720 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 5721 { 5722 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 5723 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5724 #ifdef DEBUG 5725 struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)); 5726 5727 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5728 panic("pmap_copy_page: dst page has mappings"); 5729 #endif 5730 5731 KDASSERT((src & PGOFSET) == 0); 5732 KDASSERT((dst & PGOFSET) == 0); 5733 5734 /* 5735 * Clean the source page. Hold the source page's lock for 5736 * the duration of the copy so that no other mappings can 5737 * be created while we have a potentially aliased mapping. 5738 */ 5739 #ifdef PMAP_CACHE_VIVT 5740 pmap_acquire_page_lock(src_md); 5741 (void) pmap_clean_page(src_md, true); 5742 pmap_release_page_lock(src_md); 5743 #endif 5744 5745 /* 5746 * Map the pages into the page hook points, copy them, and purge 5747 * the cache for the appropriate page. Invalidate the TLB 5748 * as required. 5749 */ 5750 const pt_entry_t nsrc_pte = L2_S_PROTO | src 5751 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 5752 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5753 l2pte_set(csrc_pte, nsrc_pte, 0); 5754 PTE_SYNC(csrc_pte); 5755 5756 const pt_entry_t ndst_pte = L2_S_PROTO | dst 5757 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) 5758 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5759 l2pte_set(cdst_pte, ndst_pte, 0); 5760 PTE_SYNC(cdst_pte); 5761 5762 cpu_tlb_flushD_SE(csrcp); 5763 cpu_tlb_flushD_SE(cdstp); 5764 cpu_cpwait(); 5765 bcopy_page(csrcp, cdstp); 5766 xscale_cache_clean_minidata(); 5767 l2pte_reset(csrc_pte); 5768 l2pte_reset(cdst_pte); 5769 PTE_SYNC(csrc_pte); 5770 PTE_SYNC(cdst_pte); 5771 } 5772 #endif /* ARM_MMU_XSCALE == 1 */ 5773 5774 /* 5775 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5776 * 5777 * Return the start and end addresses of the kernel's virtual space. 5778 * These values are setup in pmap_bootstrap and are updated as pages 5779 * are allocated. 5780 */ 5781 void 5782 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5783 { 5784 *start = virtual_avail; 5785 *end = virtual_end; 5786 } 5787 5788 /* 5789 * Helper function for pmap_grow_l2_bucket() 5790 */ 5791 static inline int 5792 pmap_grow_map(vaddr_t va, paddr_t *pap) 5793 { 5794 paddr_t pa; 5795 5796 KASSERT((va & PGOFSET) == 0); 5797 5798 if (uvm.page_init_done == false) { 5799 #ifdef PMAP_STEAL_MEMORY 5800 pv_addr_t pv; 5801 pmap_boot_pagealloc(PAGE_SIZE, 5802 #ifdef PMAP_CACHE_VIPT 5803 arm_cache_prefer_mask, 5804 va & arm_cache_prefer_mask, 5805 #else 5806 0, 0, 5807 #endif 5808 &pv); 5809 pa = pv.pv_pa; 5810 #else 5811 if (uvm_page_physget(&pa) == false) 5812 return 1; 5813 #endif /* PMAP_STEAL_MEMORY */ 5814 } else { 5815 struct vm_page *pg; 5816 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 5817 if (pg == NULL) 5818 return 1; 5819 pa = VM_PAGE_TO_PHYS(pg); 5820 /* 5821 * This new page must not have any mappings. 5822 */ 5823 struct vm_page_md *md __diagused = VM_PAGE_TO_MD(pg); 5824 KASSERT(SLIST_EMPTY(&md->pvh_list)); 5825 } 5826 5827 /* 5828 * Enter it via pmap_kenter_pa and let that routine do the hard work. 5829 */ 5830 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 5831 PMAP_KMPAGE | PMAP_PTE); 5832 5833 if (pap) 5834 *pap = pa; 5835 5836 PMAPCOUNT(pt_mappings); 5837 5838 const pmap_t kpm __diagused = pmap_kernel(); 5839 struct l2_bucket * const l2b __diagused = pmap_get_l2_bucket(kpm, va); 5840 KASSERT(l2b != NULL); 5841 5842 pt_entry_t * const ptep __diagused = &l2b->l2b_kva[l2pte_index(va)]; 5843 const pt_entry_t pte __diagused = *ptep; 5844 KASSERT(l2pte_valid_p(pte)); 5845 KASSERT((pte & L2_S_CACHE_MASK) == pte_l2_s_cache_mode_pt); 5846 5847 memset((void *)va, 0, PAGE_SIZE); 5848 5849 return 0; 5850 } 5851 5852 /* 5853 * This is the same as pmap_alloc_l2_bucket(), except that it is only 5854 * used by pmap_growkernel(). 5855 */ 5856 static inline struct l2_bucket * 5857 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 5858 { 5859 const size_t l1slot = l1pte_index(va); 5860 struct l2_dtable *l2; 5861 vaddr_t nva; 5862 5863 CTASSERT((PAGE_SIZE % L2_TABLE_SIZE_REAL) == 0); 5864 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 5865 /* 5866 * No mapping at this address, as there is 5867 * no entry in the L1 table. 5868 * Need to allocate a new l2_dtable. 5869 */ 5870 nva = pmap_kernel_l2dtable_kva; 5871 if ((nva & PGOFSET) == 0) { 5872 /* 5873 * Need to allocate a backing page 5874 */ 5875 if (pmap_grow_map(nva, NULL)) 5876 return NULL; 5877 } 5878 5879 l2 = (struct l2_dtable *)nva; 5880 nva += sizeof(struct l2_dtable); 5881 5882 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 5883 /* 5884 * The new l2_dtable straddles a page boundary. 5885 * Map in another page to cover it. 5886 */ 5887 if (pmap_grow_map(nva & ~PGOFSET, NULL)) 5888 return NULL; 5889 } 5890 5891 pmap_kernel_l2dtable_kva = nva; 5892 5893 /* 5894 * Link it into the parent pmap 5895 */ 5896 pm->pm_l2[L2_IDX(l1slot)] = l2; 5897 } 5898 5899 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 5900 5901 /* 5902 * Fetch pointer to the L2 page table associated with the address. 5903 */ 5904 if (l2b->l2b_kva == NULL) { 5905 pt_entry_t *ptep; 5906 5907 /* 5908 * No L2 page table has been allocated. Chances are, this 5909 * is because we just allocated the l2_dtable, above. 5910 */ 5911 nva = pmap_kernel_l2ptp_kva; 5912 ptep = (pt_entry_t *)nva; 5913 if ((nva & PGOFSET) == 0) { 5914 /* 5915 * Need to allocate a backing page 5916 */ 5917 if (pmap_grow_map(nva, &pmap_kernel_l2ptp_phys)) 5918 return NULL; 5919 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 5920 } 5921 5922 l2->l2_occupancy++; 5923 l2b->l2b_kva = ptep; 5924 l2b->l2b_l1slot = l1slot; 5925 l2b->l2b_pa = pmap_kernel_l2ptp_phys; 5926 5927 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 5928 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 5929 } 5930 5931 return l2b; 5932 } 5933 5934 vaddr_t 5935 pmap_growkernel(vaddr_t maxkvaddr) 5936 { 5937 UVMHIST_FUNC(__func__); 5938 UVMHIST_CALLARGS(maphist, "growing kernel from %#jx to %#jx\n", 5939 pmap_curmaxkvaddr, maxkvaddr, 0, 0); 5940 5941 pmap_t kpm = pmap_kernel(); 5942 #ifndef ARM_MMU_EXTENDED 5943 struct l1_ttable *l1; 5944 #endif 5945 int s; 5946 5947 if (maxkvaddr <= pmap_curmaxkvaddr) 5948 goto out; /* we are OK */ 5949 5950 KDASSERT(maxkvaddr <= virtual_end); 5951 5952 /* 5953 * whoops! we need to add kernel PTPs 5954 */ 5955 5956 s = splvm(); /* to be safe */ 5957 mutex_enter(&kpm_lock); 5958 5959 /* Map 1MB at a time */ 5960 size_t l1slot = l1pte_index(pmap_curmaxkvaddr); 5961 #ifdef ARM_MMU_EXTENDED 5962 pd_entry_t * const spdep = &kpm->pm_l1[l1slot]; 5963 pd_entry_t *pdep = spdep; 5964 #endif 5965 for (;pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE, 5966 #ifdef ARM_MMU_EXTENDED 5967 pdep++, 5968 #endif 5969 l1slot++) { 5970 struct l2_bucket *l2b = 5971 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 5972 KASSERT(l2b != NULL); 5973 5974 const pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 5975 | L1_C_DOM(PMAP_DOMAIN_KERNEL); 5976 #ifdef ARM_MMU_EXTENDED 5977 KASSERT(*pdep == 0); 5978 l1pte_setone(pdep, npde); 5979 #else 5980 /* Distribute new L1 entry to all other L1s */ 5981 SLIST_FOREACH(l1, &l1_list, l1_link) { 5982 pd_entry_t * const pdep = &l1->l1_kva[l1slot]; 5983 l1pte_setone(pdep, npde); 5984 PDE_SYNC(pdep); 5985 } 5986 #endif 5987 } 5988 #ifdef ARM_MMU_EXTENDED 5989 PDE_SYNC_RANGE(spdep, pdep - spdep); 5990 #endif 5991 5992 #ifdef PMAP_CACHE_VIVT 5993 /* 5994 * flush out the cache, expensive but growkernel will happen so 5995 * rarely 5996 */ 5997 cpu_dcache_wbinv_all(); 5998 cpu_tlb_flushD(); 5999 cpu_cpwait(); 6000 #endif 6001 6002 mutex_exit(&kpm_lock); 6003 splx(s); 6004 6005 out: 6006 return pmap_curmaxkvaddr; 6007 } 6008 6009 /************************ Utility routines ****************************/ 6010 6011 #ifndef ARM_HAS_VBAR 6012 /* 6013 * vector_page_setprot: 6014 * 6015 * Manipulate the protection of the vector page. 6016 */ 6017 void 6018 vector_page_setprot(int prot) 6019 { 6020 struct l2_bucket *l2b; 6021 pt_entry_t *ptep; 6022 6023 #if defined(CPU_ARMV7) || defined(CPU_ARM11) 6024 /* 6025 * If we are using VBAR to use the vectors in the kernel, then it's 6026 * already mapped in the kernel text so no need to anything here. 6027 */ 6028 if (vector_page != ARM_VECTORS_LOW && vector_page != ARM_VECTORS_HIGH) { 6029 KASSERT((armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0); 6030 return; 6031 } 6032 #endif 6033 6034 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 6035 KASSERT(l2b != NULL); 6036 6037 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 6038 6039 const pt_entry_t opte = *ptep; 6040 #ifdef ARM_MMU_EXTENDED 6041 const pt_entry_t npte = (opte & ~(L2_S_PROT_MASK|L2_XS_XN)) 6042 | L2_S_PROT(PTE_KERNEL, prot); 6043 #else 6044 const pt_entry_t npte = (opte & ~L2_S_PROT_MASK) 6045 | L2_S_PROT(PTE_KERNEL, prot); 6046 #endif 6047 l2pte_set(ptep, npte, opte); 6048 PTE_SYNC(ptep); 6049 cpu_tlb_flushD_SE(vector_page); 6050 cpu_cpwait(); 6051 } 6052 #endif 6053 6054 /* 6055 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 6056 * Returns true if the mapping exists, else false. 6057 * 6058 * NOTE: This function is only used by a couple of arm-specific modules. 6059 * It is not safe to take any pmap locks here, since we could be right 6060 * in the middle of debugging the pmap anyway... 6061 * 6062 * It is possible for this routine to return false even though a valid 6063 * mapping does exist. This is because we don't lock, so the metadata 6064 * state may be inconsistent. 6065 * 6066 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 6067 * a "section" mapping. 6068 */ 6069 bool 6070 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 6071 { 6072 struct l2_dtable *l2; 6073 pd_entry_t *pdep, pde; 6074 pt_entry_t *ptep; 6075 u_short l1slot; 6076 6077 if (pm->pm_l1 == NULL) 6078 return false; 6079 6080 l1slot = l1pte_index(va); 6081 *pdp = pdep = pmap_l1_kva(pm) + l1slot; 6082 pde = *pdep; 6083 6084 if (l1pte_section_p(pde)) { 6085 *ptp = NULL; 6086 return true; 6087 } 6088 6089 l2 = pm->pm_l2[L2_IDX(l1slot)]; 6090 if (l2 == NULL || 6091 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 6092 return false; 6093 } 6094 6095 *ptp = &ptep[l2pte_index(va)]; 6096 return true; 6097 } 6098 6099 bool 6100 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 6101 { 6102 6103 if (pm->pm_l1 == NULL) 6104 return false; 6105 6106 *pdp = pmap_l1_kva(pm) + l1pte_index(va); 6107 6108 return true; 6109 } 6110 6111 /************************ Bootstrapping routines ****************************/ 6112 6113 #ifndef ARM_MMU_EXTENDED 6114 static void 6115 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 6116 { 6117 int i; 6118 6119 l1->l1_kva = l1pt; 6120 l1->l1_domain_use_count = 0; 6121 l1->l1_domain_first = 0; 6122 6123 for (i = 0; i < PMAP_DOMAINS; i++) 6124 l1->l1_domain_free[i] = i + 1; 6125 6126 /* 6127 * Copy the kernel's L1 entries to each new L1. 6128 */ 6129 if (pmap_initialized) 6130 memcpy(l1pt, pmap_l1_kva(pmap_kernel()), L1_TABLE_SIZE); 6131 6132 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 6133 &l1->l1_physaddr) == false) 6134 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 6135 6136 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 6137 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 6138 } 6139 #endif /* !ARM_MMU_EXTENDED */ 6140 6141 /* 6142 * pmap_bootstrap() is called from the board-specific initarm() routine 6143 * once the kernel L1/L2 descriptors tables have been set up. 6144 * 6145 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 6146 * spread over a number of disparate files/functions. 6147 * 6148 * We are passed the following parameters 6149 * - vstart 6150 * 1MB-aligned start of managed kernel virtual memory. 6151 * - vend 6152 * 1MB-aligned end of managed kernel virtual memory. 6153 * 6154 * We use 'kernel_l1pt' to build the metadata (struct l1_ttable and 6155 * struct l2_dtable) necessary to track kernel mappings. 6156 */ 6157 #define PMAP_STATIC_L2_SIZE 16 6158 void 6159 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 6160 { 6161 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 6162 #ifndef ARM_MMU_EXTENDED 6163 static struct l1_ttable static_l1; 6164 struct l1_ttable *l1 = &static_l1; 6165 #endif 6166 struct l2_dtable *l2; 6167 struct l2_bucket *l2b; 6168 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 6169 pmap_t pm = pmap_kernel(); 6170 pt_entry_t *ptep; 6171 paddr_t pa; 6172 vsize_t size; 6173 int nptes, l2idx, l2next = 0; 6174 6175 #ifdef ARM_MMU_EXTENDED 6176 KASSERT(pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt); 6177 KASSERT(pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt); 6178 #endif 6179 6180 VPRINTF("kpm "); 6181 /* 6182 * Initialise the kernel pmap object 6183 */ 6184 curcpu()->ci_pmap_cur = pm; 6185 #ifdef ARM_MMU_EXTENDED 6186 pm->pm_l1 = l1pt; 6187 pm->pm_l1_pa = kernel_l1pt.pv_pa; 6188 VPRINTF("tlb0 "); 6189 pmap_tlb_info_init(&pmap_tlb0_info); 6190 #ifdef MULTIPROCESSOR 6191 VPRINTF("kcpusets "); 6192 pm->pm_onproc = kcpuset_running; 6193 pm->pm_active = kcpuset_running; 6194 #endif 6195 #else 6196 pm->pm_l1 = l1; 6197 #endif 6198 6199 VPRINTF("locks "); 6200 /* 6201 * pmap_kenter_pa() and pmap_kremove() may be called from interrupt 6202 * context, so its locks have to be at IPL_VM 6203 */ 6204 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_VM); 6205 mutex_init(&kpm_lock, MUTEX_DEFAULT, IPL_NONE); 6206 mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_VM); 6207 pm->pm_refs = 1; 6208 6209 VPRINTF("l1pt "); 6210 /* 6211 * Scan the L1 translation table created by initarm() and create 6212 * the required metadata for all valid mappings found in it. 6213 */ 6214 for (size_t l1slot = 0; 6215 l1slot < L1_TABLE_SIZE / sizeof(pd_entry_t); 6216 l1slot++) { 6217 pd_entry_t pde = l1pt[l1slot]; 6218 6219 /* 6220 * We're only interested in Coarse mappings. 6221 * pmap_extract() can deal with section mappings without 6222 * recourse to checking L2 metadata. 6223 */ 6224 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 6225 continue; 6226 6227 /* 6228 * Lookup the KVA of this L2 descriptor table 6229 */ 6230 pa = l1pte_pa(pde); 6231 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6232 if (ptep == NULL) { 6233 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 6234 (u_int)l1slot << L1_S_SHIFT, pa); 6235 } 6236 6237 /* 6238 * Fetch the associated L2 metadata structure. 6239 * Allocate a new one if necessary. 6240 */ 6241 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 6242 if (l2next == PMAP_STATIC_L2_SIZE) 6243 panic("pmap_bootstrap: out of static L2s"); 6244 pm->pm_l2[L2_IDX(l1slot)] = l2 = &static_l2[l2next++]; 6245 } 6246 6247 /* 6248 * One more L1 slot tracked... 6249 */ 6250 l2->l2_occupancy++; 6251 6252 /* 6253 * Fill in the details of the L2 descriptor in the 6254 * appropriate bucket. 6255 */ 6256 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 6257 l2b->l2b_kva = ptep; 6258 l2b->l2b_pa = pa; 6259 l2b->l2b_l1slot = l1slot; 6260 6261 /* 6262 * Establish an initial occupancy count for this descriptor 6263 */ 6264 for (l2idx = 0; 6265 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 6266 l2idx++) { 6267 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 6268 l2b->l2b_occupancy++; 6269 } 6270 } 6271 6272 /* 6273 * Make sure the descriptor itself has the correct cache mode. 6274 * If not, fix it, but whine about the problem. Port-meisters 6275 * should consider this a clue to fix up their initarm() 6276 * function. :) 6277 */ 6278 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep, 1)) { 6279 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6280 "L2 pte @ %p\n", ptep); 6281 } 6282 } 6283 6284 VPRINTF("cache(l1pt) "); 6285 /* 6286 * Ensure the primary (kernel) L1 has the correct cache mode for 6287 * a page table. Bitch if it is not correctly set. 6288 */ 6289 if (pmap_set_pt_cache_mode(l1pt, kernel_l1pt.pv_va, 6290 L1_TABLE_SIZE / L2_S_SIZE)) { 6291 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6292 "primary L1 @ 0x%lx\n", kernel_l1pt.pv_va); 6293 } 6294 6295 #ifdef PMAP_CACHE_VIVT 6296 cpu_dcache_wbinv_all(); 6297 cpu_tlb_flushID(); 6298 cpu_cpwait(); 6299 #endif 6300 6301 /* 6302 * now we allocate the "special" VAs which are used for tmp mappings 6303 * by the pmap (and other modules). we allocate the VAs by advancing 6304 * virtual_avail (note that there are no pages mapped at these VAs). 6305 * 6306 * Managed KVM space start from wherever initarm() tells us. 6307 */ 6308 virtual_avail = vstart; 6309 virtual_end = vend; 6310 6311 VPRINTF("specials "); 6312 #ifdef PMAP_CACHE_VIPT 6313 /* 6314 * If we have a VIPT cache, we need one page/pte per possible alias 6315 * page so we won't violate cache aliasing rules. 6316 */ 6317 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 6318 nptes = (arm_cache_prefer_mask >> L2_S_SHIFT) + 1; 6319 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6320 if (arm_pcache.icache_type != CACHE_TYPE_PIPT 6321 && arm_pcache.icache_way_size > nptes * L2_S_SIZE) { 6322 nptes = arm_pcache.icache_way_size >> L2_S_SHIFT; 6323 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6324 } 6325 #else 6326 nptes = PAGE_SIZE / L2_S_SIZE; 6327 #endif 6328 #ifdef MULTIPROCESSOR 6329 cnptes = nptes; 6330 nptes *= arm_cpu_max; 6331 #endif 6332 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 6333 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte, nptes); 6334 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 6335 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte, nptes); 6336 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 6337 if (msgbufaddr == NULL) { 6338 pmap_alloc_specials(&virtual_avail, 6339 round_page(MSGBUFSIZE) / PAGE_SIZE, 6340 (void *)&msgbufaddr, NULL); 6341 } 6342 6343 /* 6344 * Allocate a range of kernel virtual address space to be used 6345 * for L2 descriptor tables and metadata allocation in 6346 * pmap_growkernel(). 6347 */ 6348 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 6349 pmap_alloc_specials(&virtual_avail, 6350 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 6351 &pmap_kernel_l2ptp_kva, NULL); 6352 6353 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 6354 pmap_alloc_specials(&virtual_avail, 6355 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 6356 &pmap_kernel_l2dtable_kva, NULL); 6357 6358 #ifndef ARM_MMU_EXTENDED 6359 /* 6360 * init the static-global locks and global pmap list. 6361 */ 6362 mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM); 6363 6364 /* 6365 * We can now initialise the first L1's metadata. 6366 */ 6367 SLIST_INIT(&l1_list); 6368 TAILQ_INIT(&l1_lru_list); 6369 pmap_init_l1(l1, l1pt); 6370 #endif /* ARM_MMU_EXTENDED */ 6371 6372 #ifndef ARM_HAS_VBAR 6373 /* Set up vector page L1 details, if necessary */ 6374 if (vector_page < KERNEL_BASE) { 6375 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 6376 l2b = pmap_get_l2_bucket(pm, vector_page); 6377 KDASSERT(l2b != NULL); 6378 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 6379 L1_C_DOM(pmap_domain(pm)); 6380 } else 6381 pm->pm_pl1vec = NULL; 6382 #endif 6383 6384 VPRINTF("pools "); 6385 /* 6386 * Initialize the pmap cache 6387 */ 6388 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 6389 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 6390 6391 /* 6392 * Initialize the pv pool. 6393 */ 6394 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 6395 &pmap_bootstrap_pv_allocator, IPL_NONE); 6396 6397 /* 6398 * Initialize the L2 dtable pool and cache. 6399 */ 6400 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 6401 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 6402 6403 /* 6404 * Initialise the L2 descriptor table pool and cache 6405 */ 6406 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 6407 L2_TABLE_SIZE_REAL, 0, 0, "l2ptppl", NULL, IPL_NONE, 6408 pmap_l2ptp_ctor, NULL, NULL); 6409 6410 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 6411 6412 cpu_dcache_wbinv_all(); 6413 } 6414 6415 static bool 6416 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va, size_t nptes) 6417 { 6418 #ifdef ARM_MMU_EXTENDED 6419 return false; 6420 #else 6421 if (pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 6422 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 6423 return false; 6424 6425 const vaddr_t eva = va + nptes * PAGE_SIZE; 6426 int rv = 0; 6427 6428 while (va < eva) { 6429 /* 6430 * Make sure the descriptor itself has the correct cache mode 6431 */ 6432 pd_entry_t * const pdep = &kl1[l1pte_index(va)]; 6433 pd_entry_t pde = *pdep; 6434 6435 if (l1pte_section_p(pde)) { 6436 KASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0); 6437 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 6438 *pdep = (pde & ~L1_S_CACHE_MASK) | 6439 pte_l1_s_cache_mode_pt; 6440 PDE_SYNC(pdep); 6441 cpu_dcache_wbinv_range((vaddr_t)pdep, 6442 sizeof(*pdep)); 6443 rv = 1; 6444 } 6445 return rv; 6446 } 6447 vaddr_t pa = l1pte_pa(pde); 6448 pt_entry_t *ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6449 if (ptep == NULL) 6450 panic("pmap_bootstrap: No PTP for va %#lx\n", va); 6451 6452 ptep += l2pte_index(va); 6453 const pt_entry_t opte = *ptep; 6454 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 6455 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 6456 | pte_l2_s_cache_mode_pt; 6457 l2pte_set(ptep, npte, opte); 6458 PTE_SYNC(ptep); 6459 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 6460 rv = 1; 6461 } 6462 va += PAGE_SIZE; 6463 } 6464 6465 return rv; 6466 #endif 6467 } 6468 6469 static void 6470 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 6471 { 6472 vaddr_t va = *availp; 6473 struct l2_bucket *l2b; 6474 6475 if (ptep) { 6476 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6477 if (l2b == NULL) 6478 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 6479 6480 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 6481 } 6482 6483 *vap = va; 6484 *availp = va + (PAGE_SIZE * pages); 6485 } 6486 6487 void 6488 pmap_init(void) 6489 { 6490 6491 /* 6492 * Set the available memory vars - These do not map to real memory 6493 * addresses and cannot as the physical memory is fragmented. 6494 * They are used by ps for %mem calculations. 6495 * One could argue whether this should be the entire memory or just 6496 * the memory that is useable in a user process. 6497 */ 6498 avail_start = ptoa(uvm_physseg_get_avail_start(uvm_physseg_get_first())); 6499 avail_end = ptoa(uvm_physseg_get_avail_end(uvm_physseg_get_last())); 6500 6501 /* 6502 * Now we need to free enough pv_entry structures to allow us to get 6503 * the kmem_map/kmem_object allocated and inited (done after this 6504 * function is finished). to do this we allocate one bootstrap page out 6505 * of kernel_map and use it to provide an initial pool of pv_entry 6506 * structures. we never free this page. 6507 */ 6508 pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 6509 6510 #ifdef ARM_MMU_EXTENDED 6511 /* 6512 * Initialise the L1 pool and cache. 6513 */ 6514 6515 pool_cache_bootstrap(&pmap_l1tt_cache, L1TT_SIZE, L1TT_SIZE, 6516 0, 0, "l1ttpl", &pmap_l1tt_allocator, IPL_NONE, pmap_l1tt_ctor, 6517 NULL, NULL); 6518 6519 int error __diagused = pmap_maxproc_set(maxproc); 6520 KASSERT(error == 0); 6521 6522 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 6523 #endif 6524 6525 pmap_initialized = true; 6526 } 6527 6528 static vaddr_t last_bootstrap_page = 0; 6529 static void *free_bootstrap_pages = NULL; 6530 6531 static void * 6532 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 6533 { 6534 extern void *pool_page_alloc(struct pool *, int); 6535 vaddr_t new_page; 6536 void *rv; 6537 6538 if (pmap_initialized) 6539 return pool_page_alloc(pp, flags); 6540 6541 if (free_bootstrap_pages) { 6542 rv = free_bootstrap_pages; 6543 free_bootstrap_pages = *((void **)rv); 6544 return rv; 6545 } 6546 6547 KASSERT(kernel_map != NULL); 6548 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 6549 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 6550 6551 KASSERT(new_page > last_bootstrap_page); 6552 last_bootstrap_page = new_page; 6553 return (void *)new_page; 6554 } 6555 6556 static void 6557 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 6558 { 6559 extern void pool_page_free(struct pool *, void *); 6560 6561 if ((vaddr_t)v <= last_bootstrap_page) { 6562 *((void **)v) = free_bootstrap_pages; 6563 free_bootstrap_pages = v; 6564 return; 6565 } 6566 6567 if (pmap_initialized) { 6568 pool_page_free(pp, v); 6569 return; 6570 } 6571 } 6572 6573 6574 #if defined(ARM_MMU_EXTENDED) 6575 static void * 6576 pmap_l1tt_alloc(struct pool *pp, int flags) 6577 { 6578 struct pglist plist; 6579 vaddr_t va; 6580 6581 const int waitok = flags & PR_WAITOK; 6582 6583 int error = uvm_pglistalloc(L1TT_SIZE, 0, -1, L1TT_SIZE, 0, &plist, 1, 6584 waitok); 6585 if (error) 6586 panic("Cannot allocate L1TT physical pages, %d", error); 6587 6588 struct vm_page *pg = TAILQ_FIRST(&plist); 6589 #if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS) 6590 6591 /* Allocate a L1 translation table VA */ 6592 va = uvm_km_alloc(kernel_map, L1TT_SIZE, L1TT_SIZE, UVM_KMF_VAONLY); 6593 if (va == 0) 6594 panic("Cannot allocate L1TT KVA"); 6595 6596 const vaddr_t eva = va + L1TT_SIZE; 6597 vaddr_t mva = va; 6598 while (pg && mva < eva) { 6599 paddr_t pa = VM_PAGE_TO_PHYS(pg); 6600 6601 pmap_kenter_pa(mva, pa, 6602 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6603 6604 mva += PAGE_SIZE; 6605 pg = TAILQ_NEXT(pg, pageq.queue); 6606 } 6607 KASSERTMSG(pg == NULL && mva == eva, "pg %p mva %" PRIxVADDR 6608 " eva %" PRIxVADDR, pg, mva, eva); 6609 #else 6610 bool ok; 6611 paddr_t pa = VM_PAGE_TO_PHYS(pg); 6612 va = pmap_direct_mapped_phys(pa, &ok, 0); 6613 KASSERT(ok); 6614 KASSERT(va >= KERNEL_BASE); 6615 #endif 6616 6617 return (void *)va; 6618 } 6619 6620 static void 6621 pmap_l1tt_free(struct pool *pp, void *v) 6622 { 6623 vaddr_t va = (vaddr_t)v; 6624 6625 #if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS) 6626 uvm_km_free(kernel_map, va, L1TT_SIZE, UVM_KMF_WIRED); 6627 #else 6628 #if defined(KERNEL_BASE_VOFFSET) 6629 paddr_t pa = va - KERNEL_BASE_VOFFSET; 6630 #else 6631 paddr_t pa = va - KERNEL_BASE + physical_start; 6632 #endif 6633 const paddr_t epa = pa + L1TT_SIZE; 6634 6635 for (; pa < epa; pa += PAGE_SIZE) { 6636 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 6637 uvm_pagefree(pg); 6638 } 6639 #endif 6640 } 6641 #endif 6642 6643 /* 6644 * pmap_postinit() 6645 * 6646 * This routine is called after the vm and kmem subsystems have been 6647 * initialised. This allows the pmap code to perform any initialisation 6648 * that can only be done once the memory allocation is in place. 6649 */ 6650 void 6651 pmap_postinit(void) 6652 { 6653 #ifndef ARM_MMU_EXTENDED 6654 extern paddr_t physical_start, physical_end; 6655 struct l1_ttable *l1; 6656 struct pglist plist; 6657 struct vm_page *m; 6658 pd_entry_t *pdep; 6659 vaddr_t va, eva; 6660 u_int loop, needed; 6661 int error; 6662 #endif 6663 6664 pool_cache_setlowat(&pmap_l2ptp_cache, (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 6665 pool_cache_setlowat(&pmap_l2dtable_cache, 6666 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 6667 6668 #ifndef ARM_MMU_EXTENDED 6669 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 6670 needed -= 1; 6671 6672 l1 = kmem_alloc(sizeof(*l1) * needed, KM_SLEEP); 6673 6674 for (loop = 0; loop < needed; loop++, l1++) { 6675 /* Allocate a L1 page table */ 6676 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 6677 if (va == 0) 6678 panic("Cannot allocate L1 KVM"); 6679 6680 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 6681 physical_end, L1_TABLE_SIZE, 0, &plist, 1, 1); 6682 if (error) 6683 panic("Cannot allocate L1 physical pages"); 6684 6685 m = TAILQ_FIRST(&plist); 6686 eva = va + L1_TABLE_SIZE; 6687 pdep = (pd_entry_t *)va; 6688 6689 while (m && va < eva) { 6690 paddr_t pa = VM_PAGE_TO_PHYS(m); 6691 6692 pmap_kenter_pa(va, pa, 6693 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6694 6695 va += PAGE_SIZE; 6696 m = TAILQ_NEXT(m, pageq.queue); 6697 } 6698 6699 #ifdef DIAGNOSTIC 6700 if (m) 6701 panic("pmap_alloc_l1pt: pglist not empty"); 6702 #endif /* DIAGNOSTIC */ 6703 6704 pmap_init_l1(l1, pdep); 6705 } 6706 6707 #ifdef DEBUG 6708 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 6709 needed); 6710 #endif 6711 #endif /* !ARM_MMU_EXTENDED */ 6712 } 6713 6714 /* 6715 * Note that the following routines are used by board-specific initialisation 6716 * code to configure the initial kernel page tables. 6717 * 6718 */ 6719 6720 /* 6721 * This list exists for the benefit of pmap_map_chunk(). It keeps track 6722 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 6723 * find them as necessary. 6724 * 6725 * Note that the data on this list MUST remain valid after initarm() returns, 6726 * as pmap_bootstrap() uses it to construct L2 table metadata. 6727 */ 6728 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 6729 6730 static vaddr_t 6731 kernel_pt_lookup(paddr_t pa) 6732 { 6733 pv_addr_t *pv; 6734 6735 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 6736 if (pv->pv_pa == (pa & ~PGOFSET)) 6737 return pv->pv_va | (pa & PGOFSET); 6738 } 6739 return 0; 6740 } 6741 6742 /* 6743 * pmap_map_section: 6744 * 6745 * Create a single section mapping. 6746 */ 6747 void 6748 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6749 { 6750 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6751 const size_t l1slot = l1pte_index(va); 6752 pd_entry_t fl; 6753 6754 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 6755 6756 switch (cache) { 6757 case PTE_NOCACHE: 6758 fl = pte_l1_s_nocache_mode; 6759 break; 6760 6761 case PTE_CACHE: 6762 fl = pte_l1_s_cache_mode; 6763 break; 6764 6765 case PTE_PAGETABLE: 6766 fl = pte_l1_s_cache_mode_pt; 6767 break; 6768 6769 case PTE_DEV: 6770 default: 6771 fl = 0; 6772 break; 6773 } 6774 6775 const pd_entry_t npde = L1_S_PROTO | pa | 6776 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6777 l1pte_setone(pdep + l1slot, npde); 6778 PDE_SYNC(pdep + l1slot); 6779 } 6780 6781 /* 6782 * pmap_map_entry: 6783 * 6784 * Create a single page mapping. 6785 */ 6786 void 6787 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6788 { 6789 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6790 const size_t l1slot = l1pte_index(va); 6791 pt_entry_t npte; 6792 pt_entry_t *ptep; 6793 6794 KASSERT(((va | pa) & PGOFSET) == 0); 6795 6796 switch (cache) { 6797 case PTE_NOCACHE: 6798 npte = pte_l2_s_nocache_mode; 6799 break; 6800 6801 case PTE_CACHE: 6802 npte = pte_l2_s_cache_mode; 6803 break; 6804 6805 case PTE_PAGETABLE: 6806 npte = pte_l2_s_cache_mode_pt; 6807 break; 6808 6809 default: 6810 npte = 0; 6811 break; 6812 } 6813 6814 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6815 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 6816 6817 ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6818 if (ptep == NULL) 6819 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 6820 6821 npte |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 6822 #ifdef ARM_MMU_EXTENDED 6823 if (prot & VM_PROT_EXECUTE) { 6824 npte &= ~L2_XS_XN; 6825 } 6826 #endif 6827 ptep += l2pte_index(va); 6828 l2pte_set(ptep, npte, 0); 6829 PTE_SYNC(ptep); 6830 } 6831 6832 /* 6833 * pmap_link_l2pt: 6834 * 6835 * Link the L2 page table specified by "l2pv" into the L1 6836 * page table at the slot for "va". 6837 */ 6838 void 6839 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 6840 { 6841 pd_entry_t * const pdep = (pd_entry_t *) l1pt + l1pte_index(va); 6842 6843 KASSERT((va & ((L1_S_SIZE * (PAGE_SIZE / L2_T_SIZE)) - 1)) == 0); 6844 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 6845 6846 const pd_entry_t npde = L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO 6847 | l2pv->pv_pa; 6848 6849 l1pte_set(pdep, npde); 6850 PDE_SYNC_RANGE(pdep, PAGE_SIZE / L2_T_SIZE); 6851 6852 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 6853 } 6854 6855 /* 6856 * pmap_map_chunk: 6857 * 6858 * Map a chunk of memory using the most efficient mappings 6859 * possible (section, large page, small page) into the 6860 * provided L1 and L2 tables at the specified virtual address. 6861 */ 6862 vsize_t 6863 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 6864 int prot, int cache) 6865 { 6866 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6867 pt_entry_t f1, f2s, f2l; 6868 vsize_t resid; 6869 6870 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 6871 6872 if (l1pt == 0) 6873 panic("pmap_map_chunk: no L1 table provided"); 6874 6875 // VPRINTF("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 6876 // "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 6877 6878 switch (cache) { 6879 case PTE_NOCACHE: 6880 f1 = pte_l1_s_nocache_mode; 6881 f2l = pte_l2_l_nocache_mode; 6882 f2s = pte_l2_s_nocache_mode; 6883 break; 6884 6885 case PTE_CACHE: 6886 f1 = pte_l1_s_cache_mode; 6887 f2l = pte_l2_l_cache_mode; 6888 f2s = pte_l2_s_cache_mode; 6889 break; 6890 6891 case PTE_PAGETABLE: 6892 f1 = pte_l1_s_cache_mode_pt; 6893 f2l = pte_l2_l_cache_mode_pt; 6894 f2s = pte_l2_s_cache_mode_pt; 6895 break; 6896 6897 case PTE_DEV: 6898 default: 6899 f1 = 0; 6900 f2l = 0; 6901 f2s = 0; 6902 break; 6903 } 6904 6905 size = resid; 6906 6907 while (resid > 0) { 6908 const size_t l1slot = l1pte_index(va); 6909 #ifdef ARM_MMU_EXTENDED 6910 /* See if we can use a supersection mapping. */ 6911 if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) { 6912 /* Supersection are always domain 0 */ 6913 const pd_entry_t npde = L1_SS_PROTO | pa 6914 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6915 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6916 | L1_S_PROT(PTE_KERNEL, prot) | f1; 6917 VPRINTF("sS"); 6918 l1pte_set(&pdep[l1slot], npde); 6919 PDE_SYNC_RANGE(&pdep[l1slot], L1_SS_SIZE / L1_S_SIZE); 6920 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6921 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6922 va += L1_SS_SIZE; 6923 pa += L1_SS_SIZE; 6924 resid -= L1_SS_SIZE; 6925 continue; 6926 } 6927 #endif 6928 /* See if we can use a section mapping. */ 6929 if (L1_S_MAPPABLE_P(va, pa, resid)) { 6930 const pd_entry_t npde = L1_S_PROTO | pa 6931 #ifdef ARM_MMU_EXTENDED 6932 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6933 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6934 #endif 6935 | L1_S_PROT(PTE_KERNEL, prot) | f1 6936 | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6937 VPRINTF("S"); 6938 l1pte_set(&pdep[l1slot], npde); 6939 PDE_SYNC(&pdep[l1slot]); 6940 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6941 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6942 va += L1_S_SIZE; 6943 pa += L1_S_SIZE; 6944 resid -= L1_S_SIZE; 6945 continue; 6946 } 6947 6948 /* 6949 * Ok, we're going to use an L2 table. Make sure 6950 * one is actually in the corresponding L1 slot 6951 * for the current VA. 6952 */ 6953 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6954 panic("%s: no L2 table for VA %#lx", __func__, va); 6955 6956 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6957 if (ptep == NULL) 6958 panic("%s: can't find L2 table for VA %#lx", __func__, 6959 va); 6960 6961 ptep += l2pte_index(va); 6962 6963 /* See if we can use a L2 large page mapping. */ 6964 if (L2_L_MAPPABLE_P(va, pa, resid)) { 6965 const pt_entry_t npte = L2_L_PROTO | pa 6966 #ifdef ARM_MMU_EXTENDED 6967 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_L_XN) 6968 | (va & 0x80000000 ? 0 : L2_XS_nG) 6969 #endif 6970 | L2_L_PROT(PTE_KERNEL, prot) | f2l; 6971 VPRINTF("L"); 6972 l2pte_set(ptep, npte, 0); 6973 PTE_SYNC_RANGE(ptep, L2_L_SIZE / L2_S_SIZE); 6974 va += L2_L_SIZE; 6975 pa += L2_L_SIZE; 6976 resid -= L2_L_SIZE; 6977 continue; 6978 } 6979 6980 VPRINTF("P"); 6981 /* Use a small page mapping. */ 6982 pt_entry_t npte = L2_S_PROTO | pa 6983 #ifdef ARM_MMU_EXTENDED 6984 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_XN) 6985 | (va & 0x80000000 ? 0 : L2_XS_nG) 6986 #endif 6987 | L2_S_PROT(PTE_KERNEL, prot) | f2s; 6988 #ifdef ARM_MMU_EXTENDED 6989 npte &= ((prot & VM_PROT_EXECUTE) ? ~L2_XS_XN : ~0); 6990 #endif 6991 l2pte_set(ptep, npte, 0); 6992 PTE_SYNC(ptep); 6993 va += PAGE_SIZE; 6994 pa += PAGE_SIZE; 6995 resid -= PAGE_SIZE; 6996 } 6997 VPRINTF("\n"); 6998 return size; 6999 } 7000 7001 /* 7002 * pmap_unmap_chunk: 7003 * 7004 * Unmap a chunk of memory that was previously pmap_map_chunk 7005 */ 7006 void 7007 pmap_unmap_chunk(vaddr_t l1pt, vaddr_t va, vsize_t size) 7008 { 7009 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 7010 const size_t l1slot = l1pte_index(va); 7011 7012 KASSERT(size == L1_SS_SIZE || size == L1_S_SIZE); 7013 7014 l1pte_set(&pdep[l1slot], 0); 7015 PDE_SYNC_RANGE(&pdep[l1slot], size / L1_S_SIZE); 7016 7017 pmap_tlb_flush_SE(pmap_kernel(), va, PVF_REF); 7018 } 7019 7020 7021 7022 /********************** Static device map routines ***************************/ 7023 7024 static const struct pmap_devmap *pmap_devmap_table; 7025 7026 /* 7027 * Register the devmap table. This is provided in case early console 7028 * initialization needs to register mappings created by bootstrap code 7029 * before pmap_devmap_bootstrap() is called. 7030 */ 7031 void 7032 pmap_devmap_register(const struct pmap_devmap *table) 7033 { 7034 7035 pmap_devmap_table = table; 7036 } 7037 7038 /* 7039 * Map all of the static regions in the devmap table, and remember 7040 * the devmap table so other parts of the kernel can look up entries 7041 * later. 7042 */ 7043 void 7044 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 7045 { 7046 int i; 7047 7048 pmap_devmap_table = table; 7049 7050 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7051 const struct pmap_devmap *pdp = &pmap_devmap_table[i]; 7052 7053 KASSERTMSG(VADDR_MAX - pdp->pd_va >= pdp->pd_size - 1, "va %" PRIxVADDR 7054 " sz %" PRIxPSIZE, pdp->pd_va, pdp->pd_size); 7055 KASSERTMSG(PADDR_MAX - pdp->pd_pa >= pdp->pd_size - 1, "pa %" PRIxPADDR 7056 " sz %" PRIxPSIZE, pdp->pd_pa, pdp->pd_size); 7057 VPRINTF("devmap: %08lx -> %08lx @ %08lx\n", pdp->pd_pa, 7058 pdp->pd_pa + pdp->pd_size - 1, pdp->pd_va); 7059 7060 pmap_map_chunk(l1pt, pdp->pd_va, pdp->pd_pa, pdp->pd_size, 7061 pdp->pd_prot, pdp->pd_cache); 7062 } 7063 } 7064 7065 const struct pmap_devmap * 7066 pmap_devmap_find_pa(paddr_t pa, psize_t size) 7067 { 7068 uint64_t endpa; 7069 int i; 7070 7071 if (pmap_devmap_table == NULL) 7072 return NULL; 7073 7074 endpa = (uint64_t)pa + (uint64_t)(size - 1); 7075 7076 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7077 if (pa >= pmap_devmap_table[i].pd_pa && 7078 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 7079 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 7080 return &pmap_devmap_table[i]; 7081 } 7082 7083 return NULL; 7084 } 7085 7086 const struct pmap_devmap * 7087 pmap_devmap_find_va(vaddr_t va, vsize_t size) 7088 { 7089 int i; 7090 7091 if (pmap_devmap_table == NULL) 7092 return NULL; 7093 7094 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7095 if (va >= pmap_devmap_table[i].pd_va && 7096 va + size - 1 <= pmap_devmap_table[i].pd_va + 7097 pmap_devmap_table[i].pd_size - 1) 7098 return &pmap_devmap_table[i]; 7099 } 7100 7101 return NULL; 7102 } 7103 7104 /********************** PTE initialization routines **************************/ 7105 7106 /* 7107 * These routines are called when the CPU type is identified to set up 7108 * the PTE prototypes, cache modes, etc. 7109 * 7110 * The variables are always here, just in case modules need to reference 7111 * them (though, they shouldn't). 7112 */ 7113 7114 pt_entry_t pte_l1_s_nocache_mode; 7115 pt_entry_t pte_l1_s_cache_mode; 7116 pt_entry_t pte_l1_s_wc_mode; 7117 pt_entry_t pte_l1_s_cache_mode_pt; 7118 pt_entry_t pte_l1_s_cache_mask; 7119 7120 pt_entry_t pte_l2_l_nocache_mode; 7121 pt_entry_t pte_l2_l_cache_mode; 7122 pt_entry_t pte_l2_l_wc_mode; 7123 pt_entry_t pte_l2_l_cache_mode_pt; 7124 pt_entry_t pte_l2_l_cache_mask; 7125 7126 pt_entry_t pte_l2_s_nocache_mode; 7127 pt_entry_t pte_l2_s_cache_mode; 7128 pt_entry_t pte_l2_s_wc_mode; 7129 pt_entry_t pte_l2_s_cache_mode_pt; 7130 pt_entry_t pte_l2_s_cache_mask; 7131 7132 pt_entry_t pte_l1_s_prot_u; 7133 pt_entry_t pte_l1_s_prot_w; 7134 pt_entry_t pte_l1_s_prot_ro; 7135 pt_entry_t pte_l1_s_prot_mask; 7136 7137 pt_entry_t pte_l2_s_prot_u; 7138 pt_entry_t pte_l2_s_prot_w; 7139 pt_entry_t pte_l2_s_prot_ro; 7140 pt_entry_t pte_l2_s_prot_mask; 7141 7142 pt_entry_t pte_l2_l_prot_u; 7143 pt_entry_t pte_l2_l_prot_w; 7144 pt_entry_t pte_l2_l_prot_ro; 7145 pt_entry_t pte_l2_l_prot_mask; 7146 7147 pt_entry_t pte_l1_ss_proto; 7148 pt_entry_t pte_l1_s_proto; 7149 pt_entry_t pte_l1_c_proto; 7150 pt_entry_t pte_l2_s_proto; 7151 7152 void (*pmap_copy_page_func)(paddr_t, paddr_t); 7153 void (*pmap_zero_page_func)(paddr_t); 7154 7155 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 7156 void 7157 pmap_pte_init_generic(void) 7158 { 7159 7160 pte_l1_s_nocache_mode = 0; 7161 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7162 pte_l1_s_wc_mode = L1_S_B; 7163 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 7164 7165 pte_l2_l_nocache_mode = 0; 7166 pte_l2_l_cache_mode = L2_B|L2_C; 7167 pte_l2_l_wc_mode = L2_B; 7168 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 7169 7170 pte_l2_s_nocache_mode = 0; 7171 pte_l2_s_cache_mode = L2_B|L2_C; 7172 pte_l2_s_wc_mode = L2_B; 7173 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 7174 7175 /* 7176 * If we have a write-through cache, set B and C. If 7177 * we have a write-back cache, then we assume setting 7178 * only C will make those pages write-through (except for those 7179 * Cortex CPUs which can read the L1 caches). 7180 */ 7181 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop 7182 #if ARM_MMU_V7 > 0 7183 || CPU_ID_CORTEX_P(curcpu()->ci_arm_cpuid) 7184 #endif 7185 #if ARM_MMU_V6 > 0 7186 || CPU_ID_ARM11_P(curcpu()->ci_arm_cpuid) /* arm116 errata 399234 */ 7187 #endif 7188 || false) { 7189 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7190 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7191 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7192 } else { 7193 pte_l1_s_cache_mode_pt = L1_S_C; /* write through */ 7194 pte_l2_l_cache_mode_pt = L2_C; /* write through */ 7195 pte_l2_s_cache_mode_pt = L2_C; /* write through */ 7196 } 7197 7198 pte_l1_s_prot_u = L1_S_PROT_U_generic; 7199 pte_l1_s_prot_w = L1_S_PROT_W_generic; 7200 pte_l1_s_prot_ro = L1_S_PROT_RO_generic; 7201 pte_l1_s_prot_mask = L1_S_PROT_MASK_generic; 7202 7203 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7204 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7205 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7206 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7207 7208 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7209 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7210 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7211 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7212 7213 pte_l1_ss_proto = L1_SS_PROTO_generic; 7214 pte_l1_s_proto = L1_S_PROTO_generic; 7215 pte_l1_c_proto = L1_C_PROTO_generic; 7216 pte_l2_s_proto = L2_S_PROTO_generic; 7217 7218 pmap_copy_page_func = pmap_copy_page_generic; 7219 pmap_zero_page_func = pmap_zero_page_generic; 7220 } 7221 7222 #if defined(CPU_ARM8) 7223 void 7224 pmap_pte_init_arm8(void) 7225 { 7226 7227 /* 7228 * ARM8 is compatible with generic, but we need to use 7229 * the page tables uncached. 7230 */ 7231 pmap_pte_init_generic(); 7232 7233 pte_l1_s_cache_mode_pt = 0; 7234 pte_l2_l_cache_mode_pt = 0; 7235 pte_l2_s_cache_mode_pt = 0; 7236 } 7237 #endif /* CPU_ARM8 */ 7238 7239 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 7240 void 7241 pmap_pte_init_arm9(void) 7242 { 7243 7244 /* 7245 * ARM9 is compatible with generic, but we want to use 7246 * write-through caching for now. 7247 */ 7248 pmap_pte_init_generic(); 7249 7250 pte_l1_s_cache_mode = L1_S_C; 7251 pte_l2_l_cache_mode = L2_C; 7252 pte_l2_s_cache_mode = L2_C; 7253 7254 pte_l1_s_wc_mode = L1_S_B; 7255 pte_l2_l_wc_mode = L2_B; 7256 pte_l2_s_wc_mode = L2_B; 7257 7258 pte_l1_s_cache_mode_pt = L1_S_C; 7259 pte_l2_l_cache_mode_pt = L2_C; 7260 pte_l2_s_cache_mode_pt = L2_C; 7261 } 7262 #endif /* CPU_ARM9 && ARM9_CACHE_WRITE_THROUGH */ 7263 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 7264 7265 #if defined(CPU_ARM10) 7266 void 7267 pmap_pte_init_arm10(void) 7268 { 7269 7270 /* 7271 * ARM10 is compatible with generic, but we want to use 7272 * write-through caching for now. 7273 */ 7274 pmap_pte_init_generic(); 7275 7276 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 7277 pte_l2_l_cache_mode = L2_B | L2_C; 7278 pte_l2_s_cache_mode = L2_B | L2_C; 7279 7280 pte_l1_s_cache_mode = L1_S_B; 7281 pte_l2_l_cache_mode = L2_B; 7282 pte_l2_s_cache_mode = L2_B; 7283 7284 pte_l1_s_cache_mode_pt = L1_S_C; 7285 pte_l2_l_cache_mode_pt = L2_C; 7286 pte_l2_s_cache_mode_pt = L2_C; 7287 7288 } 7289 #endif /* CPU_ARM10 */ 7290 7291 #if defined(CPU_ARM11) && defined(ARM11_CACHE_WRITE_THROUGH) 7292 void 7293 pmap_pte_init_arm11(void) 7294 { 7295 7296 /* 7297 * ARM11 is compatible with generic, but we want to use 7298 * write-through caching for now. 7299 */ 7300 pmap_pte_init_generic(); 7301 7302 pte_l1_s_cache_mode = L1_S_C; 7303 pte_l2_l_cache_mode = L2_C; 7304 pte_l2_s_cache_mode = L2_C; 7305 7306 pte_l1_s_wc_mode = L1_S_B; 7307 pte_l2_l_wc_mode = L2_B; 7308 pte_l2_s_wc_mode = L2_B; 7309 7310 pte_l1_s_cache_mode_pt = L1_S_C; 7311 pte_l2_l_cache_mode_pt = L2_C; 7312 pte_l2_s_cache_mode_pt = L2_C; 7313 } 7314 #endif /* CPU_ARM11 && ARM11_CACHE_WRITE_THROUGH */ 7315 7316 #if ARM_MMU_SA1 == 1 7317 void 7318 pmap_pte_init_sa1(void) 7319 { 7320 7321 /* 7322 * The StrongARM SA-1 cache does not have a write-through 7323 * mode. So, do the generic initialization, then reset 7324 * the page table cache mode to B=1,C=1, and note that 7325 * the PTEs need to be sync'd. 7326 */ 7327 pmap_pte_init_generic(); 7328 7329 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7330 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7331 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7332 7333 pmap_needs_pte_sync = 1; 7334 } 7335 #endif /* ARM_MMU_SA1 == 1*/ 7336 7337 #if ARM_MMU_XSCALE == 1 7338 #if (ARM_NMMUS > 1) 7339 static u_int xscale_use_minidata; 7340 #endif 7341 7342 void 7343 pmap_pte_init_xscale(void) 7344 { 7345 uint32_t auxctl; 7346 int write_through = 0; 7347 7348 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7349 pte_l1_s_wc_mode = L1_S_B; 7350 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 7351 7352 pte_l2_l_cache_mode = L2_B|L2_C; 7353 pte_l2_l_wc_mode = L2_B; 7354 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 7355 7356 pte_l2_s_cache_mode = L2_B|L2_C; 7357 pte_l2_s_wc_mode = L2_B; 7358 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 7359 7360 pte_l1_s_cache_mode_pt = L1_S_C; 7361 pte_l2_l_cache_mode_pt = L2_C; 7362 pte_l2_s_cache_mode_pt = L2_C; 7363 7364 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 7365 /* 7366 * The XScale core has an enhanced mode where writes that 7367 * miss the cache cause a cache line to be allocated. This 7368 * is significantly faster than the traditional, write-through 7369 * behavior of this case. 7370 */ 7371 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 7372 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 7373 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 7374 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 7375 7376 #ifdef XSCALE_CACHE_WRITE_THROUGH 7377 /* 7378 * Some versions of the XScale core have various bugs in 7379 * their cache units, the work-around for which is to run 7380 * the cache in write-through mode. Unfortunately, this 7381 * has a major (negative) impact on performance. So, we 7382 * go ahead and run fast-and-loose, in the hopes that we 7383 * don't line up the planets in a way that will trip the 7384 * bugs. 7385 * 7386 * However, we give you the option to be slow-but-correct. 7387 */ 7388 write_through = 1; 7389 #elif defined(XSCALE_CACHE_WRITE_BACK) 7390 /* force write back cache mode */ 7391 write_through = 0; 7392 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 7393 /* 7394 * Intel PXA2[15]0 processors are known to have a bug in 7395 * write-back cache on revision 4 and earlier (stepping 7396 * A[01] and B[012]). Fixed for C0 and later. 7397 */ 7398 { 7399 uint32_t id, type; 7400 7401 id = cpufunc_id(); 7402 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 7403 7404 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 7405 if ((id & CPU_ID_REVISION_MASK) < 5) { 7406 /* write through for stepping A0-1 and B0-2 */ 7407 write_through = 1; 7408 } 7409 } 7410 } 7411 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 7412 7413 if (write_through) { 7414 pte_l1_s_cache_mode = L1_S_C; 7415 pte_l2_l_cache_mode = L2_C; 7416 pte_l2_s_cache_mode = L2_C; 7417 } 7418 7419 #if (ARM_NMMUS > 1) 7420 xscale_use_minidata = 1; 7421 #endif 7422 7423 pte_l1_s_prot_u = L1_S_PROT_U_xscale; 7424 pte_l1_s_prot_w = L1_S_PROT_W_xscale; 7425 pte_l1_s_prot_ro = L1_S_PROT_RO_xscale; 7426 pte_l1_s_prot_mask = L1_S_PROT_MASK_xscale; 7427 7428 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 7429 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 7430 pte_l2_s_prot_ro = L2_S_PROT_RO_xscale; 7431 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 7432 7433 pte_l2_l_prot_u = L2_L_PROT_U_xscale; 7434 pte_l2_l_prot_w = L2_L_PROT_W_xscale; 7435 pte_l2_l_prot_ro = L2_L_PROT_RO_xscale; 7436 pte_l2_l_prot_mask = L2_L_PROT_MASK_xscale; 7437 7438 pte_l1_ss_proto = L1_SS_PROTO_xscale; 7439 pte_l1_s_proto = L1_S_PROTO_xscale; 7440 pte_l1_c_proto = L1_C_PROTO_xscale; 7441 pte_l2_s_proto = L2_S_PROTO_xscale; 7442 7443 pmap_copy_page_func = pmap_copy_page_xscale; 7444 pmap_zero_page_func = pmap_zero_page_xscale; 7445 7446 /* 7447 * Disable ECC protection of page table access, for now. 7448 */ 7449 auxctl = armreg_auxctl_read(); 7450 auxctl &= ~XSCALE_AUXCTL_P; 7451 armreg_auxctl_write(auxctl); 7452 } 7453 7454 /* 7455 * xscale_setup_minidata: 7456 * 7457 * Set up the mini-data cache clean area. We require the 7458 * caller to allocate the right amount of physically and 7459 * virtually contiguous space. 7460 */ 7461 void 7462 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 7463 { 7464 extern vaddr_t xscale_minidata_clean_addr; 7465 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 7466 pd_entry_t *pde = (pd_entry_t *) l1pt; 7467 vsize_t size; 7468 uint32_t auxctl; 7469 7470 xscale_minidata_clean_addr = va; 7471 7472 /* Round it to page size. */ 7473 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 7474 7475 for (; size != 0; 7476 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 7477 const size_t l1slot = l1pte_index(va); 7478 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pde[l1slot])); 7479 if (ptep == NULL) 7480 panic("xscale_setup_minidata: can't find L2 table for " 7481 "VA 0x%08lx", va); 7482 7483 ptep += l2pte_index(va); 7484 pt_entry_t opte = *ptep; 7485 l2pte_set(ptep, 7486 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 7487 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X), opte); 7488 } 7489 7490 /* 7491 * Configure the mini-data cache for write-back with 7492 * read/write-allocate. 7493 * 7494 * NOTE: In order to reconfigure the mini-data cache, we must 7495 * make sure it contains no valid data! In order to do that, 7496 * we must issue a global data cache invalidate command! 7497 * 7498 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 7499 * THIS IS VERY IMPORTANT! 7500 */ 7501 7502 /* Invalidate data and mini-data. */ 7503 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 7504 auxctl = armreg_auxctl_read(); 7505 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 7506 armreg_auxctl_write(auxctl); 7507 } 7508 7509 /* 7510 * Change the PTEs for the specified kernel mappings such that they 7511 * will use the mini data cache instead of the main data cache. 7512 */ 7513 void 7514 pmap_uarea(vaddr_t va) 7515 { 7516 vaddr_t next_bucket, eva; 7517 7518 #if (ARM_NMMUS > 1) 7519 if (xscale_use_minidata == 0) 7520 return; 7521 #endif 7522 7523 eva = va + USPACE; 7524 7525 while (va < eva) { 7526 next_bucket = L2_NEXT_BUCKET_VA(va); 7527 if (next_bucket > eva) 7528 next_bucket = eva; 7529 7530 struct l2_bucket *l2b = pmap_get_l2_bucket(pmap_kernel(), va); 7531 KDASSERT(l2b != NULL); 7532 7533 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 7534 pt_entry_t *ptep = sptep; 7535 7536 while (va < next_bucket) { 7537 const pt_entry_t opte = *ptep; 7538 if (!l2pte_minidata_p(opte)) { 7539 cpu_dcache_wbinv_range(va, PAGE_SIZE); 7540 cpu_tlb_flushD_SE(va); 7541 l2pte_set(ptep, opte & ~L2_B, opte); 7542 } 7543 ptep += PAGE_SIZE / L2_S_SIZE; 7544 va += PAGE_SIZE; 7545 } 7546 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 7547 } 7548 cpu_cpwait(); 7549 } 7550 #endif /* ARM_MMU_XSCALE == 1 */ 7551 7552 7553 #if defined(CPU_ARM11MPCORE) 7554 void 7555 pmap_pte_init_arm11mpcore(void) 7556 { 7557 7558 /* cache mode is controlled by 5 bits (B, C, TEX[2:0]) */ 7559 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6; 7560 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6; 7561 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7562 /* use extended small page (without APn, with TEX) */ 7563 pte_l2_s_cache_mask = L2_XS_CACHE_MASK_armv6; 7564 #else 7565 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6c; 7566 #endif 7567 7568 /* write-back, write-allocate */ 7569 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7570 pte_l2_l_cache_mode = L2_C | L2_B | L2_V6_L_TEX(0x01); 7571 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7572 pte_l2_s_cache_mode = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7573 #else 7574 /* no TEX. read-allocate */ 7575 pte_l2_s_cache_mode = L2_C | L2_B; 7576 #endif 7577 /* 7578 * write-back, write-allocate for page tables. 7579 */ 7580 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7581 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_V6_L_TEX(0x01); 7582 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7583 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7584 #else 7585 pte_l2_s_cache_mode_pt = L2_C | L2_B; 7586 #endif 7587 7588 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7589 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7590 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7591 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7592 7593 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7594 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7595 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7596 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7597 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7598 7599 #else 7600 /* with AP[0..3] */ 7601 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7602 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7603 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7604 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7605 #endif 7606 7607 #ifdef ARM11MPCORE_COMPAT_MMU 7608 /* with AP[0..3] */ 7609 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7610 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7611 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7612 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7613 7614 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7615 pte_l1_s_proto = L1_S_PROTO_armv6; 7616 pte_l1_c_proto = L1_C_PROTO_armv6; 7617 pte_l2_s_proto = L2_S_PROTO_armv6c; 7618 #else 7619 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7620 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7621 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7622 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7623 7624 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7625 pte_l1_s_proto = L1_S_PROTO_armv6; 7626 pte_l1_c_proto = L1_C_PROTO_armv6; 7627 pte_l2_s_proto = L2_S_PROTO_armv6n; 7628 #endif 7629 7630 pmap_copy_page_func = pmap_copy_page_generic; 7631 pmap_zero_page_func = pmap_zero_page_generic; 7632 pmap_needs_pte_sync = 1; 7633 } 7634 #endif /* CPU_ARM11MPCORE */ 7635 7636 7637 #if ARM_MMU_V6 == 1 7638 void 7639 pmap_pte_init_armv6(void) 7640 { 7641 /* 7642 * The ARMv6-A MMU is mostly compatible with generic. If the 7643 * AP field is zero, that now means "no access" rather than 7644 * read-only. The prototypes are a little different because of 7645 * the XN bit. 7646 */ 7647 pmap_pte_init_generic(); 7648 7649 pte_l1_s_nocache_mode = L1_S_XS_TEX(1); 7650 pte_l2_l_nocache_mode = L2_XS_L_TEX(1); 7651 pte_l2_s_nocache_mode = L2_XS_T_TEX(1); 7652 7653 #ifdef ARM11_COMPAT_MMU 7654 /* with AP[0..3] */ 7655 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7656 #else 7657 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6n; 7658 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6n; 7659 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6n; 7660 7661 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7662 pte_l1_s_proto = L1_S_PROTO_armv6; 7663 pte_l1_c_proto = L1_C_PROTO_armv6; 7664 pte_l2_s_proto = L2_S_PROTO_armv6n; 7665 7666 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7667 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7668 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7669 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7670 7671 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7672 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7673 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7674 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7675 7676 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7677 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7678 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7679 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7680 7681 #endif 7682 } 7683 #endif /* ARM_MMU_V6 */ 7684 7685 #if ARM_MMU_V7 == 1 7686 void 7687 pmap_pte_init_armv7(void) 7688 { 7689 /* 7690 * The ARMv7-A MMU is mostly compatible with generic. If the 7691 * AP field is zero, that now means "no access" rather than 7692 * read-only. The prototypes are a little different because of 7693 * the XN bit. 7694 */ 7695 pmap_pte_init_generic(); 7696 7697 pmap_needs_pte_sync = 1; 7698 7699 pte_l1_s_nocache_mode = L1_S_XS_TEX(1); 7700 pte_l2_l_nocache_mode = L2_XS_L_TEX(1); 7701 pte_l2_s_nocache_mode = L2_XS_T_TEX(1); 7702 7703 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv7; 7704 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv7; 7705 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv7; 7706 7707 /* 7708 * If the core support coherent walk then updates to translation tables 7709 * do not require a clean to the point of unification to ensure 7710 * visibility by subsequent translation table walks. That means we can 7711 * map everything shareable and cached and the right thing will happen. 7712 */ 7713 if (__SHIFTOUT(armreg_mmfr3_read(), __BITS(23,20))) { 7714 pmap_needs_pte_sync = 0; 7715 7716 /* 7717 * write-back, no write-allocate, shareable for normal pages. 7718 */ 7719 pte_l1_s_cache_mode |= L1_S_V6_S; 7720 pte_l2_l_cache_mode |= L2_XS_S; 7721 pte_l2_s_cache_mode |= L2_XS_S; 7722 } 7723 7724 /* 7725 * Page tables are just all other memory. We can use write-back since 7726 * pmap_needs_pte_sync is 1 (or the MMU can read out of cache). 7727 */ 7728 pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode; 7729 pte_l2_l_cache_mode_pt = pte_l2_l_cache_mode; 7730 pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; 7731 7732 /* 7733 * Check the Memory Model Features to see if this CPU supports 7734 * the TLBIASID coproc op. 7735 */ 7736 if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(16,19)) >= 2) { 7737 arm_has_tlbiasid_p = true; 7738 } else if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(12,15)) >= 2) { 7739 arm_has_tlbiasid_p = true; 7740 } 7741 7742 /* 7743 * Check the MPIDR to see if this CPU supports MP extensions. 7744 */ 7745 #ifdef MULTIPROCESSOR 7746 arm_has_mpext_p = (armreg_mpidr_read() & (MPIDR_MP|MPIDR_U)) == MPIDR_MP; 7747 #else 7748 arm_has_mpext_p = false; 7749 #endif 7750 7751 pte_l1_s_prot_u = L1_S_PROT_U_armv7; 7752 pte_l1_s_prot_w = L1_S_PROT_W_armv7; 7753 pte_l1_s_prot_ro = L1_S_PROT_RO_armv7; 7754 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv7; 7755 7756 pte_l2_s_prot_u = L2_S_PROT_U_armv7; 7757 pte_l2_s_prot_w = L2_S_PROT_W_armv7; 7758 pte_l2_s_prot_ro = L2_S_PROT_RO_armv7; 7759 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv7; 7760 7761 pte_l2_l_prot_u = L2_L_PROT_U_armv7; 7762 pte_l2_l_prot_w = L2_L_PROT_W_armv7; 7763 pte_l2_l_prot_ro = L2_L_PROT_RO_armv7; 7764 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv7; 7765 7766 pte_l1_ss_proto = L1_SS_PROTO_armv7; 7767 pte_l1_s_proto = L1_S_PROTO_armv7; 7768 pte_l1_c_proto = L1_C_PROTO_armv7; 7769 pte_l2_s_proto = L2_S_PROTO_armv7; 7770 7771 } 7772 #endif /* ARM_MMU_V7 */ 7773 7774 /* 7775 * return the PA of the current L1 table, for use when handling a crash dump 7776 */ 7777 uint32_t 7778 pmap_kernel_L1_addr(void) 7779 { 7780 #ifdef ARM_MMU_EXTENDED 7781 return pmap_kernel()->pm_l1_pa; 7782 #else 7783 return pmap_kernel()->pm_l1->l1_physaddr; 7784 #endif 7785 } 7786 7787 #if defined(DDB) 7788 /* 7789 * A couple of ddb-callable functions for dumping pmaps 7790 */ 7791 void pmap_dump(pmap_t); 7792 7793 static pt_entry_t ncptes[64]; 7794 static void pmap_dump_ncpg(pmap_t); 7795 7796 void 7797 pmap_dump(pmap_t pm) 7798 { 7799 struct l2_dtable *l2; 7800 struct l2_bucket *l2b; 7801 pt_entry_t *ptep, pte; 7802 vaddr_t l2_va, l2b_va, va; 7803 int i, j, k, occ, rows = 0; 7804 7805 if (pm == pmap_kernel()) 7806 printf("pmap_kernel (%p): ", pm); 7807 else 7808 printf("user pmap (%p): ", pm); 7809 7810 #ifdef ARM_MMU_EXTENDED 7811 printf("l1 at %p\n", pmap_l1_kva(pm)); 7812 #else 7813 printf("domain %d, l1 at %p\n", pmap_domain(pm), pmap_l1_kva(pm)); 7814 #endif 7815 7816 l2_va = 0; 7817 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 7818 l2 = pm->pm_l2[i]; 7819 7820 if (l2 == NULL || l2->l2_occupancy == 0) 7821 continue; 7822 7823 l2b_va = l2_va; 7824 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 7825 l2b = &l2->l2_bucket[j]; 7826 7827 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 7828 continue; 7829 7830 ptep = l2b->l2b_kva; 7831 7832 for (k = 0; k < 256 && ptep[k] == 0; k++) 7833 ; 7834 7835 k &= ~63; 7836 occ = l2b->l2b_occupancy; 7837 va = l2b_va + (k * 4096); 7838 for (; k < 256; k++, va += 0x1000) { 7839 char ch = ' '; 7840 if ((k % 64) == 0) { 7841 if ((rows % 8) == 0) { 7842 printf( 7843 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 7844 } 7845 printf("%08lx: ", va); 7846 } 7847 7848 ncptes[k & 63] = 0; 7849 pte = ptep[k]; 7850 if (pte == 0) { 7851 ch = '.'; 7852 } else { 7853 occ--; 7854 switch (pte & 0x4c) { 7855 case 0x00: 7856 ch = 'N'; /* No cache No buff */ 7857 break; 7858 case 0x04: 7859 ch = 'B'; /* No cache buff */ 7860 break; 7861 case 0x08: 7862 ch = 'C'; /* Cache No buff */ 7863 break; 7864 case 0x0c: 7865 ch = 'F'; /* Cache Buff */ 7866 break; 7867 case 0x40: 7868 ch = 'D'; 7869 break; 7870 case 0x48: 7871 ch = 'm'; /* Xscale mini-data */ 7872 break; 7873 default: 7874 ch = '?'; 7875 break; 7876 } 7877 7878 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 7879 ch += 0x20; 7880 7881 if ((pte & 0xc) == 0) 7882 ncptes[k & 63] = pte; 7883 } 7884 7885 if ((k % 64) == 63) { 7886 rows++; 7887 printf("%c\n", ch); 7888 pmap_dump_ncpg(pm); 7889 if (occ == 0) 7890 break; 7891 } else 7892 printf("%c", ch); 7893 } 7894 } 7895 } 7896 } 7897 7898 static void 7899 pmap_dump_ncpg(pmap_t pm) 7900 { 7901 struct vm_page *pg; 7902 struct vm_page_md *md; 7903 struct pv_entry *pv; 7904 int i; 7905 7906 for (i = 0; i < 63; i++) { 7907 if (ncptes[i] == 0) 7908 continue; 7909 7910 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 7911 if (pg == NULL) 7912 continue; 7913 md = VM_PAGE_TO_MD(pg); 7914 7915 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 7916 VM_PAGE_TO_PHYS(pg), 7917 md->krw_mappings, md->kro_mappings, 7918 md->urw_mappings, md->uro_mappings); 7919 7920 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 7921 printf(" %c va 0x%08lx, flags 0x%x\n", 7922 (pm == pv->pv_pmap) ? '*' : ' ', 7923 pv->pv_va, pv->pv_flags); 7924 } 7925 } 7926 } 7927 #endif 7928 7929 #ifdef PMAP_STEAL_MEMORY 7930 void 7931 pmap_boot_pageadd(pv_addr_t *newpv) 7932 { 7933 pv_addr_t *pv, *npv; 7934 7935 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 7936 if (newpv->pv_pa < pv->pv_va) { 7937 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 7938 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 7939 newpv->pv_size += pv->pv_size; 7940 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 7941 } 7942 pv = NULL; 7943 } else { 7944 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 7945 pv = npv) { 7946 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 7947 KASSERT(pv->pv_pa < newpv->pv_pa); 7948 if (newpv->pv_pa > npv->pv_pa) 7949 continue; 7950 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 7951 pv->pv_size += newpv->pv_size; 7952 return; 7953 } 7954 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 7955 break; 7956 newpv->pv_size += npv->pv_size; 7957 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7958 SLIST_REMOVE_AFTER(newpv, pv_list); 7959 return; 7960 } 7961 } 7962 } 7963 7964 if (pv) { 7965 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7966 } else { 7967 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 7968 } 7969 } 7970 7971 void 7972 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 7973 pv_addr_t *rpv) 7974 { 7975 pv_addr_t *pv, **pvp; 7976 7977 KASSERT(amount & PGOFSET); 7978 KASSERT((mask & PGOFSET) == 0); 7979 KASSERT((match & PGOFSET) == 0); 7980 KASSERT(amount != 0); 7981 7982 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7983 (pv = *pvp) != NULL; 7984 pvp = &SLIST_NEXT(pv, pv_list)) { 7985 pv_addr_t *newpv; 7986 psize_t off; 7987 /* 7988 * If this entry is too small to satisfy the request... 7989 */ 7990 KASSERT(pv->pv_size > 0); 7991 if (pv->pv_size < amount) 7992 continue; 7993 7994 for (off = 0; off <= mask; off += PAGE_SIZE) { 7995 if (((pv->pv_pa + off) & mask) == match 7996 && off + amount <= pv->pv_size) 7997 break; 7998 } 7999 if (off > mask) 8000 continue; 8001 8002 rpv->pv_va = pv->pv_va + off; 8003 rpv->pv_pa = pv->pv_pa + off; 8004 rpv->pv_size = amount; 8005 pv->pv_size -= amount; 8006 if (pv->pv_size == 0) { 8007 KASSERT(off == 0); 8008 KASSERT((vaddr_t) pv == rpv->pv_va); 8009 *pvp = SLIST_NEXT(pv, pv_list); 8010 } else if (off == 0) { 8011 KASSERT((vaddr_t) pv == rpv->pv_va); 8012 newpv = (pv_addr_t *) (rpv->pv_va + amount); 8013 *newpv = *pv; 8014 newpv->pv_pa += amount; 8015 newpv->pv_va += amount; 8016 *pvp = newpv; 8017 } else if (off < pv->pv_size) { 8018 newpv = (pv_addr_t *) (rpv->pv_va + amount); 8019 *newpv = *pv; 8020 newpv->pv_size -= off; 8021 newpv->pv_pa += off + amount; 8022 newpv->pv_va += off + amount; 8023 8024 SLIST_NEXT(pv, pv_list) = newpv; 8025 pv->pv_size = off; 8026 } else { 8027 KASSERT((vaddr_t) pv != rpv->pv_va); 8028 } 8029 memset((void *)rpv->pv_va, 0, amount); 8030 return; 8031 } 8032 8033 if (!uvm_physseg_valid_p(uvm_physseg_get_first())) 8034 panic("pmap_boot_pagealloc: couldn't allocate memory"); 8035 8036 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 8037 (pv = *pvp) != NULL; 8038 pvp = &SLIST_NEXT(pv, pv_list)) { 8039 if (SLIST_NEXT(pv, pv_list) == NULL) 8040 break; 8041 } 8042 KASSERT(mask == 0); 8043 8044 for (uvm_physseg_t ups = uvm_physseg_get_first(); 8045 uvm_physseg_valid_p(ups); 8046 ups = uvm_physseg_get_next(ups)) { 8047 8048 paddr_t spn = uvm_physseg_get_start(ups); 8049 paddr_t epn = uvm_physseg_get_end(ups); 8050 if (spn == atop(pv->pv_pa + pv->pv_size) 8051 && pv->pv_va + pv->pv_size <= ptoa(epn)) { 8052 rpv->pv_va = pv->pv_va; 8053 rpv->pv_pa = pv->pv_pa; 8054 rpv->pv_size = amount; 8055 *pvp = NULL; 8056 pmap_map_chunk(kernel_l1pt.pv_va, 8057 ptoa(spn) + (pv->pv_va - pv->pv_pa), 8058 ptoa(spn), 8059 amount - pv->pv_size, 8060 VM_PROT_READ|VM_PROT_WRITE, 8061 PTE_CACHE); 8062 8063 uvm_physseg_unplug(spn, atop(amount - pv->pv_size)); 8064 memset((void *)rpv->pv_va, 0, rpv->pv_size); 8065 return; 8066 } 8067 } 8068 8069 panic("pmap_boot_pagealloc: couldn't allocate memory"); 8070 } 8071 8072 vaddr_t 8073 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 8074 { 8075 pv_addr_t pv; 8076 8077 pmap_boot_pagealloc(size, 0, 0, &pv); 8078 8079 return pv.pv_va; 8080 } 8081 #endif /* PMAP_STEAL_MEMORY */ 8082 8083 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 8084 { 8085 sysctl_createv(clog, 0, NULL, NULL, 8086 CTLFLAG_PERMANENT, 8087 CTLTYPE_NODE, "machdep", NULL, 8088 NULL, 0, NULL, 0, 8089 CTL_MACHDEP, CTL_EOL); 8090 8091 sysctl_createv(clog, 0, NULL, NULL, 8092 CTLFLAG_PERMANENT, 8093 CTLTYPE_INT, "kmpages", 8094 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 8095 NULL, 0, &pmap_kmpages, 0, 8096 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 8097 } 8098 8099 #ifdef PMAP_NEED_ALLOC_POOLPAGE 8100 struct vm_page * 8101 arm_pmap_alloc_poolpage(int flags) 8102 { 8103 /* 8104 * On some systems, only some pages may be "coherent" for dma and we 8105 * want to prefer those for pool pages (think mbufs) but fallback to 8106 * any page if none is available. 8107 */ 8108 if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) { 8109 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 8110 UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist); 8111 } 8112 8113 return uvm_pagealloc(NULL, 0, NULL, flags); 8114 } 8115 #endif 8116 8117 #if defined(ARM_MMU_EXTENDED) && defined(MULTIPROCESSOR) 8118 void 8119 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 8120 { 8121 /* nothing */ 8122 } 8123 8124 int 8125 pic_ipi_shootdown(void *arg) 8126 { 8127 #if PMAP_TLB_NEED_SHOOTDOWN 8128 pmap_tlb_shootdown_process(); 8129 #endif 8130 return 1; 8131 } 8132 #endif /* ARM_MMU_EXTENDED && MULTIPROCESSOR */ 8133 8134 8135 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 8136 vaddr_t 8137 pmap_direct_mapped_phys(paddr_t pa, bool *ok_p, vaddr_t va) 8138 { 8139 bool ok = false; 8140 if (physical_start <= pa && pa < physical_end) { 8141 #ifdef KERNEL_BASE_VOFFSET 8142 const vaddr_t newva = pa + KERNEL_BASE_VOFFSET; 8143 #else 8144 const vaddr_t newva = KERNEL_BASE + pa - physical_start; 8145 #endif 8146 #ifdef ARM_MMU_EXTENDED 8147 if (newva >= KERNEL_BASE && newva < pmap_directlimit) { 8148 #endif 8149 va = newva; 8150 ok = true; 8151 #ifdef ARM_MMU_EXTENDED 8152 } 8153 #endif 8154 } 8155 KASSERT(ok_p); 8156 *ok_p = ok; 8157 return va; 8158 } 8159 8160 vaddr_t 8161 pmap_map_poolpage(paddr_t pa) 8162 { 8163 bool ok __diagused; 8164 vaddr_t va = pmap_direct_mapped_phys(pa, &ok, 0); 8165 KASSERTMSG(ok, "pa %#lx not direct mappable", pa); 8166 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 8167 if (arm_cache_prefer_mask != 0) { 8168 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 8169 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 8170 pmap_acquire_page_lock(md); 8171 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 8172 pmap_release_page_lock(md); 8173 } 8174 #endif 8175 return va; 8176 } 8177 8178 paddr_t 8179 pmap_unmap_poolpage(vaddr_t va) 8180 { 8181 KASSERT(va >= KERNEL_BASE); 8182 #ifdef PMAP_CACHE_VIVT 8183 cpu_idcache_wbinv_range(va, PAGE_SIZE); 8184 #endif 8185 #if defined(KERNEL_BASE_VOFFSET) 8186 return va - KERNEL_BASE_VOFFSET; 8187 #else 8188 return va - KERNEL_BASE + physical_start; 8189 #endif 8190 } 8191 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 8192
Indexes created Mon Sep 29 21:09:56 GMT 2025