pmap.c revision 1.413
1 /* $NetBSD: pmap.c,v 1.413 2020/05/27 06:41:58 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999, 2020 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependent vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of in kernel VA space, 166 * then MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* Include header files */ 183 184 #include "opt_arm_debug.h" 185 #include "opt_cpuoptions.h" 186 #include "opt_ddb.h" 187 #include "opt_lockdebug.h" 188 #include "opt_multiprocessor.h" 189 190 #ifdef MULTIPROCESSOR 191 #define _INTR_PRIVATE 192 #endif 193 194 #include <sys/cdefs.h> 195 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.413 2020/05/27 06:41:58 skrll Exp $"); 196 197 #include <sys/atomic.h> 198 #include <sys/param.h> 199 #include <sys/types.h> 200 #include <sys/bus.h> 201 #include <sys/cpu.h> 202 #include <sys/intr.h> 203 #include <sys/kernel.h> 204 #include <sys/kernhist.h> 205 #include <sys/kmem.h> 206 #include <sys/pool.h> 207 #include <sys/proc.h> 208 #include <sys/sysctl.h> 209 #include <sys/systm.h> 210 211 #include <uvm/uvm.h> 212 #include <uvm/pmap/pmap_pvt.h> 213 214 #include <arm/locore.h> 215 216 #ifdef DDB 217 #include <arm/db_machdep.h> 218 #endif 219 220 #ifdef VERBOSE_INIT_ARM 221 #define VPRINTF(...) printf(__VA_ARGS__) 222 #else 223 #define VPRINTF(...) __nothing 224 #endif 225 226 /* 227 * pmap_kernel() points here 228 */ 229 static struct pmap kernel_pmap_store = { 230 #ifndef ARM_MMU_EXTENDED 231 .pm_activated = true, 232 .pm_domain = PMAP_DOMAIN_KERNEL, 233 .pm_cstate.cs_all = PMAP_CACHE_STATE_ALL, 234 #endif 235 }; 236 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 237 #undef pmap_kernel 238 #define pmap_kernel() (&kernel_pmap_store) 239 #ifdef PMAP_NEED_ALLOC_POOLPAGE 240 int arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT; 241 #endif 242 243 /* 244 * Pool and cache that pmap structures are allocated from. 245 * We use a cache to avoid clearing the pm_l2[] array (1KB) 246 * in pmap_create(). 247 */ 248 static struct pool_cache pmap_cache; 249 250 /* 251 * Pool of PV structures 252 */ 253 static struct pool pmap_pv_pool; 254 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 255 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 256 static struct pool_allocator pmap_bootstrap_pv_allocator = { 257 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 258 }; 259 260 /* 261 * Pool and cache of l2_dtable structures. 262 * We use a cache to avoid clearing the structures when they're 263 * allocated. (196 bytes) 264 */ 265 static struct pool_cache pmap_l2dtable_cache; 266 static vaddr_t pmap_kernel_l2dtable_kva; 267 268 /* 269 * Pool and cache of L2 page descriptors. 270 * We use a cache to avoid clearing the descriptor table 271 * when they're allocated. (1KB) 272 */ 273 static struct pool_cache pmap_l2ptp_cache; 274 static vaddr_t pmap_kernel_l2ptp_kva; 275 static paddr_t pmap_kernel_l2ptp_phys; 276 277 #ifdef PMAPCOUNTERS 278 #define PMAP_EVCNT_INITIALIZER(name) \ 279 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 280 281 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 282 static struct evcnt pmap_ev_vac_clean_one = 283 PMAP_EVCNT_INITIALIZER("clean page (1 color)"); 284 static struct evcnt pmap_ev_vac_flush_one = 285 PMAP_EVCNT_INITIALIZER("flush page (1 color)"); 286 static struct evcnt pmap_ev_vac_flush_lots = 287 PMAP_EVCNT_INITIALIZER("flush page (2+ colors)"); 288 static struct evcnt pmap_ev_vac_flush_lots2 = 289 PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)"); 290 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one); 291 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one); 292 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots); 293 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2); 294 295 static struct evcnt pmap_ev_vac_color_new = 296 PMAP_EVCNT_INITIALIZER("new page color"); 297 static struct evcnt pmap_ev_vac_color_reuse = 298 PMAP_EVCNT_INITIALIZER("ok first page color"); 299 static struct evcnt pmap_ev_vac_color_ok = 300 PMAP_EVCNT_INITIALIZER("ok page color"); 301 static struct evcnt pmap_ev_vac_color_blind = 302 PMAP_EVCNT_INITIALIZER("blind page color"); 303 static struct evcnt pmap_ev_vac_color_change = 304 PMAP_EVCNT_INITIALIZER("change page color"); 305 static struct evcnt pmap_ev_vac_color_erase = 306 PMAP_EVCNT_INITIALIZER("erase page color"); 307 static struct evcnt pmap_ev_vac_color_none = 308 PMAP_EVCNT_INITIALIZER("no page color"); 309 static struct evcnt pmap_ev_vac_color_restore = 310 PMAP_EVCNT_INITIALIZER("restore page color"); 311 312 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 313 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 314 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 315 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 316 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 317 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 318 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 319 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 320 #endif 321 322 static struct evcnt pmap_ev_mappings = 323 PMAP_EVCNT_INITIALIZER("pages mapped"); 324 static struct evcnt pmap_ev_unmappings = 325 PMAP_EVCNT_INITIALIZER("pages unmapped"); 326 static struct evcnt pmap_ev_remappings = 327 PMAP_EVCNT_INITIALIZER("pages remapped"); 328 329 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 330 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 331 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 332 333 static struct evcnt pmap_ev_kernel_mappings = 334 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 335 static struct evcnt pmap_ev_kernel_unmappings = 336 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 337 static struct evcnt pmap_ev_kernel_remappings = 338 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 339 340 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 341 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 342 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 343 344 static struct evcnt pmap_ev_kenter_mappings = 345 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 346 static struct evcnt pmap_ev_kenter_unmappings = 347 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 348 static struct evcnt pmap_ev_kenter_remappings = 349 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 350 static struct evcnt pmap_ev_pt_mappings = 351 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 352 353 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 354 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 355 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 356 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 357 358 static struct evcnt pmap_ev_fixup_mod = 359 PMAP_EVCNT_INITIALIZER("page modification emulations"); 360 static struct evcnt pmap_ev_fixup_ref = 361 PMAP_EVCNT_INITIALIZER("page reference emulations"); 362 static struct evcnt pmap_ev_fixup_exec = 363 PMAP_EVCNT_INITIALIZER("exec pages fixed up"); 364 static struct evcnt pmap_ev_fixup_pdes = 365 PMAP_EVCNT_INITIALIZER("pdes fixed up"); 366 #ifndef ARM_MMU_EXTENDED 367 static struct evcnt pmap_ev_fixup_ptesync = 368 PMAP_EVCNT_INITIALIZER("ptesync fixed"); 369 #endif 370 371 EVCNT_ATTACH_STATIC(pmap_ev_fixup_mod); 372 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ref); 373 EVCNT_ATTACH_STATIC(pmap_ev_fixup_exec); 374 EVCNT_ATTACH_STATIC(pmap_ev_fixup_pdes); 375 #ifndef ARM_MMU_EXTENDED 376 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ptesync); 377 #endif 378 379 #ifdef PMAP_CACHE_VIPT 380 static struct evcnt pmap_ev_exec_mappings = 381 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 382 static struct evcnt pmap_ev_exec_cached = 383 PMAP_EVCNT_INITIALIZER("exec pages cached"); 384 385 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 386 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 387 388 static struct evcnt pmap_ev_exec_synced = 389 PMAP_EVCNT_INITIALIZER("exec pages synced"); 390 static struct evcnt pmap_ev_exec_synced_map = 391 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 392 static struct evcnt pmap_ev_exec_synced_unmap = 393 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 394 static struct evcnt pmap_ev_exec_synced_remap = 395 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 396 static struct evcnt pmap_ev_exec_synced_clearbit = 397 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 398 #ifndef ARM_MMU_EXTENDED 399 static struct evcnt pmap_ev_exec_synced_kremove = 400 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 401 #endif 402 403 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 404 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 405 #ifndef ARM_MMU_EXTENDED 406 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 407 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 408 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 409 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 410 #endif 411 412 static struct evcnt pmap_ev_exec_discarded_unmap = 413 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 414 static struct evcnt pmap_ev_exec_discarded_zero = 415 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 416 static struct evcnt pmap_ev_exec_discarded_copy = 417 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 418 static struct evcnt pmap_ev_exec_discarded_page_protect = 419 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 420 static struct evcnt pmap_ev_exec_discarded_clearbit = 421 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 422 static struct evcnt pmap_ev_exec_discarded_kremove = 423 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 424 #ifdef ARM_MMU_EXTENDED 425 static struct evcnt pmap_ev_exec_discarded_modfixup = 426 PMAP_EVCNT_INITIALIZER("exec pages discarded (MF)"); 427 #endif 428 429 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 430 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 431 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 432 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 433 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 434 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 435 #ifdef ARM_MMU_EXTENDED 436 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_modfixup); 437 #endif 438 #endif /* PMAP_CACHE_VIPT */ 439 440 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 441 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 442 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 443 444 EVCNT_ATTACH_STATIC(pmap_ev_updates); 445 EVCNT_ATTACH_STATIC(pmap_ev_collects); 446 EVCNT_ATTACH_STATIC(pmap_ev_activations); 447 448 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 449 #else 450 #define PMAPCOUNT(x) ((void)0) 451 #endif 452 453 #ifdef ARM_MMU_EXTENDED 454 void pmap_md_pdetab_activate(pmap_t, struct lwp *); 455 void pmap_md_pdetab_deactivate(pmap_t pm); 456 #endif 457 458 /* 459 * pmap copy/zero page, and mem(5) hook point 460 */ 461 static pt_entry_t *csrc_pte, *cdst_pte; 462 static vaddr_t csrcp, cdstp; 463 #ifdef MULTIPROCESSOR 464 static size_t cnptes; 465 #define cpu_csrc_pte(o) (csrc_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 466 #define cpu_cdst_pte(o) (cdst_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 467 #define cpu_csrcp(o) (csrcp + L2_S_SIZE * cnptes * cpu_number() + (o)) 468 #define cpu_cdstp(o) (cdstp + L2_S_SIZE * cnptes * cpu_number() + (o)) 469 #else 470 #define cpu_csrc_pte(o) (csrc_pte + ((o) >> L2_S_SHIFT)) 471 #define cpu_cdst_pte(o) (cdst_pte + ((o) >> L2_S_SHIFT)) 472 #define cpu_csrcp(o) (csrcp + (o)) 473 #define cpu_cdstp(o) (cdstp + (o)) 474 #endif 475 vaddr_t memhook; /* used by mem.c & others */ 476 kmutex_t memlock __cacheline_aligned; /* used by mem.c & others */ 477 kmutex_t pmap_lock __cacheline_aligned; 478 kmutex_t kpm_lock __cacheline_aligned; 479 extern void *msgbufaddr; 480 int pmap_kmpages; 481 /* 482 * Flag to indicate if pmap_init() has done its thing 483 */ 484 bool pmap_initialized; 485 486 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 487 /* 488 * Virtual end of direct-mapped memory 489 */ 490 vaddr_t pmap_directlimit; 491 #endif 492 493 /* 494 * Misc. locking data structures 495 */ 496 497 static inline void 498 pmap_acquire_pmap_lock(pmap_t pm) 499 { 500 #if defined(MULTIPROCESSOR) && defined(DDB) 501 if (__predict_false(db_onproc != NULL)) 502 return; 503 #endif 504 505 mutex_enter(&pm->pm_lock); 506 } 507 508 static inline void 509 pmap_release_pmap_lock(pmap_t pm) 510 { 511 #if defined(MULTIPROCESSOR) && defined(DDB) 512 if (__predict_false(db_onproc != NULL)) 513 return; 514 #endif 515 mutex_exit(&pm->pm_lock); 516 } 517 518 static inline void 519 pmap_acquire_page_lock(struct vm_page_md *md) 520 { 521 mutex_enter(&pmap_lock); 522 } 523 524 static inline void 525 pmap_release_page_lock(struct vm_page_md *md) 526 { 527 mutex_exit(&pmap_lock); 528 } 529 530 #ifdef DIAGNOSTIC 531 static inline int 532 pmap_page_locked_p(struct vm_page_md *md) 533 { 534 return mutex_owned(&pmap_lock); 535 } 536 #endif 537 538 539 /* 540 * Metadata for L1 translation tables. 541 */ 542 #ifndef ARM_MMU_EXTENDED 543 struct l1_ttable { 544 /* Entry on the L1 Table list */ 545 SLIST_ENTRY(l1_ttable) l1_link; 546 547 /* Entry on the L1 Least Recently Used list */ 548 TAILQ_ENTRY(l1_ttable) l1_lru; 549 550 /* Track how many domains are allocated from this L1 */ 551 volatile u_int l1_domain_use_count; 552 553 /* 554 * A free-list of domain numbers for this L1. 555 * We avoid using ffs() and a bitmap to track domains since ffs() 556 * is slow on ARM. 557 */ 558 uint8_t l1_domain_first; 559 uint8_t l1_domain_free[PMAP_DOMAINS]; 560 561 /* Physical address of this L1 page table */ 562 paddr_t l1_physaddr; 563 564 /* KVA of this L1 page table */ 565 pd_entry_t *l1_kva; 566 }; 567 568 /* 569 * L1 Page Tables are tracked using a Least Recently Used list. 570 * - New L1s are allocated from the HEAD. 571 * - Freed L1s are added to the TAIL. 572 * - Recently accessed L1s (where an 'access' is some change to one of 573 * the userland pmaps which owns this L1) are moved to the TAIL. 574 */ 575 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 576 static kmutex_t l1_lru_lock __cacheline_aligned; 577 578 /* 579 * A list of all L1 tables 580 */ 581 static SLIST_HEAD(, l1_ttable) l1_list; 582 #endif /* ARM_MMU_EXTENDED */ 583 584 /* 585 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 586 * 587 * This is normally 16MB worth L2 page descriptors for any given pmap. 588 * Reference counts are maintained for L2 descriptors so they can be 589 * freed when empty. 590 */ 591 struct l2_bucket { 592 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 593 paddr_t l2b_pa; /* Physical address of same */ 594 u_short l2b_l1slot; /* This L2 table's L1 index */ 595 u_short l2b_occupancy; /* How many active descriptors */ 596 }; 597 598 struct l2_dtable { 599 /* The number of L2 page descriptors allocated to this l2_dtable */ 600 u_int l2_occupancy; 601 602 /* List of L2 page descriptors */ 603 struct l2_bucket l2_bucket[L2_BUCKET_SIZE]; 604 }; 605 606 /* 607 * Given an L1 table index, calculate the corresponding l2_dtable index 608 * and bucket index within the l2_dtable. 609 */ 610 #define L2_BUCKET_XSHIFT (L2_BUCKET_XLOG2 - L1_S_SHIFT) 611 #define L2_BUCKET_XFRAME (~(vaddr_t)0 << L2_BUCKET_XLOG2) 612 #define L2_BUCKET_IDX(l1slot) ((l1slot) >> L2_BUCKET_XSHIFT) 613 #define L2_IDX(l1slot) (L2_BUCKET_IDX(l1slot) >> L2_BUCKET_LOG2) 614 #define L2_BUCKET(l1slot) (L2_BUCKET_IDX(l1slot) & (L2_BUCKET_SIZE - 1)) 615 616 __CTASSERT(0x100000000ULL == ((uint64_t)L2_SIZE * L2_BUCKET_SIZE * L1_S_SIZE)); 617 __CTASSERT(L2_BUCKET_XFRAME == ~(L2_BUCKET_XSIZE-1)); 618 619 /* 620 * Given a virtual address, this macro returns the 621 * virtual address required to drop into the next L2 bucket. 622 */ 623 #define L2_NEXT_BUCKET_VA(va) (((va) & L2_BUCKET_XFRAME) + L2_BUCKET_XSIZE) 624 625 /* 626 * L2 allocation. 627 */ 628 #define pmap_alloc_l2_dtable() \ 629 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 630 #define pmap_free_l2_dtable(l2) \ 631 pool_cache_put(&pmap_l2dtable_cache, (l2)) 632 #define pmap_alloc_l2_ptp(pap) \ 633 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 634 PR_NOWAIT, (pap))) 635 636 /* 637 * We try to map the page tables write-through, if possible. However, not 638 * all CPUs have a write-through cache mode, so on those we have to sync 639 * the cache when we frob page tables. 640 * 641 * We try to evaluate this at compile time, if possible. However, it's 642 * not always possible to do that, hence this run-time var. 643 */ 644 int pmap_needs_pte_sync; 645 646 /* 647 * Real definition of pv_entry. 648 */ 649 struct pv_entry { 650 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 651 pmap_t pv_pmap; /* pmap where mapping lies */ 652 vaddr_t pv_va; /* virtual address for mapping */ 653 u_int pv_flags; /* flags */ 654 }; 655 656 /* 657 * Macros to determine if a mapping might be resident in the 658 * instruction/data cache and/or TLB 659 */ 660 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED) 661 /* 662 * Speculative loads by Cortex cores can cause TLB entries to be filled even if 663 * there are no explicit accesses, so there may be always be TLB entries to 664 * flush. If we used ASIDs then this would not be a problem. 665 */ 666 #define PV_BEEN_EXECD(f) (((f) & PVF_EXEC) == PVF_EXEC) 667 #define PV_BEEN_REFD(f) (true) 668 #else 669 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 670 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 671 #endif 672 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 673 #define PV_IS_KENTRY_P(f) (((f) & PVF_KENTRY) != 0) 674 #define PV_IS_WRITE_P(f) (((f) & PVF_WRITE) != 0) 675 676 /* 677 * Local prototypes 678 */ 679 static bool pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t, size_t); 680 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 681 pt_entry_t **); 682 static bool pmap_is_current(pmap_t) __unused; 683 static bool pmap_is_cached(pmap_t); 684 static void pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *, 685 pmap_t, vaddr_t, u_int); 686 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t); 687 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 688 static u_int pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t, 689 u_int, u_int); 690 691 static void pmap_pinit(pmap_t); 692 static int pmap_pmap_ctor(void *, void *, int); 693 694 static void pmap_alloc_l1(pmap_t); 695 static void pmap_free_l1(pmap_t); 696 #ifndef ARM_MMU_EXTENDED 697 static void pmap_use_l1(pmap_t); 698 #endif 699 700 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 701 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 702 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 703 static int pmap_l2ptp_ctor(void *, void *, int); 704 static int pmap_l2dtable_ctor(void *, void *, int); 705 706 static void pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 707 #ifdef PMAP_CACHE_VIVT 708 static void pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 709 static void pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 710 #endif 711 712 static void pmap_clearbit(struct vm_page_md *, paddr_t, u_int); 713 #ifdef PMAP_CACHE_VIVT 714 static bool pmap_clean_page(struct vm_page_md *, bool); 715 #endif 716 #ifdef PMAP_CACHE_VIPT 717 static void pmap_syncicache_page(struct vm_page_md *, paddr_t); 718 enum pmap_flush_op { 719 PMAP_FLUSH_PRIMARY, 720 PMAP_FLUSH_SECONDARY, 721 PMAP_CLEAN_PRIMARY 722 }; 723 #ifndef ARM_MMU_EXTENDED 724 static void pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op); 725 #endif 726 #endif 727 static void pmap_page_remove(struct vm_page_md *, paddr_t); 728 static void pmap_pv_remove(paddr_t); 729 730 #ifndef ARM_MMU_EXTENDED 731 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 732 #endif 733 static vaddr_t kernel_pt_lookup(paddr_t); 734 735 #ifdef ARM_MMU_EXTENDED 736 static struct pool_cache pmap_l1tt_cache; 737 738 static int pmap_l1tt_ctor(void *, void *, int); 739 static void * pmap_l1tt_alloc(struct pool *, int); 740 static void pmap_l1tt_free(struct pool *, void *); 741 742 static struct pool_allocator pmap_l1tt_allocator = { 743 .pa_alloc = pmap_l1tt_alloc, 744 .pa_free = pmap_l1tt_free, 745 .pa_pagesz = L1TT_SIZE, 746 }; 747 #endif 748 749 /* 750 * Misc variables 751 */ 752 vaddr_t virtual_avail; 753 vaddr_t virtual_end; 754 vaddr_t pmap_curmaxkvaddr; 755 756 paddr_t avail_start; 757 paddr_t avail_end; 758 759 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 760 pv_addr_t kernelpages; 761 pv_addr_t kernel_l1pt; 762 pv_addr_t systempage; 763 764 #ifdef PMAP_CACHE_VIPT 765 #define PMAP_VALIDATE_MD_PAGE(md) \ 766 KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \ 767 "(md) %p: attrs=%#x urw=%u krw=%u", (md), \ 768 (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings); 769 #endif /* PMAP_CACHE_VIPT */ 770 /* 771 * A bunch of routines to conditionally flush the caches/TLB depending 772 * on whether the specified pmap actually needs to be flushed at any 773 * given time. 774 */ 775 static inline void 776 pmap_tlb_flush_SE(pmap_t pm, vaddr_t va, u_int flags) 777 { 778 #ifdef ARM_MMU_EXTENDED 779 pmap_tlb_invalidate_addr(pm, va); 780 #else 781 if (pm->pm_cstate.cs_tlb_id != 0) { 782 if (PV_BEEN_EXECD(flags)) { 783 cpu_tlb_flushID_SE(va); 784 } else if (PV_BEEN_REFD(flags)) { 785 cpu_tlb_flushD_SE(va); 786 } 787 } 788 #endif /* ARM_MMU_EXTENDED */ 789 } 790 791 #ifndef ARM_MMU_EXTENDED 792 static inline void 793 pmap_tlb_flushID(pmap_t pm) 794 { 795 if (pm->pm_cstate.cs_tlb_id) { 796 cpu_tlb_flushID(); 797 #if ARM_MMU_V7 == 0 798 /* 799 * Speculative loads by Cortex cores can cause TLB entries to 800 * be filled even if there are no explicit accesses, so there 801 * may be always be TLB entries to flush. If we used ASIDs 802 * then it would not be a problem. 803 * This is not true for other CPUs. 804 */ 805 pm->pm_cstate.cs_tlb = 0; 806 #endif /* ARM_MMU_V7 */ 807 } 808 } 809 810 static inline void 811 pmap_tlb_flushD(pmap_t pm) 812 { 813 if (pm->pm_cstate.cs_tlb_d) { 814 cpu_tlb_flushD(); 815 #if ARM_MMU_V7 == 0 816 /* 817 * Speculative loads by Cortex cores can cause TLB entries to 818 * be filled even if there are no explicit accesses, so there 819 * may be always be TLB entries to flush. If we used ASIDs 820 * then it would not be a problem. 821 * This is not true for other CPUs. 822 */ 823 pm->pm_cstate.cs_tlb_d = 0; 824 #endif /* ARM_MMU_V7 */ 825 } 826 } 827 #endif /* ARM_MMU_EXTENDED */ 828 829 #ifdef PMAP_CACHE_VIVT 830 static inline void 831 pmap_cache_wbinv_page(pmap_t pm, vaddr_t va, bool do_inv, u_int flags) 832 { 833 if (PV_BEEN_EXECD(flags) && pm->pm_cstate.cs_cache_id) { 834 cpu_idcache_wbinv_range(va, PAGE_SIZE); 835 } else if (PV_BEEN_REFD(flags) && pm->pm_cstate.cs_cache_d) { 836 if (do_inv) { 837 if (flags & PVF_WRITE) 838 cpu_dcache_wbinv_range(va, PAGE_SIZE); 839 else 840 cpu_dcache_inv_range(va, PAGE_SIZE); 841 } else if (flags & PVF_WRITE) { 842 cpu_dcache_wb_range(va, PAGE_SIZE); 843 } 844 } 845 } 846 847 static inline void 848 pmap_cache_wbinv_all(pmap_t pm, u_int flags) 849 { 850 if (PV_BEEN_EXECD(flags)) { 851 if (pm->pm_cstate.cs_cache_id) { 852 cpu_idcache_wbinv_all(); 853 pm->pm_cstate.cs_cache = 0; 854 } 855 } else if (pm->pm_cstate.cs_cache_d) { 856 cpu_dcache_wbinv_all(); 857 pm->pm_cstate.cs_cache_d = 0; 858 } 859 } 860 #endif /* PMAP_CACHE_VIVT */ 861 862 static inline uint8_t 863 pmap_domain(pmap_t pm) 864 { 865 #ifdef ARM_MMU_EXTENDED 866 return pm == pmap_kernel() ? PMAP_DOMAIN_KERNEL : PMAP_DOMAIN_USER; 867 #else 868 return pm->pm_domain; 869 #endif 870 } 871 872 static inline pd_entry_t * 873 pmap_l1_kva(pmap_t pm) 874 { 875 #ifdef ARM_MMU_EXTENDED 876 return pm->pm_l1; 877 #else 878 return pm->pm_l1->l1_kva; 879 #endif 880 } 881 882 static inline bool 883 pmap_is_current(pmap_t pm) 884 { 885 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 886 return true; 887 888 return false; 889 } 890 891 static inline bool 892 pmap_is_cached(pmap_t pm) 893 { 894 #ifdef ARM_MMU_EXTENDED 895 if (pm == pmap_kernel()) 896 return true; 897 #ifdef MULTIPROCESSOR 898 // Is this pmap active on any CPU? 899 if (!kcpuset_iszero(pm->pm_active)) 900 return true; 901 #else 902 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 903 // Is this pmap active? 904 if (PMAP_PAI_ASIDVALID_P(PMAP_PAI(pm, ti), ti)) 905 return true; 906 #endif 907 #else 908 struct cpu_info * const ci = curcpu(); 909 if (pm == pmap_kernel() || ci->ci_pmap_lastuser == NULL 910 || ci->ci_pmap_lastuser == pm) 911 return true; 912 #endif /* ARM_MMU_EXTENDED */ 913 914 return false; 915 } 916 917 /* 918 * PTE_SYNC_CURRENT: 919 * 920 * Make sure the pte is written out to RAM. 921 * We need to do this for one of two cases: 922 * - We're dealing with the kernel pmap 923 * - There is no pmap active in the cache/tlb. 924 * - The specified pmap is 'active' in the cache/tlb. 925 */ 926 927 #ifdef PMAP_INCLUDE_PTE_SYNC 928 static inline void 929 pmap_pte_sync_current(pmap_t pm, pt_entry_t *ptep) 930 { 931 if (PMAP_NEEDS_PTE_SYNC && pmap_is_cached(pm)) 932 PTE_SYNC(ptep); 933 arm_dsb(); 934 } 935 936 # define PTE_SYNC_CURRENT(pm, ptep) pmap_pte_sync_current(pm, ptep) 937 #else 938 # define PTE_SYNC_CURRENT(pm, ptep) __nothing 939 #endif 940 941 /* 942 * main pv_entry manipulation functions: 943 * pmap_enter_pv: enter a mapping onto a vm_page list 944 * pmap_remove_pv: remove a mapping from a vm_page list 945 * 946 * NOTE: pmap_enter_pv expects to lock the pvh itself 947 * pmap_remove_pv expects the caller to lock the pvh before calling 948 */ 949 950 /* 951 * pmap_enter_pv: enter a mapping onto a vm_page lst 952 * 953 * => caller should hold the proper lock on pmap_main_lock 954 * => caller should have pmap locked 955 * => we will gain the lock on the vm_page and allocate the new pv_entry 956 * => caller should adjust ptp's wire_count before calling 957 * => caller should not adjust pmap's wire_count 958 */ 959 static void 960 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm, 961 vaddr_t va, u_int flags) 962 { 963 UVMHIST_FUNC(__func__); 964 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 965 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 966 UVMHIST_LOG(maphist, "...pv %#jx flags %#jx", 967 (uintptr_t)pv, flags, 0, 0); 968 969 struct pv_entry **pvp; 970 971 pv->pv_pmap = pm; 972 pv->pv_va = va; 973 pv->pv_flags = flags; 974 975 pvp = &SLIST_FIRST(&md->pvh_list); 976 #ifdef PMAP_CACHE_VIPT 977 /* 978 * Insert unmanaged entries, writeable first, at the head of 979 * the pv list. 980 */ 981 if (__predict_true(!PV_IS_KENTRY_P(flags))) { 982 while (*pvp != NULL && PV_IS_KENTRY_P((*pvp)->pv_flags)) 983 pvp = &SLIST_NEXT(*pvp, pv_link); 984 } 985 if (!PV_IS_WRITE_P(flags)) { 986 while (*pvp != NULL && PV_IS_WRITE_P((*pvp)->pv_flags)) 987 pvp = &SLIST_NEXT(*pvp, pv_link); 988 } 989 #endif 990 SLIST_NEXT(pv, pv_link) = *pvp; /* add to ... */ 991 *pvp = pv; /* ... locked list */ 992 md->pvh_attrs |= flags & (PVF_REF | PVF_MOD); 993 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 994 if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE) 995 md->pvh_attrs |= PVF_KMOD; 996 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 997 md->pvh_attrs |= PVF_DIRTY; 998 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 999 #endif 1000 if (pm == pmap_kernel()) { 1001 PMAPCOUNT(kernel_mappings); 1002 if (flags & PVF_WRITE) 1003 md->krw_mappings++; 1004 else 1005 md->kro_mappings++; 1006 } else { 1007 if (flags & PVF_WRITE) 1008 md->urw_mappings++; 1009 else 1010 md->uro_mappings++; 1011 } 1012 1013 #ifdef PMAP_CACHE_VIPT 1014 #ifndef ARM_MMU_EXTENDED 1015 /* 1016 * Even though pmap_vac_me_harder will set PVF_WRITE for us, 1017 * do it here as well to keep the mappings & KVF_WRITE consistent. 1018 */ 1019 if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) { 1020 md->pvh_attrs |= PVF_WRITE; 1021 } 1022 #endif 1023 /* 1024 * If this is an exec mapping and its the first exec mapping 1025 * for this page, make sure to sync the I-cache. 1026 */ 1027 if (PV_IS_EXEC_P(flags)) { 1028 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 1029 pmap_syncicache_page(md, pa); 1030 PMAPCOUNT(exec_synced_map); 1031 } 1032 PMAPCOUNT(exec_mappings); 1033 } 1034 #endif 1035 1036 PMAPCOUNT(mappings); 1037 1038 if (pv->pv_flags & PVF_WIRED) 1039 ++pm->pm_stats.wired_count; 1040 } 1041 1042 /* 1043 * 1044 * pmap_find_pv: Find a pv entry 1045 * 1046 * => caller should hold lock on vm_page 1047 */ 1048 static inline struct pv_entry * 1049 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va) 1050 { 1051 struct pv_entry *pv; 1052 1053 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1054 if (pm == pv->pv_pmap && va == pv->pv_va) 1055 break; 1056 } 1057 1058 return pv; 1059 } 1060 1061 /* 1062 * pmap_remove_pv: try to remove a mapping from a pv_list 1063 * 1064 * => caller should hold proper lock on pmap_main_lock 1065 * => pmap should be locked 1066 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1067 * => caller should adjust ptp's wire_count and free PTP if needed 1068 * => caller should NOT adjust pmap's wire_count 1069 * => we return the removed pv 1070 */ 1071 static struct pv_entry * 1072 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1073 { 1074 UVMHIST_FUNC(__func__); 1075 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 1076 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 1077 1078 struct pv_entry *pv, **prevptr; 1079 1080 prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */ 1081 pv = *prevptr; 1082 1083 while (pv) { 1084 if (pv->pv_pmap == pm && pv->pv_va == va) { /* match? */ 1085 UVMHIST_LOG(maphist, "pm %#jx md %#jx flags %#jx", 1086 (uintptr_t)pm, (uintptr_t)md, pv->pv_flags, 0); 1087 if (pv->pv_flags & PVF_WIRED) { 1088 --pm->pm_stats.wired_count; 1089 } 1090 *prevptr = SLIST_NEXT(pv, pv_link); /* remove it! */ 1091 if (pm == pmap_kernel()) { 1092 PMAPCOUNT(kernel_unmappings); 1093 if (pv->pv_flags & PVF_WRITE) 1094 md->krw_mappings--; 1095 else 1096 md->kro_mappings--; 1097 } else { 1098 if (pv->pv_flags & PVF_WRITE) 1099 md->urw_mappings--; 1100 else 1101 md->uro_mappings--; 1102 } 1103 1104 PMAPCOUNT(unmappings); 1105 #ifdef PMAP_CACHE_VIPT 1106 /* 1107 * If this page has had an exec mapping, then if 1108 * this was the last mapping, discard the contents, 1109 * otherwise sync the i-cache for this page. 1110 */ 1111 if (PV_IS_EXEC_P(md->pvh_attrs)) { 1112 if (SLIST_EMPTY(&md->pvh_list)) { 1113 md->pvh_attrs &= ~PVF_EXEC; 1114 PMAPCOUNT(exec_discarded_unmap); 1115 } else if (pv->pv_flags & PVF_WRITE) { 1116 pmap_syncicache_page(md, pa); 1117 PMAPCOUNT(exec_synced_unmap); 1118 } 1119 } 1120 #endif /* PMAP_CACHE_VIPT */ 1121 break; 1122 } 1123 prevptr = &SLIST_NEXT(pv, pv_link); /* previous pointer */ 1124 pv = *prevptr; /* advance */ 1125 } 1126 1127 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1128 /* 1129 * If we no longer have a WRITEABLE KENTRY at the head of list, 1130 * clear the KMOD attribute from the page. 1131 */ 1132 if (SLIST_FIRST(&md->pvh_list) == NULL 1133 || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE) 1134 md->pvh_attrs &= ~PVF_KMOD; 1135 1136 /* 1137 * If this was a writeable page and there are no more writeable 1138 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1139 * the contents to memory. 1140 */ 1141 if (arm_cache_prefer_mask != 0) { 1142 if (md->krw_mappings + md->urw_mappings == 0) 1143 md->pvh_attrs &= ~PVF_WRITE; 1144 PMAP_VALIDATE_MD_PAGE(md); 1145 } 1146 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1147 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1148 1149 /* return removed pv */ 1150 return pv; 1151 } 1152 1153 /* 1154 * 1155 * pmap_modify_pv: Update pv flags 1156 * 1157 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1158 * => caller should NOT adjust pmap's wire_count 1159 * => caller must call pmap_vac_me_harder() if writable status of a page 1160 * may have changed. 1161 * => we return the old flags 1162 * 1163 * Modify a physical-virtual mapping in the pv table 1164 */ 1165 static u_int 1166 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va, 1167 u_int clr_mask, u_int set_mask) 1168 { 1169 struct pv_entry *npv; 1170 u_int flags, oflags; 1171 UVMHIST_FUNC(__func__); 1172 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 1173 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 1174 UVMHIST_LOG(maphist, "... clr %#jx set %#jx", clr_mask, set_mask, 0, 0); 1175 1176 KASSERT(!PV_IS_KENTRY_P(clr_mask)); 1177 KASSERT(!PV_IS_KENTRY_P(set_mask)); 1178 1179 if ((npv = pmap_find_pv(md, pm, va)) == NULL) { 1180 UVMHIST_LOG(maphist, "<--- done (not found)", 0, 0, 0, 0); 1181 return 0; 1182 } 1183 1184 /* 1185 * There is at least one VA mapping this page. 1186 */ 1187 1188 if (clr_mask & (PVF_REF | PVF_MOD)) { 1189 md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1190 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1191 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1192 md->pvh_attrs |= PVF_DIRTY; 1193 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1194 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1195 } 1196 1197 oflags = npv->pv_flags; 1198 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1199 1200 if ((flags ^ oflags) & PVF_WIRED) { 1201 if (flags & PVF_WIRED) 1202 ++pm->pm_stats.wired_count; 1203 else 1204 --pm->pm_stats.wired_count; 1205 } 1206 1207 if ((flags ^ oflags) & PVF_WRITE) { 1208 if (pm == pmap_kernel()) { 1209 if (flags & PVF_WRITE) { 1210 md->krw_mappings++; 1211 md->kro_mappings--; 1212 } else { 1213 md->kro_mappings++; 1214 md->krw_mappings--; 1215 } 1216 } else { 1217 if (flags & PVF_WRITE) { 1218 md->urw_mappings++; 1219 md->uro_mappings--; 1220 } else { 1221 md->uro_mappings++; 1222 md->urw_mappings--; 1223 } 1224 } 1225 } 1226 #ifdef PMAP_CACHE_VIPT 1227 if (arm_cache_prefer_mask != 0) { 1228 if (md->urw_mappings + md->krw_mappings == 0) { 1229 md->pvh_attrs &= ~PVF_WRITE; 1230 } else { 1231 md->pvh_attrs |= PVF_WRITE; 1232 } 1233 } 1234 /* 1235 * We have two cases here: the first is from enter_pv (new exec 1236 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1237 * Since in latter, pmap_enter_pv won't do anything, we just have 1238 * to do what pmap_remove_pv would do. 1239 */ 1240 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs)) 1241 || (PV_IS_EXEC_P(md->pvh_attrs) 1242 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1243 pmap_syncicache_page(md, pa); 1244 PMAPCOUNT(exec_synced_remap); 1245 } 1246 #ifndef ARM_MMU_EXTENDED 1247 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1248 #endif /* !ARM_MMU_EXTENDED */ 1249 #endif /* PMAP_CACHE_VIPT */ 1250 1251 PMAPCOUNT(remappings); 1252 1253 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1254 1255 return oflags; 1256 } 1257 1258 1259 #if defined(ARM_MMU_EXTENDED) 1260 int 1261 pmap_maxproc_set(int nmaxproc) 1262 { 1263 static const char pmap_l1ttpool_warnmsg[] = 1264 "WARNING: l1ttpool limit reached; increase kern.maxproc"; 1265 1266 pool_cache_prime(&pmap_l1tt_cache, nmaxproc); 1267 1268 /* 1269 * Set the hard limit on the pmap_l1tt_cache to the number 1270 * of processes the kernel is to support. Log the limit 1271 * reached message max once a minute. 1272 */ 1273 pool_cache_sethardlimit(&pmap_l1tt_cache, nmaxproc, 1274 pmap_l1ttpool_warnmsg, 60); 1275 1276 return 0; 1277 } 1278 1279 #endif 1280 1281 /* 1282 * Allocate an L1 translation table for the specified pmap. 1283 * This is called at pmap creation time. 1284 */ 1285 static void 1286 pmap_alloc_l1(pmap_t pm) 1287 { 1288 #ifdef ARM_MMU_EXTENDED 1289 vaddr_t va = (vaddr_t)pool_cache_get_paddr(&pmap_l1tt_cache, PR_WAITOK, 1290 &pm->pm_l1_pa); 1291 1292 pm->pm_l1 = (pd_entry_t *)va; 1293 PTE_SYNC_RANGE(pm->pm_l1, L1TT_SIZE / sizeof(pt_entry_t)); 1294 #else 1295 struct l1_ttable *l1; 1296 uint8_t domain; 1297 1298 /* 1299 * Remove the L1 at the head of the LRU list 1300 */ 1301 mutex_spin_enter(&l1_lru_lock); 1302 l1 = TAILQ_FIRST(&l1_lru_list); 1303 KDASSERT(l1 != NULL); 1304 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1305 1306 /* 1307 * Pick the first available domain number, and update 1308 * the link to the next number. 1309 */ 1310 domain = l1->l1_domain_first; 1311 l1->l1_domain_first = l1->l1_domain_free[domain]; 1312 1313 /* 1314 * If there are still free domain numbers in this L1, 1315 * put it back on the TAIL of the LRU list. 1316 */ 1317 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1318 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1319 1320 mutex_spin_exit(&l1_lru_lock); 1321 1322 /* 1323 * Fix up the relevant bits in the pmap structure 1324 */ 1325 pm->pm_l1 = l1; 1326 pm->pm_domain = domain + 1; 1327 #endif 1328 } 1329 1330 /* 1331 * Free an L1 translation table. 1332 * This is called at pmap destruction time. 1333 */ 1334 static void 1335 pmap_free_l1(pmap_t pm) 1336 { 1337 #ifdef ARM_MMU_EXTENDED 1338 pool_cache_put_paddr(&pmap_l1tt_cache, (void *)pm->pm_l1, pm->pm_l1_pa); 1339 1340 pm->pm_l1 = NULL; 1341 pm->pm_l1_pa = 0; 1342 #else 1343 struct l1_ttable *l1 = pm->pm_l1; 1344 1345 mutex_spin_enter(&l1_lru_lock); 1346 1347 /* 1348 * If this L1 is currently on the LRU list, remove it. 1349 */ 1350 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1351 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1352 1353 /* 1354 * Free up the domain number which was allocated to the pmap 1355 */ 1356 l1->l1_domain_free[pmap_domain(pm) - 1] = l1->l1_domain_first; 1357 l1->l1_domain_first = pmap_domain(pm) - 1; 1358 l1->l1_domain_use_count--; 1359 1360 /* 1361 * The L1 now must have at least 1 free domain, so add 1362 * it back to the LRU list. If the use count is zero, 1363 * put it at the head of the list, otherwise it goes 1364 * to the tail. 1365 */ 1366 if (l1->l1_domain_use_count == 0) 1367 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1368 else 1369 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1370 1371 mutex_spin_exit(&l1_lru_lock); 1372 #endif /* ARM_MMU_EXTENDED */ 1373 } 1374 1375 #ifndef ARM_MMU_EXTENDED 1376 static inline void 1377 pmap_use_l1(pmap_t pm) 1378 { 1379 struct l1_ttable *l1; 1380 1381 /* 1382 * Do nothing if we're in interrupt context. 1383 * Access to an L1 by the kernel pmap must not affect 1384 * the LRU list. 1385 */ 1386 if (cpu_intr_p() || pm == pmap_kernel()) 1387 return; 1388 1389 l1 = pm->pm_l1; 1390 1391 /* 1392 * If the L1 is not currently on the LRU list, just return 1393 */ 1394 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1395 return; 1396 1397 mutex_spin_enter(&l1_lru_lock); 1398 1399 /* 1400 * Check the use count again, now that we've acquired the lock 1401 */ 1402 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1403 mutex_spin_exit(&l1_lru_lock); 1404 return; 1405 } 1406 1407 /* 1408 * Move the L1 to the back of the LRU list 1409 */ 1410 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1411 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1412 1413 mutex_spin_exit(&l1_lru_lock); 1414 } 1415 #endif /* !ARM_MMU_EXTENDED */ 1416 1417 /* 1418 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1419 * 1420 * Free an L2 descriptor table. 1421 */ 1422 static inline void 1423 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1424 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1425 #else 1426 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1427 #endif 1428 { 1429 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1430 /* 1431 * Note: With a write-back cache, we may need to sync this 1432 * L2 table before re-using it. 1433 * This is because it may have belonged to a non-current 1434 * pmap, in which case the cache syncs would have been 1435 * skipped for the pages that were being unmapped. If the 1436 * L2 table were then to be immediately re-allocated to 1437 * the *current* pmap, it may well contain stale mappings 1438 * which have not yet been cleared by a cache write-back 1439 * and so would still be visible to the mmu. 1440 */ 1441 if (need_sync) 1442 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1443 #endif /* PMAP_INCLUDE_PTE_SYNC && PMAP_CACHE_VIVT */ 1444 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1445 } 1446 1447 /* 1448 * Returns a pointer to the L2 bucket associated with the specified pmap 1449 * and VA, or NULL if no L2 bucket exists for the address. 1450 */ 1451 static inline struct l2_bucket * 1452 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1453 { 1454 const size_t l1slot = l1pte_index(va); 1455 struct l2_dtable *l2; 1456 struct l2_bucket *l2b; 1457 1458 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL || 1459 (l2b = &l2->l2_bucket[L2_BUCKET(l1slot)])->l2b_kva == NULL) 1460 return NULL; 1461 1462 return l2b; 1463 } 1464 1465 /* 1466 * Returns a pointer to the L2 bucket associated with the specified pmap 1467 * and VA. 1468 * 1469 * If no L2 bucket exists, perform the necessary allocations to put an L2 1470 * bucket/page table in place. 1471 * 1472 * Note that if a new L2 bucket/page was allocated, the caller *must* 1473 * increment the bucket occupancy counter appropriately *before* 1474 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1475 * the bucket/page in the meantime. 1476 */ 1477 static struct l2_bucket * 1478 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1479 { 1480 const size_t l1slot = l1pte_index(va); 1481 struct l2_dtable *l2; 1482 1483 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 1484 /* 1485 * No mapping at this address, as there is 1486 * no entry in the L1 table. 1487 * Need to allocate a new l2_dtable. 1488 */ 1489 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1490 return NULL; 1491 1492 /* 1493 * Link it into the parent pmap 1494 */ 1495 pm->pm_l2[L2_IDX(l1slot)] = l2; 1496 } 1497 1498 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 1499 1500 /* 1501 * Fetch pointer to the L2 page table associated with the address. 1502 */ 1503 if (l2b->l2b_kva == NULL) { 1504 pt_entry_t *ptep; 1505 1506 /* 1507 * No L2 page table has been allocated. Chances are, this 1508 * is because we just allocated the l2_dtable, above. 1509 */ 1510 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_pa)) == NULL) { 1511 /* 1512 * Oops, no more L2 page tables available at this 1513 * time. We may need to deallocate the l2_dtable 1514 * if we allocated a new one above. 1515 */ 1516 if (l2->l2_occupancy == 0) { 1517 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1518 pmap_free_l2_dtable(l2); 1519 } 1520 return NULL; 1521 } 1522 1523 l2->l2_occupancy++; 1524 l2b->l2b_kva = ptep; 1525 l2b->l2b_l1slot = l1slot; 1526 1527 #ifdef ARM_MMU_EXTENDED 1528 /* 1529 * We know there will be a mapping here, so simply 1530 * enter this PTP into the L1 now. 1531 */ 1532 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1533 pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 1534 | L1_C_DOM(pmap_domain(pm)); 1535 KASSERT(*pdep == 0); 1536 l1pte_setone(pdep, npde); 1537 PDE_SYNC(pdep); 1538 #endif 1539 } 1540 1541 return l2b; 1542 } 1543 1544 /* 1545 * One or more mappings in the specified L2 descriptor table have just been 1546 * invalidated. 1547 * 1548 * Garbage collect the metadata and descriptor table itself if necessary. 1549 * 1550 * The pmap lock must be acquired when this is called (not necessary 1551 * for the kernel pmap). 1552 */ 1553 static void 1554 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1555 { 1556 KDASSERT(count <= l2b->l2b_occupancy); 1557 1558 /* 1559 * Update the bucket's reference count according to how many 1560 * PTEs the caller has just invalidated. 1561 */ 1562 l2b->l2b_occupancy -= count; 1563 1564 /* 1565 * Note: 1566 * 1567 * Level 2 page tables allocated to the kernel pmap are never freed 1568 * as that would require checking all Level 1 page tables and 1569 * removing any references to the Level 2 page table. See also the 1570 * comment elsewhere about never freeing bootstrap L2 descriptors. 1571 * 1572 * We make do with just invalidating the mapping in the L2 table. 1573 * 1574 * This isn't really a big deal in practice and, in fact, leads 1575 * to a performance win over time as we don't need to continually 1576 * alloc/free. 1577 */ 1578 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1579 return; 1580 1581 /* 1582 * There are no more valid mappings in this level 2 page table. 1583 * Go ahead and NULL-out the pointer in the bucket, then 1584 * free the page table. 1585 */ 1586 const size_t l1slot = l2b->l2b_l1slot; 1587 pt_entry_t * const ptep = l2b->l2b_kva; 1588 l2b->l2b_kva = NULL; 1589 1590 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1591 pd_entry_t pde __diagused = *pdep; 1592 1593 #ifdef ARM_MMU_EXTENDED 1594 /* 1595 * Invalidate the L1 slot. 1596 */ 1597 KASSERT((pde & L1_TYPE_MASK) == L1_TYPE_C); 1598 #else 1599 /* 1600 * If the L1 slot matches the pmap's domain number, then invalidate it. 1601 */ 1602 if ((pde & (L1_C_DOM_MASK|L1_TYPE_MASK)) 1603 == (L1_C_DOM(pmap_domain(pm))|L1_TYPE_C)) { 1604 #endif 1605 l1pte_setone(pdep, 0); 1606 PDE_SYNC(pdep); 1607 #ifndef ARM_MMU_EXTENDED 1608 } 1609 #endif 1610 1611 /* 1612 * Release the L2 descriptor table back to the pool cache. 1613 */ 1614 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1615 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_pa); 1616 #else 1617 pmap_free_l2_ptp(ptep, l2b->l2b_pa); 1618 #endif 1619 1620 /* 1621 * Update the reference count in the associated l2_dtable 1622 */ 1623 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 1624 if (--l2->l2_occupancy > 0) 1625 return; 1626 1627 /* 1628 * There are no more valid mappings in any of the Level 1 1629 * slots managed by this l2_dtable. Go ahead and NULL-out 1630 * the pointer in the parent pmap and free the l2_dtable. 1631 */ 1632 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1633 pmap_free_l2_dtable(l2); 1634 } 1635 1636 #if defined(ARM_MMU_EXTENDED) 1637 /* 1638 * Pool cache constructors for L1 translation tables 1639 */ 1640 1641 static int 1642 pmap_l1tt_ctor(void *arg, void *v, int flags) 1643 { 1644 #ifndef PMAP_INCLUDE_PTE_SYNC 1645 #error not supported 1646 #endif 1647 1648 memset(v, 0, L1TT_SIZE); 1649 PTE_SYNC_RANGE(v, L1TT_SIZE / sizeof(pt_entry_t)); 1650 return 0; 1651 } 1652 #endif 1653 1654 /* 1655 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1656 * structures. 1657 */ 1658 static int 1659 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1660 { 1661 #ifndef PMAP_INCLUDE_PTE_SYNC 1662 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1663 1664 /* 1665 * The mappings for these page tables were initially made using 1666 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1667 * mode will not be right for page table mappings. To avoid 1668 * polluting the pmap_kenter_pa() code with a special case for 1669 * page tables, we simply fix up the cache-mode here if it's not 1670 * correct. 1671 */ 1672 if (pte_l2_s_cache_mode != pte_l2_s_cache_mode_pt) { 1673 const struct l2_bucket * const l2b = 1674 pmap_get_l2_bucket(pmap_kernel(), va); 1675 KASSERTMSG(l2b != NULL, "%#lx", va); 1676 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 1677 const pt_entry_t opte = *ptep; 1678 1679 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1680 /* 1681 * Page tables must have the cache-mode set correctly. 1682 */ 1683 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1684 | pte_l2_s_cache_mode_pt; 1685 l2pte_set(ptep, npte, opte); 1686 PTE_SYNC(ptep); 1687 cpu_tlb_flushD_SE(va); 1688 cpu_cpwait(); 1689 } 1690 } 1691 #endif 1692 1693 memset(v, 0, L2_TABLE_SIZE_REAL); 1694 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1695 return 0; 1696 } 1697 1698 static int 1699 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1700 { 1701 1702 memset(v, 0, sizeof(struct l2_dtable)); 1703 return 0; 1704 } 1705 1706 static int 1707 pmap_pmap_ctor(void *arg, void *v, int flags) 1708 { 1709 1710 memset(v, 0, sizeof(struct pmap)); 1711 return 0; 1712 } 1713 1714 static void 1715 pmap_pinit(pmap_t pm) 1716 { 1717 #ifndef ARM_HAS_VBAR 1718 struct l2_bucket *l2b; 1719 1720 if (vector_page < KERNEL_BASE) { 1721 /* 1722 * Map the vector page. 1723 */ 1724 pmap_enter(pm, vector_page, systempage.pv_pa, 1725 VM_PROT_READ | VM_PROT_EXECUTE, 1726 VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED); 1727 pmap_update(pm); 1728 1729 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 1730 l2b = pmap_get_l2_bucket(pm, vector_page); 1731 KASSERTMSG(l2b != NULL, "%#lx", vector_page); 1732 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 1733 L1_C_DOM(pmap_domain(pm)); 1734 } else 1735 pm->pm_pl1vec = NULL; 1736 #endif 1737 } 1738 1739 #ifdef PMAP_CACHE_VIVT 1740 /* 1741 * Since we have a virtually indexed cache, we may need to inhibit caching if 1742 * there is more than one mapping and at least one of them is writable. 1743 * Since we purge the cache on every context switch, we only need to check for 1744 * other mappings within the same pmap, or kernel_pmap. 1745 * This function is also called when a page is unmapped, to possibly reenable 1746 * caching on any remaining mappings. 1747 * 1748 * The code implements the following logic, where: 1749 * 1750 * KW = # of kernel read/write pages 1751 * KR = # of kernel read only pages 1752 * UW = # of user read/write pages 1753 * UR = # of user read only pages 1754 * 1755 * KC = kernel mapping is cacheable 1756 * UC = user mapping is cacheable 1757 * 1758 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1759 * +--------------------------------------------- 1760 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1761 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1762 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1763 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1764 */ 1765 1766 static const int pmap_vac_flags[4][4] = { 1767 {-1, 0, 0, PVF_KNC}, 1768 {0, 0, PVF_NC, PVF_NC}, 1769 {0, PVF_NC, PVF_NC, PVF_NC}, 1770 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1771 }; 1772 1773 static inline int 1774 pmap_get_vac_flags(const struct vm_page_md *md) 1775 { 1776 int kidx, uidx; 1777 1778 kidx = 0; 1779 if (md->kro_mappings || md->krw_mappings > 1) 1780 kidx |= 1; 1781 if (md->krw_mappings) 1782 kidx |= 2; 1783 1784 uidx = 0; 1785 if (md->uro_mappings || md->urw_mappings > 1) 1786 uidx |= 1; 1787 if (md->urw_mappings) 1788 uidx |= 2; 1789 1790 return pmap_vac_flags[uidx][kidx]; 1791 } 1792 1793 static inline void 1794 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1795 { 1796 int nattr; 1797 1798 nattr = pmap_get_vac_flags(md); 1799 1800 if (nattr < 0) { 1801 md->pvh_attrs &= ~PVF_NC; 1802 return; 1803 } 1804 1805 if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0) 1806 return; 1807 1808 if (pm == pmap_kernel()) 1809 pmap_vac_me_kpmap(md, pa, pm, va); 1810 else 1811 pmap_vac_me_user(md, pa, pm, va); 1812 1813 md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr; 1814 } 1815 1816 static void 1817 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1818 { 1819 u_int u_cacheable, u_entries; 1820 struct pv_entry *pv; 1821 pmap_t last_pmap = pm; 1822 1823 /* 1824 * Pass one, see if there are both kernel and user pmaps for 1825 * this page. Calculate whether there are user-writable or 1826 * kernel-writable pages. 1827 */ 1828 u_cacheable = 0; 1829 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1830 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1831 u_cacheable++; 1832 } 1833 1834 u_entries = md->urw_mappings + md->uro_mappings; 1835 1836 /* 1837 * We know we have just been updating a kernel entry, so if 1838 * all user pages are already cacheable, then there is nothing 1839 * further to do. 1840 */ 1841 if (md->k_mappings == 0 && u_cacheable == u_entries) 1842 return; 1843 1844 if (u_entries) { 1845 /* 1846 * Scan over the list again, for each entry, if it 1847 * might not be set correctly, call pmap_vac_me_user 1848 * to recalculate the settings. 1849 */ 1850 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1851 /* 1852 * We know kernel mappings will get set 1853 * correctly in other calls. We also know 1854 * that if the pmap is the same as last_pmap 1855 * then we've just handled this entry. 1856 */ 1857 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1858 continue; 1859 1860 /* 1861 * If there are kernel entries and this page 1862 * is writable but non-cacheable, then we can 1863 * skip this entry also. 1864 */ 1865 if (md->k_mappings && 1866 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1867 (PVF_NC | PVF_WRITE)) 1868 continue; 1869 1870 /* 1871 * Similarly if there are no kernel-writable 1872 * entries and the page is already 1873 * read-only/cacheable. 1874 */ 1875 if (md->krw_mappings == 0 && 1876 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1877 continue; 1878 1879 /* 1880 * For some of the remaining cases, we know 1881 * that we must recalculate, but for others we 1882 * can't tell if they are correct or not, so 1883 * we recalculate anyway. 1884 */ 1885 pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0); 1886 } 1887 1888 if (md->k_mappings == 0) 1889 return; 1890 } 1891 1892 pmap_vac_me_user(md, pa, pm, va); 1893 } 1894 1895 static void 1896 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1897 { 1898 pmap_t kpmap = pmap_kernel(); 1899 struct pv_entry *pv, *npv = NULL; 1900 u_int entries = 0; 1901 u_int writable = 0; 1902 u_int cacheable_entries = 0; 1903 u_int kern_cacheable = 0; 1904 u_int other_writable = 0; 1905 1906 /* 1907 * Count mappings and writable mappings in this pmap. 1908 * Include kernel mappings as part of our own. 1909 * Keep a pointer to the first one. 1910 */ 1911 npv = NULL; 1912 KASSERT(pmap_page_locked_p(md)); 1913 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1914 /* Count mappings in the same pmap */ 1915 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1916 if (entries++ == 0) 1917 npv = pv; 1918 1919 /* Cacheable mappings */ 1920 if ((pv->pv_flags & PVF_NC) == 0) { 1921 cacheable_entries++; 1922 if (kpmap == pv->pv_pmap) 1923 kern_cacheable++; 1924 } 1925 1926 /* Writable mappings */ 1927 if (pv->pv_flags & PVF_WRITE) 1928 ++writable; 1929 } else if (pv->pv_flags & PVF_WRITE) 1930 other_writable = 1; 1931 } 1932 1933 /* 1934 * Enable or disable caching as necessary. 1935 * Note: the first entry might be part of the kernel pmap, 1936 * so we can't assume this is indicative of the state of the 1937 * other (maybe non-kpmap) entries. 1938 */ 1939 if ((entries > 1 && writable) || 1940 (entries > 0 && pm == kpmap && other_writable)) { 1941 if (cacheable_entries == 0) { 1942 return; 1943 } 1944 1945 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1946 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1947 (pv->pv_flags & PVF_NC)) 1948 continue; 1949 1950 pv->pv_flags |= PVF_NC; 1951 1952 struct l2_bucket * const l2b 1953 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1954 KASSERTMSG(l2b != NULL, "%#lx", va); 1955 pt_entry_t * const ptep 1956 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1957 const pt_entry_t opte = *ptep; 1958 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 1959 1960 if ((va != pv->pv_va || pm != pv->pv_pmap) 1961 && l2pte_valid_p(opte)) { 1962 pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va, 1963 true, pv->pv_flags); 1964 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1965 pv->pv_flags); 1966 } 1967 1968 l2pte_set(ptep, npte, opte); 1969 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1970 } 1971 cpu_cpwait(); 1972 } else if (entries > cacheable_entries) { 1973 /* 1974 * Turn cacheing back on for some pages. If it is a kernel 1975 * page, only do so if there are no other writable pages. 1976 */ 1977 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1978 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1979 (kpmap != pv->pv_pmap || other_writable))) 1980 continue; 1981 1982 pv->pv_flags &= ~PVF_NC; 1983 1984 struct l2_bucket * const l2b 1985 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1986 KASSERTMSG(l2b != NULL, "%#lx", va); 1987 pt_entry_t * const ptep 1988 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1989 const pt_entry_t opte = *ptep; 1990 pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1991 | pte_l2_s_cache_mode; 1992 1993 if (l2pte_valid_p(opte)) { 1994 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1995 pv->pv_flags); 1996 } 1997 1998 l2pte_set(ptep, npte, opte); 1999 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2000 } 2001 } 2002 } 2003 #endif 2004 2005 #ifdef PMAP_CACHE_VIPT 2006 static void 2007 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 2008 { 2009 2010 #ifndef ARM_MMU_EXTENDED 2011 struct pv_entry *pv; 2012 vaddr_t tst_mask; 2013 bool bad_alias; 2014 const u_int 2015 rw_mappings = md->urw_mappings + md->krw_mappings, 2016 ro_mappings = md->uro_mappings + md->kro_mappings; 2017 2018 /* do we need to do anything? */ 2019 if (arm_cache_prefer_mask == 0) 2020 return; 2021 2022 UVMHIST_FUNC(__func__); 2023 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx pm %#jx va %#jx", 2024 (uintptr_t)md, (uintptr_t)pa, (uintptr_t)pm, va); 2025 2026 KASSERT(!va || pm); 2027 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2028 2029 /* Already a conflict? */ 2030 if (__predict_false(md->pvh_attrs & PVF_NC)) { 2031 /* just an add, things are already non-cached */ 2032 KASSERT(!(md->pvh_attrs & PVF_DIRTY)); 2033 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2034 bad_alias = false; 2035 if (va) { 2036 PMAPCOUNT(vac_color_none); 2037 bad_alias = true; 2038 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2039 goto fixup; 2040 } 2041 pv = SLIST_FIRST(&md->pvh_list); 2042 /* the list can't be empty because it would be cachable */ 2043 if (md->pvh_attrs & PVF_KMPAGE) { 2044 tst_mask = md->pvh_attrs; 2045 } else { 2046 KASSERT(pv); 2047 tst_mask = pv->pv_va; 2048 pv = SLIST_NEXT(pv, pv_link); 2049 } 2050 /* 2051 * Only check for a bad alias if we have writable mappings. 2052 */ 2053 tst_mask &= arm_cache_prefer_mask; 2054 if (rw_mappings > 0) { 2055 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 2056 /* if there's a bad alias, stop checking. */ 2057 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 2058 bad_alias = true; 2059 } 2060 md->pvh_attrs |= PVF_WRITE; 2061 if (!bad_alias) 2062 md->pvh_attrs |= PVF_DIRTY; 2063 } else { 2064 /* 2065 * We have only read-only mappings. Let's see if there 2066 * are multiple colors in use or if we mapped a KMPAGE. 2067 * If the latter, we have a bad alias. If the former, 2068 * we need to remember that. 2069 */ 2070 for (; pv; pv = SLIST_NEXT(pv, pv_link)) { 2071 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) { 2072 if (md->pvh_attrs & PVF_KMPAGE) 2073 bad_alias = true; 2074 break; 2075 } 2076 } 2077 md->pvh_attrs &= ~PVF_WRITE; 2078 /* 2079 * No KMPAGE and we exited early, so we must have 2080 * multiple color mappings. 2081 */ 2082 if (!bad_alias && pv != NULL) 2083 md->pvh_attrs |= PVF_MULTCLR; 2084 } 2085 2086 /* If no conflicting colors, set everything back to cached */ 2087 if (!bad_alias) { 2088 #ifdef DEBUG 2089 if ((md->pvh_attrs & PVF_WRITE) 2090 || ro_mappings < 2) { 2091 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2092 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2093 } 2094 #endif 2095 md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 2096 md->pvh_attrs |= tst_mask | PVF_COLORED; 2097 /* 2098 * Restore DIRTY bit if page is modified 2099 */ 2100 if (md->pvh_attrs & PVF_DMOD) 2101 md->pvh_attrs |= PVF_DIRTY; 2102 PMAPCOUNT(vac_color_restore); 2103 } else { 2104 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2105 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2106 } 2107 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2108 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2109 } else if (!va) { 2110 KASSERT(pmap_is_page_colored_p(md)); 2111 KASSERT(!(md->pvh_attrs & PVF_WRITE) 2112 || (md->pvh_attrs & PVF_DIRTY)); 2113 if (rw_mappings == 0) { 2114 md->pvh_attrs &= ~PVF_WRITE; 2115 if (ro_mappings == 1 2116 && (md->pvh_attrs & PVF_MULTCLR)) { 2117 /* 2118 * If this is the last readonly mapping 2119 * but it doesn't match the current color 2120 * for the page, change the current color 2121 * to match this last readonly mapping. 2122 */ 2123 pv = SLIST_FIRST(&md->pvh_list); 2124 tst_mask = (md->pvh_attrs ^ pv->pv_va) 2125 & arm_cache_prefer_mask; 2126 if (tst_mask) { 2127 md->pvh_attrs ^= tst_mask; 2128 PMAPCOUNT(vac_color_change); 2129 } 2130 } 2131 } 2132 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2133 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2134 return; 2135 } else if (!pmap_is_page_colored_p(md)) { 2136 /* not colored so we just use its color */ 2137 KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 2138 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2139 PMAPCOUNT(vac_color_new); 2140 md->pvh_attrs &= PAGE_SIZE - 1; 2141 md->pvh_attrs |= PVF_COLORED 2142 | (va & arm_cache_prefer_mask) 2143 | (rw_mappings > 0 ? PVF_WRITE : 0); 2144 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2145 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2146 return; 2147 } else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 2148 bad_alias = false; 2149 if (rw_mappings > 0) { 2150 /* 2151 * We now have writeable mappings and if we have 2152 * readonly mappings in more than once color, we have 2153 * an aliasing problem. Regardless mark the page as 2154 * writeable. 2155 */ 2156 if (md->pvh_attrs & PVF_MULTCLR) { 2157 if (ro_mappings < 2) { 2158 /* 2159 * If we only have less than two 2160 * read-only mappings, just flush the 2161 * non-primary colors from the cache. 2162 */ 2163 pmap_flush_page(md, pa, 2164 PMAP_FLUSH_SECONDARY); 2165 } else { 2166 bad_alias = true; 2167 } 2168 } 2169 md->pvh_attrs |= PVF_WRITE; 2170 } 2171 /* If no conflicting colors, set everything back to cached */ 2172 if (!bad_alias) { 2173 #ifdef DEBUG 2174 if (rw_mappings > 0 2175 || (md->pvh_attrs & PMAP_KMPAGE)) { 2176 tst_mask = md->pvh_attrs & arm_cache_prefer_mask; 2177 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2178 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2179 } 2180 #endif 2181 if (SLIST_EMPTY(&md->pvh_list)) 2182 PMAPCOUNT(vac_color_reuse); 2183 else 2184 PMAPCOUNT(vac_color_ok); 2185 2186 /* matching color, just return */ 2187 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2188 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2189 return; 2190 } 2191 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2192 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2193 2194 /* color conflict. evict from cache. */ 2195 2196 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2197 md->pvh_attrs &= ~PVF_COLORED; 2198 md->pvh_attrs |= PVF_NC; 2199 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2200 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2201 PMAPCOUNT(vac_color_erase); 2202 } else if (rw_mappings == 0 2203 && (md->pvh_attrs & PVF_KMPAGE) == 0) { 2204 KASSERT((md->pvh_attrs & PVF_WRITE) == 0); 2205 2206 /* 2207 * If the page has dirty cache lines, clean it. 2208 */ 2209 if (md->pvh_attrs & PVF_DIRTY) 2210 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 2211 2212 /* 2213 * If this is the first remapping (we know that there are no 2214 * writeable mappings), then this is a simple color change. 2215 * Otherwise this is a seconary r/o mapping, which means 2216 * we don't have to do anything. 2217 */ 2218 if (ro_mappings == 1) { 2219 KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2220 md->pvh_attrs &= PAGE_SIZE - 1; 2221 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2222 PMAPCOUNT(vac_color_change); 2223 } else { 2224 PMAPCOUNT(vac_color_blind); 2225 } 2226 md->pvh_attrs |= PVF_MULTCLR; 2227 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2228 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2229 return; 2230 } else { 2231 if (rw_mappings > 0) 2232 md->pvh_attrs |= PVF_WRITE; 2233 2234 /* color conflict. evict from cache. */ 2235 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2236 2237 /* the list can't be empty because this was a enter/modify */ 2238 pv = SLIST_FIRST(&md->pvh_list); 2239 if ((md->pvh_attrs & PVF_KMPAGE) == 0) { 2240 KASSERT(pv); 2241 /* 2242 * If there's only one mapped page, change color to the 2243 * page's new color and return. Restore the DIRTY bit 2244 * that was erased by pmap_flush_page. 2245 */ 2246 if (SLIST_NEXT(pv, pv_link) == NULL) { 2247 md->pvh_attrs &= PAGE_SIZE - 1; 2248 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2249 if (md->pvh_attrs & PVF_DMOD) 2250 md->pvh_attrs |= PVF_DIRTY; 2251 PMAPCOUNT(vac_color_change); 2252 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2253 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2254 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2255 return; 2256 } 2257 } 2258 bad_alias = true; 2259 md->pvh_attrs &= ~PVF_COLORED; 2260 md->pvh_attrs |= PVF_NC; 2261 PMAPCOUNT(vac_color_erase); 2262 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2263 } 2264 2265 fixup: 2266 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2267 2268 /* 2269 * Turn cacheing on/off for all pages. 2270 */ 2271 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2272 struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap, 2273 pv->pv_va); 2274 KASSERTMSG(l2b != NULL, "%#lx", va); 2275 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2276 const pt_entry_t opte = *ptep; 2277 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 2278 if (bad_alias) { 2279 pv->pv_flags |= PVF_NC; 2280 } else { 2281 pv->pv_flags &= ~PVF_NC; 2282 npte |= pte_l2_s_cache_mode; 2283 } 2284 2285 if (opte == npte) /* only update is there's a change */ 2286 continue; 2287 2288 if (l2pte_valid_p(opte)) { 2289 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags); 2290 } 2291 2292 l2pte_set(ptep, npte, opte); 2293 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2294 } 2295 #endif /* !ARM_MMU_EXTENDED */ 2296 } 2297 #endif /* PMAP_CACHE_VIPT */ 2298 2299 2300 /* 2301 * Modify pte bits for all ptes corresponding to the given physical address. 2302 * We use `maskbits' rather than `clearbits' because we're always passing 2303 * constants and the latter would require an extra inversion at run-time. 2304 */ 2305 static void 2306 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits) 2307 { 2308 struct pv_entry *pv; 2309 #ifdef PMAP_CACHE_VIPT 2310 const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs); 2311 bool need_syncicache = false; 2312 #ifdef ARM_MMU_EXTENDED 2313 const u_int execbits = (maskbits & PVF_EXEC) ? L2_XS_XN : 0; 2314 #else 2315 const u_int execbits = 0; 2316 bool need_vac_me_harder = false; 2317 #endif 2318 #else 2319 const u_int execbits = 0; 2320 #endif 2321 2322 UVMHIST_FUNC(__func__); 2323 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx maskbits %#jx", 2324 (uintptr_t)md, pa, maskbits, 0); 2325 2326 #ifdef PMAP_CACHE_VIPT 2327 /* 2328 * If we might want to sync the I-cache and we've modified it, 2329 * then we know we definitely need to sync or discard it. 2330 */ 2331 if (want_syncicache) { 2332 if (md->pvh_attrs & PVF_MOD) { 2333 need_syncicache = true; 2334 } 2335 } 2336 #endif 2337 KASSERT(pmap_page_locked_p(md)); 2338 2339 /* 2340 * Clear saved attributes (modify, reference) 2341 */ 2342 md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2343 2344 if (SLIST_EMPTY(&md->pvh_list)) { 2345 #if defined(PMAP_CACHE_VIPT) 2346 if (need_syncicache) { 2347 /* 2348 * No one has it mapped, so just discard it. The next 2349 * exec remapping will cause it to be synced. 2350 */ 2351 md->pvh_attrs &= ~PVF_EXEC; 2352 PMAPCOUNT(exec_discarded_clearbit); 2353 } 2354 #endif 2355 return; 2356 } 2357 2358 /* 2359 * Loop over all current mappings setting/clearing as appropos 2360 */ 2361 for (pv = SLIST_FIRST(&md->pvh_list); pv != NULL;) { 2362 pmap_t pm = pv->pv_pmap; 2363 const vaddr_t va = pv->pv_va; 2364 const u_int oflags = pv->pv_flags; 2365 #ifndef ARM_MMU_EXTENDED 2366 /* 2367 * Kernel entries are unmanaged and as such not to be changed. 2368 */ 2369 if (PV_IS_KENTRY_P(oflags)) { 2370 pv = SLIST_NEXT(pv, pv_link); 2371 continue; 2372 } 2373 #endif 2374 2375 /* 2376 * Try to get a hold on the pmap's lock. We must do this 2377 * while still holding the page locked, to know that the 2378 * page is still associated with the pmap and the mapping is 2379 * in place. If a hold can't be had, unlock and wait for 2380 * the pmap's lock to become available and retry. The pmap 2381 * must be ref'd over this dance to stop it disappearing 2382 * behind us. 2383 */ 2384 if (!mutex_tryenter(&pm->pm_lock)) { 2385 pmap_reference(pm); 2386 pmap_release_page_lock(md); 2387 pmap_acquire_pmap_lock(pm); 2388 /* nothing, just wait for it */ 2389 pmap_release_pmap_lock(pm); 2390 pmap_destroy(pm); 2391 /* Restart from the beginning. */ 2392 pmap_acquire_page_lock(md); 2393 pv = SLIST_FIRST(&md->pvh_list); 2394 continue; 2395 } 2396 pv->pv_flags &= ~maskbits; 2397 2398 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va); 2399 KASSERTMSG(l2b != NULL, "%#lx", va); 2400 2401 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 2402 const pt_entry_t opte = *ptep; 2403 pt_entry_t npte = opte | execbits; 2404 2405 #ifdef ARM_MMU_EXTENDED 2406 KASSERT((opte & L2_XS_nG) == (pm == pmap_kernel() ? 0 : L2_XS_nG)); 2407 #endif 2408 2409 UVMHIST_LOG(maphist, "pv %#jx pm %#jx va %#jx flag %#jx", 2410 (uintptr_t)pv, (uintptr_t)pm, va, oflags); 2411 2412 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2413 #ifdef PMAP_CACHE_VIVT 2414 if ((oflags & PVF_NC)) { 2415 /* 2416 * Entry is not cacheable: 2417 * 2418 * Don't turn caching on again if this is a 2419 * modified emulation. This would be 2420 * inconsitent with the settings created by 2421 * pmap_vac_me_harder(). Otherwise, it's safe 2422 * to re-enable cacheing. 2423 * 2424 * There's no need to call pmap_vac_me_harder() 2425 * here: all pages are losing their write 2426 * permission. 2427 */ 2428 if (maskbits & PVF_WRITE) { 2429 npte |= pte_l2_s_cache_mode; 2430 pv->pv_flags &= ~PVF_NC; 2431 } 2432 } else if (l2pte_writable_p(opte)) { 2433 /* 2434 * Entry is writable/cacheable: check if pmap 2435 * is current if it is flush it, otherwise it 2436 * won't be in the cache 2437 */ 2438 pmap_cache_wbinv_page(pm, va, 2439 (maskbits & PVF_REF) != 0, 2440 oflags|PVF_WRITE); 2441 } 2442 #endif 2443 2444 /* make the pte read only */ 2445 npte = l2pte_set_readonly(npte); 2446 2447 if ((maskbits & oflags & PVF_WRITE)) { 2448 /* 2449 * Keep alias accounting up to date 2450 */ 2451 if (pm == pmap_kernel()) { 2452 md->krw_mappings--; 2453 md->kro_mappings++; 2454 } else { 2455 md->urw_mappings--; 2456 md->uro_mappings++; 2457 } 2458 #ifdef PMAP_CACHE_VIPT 2459 if (arm_cache_prefer_mask != 0) { 2460 if (md->urw_mappings + md->krw_mappings == 0) { 2461 md->pvh_attrs &= ~PVF_WRITE; 2462 } else { 2463 PMAP_VALIDATE_MD_PAGE(md); 2464 } 2465 } 2466 if (want_syncicache) 2467 need_syncicache = true; 2468 #ifndef ARM_MMU_EXTENDED 2469 need_vac_me_harder = true; 2470 #endif 2471 #endif /* PMAP_CACHE_VIPT */ 2472 } 2473 } 2474 2475 if (maskbits & PVF_REF) { 2476 if (true 2477 #ifndef ARM_MMU_EXTENDED 2478 && (oflags & PVF_NC) == 0 2479 #endif 2480 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0 2481 && l2pte_valid_p(npte)) { 2482 #ifdef PMAP_CACHE_VIVT 2483 /* 2484 * Check npte here; we may have already 2485 * done the wbinv above, and the validity 2486 * of the PTE is the same for opte and 2487 * npte. 2488 */ 2489 pmap_cache_wbinv_page(pm, va, true, oflags); 2490 #endif 2491 } 2492 2493 /* 2494 * Make the PTE invalid so that we will take a 2495 * page fault the next time the mapping is 2496 * referenced. 2497 */ 2498 npte &= ~L2_TYPE_MASK; 2499 npte |= L2_TYPE_INV; 2500 } 2501 2502 if (npte != opte) { 2503 l2pte_reset(ptep); 2504 PTE_SYNC(ptep); 2505 2506 /* Flush the TLB entry if a current pmap. */ 2507 pmap_tlb_flush_SE(pm, va, oflags); 2508 2509 l2pte_set(ptep, npte, 0); 2510 PTE_SYNC(ptep); 2511 } 2512 2513 pmap_release_pmap_lock(pm); 2514 2515 UVMHIST_LOG(maphist, "pm %#jx va %#jx opte %#jx npte %#jx", 2516 (uintptr_t)pm, va, opte, npte); 2517 2518 /* Move to next entry. */ 2519 pv = SLIST_NEXT(pv, pv_link); 2520 } 2521 2522 #if defined(PMAP_CACHE_VIPT) 2523 /* 2524 * If we need to sync the I-cache and we haven't done it yet, do it. 2525 */ 2526 if (need_syncicache) { 2527 pmap_syncicache_page(md, pa); 2528 PMAPCOUNT(exec_synced_clearbit); 2529 } 2530 #ifndef ARM_MMU_EXTENDED 2531 /* 2532 * If we are changing this to read-only, we need to call vac_me_harder 2533 * so we can change all the read-only pages to cacheable. We pretend 2534 * this as a page deletion. 2535 */ 2536 if (need_vac_me_harder) { 2537 if (md->pvh_attrs & PVF_NC) 2538 pmap_vac_me_harder(md, pa, NULL, 0); 2539 } 2540 #endif /* !ARM_MMU_EXTENDED */ 2541 #endif /* PMAP_CACHE_VIPT */ 2542 } 2543 2544 /* 2545 * pmap_clean_page() 2546 * 2547 * This is a local function used to work out the best strategy to clean 2548 * a single page referenced by its entry in the PV table. It's used by 2549 * pmap_copy_page, pmap_zero_page and maybe some others later on. 2550 * 2551 * Its policy is effectively: 2552 * o If there are no mappings, we don't bother doing anything with the cache. 2553 * o If there is one mapping, we clean just that page. 2554 * o If there are multiple mappings, we clean the entire cache. 2555 * 2556 * So that some functions can be further optimised, it returns 0 if it didn't 2557 * clean the entire cache, or 1 if it did. 2558 * 2559 * XXX One bug in this routine is that if the pv_entry has a single page 2560 * mapped at 0x00000000 a whole cache clean will be performed rather than 2561 * just the 1 page. Since this should not occur in everyday use and if it does 2562 * it will just result in not the most efficient clean for the page. 2563 */ 2564 #ifdef PMAP_CACHE_VIVT 2565 static bool 2566 pmap_clean_page(struct vm_page_md *md, bool is_src) 2567 { 2568 struct pv_entry *pv; 2569 pmap_t pm_to_clean = NULL; 2570 bool cache_needs_cleaning = false; 2571 vaddr_t page_to_clean = 0; 2572 u_int flags = 0; 2573 2574 /* 2575 * Since we flush the cache each time we change to a different 2576 * user vmspace, we only need to flush the page if it is in the 2577 * current pmap. 2578 */ 2579 KASSERT(pmap_page_locked_p(md)); 2580 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2581 if (pmap_is_current(pv->pv_pmap)) { 2582 flags |= pv->pv_flags; 2583 /* 2584 * The page is mapped non-cacheable in 2585 * this map. No need to flush the cache. 2586 */ 2587 if (pv->pv_flags & PVF_NC) { 2588 #ifdef DIAGNOSTIC 2589 KASSERT(!cache_needs_cleaning); 2590 #endif 2591 break; 2592 } else if (is_src && (pv->pv_flags & PVF_WRITE) == 0) 2593 continue; 2594 if (cache_needs_cleaning) { 2595 page_to_clean = 0; 2596 break; 2597 } else { 2598 page_to_clean = pv->pv_va; 2599 pm_to_clean = pv->pv_pmap; 2600 } 2601 cache_needs_cleaning = true; 2602 } 2603 } 2604 2605 if (page_to_clean) { 2606 pmap_cache_wbinv_page(pm_to_clean, page_to_clean, 2607 !is_src, flags | PVF_REF); 2608 } else if (cache_needs_cleaning) { 2609 pmap_t const pm = curproc->p_vmspace->vm_map.pmap; 2610 2611 pmap_cache_wbinv_all(pm, flags); 2612 return true; 2613 } 2614 return false; 2615 } 2616 #endif 2617 2618 #ifdef PMAP_CACHE_VIPT 2619 /* 2620 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2621 * right cache alias to make sure we flush the right stuff. 2622 */ 2623 void 2624 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa) 2625 { 2626 pmap_t kpm = pmap_kernel(); 2627 const size_t way_size = arm_pcache.icache_type == CACHE_TYPE_PIPT 2628 ? PAGE_SIZE 2629 : arm_pcache.icache_way_size; 2630 2631 UVMHIST_FUNC(__func__); 2632 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx (attrs=%#jx)", 2633 (uintptr_t)md, pa, md->pvh_attrs, 0); 2634 2635 /* 2636 * No need to clean the page if it's non-cached. 2637 */ 2638 #ifndef ARM_MMU_EXTENDED 2639 if (md->pvh_attrs & PVF_NC) 2640 return; 2641 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED); 2642 #endif 2643 2644 pt_entry_t * const ptep = cpu_cdst_pte(0); 2645 const vaddr_t dstp = cpu_cdstp(0); 2646 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 2647 if (way_size <= PAGE_SIZE) { 2648 bool ok = false; 2649 vaddr_t vdstp = pmap_direct_mapped_phys(pa, &ok, dstp); 2650 if (ok) { 2651 cpu_icache_sync_range(vdstp, way_size); 2652 return; 2653 } 2654 } 2655 #endif 2656 2657 /* 2658 * We don't worry about the color of the exec page, we map the 2659 * same page to pages in the way and then do the icache_sync on 2660 * the entire way making sure we are cleaned. 2661 */ 2662 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 2663 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 2664 2665 for (size_t i = 0, j = 0; i < way_size; 2666 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2667 l2pte_reset(ptep + j); 2668 PTE_SYNC(ptep + j); 2669 2670 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2671 /* 2672 * Set up a PTE with to flush these cache lines. 2673 */ 2674 l2pte_set(ptep + j, npte, 0); 2675 } 2676 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2677 2678 /* 2679 * Flush it. 2680 */ 2681 cpu_icache_sync_range(dstp, way_size); 2682 2683 for (size_t i = 0, j = 0; i < way_size; 2684 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2685 /* 2686 * Unmap the page(s). 2687 */ 2688 l2pte_reset(ptep + j); 2689 PTE_SYNC(ptep + j); 2690 2691 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2692 } 2693 2694 md->pvh_attrs |= PVF_EXEC; 2695 PMAPCOUNT(exec_synced); 2696 } 2697 2698 #ifndef ARM_MMU_EXTENDED 2699 void 2700 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush) 2701 { 2702 vsize_t va_offset, end_va; 2703 bool wbinv_p; 2704 2705 if (arm_cache_prefer_mask == 0) 2706 return; 2707 2708 UVMHIST_FUNC(__func__); 2709 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx op %#jx", 2710 (uintptr_t)md, pa, op, 0); 2711 2712 switch (flush) { 2713 case PMAP_FLUSH_PRIMARY: 2714 if (md->pvh_attrs & PVF_MULTCLR) { 2715 va_offset = 0; 2716 end_va = arm_cache_prefer_mask; 2717 md->pvh_attrs &= ~PVF_MULTCLR; 2718 PMAPCOUNT(vac_flush_lots); 2719 } else { 2720 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2721 end_va = va_offset; 2722 PMAPCOUNT(vac_flush_one); 2723 } 2724 /* 2725 * Mark that the page is no longer dirty. 2726 */ 2727 md->pvh_attrs &= ~PVF_DIRTY; 2728 wbinv_p = true; 2729 break; 2730 case PMAP_FLUSH_SECONDARY: 2731 va_offset = 0; 2732 end_va = arm_cache_prefer_mask; 2733 wbinv_p = true; 2734 md->pvh_attrs &= ~PVF_MULTCLR; 2735 PMAPCOUNT(vac_flush_lots); 2736 break; 2737 case PMAP_CLEAN_PRIMARY: 2738 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2739 end_va = va_offset; 2740 wbinv_p = false; 2741 /* 2742 * Mark that the page is no longer dirty. 2743 */ 2744 if ((md->pvh_attrs & PVF_DMOD) == 0) 2745 md->pvh_attrs &= ~PVF_DIRTY; 2746 PMAPCOUNT(vac_clean_one); 2747 break; 2748 default: 2749 return; 2750 } 2751 2752 KASSERT(!(md->pvh_attrs & PVF_NC)); 2753 2754 UVMHIST_LOG(maphist, "md %#jx (attrs=%#jx)", (uintptr_t)md, 2755 md->pvh_attrs, 0, 0); 2756 2757 const size_t scache_line_size = arm_scache.dcache_line_size; 2758 2759 for (; va_offset <= end_va; va_offset += PAGE_SIZE) { 2760 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 2761 const vaddr_t dstp = cpu_cdstp(va_offset); 2762 const pt_entry_t opte = *ptep; 2763 2764 if (flush == PMAP_FLUSH_SECONDARY 2765 && va_offset == (md->pvh_attrs & arm_cache_prefer_mask)) 2766 continue; 2767 2768 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2769 /* 2770 * Set up a PTE with the right coloring to flush 2771 * existing cache entries. 2772 */ 2773 const pt_entry_t npte = L2_S_PROTO 2774 | pa 2775 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2776 | pte_l2_s_cache_mode; 2777 l2pte_set(ptep, npte, opte); 2778 PTE_SYNC(ptep); 2779 2780 /* 2781 * Flush it. Make sure to flush secondary cache too since 2782 * bus_dma will ignore uncached pages. 2783 */ 2784 if (scache_line_size != 0) { 2785 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2786 if (wbinv_p) { 2787 cpu_sdcache_wbinv_range(dstp, pa, PAGE_SIZE); 2788 cpu_dcache_inv_range(dstp, PAGE_SIZE); 2789 } else { 2790 cpu_sdcache_wb_range(dstp, pa, PAGE_SIZE); 2791 } 2792 } else { 2793 if (wbinv_p) { 2794 cpu_dcache_wbinv_range(dstp, PAGE_SIZE); 2795 } else { 2796 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2797 } 2798 } 2799 2800 /* 2801 * Restore the page table entry since we might have interrupted 2802 * pmap_zero_page or pmap_copy_page which was already using 2803 * this pte. 2804 */ 2805 if (opte) { 2806 l2pte_set(ptep, opte, npte); 2807 } else { 2808 l2pte_reset(ptep); 2809 } 2810 PTE_SYNC(ptep); 2811 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2812 } 2813 } 2814 #endif /* ARM_MMU_EXTENDED */ 2815 #endif /* PMAP_CACHE_VIPT */ 2816 2817 /* 2818 * Routine: pmap_page_remove 2819 * Function: 2820 * Removes this physical page from 2821 * all physical maps in which it resides. 2822 * Reflects back modify bits to the pager. 2823 */ 2824 static void 2825 pmap_page_remove(struct vm_page_md *md, paddr_t pa) 2826 { 2827 struct l2_bucket *l2b; 2828 struct pv_entry *pv; 2829 pt_entry_t *ptep; 2830 #ifndef ARM_MMU_EXTENDED 2831 bool flush = false; 2832 #endif 2833 u_int flags = 0; 2834 2835 UVMHIST_FUNC(__func__); 2836 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx", (uintptr_t)md, pa, 0, 0); 2837 2838 struct pv_entry **pvp = &SLIST_FIRST(&md->pvh_list); 2839 pmap_acquire_page_lock(md); 2840 if (*pvp == NULL) { 2841 #ifdef PMAP_CACHE_VIPT 2842 /* 2843 * We *know* the page contents are about to be replaced. 2844 * Discard the exec contents 2845 */ 2846 if (PV_IS_EXEC_P(md->pvh_attrs)) 2847 PMAPCOUNT(exec_discarded_page_protect); 2848 md->pvh_attrs &= ~PVF_EXEC; 2849 PMAP_VALIDATE_MD_PAGE(md); 2850 #endif 2851 pmap_release_page_lock(md); 2852 return; 2853 } 2854 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 2855 KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md)); 2856 #endif 2857 2858 /* 2859 * Clear alias counts 2860 */ 2861 #ifdef PMAP_CACHE_VIVT 2862 md->k_mappings = 0; 2863 #endif 2864 md->urw_mappings = md->uro_mappings = 0; 2865 2866 #ifdef PMAP_CACHE_VIVT 2867 pmap_clean_page(md, false); 2868 #endif 2869 2870 for (pv = *pvp; pv != NULL;) { 2871 pmap_t pm = pv->pv_pmap; 2872 #ifndef ARM_MMU_EXTENDED 2873 if (flush == false && pmap_is_current(pm)) 2874 flush = true; 2875 #endif 2876 2877 #ifdef PMAP_CACHE_VIPT 2878 if (pm == pmap_kernel() && PV_IS_KENTRY_P(pv->pv_flags)) { 2879 /* If this was unmanaged mapping, it must be ignored. */ 2880 pvp = &SLIST_NEXT(pv, pv_link); 2881 pv = *pvp; 2882 continue; 2883 } 2884 #endif 2885 2886 /* 2887 * Try to get a hold on the pmap's lock. We must do this 2888 * while still holding the page locked, to know that the 2889 * page is still associated with the pmap and the mapping is 2890 * in place. If a hold can't be had, unlock and wait for 2891 * the pmap's lock to become available and retry. The pmap 2892 * must be ref'd over this dance to stop it disappearing 2893 * behind us. 2894 */ 2895 if (!mutex_tryenter(&pm->pm_lock)) { 2896 pmap_reference(pm); 2897 pmap_release_page_lock(md); 2898 pmap_acquire_pmap_lock(pm); 2899 /* nothing, just wait for it */ 2900 pmap_release_pmap_lock(pm); 2901 pmap_destroy(pm); 2902 /* Restart from the beginning. */ 2903 pmap_acquire_page_lock(md); 2904 pvp = &SLIST_FIRST(&md->pvh_list); 2905 pv = *pvp; 2906 continue; 2907 } 2908 2909 if (pm == pmap_kernel()) { 2910 #ifdef PMAP_CACHE_VIPT 2911 if (pv->pv_flags & PVF_WRITE) 2912 md->krw_mappings--; 2913 else 2914 md->kro_mappings--; 2915 #endif 2916 PMAPCOUNT(kernel_unmappings); 2917 } 2918 *pvp = SLIST_NEXT(pv, pv_link); /* remove from list */ 2919 PMAPCOUNT(unmappings); 2920 2921 pmap_release_page_lock(md); 2922 2923 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2924 KASSERTMSG(l2b != NULL, "%#lx", pv->pv_va); 2925 2926 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2927 2928 /* 2929 * Update statistics 2930 */ 2931 --pm->pm_stats.resident_count; 2932 2933 /* Wired bit */ 2934 if (pv->pv_flags & PVF_WIRED) 2935 --pm->pm_stats.wired_count; 2936 2937 flags |= pv->pv_flags; 2938 2939 /* 2940 * Invalidate the PTEs. 2941 */ 2942 l2pte_reset(ptep); 2943 PTE_SYNC_CURRENT(pm, ptep); 2944 2945 #ifdef ARM_MMU_EXTENDED 2946 pmap_tlb_invalidate_addr(pm, pv->pv_va); 2947 #endif 2948 2949 pmap_free_l2_bucket(pm, l2b, PAGE_SIZE / L2_S_SIZE); 2950 2951 pmap_release_pmap_lock(pm); 2952 2953 pool_put(&pmap_pv_pool, pv); 2954 pmap_acquire_page_lock(md); 2955 2956 /* 2957 * Restart at the beginning of the list. 2958 */ 2959 pvp = &SLIST_FIRST(&md->pvh_list); 2960 pv = *pvp; 2961 } 2962 /* 2963 * if we reach the end of the list and there are still mappings, they 2964 * might be able to be cached now. And they must be kernel mappings. 2965 */ 2966 if (!SLIST_EMPTY(&md->pvh_list)) { 2967 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 2968 } 2969 2970 #ifdef PMAP_CACHE_VIPT 2971 /* 2972 * Its EXEC cache is now gone. 2973 */ 2974 if (PV_IS_EXEC_P(md->pvh_attrs)) 2975 PMAPCOUNT(exec_discarded_page_protect); 2976 md->pvh_attrs &= ~PVF_EXEC; 2977 KASSERT(md->urw_mappings == 0); 2978 KASSERT(md->uro_mappings == 0); 2979 #ifndef ARM_MMU_EXTENDED 2980 if (arm_cache_prefer_mask != 0) { 2981 if (md->krw_mappings == 0) 2982 md->pvh_attrs &= ~PVF_WRITE; 2983 PMAP_VALIDATE_MD_PAGE(md); 2984 } 2985 #endif /* ARM_MMU_EXTENDED */ 2986 #endif /* PMAP_CACHE_VIPT */ 2987 pmap_release_page_lock(md); 2988 2989 #ifndef ARM_MMU_EXTENDED 2990 if (flush) { 2991 /* 2992 * Note: We can't use pmap_tlb_flush{I,D}() here since that 2993 * would need a subsequent call to pmap_update() to ensure 2994 * curpm->pm_cstate.cs_all is reset. Our callers are not 2995 * required to do that (see pmap(9)), so we can't modify 2996 * the current pmap's state. 2997 */ 2998 if (PV_BEEN_EXECD(flags)) 2999 cpu_tlb_flushID(); 3000 else 3001 cpu_tlb_flushD(); 3002 } 3003 cpu_cpwait(); 3004 #endif /* ARM_MMU_EXTENDED */ 3005 } 3006 3007 /* 3008 * pmap_t pmap_create(void) 3009 * 3010 * Create a new pmap structure from scratch. 3011 */ 3012 pmap_t 3013 pmap_create(void) 3014 { 3015 pmap_t pm; 3016 3017 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 3018 3019 mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_NONE); 3020 3021 pm->pm_refs = 1; 3022 pm->pm_stats.wired_count = 0; 3023 pm->pm_stats.resident_count = 1; 3024 #ifdef ARM_MMU_EXTENDED 3025 #ifdef MULTIPROCESSOR 3026 kcpuset_create(&pm->pm_active, true); 3027 kcpuset_create(&pm->pm_onproc, true); 3028 #endif 3029 #else 3030 pm->pm_cstate.cs_all = 0; 3031 #endif 3032 pmap_alloc_l1(pm); 3033 3034 /* 3035 * Note: The pool cache ensures that the pm_l2[] array is already 3036 * initialised to zero. 3037 */ 3038 3039 pmap_pinit(pm); 3040 3041 return pm; 3042 } 3043 3044 u_int 3045 arm32_mmap_flags(paddr_t pa) 3046 { 3047 /* 3048 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff 3049 * and we're using the upper bits in page numbers to pass flags around 3050 * so we might as well use the same bits 3051 */ 3052 return (u_int)pa & PMAP_MD_MASK; 3053 } 3054 /* 3055 * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 3056 * u_int flags) 3057 * 3058 * Insert the given physical page (p) at 3059 * the specified virtual address (v) in the 3060 * target physical map with the protection requested. 3061 * 3062 * NB: This is the only routine which MAY NOT lazy-evaluate 3063 * or lose information. That is, this routine must actually 3064 * insert this page into the given map NOW. 3065 */ 3066 int 3067 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3068 { 3069 struct l2_bucket *l2b; 3070 struct vm_page *pg, *opg; 3071 u_int nflags; 3072 u_int oflags; 3073 const bool kpm_p = (pm == pmap_kernel()); 3074 #ifdef ARM_HAS_VBAR 3075 const bool vector_page_p = false; 3076 #else 3077 const bool vector_page_p = (va == vector_page); 3078 #endif 3079 struct pmap_page *pp = pmap_pv_tracked(pa); 3080 struct pv_entry *new_pv = NULL; 3081 struct pv_entry *old_pv = NULL; 3082 int error = 0; 3083 3084 UVMHIST_FUNC(__func__); 3085 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pa %#jx prot %#jx", 3086 (uintptr_t)pm, va, pa, prot); 3087 UVMHIST_LOG(maphist, " flag %#jx", flags, 0, 0, 0); 3088 3089 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 3090 KDASSERT(((va | pa) & PGOFSET) == 0); 3091 3092 /* 3093 * Get a pointer to the page. Later on in this function, we 3094 * test for a managed page by checking pg != NULL. 3095 */ 3096 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 3097 /* 3098 * if we may need a new pv entry allocate if now, as we can't do it 3099 * with the kernel_pmap locked 3100 */ 3101 if (pg || pp) 3102 new_pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3103 3104 nflags = 0; 3105 if (prot & VM_PROT_WRITE) 3106 nflags |= PVF_WRITE; 3107 if (prot & VM_PROT_EXECUTE) 3108 nflags |= PVF_EXEC; 3109 if (flags & PMAP_WIRED) 3110 nflags |= PVF_WIRED; 3111 3112 pmap_acquire_pmap_lock(pm); 3113 3114 /* 3115 * Fetch the L2 bucket which maps this page, allocating one if 3116 * necessary for user pmaps. 3117 */ 3118 if (kpm_p) { 3119 l2b = pmap_get_l2_bucket(pm, va); 3120 } else { 3121 l2b = pmap_alloc_l2_bucket(pm, va); 3122 } 3123 if (l2b == NULL) { 3124 if (flags & PMAP_CANFAIL) { 3125 pmap_release_pmap_lock(pm); 3126 error = ENOMEM; 3127 goto free_pv; 3128 } 3129 panic("pmap_enter: failed to allocate L2 bucket"); 3130 } 3131 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)]; 3132 const pt_entry_t opte = *ptep; 3133 pt_entry_t npte = pa; 3134 oflags = 0; 3135 3136 if (opte) { 3137 /* 3138 * There is already a mapping at this address. 3139 * If the physical address is different, lookup the 3140 * vm_page. 3141 */ 3142 if (l2pte_pa(opte) != pa) { 3143 KASSERT(!pmap_pv_tracked(pa)); 3144 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3145 } else 3146 opg = pg; 3147 } else 3148 opg = NULL; 3149 3150 if (pg || pp) { 3151 KASSERT((pg != NULL) != (pp != NULL)); 3152 struct vm_page_md *md = (pg != NULL) ? VM_PAGE_TO_MD(pg) : 3153 PMAP_PAGE_TO_MD(pp); 3154 3155 /* 3156 * This is to be a managed mapping. 3157 */ 3158 pmap_acquire_page_lock(md); 3159 if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) { 3160 /* 3161 * - The access type indicates that we don't need 3162 * to do referenced emulation. 3163 * OR 3164 * - The physical page has already been referenced 3165 * so no need to re-do referenced emulation here. 3166 */ 3167 npte |= l2pte_set_readonly(L2_S_PROTO); 3168 3169 nflags |= PVF_REF; 3170 3171 if ((prot & VM_PROT_WRITE) != 0 && 3172 ((flags & VM_PROT_WRITE) != 0 || 3173 (md->pvh_attrs & PVF_MOD) != 0)) { 3174 /* 3175 * This is a writable mapping, and the 3176 * page's mod state indicates it has 3177 * already been modified. Make it 3178 * writable from the outset. 3179 */ 3180 npte = l2pte_set_writable(npte); 3181 nflags |= PVF_MOD; 3182 } 3183 3184 #ifdef ARM_MMU_EXTENDED 3185 /* 3186 * If the page has been cleaned, then the pvh_attrs 3187 * will have PVF_EXEC set, so mark it execute so we 3188 * don't get an access fault when trying to execute 3189 * from it. 3190 */ 3191 if (md->pvh_attrs & nflags & PVF_EXEC) { 3192 npte &= ~L2_XS_XN; 3193 } 3194 #endif 3195 } else { 3196 /* 3197 * Need to do page referenced emulation. 3198 */ 3199 npte |= L2_TYPE_INV; 3200 } 3201 3202 if (flags & ARM32_MMAP_WRITECOMBINE) { 3203 npte |= pte_l2_s_wc_mode; 3204 } else 3205 npte |= pte_l2_s_cache_mode; 3206 3207 if (pg != NULL && pg == opg) { 3208 /* 3209 * We're changing the attrs of an existing mapping. 3210 */ 3211 oflags = pmap_modify_pv(md, pa, pm, va, 3212 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3213 PVF_MOD | PVF_REF, nflags); 3214 3215 #ifdef PMAP_CACHE_VIVT 3216 /* 3217 * We may need to flush the cache if we're 3218 * doing rw-ro... 3219 */ 3220 if (pm->pm_cstate.cs_cache_d && 3221 (oflags & PVF_NC) == 0 && 3222 l2pte_writable_p(opte) && 3223 (prot & VM_PROT_WRITE) == 0) 3224 cpu_dcache_wb_range(va, PAGE_SIZE); 3225 #endif 3226 } else { 3227 struct pv_entry *pv; 3228 /* 3229 * New mapping, or changing the backing page 3230 * of an existing mapping. 3231 */ 3232 if (opg) { 3233 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3234 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3235 3236 /* 3237 * Replacing an existing mapping with a new one. 3238 * It is part of our managed memory so we 3239 * must remove it from the PV list 3240 */ 3241 pv = pmap_remove_pv(omd, opa, pm, va); 3242 pmap_vac_me_harder(omd, opa, pm, 0); 3243 oflags = pv->pv_flags; 3244 3245 #ifdef PMAP_CACHE_VIVT 3246 /* 3247 * If the old mapping was valid (ref/mod 3248 * emulation creates 'invalid' mappings 3249 * initially) then make sure to frob 3250 * the cache. 3251 */ 3252 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3253 pmap_cache_wbinv_page(pm, va, true, 3254 oflags); 3255 } 3256 #endif 3257 } else { 3258 pv = new_pv; 3259 new_pv = NULL; 3260 if (pv == NULL) { 3261 pmap_release_page_lock(md); 3262 pmap_release_pmap_lock(pm); 3263 if ((flags & PMAP_CANFAIL) == 0) 3264 panic("pmap_enter: " 3265 "no pv entries"); 3266 3267 pmap_free_l2_bucket(pm, l2b, 0); 3268 UVMHIST_LOG(maphist, " <-- done (ENOMEM)", 3269 0, 0, 0, 0); 3270 return ENOMEM; 3271 } 3272 } 3273 3274 pmap_enter_pv(md, pa, pv, pm, va, nflags); 3275 } 3276 pmap_release_page_lock(md); 3277 } else { 3278 /* 3279 * We're mapping an unmanaged page. 3280 * These are always readable, and possibly writable, from 3281 * the get go as we don't need to track ref/mod status. 3282 */ 3283 npte |= l2pte_set_readonly(L2_S_PROTO); 3284 if (prot & VM_PROT_WRITE) 3285 npte = l2pte_set_writable(npte); 3286 3287 /* 3288 * Make sure the vector table is mapped cacheable 3289 */ 3290 if ((vector_page_p && !kpm_p) 3291 || (flags & ARM32_MMAP_CACHEABLE)) { 3292 npte |= pte_l2_s_cache_mode; 3293 #ifdef ARM_MMU_EXTENDED 3294 npte &= ~L2_XS_XN; /* and executable */ 3295 #endif 3296 } else if (flags & ARM32_MMAP_WRITECOMBINE) { 3297 npte |= pte_l2_s_wc_mode; 3298 } 3299 if (opg) { 3300 /* 3301 * Looks like there's an existing 'managed' mapping 3302 * at this address. 3303 */ 3304 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3305 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3306 3307 pmap_acquire_page_lock(omd); 3308 old_pv = pmap_remove_pv(omd, opa, pm, va); 3309 pmap_vac_me_harder(omd, opa, pm, 0); 3310 oflags = old_pv->pv_flags; 3311 pmap_release_page_lock(omd); 3312 3313 #ifdef PMAP_CACHE_VIVT 3314 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3315 pmap_cache_wbinv_page(pm, va, true, oflags); 3316 } 3317 #endif 3318 } 3319 } 3320 3321 /* 3322 * Make sure userland mappings get the right permissions 3323 */ 3324 if (!vector_page_p && !kpm_p) { 3325 npte |= L2_S_PROT_U; 3326 #ifdef ARM_MMU_EXTENDED 3327 npte |= L2_XS_nG; /* user pages are not global */ 3328 #endif 3329 } 3330 3331 /* 3332 * Keep the stats up to date 3333 */ 3334 if (opte == 0) { 3335 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3336 pm->pm_stats.resident_count++; 3337 } 3338 3339 UVMHIST_LOG(maphist, " opte %#jx npte %#jx", opte, npte, 0, 0); 3340 3341 #if defined(ARM_MMU_EXTENDED) 3342 /* 3343 * If exec protection was requested but the page hasn't been synced, 3344 * sync it now and allow execution from it. 3345 */ 3346 if ((nflags & PVF_EXEC) && (npte & L2_XS_XN)) { 3347 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3348 npte &= ~L2_XS_XN; 3349 pmap_syncicache_page(md, pa); 3350 PMAPCOUNT(exec_synced_map); 3351 } 3352 #endif 3353 /* 3354 * If this is just a wiring change, the two PTEs will be 3355 * identical, so there's no need to update the page table. 3356 */ 3357 if (npte != opte) { 3358 l2pte_reset(ptep); 3359 PTE_SYNC(ptep); 3360 if (l2pte_valid_p(opte)) { 3361 pmap_tlb_flush_SE(pm, va, oflags); 3362 } 3363 l2pte_set(ptep, npte, 0); 3364 PTE_SYNC(ptep); 3365 #ifndef ARM_MMU_EXTENDED 3366 bool is_cached = pmap_is_cached(pm); 3367 if (is_cached) { 3368 /* 3369 * We only need to frob the cache/tlb if this pmap 3370 * is current 3371 */ 3372 if (!vector_page_p && l2pte_valid_p(npte)) { 3373 /* 3374 * This mapping is likely to be accessed as 3375 * soon as we return to userland. Fix up the 3376 * L1 entry to avoid taking another 3377 * page/domain fault. 3378 */ 3379 pd_entry_t *pdep = pmap_l1_kva(pm) 3380 + l1pte_index(va); 3381 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa 3382 | L1_C_DOM(pmap_domain(pm)); 3383 if (*pdep != pde) { 3384 l1pte_setone(pdep, pde); 3385 PDE_SYNC(pdep); 3386 } 3387 } 3388 } 3389 3390 UVMHIST_LOG(maphist, " is_cached %jd cs 0x%08jx", 3391 is_cached, pm->pm_cstate.cs_all, 0, 0); 3392 3393 if (pg != NULL) { 3394 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3395 3396 pmap_acquire_page_lock(md); 3397 pmap_vac_me_harder(md, pa, pm, va); 3398 pmap_release_page_lock(md); 3399 } 3400 #endif 3401 } 3402 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 3403 if (pg) { 3404 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3405 3406 pmap_acquire_page_lock(md); 3407 #ifndef ARM_MMU_EXTENDED 3408 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3409 #endif 3410 PMAP_VALIDATE_MD_PAGE(md); 3411 pmap_release_page_lock(md); 3412 } 3413 #endif 3414 3415 pmap_release_pmap_lock(pm); 3416 3417 3418 if (old_pv) 3419 pool_put(&pmap_pv_pool, old_pv); 3420 free_pv: 3421 if (new_pv) 3422 pool_put(&pmap_pv_pool, new_pv); 3423 return error; 3424 } 3425 3426 /* 3427 * pmap_remove() 3428 * 3429 * pmap_remove is responsible for nuking a number of mappings for a range 3430 * of virtual address space in the current pmap. To do this efficiently 3431 * is interesting, because in a number of cases a wide virtual address 3432 * range may be supplied that contains few actual mappings. So, the 3433 * optimisations are: 3434 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3435 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3436 * maybe do just a partial cache clean. This path of execution is 3437 * complicated by the fact that the cache must be flushed _before_ 3438 * the PTE is nuked, being a VAC :-) 3439 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3440 * all invalidations until pmap_update(), since pmap_remove_all() has 3441 * already flushed the cache. 3442 * 4. Maybe later fast-case a single page, but I don't think this is 3443 * going to make _that_ much difference overall. 3444 */ 3445 3446 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3447 3448 void 3449 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 3450 { 3451 SLIST_HEAD(,pv_entry) opv_list; 3452 struct pv_entry *pv, *npv; 3453 UVMHIST_FUNC(__func__); 3454 UVMHIST_CALLARGS(maphist, " (pm=%#jx, sva=%#jx, eva=%#jx)", 3455 (uintptr_t)pm, sva, eva, 0); 3456 3457 #ifdef PMAP_FAULTINFO 3458 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 3459 curpcb->pcb_faultinfo.pfi_repeats = 0; 3460 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 3461 #endif 3462 3463 SLIST_INIT(&opv_list); 3464 /* 3465 * we lock in the pmap => pv_head direction 3466 */ 3467 pmap_acquire_pmap_lock(pm); 3468 3469 #ifndef ARM_MMU_EXTENDED 3470 u_int cleanlist_idx, total, cnt; 3471 struct { 3472 vaddr_t va; 3473 pt_entry_t *ptep; 3474 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3475 3476 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3477 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3478 if (pm->pm_cstate.cs_tlb == 0) 3479 pm->pm_remove_all = true; 3480 } else 3481 cleanlist_idx = 0; 3482 total = 0; 3483 #endif 3484 3485 while (sva < eva) { 3486 /* 3487 * Do one L2 bucket's worth at a time. 3488 */ 3489 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(sva); 3490 if (next_bucket > eva) 3491 next_bucket = eva; 3492 3493 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva); 3494 if (l2b == NULL) { 3495 sva = next_bucket; 3496 continue; 3497 } 3498 3499 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3500 u_int mappings = 0; 3501 3502 for (;sva < next_bucket; 3503 sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) { 3504 pt_entry_t opte = *ptep; 3505 3506 if (opte == 0) { 3507 /* Nothing here, move along */ 3508 continue; 3509 } 3510 3511 u_int flags = PVF_REF; 3512 paddr_t pa = l2pte_pa(opte); 3513 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 3514 3515 /* 3516 * Update flags. In a number of circumstances, 3517 * we could cluster a lot of these and do a 3518 * number of sequential pages in one go. 3519 */ 3520 if (pg != NULL) { 3521 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3522 3523 pmap_acquire_page_lock(md); 3524 pv = pmap_remove_pv(md, pa, pm, sva); 3525 pmap_vac_me_harder(md, pa, pm, 0); 3526 pmap_release_page_lock(md); 3527 if (pv != NULL) { 3528 if (pm->pm_remove_all == false) { 3529 flags = pv->pv_flags; 3530 } 3531 SLIST_INSERT_HEAD(&opv_list, 3532 pv, pv_link); 3533 } 3534 } 3535 mappings += PAGE_SIZE / L2_S_SIZE; 3536 3537 if (!l2pte_valid_p(opte)) { 3538 /* 3539 * Ref/Mod emulation is still active for this 3540 * mapping, therefore it is has not yet been 3541 * accessed. No need to frob the cache/tlb. 3542 */ 3543 l2pte_reset(ptep); 3544 PTE_SYNC_CURRENT(pm, ptep); 3545 continue; 3546 } 3547 3548 #ifdef ARM_MMU_EXTENDED 3549 l2pte_reset(ptep); 3550 PTE_SYNC(ptep); 3551 if (__predict_false(pm->pm_remove_all == false)) { 3552 pmap_tlb_flush_SE(pm, sva, flags); 3553 } 3554 #else 3555 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3556 /* Add to the clean list. */ 3557 cleanlist[cleanlist_idx].ptep = ptep; 3558 cleanlist[cleanlist_idx].va = 3559 sva | (flags & PVF_EXEC); 3560 cleanlist_idx++; 3561 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3562 /* Nuke everything if needed. */ 3563 #ifdef PMAP_CACHE_VIVT 3564 pmap_cache_wbinv_all(pm, PVF_EXEC); 3565 #endif 3566 /* 3567 * Roll back the previous PTE list, 3568 * and zero out the current PTE. 3569 */ 3570 for (cnt = 0; 3571 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3572 l2pte_reset(cleanlist[cnt].ptep); 3573 PTE_SYNC(cleanlist[cnt].ptep); 3574 } 3575 l2pte_reset(ptep); 3576 PTE_SYNC(ptep); 3577 cleanlist_idx++; 3578 pm->pm_remove_all = true; 3579 } else { 3580 l2pte_reset(ptep); 3581 PTE_SYNC(ptep); 3582 if (pm->pm_remove_all == false) { 3583 pmap_tlb_flush_SE(pm, sva, flags); 3584 } 3585 } 3586 #endif 3587 } 3588 3589 #ifndef ARM_MMU_EXTENDED 3590 /* 3591 * Deal with any left overs 3592 */ 3593 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3594 total += cleanlist_idx; 3595 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3596 l2pte_reset(cleanlist[cnt].ptep); 3597 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3598 vaddr_t va = cleanlist[cnt].va; 3599 if (pm->pm_cstate.cs_all != 0) { 3600 vaddr_t clva = va & ~PAGE_MASK; 3601 u_int flags = va & PVF_EXEC; 3602 #ifdef PMAP_CACHE_VIVT 3603 pmap_cache_wbinv_page(pm, clva, true, 3604 PVF_REF | PVF_WRITE | flags); 3605 #endif 3606 pmap_tlb_flush_SE(pm, clva, 3607 PVF_REF | flags); 3608 } 3609 } 3610 3611 /* 3612 * If it looks like we're removing a whole bunch 3613 * of mappings, it's faster to just write-back 3614 * the whole cache now and defer TLB flushes until 3615 * pmap_update() is called. 3616 */ 3617 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3618 cleanlist_idx = 0; 3619 else { 3620 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3621 #ifdef PMAP_CACHE_VIVT 3622 pmap_cache_wbinv_all(pm, PVF_EXEC); 3623 #endif 3624 pm->pm_remove_all = true; 3625 } 3626 } 3627 #endif /* ARM_MMU_EXTENDED */ 3628 3629 pmap_free_l2_bucket(pm, l2b, mappings); 3630 pm->pm_stats.resident_count -= mappings / (PAGE_SIZE/L2_S_SIZE); 3631 } 3632 3633 pmap_release_pmap_lock(pm); 3634 SLIST_FOREACH_SAFE(pv, &opv_list, pv_link, npv) { 3635 pool_put(&pmap_pv_pool, pv); 3636 } 3637 } 3638 3639 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3640 static struct pv_entry * 3641 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3642 { 3643 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3644 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3645 struct pv_entry *pv; 3646 3647 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC)); 3648 KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0); 3649 KASSERT(pmap_page_locked_p(md)); 3650 3651 pv = pmap_remove_pv(md, pa, pmap_kernel(), va); 3652 KASSERTMSG(pv, "pg %p (pa #%lx) va %#lx", pg, pa, va); 3653 KASSERT(PV_IS_KENTRY_P(pv->pv_flags)); 3654 3655 /* 3656 * We are removing a writeable mapping to a cached exec page, if 3657 * it's the last mapping then clear its execness otherwise sync 3658 * the page to the icache. 3659 */ 3660 if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3661 && (pv->pv_flags & PVF_WRITE) != 0) { 3662 if (SLIST_EMPTY(&md->pvh_list)) { 3663 md->pvh_attrs &= ~PVF_EXEC; 3664 PMAPCOUNT(exec_discarded_kremove); 3665 } else { 3666 pmap_syncicache_page(md, pa); 3667 PMAPCOUNT(exec_synced_kremove); 3668 } 3669 } 3670 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 3671 3672 return pv; 3673 } 3674 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 3675 3676 /* 3677 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3678 * 3679 * We assume there is already sufficient KVM space available 3680 * to do this, as we can't allocate L2 descriptor tables/metadata 3681 * from here. 3682 */ 3683 void 3684 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3685 { 3686 #ifdef PMAP_CACHE_VIVT 3687 struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3688 #endif 3689 #ifdef PMAP_CACHE_VIPT 3690 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3691 struct vm_page *opg; 3692 #ifndef ARM_MMU_EXTENDED 3693 struct pv_entry *pv = NULL; 3694 #endif 3695 #endif 3696 struct vm_page_md *md = pg != NULL ? VM_PAGE_TO_MD(pg) : NULL; 3697 3698 UVMHIST_FUNC(__func__); 3699 3700 if (pmap_initialized) { 3701 UVMHIST_CALLARGS(maphist, 3702 "va=%#jx, pa=%#jx, prot=%#jx, flags=%#jx", va, pa, prot, 3703 flags); 3704 } 3705 3706 pmap_t kpm = pmap_kernel(); 3707 pmap_acquire_pmap_lock(kpm); 3708 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3709 const size_t l1slot __diagused = l1pte_index(va); 3710 KASSERTMSG(l2b != NULL, 3711 "va %#lx pa %#lx prot %d maxkvaddr %#lx: l2 %p l2b %p kva %p", 3712 va, pa, prot, pmap_curmaxkvaddr, kpm->pm_l2[L2_IDX(l1slot)], 3713 kpm->pm_l2[L2_IDX(l1slot)] 3714 ? &kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)] 3715 : NULL, 3716 kpm->pm_l2[L2_IDX(l1slot)] 3717 ? kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)].l2b_kva 3718 : NULL); 3719 KASSERT(l2b->l2b_kva != NULL); 3720 3721 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 3722 const pt_entry_t opte = *ptep; 3723 3724 if (opte == 0) { 3725 PMAPCOUNT(kenter_mappings); 3726 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3727 } else { 3728 PMAPCOUNT(kenter_remappings); 3729 #ifdef PMAP_CACHE_VIPT 3730 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3731 #if !defined(ARM_MMU_EXTENDED) || defined(DIAGNOSTIC) 3732 struct vm_page_md *omd __diagused = VM_PAGE_TO_MD(opg); 3733 #endif 3734 if (opg && arm_cache_prefer_mask != 0) { 3735 KASSERT(opg != pg); 3736 KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0); 3737 KASSERT((flags & PMAP_KMPAGE) == 0); 3738 #ifndef ARM_MMU_EXTENDED 3739 pmap_acquire_page_lock(omd); 3740 pv = pmap_kremove_pg(opg, va); 3741 pmap_release_page_lock(omd); 3742 #endif 3743 } 3744 #endif 3745 if (l2pte_valid_p(opte)) { 3746 l2pte_reset(ptep); 3747 PTE_SYNC(ptep); 3748 #ifdef PMAP_CACHE_VIVT 3749 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3750 #endif 3751 cpu_tlb_flushD_SE(va); 3752 cpu_cpwait(); 3753 } 3754 } 3755 pmap_release_pmap_lock(kpm); 3756 pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 3757 3758 if (flags & PMAP_PTE) { 3759 KASSERT((flags & PMAP_CACHE_MASK) == 0); 3760 if (!(flags & PMAP_NOCACHE)) 3761 npte |= pte_l2_s_cache_mode_pt; 3762 } else { 3763 switch (flags & (PMAP_CACHE_MASK | PMAP_DEV_MASK)) { 3764 case PMAP_DEV ... PMAP_DEV | PMAP_CACHE_MASK: 3765 break; 3766 case PMAP_NOCACHE: 3767 npte |= pte_l2_s_nocache_mode; 3768 break; 3769 case PMAP_WRITE_COMBINE: 3770 npte |= pte_l2_s_wc_mode; 3771 break; 3772 default: 3773 npte |= pte_l2_s_cache_mode; 3774 break; 3775 } 3776 } 3777 #ifdef ARM_MMU_EXTENDED 3778 if (prot & VM_PROT_EXECUTE) 3779 npte &= ~L2_XS_XN; 3780 #endif 3781 l2pte_set(ptep, npte, 0); 3782 PTE_SYNC(ptep); 3783 3784 if (pg) { 3785 if (flags & PMAP_KMPAGE) { 3786 KASSERT(md->urw_mappings == 0); 3787 KASSERT(md->uro_mappings == 0); 3788 KASSERT(md->krw_mappings == 0); 3789 KASSERT(md->kro_mappings == 0); 3790 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3791 KASSERT(pv == NULL); 3792 KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0); 3793 KASSERT((md->pvh_attrs & PVF_NC) == 0); 3794 /* if there is a color conflict, evict from cache. */ 3795 if (pmap_is_page_colored_p(md) 3796 && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) { 3797 PMAPCOUNT(vac_color_change); 3798 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 3799 } else if (md->pvh_attrs & PVF_MULTCLR) { 3800 /* 3801 * If this page has multiple colors, expunge 3802 * them. 3803 */ 3804 PMAPCOUNT(vac_flush_lots2); 3805 pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY); 3806 } 3807 /* 3808 * Since this is a KMPAGE, there can be no contention 3809 * for this page so don't lock it. 3810 */ 3811 md->pvh_attrs &= PAGE_SIZE - 1; 3812 md->pvh_attrs |= PVF_KMPAGE | PVF_COLORED | PVF_DIRTY 3813 | (va & arm_cache_prefer_mask); 3814 #else /* !PMAP_CACHE_VIPT || ARM_MMU_EXTENDED */ 3815 md->pvh_attrs |= PVF_KMPAGE; 3816 #endif 3817 atomic_inc_32(&pmap_kmpages); 3818 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3819 } else if (arm_cache_prefer_mask != 0) { 3820 if (pv == NULL) { 3821 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3822 KASSERT(pv != NULL); 3823 } 3824 pmap_acquire_page_lock(md); 3825 pmap_enter_pv(md, pa, pv, pmap_kernel(), va, 3826 PVF_WIRED | PVF_KENTRY 3827 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3828 if ((prot & VM_PROT_WRITE) 3829 && !(md->pvh_attrs & PVF_NC)) 3830 md->pvh_attrs |= PVF_DIRTY; 3831 KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3832 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 3833 pmap_release_page_lock(md); 3834 #endif 3835 } 3836 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3837 } else { 3838 if (pv != NULL) 3839 pool_put(&pmap_pv_pool, pv); 3840 #endif 3841 } 3842 if (pmap_initialized) { 3843 UVMHIST_LOG(maphist, " <-- done (ptep %#jx: %#jx -> %#jx)", 3844 (uintptr_t)ptep, opte, npte, 0); 3845 } 3846 3847 } 3848 3849 void 3850 pmap_kremove(vaddr_t va, vsize_t len) 3851 { 3852 #ifdef UVMHIST 3853 u_int total_mappings = 0; 3854 #endif 3855 3856 PMAPCOUNT(kenter_unmappings); 3857 3858 UVMHIST_FUNC(__func__); 3859 UVMHIST_CALLARGS(maphist, " (va=%#jx, len=%#jx)", va, len, 0, 0); 3860 3861 const vaddr_t eva = va + len; 3862 pmap_t kpm = pmap_kernel(); 3863 3864 pmap_acquire_pmap_lock(kpm); 3865 3866 while (va < eva) { 3867 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(va); 3868 if (next_bucket > eva) 3869 next_bucket = eva; 3870 3871 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3872 KDASSERT(l2b != NULL); 3873 3874 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 3875 pt_entry_t *ptep = sptep; 3876 u_int mappings = 0; 3877 3878 while (va < next_bucket) { 3879 const pt_entry_t opte = *ptep; 3880 struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3881 if (opg != NULL) { 3882 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3883 3884 if (omd->pvh_attrs & PVF_KMPAGE) { 3885 KASSERT(omd->urw_mappings == 0); 3886 KASSERT(omd->uro_mappings == 0); 3887 KASSERT(omd->krw_mappings == 0); 3888 KASSERT(omd->kro_mappings == 0); 3889 omd->pvh_attrs &= ~PVF_KMPAGE; 3890 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3891 if (arm_cache_prefer_mask != 0) { 3892 omd->pvh_attrs &= ~PVF_WRITE; 3893 } 3894 #endif 3895 atomic_dec_32(&pmap_kmpages); 3896 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3897 } else if (arm_cache_prefer_mask != 0) { 3898 pmap_acquire_page_lock(omd); 3899 pool_put(&pmap_pv_pool, 3900 pmap_kremove_pg(opg, va)); 3901 pmap_release_page_lock(omd); 3902 #endif 3903 } 3904 } 3905 if (l2pte_valid_p(opte)) { 3906 l2pte_reset(ptep); 3907 PTE_SYNC(ptep); 3908 #ifdef PMAP_CACHE_VIVT 3909 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3910 #endif 3911 cpu_tlb_flushD_SE(va); 3912 3913 mappings += PAGE_SIZE / L2_S_SIZE; 3914 } 3915 va += PAGE_SIZE; 3916 ptep += PAGE_SIZE / L2_S_SIZE; 3917 } 3918 KDASSERTMSG(mappings <= l2b->l2b_occupancy, "%u %u", 3919 mappings, l2b->l2b_occupancy); 3920 l2b->l2b_occupancy -= mappings; 3921 //PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3922 #ifdef UVMHIST 3923 total_mappings += mappings; 3924 #endif 3925 } 3926 pmap_release_pmap_lock(kpm); 3927 cpu_cpwait(); 3928 UVMHIST_LOG(maphist, " <--- done (%ju mappings removed)", 3929 total_mappings, 0, 0, 0); 3930 } 3931 3932 bool 3933 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3934 { 3935 3936 return pmap_extract_coherency(pm, va, pap, NULL); 3937 } 3938 3939 bool 3940 pmap_extract_coherency(pmap_t pm, vaddr_t va, paddr_t *pap, bool *coherentp) 3941 { 3942 struct l2_dtable *l2; 3943 pd_entry_t *pdep, pde; 3944 pt_entry_t *ptep, pte; 3945 paddr_t pa; 3946 u_int l1slot; 3947 bool coherent; 3948 3949 pmap_acquire_pmap_lock(pm); 3950 3951 l1slot = l1pte_index(va); 3952 pdep = pmap_l1_kva(pm) + l1slot; 3953 pde = *pdep; 3954 3955 if (l1pte_section_p(pde)) { 3956 /* 3957 * These should only happen for pmap_kernel() 3958 */ 3959 KDASSERT(pm == pmap_kernel()); 3960 pmap_release_pmap_lock(pm); 3961 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 3962 if (l1pte_supersection_p(pde)) { 3963 pa = (pde & L1_SS_FRAME) | (va & L1_SS_OFFSET); 3964 } else 3965 #endif 3966 pa = (pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3967 coherent = (pde & L1_S_CACHE_MASK) == 0; 3968 } else { 3969 /* 3970 * Note that we can't rely on the validity of the L1 3971 * descriptor as an indication that a mapping exists. 3972 * We have to look it up in the L2 dtable. 3973 */ 3974 l2 = pm->pm_l2[L2_IDX(l1slot)]; 3975 3976 if (l2 == NULL || 3977 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 3978 pmap_release_pmap_lock(pm); 3979 return false; 3980 } 3981 3982 pte = ptep[l2pte_index(va)]; 3983 pmap_release_pmap_lock(pm); 3984 3985 if (pte == 0) 3986 return false; 3987 3988 switch (pte & L2_TYPE_MASK) { 3989 case L2_TYPE_L: 3990 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3991 coherent = (pte & L2_L_CACHE_MASK) == 0; 3992 break; 3993 3994 default: 3995 pa = (pte & ~PAGE_MASK) | (va & PAGE_MASK); 3996 coherent = (pte & L2_S_CACHE_MASK) == 0; 3997 break; 3998 } 3999 } 4000 4001 if (pap != NULL) 4002 *pap = pa; 4003 4004 if (coherentp != NULL) 4005 *coherentp = (pm == pmap_kernel() && coherent); 4006 4007 return true; 4008 } 4009 4010 /* 4011 * pmap_pv_remove: remove an unmanaged pv-tracked page from all pmaps 4012 * that map it 4013 */ 4014 4015 static void 4016 pmap_pv_remove(paddr_t pa) 4017 { 4018 struct pmap_page *pp; 4019 4020 pp = pmap_pv_tracked(pa); 4021 if (pp == NULL) 4022 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 4023 pa); 4024 4025 struct vm_page_md *md = PMAP_PAGE_TO_MD(pp); 4026 pmap_page_remove(md, pa); 4027 } 4028 4029 void 4030 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 4031 { 4032 4033 /* the only case is remove at the moment */ 4034 KASSERT(prot == VM_PROT_NONE); 4035 pmap_pv_remove(pa); 4036 } 4037 4038 void 4039 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 4040 { 4041 struct l2_bucket *l2b; 4042 vaddr_t next_bucket; 4043 4044 UVMHIST_FUNC(__func__); 4045 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx...#%jx prot %#jx", 4046 (uintptr_t)pm, sva, eva, prot); 4047 4048 if ((prot & VM_PROT_READ) == 0) { 4049 pmap_remove(pm, sva, eva); 4050 return; 4051 } 4052 4053 if (prot & VM_PROT_WRITE) { 4054 /* 4055 * If this is a read->write transition, just ignore it and let 4056 * uvm_fault() take care of it later. 4057 */ 4058 return; 4059 } 4060 4061 pmap_acquire_pmap_lock(pm); 4062 4063 #ifndef ARM_MMU_EXTENDED 4064 const bool flush = eva - sva >= PAGE_SIZE * 4; 4065 u_int flags = 0; 4066 #endif 4067 u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 4068 4069 while (sva < eva) { 4070 next_bucket = L2_NEXT_BUCKET_VA(sva); 4071 if (next_bucket > eva) 4072 next_bucket = eva; 4073 4074 l2b = pmap_get_l2_bucket(pm, sva); 4075 if (l2b == NULL) { 4076 sva = next_bucket; 4077 continue; 4078 } 4079 4080 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4081 4082 while (sva < next_bucket) { 4083 const pt_entry_t opte = *ptep; 4084 if (l2pte_valid_p(opte) && l2pte_writable_p(opte)) { 4085 struct vm_page *pg; 4086 #ifndef ARM_MMU_EXTENDED 4087 u_int f; 4088 #endif 4089 4090 #ifdef PMAP_CACHE_VIVT 4091 /* 4092 * OK, at this point, we know we're doing 4093 * write-protect operation. If the pmap is 4094 * active, write-back the page. 4095 */ 4096 pmap_cache_wbinv_page(pm, sva, false, 4097 PVF_REF | PVF_WRITE); 4098 #endif 4099 4100 pg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 4101 pt_entry_t npte = l2pte_set_readonly(opte); 4102 l2pte_reset(ptep); 4103 PTE_SYNC(ptep); 4104 #ifdef ARM_MMU_EXTENDED 4105 pmap_tlb_flush_SE(pm, sva, PVF_REF); 4106 #endif 4107 l2pte_set(ptep, npte, 0); 4108 PTE_SYNC(ptep); 4109 4110 if (pg != NULL) { 4111 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4112 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4113 4114 pmap_acquire_page_lock(md); 4115 #ifndef ARM_MMU_EXTENDED 4116 f = 4117 #endif 4118 pmap_modify_pv(md, pa, pm, sva, 4119 clr_mask, 0); 4120 pmap_vac_me_harder(md, pa, pm, sva); 4121 pmap_release_page_lock(md); 4122 #ifndef ARM_MMU_EXTENDED 4123 } else { 4124 f = PVF_REF | PVF_EXEC; 4125 } 4126 4127 if (flush) { 4128 flags |= f; 4129 } else { 4130 pmap_tlb_flush_SE(pm, sva, f); 4131 #endif 4132 } 4133 } 4134 4135 sva += PAGE_SIZE; 4136 ptep += PAGE_SIZE / L2_S_SIZE; 4137 } 4138 } 4139 4140 #ifndef ARM_MMU_EXTENDED 4141 if (flush) { 4142 if (PV_BEEN_EXECD(flags)) { 4143 pmap_tlb_flushID(pm); 4144 } else if (PV_BEEN_REFD(flags)) { 4145 pmap_tlb_flushD(pm); 4146 } 4147 } 4148 #endif 4149 4150 pmap_release_pmap_lock(pm); 4151 } 4152 4153 void 4154 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 4155 { 4156 struct l2_bucket *l2b; 4157 pt_entry_t *ptep; 4158 vaddr_t next_bucket; 4159 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 4160 4161 UVMHIST_FUNC(__func__); 4162 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx...#%jx", 4163 (uintptr_t)pm, sva, eva, 0); 4164 4165 pmap_acquire_pmap_lock(pm); 4166 4167 while (sva < eva) { 4168 next_bucket = L2_NEXT_BUCKET_VA(sva); 4169 if (next_bucket > eva) 4170 next_bucket = eva; 4171 4172 l2b = pmap_get_l2_bucket(pm, sva); 4173 if (l2b == NULL) { 4174 sva = next_bucket; 4175 continue; 4176 } 4177 4178 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4179 sva < next_bucket; 4180 sva += page_size, 4181 ptep += PAGE_SIZE / L2_S_SIZE, 4182 page_size = PAGE_SIZE) { 4183 if (l2pte_valid_p(*ptep)) { 4184 cpu_icache_sync_range(sva, 4185 uimin(page_size, eva - sva)); 4186 } 4187 } 4188 } 4189 4190 pmap_release_pmap_lock(pm); 4191 } 4192 4193 void 4194 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 4195 { 4196 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4197 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4198 4199 UVMHIST_FUNC(__func__); 4200 UVMHIST_CALLARGS(maphist, "md %#jx pa %#jx prot %#jx", 4201 (uintptr_t)md, pa, prot, 0); 4202 4203 switch(prot) { 4204 case VM_PROT_READ|VM_PROT_WRITE: 4205 #if defined(ARM_MMU_EXTENDED) 4206 pmap_acquire_page_lock(md); 4207 pmap_clearbit(md, pa, PVF_EXEC); 4208 pmap_release_page_lock(md); 4209 break; 4210 #endif 4211 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 4212 break; 4213 4214 case VM_PROT_READ: 4215 #if defined(ARM_MMU_EXTENDED) 4216 pmap_acquire_page_lock(md); 4217 pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC); 4218 pmap_release_page_lock(md); 4219 break; 4220 #endif 4221 case VM_PROT_READ|VM_PROT_EXECUTE: 4222 pmap_acquire_page_lock(md); 4223 pmap_clearbit(md, pa, PVF_WRITE); 4224 pmap_release_page_lock(md); 4225 break; 4226 4227 default: 4228 pmap_page_remove(md, pa); 4229 break; 4230 } 4231 } 4232 4233 /* 4234 * pmap_clear_modify: 4235 * 4236 * Clear the "modified" attribute for a page. 4237 */ 4238 bool 4239 pmap_clear_modify(struct vm_page *pg) 4240 { 4241 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4242 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4243 bool rv; 4244 4245 pmap_acquire_page_lock(md); 4246 4247 if (md->pvh_attrs & PVF_MOD) { 4248 rv = true; 4249 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4250 /* 4251 * If we are going to clear the modified bit and there are 4252 * no other modified bits set, flush the page to memory and 4253 * mark it clean. 4254 */ 4255 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD) 4256 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 4257 #endif 4258 pmap_clearbit(md, pa, PVF_MOD); 4259 } else { 4260 rv = false; 4261 } 4262 pmap_release_page_lock(md); 4263 4264 return rv; 4265 } 4266 4267 /* 4268 * pmap_clear_reference: 4269 * 4270 * Clear the "referenced" attribute for a page. 4271 */ 4272 bool 4273 pmap_clear_reference(struct vm_page *pg) 4274 { 4275 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4276 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4277 bool rv; 4278 4279 pmap_acquire_page_lock(md); 4280 4281 if (md->pvh_attrs & PVF_REF) { 4282 rv = true; 4283 pmap_clearbit(md, pa, PVF_REF); 4284 } else { 4285 rv = false; 4286 } 4287 pmap_release_page_lock(md); 4288 4289 return rv; 4290 } 4291 4292 /* 4293 * pmap_is_modified: 4294 * 4295 * Test if a page has the "modified" attribute. 4296 */ 4297 /* See <arm/arm32/pmap.h> */ 4298 4299 /* 4300 * pmap_is_referenced: 4301 * 4302 * Test if a page has the "referenced" attribute. 4303 */ 4304 /* See <arm/arm32/pmap.h> */ 4305 4306 #if defined(ARM_MMU_EXTENDED) && 0 4307 int 4308 pmap_prefetchabt_fixup(void *v) 4309 { 4310 struct trapframe * const tf = v; 4311 vaddr_t va = trunc_page(tf->tf_pc); 4312 int rv = ABORT_FIXUP_FAILED; 4313 4314 if (!TRAP_USERMODE(tf) && va < VM_MAXUSER_ADDRESS) 4315 return rv; 4316 4317 kpreempt_disable(); 4318 pmap_t pm = curcpu()->ci_pmap_cur; 4319 const size_t l1slot = l1pte_index(va); 4320 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 4321 if (l2 == NULL) 4322 goto out; 4323 4324 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4325 if (l2b->l2b_kva == NULL) 4326 goto out; 4327 4328 /* 4329 * Check the PTE itself. 4330 */ 4331 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4332 const pt_entry_t opte = *ptep; 4333 if ((opte & L2_S_PROT_U) == 0 || (opte & L2_XS_XN) == 0) 4334 goto out; 4335 4336 paddr_t pa = l2pte_pa(opte); 4337 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4338 KASSERT(pg != NULL); 4339 4340 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4341 4342 pmap_acquire_page_lock(md); 4343 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4344 KASSERT(pv != NULL); 4345 4346 if (PV_IS_EXEC_P(pv->pv_flags)) { 4347 l2pte_reset(ptep); 4348 PTE_SYNC(ptep); 4349 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4350 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 4351 pmap_syncicache_page(md, pa); 4352 } 4353 rv = ABORT_FIXUP_RETURN; 4354 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4355 PTE_SYNC(ptep); 4356 } 4357 pmap_release_page_lock(md); 4358 4359 out: 4360 kpreempt_enable(); 4361 return rv; 4362 } 4363 #endif 4364 4365 int 4366 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 4367 { 4368 struct l2_dtable *l2; 4369 struct l2_bucket *l2b; 4370 paddr_t pa; 4371 const size_t l1slot = l1pte_index(va); 4372 int rv = 0; 4373 4374 UVMHIST_FUNC(__func__); 4375 UVMHIST_CALLARGS(maphist, "pm=%#jx, va=%#jx, ftype=%#jx, user=%jd", 4376 (uintptr_t)pm, va, ftype, user); 4377 4378 va = trunc_page(va); 4379 4380 KASSERT(!user || (pm != pmap_kernel())); 4381 4382 #ifdef ARM_MMU_EXTENDED 4383 UVMHIST_LOG(maphist, " ti=%#jx pai=%#jx asid=%#jx", 4384 (uintptr_t)cpu_tlb_info(curcpu()), 4385 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu())), 4386 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu()))->pai_asid, 0); 4387 #endif 4388 4389 pmap_acquire_pmap_lock(pm); 4390 4391 /* 4392 * If there is no l2_dtable for this address, then the process 4393 * has no business accessing it. 4394 * 4395 * Note: This will catch userland processes trying to access 4396 * kernel addresses. 4397 */ 4398 l2 = pm->pm_l2[L2_IDX(l1slot)]; 4399 if (l2 == NULL) { 4400 UVMHIST_LOG(maphist, " no l2 for l1slot %#jx", l1slot, 0, 0, 0); 4401 goto out; 4402 } 4403 4404 /* 4405 * Likewise if there is no L2 descriptor table 4406 */ 4407 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4408 if (l2b->l2b_kva == NULL) { 4409 UVMHIST_LOG(maphist, " <-- done (no ptep for l1slot %#jx)", 4410 l1slot, 0, 0, 0); 4411 goto out; 4412 } 4413 4414 /* 4415 * Check the PTE itself. 4416 */ 4417 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4418 pt_entry_t const opte = *ptep; 4419 if (opte == 0 || (opte & L2_TYPE_MASK) == L2_TYPE_L) { 4420 UVMHIST_LOG(maphist, " <-- done (empty pde for l1slot %#jx)", 4421 l1slot, 0, 0, 0); 4422 goto out; 4423 } 4424 4425 #ifndef ARM_HAS_VBAR 4426 /* 4427 * Catch a userland access to the vector page mapped at 0x0 4428 */ 4429 if (user && (opte & L2_S_PROT_U) == 0) { 4430 UVMHIST_LOG(maphist, " <-- done (vector_page)", 0, 0, 0, 0); 4431 goto out; 4432 } 4433 #endif 4434 4435 pa = l2pte_pa(opte); 4436 4437 if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(opte)) { 4438 /* 4439 * This looks like a good candidate for "page modified" 4440 * emulation... 4441 */ 4442 struct pv_entry *pv; 4443 struct vm_page *pg; 4444 4445 /* Extract the physical address of the page */ 4446 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4447 UVMHIST_LOG(maphist, " <-- done (mod/ref unmanaged page)", 0, 0, 0, 0); 4448 goto out; 4449 } 4450 4451 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4452 4453 /* Get the current flags for this page. */ 4454 pmap_acquire_page_lock(md); 4455 pv = pmap_find_pv(md, pm, va); 4456 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4457 pmap_release_page_lock(md); 4458 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: no PV)", 0, 0, 0, 0); 4459 goto out; 4460 } 4461 4462 /* 4463 * Do the flags say this page is writable? If not then it 4464 * is a genuine write fault. If yes then the write fault is 4465 * our fault as we did not reflect the write access in the 4466 * PTE. Now we know a write has occurred we can correct this 4467 * and also set the modified bit 4468 */ 4469 if ((pv->pv_flags & PVF_WRITE) == 0) { 4470 pmap_release_page_lock(md); 4471 goto out; 4472 } 4473 4474 md->pvh_attrs |= PVF_REF | PVF_MOD; 4475 pv->pv_flags |= PVF_REF | PVF_MOD; 4476 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4477 /* 4478 * If there are cacheable mappings for this page, mark it dirty. 4479 */ 4480 if ((md->pvh_attrs & PVF_NC) == 0) 4481 md->pvh_attrs |= PVF_DIRTY; 4482 #endif 4483 #ifdef ARM_MMU_EXTENDED 4484 if (md->pvh_attrs & PVF_EXEC) { 4485 md->pvh_attrs &= ~PVF_EXEC; 4486 PMAPCOUNT(exec_discarded_modfixup); 4487 } 4488 #endif 4489 pmap_release_page_lock(md); 4490 4491 /* 4492 * Re-enable write permissions for the page. No need to call 4493 * pmap_vac_me_harder(), since this is just a 4494 * modified-emulation fault, and the PVF_WRITE bit isn't 4495 * changing. We've already set the cacheable bits based on 4496 * the assumption that we can write to this page. 4497 */ 4498 const pt_entry_t npte = 4499 l2pte_set_writable((opte & ~L2_TYPE_MASK) | L2_S_PROTO) 4500 #ifdef ARM_MMU_EXTENDED 4501 | (pm != pmap_kernel() ? L2_XS_nG : 0) 4502 #endif 4503 | 0; 4504 l2pte_reset(ptep); 4505 PTE_SYNC(ptep); 4506 pmap_tlb_flush_SE(pm, va, 4507 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4508 l2pte_set(ptep, npte, 0); 4509 PTE_SYNC(ptep); 4510 PMAPCOUNT(fixup_mod); 4511 rv = 1; 4512 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: changed pte " 4513 "from %#jx to %#jx)", opte, npte, 0, 0); 4514 } else if ((opte & L2_TYPE_MASK) == L2_TYPE_INV) { 4515 /* 4516 * This looks like a good candidate for "page referenced" 4517 * emulation. 4518 */ 4519 struct vm_page *pg; 4520 4521 /* Extract the physical address of the page */ 4522 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4523 UVMHIST_LOG(maphist, " <-- done (ref emul: unmanaged page)", 0, 0, 0, 0); 4524 goto out; 4525 } 4526 4527 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4528 4529 /* Get the current flags for this page. */ 4530 pmap_acquire_page_lock(md); 4531 struct pv_entry *pv = pmap_find_pv(md, pm, va); 4532 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4533 pmap_release_page_lock(md); 4534 UVMHIST_LOG(maphist, " <-- done (ref emul no PV)", 0, 0, 0, 0); 4535 goto out; 4536 } 4537 4538 md->pvh_attrs |= PVF_REF; 4539 pv->pv_flags |= PVF_REF; 4540 4541 pt_entry_t npte = 4542 l2pte_set_readonly((opte & ~L2_TYPE_MASK) | L2_S_PROTO); 4543 #ifdef ARM_MMU_EXTENDED 4544 if (pm != pmap_kernel()) { 4545 npte |= L2_XS_nG; 4546 } 4547 /* 4548 * If we got called from prefetch abort, then ftype will have 4549 * VM_PROT_EXECUTE set. Now see if we have no-execute set in 4550 * the PTE. 4551 */ 4552 if (user && (ftype & VM_PROT_EXECUTE) && (npte & L2_XS_XN)) { 4553 /* 4554 * Is this a mapping of an executable page? 4555 */ 4556 if ((pv->pv_flags & PVF_EXEC) == 0) { 4557 pmap_release_page_lock(md); 4558 UVMHIST_LOG(maphist, " <-- done (ref emul: no exec)", 4559 0, 0, 0, 0); 4560 goto out; 4561 } 4562 /* 4563 * If we haven't synced the page, do so now. 4564 */ 4565 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4566 UVMHIST_LOG(maphist, " ref emul: syncicache " 4567 "page #%#jx", pa, 0, 0, 0); 4568 pmap_syncicache_page(md, pa); 4569 PMAPCOUNT(fixup_exec); 4570 } 4571 npte &= ~L2_XS_XN; 4572 } 4573 #endif /* ARM_MMU_EXTENDED */ 4574 pmap_release_page_lock(md); 4575 l2pte_reset(ptep); 4576 PTE_SYNC(ptep); 4577 pmap_tlb_flush_SE(pm, va, 4578 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4579 l2pte_set(ptep, npte, 0); 4580 PTE_SYNC(ptep); 4581 PMAPCOUNT(fixup_ref); 4582 rv = 1; 4583 UVMHIST_LOG(maphist, " <-- done (ref emul: changed pte from " 4584 "%#jx to %#jx)", opte, npte, 0, 0); 4585 #ifdef ARM_MMU_EXTENDED 4586 } else if (user && (ftype & VM_PROT_EXECUTE) && (opte & L2_XS_XN)) { 4587 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4588 if (pg == NULL) { 4589 UVMHIST_LOG(maphist, " <-- done (unmanaged page)", 0, 0, 0, 0); 4590 goto out; 4591 } 4592 4593 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4594 4595 /* Get the current flags for this page. */ 4596 pmap_acquire_page_lock(md); 4597 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4598 if (pv == NULL || (pv->pv_flags & PVF_EXEC) == 0) { 4599 pmap_release_page_lock(md); 4600 UVMHIST_LOG(maphist, " <-- done (no PV or not EXEC)", 0, 0, 0, 0); 4601 goto out; 4602 } 4603 4604 /* 4605 * If we haven't synced the page, do so now. 4606 */ 4607 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4608 UVMHIST_LOG(maphist, "syncicache page #%#jx", 4609 pa, 0, 0, 0); 4610 pmap_syncicache_page(md, pa); 4611 } 4612 pmap_release_page_lock(md); 4613 /* 4614 * Turn off no-execute. 4615 */ 4616 KASSERT(opte & L2_XS_nG); 4617 l2pte_reset(ptep); 4618 PTE_SYNC(ptep); 4619 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4620 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4621 PTE_SYNC(ptep); 4622 rv = 1; 4623 PMAPCOUNT(fixup_exec); 4624 UVMHIST_LOG(maphist, "exec: changed pte from %#jx to %#jx", 4625 opte, opte & ~L2_XS_XN, 0, 0); 4626 #endif 4627 } 4628 4629 #ifndef ARM_MMU_EXTENDED 4630 /* 4631 * We know there is a valid mapping here, so simply 4632 * fix up the L1 if necessary. 4633 */ 4634 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 4635 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa | L1_C_DOM(pmap_domain(pm)); 4636 if (*pdep != pde) { 4637 l1pte_setone(pdep, pde); 4638 PDE_SYNC(pdep); 4639 rv = 1; 4640 PMAPCOUNT(fixup_pdes); 4641 } 4642 #endif 4643 4644 #ifdef CPU_SA110 4645 /* 4646 * There are bugs in the rev K SA110. This is a check for one 4647 * of them. 4648 */ 4649 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 4650 curcpu()->ci_arm_cpurev < 3) { 4651 /* Always current pmap */ 4652 if (l2pte_valid_p(opte)) { 4653 extern int kernel_debug; 4654 if (kernel_debug & 1) { 4655 struct proc *p = curlwp->l_proc; 4656 printf("prefetch_abort: page is already " 4657 "mapped - pte=%p *pte=%08x\n", ptep, opte); 4658 printf("prefetch_abort: pc=%08lx proc=%p " 4659 "process=%s\n", va, p, p->p_comm); 4660 printf("prefetch_abort: far=%08x fs=%x\n", 4661 cpu_faultaddress(), cpu_faultstatus()); 4662 } 4663 #ifdef DDB 4664 if (kernel_debug & 2) 4665 Debugger(); 4666 #endif 4667 rv = 1; 4668 } 4669 } 4670 #endif /* CPU_SA110 */ 4671 4672 #ifndef ARM_MMU_EXTENDED 4673 /* 4674 * If 'rv == 0' at this point, it generally indicates that there is a 4675 * stale TLB entry for the faulting address. That might be due to a 4676 * wrong setting of pmap_needs_pte_sync. So set it and retry. 4677 */ 4678 if (rv == 0 4679 && pm->pm_l1->l1_domain_use_count == 1 4680 && pmap_needs_pte_sync == 0) { 4681 pmap_needs_pte_sync = 1; 4682 PTE_SYNC(ptep); 4683 PMAPCOUNT(fixup_ptesync); 4684 rv = 1; 4685 } 4686 #endif 4687 4688 #ifndef MULTIPROCESSOR 4689 #if defined(DEBUG) || 1 4690 /* 4691 * If 'rv == 0' at this point, it generally indicates that there is a 4692 * stale TLB entry for the faulting address. This happens when two or 4693 * more processes are sharing an L1. Since we don't flush the TLB on 4694 * a context switch between such processes, we can take domain faults 4695 * for mappings which exist at the same VA in both processes. EVEN IF 4696 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 4697 * example. 4698 * 4699 * This is extremely likely to happen if pmap_enter() updated the L1 4700 * entry for a recently entered mapping. In this case, the TLB is 4701 * flushed for the new mapping, but there may still be TLB entries for 4702 * other mappings belonging to other processes in the 1MB range 4703 * covered by the L1 entry. 4704 * 4705 * Since 'rv == 0', we know that the L1 already contains the correct 4706 * value, so the fault must be due to a stale TLB entry. 4707 * 4708 * Since we always need to flush the TLB anyway in the case where we 4709 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 4710 * stale TLB entries dynamically. 4711 * 4712 * However, the above condition can ONLY happen if the current L1 is 4713 * being shared. If it happens when the L1 is unshared, it indicates 4714 * that other parts of the pmap are not doing their job WRT managing 4715 * the TLB. 4716 */ 4717 if (rv == 0 4718 #ifndef ARM_MMU_EXTENDED 4719 && pm->pm_l1->l1_domain_use_count == 1 4720 #endif 4721 && true) { 4722 #ifdef DEBUG 4723 extern int last_fault_code; 4724 #else 4725 int last_fault_code = ftype & VM_PROT_EXECUTE 4726 ? armreg_ifsr_read() 4727 : armreg_dfsr_read(); 4728 #endif 4729 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 4730 pm, va, ftype); 4731 printf("fixup: l2 %p, l2b %p, ptep %p, pte %#x\n", 4732 l2, l2b, ptep, opte); 4733 4734 #ifndef ARM_MMU_EXTENDED 4735 printf("fixup: pdep %p, pde %#x, fsr %#x\n", 4736 pdep, pde, last_fault_code); 4737 #else 4738 printf("fixup: pdep %p, pde %#x, ttbcr %#x\n", 4739 &pmap_l1_kva(pm)[l1slot], pmap_l1_kva(pm)[l1slot], 4740 armreg_ttbcr_read()); 4741 printf("fixup: fsr %#x cpm %p casid %#x contextidr %#x dacr %#x\n", 4742 last_fault_code, curcpu()->ci_pmap_cur, 4743 curcpu()->ci_pmap_asid_cur, 4744 armreg_contextidr_read(), armreg_dacr_read()); 4745 #ifdef _ARM_ARCH_7 4746 if (ftype & VM_PROT_WRITE) 4747 armreg_ats1cuw_write(va); 4748 else 4749 armreg_ats1cur_write(va); 4750 arm_isb(); 4751 printf("fixup: par %#x\n", armreg_par_read()); 4752 #endif 4753 #endif 4754 #ifdef DDB 4755 extern int kernel_debug; 4756 4757 if (kernel_debug & 2) { 4758 pmap_release_pmap_lock(pm); 4759 #ifdef UVMHIST 4760 KERNHIST_DUMP(maphist); 4761 #endif 4762 cpu_Debugger(); 4763 pmap_acquire_pmap_lock(pm); 4764 } 4765 #endif 4766 } 4767 #endif 4768 #endif 4769 4770 #ifndef ARM_MMU_EXTENDED 4771 /* Flush the TLB in the shared L1 case - see comment above */ 4772 pmap_tlb_flush_SE(pm, va, 4773 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4774 #endif 4775 4776 rv = 1; 4777 4778 out: 4779 pmap_release_pmap_lock(pm); 4780 4781 return rv; 4782 } 4783 4784 /* 4785 * Routine: pmap_procwr 4786 * 4787 * Function: 4788 * Synchronize caches corresponding to [addr, addr+len) in p. 4789 * 4790 */ 4791 void 4792 pmap_procwr(struct proc *p, vaddr_t va, int len) 4793 { 4794 #ifndef ARM_MMU_EXTENDED 4795 4796 /* We only need to do anything if it is the current process. */ 4797 if (p == curproc) 4798 cpu_icache_sync_range(va, len); 4799 #endif 4800 } 4801 4802 /* 4803 * Routine: pmap_unwire 4804 * Function: Clear the wired attribute for a map/virtual-address pair. 4805 * 4806 * In/out conditions: 4807 * The mapping must already exist in the pmap. 4808 */ 4809 void 4810 pmap_unwire(pmap_t pm, vaddr_t va) 4811 { 4812 struct l2_bucket *l2b; 4813 pt_entry_t *ptep, pte; 4814 struct vm_page *pg; 4815 paddr_t pa; 4816 4817 UVMHIST_FUNC(__func__); 4818 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx", (uintptr_t)pm, va, 0, 0); 4819 4820 pmap_acquire_pmap_lock(pm); 4821 4822 l2b = pmap_get_l2_bucket(pm, va); 4823 KDASSERT(l2b != NULL); 4824 4825 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4826 pte = *ptep; 4827 4828 /* Extract the physical address of the page */ 4829 pa = l2pte_pa(pte); 4830 4831 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4832 /* Update the wired bit in the pv entry for this page. */ 4833 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4834 4835 pmap_acquire_page_lock(md); 4836 (void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0); 4837 pmap_release_page_lock(md); 4838 } 4839 4840 pmap_release_pmap_lock(pm); 4841 } 4842 4843 #ifdef ARM_MMU_EXTENDED 4844 void 4845 pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) 4846 { 4847 UVMHIST_FUNC(__func__); 4848 struct cpu_info * const ci = curcpu(); 4849 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); 4850 4851 UVMHIST_CALLARGS(maphist, "pm %#jx (pm->pm_l1_pa %08jx asid %ju)", 4852 (uintptr_t)pm, pm->pm_l1_pa, pai->pai_asid, 0); 4853 4854 /* 4855 * Assume that TTBR1 has only global mappings and TTBR0 only 4856 * has non-global mappings. To prevent speculation from doing 4857 * evil things we disable translation table walks using TTBR0 4858 * before setting the CONTEXTIDR (ASID) or new TTBR0 value. 4859 * Once both are set, table walks are reenabled. 4860 */ 4861 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4862 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4863 arm_isb(); 4864 4865 pmap_tlb_asid_acquire(pm, l); 4866 4867 cpu_setttb(pm->pm_l1_pa, pai->pai_asid); 4868 /* 4869 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 4870 * have been updated. 4871 */ 4872 arm_isb(); 4873 4874 if (pm != pmap_kernel()) { 4875 armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); 4876 } 4877 cpu_cpwait(); 4878 4879 KASSERTMSG(ci->ci_pmap_asid_cur == pai->pai_asid, "%u vs %u", 4880 ci->ci_pmap_asid_cur, pai->pai_asid); 4881 ci->ci_pmap_cur = pm; 4882 } 4883 4884 void 4885 pmap_md_pdetab_deactivate(pmap_t pm) 4886 { 4887 4888 UVMHIST_FUNC(__func__); 4889 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 4890 4891 kpreempt_disable(); 4892 struct cpu_info * const ci = curcpu(); 4893 /* 4894 * Disable translation table walks from TTBR0 while no pmap has been 4895 * activated. 4896 */ 4897 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4898 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4899 arm_isb(); 4900 pmap_tlb_asid_deactivate(pm); 4901 cpu_setttb(pmap_kernel()->pm_l1_pa, KERNEL_PID); 4902 arm_isb(); 4903 4904 ci->ci_pmap_cur = pmap_kernel(); 4905 KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", 4906 ci->ci_pmap_asid_cur); 4907 kpreempt_enable(); 4908 } 4909 #endif 4910 4911 void 4912 pmap_activate(struct lwp *l) 4913 { 4914 extern int block_userspace_access; 4915 pmap_t npm = l->l_proc->p_vmspace->vm_map.pmap; 4916 4917 UVMHIST_FUNC(__func__); 4918 UVMHIST_CALLARGS(maphist, "l=%#jx pm=%#jx", (uintptr_t)l, 4919 (uintptr_t)npm, 0, 0); 4920 4921 struct cpu_info * const ci = curcpu(); 4922 4923 /* 4924 * If activating a non-current lwp or the current lwp is 4925 * already active, just return. 4926 */ 4927 if (false 4928 || l != curlwp 4929 #ifdef ARM_MMU_EXTENDED 4930 || (ci->ci_pmap_cur == npm && 4931 (npm == pmap_kernel() 4932 /* || PMAP_PAI_ASIDVALID_P(pai, cpu_tlb_info(ci)) */)) 4933 #else 4934 || npm->pm_activated == true 4935 #endif 4936 || false) { 4937 UVMHIST_LOG(maphist, " <-- (same pmap)", (uintptr_t)curlwp, 4938 (uintptr_t)l, 0, 0); 4939 return; 4940 } 4941 4942 #ifndef ARM_MMU_EXTENDED 4943 const uint32_t ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4944 | (DOMAIN_CLIENT << (pmap_domain(npm) * 2)); 4945 4946 /* 4947 * If TTB and DACR are unchanged, short-circuit all the 4948 * TLB/cache management stuff. 4949 */ 4950 pmap_t opm = ci->ci_lastlwp 4951 ? ci->ci_lastlwp->l_proc->p_vmspace->vm_map.pmap 4952 : NULL; 4953 if (opm != NULL) { 4954 uint32_t odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4955 | (DOMAIN_CLIENT << (pmap_domain(opm) * 2)); 4956 4957 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4958 goto all_done; 4959 } 4960 #endif /* !ARM_MMU_EXTENDED */ 4961 4962 PMAPCOUNT(activations); 4963 block_userspace_access = 1; 4964 4965 #ifndef ARM_MMU_EXTENDED 4966 /* 4967 * If switching to a user vmspace which is different to the 4968 * most recent one, and the most recent one is potentially 4969 * live in the cache, we must write-back and invalidate the 4970 * entire cache. 4971 */ 4972 pmap_t rpm = ci->ci_pmap_lastuser; 4973 4974 /* 4975 * XXXSCW: There's a corner case here which can leave turds in the 4976 * cache as reported in kern/41058. They're probably left over during 4977 * tear-down and switching away from an exiting process. Until the root 4978 * cause is identified and fixed, zap the cache when switching pmaps. 4979 * This will result in a few unnecessary cache flushes, but that's 4980 * better than silently corrupting data. 4981 */ 4982 #if 0 4983 if (npm != pmap_kernel() && rpm && npm != rpm && 4984 rpm->pm_cstate.cs_cache) { 4985 rpm->pm_cstate.cs_cache = 0; 4986 #ifdef PMAP_CACHE_VIVT 4987 cpu_idcache_wbinv_all(); 4988 #endif 4989 } 4990 #else 4991 if (rpm) { 4992 rpm->pm_cstate.cs_cache = 0; 4993 if (npm == pmap_kernel()) 4994 ci->ci_pmap_lastuser = NULL; 4995 #ifdef PMAP_CACHE_VIVT 4996 cpu_idcache_wbinv_all(); 4997 #endif 4998 } 4999 #endif 5000 5001 /* No interrupts while we frob the TTB/DACR */ 5002 uint32_t oldirqstate = disable_interrupts(IF32_bits); 5003 #endif /* !ARM_MMU_EXTENDED */ 5004 5005 #ifndef ARM_HAS_VBAR 5006 /* 5007 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 5008 * entry corresponding to 'vector_page' in the incoming L1 table 5009 * before switching to it otherwise subsequent interrupts/exceptions 5010 * (including domain faults!) will jump into hyperspace. 5011 */ 5012 if (npm->pm_pl1vec != NULL) { 5013 cpu_tlb_flushID_SE((u_int)vector_page); 5014 cpu_cpwait(); 5015 *npm->pm_pl1vec = npm->pm_l1vec; 5016 PTE_SYNC(npm->pm_pl1vec); 5017 } 5018 #endif 5019 5020 #ifdef ARM_MMU_EXTENDED 5021 pmap_md_pdetab_activate(npm, l); 5022 #else 5023 cpu_domains(ndacr); 5024 if (npm == pmap_kernel() || npm == rpm) { 5025 /* 5026 * Switching to a kernel thread, or back to the 5027 * same user vmspace as before... Simply update 5028 * the TTB (no TLB flush required) 5029 */ 5030 cpu_setttb(npm->pm_l1->l1_physaddr, false); 5031 cpu_cpwait(); 5032 } else { 5033 /* 5034 * Otherwise, update TTB and flush TLB 5035 */ 5036 cpu_context_switch(npm->pm_l1->l1_physaddr); 5037 if (rpm != NULL) 5038 rpm->pm_cstate.cs_tlb = 0; 5039 } 5040 5041 restore_interrupts(oldirqstate); 5042 #endif /* ARM_MMU_EXTENDED */ 5043 5044 block_userspace_access = 0; 5045 5046 #ifndef ARM_MMU_EXTENDED 5047 all_done: 5048 /* 5049 * The new pmap is resident. Make sure it's marked 5050 * as resident in the cache/TLB. 5051 */ 5052 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5053 if (npm != pmap_kernel()) 5054 ci->ci_pmap_lastuser = npm; 5055 5056 /* The old pmap is not longer active */ 5057 if (opm != npm) { 5058 if (opm != NULL) 5059 opm->pm_activated = false; 5060 5061 /* But the new one is */ 5062 npm->pm_activated = true; 5063 } 5064 ci->ci_pmap_cur = npm; 5065 #endif 5066 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5067 } 5068 5069 void 5070 pmap_deactivate(struct lwp *l) 5071 { 5072 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 5073 5074 UVMHIST_FUNC(__func__); 5075 UVMHIST_CALLARGS(maphist, "l=%#jx (pm=%#jx)", (uintptr_t)l, 5076 (uintptr_t)pm, 0, 0); 5077 5078 #ifdef ARM_MMU_EXTENDED 5079 pmap_md_pdetab_deactivate(pm); 5080 #else 5081 /* 5082 * If the process is exiting, make sure pmap_activate() does 5083 * a full MMU context-switch and cache flush, which we might 5084 * otherwise skip. See PR port-arm/38950. 5085 */ 5086 if (l->l_proc->p_sflag & PS_WEXIT) 5087 curcpu()->ci_lastlwp = NULL; 5088 5089 pm->pm_activated = false; 5090 #endif 5091 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5092 } 5093 5094 void 5095 pmap_update(pmap_t pm) 5096 { 5097 5098 UVMHIST_FUNC(__func__); 5099 UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5100 pm->pm_remove_all, 0, 0); 5101 5102 #ifndef ARM_MMU_EXTENDED 5103 if (pm->pm_remove_all) { 5104 /* 5105 * Finish up the pmap_remove_all() optimisation by flushing 5106 * the TLB. 5107 */ 5108 pmap_tlb_flushID(pm); 5109 pm->pm_remove_all = false; 5110 } 5111 5112 if (pmap_is_current(pm)) { 5113 /* 5114 * If we're dealing with a current userland pmap, move its L1 5115 * to the end of the LRU. 5116 */ 5117 if (pm != pmap_kernel()) 5118 pmap_use_l1(pm); 5119 5120 /* 5121 * We can assume we're done with frobbing the cache/tlb for 5122 * now. Make sure any future pmap ops don't skip cache/tlb 5123 * flushes. 5124 */ 5125 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5126 } 5127 #else 5128 5129 kpreempt_disable(); 5130 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 5131 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 5132 if (pending && pmap_tlb_shootdown_bystanders(pmap)) { 5133 PMAP_COUNT(shootdown_ipis); 5134 } 5135 #endif 5136 5137 /* 5138 * If pmap_remove_all was called, we deactivated ourselves and released 5139 * our ASID. Now we have to reactivate ourselves. 5140 */ 5141 if (__predict_false(pm->pm_remove_all)) { 5142 pm->pm_remove_all = false; 5143 5144 KASSERT(pm != pmap_kernel()); 5145 pmap_md_pdetab_activate(pm, curlwp); 5146 } 5147 5148 if (arm_has_mpext_p) 5149 armreg_bpiallis_write(0); 5150 else 5151 armreg_bpiall_write(0); 5152 5153 kpreempt_enable(); 5154 5155 KASSERTMSG(pm == pmap_kernel() 5156 || curcpu()->ci_pmap_cur != pm 5157 || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, 5158 "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, 5159 pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, 5160 curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur); 5161 #endif 5162 5163 PMAPCOUNT(updates); 5164 5165 /* 5166 * make sure TLB/cache operations have completed. 5167 */ 5168 cpu_cpwait(); 5169 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5170 } 5171 5172 bool 5173 pmap_remove_all(pmap_t pm) 5174 { 5175 5176 /* 5177 * The vmspace described by this pmap is about to be torn down. 5178 * Until pmap_update() is called, UVM will only make calls 5179 * to pmap_remove(). We can make life much simpler by flushing 5180 * the cache now, and deferring TLB invalidation to pmap_update(). 5181 */ 5182 #ifdef PMAP_CACHE_VIVT 5183 pmap_cache_wbinv_all(pm, PVF_EXEC); 5184 #endif 5185 #ifdef ARM_MMU_EXTENDED 5186 #ifdef MULTIPROCESSOR 5187 struct cpu_info * const ci = curcpu(); 5188 // This should be the last CPU with this pmap onproc 5189 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 5190 if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) 5191 #endif 5192 pmap_tlb_asid_deactivate(pm); 5193 #ifdef MULTIPROCESSOR 5194 KASSERT(kcpuset_iszero(pm->pm_onproc)); 5195 #endif 5196 5197 pmap_tlb_asid_release_all(pm); 5198 #endif 5199 pm->pm_remove_all = true; 5200 return false; 5201 } 5202 5203 /* 5204 * Retire the given physical map from service. 5205 * Should only be called if the map contains no valid mappings. 5206 */ 5207 void 5208 pmap_destroy(pmap_t pm) 5209 { 5210 UVMHIST_FUNC(__func__); 5211 UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5212 pm ? pm->pm_remove_all : 0, 0, 0); 5213 5214 if (pm == NULL) 5215 return; 5216 5217 if (pm->pm_remove_all) { 5218 #ifdef ARM_MMU_EXTENDED 5219 pmap_tlb_asid_release_all(pm); 5220 #else 5221 pmap_tlb_flushID(pm); 5222 #endif 5223 pm->pm_remove_all = false; 5224 } 5225 5226 /* 5227 * Drop reference count 5228 */ 5229 if (atomic_dec_uint_nv(&pm->pm_refs) > 0) { 5230 #ifndef ARM_MMU_EXTENDED 5231 if (pmap_is_current(pm)) { 5232 if (pm != pmap_kernel()) 5233 pmap_use_l1(pm); 5234 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5235 } 5236 #endif 5237 return; 5238 } 5239 5240 /* 5241 * reference count is zero, free pmap resources and then free pmap. 5242 */ 5243 5244 #ifndef ARM_HAS_VBAR 5245 if (vector_page < KERNEL_BASE) { 5246 KDASSERT(!pmap_is_current(pm)); 5247 5248 /* Remove the vector page mapping */ 5249 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 5250 pmap_update(pm); 5251 } 5252 #endif 5253 5254 pmap_free_l1(pm); 5255 5256 #ifdef ARM_MMU_EXTENDED 5257 #ifdef MULTIPROCESSOR 5258 kcpuset_destroy(pm->pm_active); 5259 kcpuset_destroy(pm->pm_onproc); 5260 #endif 5261 #else 5262 struct cpu_info * const ci = curcpu(); 5263 if (ci->ci_pmap_lastuser == pm) 5264 ci->ci_pmap_lastuser = NULL; 5265 #endif 5266 5267 mutex_destroy(&pm->pm_lock); 5268 pool_cache_put(&pmap_cache, pm); 5269 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5270 } 5271 5272 5273 /* 5274 * void pmap_reference(pmap_t pm) 5275 * 5276 * Add a reference to the specified pmap. 5277 */ 5278 void 5279 pmap_reference(pmap_t pm) 5280 { 5281 5282 if (pm == NULL) 5283 return; 5284 5285 #ifndef ARM_MMU_EXTENDED 5286 pmap_use_l1(pm); 5287 #endif 5288 5289 atomic_inc_uint(&pm->pm_refs); 5290 } 5291 5292 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 5293 5294 static struct evcnt pmap_prefer_nochange_ev = 5295 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 5296 static struct evcnt pmap_prefer_change_ev = 5297 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 5298 5299 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 5300 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 5301 5302 void 5303 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 5304 { 5305 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 5306 vaddr_t va = *vap; 5307 vaddr_t diff = (hint - va) & mask; 5308 if (diff == 0) { 5309 pmap_prefer_nochange_ev.ev_count++; 5310 } else { 5311 pmap_prefer_change_ev.ev_count++; 5312 if (__predict_false(td)) 5313 va -= mask + 1; 5314 *vap = va + diff; 5315 } 5316 } 5317 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */ 5318 5319 /* 5320 * pmap_zero_page() 5321 * 5322 * Zero a given physical page by mapping it at a page hook point. 5323 * In doing the zero page op, the page we zero is mapped cachable, as with 5324 * StrongARM accesses to non-cached pages are non-burst making writing 5325 * _any_ bulk data very slow. 5326 */ 5327 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5328 void 5329 pmap_zero_page_generic(paddr_t pa) 5330 { 5331 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5332 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5333 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5334 #endif 5335 #if defined(PMAP_CACHE_VIPT) 5336 /* Choose the last page color it had, if any */ 5337 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5338 #else 5339 const vsize_t va_offset = 0; 5340 #endif 5341 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5342 /* 5343 * Is this page mapped at its natural color? 5344 * If we have all of memory mapped, then just convert PA to VA. 5345 */ 5346 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5347 || va_offset == (pa & arm_cache_prefer_mask); 5348 const vaddr_t vdstp = okcolor 5349 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5350 : cpu_cdstp(va_offset); 5351 #else 5352 const bool okcolor = false; 5353 const vaddr_t vdstp = cpu_cdstp(va_offset); 5354 #endif 5355 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5356 5357 5358 #ifdef DEBUG 5359 if (!SLIST_EMPTY(&md->pvh_list)) 5360 panic("pmap_zero_page: page has mappings"); 5361 #endif 5362 5363 KDASSERT((pa & PGOFSET) == 0); 5364 5365 if (!okcolor) { 5366 /* 5367 * Hook in the page, zero it, and purge the cache for that 5368 * zeroed page. Invalidate the TLB as needed. 5369 */ 5370 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 5371 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); 5372 l2pte_set(ptep, npte, 0); 5373 PTE_SYNC(ptep); 5374 cpu_tlb_flushD_SE(vdstp); 5375 cpu_cpwait(); 5376 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) \ 5377 && !defined(ARM_MMU_EXTENDED) 5378 /* 5379 * If we are direct-mapped and our color isn't ok, then before 5380 * we bzero the page invalidate its contents from the cache and 5381 * reset the color to its natural color. 5382 */ 5383 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5384 md->pvh_attrs &= ~arm_cache_prefer_mask; 5385 md->pvh_attrs |= (pa & arm_cache_prefer_mask); 5386 #endif 5387 } 5388 bzero_page(vdstp); 5389 if (!okcolor) { 5390 /* 5391 * Unmap the page. 5392 */ 5393 l2pte_reset(ptep); 5394 PTE_SYNC(ptep); 5395 cpu_tlb_flushD_SE(vdstp); 5396 #ifdef PMAP_CACHE_VIVT 5397 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5398 #endif 5399 } 5400 #ifdef PMAP_CACHE_VIPT 5401 /* 5402 * This page is now cache resident so it now has a page color. 5403 * Any contents have been obliterated so clear the EXEC flag. 5404 */ 5405 #ifndef ARM_MMU_EXTENDED 5406 if (!pmap_is_page_colored_p(md)) { 5407 PMAPCOUNT(vac_color_new); 5408 md->pvh_attrs |= PVF_COLORED; 5409 } 5410 md->pvh_attrs |= PVF_DIRTY; 5411 #endif 5412 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5413 md->pvh_attrs &= ~PVF_EXEC; 5414 PMAPCOUNT(exec_discarded_zero); 5415 } 5416 #endif 5417 } 5418 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5419 5420 #if ARM_MMU_XSCALE == 1 5421 void 5422 pmap_zero_page_xscale(paddr_t pa) 5423 { 5424 #ifdef DEBUG 5425 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5426 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5427 5428 if (!SLIST_EMPTY(&md->pvh_list)) 5429 panic("pmap_zero_page: page has mappings"); 5430 #endif 5431 5432 KDASSERT((pa & PGOFSET) == 0); 5433 5434 /* 5435 * Hook in the page, zero it, and purge the cache for that 5436 * zeroed page. Invalidate the TLB as needed. 5437 */ 5438 5439 pt_entry_t npte = L2_S_PROTO | pa | 5440 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 5441 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5442 l2pte_set(cdst_pte, npte, 0); 5443 PTE_SYNC(cdst_pte); 5444 cpu_tlb_flushD_SE(cdstp); 5445 cpu_cpwait(); 5446 bzero_page(cdstp); 5447 xscale_cache_clean_minidata(); 5448 l2pte_reset(cdst_pte); 5449 PTE_SYNC(cdst_pte); 5450 } 5451 #endif /* ARM_MMU_XSCALE == 1 */ 5452 5453 /* pmap_pageidlezero() 5454 * 5455 * The same as above, except that we assume that the page is not 5456 * mapped. This means we never have to flush the cache first. Called 5457 * from the idle loop. 5458 */ 5459 bool 5460 pmap_pageidlezero(paddr_t pa) 5461 { 5462 bool rv = true; 5463 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5464 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 5465 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5466 #endif 5467 #ifdef PMAP_CACHE_VIPT 5468 /* Choose the last page color it had, if any */ 5469 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5470 #else 5471 const vsize_t va_offset = 0; 5472 #endif 5473 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 5474 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5475 || va_offset == (pa & arm_cache_prefer_mask); 5476 const vaddr_t vdstp = okcolor 5477 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5478 : cpu_cdstp(va_offset); 5479 #else 5480 const bool okcolor = false; 5481 const vaddr_t vdstp = cpu_cdstp(va_offset); 5482 #endif 5483 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5484 5485 5486 #ifdef DEBUG 5487 if (!SLIST_EMPTY(&md->pvh_list)) 5488 panic("pmap_pageidlezero: page has mappings"); 5489 #endif 5490 5491 KDASSERT((pa & PGOFSET) == 0); 5492 5493 if (!okcolor) { 5494 /* 5495 * Hook in the page, zero it, and purge the cache for that 5496 * zeroed page. Invalidate the TLB as needed. 5497 */ 5498 const pt_entry_t npte = L2_S_PROTO | pa | 5499 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5500 l2pte_set(ptep, npte, 0); 5501 PTE_SYNC(ptep); 5502 cpu_tlb_flushD_SE(vdstp); 5503 cpu_cpwait(); 5504 } 5505 5506 uint64_t *ptr = (uint64_t *)vdstp; 5507 for (size_t i = 0; i < PAGE_SIZE / sizeof(*ptr); i++) { 5508 if (sched_curcpu_runnable_p() != 0) { 5509 /* 5510 * A process has become ready. Abort now, 5511 * so we don't keep it waiting while we 5512 * do slow memory access to finish this 5513 * page. 5514 */ 5515 rv = false; 5516 break; 5517 } 5518 *ptr++ = 0; 5519 } 5520 5521 #ifdef PMAP_CACHE_VIVT 5522 if (rv) 5523 /* 5524 * if we aborted we'll rezero this page again later so don't 5525 * purge it unless we finished it 5526 */ 5527 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5528 #elif defined(PMAP_CACHE_VIPT) 5529 /* 5530 * This page is now cache resident so it now has a page color. 5531 * Any contents have been obliterated so clear the EXEC flag. 5532 */ 5533 #ifndef ARM_MMU_EXTENDED 5534 if (!pmap_is_page_colored_p(md)) { 5535 PMAPCOUNT(vac_color_new); 5536 md->pvh_attrs |= PVF_COLORED; 5537 } 5538 #endif 5539 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5540 md->pvh_attrs &= ~PVF_EXEC; 5541 PMAPCOUNT(exec_discarded_zero); 5542 } 5543 #endif 5544 /* 5545 * Unmap the page. 5546 */ 5547 if (!okcolor) { 5548 l2pte_reset(ptep); 5549 PTE_SYNC(ptep); 5550 cpu_tlb_flushD_SE(vdstp); 5551 } 5552 5553 return rv; 5554 } 5555 5556 /* 5557 * pmap_copy_page() 5558 * 5559 * Copy one physical page into another, by mapping the pages into 5560 * hook points. The same comment regarding cachability as in 5561 * pmap_zero_page also applies here. 5562 */ 5563 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5564 void 5565 pmap_copy_page_generic(paddr_t src, paddr_t dst) 5566 { 5567 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 5568 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5569 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5570 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 5571 struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg); 5572 #endif 5573 #ifdef PMAP_CACHE_VIPT 5574 const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask; 5575 const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask; 5576 #else 5577 const vsize_t src_va_offset = 0; 5578 const vsize_t dst_va_offset = 0; 5579 #endif 5580 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5581 /* 5582 * Is this page mapped at its natural color? 5583 * If we have all of memory mapped, then just convert PA to VA. 5584 */ 5585 bool src_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5586 || src_va_offset == (src & arm_cache_prefer_mask); 5587 bool dst_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5588 || dst_va_offset == (dst & arm_cache_prefer_mask); 5589 const vaddr_t vsrcp = src_okcolor 5590 ? pmap_direct_mapped_phys(src, &src_okcolor, 5591 cpu_csrcp(src_va_offset)) 5592 : cpu_csrcp(src_va_offset); 5593 const vaddr_t vdstp = pmap_direct_mapped_phys(dst, &dst_okcolor, 5594 cpu_cdstp(dst_va_offset)); 5595 #else 5596 const bool src_okcolor = false; 5597 const bool dst_okcolor = false; 5598 const vaddr_t vsrcp = cpu_csrcp(src_va_offset); 5599 const vaddr_t vdstp = cpu_cdstp(dst_va_offset); 5600 #endif 5601 pt_entry_t * const src_ptep = cpu_csrc_pte(src_va_offset); 5602 pt_entry_t * const dst_ptep = cpu_cdst_pte(dst_va_offset); 5603 5604 #ifdef DEBUG 5605 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5606 panic("pmap_copy_page: dst page has mappings"); 5607 #endif 5608 5609 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5610 KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC)); 5611 #endif 5612 KDASSERT((src & PGOFSET) == 0); 5613 KDASSERT((dst & PGOFSET) == 0); 5614 5615 /* 5616 * Clean the source page. Hold the source page's lock for 5617 * the duration of the copy so that no other mappings can 5618 * be created while we have a potentially aliased mapping. 5619 */ 5620 #ifdef PMAP_CACHE_VIVT 5621 pmap_acquire_page_lock(src_md); 5622 (void) pmap_clean_page(src_md, true); 5623 pmap_release_page_lock(src_md); 5624 #endif 5625 5626 /* 5627 * Map the pages into the page hook points, copy them, and purge 5628 * the cache for the appropriate page. Invalidate the TLB 5629 * as required. 5630 */ 5631 if (!src_okcolor) { 5632 const pt_entry_t nsrc_pte = L2_S_PROTO 5633 | src 5634 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5635 | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 5636 #else // defined(PMAP_CACHE_VIVT) || defined(ARM_MMU_EXTENDED) 5637 | pte_l2_s_cache_mode 5638 #endif 5639 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 5640 l2pte_set(src_ptep, nsrc_pte, 0); 5641 PTE_SYNC(src_ptep); 5642 cpu_tlb_flushD_SE(vsrcp); 5643 cpu_cpwait(); 5644 } 5645 if (!dst_okcolor) { 5646 const pt_entry_t ndst_pte = L2_S_PROTO | dst | 5647 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5648 l2pte_set(dst_ptep, ndst_pte, 0); 5649 PTE_SYNC(dst_ptep); 5650 cpu_tlb_flushD_SE(vdstp); 5651 cpu_cpwait(); 5652 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 5653 /* 5654 * If we are direct-mapped and our color isn't ok, then before 5655 * we bcopy to the new page invalidate its contents from the 5656 * cache and reset its color to its natural color. 5657 */ 5658 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5659 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5660 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5661 #endif 5662 } 5663 bcopy_page(vsrcp, vdstp); 5664 #ifdef PMAP_CACHE_VIVT 5665 cpu_dcache_inv_range(vsrcp, PAGE_SIZE); 5666 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5667 #endif 5668 /* 5669 * Unmap the pages. 5670 */ 5671 if (!src_okcolor) { 5672 l2pte_reset(src_ptep); 5673 PTE_SYNC(src_ptep); 5674 cpu_tlb_flushD_SE(vsrcp); 5675 cpu_cpwait(); 5676 } 5677 if (!dst_okcolor) { 5678 l2pte_reset(dst_ptep); 5679 PTE_SYNC(dst_ptep); 5680 cpu_tlb_flushD_SE(vdstp); 5681 cpu_cpwait(); 5682 } 5683 #ifdef PMAP_CACHE_VIPT 5684 /* 5685 * Now that the destination page is in the cache, mark it as colored. 5686 * If this was an exec page, discard it. 5687 */ 5688 pmap_acquire_page_lock(dst_md); 5689 #ifndef ARM_MMU_EXTENDED 5690 if (arm_pcache.cache_type == CACHE_TYPE_PIPT) { 5691 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5692 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5693 } 5694 if (!pmap_is_page_colored_p(dst_md)) { 5695 PMAPCOUNT(vac_color_new); 5696 dst_md->pvh_attrs |= PVF_COLORED; 5697 } 5698 dst_md->pvh_attrs |= PVF_DIRTY; 5699 #endif 5700 if (PV_IS_EXEC_P(dst_md->pvh_attrs)) { 5701 dst_md->pvh_attrs &= ~PVF_EXEC; 5702 PMAPCOUNT(exec_discarded_copy); 5703 } 5704 pmap_release_page_lock(dst_md); 5705 #endif 5706 } 5707 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5708 5709 #if ARM_MMU_XSCALE == 1 5710 void 5711 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 5712 { 5713 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 5714 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5715 #ifdef DEBUG 5716 struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)); 5717 5718 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5719 panic("pmap_copy_page: dst page has mappings"); 5720 #endif 5721 5722 KDASSERT((src & PGOFSET) == 0); 5723 KDASSERT((dst & PGOFSET) == 0); 5724 5725 /* 5726 * Clean the source page. Hold the source page's lock for 5727 * the duration of the copy so that no other mappings can 5728 * be created while we have a potentially aliased mapping. 5729 */ 5730 #ifdef PMAP_CACHE_VIVT 5731 pmap_acquire_page_lock(src_md); 5732 (void) pmap_clean_page(src_md, true); 5733 pmap_release_page_lock(src_md); 5734 #endif 5735 5736 /* 5737 * Map the pages into the page hook points, copy them, and purge 5738 * the cache for the appropriate page. Invalidate the TLB 5739 * as required. 5740 */ 5741 const pt_entry_t nsrc_pte = L2_S_PROTO | src 5742 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 5743 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5744 l2pte_set(csrc_pte, nsrc_pte, 0); 5745 PTE_SYNC(csrc_pte); 5746 5747 const pt_entry_t ndst_pte = L2_S_PROTO | dst 5748 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) 5749 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5750 l2pte_set(cdst_pte, ndst_pte, 0); 5751 PTE_SYNC(cdst_pte); 5752 5753 cpu_tlb_flushD_SE(csrcp); 5754 cpu_tlb_flushD_SE(cdstp); 5755 cpu_cpwait(); 5756 bcopy_page(csrcp, cdstp); 5757 xscale_cache_clean_minidata(); 5758 l2pte_reset(csrc_pte); 5759 l2pte_reset(cdst_pte); 5760 PTE_SYNC(csrc_pte); 5761 PTE_SYNC(cdst_pte); 5762 } 5763 #endif /* ARM_MMU_XSCALE == 1 */ 5764 5765 /* 5766 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5767 * 5768 * Return the start and end addresses of the kernel's virtual space. 5769 * These values are setup in pmap_bootstrap and are updated as pages 5770 * are allocated. 5771 */ 5772 void 5773 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5774 { 5775 *start = virtual_avail; 5776 *end = virtual_end; 5777 } 5778 5779 /* 5780 * Helper function for pmap_grow_l2_bucket() 5781 */ 5782 static inline int 5783 pmap_grow_map(vaddr_t va, paddr_t *pap) 5784 { 5785 paddr_t pa; 5786 5787 KASSERT((va & PGOFSET) == 0); 5788 5789 if (uvm.page_init_done == false) { 5790 #ifdef PMAP_STEAL_MEMORY 5791 pv_addr_t pv; 5792 pmap_boot_pagealloc(PAGE_SIZE, 5793 #ifdef PMAP_CACHE_VIPT 5794 arm_cache_prefer_mask, 5795 va & arm_cache_prefer_mask, 5796 #else 5797 0, 0, 5798 #endif 5799 &pv); 5800 pa = pv.pv_pa; 5801 #else 5802 if (uvm_page_physget(&pa) == false) 5803 return 1; 5804 #endif /* PMAP_STEAL_MEMORY */ 5805 } else { 5806 struct vm_page *pg; 5807 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 5808 if (pg == NULL) 5809 return 1; 5810 pa = VM_PAGE_TO_PHYS(pg); 5811 /* 5812 * This new page must not have any mappings. 5813 */ 5814 struct vm_page_md *md __diagused = VM_PAGE_TO_MD(pg); 5815 KASSERT(SLIST_EMPTY(&md->pvh_list)); 5816 } 5817 5818 /* 5819 * Enter it via pmap_kenter_pa and let that routine do the hard work. 5820 */ 5821 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, 5822 PMAP_KMPAGE | PMAP_PTE); 5823 5824 if (pap) 5825 *pap = pa; 5826 5827 PMAPCOUNT(pt_mappings); 5828 5829 const pmap_t kpm __diagused = pmap_kernel(); 5830 struct l2_bucket * const l2b __diagused = pmap_get_l2_bucket(kpm, va); 5831 KASSERT(l2b != NULL); 5832 5833 pt_entry_t * const ptep __diagused = &l2b->l2b_kva[l2pte_index(va)]; 5834 const pt_entry_t pte __diagused = *ptep; 5835 KASSERT(l2pte_valid_p(pte)); 5836 KASSERT((pte & L2_S_CACHE_MASK) == pte_l2_s_cache_mode_pt); 5837 5838 memset((void *)va, 0, PAGE_SIZE); 5839 5840 return 0; 5841 } 5842 5843 /* 5844 * This is the same as pmap_alloc_l2_bucket(), except that it is only 5845 * used by pmap_growkernel(). 5846 */ 5847 static inline struct l2_bucket * 5848 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 5849 { 5850 const size_t l1slot = l1pte_index(va); 5851 struct l2_dtable *l2; 5852 vaddr_t nva; 5853 5854 CTASSERT((PAGE_SIZE % L2_TABLE_SIZE_REAL) == 0); 5855 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 5856 /* 5857 * No mapping at this address, as there is 5858 * no entry in the L1 table. 5859 * Need to allocate a new l2_dtable. 5860 */ 5861 nva = pmap_kernel_l2dtable_kva; 5862 if ((nva & PGOFSET) == 0) { 5863 /* 5864 * Need to allocate a backing page 5865 */ 5866 if (pmap_grow_map(nva, NULL)) 5867 return NULL; 5868 } 5869 5870 l2 = (struct l2_dtable *)nva; 5871 nva += sizeof(struct l2_dtable); 5872 5873 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 5874 /* 5875 * The new l2_dtable straddles a page boundary. 5876 * Map in another page to cover it. 5877 */ 5878 if (pmap_grow_map(nva & ~PGOFSET, NULL)) 5879 return NULL; 5880 } 5881 5882 pmap_kernel_l2dtable_kva = nva; 5883 5884 /* 5885 * Link it into the parent pmap 5886 */ 5887 pm->pm_l2[L2_IDX(l1slot)] = l2; 5888 } 5889 5890 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 5891 5892 /* 5893 * Fetch pointer to the L2 page table associated with the address. 5894 */ 5895 if (l2b->l2b_kva == NULL) { 5896 pt_entry_t *ptep; 5897 5898 /* 5899 * No L2 page table has been allocated. Chances are, this 5900 * is because we just allocated the l2_dtable, above. 5901 */ 5902 nva = pmap_kernel_l2ptp_kva; 5903 ptep = (pt_entry_t *)nva; 5904 if ((nva & PGOFSET) == 0) { 5905 /* 5906 * Need to allocate a backing page 5907 */ 5908 if (pmap_grow_map(nva, &pmap_kernel_l2ptp_phys)) 5909 return NULL; 5910 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 5911 } 5912 5913 l2->l2_occupancy++; 5914 l2b->l2b_kva = ptep; 5915 l2b->l2b_l1slot = l1slot; 5916 l2b->l2b_pa = pmap_kernel_l2ptp_phys; 5917 5918 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 5919 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 5920 } 5921 5922 return l2b; 5923 } 5924 5925 vaddr_t 5926 pmap_growkernel(vaddr_t maxkvaddr) 5927 { 5928 UVMHIST_FUNC(__func__); 5929 UVMHIST_CALLARGS(maphist, "growing kernel from %#jx to %#jx\n", 5930 pmap_curmaxkvaddr, maxkvaddr, 0, 0); 5931 5932 pmap_t kpm = pmap_kernel(); 5933 #ifndef ARM_MMU_EXTENDED 5934 struct l1_ttable *l1; 5935 #endif 5936 int s; 5937 5938 if (maxkvaddr <= pmap_curmaxkvaddr) 5939 goto out; /* we are OK */ 5940 5941 KDASSERT(maxkvaddr <= virtual_end); 5942 5943 /* 5944 * whoops! we need to add kernel PTPs 5945 */ 5946 5947 s = splvm(); /* to be safe */ 5948 mutex_enter(&kpm_lock); 5949 5950 /* Map 1MB at a time */ 5951 size_t l1slot = l1pte_index(pmap_curmaxkvaddr); 5952 #ifdef ARM_MMU_EXTENDED 5953 pd_entry_t * const spdep = &kpm->pm_l1[l1slot]; 5954 pd_entry_t *pdep = spdep; 5955 #endif 5956 for (;pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE, 5957 #ifdef ARM_MMU_EXTENDED 5958 pdep++, 5959 #endif 5960 l1slot++) { 5961 struct l2_bucket *l2b = 5962 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 5963 KASSERT(l2b != NULL); 5964 5965 const pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 5966 | L1_C_DOM(PMAP_DOMAIN_KERNEL); 5967 #ifdef ARM_MMU_EXTENDED 5968 KASSERT(*pdep == 0); 5969 l1pte_setone(pdep, npde); 5970 #else 5971 /* Distribute new L1 entry to all other L1s */ 5972 SLIST_FOREACH(l1, &l1_list, l1_link) { 5973 pd_entry_t * const pdep = &l1->l1_kva[l1slot]; 5974 l1pte_setone(pdep, npde); 5975 PDE_SYNC(pdep); 5976 } 5977 #endif 5978 } 5979 #ifdef ARM_MMU_EXTENDED 5980 PDE_SYNC_RANGE(spdep, pdep - spdep); 5981 #endif 5982 5983 #ifdef PMAP_CACHE_VIVT 5984 /* 5985 * flush out the cache, expensive but growkernel will happen so 5986 * rarely 5987 */ 5988 cpu_dcache_wbinv_all(); 5989 cpu_tlb_flushD(); 5990 cpu_cpwait(); 5991 #endif 5992 5993 mutex_exit(&kpm_lock); 5994 splx(s); 5995 5996 out: 5997 return pmap_curmaxkvaddr; 5998 } 5999 6000 /************************ Utility routines ****************************/ 6001 6002 #ifndef ARM_HAS_VBAR 6003 /* 6004 * vector_page_setprot: 6005 * 6006 * Manipulate the protection of the vector page. 6007 */ 6008 void 6009 vector_page_setprot(int prot) 6010 { 6011 struct l2_bucket *l2b; 6012 pt_entry_t *ptep; 6013 6014 #if defined(CPU_ARMV7) || defined(CPU_ARM11) 6015 /* 6016 * If we are using VBAR to use the vectors in the kernel, then it's 6017 * already mapped in the kernel text so no need to anything here. 6018 */ 6019 if (vector_page != ARM_VECTORS_LOW && vector_page != ARM_VECTORS_HIGH) { 6020 KASSERT((armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0); 6021 return; 6022 } 6023 #endif 6024 6025 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 6026 KASSERT(l2b != NULL); 6027 6028 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 6029 6030 const pt_entry_t opte = *ptep; 6031 #ifdef ARM_MMU_EXTENDED 6032 const pt_entry_t npte = (opte & ~(L2_S_PROT_MASK|L2_XS_XN)) 6033 | L2_S_PROT(PTE_KERNEL, prot); 6034 #else 6035 const pt_entry_t npte = (opte & ~L2_S_PROT_MASK) 6036 | L2_S_PROT(PTE_KERNEL, prot); 6037 #endif 6038 l2pte_set(ptep, npte, opte); 6039 PTE_SYNC(ptep); 6040 cpu_tlb_flushD_SE(vector_page); 6041 cpu_cpwait(); 6042 } 6043 #endif 6044 6045 /* 6046 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 6047 * Returns true if the mapping exists, else false. 6048 * 6049 * NOTE: This function is only used by a couple of arm-specific modules. 6050 * It is not safe to take any pmap locks here, since we could be right 6051 * in the middle of debugging the pmap anyway... 6052 * 6053 * It is possible for this routine to return false even though a valid 6054 * mapping does exist. This is because we don't lock, so the metadata 6055 * state may be inconsistent. 6056 * 6057 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 6058 * a "section" mapping. 6059 */ 6060 bool 6061 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 6062 { 6063 struct l2_dtable *l2; 6064 pd_entry_t *pdep, pde; 6065 pt_entry_t *ptep; 6066 u_short l1slot; 6067 6068 if (pm->pm_l1 == NULL) 6069 return false; 6070 6071 l1slot = l1pte_index(va); 6072 *pdp = pdep = pmap_l1_kva(pm) + l1slot; 6073 pde = *pdep; 6074 6075 if (l1pte_section_p(pde)) { 6076 *ptp = NULL; 6077 return true; 6078 } 6079 6080 l2 = pm->pm_l2[L2_IDX(l1slot)]; 6081 if (l2 == NULL || 6082 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 6083 return false; 6084 } 6085 6086 *ptp = &ptep[l2pte_index(va)]; 6087 return true; 6088 } 6089 6090 bool 6091 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 6092 { 6093 6094 if (pm->pm_l1 == NULL) 6095 return false; 6096 6097 *pdp = pmap_l1_kva(pm) + l1pte_index(va); 6098 6099 return true; 6100 } 6101 6102 /************************ Bootstrapping routines ****************************/ 6103 6104 #ifndef ARM_MMU_EXTENDED 6105 static void 6106 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 6107 { 6108 int i; 6109 6110 l1->l1_kva = l1pt; 6111 l1->l1_domain_use_count = 0; 6112 l1->l1_domain_first = 0; 6113 6114 for (i = 0; i < PMAP_DOMAINS; i++) 6115 l1->l1_domain_free[i] = i + 1; 6116 6117 /* 6118 * Copy the kernel's L1 entries to each new L1. 6119 */ 6120 if (pmap_initialized) 6121 memcpy(l1pt, pmap_l1_kva(pmap_kernel()), L1_TABLE_SIZE); 6122 6123 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 6124 &l1->l1_physaddr) == false) 6125 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 6126 6127 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 6128 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 6129 } 6130 #endif /* !ARM_MMU_EXTENDED */ 6131 6132 /* 6133 * pmap_bootstrap() is called from the board-specific initarm() routine 6134 * once the kernel L1/L2 descriptors tables have been set up. 6135 * 6136 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 6137 * spread over a number of disparate files/functions. 6138 * 6139 * We are passed the following parameters 6140 * - vstart 6141 * 1MB-aligned start of managed kernel virtual memory. 6142 * - vend 6143 * 1MB-aligned end of managed kernel virtual memory. 6144 * 6145 * We use 'kernel_l1pt' to build the metadata (struct l1_ttable and 6146 * struct l2_dtable) necessary to track kernel mappings. 6147 */ 6148 #define PMAP_STATIC_L2_SIZE 16 6149 void 6150 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 6151 { 6152 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 6153 #ifndef ARM_MMU_EXTENDED 6154 static struct l1_ttable static_l1; 6155 struct l1_ttable *l1 = &static_l1; 6156 #endif 6157 struct l2_dtable *l2; 6158 struct l2_bucket *l2b; 6159 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 6160 pmap_t pm = pmap_kernel(); 6161 pt_entry_t *ptep; 6162 paddr_t pa; 6163 vsize_t size; 6164 int nptes, l2idx, l2next = 0; 6165 6166 #ifdef ARM_MMU_EXTENDED 6167 KASSERT(pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt); 6168 KASSERT(pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt); 6169 #endif 6170 6171 VPRINTF("kpm "); 6172 /* 6173 * Initialise the kernel pmap object 6174 */ 6175 curcpu()->ci_pmap_cur = pm; 6176 #ifdef ARM_MMU_EXTENDED 6177 pm->pm_l1 = l1pt; 6178 pm->pm_l1_pa = kernel_l1pt.pv_pa; 6179 VPRINTF("tlb0 "); 6180 pmap_tlb_info_init(&pmap_tlb0_info); 6181 #ifdef MULTIPROCESSOR 6182 VPRINTF("kcpusets "); 6183 pm->pm_onproc = kcpuset_running; 6184 pm->pm_active = kcpuset_running; 6185 #endif 6186 #else 6187 pm->pm_l1 = l1; 6188 #endif 6189 6190 VPRINTF("locks "); 6191 /* 6192 * pmap_kenter_pa() and pmap_kremove() may be called from interrupt 6193 * context, so its locks have to be at IPL_VM 6194 */ 6195 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_VM); 6196 mutex_init(&kpm_lock, MUTEX_DEFAULT, IPL_NONE); 6197 mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_VM); 6198 pm->pm_refs = 1; 6199 6200 VPRINTF("l1pt "); 6201 /* 6202 * Scan the L1 translation table created by initarm() and create 6203 * the required metadata for all valid mappings found in it. 6204 */ 6205 for (size_t l1slot = 0; 6206 l1slot < L1_TABLE_SIZE / sizeof(pd_entry_t); 6207 l1slot++) { 6208 pd_entry_t pde = l1pt[l1slot]; 6209 6210 /* 6211 * We're only interested in Coarse mappings. 6212 * pmap_extract() can deal with section mappings without 6213 * recourse to checking L2 metadata. 6214 */ 6215 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 6216 continue; 6217 6218 /* 6219 * Lookup the KVA of this L2 descriptor table 6220 */ 6221 pa = l1pte_pa(pde); 6222 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6223 if (ptep == NULL) { 6224 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 6225 (u_int)l1slot << L1_S_SHIFT, pa); 6226 } 6227 6228 /* 6229 * Fetch the associated L2 metadata structure. 6230 * Allocate a new one if necessary. 6231 */ 6232 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 6233 if (l2next == PMAP_STATIC_L2_SIZE) 6234 panic("pmap_bootstrap: out of static L2s"); 6235 pm->pm_l2[L2_IDX(l1slot)] = l2 = &static_l2[l2next++]; 6236 } 6237 6238 /* 6239 * One more L1 slot tracked... 6240 */ 6241 l2->l2_occupancy++; 6242 6243 /* 6244 * Fill in the details of the L2 descriptor in the 6245 * appropriate bucket. 6246 */ 6247 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 6248 l2b->l2b_kva = ptep; 6249 l2b->l2b_pa = pa; 6250 l2b->l2b_l1slot = l1slot; 6251 6252 /* 6253 * Establish an initial occupancy count for this descriptor 6254 */ 6255 for (l2idx = 0; 6256 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 6257 l2idx++) { 6258 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 6259 l2b->l2b_occupancy++; 6260 } 6261 } 6262 6263 /* 6264 * Make sure the descriptor itself has the correct cache mode. 6265 * If not, fix it, but whine about the problem. Port-meisters 6266 * should consider this a clue to fix up their initarm() 6267 * function. :) 6268 */ 6269 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep, 1)) { 6270 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6271 "L2 pte @ %p\n", ptep); 6272 } 6273 } 6274 6275 VPRINTF("cache(l1pt) "); 6276 /* 6277 * Ensure the primary (kernel) L1 has the correct cache mode for 6278 * a page table. Bitch if it is not correctly set. 6279 */ 6280 if (pmap_set_pt_cache_mode(l1pt, kernel_l1pt.pv_va, 6281 L1_TABLE_SIZE / L2_S_SIZE)) { 6282 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6283 "primary L1 @ 0x%lx\n", kernel_l1pt.pv_va); 6284 } 6285 6286 #ifdef PMAP_CACHE_VIVT 6287 cpu_dcache_wbinv_all(); 6288 cpu_tlb_flushID(); 6289 cpu_cpwait(); 6290 #endif 6291 6292 /* 6293 * now we allocate the "special" VAs which are used for tmp mappings 6294 * by the pmap (and other modules). we allocate the VAs by advancing 6295 * virtual_avail (note that there are no pages mapped at these VAs). 6296 * 6297 * Managed KVM space start from wherever initarm() tells us. 6298 */ 6299 virtual_avail = vstart; 6300 virtual_end = vend; 6301 6302 VPRINTF("specials "); 6303 #ifdef PMAP_CACHE_VIPT 6304 /* 6305 * If we have a VIPT cache, we need one page/pte per possible alias 6306 * page so we won't violate cache aliasing rules. 6307 */ 6308 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 6309 nptes = (arm_cache_prefer_mask >> L2_S_SHIFT) + 1; 6310 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6311 if (arm_pcache.icache_type != CACHE_TYPE_PIPT 6312 && arm_pcache.icache_way_size > nptes * L2_S_SIZE) { 6313 nptes = arm_pcache.icache_way_size >> L2_S_SHIFT; 6314 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6315 } 6316 #else 6317 nptes = PAGE_SIZE / L2_S_SIZE; 6318 #endif 6319 #ifdef MULTIPROCESSOR 6320 cnptes = nptes; 6321 nptes *= arm_cpu_max; 6322 #endif 6323 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 6324 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte, nptes); 6325 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 6326 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte, nptes); 6327 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 6328 if (msgbufaddr == NULL) { 6329 pmap_alloc_specials(&virtual_avail, 6330 round_page(MSGBUFSIZE) / PAGE_SIZE, 6331 (void *)&msgbufaddr, NULL); 6332 } 6333 6334 /* 6335 * Allocate a range of kernel virtual address space to be used 6336 * for L2 descriptor tables and metadata allocation in 6337 * pmap_growkernel(). 6338 */ 6339 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 6340 pmap_alloc_specials(&virtual_avail, 6341 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 6342 &pmap_kernel_l2ptp_kva, NULL); 6343 6344 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 6345 pmap_alloc_specials(&virtual_avail, 6346 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 6347 &pmap_kernel_l2dtable_kva, NULL); 6348 6349 #ifndef ARM_MMU_EXTENDED 6350 /* 6351 * init the static-global locks and global pmap list. 6352 */ 6353 mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM); 6354 6355 /* 6356 * We can now initialise the first L1's metadata. 6357 */ 6358 SLIST_INIT(&l1_list); 6359 TAILQ_INIT(&l1_lru_list); 6360 pmap_init_l1(l1, l1pt); 6361 #endif /* ARM_MMU_EXTENDED */ 6362 6363 #ifndef ARM_HAS_VBAR 6364 /* Set up vector page L1 details, if necessary */ 6365 if (vector_page < KERNEL_BASE) { 6366 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 6367 l2b = pmap_get_l2_bucket(pm, vector_page); 6368 KDASSERT(l2b != NULL); 6369 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 6370 L1_C_DOM(pmap_domain(pm)); 6371 } else 6372 pm->pm_pl1vec = NULL; 6373 #endif 6374 6375 VPRINTF("pools "); 6376 /* 6377 * Initialize the pmap cache 6378 */ 6379 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 6380 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 6381 6382 /* 6383 * Initialize the pv pool. 6384 */ 6385 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 6386 &pmap_bootstrap_pv_allocator, IPL_NONE); 6387 6388 /* 6389 * Initialize the L2 dtable pool and cache. 6390 */ 6391 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 6392 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 6393 6394 /* 6395 * Initialise the L2 descriptor table pool and cache 6396 */ 6397 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 6398 L2_TABLE_SIZE_REAL, 0, 0, "l2ptppl", NULL, IPL_NONE, 6399 pmap_l2ptp_ctor, NULL, NULL); 6400 6401 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 6402 6403 cpu_dcache_wbinv_all(); 6404 } 6405 6406 static bool 6407 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va, size_t nptes) 6408 { 6409 #ifdef ARM_MMU_EXTENDED 6410 return false; 6411 #else 6412 if (pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 6413 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 6414 return false; 6415 6416 const vaddr_t eva = va + nptes * PAGE_SIZE; 6417 int rv = 0; 6418 6419 while (va < eva) { 6420 /* 6421 * Make sure the descriptor itself has the correct cache mode 6422 */ 6423 pd_entry_t * const pdep = &kl1[l1pte_index(va)]; 6424 pd_entry_t pde = *pdep; 6425 6426 if (l1pte_section_p(pde)) { 6427 KASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0); 6428 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 6429 *pdep = (pde & ~L1_S_CACHE_MASK) | 6430 pte_l1_s_cache_mode_pt; 6431 PDE_SYNC(pdep); 6432 cpu_dcache_wbinv_range((vaddr_t)pdep, 6433 sizeof(*pdep)); 6434 rv = 1; 6435 } 6436 return rv; 6437 } 6438 vaddr_t pa = l1pte_pa(pde); 6439 pt_entry_t *ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6440 if (ptep == NULL) 6441 panic("pmap_bootstrap: No PTP for va %#lx\n", va); 6442 6443 ptep += l2pte_index(va); 6444 const pt_entry_t opte = *ptep; 6445 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 6446 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 6447 | pte_l2_s_cache_mode_pt; 6448 l2pte_set(ptep, npte, opte); 6449 PTE_SYNC(ptep); 6450 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 6451 rv = 1; 6452 } 6453 va += PAGE_SIZE; 6454 } 6455 6456 return rv; 6457 #endif 6458 } 6459 6460 static void 6461 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 6462 { 6463 vaddr_t va = *availp; 6464 struct l2_bucket *l2b; 6465 6466 if (ptep) { 6467 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6468 if (l2b == NULL) 6469 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 6470 6471 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 6472 } 6473 6474 *vap = va; 6475 *availp = va + (PAGE_SIZE * pages); 6476 } 6477 6478 void 6479 pmap_init(void) 6480 { 6481 6482 /* 6483 * Set the available memory vars - These do not map to real memory 6484 * addresses and cannot as the physical memory is fragmented. 6485 * They are used by ps for %mem calculations. 6486 * One could argue whether this should be the entire memory or just 6487 * the memory that is useable in a user process. 6488 */ 6489 avail_start = ptoa(uvm_physseg_get_avail_start(uvm_physseg_get_first())); 6490 avail_end = ptoa(uvm_physseg_get_avail_end(uvm_physseg_get_last())); 6491 6492 /* 6493 * Now we need to free enough pv_entry structures to allow us to get 6494 * the kmem_map/kmem_object allocated and inited (done after this 6495 * function is finished). to do this we allocate one bootstrap page out 6496 * of kernel_map and use it to provide an initial pool of pv_entry 6497 * structures. we never free this page. 6498 */ 6499 pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 6500 6501 #ifdef ARM_MMU_EXTENDED 6502 /* 6503 * Initialise the L1 pool and cache. 6504 */ 6505 6506 pool_cache_bootstrap(&pmap_l1tt_cache, L1TT_SIZE, L1TT_SIZE, 6507 0, 0, "l1ttpl", &pmap_l1tt_allocator, IPL_NONE, pmap_l1tt_ctor, 6508 NULL, NULL); 6509 6510 int error __diagused = pmap_maxproc_set(maxproc); 6511 KASSERT(error == 0); 6512 6513 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 6514 #endif 6515 6516 pmap_initialized = true; 6517 } 6518 6519 static vaddr_t last_bootstrap_page = 0; 6520 static void *free_bootstrap_pages = NULL; 6521 6522 static void * 6523 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 6524 { 6525 extern void *pool_page_alloc(struct pool *, int); 6526 vaddr_t new_page; 6527 void *rv; 6528 6529 if (pmap_initialized) 6530 return pool_page_alloc(pp, flags); 6531 6532 if (free_bootstrap_pages) { 6533 rv = free_bootstrap_pages; 6534 free_bootstrap_pages = *((void **)rv); 6535 return rv; 6536 } 6537 6538 KASSERT(kernel_map != NULL); 6539 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 6540 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 6541 6542 KASSERT(new_page > last_bootstrap_page); 6543 last_bootstrap_page = new_page; 6544 return (void *)new_page; 6545 } 6546 6547 static void 6548 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 6549 { 6550 extern void pool_page_free(struct pool *, void *); 6551 6552 if ((vaddr_t)v <= last_bootstrap_page) { 6553 *((void **)v) = free_bootstrap_pages; 6554 free_bootstrap_pages = v; 6555 return; 6556 } 6557 6558 if (pmap_initialized) { 6559 pool_page_free(pp, v); 6560 return; 6561 } 6562 } 6563 6564 6565 #if defined(ARM_MMU_EXTENDED) 6566 static void * 6567 pmap_l1tt_alloc(struct pool *pp, int flags) 6568 { 6569 struct pglist plist; 6570 vaddr_t va; 6571 6572 const int waitok = flags & PR_WAITOK; 6573 6574 int error = uvm_pglistalloc(L1TT_SIZE, 0, -1, L1TT_SIZE, 0, &plist, 1, 6575 waitok); 6576 if (error) 6577 panic("Cannot allocate L1TT physical pages, %d", error); 6578 6579 struct vm_page *pg = TAILQ_FIRST(&plist); 6580 #if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS) 6581 6582 /* Allocate a L1 translation table VA */ 6583 va = uvm_km_alloc(kernel_map, L1TT_SIZE, L1TT_SIZE, UVM_KMF_VAONLY); 6584 if (va == 0) 6585 panic("Cannot allocate L1TT KVA"); 6586 6587 const vaddr_t eva = va + L1TT_SIZE; 6588 vaddr_t mva = va; 6589 while (pg && mva < eva) { 6590 paddr_t pa = VM_PAGE_TO_PHYS(pg); 6591 6592 pmap_kenter_pa(mva, pa, 6593 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6594 6595 mva += PAGE_SIZE; 6596 pg = TAILQ_NEXT(pg, pageq.queue); 6597 } 6598 KASSERTMSG(pg == NULL && mva == eva, "pg %p mva %" PRIxVADDR 6599 " eva %" PRIxVADDR, pg, mva, eva); 6600 #else 6601 bool ok; 6602 paddr_t pa = VM_PAGE_TO_PHYS(pg); 6603 va = pmap_direct_mapped_phys(pa, &ok, 0); 6604 KASSERT(ok); 6605 KASSERT(va >= KERNEL_BASE); 6606 #endif 6607 6608 return (void *)va; 6609 } 6610 6611 static void 6612 pmap_l1tt_free(struct pool *pp, void *v) 6613 { 6614 vaddr_t va = (vaddr_t)v; 6615 6616 #if !defined( __HAVE_MM_MD_DIRECT_MAPPED_PHYS) 6617 uvm_km_free(kernel_map, va, L1TT_SIZE, UVM_KMF_WIRED); 6618 #else 6619 #if defined(KERNEL_BASE_VOFFSET) 6620 paddr_t pa = va - KERNEL_BASE_VOFFSET; 6621 #else 6622 paddr_t pa = va - KERNEL_BASE + physical_start; 6623 #endif 6624 const paddr_t epa = pa + L1TT_SIZE; 6625 6626 for (; pa < epa; pa += PAGE_SIZE) { 6627 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 6628 uvm_pagefree(pg); 6629 } 6630 #endif 6631 } 6632 #endif 6633 6634 /* 6635 * pmap_postinit() 6636 * 6637 * This routine is called after the vm and kmem subsystems have been 6638 * initialised. This allows the pmap code to perform any initialisation 6639 * that can only be done once the memory allocation is in place. 6640 */ 6641 void 6642 pmap_postinit(void) 6643 { 6644 #ifndef ARM_MMU_EXTENDED 6645 extern paddr_t physical_start, physical_end; 6646 struct l1_ttable *l1; 6647 struct pglist plist; 6648 struct vm_page *m; 6649 pd_entry_t *pdep; 6650 vaddr_t va, eva; 6651 u_int loop, needed; 6652 int error; 6653 #endif 6654 6655 pool_cache_setlowat(&pmap_l2ptp_cache, (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 6656 pool_cache_setlowat(&pmap_l2dtable_cache, 6657 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 6658 6659 #ifndef ARM_MMU_EXTENDED 6660 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 6661 needed -= 1; 6662 6663 l1 = kmem_alloc(sizeof(*l1) * needed, KM_SLEEP); 6664 6665 for (loop = 0; loop < needed; loop++, l1++) { 6666 /* Allocate a L1 page table */ 6667 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 6668 if (va == 0) 6669 panic("Cannot allocate L1 KVM"); 6670 6671 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 6672 physical_end, L1_TABLE_SIZE, 0, &plist, 1, 1); 6673 if (error) 6674 panic("Cannot allocate L1 physical pages"); 6675 6676 m = TAILQ_FIRST(&plist); 6677 eva = va + L1_TABLE_SIZE; 6678 pdep = (pd_entry_t *)va; 6679 6680 while (m && va < eva) { 6681 paddr_t pa = VM_PAGE_TO_PHYS(m); 6682 6683 pmap_kenter_pa(va, pa, 6684 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6685 6686 va += PAGE_SIZE; 6687 m = TAILQ_NEXT(m, pageq.queue); 6688 } 6689 6690 #ifdef DIAGNOSTIC 6691 if (m) 6692 panic("pmap_alloc_l1pt: pglist not empty"); 6693 #endif /* DIAGNOSTIC */ 6694 6695 pmap_init_l1(l1, pdep); 6696 } 6697 6698 #ifdef DEBUG 6699 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 6700 needed); 6701 #endif 6702 #endif /* !ARM_MMU_EXTENDED */ 6703 } 6704 6705 /* 6706 * Note that the following routines are used by board-specific initialisation 6707 * code to configure the initial kernel page tables. 6708 * 6709 */ 6710 6711 /* 6712 * This list exists for the benefit of pmap_map_chunk(). It keeps track 6713 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 6714 * find them as necessary. 6715 * 6716 * Note that the data on this list MUST remain valid after initarm() returns, 6717 * as pmap_bootstrap() uses it to construct L2 table metadata. 6718 */ 6719 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 6720 6721 static vaddr_t 6722 kernel_pt_lookup(paddr_t pa) 6723 { 6724 pv_addr_t *pv; 6725 6726 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 6727 if (pv->pv_pa == (pa & ~PGOFSET)) 6728 return pv->pv_va | (pa & PGOFSET); 6729 } 6730 return 0; 6731 } 6732 6733 /* 6734 * pmap_map_section: 6735 * 6736 * Create a single section mapping. 6737 */ 6738 void 6739 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6740 { 6741 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6742 const size_t l1slot = l1pte_index(va); 6743 pd_entry_t fl; 6744 6745 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 6746 6747 switch (cache) { 6748 case PTE_NOCACHE: 6749 fl = pte_l1_s_nocache_mode; 6750 break; 6751 6752 case PTE_CACHE: 6753 fl = pte_l1_s_cache_mode; 6754 break; 6755 6756 case PTE_PAGETABLE: 6757 fl = pte_l1_s_cache_mode_pt; 6758 break; 6759 6760 case PTE_DEV: 6761 default: 6762 fl = 0; 6763 break; 6764 } 6765 6766 const pd_entry_t npde = L1_S_PROTO | pa | 6767 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6768 l1pte_setone(pdep + l1slot, npde); 6769 PDE_SYNC(pdep + l1slot); 6770 } 6771 6772 /* 6773 * pmap_map_entry: 6774 * 6775 * Create a single page mapping. 6776 */ 6777 void 6778 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6779 { 6780 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6781 const size_t l1slot = l1pte_index(va); 6782 pt_entry_t npte; 6783 pt_entry_t *ptep; 6784 6785 KASSERT(((va | pa) & PGOFSET) == 0); 6786 6787 switch (cache) { 6788 case PTE_NOCACHE: 6789 npte = pte_l2_s_nocache_mode; 6790 break; 6791 6792 case PTE_CACHE: 6793 npte = pte_l2_s_cache_mode; 6794 break; 6795 6796 case PTE_PAGETABLE: 6797 npte = pte_l2_s_cache_mode_pt; 6798 break; 6799 6800 default: 6801 npte = 0; 6802 break; 6803 } 6804 6805 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6806 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 6807 6808 ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6809 if (ptep == NULL) 6810 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 6811 6812 npte |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 6813 #ifdef ARM_MMU_EXTENDED 6814 if (prot & VM_PROT_EXECUTE) { 6815 npte &= ~L2_XS_XN; 6816 } 6817 #endif 6818 ptep += l2pte_index(va); 6819 l2pte_set(ptep, npte, 0); 6820 PTE_SYNC(ptep); 6821 } 6822 6823 /* 6824 * pmap_link_l2pt: 6825 * 6826 * Link the L2 page table specified by "l2pv" into the L1 6827 * page table at the slot for "va". 6828 */ 6829 void 6830 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 6831 { 6832 pd_entry_t * const pdep = (pd_entry_t *) l1pt + l1pte_index(va); 6833 6834 KASSERT((va & ((L1_S_SIZE * (PAGE_SIZE / L2_T_SIZE)) - 1)) == 0); 6835 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 6836 6837 const pd_entry_t npde = L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO 6838 | l2pv->pv_pa; 6839 6840 l1pte_set(pdep, npde); 6841 PDE_SYNC_RANGE(pdep, PAGE_SIZE / L2_T_SIZE); 6842 6843 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 6844 } 6845 6846 /* 6847 * pmap_map_chunk: 6848 * 6849 * Map a chunk of memory using the most efficient mappings 6850 * possible (section, large page, small page) into the 6851 * provided L1 and L2 tables at the specified virtual address. 6852 */ 6853 vsize_t 6854 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 6855 int prot, int cache) 6856 { 6857 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6858 pt_entry_t f1, f2s, f2l; 6859 vsize_t resid; 6860 6861 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 6862 6863 if (l1pt == 0) 6864 panic("pmap_map_chunk: no L1 table provided"); 6865 6866 // VPRINTF("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 6867 // "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 6868 6869 switch (cache) { 6870 case PTE_NOCACHE: 6871 f1 = pte_l1_s_nocache_mode; 6872 f2l = pte_l2_l_nocache_mode; 6873 f2s = pte_l2_s_nocache_mode; 6874 break; 6875 6876 case PTE_CACHE: 6877 f1 = pte_l1_s_cache_mode; 6878 f2l = pte_l2_l_cache_mode; 6879 f2s = pte_l2_s_cache_mode; 6880 break; 6881 6882 case PTE_PAGETABLE: 6883 f1 = pte_l1_s_cache_mode_pt; 6884 f2l = pte_l2_l_cache_mode_pt; 6885 f2s = pte_l2_s_cache_mode_pt; 6886 break; 6887 6888 case PTE_DEV: 6889 default: 6890 f1 = 0; 6891 f2l = 0; 6892 f2s = 0; 6893 break; 6894 } 6895 6896 size = resid; 6897 6898 while (resid > 0) { 6899 const size_t l1slot = l1pte_index(va); 6900 #ifdef ARM_MMU_EXTENDED 6901 /* See if we can use a supersection mapping. */ 6902 if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) { 6903 /* Supersection are always domain 0 */ 6904 const pd_entry_t npde = L1_SS_PROTO | pa 6905 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6906 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6907 | L1_S_PROT(PTE_KERNEL, prot) | f1; 6908 VPRINTF("sS"); 6909 l1pte_set(&pdep[l1slot], npde); 6910 PDE_SYNC_RANGE(&pdep[l1slot], L1_SS_SIZE / L1_S_SIZE); 6911 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6912 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6913 va += L1_SS_SIZE; 6914 pa += L1_SS_SIZE; 6915 resid -= L1_SS_SIZE; 6916 continue; 6917 } 6918 #endif 6919 /* See if we can use a section mapping. */ 6920 if (L1_S_MAPPABLE_P(va, pa, resid)) { 6921 const pd_entry_t npde = L1_S_PROTO | pa 6922 #ifdef ARM_MMU_EXTENDED 6923 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6924 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6925 #endif 6926 | L1_S_PROT(PTE_KERNEL, prot) | f1 6927 | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6928 VPRINTF("S"); 6929 l1pte_set(&pdep[l1slot], npde); 6930 PDE_SYNC(&pdep[l1slot]); 6931 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6932 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6933 va += L1_S_SIZE; 6934 pa += L1_S_SIZE; 6935 resid -= L1_S_SIZE; 6936 continue; 6937 } 6938 6939 /* 6940 * Ok, we're going to use an L2 table. Make sure 6941 * one is actually in the corresponding L1 slot 6942 * for the current VA. 6943 */ 6944 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6945 panic("%s: no L2 table for VA %#lx", __func__, va); 6946 6947 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6948 if (ptep == NULL) 6949 panic("%s: can't find L2 table for VA %#lx", __func__, 6950 va); 6951 6952 ptep += l2pte_index(va); 6953 6954 /* See if we can use a L2 large page mapping. */ 6955 if (L2_L_MAPPABLE_P(va, pa, resid)) { 6956 const pt_entry_t npte = L2_L_PROTO | pa 6957 #ifdef ARM_MMU_EXTENDED 6958 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_L_XN) 6959 | (va & 0x80000000 ? 0 : L2_XS_nG) 6960 #endif 6961 | L2_L_PROT(PTE_KERNEL, prot) | f2l; 6962 VPRINTF("L"); 6963 l2pte_set(ptep, npte, 0); 6964 PTE_SYNC_RANGE(ptep, L2_L_SIZE / L2_S_SIZE); 6965 va += L2_L_SIZE; 6966 pa += L2_L_SIZE; 6967 resid -= L2_L_SIZE; 6968 continue; 6969 } 6970 6971 VPRINTF("P"); 6972 /* Use a small page mapping. */ 6973 pt_entry_t npte = L2_S_PROTO | pa 6974 #ifdef ARM_MMU_EXTENDED 6975 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_XN) 6976 | (va & 0x80000000 ? 0 : L2_XS_nG) 6977 #endif 6978 | L2_S_PROT(PTE_KERNEL, prot) | f2s; 6979 #ifdef ARM_MMU_EXTENDED 6980 npte &= ((prot & VM_PROT_EXECUTE) ? ~L2_XS_XN : ~0); 6981 #endif 6982 l2pte_set(ptep, npte, 0); 6983 PTE_SYNC(ptep); 6984 va += PAGE_SIZE; 6985 pa += PAGE_SIZE; 6986 resid -= PAGE_SIZE; 6987 } 6988 VPRINTF("\n"); 6989 return size; 6990 } 6991 6992 /* 6993 * pmap_unmap_chunk: 6994 * 6995 * Unmap a chunk of memory that was previously pmap_map_chunk 6996 */ 6997 void 6998 pmap_unmap_chunk(vaddr_t l1pt, vaddr_t va, vsize_t size) 6999 { 7000 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 7001 const size_t l1slot = l1pte_index(va); 7002 7003 KASSERT(size == L1_SS_SIZE || size == L1_S_SIZE); 7004 7005 l1pte_set(&pdep[l1slot], 0); 7006 PDE_SYNC_RANGE(&pdep[l1slot], size / L1_S_SIZE); 7007 7008 pmap_tlb_flush_SE(pmap_kernel(), va, PVF_REF); 7009 } 7010 7011 7012 7013 /********************** Static device map routines ***************************/ 7014 7015 static const struct pmap_devmap *pmap_devmap_table; 7016 7017 /* 7018 * Register the devmap table. This is provided in case early console 7019 * initialization needs to register mappings created by bootstrap code 7020 * before pmap_devmap_bootstrap() is called. 7021 */ 7022 void 7023 pmap_devmap_register(const struct pmap_devmap *table) 7024 { 7025 7026 pmap_devmap_table = table; 7027 } 7028 7029 /* 7030 * Map all of the static regions in the devmap table, and remember 7031 * the devmap table so other parts of the kernel can look up entries 7032 * later. 7033 */ 7034 void 7035 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 7036 { 7037 int i; 7038 7039 pmap_devmap_table = table; 7040 7041 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7042 const struct pmap_devmap *pdp = &pmap_devmap_table[i]; 7043 7044 KASSERTMSG(VADDR_MAX - pdp->pd_va >= pdp->pd_size - 1, "va %" PRIxVADDR 7045 " sz %" PRIxPSIZE, pdp->pd_va, pdp->pd_size); 7046 KASSERTMSG(PADDR_MAX - pdp->pd_pa >= pdp->pd_size - 1, "pa %" PRIxPADDR 7047 " sz %" PRIxPSIZE, pdp->pd_pa, pdp->pd_size); 7048 VPRINTF("devmap: %08lx -> %08lx @ %08lx\n", pdp->pd_pa, 7049 pdp->pd_pa + pdp->pd_size - 1, pdp->pd_va); 7050 7051 pmap_map_chunk(l1pt, pdp->pd_va, pdp->pd_pa, pdp->pd_size, 7052 pdp->pd_prot, pdp->pd_cache); 7053 } 7054 } 7055 7056 const struct pmap_devmap * 7057 pmap_devmap_find_pa(paddr_t pa, psize_t size) 7058 { 7059 uint64_t endpa; 7060 int i; 7061 7062 if (pmap_devmap_table == NULL) 7063 return NULL; 7064 7065 endpa = (uint64_t)pa + (uint64_t)(size - 1); 7066 7067 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7068 if (pa >= pmap_devmap_table[i].pd_pa && 7069 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 7070 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 7071 return &pmap_devmap_table[i]; 7072 } 7073 7074 return NULL; 7075 } 7076 7077 const struct pmap_devmap * 7078 pmap_devmap_find_va(vaddr_t va, vsize_t size) 7079 { 7080 int i; 7081 7082 if (pmap_devmap_table == NULL) 7083 return NULL; 7084 7085 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 7086 if (va >= pmap_devmap_table[i].pd_va && 7087 va + size - 1 <= pmap_devmap_table[i].pd_va + 7088 pmap_devmap_table[i].pd_size - 1) 7089 return &pmap_devmap_table[i]; 7090 } 7091 7092 return NULL; 7093 } 7094 7095 /********************** PTE initialization routines **************************/ 7096 7097 /* 7098 * These routines are called when the CPU type is identified to set up 7099 * the PTE prototypes, cache modes, etc. 7100 * 7101 * The variables are always here, just in case modules need to reference 7102 * them (though, they shouldn't). 7103 */ 7104 7105 pt_entry_t pte_l1_s_nocache_mode; 7106 pt_entry_t pte_l1_s_cache_mode; 7107 pt_entry_t pte_l1_s_wc_mode; 7108 pt_entry_t pte_l1_s_cache_mode_pt; 7109 pt_entry_t pte_l1_s_cache_mask; 7110 7111 pt_entry_t pte_l2_l_nocache_mode; 7112 pt_entry_t pte_l2_l_cache_mode; 7113 pt_entry_t pte_l2_l_wc_mode; 7114 pt_entry_t pte_l2_l_cache_mode_pt; 7115 pt_entry_t pte_l2_l_cache_mask; 7116 7117 pt_entry_t pte_l2_s_nocache_mode; 7118 pt_entry_t pte_l2_s_cache_mode; 7119 pt_entry_t pte_l2_s_wc_mode; 7120 pt_entry_t pte_l2_s_cache_mode_pt; 7121 pt_entry_t pte_l2_s_cache_mask; 7122 7123 pt_entry_t pte_l1_s_prot_u; 7124 pt_entry_t pte_l1_s_prot_w; 7125 pt_entry_t pte_l1_s_prot_ro; 7126 pt_entry_t pte_l1_s_prot_mask; 7127 7128 pt_entry_t pte_l2_s_prot_u; 7129 pt_entry_t pte_l2_s_prot_w; 7130 pt_entry_t pte_l2_s_prot_ro; 7131 pt_entry_t pte_l2_s_prot_mask; 7132 7133 pt_entry_t pte_l2_l_prot_u; 7134 pt_entry_t pte_l2_l_prot_w; 7135 pt_entry_t pte_l2_l_prot_ro; 7136 pt_entry_t pte_l2_l_prot_mask; 7137 7138 pt_entry_t pte_l1_ss_proto; 7139 pt_entry_t pte_l1_s_proto; 7140 pt_entry_t pte_l1_c_proto; 7141 pt_entry_t pte_l2_s_proto; 7142 7143 void (*pmap_copy_page_func)(paddr_t, paddr_t); 7144 void (*pmap_zero_page_func)(paddr_t); 7145 7146 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 7147 void 7148 pmap_pte_init_generic(void) 7149 { 7150 7151 pte_l1_s_nocache_mode = 0; 7152 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7153 pte_l1_s_wc_mode = L1_S_B; 7154 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 7155 7156 pte_l2_l_nocache_mode = 0; 7157 pte_l2_l_cache_mode = L2_B|L2_C; 7158 pte_l2_l_wc_mode = L2_B; 7159 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 7160 7161 pte_l2_s_nocache_mode = 0; 7162 pte_l2_s_cache_mode = L2_B|L2_C; 7163 pte_l2_s_wc_mode = L2_B; 7164 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 7165 7166 /* 7167 * If we have a write-through cache, set B and C. If 7168 * we have a write-back cache, then we assume setting 7169 * only C will make those pages write-through (except for those 7170 * Cortex CPUs which can read the L1 caches). 7171 */ 7172 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop 7173 #if ARM_MMU_V7 > 0 7174 || CPU_ID_CORTEX_P(curcpu()->ci_arm_cpuid) 7175 #endif 7176 #if ARM_MMU_V6 > 0 7177 || CPU_ID_ARM11_P(curcpu()->ci_arm_cpuid) /* arm116 errata 399234 */ 7178 #endif 7179 || false) { 7180 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7181 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7182 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7183 } else { 7184 pte_l1_s_cache_mode_pt = L1_S_C; /* write through */ 7185 pte_l2_l_cache_mode_pt = L2_C; /* write through */ 7186 pte_l2_s_cache_mode_pt = L2_C; /* write through */ 7187 } 7188 7189 pte_l1_s_prot_u = L1_S_PROT_U_generic; 7190 pte_l1_s_prot_w = L1_S_PROT_W_generic; 7191 pte_l1_s_prot_ro = L1_S_PROT_RO_generic; 7192 pte_l1_s_prot_mask = L1_S_PROT_MASK_generic; 7193 7194 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7195 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7196 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7197 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7198 7199 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7200 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7201 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7202 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7203 7204 pte_l1_ss_proto = L1_SS_PROTO_generic; 7205 pte_l1_s_proto = L1_S_PROTO_generic; 7206 pte_l1_c_proto = L1_C_PROTO_generic; 7207 pte_l2_s_proto = L2_S_PROTO_generic; 7208 7209 pmap_copy_page_func = pmap_copy_page_generic; 7210 pmap_zero_page_func = pmap_zero_page_generic; 7211 } 7212 7213 #if defined(CPU_ARM8) 7214 void 7215 pmap_pte_init_arm8(void) 7216 { 7217 7218 /* 7219 * ARM8 is compatible with generic, but we need to use 7220 * the page tables uncached. 7221 */ 7222 pmap_pte_init_generic(); 7223 7224 pte_l1_s_cache_mode_pt = 0; 7225 pte_l2_l_cache_mode_pt = 0; 7226 pte_l2_s_cache_mode_pt = 0; 7227 } 7228 #endif /* CPU_ARM8 */ 7229 7230 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 7231 void 7232 pmap_pte_init_arm9(void) 7233 { 7234 7235 /* 7236 * ARM9 is compatible with generic, but we want to use 7237 * write-through caching for now. 7238 */ 7239 pmap_pte_init_generic(); 7240 7241 pte_l1_s_cache_mode = L1_S_C; 7242 pte_l2_l_cache_mode = L2_C; 7243 pte_l2_s_cache_mode = L2_C; 7244 7245 pte_l1_s_wc_mode = L1_S_B; 7246 pte_l2_l_wc_mode = L2_B; 7247 pte_l2_s_wc_mode = L2_B; 7248 7249 pte_l1_s_cache_mode_pt = L1_S_C; 7250 pte_l2_l_cache_mode_pt = L2_C; 7251 pte_l2_s_cache_mode_pt = L2_C; 7252 } 7253 #endif /* CPU_ARM9 && ARM9_CACHE_WRITE_THROUGH */ 7254 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 7255 7256 #if defined(CPU_ARM10) 7257 void 7258 pmap_pte_init_arm10(void) 7259 { 7260 7261 /* 7262 * ARM10 is compatible with generic, but we want to use 7263 * write-through caching for now. 7264 */ 7265 pmap_pte_init_generic(); 7266 7267 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 7268 pte_l2_l_cache_mode = L2_B | L2_C; 7269 pte_l2_s_cache_mode = L2_B | L2_C; 7270 7271 pte_l1_s_cache_mode = L1_S_B; 7272 pte_l2_l_cache_mode = L2_B; 7273 pte_l2_s_cache_mode = L2_B; 7274 7275 pte_l1_s_cache_mode_pt = L1_S_C; 7276 pte_l2_l_cache_mode_pt = L2_C; 7277 pte_l2_s_cache_mode_pt = L2_C; 7278 7279 } 7280 #endif /* CPU_ARM10 */ 7281 7282 #if defined(CPU_ARM11) && defined(ARM11_CACHE_WRITE_THROUGH) 7283 void 7284 pmap_pte_init_arm11(void) 7285 { 7286 7287 /* 7288 * ARM11 is compatible with generic, but we want to use 7289 * write-through caching for now. 7290 */ 7291 pmap_pte_init_generic(); 7292 7293 pte_l1_s_cache_mode = L1_S_C; 7294 pte_l2_l_cache_mode = L2_C; 7295 pte_l2_s_cache_mode = L2_C; 7296 7297 pte_l1_s_wc_mode = L1_S_B; 7298 pte_l2_l_wc_mode = L2_B; 7299 pte_l2_s_wc_mode = L2_B; 7300 7301 pte_l1_s_cache_mode_pt = L1_S_C; 7302 pte_l2_l_cache_mode_pt = L2_C; 7303 pte_l2_s_cache_mode_pt = L2_C; 7304 } 7305 #endif /* CPU_ARM11 && ARM11_CACHE_WRITE_THROUGH */ 7306 7307 #if ARM_MMU_SA1 == 1 7308 void 7309 pmap_pte_init_sa1(void) 7310 { 7311 7312 /* 7313 * The StrongARM SA-1 cache does not have a write-through 7314 * mode. So, do the generic initialization, then reset 7315 * the page table cache mode to B=1,C=1, and note that 7316 * the PTEs need to be sync'd. 7317 */ 7318 pmap_pte_init_generic(); 7319 7320 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7321 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7322 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7323 7324 pmap_needs_pte_sync = 1; 7325 } 7326 #endif /* ARM_MMU_SA1 == 1*/ 7327 7328 #if ARM_MMU_XSCALE == 1 7329 #if (ARM_NMMUS > 1) 7330 static u_int xscale_use_minidata; 7331 #endif 7332 7333 void 7334 pmap_pte_init_xscale(void) 7335 { 7336 uint32_t auxctl; 7337 int write_through = 0; 7338 7339 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7340 pte_l1_s_wc_mode = L1_S_B; 7341 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 7342 7343 pte_l2_l_cache_mode = L2_B|L2_C; 7344 pte_l2_l_wc_mode = L2_B; 7345 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 7346 7347 pte_l2_s_cache_mode = L2_B|L2_C; 7348 pte_l2_s_wc_mode = L2_B; 7349 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 7350 7351 pte_l1_s_cache_mode_pt = L1_S_C; 7352 pte_l2_l_cache_mode_pt = L2_C; 7353 pte_l2_s_cache_mode_pt = L2_C; 7354 7355 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 7356 /* 7357 * The XScale core has an enhanced mode where writes that 7358 * miss the cache cause a cache line to be allocated. This 7359 * is significantly faster than the traditional, write-through 7360 * behavior of this case. 7361 */ 7362 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 7363 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 7364 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 7365 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 7366 7367 #ifdef XSCALE_CACHE_WRITE_THROUGH 7368 /* 7369 * Some versions of the XScale core have various bugs in 7370 * their cache units, the work-around for which is to run 7371 * the cache in write-through mode. Unfortunately, this 7372 * has a major (negative) impact on performance. So, we 7373 * go ahead and run fast-and-loose, in the hopes that we 7374 * don't line up the planets in a way that will trip the 7375 * bugs. 7376 * 7377 * However, we give you the option to be slow-but-correct. 7378 */ 7379 write_through = 1; 7380 #elif defined(XSCALE_CACHE_WRITE_BACK) 7381 /* force write back cache mode */ 7382 write_through = 0; 7383 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 7384 /* 7385 * Intel PXA2[15]0 processors are known to have a bug in 7386 * write-back cache on revision 4 and earlier (stepping 7387 * A[01] and B[012]). Fixed for C0 and later. 7388 */ 7389 { 7390 uint32_t id, type; 7391 7392 id = cpufunc_id(); 7393 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 7394 7395 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 7396 if ((id & CPU_ID_REVISION_MASK) < 5) { 7397 /* write through for stepping A0-1 and B0-2 */ 7398 write_through = 1; 7399 } 7400 } 7401 } 7402 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 7403 7404 if (write_through) { 7405 pte_l1_s_cache_mode = L1_S_C; 7406 pte_l2_l_cache_mode = L2_C; 7407 pte_l2_s_cache_mode = L2_C; 7408 } 7409 7410 #if (ARM_NMMUS > 1) 7411 xscale_use_minidata = 1; 7412 #endif 7413 7414 pte_l1_s_prot_u = L1_S_PROT_U_xscale; 7415 pte_l1_s_prot_w = L1_S_PROT_W_xscale; 7416 pte_l1_s_prot_ro = L1_S_PROT_RO_xscale; 7417 pte_l1_s_prot_mask = L1_S_PROT_MASK_xscale; 7418 7419 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 7420 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 7421 pte_l2_s_prot_ro = L2_S_PROT_RO_xscale; 7422 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 7423 7424 pte_l2_l_prot_u = L2_L_PROT_U_xscale; 7425 pte_l2_l_prot_w = L2_L_PROT_W_xscale; 7426 pte_l2_l_prot_ro = L2_L_PROT_RO_xscale; 7427 pte_l2_l_prot_mask = L2_L_PROT_MASK_xscale; 7428 7429 pte_l1_ss_proto = L1_SS_PROTO_xscale; 7430 pte_l1_s_proto = L1_S_PROTO_xscale; 7431 pte_l1_c_proto = L1_C_PROTO_xscale; 7432 pte_l2_s_proto = L2_S_PROTO_xscale; 7433 7434 pmap_copy_page_func = pmap_copy_page_xscale; 7435 pmap_zero_page_func = pmap_zero_page_xscale; 7436 7437 /* 7438 * Disable ECC protection of page table access, for now. 7439 */ 7440 auxctl = armreg_auxctl_read(); 7441 auxctl &= ~XSCALE_AUXCTL_P; 7442 armreg_auxctl_write(auxctl); 7443 } 7444 7445 /* 7446 * xscale_setup_minidata: 7447 * 7448 * Set up the mini-data cache clean area. We require the 7449 * caller to allocate the right amount of physically and 7450 * virtually contiguous space. 7451 */ 7452 void 7453 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 7454 { 7455 extern vaddr_t xscale_minidata_clean_addr; 7456 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 7457 pd_entry_t *pde = (pd_entry_t *) l1pt; 7458 vsize_t size; 7459 uint32_t auxctl; 7460 7461 xscale_minidata_clean_addr = va; 7462 7463 /* Round it to page size. */ 7464 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 7465 7466 for (; size != 0; 7467 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 7468 const size_t l1slot = l1pte_index(va); 7469 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pde[l1slot])); 7470 if (ptep == NULL) 7471 panic("xscale_setup_minidata: can't find L2 table for " 7472 "VA 0x%08lx", va); 7473 7474 ptep += l2pte_index(va); 7475 pt_entry_t opte = *ptep; 7476 l2pte_set(ptep, 7477 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 7478 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X), opte); 7479 } 7480 7481 /* 7482 * Configure the mini-data cache for write-back with 7483 * read/write-allocate. 7484 * 7485 * NOTE: In order to reconfigure the mini-data cache, we must 7486 * make sure it contains no valid data! In order to do that, 7487 * we must issue a global data cache invalidate command! 7488 * 7489 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 7490 * THIS IS VERY IMPORTANT! 7491 */ 7492 7493 /* Invalidate data and mini-data. */ 7494 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 7495 auxctl = armreg_auxctl_read(); 7496 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 7497 armreg_auxctl_write(auxctl); 7498 } 7499 7500 /* 7501 * Change the PTEs for the specified kernel mappings such that they 7502 * will use the mini data cache instead of the main data cache. 7503 */ 7504 void 7505 pmap_uarea(vaddr_t va) 7506 { 7507 vaddr_t next_bucket, eva; 7508 7509 #if (ARM_NMMUS > 1) 7510 if (xscale_use_minidata == 0) 7511 return; 7512 #endif 7513 7514 eva = va + USPACE; 7515 7516 while (va < eva) { 7517 next_bucket = L2_NEXT_BUCKET_VA(va); 7518 if (next_bucket > eva) 7519 next_bucket = eva; 7520 7521 struct l2_bucket *l2b = pmap_get_l2_bucket(pmap_kernel(), va); 7522 KDASSERT(l2b != NULL); 7523 7524 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 7525 pt_entry_t *ptep = sptep; 7526 7527 while (va < next_bucket) { 7528 const pt_entry_t opte = *ptep; 7529 if (!l2pte_minidata_p(opte)) { 7530 cpu_dcache_wbinv_range(va, PAGE_SIZE); 7531 cpu_tlb_flushD_SE(va); 7532 l2pte_set(ptep, opte & ~L2_B, opte); 7533 } 7534 ptep += PAGE_SIZE / L2_S_SIZE; 7535 va += PAGE_SIZE; 7536 } 7537 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 7538 } 7539 cpu_cpwait(); 7540 } 7541 #endif /* ARM_MMU_XSCALE == 1 */ 7542 7543 7544 #if defined(CPU_ARM11MPCORE) 7545 void 7546 pmap_pte_init_arm11mpcore(void) 7547 { 7548 7549 /* cache mode is controlled by 5 bits (B, C, TEX[2:0]) */ 7550 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6; 7551 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6; 7552 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7553 /* use extended small page (without APn, with TEX) */ 7554 pte_l2_s_cache_mask = L2_XS_CACHE_MASK_armv6; 7555 #else 7556 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6c; 7557 #endif 7558 7559 /* write-back, write-allocate */ 7560 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7561 pte_l2_l_cache_mode = L2_C | L2_B | L2_V6_L_TEX(0x01); 7562 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7563 pte_l2_s_cache_mode = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7564 #else 7565 /* no TEX. read-allocate */ 7566 pte_l2_s_cache_mode = L2_C | L2_B; 7567 #endif 7568 /* 7569 * write-back, write-allocate for page tables. 7570 */ 7571 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7572 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_V6_L_TEX(0x01); 7573 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7574 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7575 #else 7576 pte_l2_s_cache_mode_pt = L2_C | L2_B; 7577 #endif 7578 7579 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7580 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7581 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7582 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7583 7584 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7585 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7586 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7587 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7588 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7589 7590 #else 7591 /* with AP[0..3] */ 7592 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7593 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7594 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7595 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7596 #endif 7597 7598 #ifdef ARM11MPCORE_COMPAT_MMU 7599 /* with AP[0..3] */ 7600 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7601 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7602 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7603 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7604 7605 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7606 pte_l1_s_proto = L1_S_PROTO_armv6; 7607 pte_l1_c_proto = L1_C_PROTO_armv6; 7608 pte_l2_s_proto = L2_S_PROTO_armv6c; 7609 #else 7610 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7611 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7612 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7613 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7614 7615 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7616 pte_l1_s_proto = L1_S_PROTO_armv6; 7617 pte_l1_c_proto = L1_C_PROTO_armv6; 7618 pte_l2_s_proto = L2_S_PROTO_armv6n; 7619 #endif 7620 7621 pmap_copy_page_func = pmap_copy_page_generic; 7622 pmap_zero_page_func = pmap_zero_page_generic; 7623 pmap_needs_pte_sync = 1; 7624 } 7625 #endif /* CPU_ARM11MPCORE */ 7626 7627 7628 #if ARM_MMU_V6 == 1 7629 void 7630 pmap_pte_init_armv6(void) 7631 { 7632 /* 7633 * The ARMv6-A MMU is mostly compatible with generic. If the 7634 * AP field is zero, that now means "no access" rather than 7635 * read-only. The prototypes are a little different because of 7636 * the XN bit. 7637 */ 7638 pmap_pte_init_generic(); 7639 7640 pte_l1_s_nocache_mode = L1_S_XS_TEX(1); 7641 pte_l2_l_nocache_mode = L2_XS_L_TEX(1); 7642 pte_l2_s_nocache_mode = L2_XS_T_TEX(1); 7643 7644 #ifdef ARM11_COMPAT_MMU 7645 /* with AP[0..3] */ 7646 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7647 #else 7648 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6n; 7649 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6n; 7650 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6n; 7651 7652 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7653 pte_l1_s_proto = L1_S_PROTO_armv6; 7654 pte_l1_c_proto = L1_C_PROTO_armv6; 7655 pte_l2_s_proto = L2_S_PROTO_armv6n; 7656 7657 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7658 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7659 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7660 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7661 7662 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7663 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7664 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7665 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7666 7667 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7668 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7669 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7670 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7671 7672 #endif 7673 } 7674 #endif /* ARM_MMU_V6 */ 7675 7676 #if ARM_MMU_V7 == 1 7677 void 7678 pmap_pte_init_armv7(void) 7679 { 7680 /* 7681 * The ARMv7-A MMU is mostly compatible with generic. If the 7682 * AP field is zero, that now means "no access" rather than 7683 * read-only. The prototypes are a little different because of 7684 * the XN bit. 7685 */ 7686 pmap_pte_init_generic(); 7687 7688 pmap_needs_pte_sync = 1; 7689 7690 pte_l1_s_nocache_mode = L1_S_XS_TEX(1); 7691 pte_l2_l_nocache_mode = L2_XS_L_TEX(1); 7692 pte_l2_s_nocache_mode = L2_XS_T_TEX(1); 7693 7694 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv7; 7695 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv7; 7696 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv7; 7697 7698 /* 7699 * If the core support coherent walk then updates to translation tables 7700 * do not require a clean to the point of unification to ensure 7701 * visibility by subsequent translation table walks. That means we can 7702 * map everything shareable and cached and the right thing will happen. 7703 */ 7704 if (__SHIFTOUT(armreg_mmfr3_read(), __BITS(23,20))) { 7705 pmap_needs_pte_sync = 0; 7706 7707 /* 7708 * write-back, no write-allocate, shareable for normal pages. 7709 */ 7710 pte_l1_s_cache_mode |= L1_S_V6_S; 7711 pte_l2_l_cache_mode |= L2_XS_S; 7712 pte_l2_s_cache_mode |= L2_XS_S; 7713 } 7714 7715 /* 7716 * Page tables are just all other memory. We can use write-back since 7717 * pmap_needs_pte_sync is 1 (or the MMU can read out of cache). 7718 */ 7719 pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode; 7720 pte_l2_l_cache_mode_pt = pte_l2_l_cache_mode; 7721 pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; 7722 7723 /* 7724 * Check the Memory Model Features to see if this CPU supports 7725 * the TLBIASID coproc op. 7726 */ 7727 if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(16,19)) >= 2) { 7728 arm_has_tlbiasid_p = true; 7729 } else if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(12,15)) >= 2) { 7730 arm_has_tlbiasid_p = true; 7731 } 7732 7733 /* 7734 * Check the MPIDR to see if this CPU supports MP extensions. 7735 */ 7736 #ifdef MULTIPROCESSOR 7737 arm_has_mpext_p = (armreg_mpidr_read() & (MPIDR_MP|MPIDR_U)) == MPIDR_MP; 7738 #else 7739 arm_has_mpext_p = false; 7740 #endif 7741 7742 pte_l1_s_prot_u = L1_S_PROT_U_armv7; 7743 pte_l1_s_prot_w = L1_S_PROT_W_armv7; 7744 pte_l1_s_prot_ro = L1_S_PROT_RO_armv7; 7745 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv7; 7746 7747 pte_l2_s_prot_u = L2_S_PROT_U_armv7; 7748 pte_l2_s_prot_w = L2_S_PROT_W_armv7; 7749 pte_l2_s_prot_ro = L2_S_PROT_RO_armv7; 7750 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv7; 7751 7752 pte_l2_l_prot_u = L2_L_PROT_U_armv7; 7753 pte_l2_l_prot_w = L2_L_PROT_W_armv7; 7754 pte_l2_l_prot_ro = L2_L_PROT_RO_armv7; 7755 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv7; 7756 7757 pte_l1_ss_proto = L1_SS_PROTO_armv7; 7758 pte_l1_s_proto = L1_S_PROTO_armv7; 7759 pte_l1_c_proto = L1_C_PROTO_armv7; 7760 pte_l2_s_proto = L2_S_PROTO_armv7; 7761 7762 } 7763 #endif /* ARM_MMU_V7 */ 7764 7765 /* 7766 * return the PA of the current L1 table, for use when handling a crash dump 7767 */ 7768 uint32_t 7769 pmap_kernel_L1_addr(void) 7770 { 7771 #ifdef ARM_MMU_EXTENDED 7772 return pmap_kernel()->pm_l1_pa; 7773 #else 7774 return pmap_kernel()->pm_l1->l1_physaddr; 7775 #endif 7776 } 7777 7778 #if defined(DDB) 7779 /* 7780 * A couple of ddb-callable functions for dumping pmaps 7781 */ 7782 void pmap_dump(pmap_t); 7783 7784 static pt_entry_t ncptes[64]; 7785 static void pmap_dump_ncpg(pmap_t); 7786 7787 void 7788 pmap_dump(pmap_t pm) 7789 { 7790 struct l2_dtable *l2; 7791 struct l2_bucket *l2b; 7792 pt_entry_t *ptep, pte; 7793 vaddr_t l2_va, l2b_va, va; 7794 int i, j, k, occ, rows = 0; 7795 7796 if (pm == pmap_kernel()) 7797 printf("pmap_kernel (%p): ", pm); 7798 else 7799 printf("user pmap (%p): ", pm); 7800 7801 #ifdef ARM_MMU_EXTENDED 7802 printf("l1 at %p\n", pmap_l1_kva(pm)); 7803 #else 7804 printf("domain %d, l1 at %p\n", pmap_domain(pm), pmap_l1_kva(pm)); 7805 #endif 7806 7807 l2_va = 0; 7808 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 7809 l2 = pm->pm_l2[i]; 7810 7811 if (l2 == NULL || l2->l2_occupancy == 0) 7812 continue; 7813 7814 l2b_va = l2_va; 7815 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 7816 l2b = &l2->l2_bucket[j]; 7817 7818 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 7819 continue; 7820 7821 ptep = l2b->l2b_kva; 7822 7823 for (k = 0; k < 256 && ptep[k] == 0; k++) 7824 ; 7825 7826 k &= ~63; 7827 occ = l2b->l2b_occupancy; 7828 va = l2b_va + (k * 4096); 7829 for (; k < 256; k++, va += 0x1000) { 7830 char ch = ' '; 7831 if ((k % 64) == 0) { 7832 if ((rows % 8) == 0) { 7833 printf( 7834 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 7835 } 7836 printf("%08lx: ", va); 7837 } 7838 7839 ncptes[k & 63] = 0; 7840 pte = ptep[k]; 7841 if (pte == 0) { 7842 ch = '.'; 7843 } else { 7844 occ--; 7845 switch (pte & 0x4c) { 7846 case 0x00: 7847 ch = 'N'; /* No cache No buff */ 7848 break; 7849 case 0x04: 7850 ch = 'B'; /* No cache buff */ 7851 break; 7852 case 0x08: 7853 ch = 'C'; /* Cache No buff */ 7854 break; 7855 case 0x0c: 7856 ch = 'F'; /* Cache Buff */ 7857 break; 7858 case 0x40: 7859 ch = 'D'; 7860 break; 7861 case 0x48: 7862 ch = 'm'; /* Xscale mini-data */ 7863 break; 7864 default: 7865 ch = '?'; 7866 break; 7867 } 7868 7869 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 7870 ch += 0x20; 7871 7872 if ((pte & 0xc) == 0) 7873 ncptes[k & 63] = pte; 7874 } 7875 7876 if ((k % 64) == 63) { 7877 rows++; 7878 printf("%c\n", ch); 7879 pmap_dump_ncpg(pm); 7880 if (occ == 0) 7881 break; 7882 } else 7883 printf("%c", ch); 7884 } 7885 } 7886 } 7887 } 7888 7889 static void 7890 pmap_dump_ncpg(pmap_t pm) 7891 { 7892 struct vm_page *pg; 7893 struct vm_page_md *md; 7894 struct pv_entry *pv; 7895 int i; 7896 7897 for (i = 0; i < 63; i++) { 7898 if (ncptes[i] == 0) 7899 continue; 7900 7901 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 7902 if (pg == NULL) 7903 continue; 7904 md = VM_PAGE_TO_MD(pg); 7905 7906 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 7907 VM_PAGE_TO_PHYS(pg), 7908 md->krw_mappings, md->kro_mappings, 7909 md->urw_mappings, md->uro_mappings); 7910 7911 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 7912 printf(" %c va 0x%08lx, flags 0x%x\n", 7913 (pm == pv->pv_pmap) ? '*' : ' ', 7914 pv->pv_va, pv->pv_flags); 7915 } 7916 } 7917 } 7918 #endif 7919 7920 #ifdef PMAP_STEAL_MEMORY 7921 void 7922 pmap_boot_pageadd(pv_addr_t *newpv) 7923 { 7924 pv_addr_t *pv, *npv; 7925 7926 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 7927 if (newpv->pv_pa < pv->pv_va) { 7928 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 7929 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 7930 newpv->pv_size += pv->pv_size; 7931 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 7932 } 7933 pv = NULL; 7934 } else { 7935 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 7936 pv = npv) { 7937 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 7938 KASSERT(pv->pv_pa < newpv->pv_pa); 7939 if (newpv->pv_pa > npv->pv_pa) 7940 continue; 7941 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 7942 pv->pv_size += newpv->pv_size; 7943 return; 7944 } 7945 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 7946 break; 7947 newpv->pv_size += npv->pv_size; 7948 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7949 SLIST_REMOVE_AFTER(newpv, pv_list); 7950 return; 7951 } 7952 } 7953 } 7954 7955 if (pv) { 7956 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7957 } else { 7958 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 7959 } 7960 } 7961 7962 void 7963 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 7964 pv_addr_t *rpv) 7965 { 7966 pv_addr_t *pv, **pvp; 7967 7968 KASSERT(amount & PGOFSET); 7969 KASSERT((mask & PGOFSET) == 0); 7970 KASSERT((match & PGOFSET) == 0); 7971 KASSERT(amount != 0); 7972 7973 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7974 (pv = *pvp) != NULL; 7975 pvp = &SLIST_NEXT(pv, pv_list)) { 7976 pv_addr_t *newpv; 7977 psize_t off; 7978 /* 7979 * If this entry is too small to satisfy the request... 7980 */ 7981 KASSERT(pv->pv_size > 0); 7982 if (pv->pv_size < amount) 7983 continue; 7984 7985 for (off = 0; off <= mask; off += PAGE_SIZE) { 7986 if (((pv->pv_pa + off) & mask) == match 7987 && off + amount <= pv->pv_size) 7988 break; 7989 } 7990 if (off > mask) 7991 continue; 7992 7993 rpv->pv_va = pv->pv_va + off; 7994 rpv->pv_pa = pv->pv_pa + off; 7995 rpv->pv_size = amount; 7996 pv->pv_size -= amount; 7997 if (pv->pv_size == 0) { 7998 KASSERT(off == 0); 7999 KASSERT((vaddr_t) pv == rpv->pv_va); 8000 *pvp = SLIST_NEXT(pv, pv_list); 8001 } else if (off == 0) { 8002 KASSERT((vaddr_t) pv == rpv->pv_va); 8003 newpv = (pv_addr_t *) (rpv->pv_va + amount); 8004 *newpv = *pv; 8005 newpv->pv_pa += amount; 8006 newpv->pv_va += amount; 8007 *pvp = newpv; 8008 } else if (off < pv->pv_size) { 8009 newpv = (pv_addr_t *) (rpv->pv_va + amount); 8010 *newpv = *pv; 8011 newpv->pv_size -= off; 8012 newpv->pv_pa += off + amount; 8013 newpv->pv_va += off + amount; 8014 8015 SLIST_NEXT(pv, pv_list) = newpv; 8016 pv->pv_size = off; 8017 } else { 8018 KASSERT((vaddr_t) pv != rpv->pv_va); 8019 } 8020 memset((void *)rpv->pv_va, 0, amount); 8021 return; 8022 } 8023 8024 if (!uvm_physseg_valid_p(uvm_physseg_get_first())) 8025 panic("pmap_boot_pagealloc: couldn't allocate memory"); 8026 8027 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 8028 (pv = *pvp) != NULL; 8029 pvp = &SLIST_NEXT(pv, pv_list)) { 8030 if (SLIST_NEXT(pv, pv_list) == NULL) 8031 break; 8032 } 8033 KASSERT(mask == 0); 8034 8035 for (uvm_physseg_t ups = uvm_physseg_get_first(); 8036 uvm_physseg_valid_p(ups); 8037 ups = uvm_physseg_get_next(ups)) { 8038 8039 paddr_t spn = uvm_physseg_get_start(ups); 8040 paddr_t epn = uvm_physseg_get_end(ups); 8041 if (spn == atop(pv->pv_pa + pv->pv_size) 8042 && pv->pv_va + pv->pv_size <= ptoa(epn)) { 8043 rpv->pv_va = pv->pv_va; 8044 rpv->pv_pa = pv->pv_pa; 8045 rpv->pv_size = amount; 8046 *pvp = NULL; 8047 pmap_map_chunk(kernel_l1pt.pv_va, 8048 ptoa(spn) + (pv->pv_va - pv->pv_pa), 8049 ptoa(spn), 8050 amount - pv->pv_size, 8051 VM_PROT_READ|VM_PROT_WRITE, 8052 PTE_CACHE); 8053 8054 uvm_physseg_unplug(spn, atop(amount - pv->pv_size)); 8055 memset((void *)rpv->pv_va, 0, rpv->pv_size); 8056 return; 8057 } 8058 } 8059 8060 panic("pmap_boot_pagealloc: couldn't allocate memory"); 8061 } 8062 8063 vaddr_t 8064 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 8065 { 8066 pv_addr_t pv; 8067 8068 pmap_boot_pagealloc(size, 0, 0, &pv); 8069 8070 return pv.pv_va; 8071 } 8072 #endif /* PMAP_STEAL_MEMORY */ 8073 8074 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 8075 { 8076 sysctl_createv(clog, 0, NULL, NULL, 8077 CTLFLAG_PERMANENT, 8078 CTLTYPE_NODE, "machdep", NULL, 8079 NULL, 0, NULL, 0, 8080 CTL_MACHDEP, CTL_EOL); 8081 8082 sysctl_createv(clog, 0, NULL, NULL, 8083 CTLFLAG_PERMANENT, 8084 CTLTYPE_INT, "kmpages", 8085 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 8086 NULL, 0, &pmap_kmpages, 0, 8087 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 8088 } 8089 8090 #ifdef PMAP_NEED_ALLOC_POOLPAGE 8091 struct vm_page * 8092 arm_pmap_alloc_poolpage(int flags) 8093 { 8094 /* 8095 * On some systems, only some pages may be "coherent" for dma and we 8096 * want to prefer those for pool pages (think mbufs) but fallback to 8097 * any page if none is available. 8098 */ 8099 if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) { 8100 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 8101 UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist); 8102 } 8103 8104 return uvm_pagealloc(NULL, 0, NULL, flags); 8105 } 8106 #endif 8107 8108 #if defined(ARM_MMU_EXTENDED) && defined(MULTIPROCESSOR) 8109 void 8110 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 8111 { 8112 /* nothing */ 8113 } 8114 8115 int 8116 pic_ipi_shootdown(void *arg) 8117 { 8118 #if PMAP_TLB_NEED_SHOOTDOWN 8119 pmap_tlb_shootdown_process(); 8120 #endif 8121 return 1; 8122 } 8123 #endif /* ARM_MMU_EXTENDED && MULTIPROCESSOR */ 8124 8125 8126 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 8127 vaddr_t 8128 pmap_direct_mapped_phys(paddr_t pa, bool *ok_p, vaddr_t va) 8129 { 8130 bool ok = false; 8131 if (physical_start <= pa && pa < physical_end) { 8132 #ifdef KERNEL_BASE_VOFFSET 8133 const vaddr_t newva = pa + KERNEL_BASE_VOFFSET; 8134 #else 8135 const vaddr_t newva = KERNEL_BASE + pa - physical_start; 8136 #endif 8137 #ifdef ARM_MMU_EXTENDED 8138 if (newva >= KERNEL_BASE && newva < pmap_directlimit) { 8139 #endif 8140 va = newva; 8141 ok = true; 8142 #ifdef ARM_MMU_EXTENDED 8143 } 8144 #endif 8145 } 8146 KASSERT(ok_p); 8147 *ok_p = ok; 8148 return va; 8149 } 8150 8151 vaddr_t 8152 pmap_map_poolpage(paddr_t pa) 8153 { 8154 bool ok __diagused; 8155 vaddr_t va = pmap_direct_mapped_phys(pa, &ok, 0); 8156 KASSERTMSG(ok, "pa %#lx not direct mappable", pa); 8157 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 8158 if (arm_cache_prefer_mask != 0) { 8159 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 8160 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 8161 pmap_acquire_page_lock(md); 8162 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 8163 pmap_release_page_lock(md); 8164 } 8165 #endif 8166 return va; 8167 } 8168 8169 paddr_t 8170 pmap_unmap_poolpage(vaddr_t va) 8171 { 8172 KASSERT(va >= KERNEL_BASE); 8173 #ifdef PMAP_CACHE_VIVT 8174 cpu_idcache_wbinv_range(va, PAGE_SIZE); 8175 #endif 8176 #if defined(KERNEL_BASE_VOFFSET) 8177 return va - KERNEL_BASE_VOFFSET; 8178 #else 8179 return va - KERNEL_BASE + physical_start; 8180 #endif 8181 } 8182 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 8183
Indexes created Sat Oct 25 16:10:12 GMT 2025