pmap.c revision 1.379
1 /* $NetBSD: pmap.c,v 1.379 2020/01/18 07:52:33 skrll Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependent vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of in kernel VA space, 166 * then MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* 183 * Special compilation symbols 184 * PMAP_DEBUG - Build in pmap_debug_level code 185 */ 186 187 /* Include header files */ 188 189 #include "opt_arm_debug.h" 190 #include "opt_cpuoptions.h" 191 #include "opt_pmap_debug.h" 192 #include "opt_ddb.h" 193 #include "opt_lockdebug.h" 194 #include "opt_multiprocessor.h" 195 196 #ifdef MULTIPROCESSOR 197 #define _INTR_PRIVATE 198 #endif 199 200 #include <sys/param.h> 201 #include <sys/types.h> 202 #include <sys/kernel.h> 203 #include <sys/systm.h> 204 #include <sys/proc.h> 205 #include <sys/intr.h> 206 #include <sys/pool.h> 207 #include <sys/kmem.h> 208 #include <sys/cdefs.h> 209 #include <sys/cpu.h> 210 #include <sys/sysctl.h> 211 #include <sys/bus.h> 212 #include <sys/atomic.h> 213 #include <sys/kernhist.h> 214 215 #include <uvm/uvm.h> 216 #include <uvm/pmap/pmap_pvt.h> 217 218 #include <arm/locore.h> 219 220 #ifdef DDB 221 #include <arm/db_machdep.h> 222 #endif 223 224 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.379 2020/01/18 07:52:33 skrll Exp $"); 225 226 //#define PMAP_DEBUG 227 #ifdef PMAP_DEBUG 228 229 /* XXX need to get rid of all refs to this */ 230 int pmap_debug_level = 0; 231 232 /* 233 * for switching to potentially finer grained debugging 234 */ 235 #define PDB_FOLLOW 0x0001 236 #define PDB_INIT 0x0002 237 #define PDB_ENTER 0x0004 238 #define PDB_REMOVE 0x0008 239 #define PDB_CREATE 0x0010 240 #define PDB_PTPAGE 0x0020 241 #define PDB_GROWKERN 0x0040 242 #define PDB_BITS 0x0080 243 #define PDB_COLLECT 0x0100 244 #define PDB_PROTECT 0x0200 245 #define PDB_MAP_L1 0x0400 246 #define PDB_BOOTSTRAP 0x1000 247 #define PDB_PARANOIA 0x2000 248 #define PDB_WIRING 0x4000 249 #define PDB_PVDUMP 0x8000 250 #define PDB_VAC 0x10000 251 #define PDB_KENTER 0x20000 252 #define PDB_KREMOVE 0x40000 253 #define PDB_EXEC 0x80000 254 255 int debugmap = 1; 256 int pmapdebug = 0; 257 #define NPDEBUG(_lev_,_stat_) \ 258 if (pmapdebug & (_lev_)) \ 259 ((_stat_)) 260 261 #else /* PMAP_DEBUG */ 262 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 263 #endif /* PMAP_DEBUG */ 264 265 266 #ifdef VERBOSE_INIT_ARM 267 #define VPRINTF(...) printf(__VA_ARGS__) 268 #else 269 #define VPRINTF(...) __nothing 270 #endif 271 272 /* 273 * pmap_kernel() points here 274 */ 275 static struct pmap kernel_pmap_store = { 276 #ifndef ARM_MMU_EXTENDED 277 .pm_activated = true, 278 .pm_domain = PMAP_DOMAIN_KERNEL, 279 .pm_cstate.cs_all = PMAP_CACHE_STATE_ALL, 280 #endif 281 }; 282 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 283 #undef pmap_kernel 284 #define pmap_kernel() (&kernel_pmap_store) 285 #ifdef PMAP_NEED_ALLOC_POOLPAGE 286 int arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT; 287 #endif 288 289 /* 290 * Pool and cache that pmap structures are allocated from. 291 * We use a cache to avoid clearing the pm_l2[] array (1KB) 292 * in pmap_create(). 293 */ 294 static struct pool_cache pmap_cache; 295 296 /* 297 * Pool of PV structures 298 */ 299 static struct pool pmap_pv_pool; 300 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 301 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 302 static struct pool_allocator pmap_bootstrap_pv_allocator = { 303 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 304 }; 305 306 /* 307 * Pool and cache of l2_dtable structures. 308 * We use a cache to avoid clearing the structures when they're 309 * allocated. (196 bytes) 310 */ 311 static struct pool_cache pmap_l2dtable_cache; 312 static vaddr_t pmap_kernel_l2dtable_kva; 313 314 /* 315 * Pool and cache of L2 page descriptors. 316 * We use a cache to avoid clearing the descriptor table 317 * when they're allocated. (1KB) 318 */ 319 static struct pool_cache pmap_l2ptp_cache; 320 static vaddr_t pmap_kernel_l2ptp_kva; 321 static paddr_t pmap_kernel_l2ptp_phys; 322 323 #ifdef PMAPCOUNTERS 324 #define PMAP_EVCNT_INITIALIZER(name) \ 325 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 326 327 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 328 static struct evcnt pmap_ev_vac_clean_one = 329 PMAP_EVCNT_INITIALIZER("clean page (1 color)"); 330 static struct evcnt pmap_ev_vac_flush_one = 331 PMAP_EVCNT_INITIALIZER("flush page (1 color)"); 332 static struct evcnt pmap_ev_vac_flush_lots = 333 PMAP_EVCNT_INITIALIZER("flush page (2+ colors)"); 334 static struct evcnt pmap_ev_vac_flush_lots2 = 335 PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)"); 336 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one); 337 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one); 338 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots); 339 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2); 340 341 static struct evcnt pmap_ev_vac_color_new = 342 PMAP_EVCNT_INITIALIZER("new page color"); 343 static struct evcnt pmap_ev_vac_color_reuse = 344 PMAP_EVCNT_INITIALIZER("ok first page color"); 345 static struct evcnt pmap_ev_vac_color_ok = 346 PMAP_EVCNT_INITIALIZER("ok page color"); 347 static struct evcnt pmap_ev_vac_color_blind = 348 PMAP_EVCNT_INITIALIZER("blind page color"); 349 static struct evcnt pmap_ev_vac_color_change = 350 PMAP_EVCNT_INITIALIZER("change page color"); 351 static struct evcnt pmap_ev_vac_color_erase = 352 PMAP_EVCNT_INITIALIZER("erase page color"); 353 static struct evcnt pmap_ev_vac_color_none = 354 PMAP_EVCNT_INITIALIZER("no page color"); 355 static struct evcnt pmap_ev_vac_color_restore = 356 PMAP_EVCNT_INITIALIZER("restore page color"); 357 358 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 359 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 360 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 361 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 362 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 363 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 364 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 365 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 366 #endif 367 368 static struct evcnt pmap_ev_mappings = 369 PMAP_EVCNT_INITIALIZER("pages mapped"); 370 static struct evcnt pmap_ev_unmappings = 371 PMAP_EVCNT_INITIALIZER("pages unmapped"); 372 static struct evcnt pmap_ev_remappings = 373 PMAP_EVCNT_INITIALIZER("pages remapped"); 374 375 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 376 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 377 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 378 379 static struct evcnt pmap_ev_kernel_mappings = 380 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 381 static struct evcnt pmap_ev_kernel_unmappings = 382 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 383 static struct evcnt pmap_ev_kernel_remappings = 384 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 385 386 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 387 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 388 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 389 390 static struct evcnt pmap_ev_kenter_mappings = 391 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 392 static struct evcnt pmap_ev_kenter_unmappings = 393 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 394 static struct evcnt pmap_ev_kenter_remappings = 395 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 396 static struct evcnt pmap_ev_pt_mappings = 397 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 398 399 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 400 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 401 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 402 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 403 404 static struct evcnt pmap_ev_fixup_mod = 405 PMAP_EVCNT_INITIALIZER("page modification emulations"); 406 static struct evcnt pmap_ev_fixup_ref = 407 PMAP_EVCNT_INITIALIZER("page reference emulations"); 408 static struct evcnt pmap_ev_fixup_exec = 409 PMAP_EVCNT_INITIALIZER("exec pages fixed up"); 410 static struct evcnt pmap_ev_fixup_pdes = 411 PMAP_EVCNT_INITIALIZER("pdes fixed up"); 412 #ifndef ARM_MMU_EXTENDED 413 static struct evcnt pmap_ev_fixup_ptesync = 414 PMAP_EVCNT_INITIALIZER("ptesync fixed"); 415 #endif 416 417 EVCNT_ATTACH_STATIC(pmap_ev_fixup_mod); 418 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ref); 419 EVCNT_ATTACH_STATIC(pmap_ev_fixup_exec); 420 EVCNT_ATTACH_STATIC(pmap_ev_fixup_pdes); 421 #ifndef ARM_MMU_EXTENDED 422 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ptesync); 423 #endif 424 425 #ifdef PMAP_CACHE_VIPT 426 static struct evcnt pmap_ev_exec_mappings = 427 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 428 static struct evcnt pmap_ev_exec_cached = 429 PMAP_EVCNT_INITIALIZER("exec pages cached"); 430 431 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 432 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 433 434 static struct evcnt pmap_ev_exec_synced = 435 PMAP_EVCNT_INITIALIZER("exec pages synced"); 436 static struct evcnt pmap_ev_exec_synced_map = 437 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 438 static struct evcnt pmap_ev_exec_synced_unmap = 439 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 440 static struct evcnt pmap_ev_exec_synced_remap = 441 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 442 static struct evcnt pmap_ev_exec_synced_clearbit = 443 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 444 #ifndef ARM_MMU_EXTENDED 445 static struct evcnt pmap_ev_exec_synced_kremove = 446 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 447 #endif 448 449 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 450 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 451 #ifndef ARM_MMU_EXTENDED 452 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 453 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 454 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 455 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 456 #endif 457 458 static struct evcnt pmap_ev_exec_discarded_unmap = 459 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 460 static struct evcnt pmap_ev_exec_discarded_zero = 461 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 462 static struct evcnt pmap_ev_exec_discarded_copy = 463 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 464 static struct evcnt pmap_ev_exec_discarded_page_protect = 465 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 466 static struct evcnt pmap_ev_exec_discarded_clearbit = 467 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 468 static struct evcnt pmap_ev_exec_discarded_kremove = 469 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 470 #ifdef ARM_MMU_EXTENDED 471 static struct evcnt pmap_ev_exec_discarded_modfixup = 472 PMAP_EVCNT_INITIALIZER("exec pages discarded (MF)"); 473 #endif 474 475 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 476 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 477 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 478 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 479 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 480 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 481 #ifdef ARM_MMU_EXTENDED 482 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_modfixup); 483 #endif 484 #endif /* PMAP_CACHE_VIPT */ 485 486 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 487 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 488 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 489 490 EVCNT_ATTACH_STATIC(pmap_ev_updates); 491 EVCNT_ATTACH_STATIC(pmap_ev_collects); 492 EVCNT_ATTACH_STATIC(pmap_ev_activations); 493 494 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 495 #else 496 #define PMAPCOUNT(x) ((void)0) 497 #endif 498 499 #ifdef ARM_MMU_EXTENDED 500 void pmap_md_pdetab_activate(pmap_t, struct lwp *); 501 void pmap_md_pdetab_deactivate(pmap_t pm); 502 #endif 503 504 /* 505 * pmap copy/zero page, and mem(5) hook point 506 */ 507 static pt_entry_t *csrc_pte, *cdst_pte; 508 static vaddr_t csrcp, cdstp; 509 #ifdef MULTIPROCESSOR 510 static size_t cnptes; 511 #define cpu_csrc_pte(o) (csrc_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 512 #define cpu_cdst_pte(o) (cdst_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 513 #define cpu_csrcp(o) (csrcp + L2_S_SIZE * cnptes * cpu_number() + (o)) 514 #define cpu_cdstp(o) (cdstp + L2_S_SIZE * cnptes * cpu_number() + (o)) 515 #else 516 #define cpu_csrc_pte(o) (csrc_pte + ((o) >> L2_S_SHIFT)) 517 #define cpu_cdst_pte(o) (cdst_pte + ((o) >> L2_S_SHIFT)) 518 #define cpu_csrcp(o) (csrcp + (o)) 519 #define cpu_cdstp(o) (cdstp + (o)) 520 #endif 521 vaddr_t memhook; /* used by mem.c & others */ 522 kmutex_t memlock __cacheline_aligned; /* used by mem.c & others */ 523 kmutex_t pmap_lock __cacheline_aligned; 524 kmutex_t kpm_lock __cacheline_aligned; 525 extern void *msgbufaddr; 526 int pmap_kmpages; 527 /* 528 * Flag to indicate if pmap_init() has done its thing 529 */ 530 bool pmap_initialized; 531 532 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 533 /* 534 * Virtual end of direct-mapped memory 535 */ 536 vaddr_t pmap_directlimit; 537 #endif 538 539 /* 540 * Misc. locking data structures 541 */ 542 543 static inline void 544 pmap_acquire_pmap_lock(pmap_t pm) 545 { 546 #if defined(MULTIPROCESSOR) && defined(DDB) 547 if (__predict_false(db_onproc != NULL)) 548 return; 549 #endif 550 551 mutex_enter(pm->pm_lock); 552 } 553 554 static inline void 555 pmap_release_pmap_lock(pmap_t pm) 556 { 557 #if defined(MULTIPROCESSOR) && defined(DDB) 558 if (__predict_false(db_onproc != NULL)) 559 return; 560 #endif 561 mutex_exit(pm->pm_lock); 562 } 563 564 static inline void 565 pmap_acquire_page_lock(struct vm_page_md *md) 566 { 567 mutex_enter(&pmap_lock); 568 } 569 570 static inline void 571 pmap_release_page_lock(struct vm_page_md *md) 572 { 573 mutex_exit(&pmap_lock); 574 } 575 576 #ifdef DIAGNOSTIC 577 static inline int 578 pmap_page_locked_p(struct vm_page_md *md) 579 { 580 return mutex_owned(&pmap_lock); 581 } 582 #endif 583 584 585 /* 586 * Metadata for L1 translation tables. 587 */ 588 #ifndef ARM_MMU_EXTENDED 589 struct l1_ttable { 590 /* Entry on the L1 Table list */ 591 SLIST_ENTRY(l1_ttable) l1_link; 592 593 /* Entry on the L1 Least Recently Used list */ 594 TAILQ_ENTRY(l1_ttable) l1_lru; 595 596 /* Track how many domains are allocated from this L1 */ 597 volatile u_int l1_domain_use_count; 598 599 /* 600 * A free-list of domain numbers for this L1. 601 * We avoid using ffs() and a bitmap to track domains since ffs() 602 * is slow on ARM. 603 */ 604 uint8_t l1_domain_first; 605 uint8_t l1_domain_free[PMAP_DOMAINS]; 606 607 /* Physical address of this L1 page table */ 608 paddr_t l1_physaddr; 609 610 /* KVA of this L1 page table */ 611 pd_entry_t *l1_kva; 612 }; 613 614 /* 615 * L1 Page Tables are tracked using a Least Recently Used list. 616 * - New L1s are allocated from the HEAD. 617 * - Freed L1s are added to the TAIl. 618 * - Recently accessed L1s (where an 'access' is some change to one of 619 * the userland pmaps which owns this L1) are moved to the TAIL. 620 */ 621 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 622 static kmutex_t l1_lru_lock __cacheline_aligned; 623 624 /* 625 * A list of all L1 tables 626 */ 627 static SLIST_HEAD(, l1_ttable) l1_list; 628 #endif /* ARM_MMU_EXTENDED */ 629 630 /* 631 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 632 * 633 * This is normally 16MB worth L2 page descriptors for any given pmap. 634 * Reference counts are maintained for L2 descriptors so they can be 635 * freed when empty. 636 */ 637 struct l2_bucket { 638 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 639 paddr_t l2b_pa; /* Physical address of same */ 640 u_short l2b_l1slot; /* This L2 table's L1 index */ 641 u_short l2b_occupancy; /* How many active descriptors */ 642 }; 643 644 struct l2_dtable { 645 /* The number of L2 page descriptors allocated to this l2_dtable */ 646 u_int l2_occupancy; 647 648 /* List of L2 page descriptors */ 649 struct l2_bucket l2_bucket[L2_BUCKET_SIZE]; 650 }; 651 652 /* 653 * Given an L1 table index, calculate the corresponding l2_dtable index 654 * and bucket index within the l2_dtable. 655 */ 656 #define L2_BUCKET_XSHIFT (L2_BUCKET_XLOG2 - L1_S_SHIFT) 657 #define L2_BUCKET_XFRAME (~(vaddr_t)0 << L2_BUCKET_XLOG2) 658 #define L2_BUCKET_IDX(l1slot) ((l1slot) >> L2_BUCKET_XSHIFT) 659 #define L2_IDX(l1slot) (L2_BUCKET_IDX(l1slot) >> L2_BUCKET_LOG2) 660 #define L2_BUCKET(l1slot) (L2_BUCKET_IDX(l1slot) & (L2_BUCKET_SIZE - 1)) 661 662 __CTASSERT(0x100000000ULL == ((uint64_t)L2_SIZE * L2_BUCKET_SIZE * L1_S_SIZE)); 663 __CTASSERT(L2_BUCKET_XFRAME == ~(L2_BUCKET_XSIZE-1)); 664 665 /* 666 * Given a virtual address, this macro returns the 667 * virtual address required to drop into the next L2 bucket. 668 */ 669 #define L2_NEXT_BUCKET_VA(va) (((va) & L2_BUCKET_XFRAME) + L2_BUCKET_XSIZE) 670 671 /* 672 * L2 allocation. 673 */ 674 #define pmap_alloc_l2_dtable() \ 675 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 676 #define pmap_free_l2_dtable(l2) \ 677 pool_cache_put(&pmap_l2dtable_cache, (l2)) 678 #define pmap_alloc_l2_ptp(pap) \ 679 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 680 PR_NOWAIT, (pap))) 681 682 /* 683 * We try to map the page tables write-through, if possible. However, not 684 * all CPUs have a write-through cache mode, so on those we have to sync 685 * the cache when we frob page tables. 686 * 687 * We try to evaluate this at compile time, if possible. However, it's 688 * not always possible to do that, hence this run-time var. 689 */ 690 int pmap_needs_pte_sync; 691 692 /* 693 * Real definition of pv_entry. 694 */ 695 struct pv_entry { 696 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 697 pmap_t pv_pmap; /* pmap where mapping lies */ 698 vaddr_t pv_va; /* virtual address for mapping */ 699 u_int pv_flags; /* flags */ 700 }; 701 702 /* 703 * Macros to determine if a mapping might be resident in the 704 * instruction/data cache and/or TLB 705 */ 706 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED) 707 /* 708 * Speculative loads by Cortex cores can cause TLB entries to be filled even if 709 * there are no explicit accesses, so there may be always be TLB entries to 710 * flush. If we used ASIDs then this would not be a problem. 711 */ 712 #define PV_BEEN_EXECD(f) (((f) & PVF_EXEC) == PVF_EXEC) 713 #define PV_BEEN_REFD(f) (true) 714 #else 715 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 716 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 717 #endif 718 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 719 #define PV_IS_KENTRY_P(f) (((f) & PVF_KENTRY) != 0) 720 #define PV_IS_WRITE_P(f) (((f) & PVF_WRITE) != 0) 721 722 /* 723 * Local prototypes 724 */ 725 static bool pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t, size_t); 726 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 727 pt_entry_t **); 728 static bool pmap_is_current(pmap_t) __unused; 729 static bool pmap_is_cached(pmap_t); 730 static void pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *, 731 pmap_t, vaddr_t, u_int); 732 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t); 733 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 734 static u_int pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t, 735 u_int, u_int); 736 737 static void pmap_pinit(pmap_t); 738 static int pmap_pmap_ctor(void *, void *, int); 739 740 static void pmap_alloc_l1(pmap_t); 741 static void pmap_free_l1(pmap_t); 742 #ifndef ARM_MMU_EXTENDED 743 static void pmap_use_l1(pmap_t); 744 #endif 745 746 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 747 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 748 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 749 static int pmap_l2ptp_ctor(void *, void *, int); 750 static int pmap_l2dtable_ctor(void *, void *, int); 751 752 static void pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 753 #ifdef PMAP_CACHE_VIVT 754 static void pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 755 static void pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 756 #endif 757 758 static void pmap_clearbit(struct vm_page_md *, paddr_t, u_int); 759 #ifdef PMAP_CACHE_VIVT 760 static bool pmap_clean_page(struct vm_page_md *, bool); 761 #endif 762 #ifdef PMAP_CACHE_VIPT 763 static void pmap_syncicache_page(struct vm_page_md *, paddr_t); 764 enum pmap_flush_op { 765 PMAP_FLUSH_PRIMARY, 766 PMAP_FLUSH_SECONDARY, 767 PMAP_CLEAN_PRIMARY 768 }; 769 #ifndef ARM_MMU_EXTENDED 770 static void pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op); 771 #endif 772 #endif 773 static void pmap_page_remove(struct vm_page_md *, paddr_t); 774 static void pmap_pv_remove(paddr_t); 775 776 #ifndef ARM_MMU_EXTENDED 777 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 778 #endif 779 static vaddr_t kernel_pt_lookup(paddr_t); 780 781 782 /* 783 * Misc variables 784 */ 785 vaddr_t virtual_avail; 786 vaddr_t virtual_end; 787 vaddr_t pmap_curmaxkvaddr; 788 789 paddr_t avail_start; 790 paddr_t avail_end; 791 792 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 793 pv_addr_t kernelpages; 794 pv_addr_t kernel_l1pt; 795 pv_addr_t systempage; 796 797 /* Function to set the debug level of the pmap code */ 798 799 #ifdef PMAP_DEBUG 800 void 801 pmap_debug(int level) 802 { 803 pmap_debug_level = level; 804 printf("pmap_debug: level=%d\n", pmap_debug_level); 805 } 806 #endif /* PMAP_DEBUG */ 807 808 #ifdef PMAP_CACHE_VIPT 809 #define PMAP_VALIDATE_MD_PAGE(md) \ 810 KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \ 811 "(md) %p: attrs=%#x urw=%u krw=%u", (md), \ 812 (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings); 813 #endif /* PMAP_CACHE_VIPT */ 814 /* 815 * A bunch of routines to conditionally flush the caches/TLB depending 816 * on whether the specified pmap actually needs to be flushed at any 817 * given time. 818 */ 819 static inline void 820 pmap_tlb_flush_SE(pmap_t pm, vaddr_t va, u_int flags) 821 { 822 #ifdef ARM_MMU_EXTENDED 823 pmap_tlb_invalidate_addr(pm, va); 824 #else 825 if (pm->pm_cstate.cs_tlb_id != 0) { 826 if (PV_BEEN_EXECD(flags)) { 827 cpu_tlb_flushID_SE(va); 828 } else if (PV_BEEN_REFD(flags)) { 829 cpu_tlb_flushD_SE(va); 830 } 831 } 832 #endif /* ARM_MMU_EXTENDED */ 833 } 834 835 #ifndef ARM_MMU_EXTENDED 836 static inline void 837 pmap_tlb_flushID(pmap_t pm) 838 { 839 if (pm->pm_cstate.cs_tlb_id) { 840 cpu_tlb_flushID(); 841 #if ARM_MMU_V7 == 0 842 /* 843 * Speculative loads by Cortex cores can cause TLB entries to 844 * be filled even if there are no explicit accesses, so there 845 * may be always be TLB entries to flush. If we used ASIDs 846 * then it would not be a problem. 847 * This is not true for other CPUs. 848 */ 849 pm->pm_cstate.cs_tlb = 0; 850 #endif /* ARM_MMU_V7 */ 851 } 852 } 853 854 static inline void 855 pmap_tlb_flushD(pmap_t pm) 856 { 857 if (pm->pm_cstate.cs_tlb_d) { 858 cpu_tlb_flushD(); 859 #if ARM_MMU_V7 == 0 860 /* 861 * Speculative loads by Cortex cores can cause TLB entries to 862 * be filled even if there are no explicit accesses, so there 863 * may be always be TLB entries to flush. If we used ASIDs 864 * then it would not be a problem. 865 * This is not true for other CPUs. 866 */ 867 pm->pm_cstate.cs_tlb_d = 0; 868 #endif /* ARM_MMU_V7 */ 869 } 870 } 871 #endif /* ARM_MMU_EXTENDED */ 872 873 #ifdef PMAP_CACHE_VIVT 874 static inline void 875 pmap_cache_wbinv_page(pmap_t pm, vaddr_t va, bool do_inv, u_int flags) 876 { 877 if (PV_BEEN_EXECD(flags) && pm->pm_cstate.cs_cache_id) { 878 cpu_idcache_wbinv_range(va, PAGE_SIZE); 879 } else if (PV_BEEN_REFD(flags) && pm->pm_cstate.cs_cache_d) { 880 if (do_inv) { 881 if (flags & PVF_WRITE) 882 cpu_dcache_wbinv_range(va, PAGE_SIZE); 883 else 884 cpu_dcache_inv_range(va, PAGE_SIZE); 885 } else if (flags & PVF_WRITE) { 886 cpu_dcache_wb_range(va, PAGE_SIZE); 887 } 888 } 889 } 890 891 static inline void 892 pmap_cache_wbinv_all(pmap_t pm, u_int flags) 893 { 894 if (PV_BEEN_EXECD(flags)) { 895 if (pm->pm_cstate.cs_cache_id) { 896 cpu_idcache_wbinv_all(); 897 pm->pm_cstate.cs_cache = 0; 898 } 899 } else if (pm->pm_cstate.cs_cache_d) { 900 cpu_dcache_wbinv_all(); 901 pm->pm_cstate.cs_cache_d = 0; 902 } 903 } 904 #endif /* PMAP_CACHE_VIVT */ 905 906 static inline uint8_t 907 pmap_domain(pmap_t pm) 908 { 909 #ifdef ARM_MMU_EXTENDED 910 return pm == pmap_kernel() ? PMAP_DOMAIN_KERNEL : PMAP_DOMAIN_USER; 911 #else 912 return pm->pm_domain; 913 #endif 914 } 915 916 static inline pd_entry_t * 917 pmap_l1_kva(pmap_t pm) 918 { 919 #ifdef ARM_MMU_EXTENDED 920 return pm->pm_l1; 921 #else 922 return pm->pm_l1->l1_kva; 923 #endif 924 } 925 926 static inline bool 927 pmap_is_current(pmap_t pm) 928 { 929 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 930 return true; 931 932 return false; 933 } 934 935 static inline bool 936 pmap_is_cached(pmap_t pm) 937 { 938 #ifdef ARM_MMU_EXTENDED 939 if (pm == pmap_kernel()) 940 return true; 941 #ifdef MULTIPROCESSOR 942 // Is this pmap active on any CPU? 943 if (!kcpuset_iszero(pm->pm_active)) 944 return true; 945 #else 946 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 947 // Is this pmap active? 948 if (PMAP_PAI_ASIDVALID_P(PMAP_PAI(pm, ti), ti)) 949 return true; 950 #endif 951 #else 952 struct cpu_info * const ci = curcpu(); 953 if (pm == pmap_kernel() || ci->ci_pmap_lastuser == NULL 954 || ci->ci_pmap_lastuser == pm) 955 return true; 956 #endif /* ARM_MMU_EXTENDED */ 957 958 return false; 959 } 960 961 /* 962 * PTE_SYNC_CURRENT: 963 * 964 * Make sure the pte is written out to RAM. 965 * We need to do this for one of two cases: 966 * - We're dealing with the kernel pmap 967 * - There is no pmap active in the cache/tlb. 968 * - The specified pmap is 'active' in the cache/tlb. 969 */ 970 971 #ifdef PMAP_INCLUDE_PTE_SYNC 972 static inline void 973 pmap_pte_sync_current(pmap_t pm, pt_entry_t *ptep) 974 { 975 if (PMAP_NEEDS_PTE_SYNC && pmap_is_cached(pm)) 976 PTE_SYNC(ptep); 977 arm_dsb(); 978 } 979 980 # define PTE_SYNC_CURRENT(pm, ptep) pmap_pte_sync_current(pm, ptep) 981 #else 982 # define PTE_SYNC_CURRENT(pm, ptep) __nothing 983 #endif 984 985 /* 986 * main pv_entry manipulation functions: 987 * pmap_enter_pv: enter a mapping onto a vm_page list 988 * pmap_remove_pv: remove a mapping from a vm_page list 989 * 990 * NOTE: pmap_enter_pv expects to lock the pvh itself 991 * pmap_remove_pv expects the caller to lock the pvh before calling 992 */ 993 994 /* 995 * pmap_enter_pv: enter a mapping onto a vm_page lst 996 * 997 * => caller should hold the proper lock on pmap_main_lock 998 * => caller should have pmap locked 999 * => we will gain the lock on the vm_page and allocate the new pv_entry 1000 * => caller should adjust ptp's wire_count before calling 1001 * => caller should not adjust pmap's wire_count 1002 */ 1003 static void 1004 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm, 1005 vaddr_t va, u_int flags) 1006 { 1007 struct pv_entry **pvp; 1008 1009 NPDEBUG(PDB_PVDUMP, 1010 printf("pmap_enter_pv: pm %p, md %p, flags 0x%x\n", pm, md, flags)); 1011 1012 pv->pv_pmap = pm; 1013 pv->pv_va = va; 1014 pv->pv_flags = flags; 1015 1016 pvp = &SLIST_FIRST(&md->pvh_list); 1017 #ifdef PMAP_CACHE_VIPT 1018 /* 1019 * Insert unmanaged entries, writeable first, at the head of 1020 * the pv list. 1021 */ 1022 if (__predict_true(!PV_IS_KENTRY_P(flags))) { 1023 while (*pvp != NULL && PV_IS_KENTRY_P((*pvp)->pv_flags)) 1024 pvp = &SLIST_NEXT(*pvp, pv_link); 1025 } 1026 if (!PV_IS_WRITE_P(flags)) { 1027 while (*pvp != NULL && PV_IS_WRITE_P((*pvp)->pv_flags)) 1028 pvp = &SLIST_NEXT(*pvp, pv_link); 1029 } 1030 #endif 1031 SLIST_NEXT(pv, pv_link) = *pvp; /* add to ... */ 1032 *pvp = pv; /* ... locked list */ 1033 md->pvh_attrs |= flags & (PVF_REF | PVF_MOD); 1034 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1035 if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE) 1036 md->pvh_attrs |= PVF_KMOD; 1037 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1038 md->pvh_attrs |= PVF_DIRTY; 1039 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1040 #endif 1041 if (pm == pmap_kernel()) { 1042 PMAPCOUNT(kernel_mappings); 1043 if (flags & PVF_WRITE) 1044 md->krw_mappings++; 1045 else 1046 md->kro_mappings++; 1047 } else { 1048 if (flags & PVF_WRITE) 1049 md->urw_mappings++; 1050 else 1051 md->uro_mappings++; 1052 } 1053 1054 #ifdef PMAP_CACHE_VIPT 1055 #ifndef ARM_MMU_EXTENDED 1056 /* 1057 * Even though pmap_vac_me_harder will set PVF_WRITE for us, 1058 * do it here as well to keep the mappings & KVF_WRITE consistent. 1059 */ 1060 if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) { 1061 md->pvh_attrs |= PVF_WRITE; 1062 } 1063 #endif 1064 /* 1065 * If this is an exec mapping and its the first exec mapping 1066 * for this page, make sure to sync the I-cache. 1067 */ 1068 if (PV_IS_EXEC_P(flags)) { 1069 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 1070 pmap_syncicache_page(md, pa); 1071 PMAPCOUNT(exec_synced_map); 1072 } 1073 PMAPCOUNT(exec_mappings); 1074 } 1075 #endif 1076 1077 PMAPCOUNT(mappings); 1078 1079 if (pv->pv_flags & PVF_WIRED) 1080 ++pm->pm_stats.wired_count; 1081 } 1082 1083 /* 1084 * 1085 * pmap_find_pv: Find a pv entry 1086 * 1087 * => caller should hold lock on vm_page 1088 */ 1089 static inline struct pv_entry * 1090 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va) 1091 { 1092 struct pv_entry *pv; 1093 1094 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1095 if (pm == pv->pv_pmap && va == pv->pv_va) 1096 break; 1097 } 1098 1099 return (pv); 1100 } 1101 1102 /* 1103 * pmap_remove_pv: try to remove a mapping from a pv_list 1104 * 1105 * => caller should hold proper lock on pmap_main_lock 1106 * => pmap should be locked 1107 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1108 * => caller should adjust ptp's wire_count and free PTP if needed 1109 * => caller should NOT adjust pmap's wire_count 1110 * => we return the removed pv 1111 */ 1112 static struct pv_entry * 1113 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1114 { 1115 struct pv_entry *pv, **prevptr; 1116 1117 NPDEBUG(PDB_PVDUMP, 1118 printf("pmap_remove_pv: pm %p, md %p, va 0x%08lx\n", pm, md, va)); 1119 1120 prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */ 1121 pv = *prevptr; 1122 1123 while (pv) { 1124 if (pv->pv_pmap == pm && pv->pv_va == va) { /* match? */ 1125 NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, md " 1126 "%p, flags 0x%x\n", pm, md, pv->pv_flags)); 1127 if (pv->pv_flags & PVF_WIRED) { 1128 --pm->pm_stats.wired_count; 1129 } 1130 *prevptr = SLIST_NEXT(pv, pv_link); /* remove it! */ 1131 if (pm == pmap_kernel()) { 1132 PMAPCOUNT(kernel_unmappings); 1133 if (pv->pv_flags & PVF_WRITE) 1134 md->krw_mappings--; 1135 else 1136 md->kro_mappings--; 1137 } else { 1138 if (pv->pv_flags & PVF_WRITE) 1139 md->urw_mappings--; 1140 else 1141 md->uro_mappings--; 1142 } 1143 1144 PMAPCOUNT(unmappings); 1145 #ifdef PMAP_CACHE_VIPT 1146 /* 1147 * If this page has had an exec mapping, then if 1148 * this was the last mapping, discard the contents, 1149 * otherwise sync the i-cache for this page. 1150 */ 1151 if (PV_IS_EXEC_P(md->pvh_attrs)) { 1152 if (SLIST_EMPTY(&md->pvh_list)) { 1153 md->pvh_attrs &= ~PVF_EXEC; 1154 PMAPCOUNT(exec_discarded_unmap); 1155 } else if (pv->pv_flags & PVF_WRITE) { 1156 pmap_syncicache_page(md, pa); 1157 PMAPCOUNT(exec_synced_unmap); 1158 } 1159 } 1160 #endif /* PMAP_CACHE_VIPT */ 1161 break; 1162 } 1163 prevptr = &SLIST_NEXT(pv, pv_link); /* previous pointer */ 1164 pv = *prevptr; /* advance */ 1165 } 1166 1167 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1168 /* 1169 * If we no longer have a WRITEABLE KENTRY at the head of list, 1170 * clear the KMOD attribute from the page. 1171 */ 1172 if (SLIST_FIRST(&md->pvh_list) == NULL 1173 || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE) 1174 md->pvh_attrs &= ~PVF_KMOD; 1175 1176 /* 1177 * If this was a writeable page and there are no more writeable 1178 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1179 * the contents to memory. 1180 */ 1181 if (arm_cache_prefer_mask != 0) { 1182 if (md->krw_mappings + md->urw_mappings == 0) 1183 md->pvh_attrs &= ~PVF_WRITE; 1184 PMAP_VALIDATE_MD_PAGE(md); 1185 } 1186 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1187 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1188 1189 /* return removed pv */ 1190 return pv; 1191 } 1192 1193 /* 1194 * 1195 * pmap_modify_pv: Update pv flags 1196 * 1197 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1198 * => caller should NOT adjust pmap's wire_count 1199 * => caller must call pmap_vac_me_harder() if writable status of a page 1200 * may have changed. 1201 * => we return the old flags 1202 * 1203 * Modify a physical-virtual mapping in the pv table 1204 */ 1205 static u_int 1206 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va, 1207 u_int clr_mask, u_int set_mask) 1208 { 1209 struct pv_entry *npv; 1210 u_int flags, oflags; 1211 1212 KASSERT(!PV_IS_KENTRY_P(clr_mask)); 1213 KASSERT(!PV_IS_KENTRY_P(set_mask)); 1214 1215 if ((npv = pmap_find_pv(md, pm, va)) == NULL) 1216 return (0); 1217 1218 NPDEBUG(PDB_PVDUMP, 1219 printf("pmap_modify_pv: pm %p, md %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, md, clr_mask, set_mask, npv->pv_flags)); 1220 1221 /* 1222 * There is at least one VA mapping this page. 1223 */ 1224 1225 if (clr_mask & (PVF_REF | PVF_MOD)) { 1226 md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1227 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1228 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1229 md->pvh_attrs |= PVF_DIRTY; 1230 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1231 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1232 } 1233 1234 oflags = npv->pv_flags; 1235 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1236 1237 if ((flags ^ oflags) & PVF_WIRED) { 1238 if (flags & PVF_WIRED) 1239 ++pm->pm_stats.wired_count; 1240 else 1241 --pm->pm_stats.wired_count; 1242 } 1243 1244 if ((flags ^ oflags) & PVF_WRITE) { 1245 if (pm == pmap_kernel()) { 1246 if (flags & PVF_WRITE) { 1247 md->krw_mappings++; 1248 md->kro_mappings--; 1249 } else { 1250 md->kro_mappings++; 1251 md->krw_mappings--; 1252 } 1253 } else { 1254 if (flags & PVF_WRITE) { 1255 md->urw_mappings++; 1256 md->uro_mappings--; 1257 } else { 1258 md->uro_mappings++; 1259 md->urw_mappings--; 1260 } 1261 } 1262 } 1263 #ifdef PMAP_CACHE_VIPT 1264 if (arm_cache_prefer_mask != 0) { 1265 if (md->urw_mappings + md->krw_mappings == 0) { 1266 md->pvh_attrs &= ~PVF_WRITE; 1267 } else { 1268 md->pvh_attrs |= PVF_WRITE; 1269 } 1270 } 1271 /* 1272 * We have two cases here: the first is from enter_pv (new exec 1273 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1274 * Since in latter, pmap_enter_pv won't do anything, we just have 1275 * to do what pmap_remove_pv would do. 1276 */ 1277 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs)) 1278 || (PV_IS_EXEC_P(md->pvh_attrs) 1279 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1280 pmap_syncicache_page(md, pa); 1281 PMAPCOUNT(exec_synced_remap); 1282 } 1283 #ifndef ARM_MMU_EXTENDED 1284 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1285 #endif /* !ARM_MMU_EXTENDED */ 1286 #endif /* PMAP_CACHE_VIPT */ 1287 1288 PMAPCOUNT(remappings); 1289 1290 return (oflags); 1291 } 1292 1293 /* 1294 * Allocate an L1 translation table for the specified pmap. 1295 * This is called at pmap creation time. 1296 */ 1297 static void 1298 pmap_alloc_l1(pmap_t pm) 1299 { 1300 #ifdef ARM_MMU_EXTENDED 1301 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 1302 struct vm_page *pg; 1303 bool ok __diagused; 1304 for (;;) { 1305 #ifdef PMAP_NEED_ALLOC_POOLPAGE 1306 pg = arm_pmap_alloc_poolpage(UVM_PGA_ZERO); 1307 #else 1308 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1309 #endif 1310 if (pg != NULL) 1311 break; 1312 uvm_wait("pmapl1alloc"); 1313 } 1314 pm->pm_l1_pa = VM_PAGE_TO_PHYS(pg); 1315 vaddr_t va = pmap_direct_mapped_phys(pm->pm_l1_pa, &ok, 0); 1316 KASSERT(ok); 1317 KASSERT(va >= KERNEL_BASE); 1318 1319 #else 1320 KASSERTMSG(kernel_map != NULL, "pm %p", pm); 1321 vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1322 UVM_KMF_WIRED|UVM_KMF_ZERO); 1323 KASSERT(va); 1324 pmap_extract(pmap_kernel(), va, &pm->pm_l1_pa); 1325 #endif 1326 pm->pm_l1 = (pd_entry_t *)va; 1327 PTE_SYNC_RANGE(pm->pm_l1, PAGE_SIZE / sizeof(pt_entry_t)); 1328 #else 1329 struct l1_ttable *l1; 1330 uint8_t domain; 1331 1332 /* 1333 * Remove the L1 at the head of the LRU list 1334 */ 1335 mutex_spin_enter(&l1_lru_lock); 1336 l1 = TAILQ_FIRST(&l1_lru_list); 1337 KDASSERT(l1 != NULL); 1338 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1339 1340 /* 1341 * Pick the first available domain number, and update 1342 * the link to the next number. 1343 */ 1344 domain = l1->l1_domain_first; 1345 l1->l1_domain_first = l1->l1_domain_free[domain]; 1346 1347 /* 1348 * If there are still free domain numbers in this L1, 1349 * put it back on the TAIL of the LRU list. 1350 */ 1351 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1352 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1353 1354 mutex_spin_exit(&l1_lru_lock); 1355 1356 /* 1357 * Fix up the relevant bits in the pmap structure 1358 */ 1359 pm->pm_l1 = l1; 1360 pm->pm_domain = domain + 1; 1361 #endif 1362 } 1363 1364 /* 1365 * Free an L1 translation table. 1366 * This is called at pmap destruction time. 1367 */ 1368 static void 1369 pmap_free_l1(pmap_t pm) 1370 { 1371 #ifdef ARM_MMU_EXTENDED 1372 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 1373 struct vm_page *pg = PHYS_TO_VM_PAGE(pm->pm_l1_pa); 1374 uvm_pagefree(pg); 1375 #else 1376 uvm_km_free(kernel_map, (vaddr_t)pm->pm_l1, PAGE_SIZE, UVM_KMF_WIRED); 1377 #endif 1378 pm->pm_l1 = NULL; 1379 pm->pm_l1_pa = 0; 1380 #else 1381 struct l1_ttable *l1 = pm->pm_l1; 1382 1383 mutex_spin_enter(&l1_lru_lock); 1384 1385 /* 1386 * If this L1 is currently on the LRU list, remove it. 1387 */ 1388 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1389 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1390 1391 /* 1392 * Free up the domain number which was allocated to the pmap 1393 */ 1394 l1->l1_domain_free[pmap_domain(pm) - 1] = l1->l1_domain_first; 1395 l1->l1_domain_first = pmap_domain(pm) - 1; 1396 l1->l1_domain_use_count--; 1397 1398 /* 1399 * The L1 now must have at least 1 free domain, so add 1400 * it back to the LRU list. If the use count is zero, 1401 * put it at the head of the list, otherwise it goes 1402 * to the tail. 1403 */ 1404 if (l1->l1_domain_use_count == 0) 1405 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1406 else 1407 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1408 1409 mutex_spin_exit(&l1_lru_lock); 1410 #endif /* ARM_MMU_EXTENDED */ 1411 } 1412 1413 #ifndef ARM_MMU_EXTENDED 1414 static inline void 1415 pmap_use_l1(pmap_t pm) 1416 { 1417 struct l1_ttable *l1; 1418 1419 /* 1420 * Do nothing if we're in interrupt context. 1421 * Access to an L1 by the kernel pmap must not affect 1422 * the LRU list. 1423 */ 1424 if (cpu_intr_p() || pm == pmap_kernel()) 1425 return; 1426 1427 l1 = pm->pm_l1; 1428 1429 /* 1430 * If the L1 is not currently on the LRU list, just return 1431 */ 1432 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1433 return; 1434 1435 mutex_spin_enter(&l1_lru_lock); 1436 1437 /* 1438 * Check the use count again, now that we've acquired the lock 1439 */ 1440 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1441 mutex_spin_exit(&l1_lru_lock); 1442 return; 1443 } 1444 1445 /* 1446 * Move the L1 to the back of the LRU list 1447 */ 1448 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1449 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1450 1451 mutex_spin_exit(&l1_lru_lock); 1452 } 1453 #endif /* !ARM_MMU_EXTENDED */ 1454 1455 /* 1456 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1457 * 1458 * Free an L2 descriptor table. 1459 */ 1460 static inline void 1461 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1462 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1463 #else 1464 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1465 #endif 1466 { 1467 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1468 /* 1469 * Note: With a write-back cache, we may need to sync this 1470 * L2 table before re-using it. 1471 * This is because it may have belonged to a non-current 1472 * pmap, in which case the cache syncs would have been 1473 * skipped for the pages that were being unmapped. If the 1474 * L2 table were then to be immediately re-allocated to 1475 * the *current* pmap, it may well contain stale mappings 1476 * which have not yet been cleared by a cache write-back 1477 * and so would still be visible to the mmu. 1478 */ 1479 if (need_sync) 1480 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1481 #endif /* PMAP_INCLUDE_PTE_SYNC && PMAP_CACHE_VIVT */ 1482 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1483 } 1484 1485 /* 1486 * Returns a pointer to the L2 bucket associated with the specified pmap 1487 * and VA, or NULL if no L2 bucket exists for the address. 1488 */ 1489 static inline struct l2_bucket * 1490 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1491 { 1492 const size_t l1slot = l1pte_index(va); 1493 struct l2_dtable *l2; 1494 struct l2_bucket *l2b; 1495 1496 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL || 1497 (l2b = &l2->l2_bucket[L2_BUCKET(l1slot)])->l2b_kva == NULL) 1498 return (NULL); 1499 1500 return (l2b); 1501 } 1502 1503 /* 1504 * Returns a pointer to the L2 bucket associated with the specified pmap 1505 * and VA. 1506 * 1507 * If no L2 bucket exists, perform the necessary allocations to put an L2 1508 * bucket/page table in place. 1509 * 1510 * Note that if a new L2 bucket/page was allocated, the caller *must* 1511 * increment the bucket occupancy counter appropriately *before* 1512 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1513 * the bucket/page in the meantime. 1514 */ 1515 static struct l2_bucket * 1516 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1517 { 1518 const size_t l1slot = l1pte_index(va); 1519 struct l2_dtable *l2; 1520 1521 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 1522 /* 1523 * No mapping at this address, as there is 1524 * no entry in the L1 table. 1525 * Need to allocate a new l2_dtable. 1526 */ 1527 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1528 return (NULL); 1529 1530 /* 1531 * Link it into the parent pmap 1532 */ 1533 pm->pm_l2[L2_IDX(l1slot)] = l2; 1534 } 1535 1536 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 1537 1538 /* 1539 * Fetch pointer to the L2 page table associated with the address. 1540 */ 1541 if (l2b->l2b_kva == NULL) { 1542 pt_entry_t *ptep; 1543 1544 /* 1545 * No L2 page table has been allocated. Chances are, this 1546 * is because we just allocated the l2_dtable, above. 1547 */ 1548 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_pa)) == NULL) { 1549 /* 1550 * Oops, no more L2 page tables available at this 1551 * time. We may need to deallocate the l2_dtable 1552 * if we allocated a new one above. 1553 */ 1554 if (l2->l2_occupancy == 0) { 1555 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1556 pmap_free_l2_dtable(l2); 1557 } 1558 return (NULL); 1559 } 1560 1561 l2->l2_occupancy++; 1562 l2b->l2b_kva = ptep; 1563 l2b->l2b_l1slot = l1slot; 1564 1565 #ifdef ARM_MMU_EXTENDED 1566 /* 1567 * We know there will be a mapping here, so simply 1568 * enter this PTP into the L1 now. 1569 */ 1570 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1571 pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 1572 | L1_C_DOM(pmap_domain(pm)); 1573 KASSERT(*pdep == 0); 1574 l1pte_setone(pdep, npde); 1575 PDE_SYNC(pdep); 1576 #endif 1577 } 1578 1579 return (l2b); 1580 } 1581 1582 /* 1583 * One or more mappings in the specified L2 descriptor table have just been 1584 * invalidated. 1585 * 1586 * Garbage collect the metadata and descriptor table itself if necessary. 1587 * 1588 * The pmap lock must be acquired when this is called (not necessary 1589 * for the kernel pmap). 1590 */ 1591 static void 1592 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1593 { 1594 KDASSERT(count <= l2b->l2b_occupancy); 1595 1596 /* 1597 * Update the bucket's reference count according to how many 1598 * PTEs the caller has just invalidated. 1599 */ 1600 l2b->l2b_occupancy -= count; 1601 1602 /* 1603 * Note: 1604 * 1605 * Level 2 page tables allocated to the kernel pmap are never freed 1606 * as that would require checking all Level 1 page tables and 1607 * removing any references to the Level 2 page table. See also the 1608 * comment elsewhere about never freeing bootstrap L2 descriptors. 1609 * 1610 * We make do with just invalidating the mapping in the L2 table. 1611 * 1612 * This isn't really a big deal in practice and, in fact, leads 1613 * to a performance win over time as we don't need to continually 1614 * alloc/free. 1615 */ 1616 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1617 return; 1618 1619 /* 1620 * There are no more valid mappings in this level 2 page table. 1621 * Go ahead and NULL-out the pointer in the bucket, then 1622 * free the page table. 1623 */ 1624 const size_t l1slot = l2b->l2b_l1slot; 1625 pt_entry_t * const ptep = l2b->l2b_kva; 1626 l2b->l2b_kva = NULL; 1627 1628 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1629 pd_entry_t pde __diagused = *pdep; 1630 1631 #ifdef ARM_MMU_EXTENDED 1632 /* 1633 * Invalidate the L1 slot. 1634 */ 1635 KASSERT((pde & L1_TYPE_MASK) == L1_TYPE_C); 1636 #else 1637 /* 1638 * If the L1 slot matches the pmap's domain number, then invalidate it. 1639 */ 1640 if ((pde & (L1_C_DOM_MASK|L1_TYPE_MASK)) 1641 == (L1_C_DOM(pmap_domain(pm))|L1_TYPE_C)) { 1642 #endif 1643 l1pte_setone(pdep, 0); 1644 PDE_SYNC(pdep); 1645 #ifndef ARM_MMU_EXTENDED 1646 } 1647 #endif 1648 1649 /* 1650 * Release the L2 descriptor table back to the pool cache. 1651 */ 1652 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1653 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_pa); 1654 #else 1655 pmap_free_l2_ptp(ptep, l2b->l2b_pa); 1656 #endif 1657 1658 /* 1659 * Update the reference count in the associated l2_dtable 1660 */ 1661 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 1662 if (--l2->l2_occupancy > 0) 1663 return; 1664 1665 /* 1666 * There are no more valid mappings in any of the Level 1 1667 * slots managed by this l2_dtable. Go ahead and NULL-out 1668 * the pointer in the parent pmap and free the l2_dtable. 1669 */ 1670 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1671 pmap_free_l2_dtable(l2); 1672 } 1673 1674 /* 1675 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1676 * structures. 1677 */ 1678 static int 1679 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1680 { 1681 #ifndef PMAP_INCLUDE_PTE_SYNC 1682 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1683 1684 /* 1685 * The mappings for these page tables were initially made using 1686 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1687 * mode will not be right for page table mappings. To avoid 1688 * polluting the pmap_kenter_pa() code with a special case for 1689 * page tables, we simply fix up the cache-mode here if it's not 1690 * correct. 1691 */ 1692 if (pte_l2_s_cache_mode != pte_l2_s_cache_mode_pt) { 1693 const struct l2_bucket * const l2b = 1694 pmap_get_l2_bucket(pmap_kernel(), va); 1695 KASSERTMSG(l2b != NULL, "%#lx", va); 1696 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 1697 const pt_entry_t opte = *ptep; 1698 1699 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1700 /* 1701 * Page tables must have the cache-mode set correctly. 1702 */ 1703 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1704 | pte_l2_s_cache_mode_pt; 1705 l2pte_set(ptep, npte, opte); 1706 PTE_SYNC(ptep); 1707 cpu_tlb_flushD_SE(va); 1708 cpu_cpwait(); 1709 } 1710 } 1711 #endif 1712 1713 memset(v, 0, L2_TABLE_SIZE_REAL); 1714 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1715 return (0); 1716 } 1717 1718 static int 1719 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1720 { 1721 1722 memset(v, 0, sizeof(struct l2_dtable)); 1723 return (0); 1724 } 1725 1726 static int 1727 pmap_pmap_ctor(void *arg, void *v, int flags) 1728 { 1729 1730 memset(v, 0, sizeof(struct pmap)); 1731 return (0); 1732 } 1733 1734 static void 1735 pmap_pinit(pmap_t pm) 1736 { 1737 #ifndef ARM_HAS_VBAR 1738 struct l2_bucket *l2b; 1739 1740 if (vector_page < KERNEL_BASE) { 1741 /* 1742 * Map the vector page. 1743 */ 1744 pmap_enter(pm, vector_page, systempage.pv_pa, 1745 VM_PROT_READ | VM_PROT_EXECUTE, 1746 VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED); 1747 pmap_update(pm); 1748 1749 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 1750 l2b = pmap_get_l2_bucket(pm, vector_page); 1751 KASSERTMSG(l2b != NULL, "%#lx", vector_page); 1752 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 1753 L1_C_DOM(pmap_domain(pm)); 1754 } else 1755 pm->pm_pl1vec = NULL; 1756 #endif 1757 } 1758 1759 #ifdef PMAP_CACHE_VIVT 1760 /* 1761 * Since we have a virtually indexed cache, we may need to inhibit caching if 1762 * there is more than one mapping and at least one of them is writable. 1763 * Since we purge the cache on every context switch, we only need to check for 1764 * other mappings within the same pmap, or kernel_pmap. 1765 * This function is also called when a page is unmapped, to possibly reenable 1766 * caching on any remaining mappings. 1767 * 1768 * The code implements the following logic, where: 1769 * 1770 * KW = # of kernel read/write pages 1771 * KR = # of kernel read only pages 1772 * UW = # of user read/write pages 1773 * UR = # of user read only pages 1774 * 1775 * KC = kernel mapping is cacheable 1776 * UC = user mapping is cacheable 1777 * 1778 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1779 * +--------------------------------------------- 1780 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1781 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1782 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1783 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1784 */ 1785 1786 static const int pmap_vac_flags[4][4] = { 1787 {-1, 0, 0, PVF_KNC}, 1788 {0, 0, PVF_NC, PVF_NC}, 1789 {0, PVF_NC, PVF_NC, PVF_NC}, 1790 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1791 }; 1792 1793 static inline int 1794 pmap_get_vac_flags(const struct vm_page_md *md) 1795 { 1796 int kidx, uidx; 1797 1798 kidx = 0; 1799 if (md->kro_mappings || md->krw_mappings > 1) 1800 kidx |= 1; 1801 if (md->krw_mappings) 1802 kidx |= 2; 1803 1804 uidx = 0; 1805 if (md->uro_mappings || md->urw_mappings > 1) 1806 uidx |= 1; 1807 if (md->urw_mappings) 1808 uidx |= 2; 1809 1810 return (pmap_vac_flags[uidx][kidx]); 1811 } 1812 1813 static inline void 1814 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1815 { 1816 int nattr; 1817 1818 nattr = pmap_get_vac_flags(md); 1819 1820 if (nattr < 0) { 1821 md->pvh_attrs &= ~PVF_NC; 1822 return; 1823 } 1824 1825 if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0) 1826 return; 1827 1828 if (pm == pmap_kernel()) 1829 pmap_vac_me_kpmap(md, pa, pm, va); 1830 else 1831 pmap_vac_me_user(md, pa, pm, va); 1832 1833 md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr; 1834 } 1835 1836 static void 1837 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1838 { 1839 u_int u_cacheable, u_entries; 1840 struct pv_entry *pv; 1841 pmap_t last_pmap = pm; 1842 1843 /* 1844 * Pass one, see if there are both kernel and user pmaps for 1845 * this page. Calculate whether there are user-writable or 1846 * kernel-writable pages. 1847 */ 1848 u_cacheable = 0; 1849 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1850 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1851 u_cacheable++; 1852 } 1853 1854 u_entries = md->urw_mappings + md->uro_mappings; 1855 1856 /* 1857 * We know we have just been updating a kernel entry, so if 1858 * all user pages are already cacheable, then there is nothing 1859 * further to do. 1860 */ 1861 if (md->k_mappings == 0 && u_cacheable == u_entries) 1862 return; 1863 1864 if (u_entries) { 1865 /* 1866 * Scan over the list again, for each entry, if it 1867 * might not be set correctly, call pmap_vac_me_user 1868 * to recalculate the settings. 1869 */ 1870 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1871 /* 1872 * We know kernel mappings will get set 1873 * correctly in other calls. We also know 1874 * that if the pmap is the same as last_pmap 1875 * then we've just handled this entry. 1876 */ 1877 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1878 continue; 1879 1880 /* 1881 * If there are kernel entries and this page 1882 * is writable but non-cacheable, then we can 1883 * skip this entry also. 1884 */ 1885 if (md->k_mappings && 1886 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1887 (PVF_NC | PVF_WRITE)) 1888 continue; 1889 1890 /* 1891 * Similarly if there are no kernel-writable 1892 * entries and the page is already 1893 * read-only/cacheable. 1894 */ 1895 if (md->krw_mappings == 0 && 1896 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1897 continue; 1898 1899 /* 1900 * For some of the remaining cases, we know 1901 * that we must recalculate, but for others we 1902 * can't tell if they are correct or not, so 1903 * we recalculate anyway. 1904 */ 1905 pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0); 1906 } 1907 1908 if (md->k_mappings == 0) 1909 return; 1910 } 1911 1912 pmap_vac_me_user(md, pa, pm, va); 1913 } 1914 1915 static void 1916 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1917 { 1918 pmap_t kpmap = pmap_kernel(); 1919 struct pv_entry *pv, *npv = NULL; 1920 u_int entries = 0; 1921 u_int writable = 0; 1922 u_int cacheable_entries = 0; 1923 u_int kern_cacheable = 0; 1924 u_int other_writable = 0; 1925 1926 /* 1927 * Count mappings and writable mappings in this pmap. 1928 * Include kernel mappings as part of our own. 1929 * Keep a pointer to the first one. 1930 */ 1931 npv = NULL; 1932 KASSERT(pmap_page_locked_p(md)); 1933 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1934 /* Count mappings in the same pmap */ 1935 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1936 if (entries++ == 0) 1937 npv = pv; 1938 1939 /* Cacheable mappings */ 1940 if ((pv->pv_flags & PVF_NC) == 0) { 1941 cacheable_entries++; 1942 if (kpmap == pv->pv_pmap) 1943 kern_cacheable++; 1944 } 1945 1946 /* Writable mappings */ 1947 if (pv->pv_flags & PVF_WRITE) 1948 ++writable; 1949 } else if (pv->pv_flags & PVF_WRITE) 1950 other_writable = 1; 1951 } 1952 1953 /* 1954 * Enable or disable caching as necessary. 1955 * Note: the first entry might be part of the kernel pmap, 1956 * so we can't assume this is indicative of the state of the 1957 * other (maybe non-kpmap) entries. 1958 */ 1959 if ((entries > 1 && writable) || 1960 (entries > 0 && pm == kpmap && other_writable)) { 1961 if (cacheable_entries == 0) { 1962 return; 1963 } 1964 1965 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1966 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1967 (pv->pv_flags & PVF_NC)) 1968 continue; 1969 1970 pv->pv_flags |= PVF_NC; 1971 1972 struct l2_bucket * const l2b 1973 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1974 KASSERTMSG(l2b != NULL, "%#lx", va); 1975 pt_entry_t * const ptep 1976 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1977 const pt_entry_t opte = *ptep; 1978 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 1979 1980 if ((va != pv->pv_va || pm != pv->pv_pmap) 1981 && l2pte_valid_p(opte)) { 1982 pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va, 1983 true, pv->pv_flags); 1984 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1985 pv->pv_flags); 1986 } 1987 1988 l2pte_set(ptep, npte, opte); 1989 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1990 } 1991 cpu_cpwait(); 1992 } else if (entries > cacheable_entries) { 1993 /* 1994 * Turn cacheing back on for some pages. If it is a kernel 1995 * page, only do so if there are no other writable pages. 1996 */ 1997 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1998 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1999 (kpmap != pv->pv_pmap || other_writable))) 2000 continue; 2001 2002 pv->pv_flags &= ~PVF_NC; 2003 2004 struct l2_bucket * const l2b 2005 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2006 KASSERTMSG(l2b != NULL, "%#lx", va); 2007 pt_entry_t * const ptep 2008 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2009 const pt_entry_t opte = *ptep; 2010 pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 2011 | pte_l2_s_cache_mode; 2012 2013 if (l2pte_valid_p(opte)) { 2014 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 2015 pv->pv_flags); 2016 } 2017 2018 l2pte_set(ptep, npte, opte); 2019 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2020 } 2021 } 2022 } 2023 #endif 2024 2025 #ifdef PMAP_CACHE_VIPT 2026 static void 2027 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 2028 { 2029 #ifndef ARM_MMU_EXTENDED 2030 struct pv_entry *pv; 2031 vaddr_t tst_mask; 2032 bool bad_alias; 2033 const u_int 2034 rw_mappings = md->urw_mappings + md->krw_mappings, 2035 ro_mappings = md->uro_mappings + md->kro_mappings; 2036 2037 /* do we need to do anything? */ 2038 if (arm_cache_prefer_mask == 0) 2039 return; 2040 2041 NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: md=%p, pmap=%p va=%08lx\n", 2042 md, pm, va)); 2043 2044 KASSERT(!va || pm); 2045 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2046 2047 /* Already a conflict? */ 2048 if (__predict_false(md->pvh_attrs & PVF_NC)) { 2049 /* just an add, things are already non-cached */ 2050 KASSERT(!(md->pvh_attrs & PVF_DIRTY)); 2051 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2052 bad_alias = false; 2053 if (va) { 2054 PMAPCOUNT(vac_color_none); 2055 bad_alias = true; 2056 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2057 goto fixup; 2058 } 2059 pv = SLIST_FIRST(&md->pvh_list); 2060 /* the list can't be empty because it would be cachable */ 2061 if (md->pvh_attrs & PVF_KMPAGE) { 2062 tst_mask = md->pvh_attrs; 2063 } else { 2064 KASSERT(pv); 2065 tst_mask = pv->pv_va; 2066 pv = SLIST_NEXT(pv, pv_link); 2067 } 2068 /* 2069 * Only check for a bad alias if we have writable mappings. 2070 */ 2071 tst_mask &= arm_cache_prefer_mask; 2072 if (rw_mappings > 0) { 2073 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 2074 /* if there's a bad alias, stop checking. */ 2075 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 2076 bad_alias = true; 2077 } 2078 md->pvh_attrs |= PVF_WRITE; 2079 if (!bad_alias) 2080 md->pvh_attrs |= PVF_DIRTY; 2081 } else { 2082 /* 2083 * We have only read-only mappings. Let's see if there 2084 * are multiple colors in use or if we mapped a KMPAGE. 2085 * If the latter, we have a bad alias. If the former, 2086 * we need to remember that. 2087 */ 2088 for (; pv; pv = SLIST_NEXT(pv, pv_link)) { 2089 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) { 2090 if (md->pvh_attrs & PVF_KMPAGE) 2091 bad_alias = true; 2092 break; 2093 } 2094 } 2095 md->pvh_attrs &= ~PVF_WRITE; 2096 /* 2097 * No KMPAGE and we exited early, so we must have 2098 * multiple color mappings. 2099 */ 2100 if (!bad_alias && pv != NULL) 2101 md->pvh_attrs |= PVF_MULTCLR; 2102 } 2103 2104 /* If no conflicting colors, set everything back to cached */ 2105 if (!bad_alias) { 2106 #ifdef DEBUG 2107 if ((md->pvh_attrs & PVF_WRITE) 2108 || ro_mappings < 2) { 2109 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2110 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2111 } 2112 #endif 2113 md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 2114 md->pvh_attrs |= tst_mask | PVF_COLORED; 2115 /* 2116 * Restore DIRTY bit if page is modified 2117 */ 2118 if (md->pvh_attrs & PVF_DMOD) 2119 md->pvh_attrs |= PVF_DIRTY; 2120 PMAPCOUNT(vac_color_restore); 2121 } else { 2122 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2123 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2124 } 2125 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2126 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2127 } else if (!va) { 2128 KASSERT(pmap_is_page_colored_p(md)); 2129 KASSERT(!(md->pvh_attrs & PVF_WRITE) 2130 || (md->pvh_attrs & PVF_DIRTY)); 2131 if (rw_mappings == 0) { 2132 md->pvh_attrs &= ~PVF_WRITE; 2133 if (ro_mappings == 1 2134 && (md->pvh_attrs & PVF_MULTCLR)) { 2135 /* 2136 * If this is the last readonly mapping 2137 * but it doesn't match the current color 2138 * for the page, change the current color 2139 * to match this last readonly mapping. 2140 */ 2141 pv = SLIST_FIRST(&md->pvh_list); 2142 tst_mask = (md->pvh_attrs ^ pv->pv_va) 2143 & arm_cache_prefer_mask; 2144 if (tst_mask) { 2145 md->pvh_attrs ^= tst_mask; 2146 PMAPCOUNT(vac_color_change); 2147 } 2148 } 2149 } 2150 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2151 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2152 return; 2153 } else if (!pmap_is_page_colored_p(md)) { 2154 /* not colored so we just use its color */ 2155 KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 2156 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2157 PMAPCOUNT(vac_color_new); 2158 md->pvh_attrs &= PAGE_SIZE - 1; 2159 md->pvh_attrs |= PVF_COLORED 2160 | (va & arm_cache_prefer_mask) 2161 | (rw_mappings > 0 ? PVF_WRITE : 0); 2162 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2163 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2164 return; 2165 } else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 2166 bad_alias = false; 2167 if (rw_mappings > 0) { 2168 /* 2169 * We now have writeable mappings and if we have 2170 * readonly mappings in more than once color, we have 2171 * an aliasing problem. Regardless mark the page as 2172 * writeable. 2173 */ 2174 if (md->pvh_attrs & PVF_MULTCLR) { 2175 if (ro_mappings < 2) { 2176 /* 2177 * If we only have less than two 2178 * read-only mappings, just flush the 2179 * non-primary colors from the cache. 2180 */ 2181 pmap_flush_page(md, pa, 2182 PMAP_FLUSH_SECONDARY); 2183 } else { 2184 bad_alias = true; 2185 } 2186 } 2187 md->pvh_attrs |= PVF_WRITE; 2188 } 2189 /* If no conflicting colors, set everything back to cached */ 2190 if (!bad_alias) { 2191 #ifdef DEBUG 2192 if (rw_mappings > 0 2193 || (md->pvh_attrs & PMAP_KMPAGE)) { 2194 tst_mask = md->pvh_attrs & arm_cache_prefer_mask; 2195 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2196 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2197 } 2198 #endif 2199 if (SLIST_EMPTY(&md->pvh_list)) 2200 PMAPCOUNT(vac_color_reuse); 2201 else 2202 PMAPCOUNT(vac_color_ok); 2203 2204 /* matching color, just return */ 2205 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2206 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2207 return; 2208 } 2209 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2210 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2211 2212 /* color conflict. evict from cache. */ 2213 2214 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2215 md->pvh_attrs &= ~PVF_COLORED; 2216 md->pvh_attrs |= PVF_NC; 2217 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2218 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2219 PMAPCOUNT(vac_color_erase); 2220 } else if (rw_mappings == 0 2221 && (md->pvh_attrs & PVF_KMPAGE) == 0) { 2222 KASSERT((md->pvh_attrs & PVF_WRITE) == 0); 2223 2224 /* 2225 * If the page has dirty cache lines, clean it. 2226 */ 2227 if (md->pvh_attrs & PVF_DIRTY) 2228 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 2229 2230 /* 2231 * If this is the first remapping (we know that there are no 2232 * writeable mappings), then this is a simple color change. 2233 * Otherwise this is a seconary r/o mapping, which means 2234 * we don't have to do anything. 2235 */ 2236 if (ro_mappings == 1) { 2237 KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2238 md->pvh_attrs &= PAGE_SIZE - 1; 2239 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2240 PMAPCOUNT(vac_color_change); 2241 } else { 2242 PMAPCOUNT(vac_color_blind); 2243 } 2244 md->pvh_attrs |= PVF_MULTCLR; 2245 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2246 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2247 return; 2248 } else { 2249 if (rw_mappings > 0) 2250 md->pvh_attrs |= PVF_WRITE; 2251 2252 /* color conflict. evict from cache. */ 2253 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2254 2255 /* the list can't be empty because this was a enter/modify */ 2256 pv = SLIST_FIRST(&md->pvh_list); 2257 if ((md->pvh_attrs & PVF_KMPAGE) == 0) { 2258 KASSERT(pv); 2259 /* 2260 * If there's only one mapped page, change color to the 2261 * page's new color and return. Restore the DIRTY bit 2262 * that was erased by pmap_flush_page. 2263 */ 2264 if (SLIST_NEXT(pv, pv_link) == NULL) { 2265 md->pvh_attrs &= PAGE_SIZE - 1; 2266 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2267 if (md->pvh_attrs & PVF_DMOD) 2268 md->pvh_attrs |= PVF_DIRTY; 2269 PMAPCOUNT(vac_color_change); 2270 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2271 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2272 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2273 return; 2274 } 2275 } 2276 bad_alias = true; 2277 md->pvh_attrs &= ~PVF_COLORED; 2278 md->pvh_attrs |= PVF_NC; 2279 PMAPCOUNT(vac_color_erase); 2280 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2281 } 2282 2283 fixup: 2284 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2285 2286 /* 2287 * Turn cacheing on/off for all pages. 2288 */ 2289 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2290 struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap, 2291 pv->pv_va); 2292 KASSERTMSG(l2b != NULL, "%#lx", va); 2293 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2294 const pt_entry_t opte = *ptep; 2295 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 2296 if (bad_alias) { 2297 pv->pv_flags |= PVF_NC; 2298 } else { 2299 pv->pv_flags &= ~PVF_NC; 2300 npte |= pte_l2_s_cache_mode; 2301 } 2302 2303 if (opte == npte) /* only update is there's a change */ 2304 continue; 2305 2306 if (l2pte_valid_p(opte)) { 2307 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags); 2308 } 2309 2310 l2pte_set(ptep, npte, opte); 2311 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2312 } 2313 #endif /* !ARM_MMU_EXTENDED */ 2314 } 2315 #endif /* PMAP_CACHE_VIPT */ 2316 2317 2318 /* 2319 * Modify pte bits for all ptes corresponding to the given physical address. 2320 * We use `maskbits' rather than `clearbits' because we're always passing 2321 * constants and the latter would require an extra inversion at run-time. 2322 */ 2323 static void 2324 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits) 2325 { 2326 struct pv_entry *pv; 2327 #ifdef PMAP_CACHE_VIPT 2328 const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs); 2329 bool need_syncicache = false; 2330 #ifdef ARM_MMU_EXTENDED 2331 const u_int execbits = (maskbits & PVF_EXEC) ? L2_XS_XN : 0; 2332 #else 2333 const u_int execbits = 0; 2334 bool need_vac_me_harder = false; 2335 #endif 2336 #else 2337 const u_int execbits = 0; 2338 #endif 2339 2340 NPDEBUG(PDB_BITS, 2341 printf("pmap_clearbit: md %p mask 0x%x\n", 2342 md, maskbits)); 2343 2344 #ifdef PMAP_CACHE_VIPT 2345 /* 2346 * If we might want to sync the I-cache and we've modified it, 2347 * then we know we definitely need to sync or discard it. 2348 */ 2349 if (want_syncicache) { 2350 if (md->pvh_attrs & PVF_MOD) { 2351 need_syncicache = true; 2352 } 2353 } 2354 #endif 2355 KASSERT(pmap_page_locked_p(md)); 2356 2357 /* 2358 * Clear saved attributes (modify, reference) 2359 */ 2360 md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2361 2362 if (SLIST_EMPTY(&md->pvh_list)) { 2363 #if defined(PMAP_CACHE_VIPT) 2364 if (need_syncicache) { 2365 /* 2366 * No one has it mapped, so just discard it. The next 2367 * exec remapping will cause it to be synced. 2368 */ 2369 md->pvh_attrs &= ~PVF_EXEC; 2370 PMAPCOUNT(exec_discarded_clearbit); 2371 } 2372 #endif 2373 return; 2374 } 2375 2376 /* 2377 * Loop over all current mappings setting/clearing as appropos 2378 */ 2379 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2380 pmap_t pm = pv->pv_pmap; 2381 const vaddr_t va = pv->pv_va; 2382 const u_int oflags = pv->pv_flags; 2383 #ifndef ARM_MMU_EXTENDED 2384 /* 2385 * Kernel entries are unmanaged and as such not to be changed. 2386 */ 2387 if (PV_IS_KENTRY_P(oflags)) 2388 continue; 2389 #endif 2390 pv->pv_flags &= ~maskbits; 2391 2392 pmap_release_page_lock(md); 2393 pmap_acquire_pmap_lock(pm); 2394 2395 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va); 2396 if (l2b == NULL) { 2397 pmap_release_pmap_lock(pm); 2398 pmap_acquire_page_lock(md); 2399 continue; 2400 } 2401 KASSERTMSG(l2b != NULL, "%#lx", va); 2402 2403 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 2404 const pt_entry_t opte = *ptep; 2405 pt_entry_t npte = opte | execbits; 2406 2407 #ifdef ARM_MMU_EXTENDED 2408 KASSERT((opte & L2_XS_nG) == (pm == pmap_kernel() ? 0 : L2_XS_nG)); 2409 #endif 2410 2411 NPDEBUG(PDB_BITS, 2412 printf( "%s: pv %p, pm %p, va 0x%08lx, flag 0x%x\n", 2413 __func__, pv, pm, va, oflags)); 2414 2415 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2416 #ifdef PMAP_CACHE_VIVT 2417 if ((oflags & PVF_NC)) { 2418 /* 2419 * Entry is not cacheable: 2420 * 2421 * Don't turn caching on again if this is a 2422 * modified emulation. This would be 2423 * inconsitent with the settings created by 2424 * pmap_vac_me_harder(). Otherwise, it's safe 2425 * to re-enable cacheing. 2426 * 2427 * There's no need to call pmap_vac_me_harder() 2428 * here: all pages are losing their write 2429 * permission. 2430 */ 2431 if (maskbits & PVF_WRITE) { 2432 npte |= pte_l2_s_cache_mode; 2433 pv->pv_flags &= ~PVF_NC; 2434 } 2435 } else if (l2pte_writable_p(opte)) { 2436 /* 2437 * Entry is writable/cacheable: check if pmap 2438 * is current if it is flush it, otherwise it 2439 * won't be in the cache 2440 */ 2441 pmap_cache_wbinv_page(pm, va, 2442 (maskbits & PVF_REF) != 0, 2443 oflags|PVF_WRITE); 2444 } 2445 #endif 2446 2447 /* make the pte read only */ 2448 npte = l2pte_set_readonly(npte); 2449 2450 pmap_acquire_page_lock(md); 2451 #ifdef MULTIPROCESSOR 2452 pv = pmap_find_pv(md, pm, va); 2453 #endif 2454 if (pv != NULL && (maskbits & oflags & PVF_WRITE)) { 2455 /* 2456 * Keep alias accounting up to date 2457 */ 2458 if (pm == pmap_kernel()) { 2459 md->krw_mappings--; 2460 md->kro_mappings++; 2461 } else { 2462 md->urw_mappings--; 2463 md->uro_mappings++; 2464 } 2465 #ifdef PMAP_CACHE_VIPT 2466 if (arm_cache_prefer_mask != 0) { 2467 if (md->urw_mappings + md->krw_mappings == 0) { 2468 md->pvh_attrs &= ~PVF_WRITE; 2469 } else { 2470 PMAP_VALIDATE_MD_PAGE(md); 2471 } 2472 } 2473 if (want_syncicache) 2474 need_syncicache = true; 2475 #ifndef ARM_MMU_EXTENDED 2476 need_vac_me_harder = true; 2477 #endif 2478 #endif /* PMAP_CACHE_VIPT */ 2479 } 2480 pmap_release_page_lock(md); 2481 } 2482 2483 if (maskbits & PVF_REF) { 2484 if (true 2485 #ifndef ARM_MMU_EXTENDED 2486 && (oflags & PVF_NC) == 0 2487 #endif 2488 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0 2489 && l2pte_valid_p(npte)) { 2490 #ifdef PMAP_CACHE_VIVT 2491 /* 2492 * Check npte here; we may have already 2493 * done the wbinv above, and the validity 2494 * of the PTE is the same for opte and 2495 * npte. 2496 */ 2497 pmap_cache_wbinv_page(pm, va, true, oflags); 2498 #endif 2499 } 2500 2501 /* 2502 * Make the PTE invalid so that we will take a 2503 * page fault the next time the mapping is 2504 * referenced. 2505 */ 2506 npte &= ~L2_TYPE_MASK; 2507 npte |= L2_TYPE_INV; 2508 } 2509 2510 if (npte != opte) { 2511 l2pte_reset(ptep); 2512 PTE_SYNC(ptep); 2513 2514 /* Flush the TLB entry if a current pmap. */ 2515 pmap_tlb_flush_SE(pm, va, oflags); 2516 2517 l2pte_set(ptep, npte, 0); 2518 PTE_SYNC(ptep); 2519 } 2520 2521 pmap_release_pmap_lock(pm); 2522 pmap_acquire_page_lock(md); 2523 2524 NPDEBUG(PDB_BITS, 2525 printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n", 2526 pm, va, opte, npte)); 2527 } 2528 2529 #if defined(PMAP_CACHE_VIPT) 2530 /* 2531 * If we need to sync the I-cache and we haven't done it yet, do it. 2532 */ 2533 if (need_syncicache) { 2534 pmap_release_page_lock(md); 2535 pmap_syncicache_page(md, pa); 2536 pmap_acquire_page_lock(md); 2537 PMAPCOUNT(exec_synced_clearbit); 2538 } 2539 #ifndef ARM_MMU_EXTENDED 2540 /* 2541 * If we are changing this to read-only, we need to call vac_me_harder 2542 * so we can change all the read-only pages to cacheable. We pretend 2543 * this as a page deletion. 2544 */ 2545 if (need_vac_me_harder) { 2546 if (md->pvh_attrs & PVF_NC) 2547 pmap_vac_me_harder(md, pa, NULL, 0); 2548 } 2549 #endif /* !ARM_MMU_EXTENDED */ 2550 #endif /* PMAP_CACHE_VIPT */ 2551 } 2552 2553 /* 2554 * pmap_clean_page() 2555 * 2556 * This is a local function used to work out the best strategy to clean 2557 * a single page referenced by its entry in the PV table. It's used by 2558 * pmap_copy_page, pmap_zero_page and maybe some others later on. 2559 * 2560 * Its policy is effectively: 2561 * o If there are no mappings, we don't bother doing anything with the cache. 2562 * o If there is one mapping, we clean just that page. 2563 * o If there are multiple mappings, we clean the entire cache. 2564 * 2565 * So that some functions can be further optimised, it returns 0 if it didn't 2566 * clean the entire cache, or 1 if it did. 2567 * 2568 * XXX One bug in this routine is that if the pv_entry has a single page 2569 * mapped at 0x00000000 a whole cache clean will be performed rather than 2570 * just the 1 page. Since this should not occur in everyday use and if it does 2571 * it will just result in not the most efficient clean for the page. 2572 */ 2573 #ifdef PMAP_CACHE_VIVT 2574 static bool 2575 pmap_clean_page(struct vm_page_md *md, bool is_src) 2576 { 2577 struct pv_entry *pv; 2578 pmap_t pm_to_clean = NULL; 2579 bool cache_needs_cleaning = false; 2580 vaddr_t page_to_clean = 0; 2581 u_int flags = 0; 2582 2583 /* 2584 * Since we flush the cache each time we change to a different 2585 * user vmspace, we only need to flush the page if it is in the 2586 * current pmap. 2587 */ 2588 KASSERT(pmap_page_locked_p(md)); 2589 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2590 if (pmap_is_current(pv->pv_pmap)) { 2591 flags |= pv->pv_flags; 2592 /* 2593 * The page is mapped non-cacheable in 2594 * this map. No need to flush the cache. 2595 */ 2596 if (pv->pv_flags & PVF_NC) { 2597 #ifdef DIAGNOSTIC 2598 KASSERT(!cache_needs_cleaning); 2599 #endif 2600 break; 2601 } else if (is_src && (pv->pv_flags & PVF_WRITE) == 0) 2602 continue; 2603 if (cache_needs_cleaning) { 2604 page_to_clean = 0; 2605 break; 2606 } else { 2607 page_to_clean = pv->pv_va; 2608 pm_to_clean = pv->pv_pmap; 2609 } 2610 cache_needs_cleaning = true; 2611 } 2612 } 2613 2614 if (page_to_clean) { 2615 pmap_cache_wbinv_page(pm_to_clean, page_to_clean, 2616 !is_src, flags | PVF_REF); 2617 } else if (cache_needs_cleaning) { 2618 pmap_t const pm = curproc->p_vmspace->vm_map.pmap; 2619 2620 pmap_cache_wbinv_all(pm, flags); 2621 return true; 2622 } 2623 return false; 2624 } 2625 #endif 2626 2627 #ifdef PMAP_CACHE_VIPT 2628 /* 2629 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2630 * right cache alias to make sure we flush the right stuff. 2631 */ 2632 void 2633 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa) 2634 { 2635 pmap_t kpm = pmap_kernel(); 2636 const size_t way_size = arm_pcache.icache_type == CACHE_TYPE_PIPT 2637 ? PAGE_SIZE 2638 : arm_pcache.icache_way_size; 2639 2640 NPDEBUG(PDB_EXEC, printf("pmap_syncicache_page: md=%p (attrs=%#x)\n", 2641 md, md->pvh_attrs)); 2642 /* 2643 * No need to clean the page if it's non-cached. 2644 */ 2645 #ifndef ARM_MMU_EXTENDED 2646 if (md->pvh_attrs & PVF_NC) 2647 return; 2648 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED); 2649 #endif 2650 2651 pt_entry_t * const ptep = cpu_cdst_pte(0); 2652 const vaddr_t dstp = cpu_cdstp(0); 2653 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 2654 if (way_size <= PAGE_SIZE) { 2655 bool ok = false; 2656 vaddr_t vdstp = pmap_direct_mapped_phys(pa, &ok, dstp); 2657 if (ok) { 2658 cpu_icache_sync_range(vdstp, way_size); 2659 return; 2660 } 2661 } 2662 #endif 2663 2664 /* 2665 * We don't worry about the color of the exec page, we map the 2666 * same page to pages in the way and then do the icache_sync on 2667 * the entire way making sure we are cleaned. 2668 */ 2669 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 2670 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 2671 2672 for (size_t i = 0, j = 0; i < way_size; 2673 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2674 l2pte_reset(ptep + j); 2675 PTE_SYNC(ptep + j); 2676 2677 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2678 /* 2679 * Set up a PTE with to flush these cache lines. 2680 */ 2681 l2pte_set(ptep + j, npte, 0); 2682 } 2683 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2684 2685 /* 2686 * Flush it. 2687 */ 2688 cpu_icache_sync_range(dstp, way_size); 2689 2690 for (size_t i = 0, j = 0; i < way_size; 2691 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2692 /* 2693 * Unmap the page(s). 2694 */ 2695 l2pte_reset(ptep + j); 2696 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2697 } 2698 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2699 2700 md->pvh_attrs |= PVF_EXEC; 2701 PMAPCOUNT(exec_synced); 2702 } 2703 2704 #ifndef ARM_MMU_EXTENDED 2705 void 2706 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush) 2707 { 2708 vsize_t va_offset, end_va; 2709 bool wbinv_p; 2710 2711 if (arm_cache_prefer_mask == 0) 2712 return; 2713 2714 switch (flush) { 2715 case PMAP_FLUSH_PRIMARY: 2716 if (md->pvh_attrs & PVF_MULTCLR) { 2717 va_offset = 0; 2718 end_va = arm_cache_prefer_mask; 2719 md->pvh_attrs &= ~PVF_MULTCLR; 2720 PMAPCOUNT(vac_flush_lots); 2721 } else { 2722 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2723 end_va = va_offset; 2724 PMAPCOUNT(vac_flush_one); 2725 } 2726 /* 2727 * Mark that the page is no longer dirty. 2728 */ 2729 md->pvh_attrs &= ~PVF_DIRTY; 2730 wbinv_p = true; 2731 break; 2732 case PMAP_FLUSH_SECONDARY: 2733 va_offset = 0; 2734 end_va = arm_cache_prefer_mask; 2735 wbinv_p = true; 2736 md->pvh_attrs &= ~PVF_MULTCLR; 2737 PMAPCOUNT(vac_flush_lots); 2738 break; 2739 case PMAP_CLEAN_PRIMARY: 2740 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2741 end_va = va_offset; 2742 wbinv_p = false; 2743 /* 2744 * Mark that the page is no longer dirty. 2745 */ 2746 if ((md->pvh_attrs & PVF_DMOD) == 0) 2747 md->pvh_attrs &= ~PVF_DIRTY; 2748 PMAPCOUNT(vac_clean_one); 2749 break; 2750 default: 2751 return; 2752 } 2753 2754 KASSERT(!(md->pvh_attrs & PVF_NC)); 2755 2756 NPDEBUG(PDB_VAC, printf("pmap_flush_page: md=%p (attrs=%#x)\n", 2757 md, md->pvh_attrs)); 2758 2759 const size_t scache_line_size = arm_scache.dcache_line_size; 2760 2761 for (; va_offset <= end_va; va_offset += PAGE_SIZE) { 2762 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 2763 const vaddr_t dstp = cpu_cdstp(va_offset); 2764 const pt_entry_t opte = *ptep; 2765 2766 if (flush == PMAP_FLUSH_SECONDARY 2767 && va_offset == (md->pvh_attrs & arm_cache_prefer_mask)) 2768 continue; 2769 2770 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2771 /* 2772 * Set up a PTE with the right coloring to flush 2773 * existing cache entries. 2774 */ 2775 const pt_entry_t npte = L2_S_PROTO 2776 | pa 2777 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2778 | pte_l2_s_cache_mode; 2779 l2pte_set(ptep, npte, opte); 2780 PTE_SYNC(ptep); 2781 2782 /* 2783 * Flush it. Make sure to flush secondary cache too since 2784 * bus_dma will ignore uncached pages. 2785 */ 2786 if (scache_line_size != 0) { 2787 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2788 if (wbinv_p) { 2789 cpu_sdcache_wbinv_range(dstp, pa, PAGE_SIZE); 2790 cpu_dcache_inv_range(dstp, PAGE_SIZE); 2791 } else { 2792 cpu_sdcache_wb_range(dstp, pa, PAGE_SIZE); 2793 } 2794 } else { 2795 if (wbinv_p) { 2796 cpu_dcache_wbinv_range(dstp, PAGE_SIZE); 2797 } else { 2798 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2799 } 2800 } 2801 2802 /* 2803 * Restore the page table entry since we might have interrupted 2804 * pmap_zero_page or pmap_copy_page which was already using 2805 * this pte. 2806 */ 2807 if (opte) { 2808 l2pte_set(ptep, opte, npte); 2809 } else { 2810 l2pte_reset(ptep); 2811 } 2812 PTE_SYNC(ptep); 2813 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2814 } 2815 } 2816 #endif /* ARM_MMU_EXTENDED */ 2817 #endif /* PMAP_CACHE_VIPT */ 2818 2819 /* 2820 * Routine: pmap_page_remove 2821 * Function: 2822 * Removes this physical page from 2823 * all physical maps in which it resides. 2824 * Reflects back modify bits to the pager. 2825 */ 2826 static void 2827 pmap_page_remove(struct vm_page_md *md, paddr_t pa) 2828 { 2829 struct l2_bucket *l2b; 2830 struct pv_entry *pv; 2831 pt_entry_t *ptep; 2832 #ifndef ARM_MMU_EXTENDED 2833 bool flush = false; 2834 #endif 2835 u_int flags = 0; 2836 2837 NPDEBUG(PDB_FOLLOW, 2838 printf("pmap_page_remove: md %p (0x%08lx)\n", md, 2839 pa)); 2840 2841 struct pv_entry **pvp = &SLIST_FIRST(&md->pvh_list); 2842 pmap_acquire_page_lock(md); 2843 if (*pvp == NULL) { 2844 #ifdef PMAP_CACHE_VIPT 2845 /* 2846 * We *know* the page contents are about to be replaced. 2847 * Discard the exec contents 2848 */ 2849 if (PV_IS_EXEC_P(md->pvh_attrs)) 2850 PMAPCOUNT(exec_discarded_page_protect); 2851 md->pvh_attrs &= ~PVF_EXEC; 2852 PMAP_VALIDATE_MD_PAGE(md); 2853 #endif 2854 pmap_release_page_lock(md); 2855 return; 2856 } 2857 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 2858 KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md)); 2859 #endif 2860 2861 /* 2862 * Clear alias counts 2863 */ 2864 #ifdef PMAP_CACHE_VIVT 2865 md->k_mappings = 0; 2866 #endif 2867 md->urw_mappings = md->uro_mappings = 0; 2868 2869 #ifdef PMAP_CACHE_VIVT 2870 pmap_clean_page(md, false); 2871 #endif 2872 2873 while ((pv = *pvp) != NULL) { 2874 pmap_t pm = pv->pv_pmap; 2875 #ifndef ARM_MMU_EXTENDED 2876 if (flush == false && pmap_is_current(pm)) 2877 flush = true; 2878 #endif 2879 2880 if (pm == pmap_kernel()) { 2881 #ifdef PMAP_CACHE_VIPT 2882 /* 2883 * If this was unmanaged mapping, it must be preserved. 2884 * Move it back on the list and advance the end-of-list 2885 * pointer. 2886 */ 2887 if (PV_IS_KENTRY_P(pv->pv_flags)) { 2888 *pvp = pv; 2889 pvp = &SLIST_NEXT(pv, pv_link); 2890 continue; 2891 } 2892 if (pv->pv_flags & PVF_WRITE) 2893 md->krw_mappings--; 2894 else 2895 md->kro_mappings--; 2896 #endif 2897 PMAPCOUNT(kernel_unmappings); 2898 } 2899 *pvp = SLIST_NEXT(pv, pv_link); /* remove from list */ 2900 PMAPCOUNT(unmappings); 2901 2902 pmap_release_page_lock(md); 2903 pmap_acquire_pmap_lock(pm); 2904 2905 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2906 KASSERTMSG(l2b != NULL, "%#lx", pv->pv_va); 2907 2908 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2909 2910 /* 2911 * Update statistics 2912 */ 2913 --pm->pm_stats.resident_count; 2914 2915 /* Wired bit */ 2916 if (pv->pv_flags & PVF_WIRED) 2917 --pm->pm_stats.wired_count; 2918 2919 flags |= pv->pv_flags; 2920 2921 /* 2922 * Invalidate the PTEs. 2923 */ 2924 l2pte_reset(ptep); 2925 PTE_SYNC_CURRENT(pm, ptep); 2926 2927 #ifdef ARM_MMU_EXTENDED 2928 pmap_tlb_invalidate_addr(pm, pv->pv_va); 2929 #endif 2930 2931 pmap_free_l2_bucket(pm, l2b, PAGE_SIZE / L2_S_SIZE); 2932 2933 pmap_release_pmap_lock(pm); 2934 2935 pool_put(&pmap_pv_pool, pv); 2936 pmap_acquire_page_lock(md); 2937 #ifdef MULTIPROCESSOR 2938 /* 2939 * Restart of the beginning of the list. 2940 */ 2941 pvp = &SLIST_FIRST(&md->pvh_list); 2942 #endif 2943 } 2944 /* 2945 * if we reach the end of the list and there are still mappings, they 2946 * might be able to be cached now. And they must be kernel mappings. 2947 */ 2948 if (!SLIST_EMPTY(&md->pvh_list)) { 2949 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 2950 } 2951 2952 #ifdef PMAP_CACHE_VIPT 2953 /* 2954 * Its EXEC cache is now gone. 2955 */ 2956 if (PV_IS_EXEC_P(md->pvh_attrs)) 2957 PMAPCOUNT(exec_discarded_page_protect); 2958 md->pvh_attrs &= ~PVF_EXEC; 2959 KASSERT(md->urw_mappings == 0); 2960 KASSERT(md->uro_mappings == 0); 2961 #ifndef ARM_MMU_EXTENDED 2962 if (arm_cache_prefer_mask != 0) { 2963 if (md->krw_mappings == 0) 2964 md->pvh_attrs &= ~PVF_WRITE; 2965 PMAP_VALIDATE_MD_PAGE(md); 2966 } 2967 #endif /* ARM_MMU_EXTENDED */ 2968 #endif /* PMAP_CACHE_VIPT */ 2969 pmap_release_page_lock(md); 2970 2971 #ifndef ARM_MMU_EXTENDED 2972 if (flush) { 2973 /* 2974 * Note: We can't use pmap_tlb_flush{I,D}() here since that 2975 * would need a subsequent call to pmap_update() to ensure 2976 * curpm->pm_cstate.cs_all is reset. Our callers are not 2977 * required to do that (see pmap(9)), so we can't modify 2978 * the current pmap's state. 2979 */ 2980 if (PV_BEEN_EXECD(flags)) 2981 cpu_tlb_flushID(); 2982 else 2983 cpu_tlb_flushD(); 2984 } 2985 cpu_cpwait(); 2986 #endif /* ARM_MMU_EXTENDED */ 2987 } 2988 2989 /* 2990 * pmap_t pmap_create(void) 2991 * 2992 * Create a new pmap structure from scratch. 2993 */ 2994 pmap_t 2995 pmap_create(void) 2996 { 2997 pmap_t pm; 2998 2999 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 3000 3001 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE); 3002 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 3003 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 3004 3005 pm->pm_stats.wired_count = 0; 3006 pm->pm_stats.resident_count = 1; 3007 #ifdef ARM_MMU_EXTENDED 3008 #ifdef MULTIPROCESSOR 3009 kcpuset_create(&pm->pm_active, true); 3010 kcpuset_create(&pm->pm_onproc, true); 3011 #endif 3012 #else 3013 pm->pm_cstate.cs_all = 0; 3014 #endif 3015 pmap_alloc_l1(pm); 3016 3017 /* 3018 * Note: The pool cache ensures that the pm_l2[] array is already 3019 * initialised to zero. 3020 */ 3021 3022 pmap_pinit(pm); 3023 3024 return (pm); 3025 } 3026 3027 u_int 3028 arm32_mmap_flags(paddr_t pa) 3029 { 3030 /* 3031 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff 3032 * and we're using the upper bits in page numbers to pass flags around 3033 * so we might as well use the same bits 3034 */ 3035 return (u_int)pa & PMAP_MD_MASK; 3036 } 3037 /* 3038 * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 3039 * u_int flags) 3040 * 3041 * Insert the given physical page (p) at 3042 * the specified virtual address (v) in the 3043 * target physical map with the protection requested. 3044 * 3045 * NB: This is the only routine which MAY NOT lazy-evaluate 3046 * or lose information. That is, this routine must actually 3047 * insert this page into the given map NOW. 3048 */ 3049 int 3050 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3051 { 3052 struct l2_bucket *l2b; 3053 struct vm_page *pg, *opg; 3054 u_int nflags; 3055 u_int oflags; 3056 const bool kpm_p = (pm == pmap_kernel()); 3057 #ifdef ARM_HAS_VBAR 3058 const bool vector_page_p = false; 3059 #else 3060 const bool vector_page_p = (va == vector_page); 3061 #endif 3062 struct pmap_page *pp = pmap_pv_tracked(pa); 3063 struct pv_entry *new_pv = NULL; 3064 struct pv_entry *old_pv = NULL; 3065 int error = 0; 3066 3067 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3068 3069 UVMHIST_LOG(maphist, " (pm %#jx va %#jx pa %#jx prot %#jx", 3070 (uintptr_t)pm, va, pa, prot); 3071 UVMHIST_LOG(maphist, " flag %#jx", flags, 0, 0, 0); 3072 3073 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 3074 KDASSERT(((va | pa) & PGOFSET) == 0); 3075 3076 /* 3077 * Get a pointer to the page. Later on in this function, we 3078 * test for a managed page by checking pg != NULL. 3079 */ 3080 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 3081 /* 3082 * if we may need a new pv entry allocate if now, as we can't do it 3083 * with the kernel_pmap locked 3084 */ 3085 if (pg || pp) 3086 new_pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3087 3088 nflags = 0; 3089 if (prot & VM_PROT_WRITE) 3090 nflags |= PVF_WRITE; 3091 if (prot & VM_PROT_EXECUTE) 3092 nflags |= PVF_EXEC; 3093 if (flags & PMAP_WIRED) 3094 nflags |= PVF_WIRED; 3095 3096 pmap_acquire_pmap_lock(pm); 3097 3098 /* 3099 * Fetch the L2 bucket which maps this page, allocating one if 3100 * necessary for user pmaps. 3101 */ 3102 if (kpm_p) { 3103 l2b = pmap_get_l2_bucket(pm, va); 3104 } else { 3105 l2b = pmap_alloc_l2_bucket(pm, va); 3106 } 3107 if (l2b == NULL) { 3108 if (flags & PMAP_CANFAIL) { 3109 pmap_release_pmap_lock(pm); 3110 error = ENOMEM; 3111 goto free_pv; 3112 } 3113 panic("pmap_enter: failed to allocate L2 bucket"); 3114 } 3115 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)]; 3116 const pt_entry_t opte = *ptep; 3117 pt_entry_t npte = pa; 3118 oflags = 0; 3119 3120 if (opte) { 3121 /* 3122 * There is already a mapping at this address. 3123 * If the physical address is different, lookup the 3124 * vm_page. 3125 */ 3126 if (l2pte_pa(opte) != pa) { 3127 KASSERT(!pmap_pv_tracked(pa)); 3128 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3129 } else 3130 opg = pg; 3131 } else 3132 opg = NULL; 3133 3134 if (pg || pp) { 3135 KASSERT((pg != NULL) != (pp != NULL)); 3136 struct vm_page_md *md = (pg != NULL) ? VM_PAGE_TO_MD(pg) : 3137 PMAP_PAGE_TO_MD(pp); 3138 3139 /* 3140 * This is to be a managed mapping. 3141 */ 3142 pmap_acquire_page_lock(md); 3143 if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) { 3144 /* 3145 * - The access type indicates that we don't need 3146 * to do referenced emulation. 3147 * OR 3148 * - The physical page has already been referenced 3149 * so no need to re-do referenced emulation here. 3150 */ 3151 npte |= l2pte_set_readonly(L2_S_PROTO); 3152 3153 nflags |= PVF_REF; 3154 3155 if ((prot & VM_PROT_WRITE) != 0 && 3156 ((flags & VM_PROT_WRITE) != 0 || 3157 (md->pvh_attrs & PVF_MOD) != 0)) { 3158 /* 3159 * This is a writable mapping, and the 3160 * page's mod state indicates it has 3161 * already been modified. Make it 3162 * writable from the outset. 3163 */ 3164 npte = l2pte_set_writable(npte); 3165 nflags |= PVF_MOD; 3166 } 3167 3168 #ifdef ARM_MMU_EXTENDED 3169 /* 3170 * If the page has been cleaned, then the pvh_attrs 3171 * will have PVF_EXEC set, so mark it execute so we 3172 * don't get an access fault when trying to execute 3173 * from it. 3174 */ 3175 if (md->pvh_attrs & nflags & PVF_EXEC) { 3176 npte &= ~L2_XS_XN; 3177 } 3178 #endif 3179 } else { 3180 /* 3181 * Need to do page referenced emulation. 3182 */ 3183 npte |= L2_TYPE_INV; 3184 } 3185 3186 if (flags & ARM32_MMAP_WRITECOMBINE) { 3187 npte |= pte_l2_s_wc_mode; 3188 } else 3189 npte |= pte_l2_s_cache_mode; 3190 3191 if (pg != NULL && pg == opg) { 3192 /* 3193 * We're changing the attrs of an existing mapping. 3194 */ 3195 oflags = pmap_modify_pv(md, pa, pm, va, 3196 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3197 PVF_MOD | PVF_REF, nflags); 3198 3199 #ifdef PMAP_CACHE_VIVT 3200 /* 3201 * We may need to flush the cache if we're 3202 * doing rw-ro... 3203 */ 3204 if (pm->pm_cstate.cs_cache_d && 3205 (oflags & PVF_NC) == 0 && 3206 l2pte_writable_p(opte) && 3207 (prot & VM_PROT_WRITE) == 0) 3208 cpu_dcache_wb_range(va, PAGE_SIZE); 3209 #endif 3210 } else { 3211 struct pv_entry *pv; 3212 /* 3213 * New mapping, or changing the backing page 3214 * of an existing mapping. 3215 */ 3216 if (opg) { 3217 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3218 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3219 3220 /* 3221 * Replacing an existing mapping with a new one. 3222 * It is part of our managed memory so we 3223 * must remove it from the PV list 3224 */ 3225 pv = pmap_remove_pv(omd, opa, pm, va); 3226 pmap_vac_me_harder(omd, opa, pm, 0); 3227 oflags = pv->pv_flags; 3228 3229 #ifdef PMAP_CACHE_VIVT 3230 /* 3231 * If the old mapping was valid (ref/mod 3232 * emulation creates 'invalid' mappings 3233 * initially) then make sure to frob 3234 * the cache. 3235 */ 3236 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3237 pmap_cache_wbinv_page(pm, va, true, 3238 oflags); 3239 } 3240 #endif 3241 } else { 3242 pv = new_pv; 3243 new_pv = NULL; 3244 if (pv == NULL) { 3245 pmap_release_page_lock(md); 3246 pmap_release_pmap_lock(pm); 3247 if ((flags & PMAP_CANFAIL) == 0) 3248 panic("pmap_enter: " 3249 "no pv entries"); 3250 3251 pmap_free_l2_bucket(pm, l2b, 0); 3252 UVMHIST_LOG(maphist, " <-- done (ENOMEM)", 3253 0, 0, 0, 0); 3254 return (ENOMEM); 3255 } 3256 } 3257 3258 pmap_enter_pv(md, pa, pv, pm, va, nflags); 3259 } 3260 pmap_release_page_lock(md); 3261 } else { 3262 /* 3263 * We're mapping an unmanaged page. 3264 * These are always readable, and possibly writable, from 3265 * the get go as we don't need to track ref/mod status. 3266 */ 3267 npte |= l2pte_set_readonly(L2_S_PROTO); 3268 if (prot & VM_PROT_WRITE) 3269 npte = l2pte_set_writable(npte); 3270 3271 /* 3272 * Make sure the vector table is mapped cacheable 3273 */ 3274 if ((vector_page_p && !kpm_p) 3275 || (flags & ARM32_MMAP_CACHEABLE)) { 3276 npte |= pte_l2_s_cache_mode; 3277 #ifdef ARM_MMU_EXTENDED 3278 npte &= ~L2_XS_XN; /* and executable */ 3279 #endif 3280 } else if (flags & ARM32_MMAP_WRITECOMBINE) { 3281 npte |= pte_l2_s_wc_mode; 3282 } 3283 if (opg) { 3284 /* 3285 * Looks like there's an existing 'managed' mapping 3286 * at this address. 3287 */ 3288 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3289 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3290 3291 pmap_acquire_page_lock(omd); 3292 old_pv = pmap_remove_pv(omd, opa, pm, va); 3293 pmap_vac_me_harder(omd, opa, pm, 0); 3294 oflags = old_pv->pv_flags; 3295 pmap_release_page_lock(omd); 3296 3297 #ifdef PMAP_CACHE_VIVT 3298 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3299 pmap_cache_wbinv_page(pm, va, true, oflags); 3300 } 3301 #endif 3302 } 3303 } 3304 3305 /* 3306 * Make sure userland mappings get the right permissions 3307 */ 3308 if (!vector_page_p && !kpm_p) { 3309 npte |= L2_S_PROT_U; 3310 #ifdef ARM_MMU_EXTENDED 3311 npte |= L2_XS_nG; /* user pages are not global */ 3312 #endif 3313 } 3314 3315 /* 3316 * Keep the stats up to date 3317 */ 3318 if (opte == 0) { 3319 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3320 pm->pm_stats.resident_count++; 3321 } 3322 3323 UVMHIST_LOG(maphist, " opte %#jx npte %#jx", opte, npte, 0, 0); 3324 3325 #if defined(ARM_MMU_EXTENDED) 3326 /* 3327 * If exec protection was requested but the page hasn't been synced, 3328 * sync it now and allow execution from it. 3329 */ 3330 if ((nflags & PVF_EXEC) && (npte & L2_XS_XN)) { 3331 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3332 npte &= ~L2_XS_XN; 3333 pmap_syncicache_page(md, pa); 3334 PMAPCOUNT(exec_synced_map); 3335 } 3336 #endif 3337 /* 3338 * If this is just a wiring change, the two PTEs will be 3339 * identical, so there's no need to update the page table. 3340 */ 3341 if (npte != opte) { 3342 l2pte_reset(ptep); 3343 PTE_SYNC(ptep); 3344 if (l2pte_valid_p(opte)) { 3345 pmap_tlb_flush_SE(pm, va, oflags); 3346 } 3347 l2pte_set(ptep, npte, 0); 3348 PTE_SYNC(ptep); 3349 #ifndef ARM_MMU_EXTENDED 3350 bool is_cached = pmap_is_cached(pm); 3351 if (is_cached) { 3352 /* 3353 * We only need to frob the cache/tlb if this pmap 3354 * is current 3355 */ 3356 if (!vector_page_p && l2pte_valid_p(npte)) { 3357 /* 3358 * This mapping is likely to be accessed as 3359 * soon as we return to userland. Fix up the 3360 * L1 entry to avoid taking another 3361 * page/domain fault. 3362 */ 3363 pd_entry_t *pdep = pmap_l1_kva(pm) 3364 + l1pte_index(va); 3365 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa 3366 | L1_C_DOM(pmap_domain(pm)); 3367 if (*pdep != pde) { 3368 l1pte_setone(pdep, pde); 3369 PDE_SYNC(pdep); 3370 } 3371 } 3372 } 3373 #endif /* !ARM_MMU_EXTENDED */ 3374 3375 #ifndef ARM_MMU_EXTENDED 3376 UVMHIST_LOG(maphist, " is_cached %jd cs 0x%08jx", 3377 is_cached, pm->pm_cstate.cs_all, 0, 0); 3378 3379 if (pg != NULL) { 3380 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3381 3382 pmap_acquire_page_lock(md); 3383 pmap_vac_me_harder(md, pa, pm, va); 3384 pmap_release_page_lock(md); 3385 } 3386 #endif 3387 } 3388 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 3389 if (pg) { 3390 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3391 3392 pmap_acquire_page_lock(md); 3393 #ifndef ARM_MMU_EXTENDED 3394 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3395 #endif 3396 PMAP_VALIDATE_MD_PAGE(md); 3397 pmap_release_page_lock(md); 3398 } 3399 #endif 3400 3401 pmap_release_pmap_lock(pm); 3402 3403 3404 if (old_pv) 3405 pool_put(&pmap_pv_pool, old_pv); 3406 free_pv: 3407 if (new_pv) 3408 pool_put(&pmap_pv_pool, new_pv); 3409 return (error); 3410 } 3411 3412 /* 3413 * pmap_remove() 3414 * 3415 * pmap_remove is responsible for nuking a number of mappings for a range 3416 * of virtual address space in the current pmap. To do this efficiently 3417 * is interesting, because in a number of cases a wide virtual address 3418 * range may be supplied that contains few actual mappings. So, the 3419 * optimisations are: 3420 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3421 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3422 * maybe do just a partial cache clean. This path of execution is 3423 * complicated by the fact that the cache must be flushed _before_ 3424 * the PTE is nuked, being a VAC :-) 3425 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3426 * all invalidations until pmap_update(), since pmap_remove_all() has 3427 * already flushed the cache. 3428 * 4. Maybe later fast-case a single page, but I don't think this is 3429 * going to make _that_ much difference overall. 3430 */ 3431 3432 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3433 3434 void 3435 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 3436 { 3437 SLIST_HEAD(,pv_entry) opv_list; 3438 struct pv_entry *pv, *npv; 3439 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3440 UVMHIST_LOG(maphist, " (pm=%#jx, sva=%#jx, eva=%#jx)", 3441 (uintptr_t)pm, sva, eva, 0); 3442 3443 SLIST_INIT(&opv_list); 3444 /* 3445 * we lock in the pmap => pv_head direction 3446 */ 3447 pmap_acquire_pmap_lock(pm); 3448 3449 #ifndef ARM_MMU_EXTENDED 3450 u_int cleanlist_idx, total, cnt; 3451 struct { 3452 vaddr_t va; 3453 pt_entry_t *ptep; 3454 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3455 3456 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3457 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3458 if (pm->pm_cstate.cs_tlb == 0) 3459 pm->pm_remove_all = true; 3460 } else 3461 cleanlist_idx = 0; 3462 total = 0; 3463 #endif 3464 3465 while (sva < eva) { 3466 /* 3467 * Do one L2 bucket's worth at a time. 3468 */ 3469 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(sva); 3470 if (next_bucket > eva) 3471 next_bucket = eva; 3472 3473 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva); 3474 if (l2b == NULL) { 3475 sva = next_bucket; 3476 continue; 3477 } 3478 3479 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3480 u_int mappings = 0; 3481 3482 for (;sva < next_bucket; 3483 sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) { 3484 pt_entry_t opte = *ptep; 3485 3486 if (opte == 0) { 3487 /* Nothing here, move along */ 3488 continue; 3489 } 3490 3491 u_int flags = PVF_REF; 3492 paddr_t pa = l2pte_pa(opte); 3493 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 3494 3495 /* 3496 * Update flags. In a number of circumstances, 3497 * we could cluster a lot of these and do a 3498 * number of sequential pages in one go. 3499 */ 3500 if (pg != NULL) { 3501 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3502 3503 pmap_acquire_page_lock(md); 3504 pv = pmap_remove_pv(md, pa, pm, sva); 3505 pmap_vac_me_harder(md, pa, pm, 0); 3506 pmap_release_page_lock(md); 3507 if (pv != NULL) { 3508 if (pm->pm_remove_all == false) { 3509 flags = pv->pv_flags; 3510 } 3511 SLIST_INSERT_HEAD(&opv_list, 3512 pv, pv_link); 3513 } 3514 } 3515 mappings += PAGE_SIZE / L2_S_SIZE; 3516 3517 if (!l2pte_valid_p(opte)) { 3518 /* 3519 * Ref/Mod emulation is still active for this 3520 * mapping, therefore it is has not yet been 3521 * accessed. No need to frob the cache/tlb. 3522 */ 3523 l2pte_reset(ptep); 3524 PTE_SYNC_CURRENT(pm, ptep); 3525 continue; 3526 } 3527 3528 #ifdef ARM_MMU_EXTENDED 3529 l2pte_reset(ptep); 3530 PTE_SYNC(ptep); 3531 if (__predict_false(pm->pm_remove_all == false)) { 3532 pmap_tlb_flush_SE(pm, sva, flags); 3533 } 3534 #else 3535 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3536 /* Add to the clean list. */ 3537 cleanlist[cleanlist_idx].ptep = ptep; 3538 cleanlist[cleanlist_idx].va = 3539 sva | (flags & PVF_EXEC); 3540 cleanlist_idx++; 3541 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3542 /* Nuke everything if needed. */ 3543 #ifdef PMAP_CACHE_VIVT 3544 pmap_cache_wbinv_all(pm, PVF_EXEC); 3545 #endif 3546 /* 3547 * Roll back the previous PTE list, 3548 * and zero out the current PTE. 3549 */ 3550 for (cnt = 0; 3551 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3552 l2pte_reset(cleanlist[cnt].ptep); 3553 PTE_SYNC(cleanlist[cnt].ptep); 3554 } 3555 l2pte_reset(ptep); 3556 PTE_SYNC(ptep); 3557 cleanlist_idx++; 3558 pm->pm_remove_all = true; 3559 } else { 3560 l2pte_reset(ptep); 3561 PTE_SYNC(ptep); 3562 if (pm->pm_remove_all == false) { 3563 pmap_tlb_flush_SE(pm, sva, flags); 3564 } 3565 } 3566 #endif 3567 } 3568 3569 #ifndef ARM_MMU_EXTENDED 3570 /* 3571 * Deal with any left overs 3572 */ 3573 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3574 total += cleanlist_idx; 3575 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3576 l2pte_reset(cleanlist[cnt].ptep); 3577 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3578 vaddr_t va = cleanlist[cnt].va; 3579 if (pm->pm_cstate.cs_all != 0) { 3580 vaddr_t clva = va & ~PAGE_MASK; 3581 u_int flags = va & PVF_EXEC; 3582 #ifdef PMAP_CACHE_VIVT 3583 pmap_cache_wbinv_page(pm, clva, true, 3584 PVF_REF | PVF_WRITE | flags); 3585 #endif 3586 pmap_tlb_flush_SE(pm, clva, 3587 PVF_REF | flags); 3588 } 3589 } 3590 3591 /* 3592 * If it looks like we're removing a whole bunch 3593 * of mappings, it's faster to just write-back 3594 * the whole cache now and defer TLB flushes until 3595 * pmap_update() is called. 3596 */ 3597 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3598 cleanlist_idx = 0; 3599 else { 3600 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3601 #ifdef PMAP_CACHE_VIVT 3602 pmap_cache_wbinv_all(pm, PVF_EXEC); 3603 #endif 3604 pm->pm_remove_all = true; 3605 } 3606 } 3607 #endif /* ARM_MMU_EXTENDED */ 3608 3609 pmap_free_l2_bucket(pm, l2b, mappings); 3610 pm->pm_stats.resident_count -= mappings / (PAGE_SIZE/L2_S_SIZE); 3611 } 3612 3613 pmap_release_pmap_lock(pm); 3614 SLIST_FOREACH_SAFE(pv, &opv_list, pv_link, npv) { 3615 pool_put(&pmap_pv_pool, pv); 3616 } 3617 } 3618 3619 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3620 static struct pv_entry * 3621 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3622 { 3623 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3624 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3625 struct pv_entry *pv; 3626 3627 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC)); 3628 KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0); 3629 KASSERT(pmap_page_locked_p(md)); 3630 3631 pv = pmap_remove_pv(md, pa, pmap_kernel(), va); 3632 KASSERTMSG(pv, "pg %p (pa #%lx) va %#lx", pg, pa, va); 3633 KASSERT(PV_IS_KENTRY_P(pv->pv_flags)); 3634 3635 /* 3636 * We are removing a writeable mapping to a cached exec page, if 3637 * it's the last mapping then clear its execness otherwise sync 3638 * the page to the icache. 3639 */ 3640 if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3641 && (pv->pv_flags & PVF_WRITE) != 0) { 3642 if (SLIST_EMPTY(&md->pvh_list)) { 3643 md->pvh_attrs &= ~PVF_EXEC; 3644 PMAPCOUNT(exec_discarded_kremove); 3645 } else { 3646 pmap_syncicache_page(md, pa); 3647 PMAPCOUNT(exec_synced_kremove); 3648 } 3649 } 3650 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 3651 3652 return pv; 3653 } 3654 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 3655 3656 /* 3657 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3658 * 3659 * We assume there is already sufficient KVM space available 3660 * to do this, as we can't allocate L2 descriptor tables/metadata 3661 * from here. 3662 */ 3663 void 3664 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3665 { 3666 #ifdef PMAP_CACHE_VIVT 3667 struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3668 #endif 3669 #ifdef PMAP_CACHE_VIPT 3670 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3671 struct vm_page *opg; 3672 #ifndef ARM_MMU_EXTENDED 3673 struct pv_entry *pv = NULL; 3674 #endif 3675 #endif 3676 struct vm_page_md *md = pg != NULL ? VM_PAGE_TO_MD(pg) : NULL; 3677 3678 UVMHIST_FUNC(__func__); 3679 3680 if (pmap_initialized) { 3681 UVMHIST_CALLED(maphist); 3682 UVMHIST_LOG(maphist, " (va=%#jx, pa=%#jx, prot=%#jx, flags=%#jx", 3683 va, pa, prot, flags); 3684 } 3685 3686 pmap_t kpm = pmap_kernel(); 3687 pmap_acquire_pmap_lock(kpm); 3688 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3689 const size_t l1slot __diagused = l1pte_index(va); 3690 KASSERTMSG(l2b != NULL, 3691 "va %#lx pa %#lx prot %d maxkvaddr %#lx: l2 %p l2b %p kva %p", 3692 va, pa, prot, pmap_curmaxkvaddr, kpm->pm_l2[L2_IDX(l1slot)], 3693 kpm->pm_l2[L2_IDX(l1slot)] 3694 ? &kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)] 3695 : NULL, 3696 kpm->pm_l2[L2_IDX(l1slot)] 3697 ? kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)].l2b_kva 3698 : NULL); 3699 KASSERT(l2b->l2b_kva != NULL); 3700 3701 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 3702 const pt_entry_t opte = *ptep; 3703 3704 if (opte == 0) { 3705 PMAPCOUNT(kenter_mappings); 3706 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3707 } else { 3708 PMAPCOUNT(kenter_remappings); 3709 #ifdef PMAP_CACHE_VIPT 3710 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3711 #if !defined(ARM_MMU_EXTENDED) || defined(DIAGNOSTIC) 3712 struct vm_page_md *omd __diagused = VM_PAGE_TO_MD(opg); 3713 #endif 3714 if (opg && arm_cache_prefer_mask != 0) { 3715 KASSERT(opg != pg); 3716 KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0); 3717 KASSERT((flags & PMAP_KMPAGE) == 0); 3718 #ifndef ARM_MMU_EXTENDED 3719 pmap_acquire_page_lock(omd); 3720 pv = pmap_kremove_pg(opg, va); 3721 pmap_release_page_lock(omd); 3722 #endif 3723 } 3724 #endif 3725 if (l2pte_valid_p(opte)) { 3726 l2pte_reset(ptep); 3727 PTE_SYNC(ptep); 3728 #ifdef PMAP_CACHE_VIVT 3729 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3730 #endif 3731 cpu_tlb_flushD_SE(va); 3732 cpu_cpwait(); 3733 } 3734 } 3735 pmap_release_pmap_lock(kpm); 3736 pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 3737 3738 if (flags & PMAP_PTE) { 3739 KASSERT((flags & PMAP_CACHE_MASK) == 0); 3740 if (!(flags & PMAP_NOCACHE)) 3741 npte |= pte_l2_s_cache_mode_pt; 3742 } else { 3743 switch (flags & PMAP_CACHE_MASK) { 3744 case PMAP_NOCACHE: 3745 break; 3746 case PMAP_WRITE_COMBINE: 3747 npte |= pte_l2_s_wc_mode; 3748 break; 3749 default: 3750 npte |= pte_l2_s_cache_mode; 3751 break; 3752 } 3753 } 3754 #ifdef ARM_MMU_EXTENDED 3755 if (prot & VM_PROT_EXECUTE) 3756 npte &= ~L2_XS_XN; 3757 #endif 3758 l2pte_set(ptep, npte, 0); 3759 PTE_SYNC(ptep); 3760 3761 if (pg) { 3762 if (flags & PMAP_KMPAGE) { 3763 KASSERT(md->urw_mappings == 0); 3764 KASSERT(md->uro_mappings == 0); 3765 KASSERT(md->krw_mappings == 0); 3766 KASSERT(md->kro_mappings == 0); 3767 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3768 KASSERT(pv == NULL); 3769 KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0); 3770 KASSERT((md->pvh_attrs & PVF_NC) == 0); 3771 /* if there is a color conflict, evict from cache. */ 3772 if (pmap_is_page_colored_p(md) 3773 && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) { 3774 PMAPCOUNT(vac_color_change); 3775 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 3776 } else if (md->pvh_attrs & PVF_MULTCLR) { 3777 /* 3778 * If this page has multiple colors, expunge 3779 * them. 3780 */ 3781 PMAPCOUNT(vac_flush_lots2); 3782 pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY); 3783 } 3784 /* 3785 * Since this is a KMPAGE, there can be no contention 3786 * for this page so don't lock it. 3787 */ 3788 md->pvh_attrs &= PAGE_SIZE - 1; 3789 md->pvh_attrs |= PVF_KMPAGE | PVF_COLORED | PVF_DIRTY 3790 | (va & arm_cache_prefer_mask); 3791 #else /* !PMAP_CACHE_VIPT || ARM_MMU_EXTENDED */ 3792 md->pvh_attrs |= PVF_KMPAGE; 3793 #endif 3794 atomic_inc_32(&pmap_kmpages); 3795 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3796 } else if (arm_cache_prefer_mask != 0) { 3797 if (pv == NULL) { 3798 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3799 KASSERT(pv != NULL); 3800 } 3801 pmap_acquire_page_lock(md); 3802 pmap_enter_pv(md, pa, pv, pmap_kernel(), va, 3803 PVF_WIRED | PVF_KENTRY 3804 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3805 if ((prot & VM_PROT_WRITE) 3806 && !(md->pvh_attrs & PVF_NC)) 3807 md->pvh_attrs |= PVF_DIRTY; 3808 KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3809 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 3810 pmap_release_page_lock(md); 3811 #endif 3812 } 3813 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3814 } else { 3815 if (pv != NULL) 3816 pool_put(&pmap_pv_pool, pv); 3817 #endif 3818 } 3819 if (pmap_initialized) { 3820 UVMHIST_LOG(maphist, " <-- done (ptep %#jx: %#jx -> %#jx)", 3821 (uintptr_t)ptep, opte, npte, 0); 3822 } 3823 3824 } 3825 3826 void 3827 pmap_kremove(vaddr_t va, vsize_t len) 3828 { 3829 #ifdef UVMHIST 3830 u_int total_mappings = 0; 3831 #endif 3832 3833 PMAPCOUNT(kenter_unmappings); 3834 3835 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3836 3837 UVMHIST_LOG(maphist, " (va=%#jx, len=%#jx)", va, len, 0, 0); 3838 3839 const vaddr_t eva = va + len; 3840 pmap_t kpm = pmap_kernel(); 3841 3842 pmap_acquire_pmap_lock(kpm); 3843 3844 while (va < eva) { 3845 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(va); 3846 if (next_bucket > eva) 3847 next_bucket = eva; 3848 3849 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3850 KDASSERT(l2b != NULL); 3851 3852 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 3853 pt_entry_t *ptep = sptep; 3854 u_int mappings = 0; 3855 3856 while (va < next_bucket) { 3857 const pt_entry_t opte = *ptep; 3858 struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3859 if (opg != NULL) { 3860 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3861 3862 if (omd->pvh_attrs & PVF_KMPAGE) { 3863 KASSERT(omd->urw_mappings == 0); 3864 KASSERT(omd->uro_mappings == 0); 3865 KASSERT(omd->krw_mappings == 0); 3866 KASSERT(omd->kro_mappings == 0); 3867 omd->pvh_attrs &= ~PVF_KMPAGE; 3868 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3869 if (arm_cache_prefer_mask != 0) { 3870 omd->pvh_attrs &= ~PVF_WRITE; 3871 } 3872 #endif 3873 atomic_dec_32(&pmap_kmpages); 3874 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3875 } else if (arm_cache_prefer_mask != 0) { 3876 pmap_acquire_page_lock(omd); 3877 pool_put(&pmap_pv_pool, 3878 pmap_kremove_pg(opg, va)); 3879 pmap_release_page_lock(omd); 3880 #endif 3881 } 3882 } 3883 if (l2pte_valid_p(opte)) { 3884 l2pte_reset(ptep); 3885 PTE_SYNC(ptep); 3886 #ifdef PMAP_CACHE_VIVT 3887 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3888 #endif 3889 cpu_tlb_flushD_SE(va); 3890 3891 mappings += PAGE_SIZE / L2_S_SIZE; 3892 } 3893 va += PAGE_SIZE; 3894 ptep += PAGE_SIZE / L2_S_SIZE; 3895 } 3896 KDASSERTMSG(mappings <= l2b->l2b_occupancy, "%u %u", 3897 mappings, l2b->l2b_occupancy); 3898 l2b->l2b_occupancy -= mappings; 3899 //PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3900 #ifdef UVMHIST 3901 total_mappings += mappings; 3902 #endif 3903 } 3904 pmap_release_pmap_lock(kpm); 3905 cpu_cpwait(); 3906 UVMHIST_LOG(maphist, " <--- done (%ju mappings removed)", 3907 total_mappings, 0, 0, 0); 3908 } 3909 3910 bool 3911 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3912 { 3913 3914 return pmap_extract_coherency(pm, va, pap, NULL); 3915 } 3916 3917 bool 3918 pmap_extract_coherency(pmap_t pm, vaddr_t va, paddr_t *pap, bool *coherentp) 3919 { 3920 struct l2_dtable *l2; 3921 pd_entry_t *pdep, pde; 3922 pt_entry_t *ptep, pte; 3923 paddr_t pa; 3924 u_int l1slot; 3925 bool coherent; 3926 3927 pmap_acquire_pmap_lock(pm); 3928 3929 l1slot = l1pte_index(va); 3930 pdep = pmap_l1_kva(pm) + l1slot; 3931 pde = *pdep; 3932 3933 if (l1pte_section_p(pde)) { 3934 /* 3935 * These should only happen for pmap_kernel() 3936 */ 3937 KDASSERT(pm == pmap_kernel()); 3938 pmap_release_pmap_lock(pm); 3939 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 3940 if (l1pte_supersection_p(pde)) { 3941 pa = (pde & L1_SS_FRAME) | (va & L1_SS_OFFSET); 3942 } else 3943 #endif 3944 pa = (pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3945 coherent = (pde & L1_S_CACHE_MASK) == 0; 3946 } else { 3947 /* 3948 * Note that we can't rely on the validity of the L1 3949 * descriptor as an indication that a mapping exists. 3950 * We have to look it up in the L2 dtable. 3951 */ 3952 l2 = pm->pm_l2[L2_IDX(l1slot)]; 3953 3954 if (l2 == NULL || 3955 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 3956 pmap_release_pmap_lock(pm); 3957 return false; 3958 } 3959 3960 pte = ptep[l2pte_index(va)]; 3961 pmap_release_pmap_lock(pm); 3962 3963 if (pte == 0) 3964 return false; 3965 3966 switch (pte & L2_TYPE_MASK) { 3967 case L2_TYPE_L: 3968 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3969 coherent = (pte & L2_L_CACHE_MASK) == 0; 3970 break; 3971 3972 default: 3973 pa = (pte & ~PAGE_MASK) | (va & PAGE_MASK); 3974 coherent = (pte & L2_S_CACHE_MASK) == 0; 3975 break; 3976 } 3977 } 3978 3979 if (pap != NULL) 3980 *pap = pa; 3981 3982 if (coherentp != NULL) 3983 *coherentp = (pm == pmap_kernel() && coherent); 3984 3985 return true; 3986 } 3987 3988 /* 3989 * pmap_pv_remove: remove an unmanaged pv-tracked page from all pmaps 3990 * that map it 3991 */ 3992 3993 static void 3994 pmap_pv_remove(paddr_t pa) 3995 { 3996 struct pmap_page *pp; 3997 3998 pp = pmap_pv_tracked(pa); 3999 if (pp == NULL) 4000 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 4001 pa); 4002 4003 struct vm_page_md *md = PMAP_PAGE_TO_MD(pp); 4004 pmap_page_remove(md, pa); 4005 } 4006 4007 void 4008 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 4009 { 4010 4011 /* the only case is remove at the moment */ 4012 KASSERT(prot == VM_PROT_NONE); 4013 pmap_pv_remove(pa); 4014 } 4015 4016 void 4017 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 4018 { 4019 struct l2_bucket *l2b; 4020 vaddr_t next_bucket; 4021 4022 NPDEBUG(PDB_PROTECT, 4023 printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n", 4024 pm, sva, eva, prot)); 4025 4026 if ((prot & VM_PROT_READ) == 0) { 4027 pmap_remove(pm, sva, eva); 4028 return; 4029 } 4030 4031 if (prot & VM_PROT_WRITE) { 4032 /* 4033 * If this is a read->write transition, just ignore it and let 4034 * uvm_fault() take care of it later. 4035 */ 4036 return; 4037 } 4038 4039 pmap_acquire_pmap_lock(pm); 4040 4041 #ifndef ARM_MMU_EXTENDED 4042 const bool flush = eva - sva >= PAGE_SIZE * 4; 4043 u_int flags = 0; 4044 #endif 4045 u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 4046 4047 while (sva < eva) { 4048 next_bucket = L2_NEXT_BUCKET_VA(sva); 4049 if (next_bucket > eva) 4050 next_bucket = eva; 4051 4052 l2b = pmap_get_l2_bucket(pm, sva); 4053 if (l2b == NULL) { 4054 sva = next_bucket; 4055 continue; 4056 } 4057 4058 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4059 4060 while (sva < next_bucket) { 4061 const pt_entry_t opte = *ptep; 4062 if (l2pte_valid_p(opte) && l2pte_writable_p(opte)) { 4063 struct vm_page *pg; 4064 #ifndef ARM_MMU_EXTENDED 4065 u_int f; 4066 #endif 4067 4068 #ifdef PMAP_CACHE_VIVT 4069 /* 4070 * OK, at this point, we know we're doing 4071 * write-protect operation. If the pmap is 4072 * active, write-back the page. 4073 */ 4074 pmap_cache_wbinv_page(pm, sva, false, 4075 PVF_REF | PVF_WRITE); 4076 #endif 4077 4078 pg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 4079 pt_entry_t npte = l2pte_set_readonly(opte); 4080 l2pte_reset(ptep); 4081 PTE_SYNC(ptep); 4082 #ifdef ARM_MMU_EXTENDED 4083 pmap_tlb_flush_SE(pm, sva, PVF_REF); 4084 #endif 4085 l2pte_set(ptep, npte, 0); 4086 PTE_SYNC(ptep); 4087 4088 if (pg != NULL) { 4089 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4090 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4091 4092 pmap_acquire_page_lock(md); 4093 #ifndef ARM_MMU_EXTENDED 4094 f = 4095 #endif 4096 pmap_modify_pv(md, pa, pm, sva, 4097 clr_mask, 0); 4098 pmap_vac_me_harder(md, pa, pm, sva); 4099 pmap_release_page_lock(md); 4100 #ifndef ARM_MMU_EXTENDED 4101 } else { 4102 f = PVF_REF | PVF_EXEC; 4103 } 4104 4105 if (flush) { 4106 flags |= f; 4107 } else { 4108 pmap_tlb_flush_SE(pm, sva, f); 4109 #endif 4110 } 4111 } 4112 4113 sva += PAGE_SIZE; 4114 ptep += PAGE_SIZE / L2_S_SIZE; 4115 } 4116 } 4117 4118 #ifndef ARM_MMU_EXTENDED 4119 if (flush) { 4120 if (PV_BEEN_EXECD(flags)) { 4121 pmap_tlb_flushID(pm); 4122 } else if (PV_BEEN_REFD(flags)) { 4123 pmap_tlb_flushD(pm); 4124 } 4125 } 4126 #endif 4127 4128 pmap_release_pmap_lock(pm); 4129 } 4130 4131 void 4132 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 4133 { 4134 struct l2_bucket *l2b; 4135 pt_entry_t *ptep; 4136 vaddr_t next_bucket; 4137 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 4138 4139 NPDEBUG(PDB_EXEC, 4140 printf("pmap_icache_sync_range: pm %p sva 0x%lx eva 0x%lx\n", 4141 pm, sva, eva)); 4142 4143 pmap_acquire_pmap_lock(pm); 4144 4145 while (sva < eva) { 4146 next_bucket = L2_NEXT_BUCKET_VA(sva); 4147 if (next_bucket > eva) 4148 next_bucket = eva; 4149 4150 l2b = pmap_get_l2_bucket(pm, sva); 4151 if (l2b == NULL) { 4152 sva = next_bucket; 4153 continue; 4154 } 4155 4156 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4157 sva < next_bucket; 4158 sva += page_size, 4159 ptep += PAGE_SIZE / L2_S_SIZE, 4160 page_size = PAGE_SIZE) { 4161 if (l2pte_valid_p(*ptep)) { 4162 cpu_icache_sync_range(sva, 4163 uimin(page_size, eva - sva)); 4164 } 4165 } 4166 } 4167 4168 pmap_release_pmap_lock(pm); 4169 } 4170 4171 void 4172 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 4173 { 4174 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4175 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4176 4177 NPDEBUG(PDB_PROTECT, 4178 printf("pmap_page_protect: md %p (0x%08lx), prot 0x%x\n", 4179 md, pa, prot)); 4180 4181 switch(prot) { 4182 case VM_PROT_READ|VM_PROT_WRITE: 4183 #if defined(ARM_MMU_EXTENDED) 4184 pmap_acquire_page_lock(md); 4185 pmap_clearbit(md, pa, PVF_EXEC); 4186 pmap_release_page_lock(md); 4187 break; 4188 #endif 4189 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 4190 break; 4191 4192 case VM_PROT_READ: 4193 #if defined(ARM_MMU_EXTENDED) 4194 pmap_acquire_page_lock(md); 4195 pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC); 4196 pmap_release_page_lock(md); 4197 break; 4198 #endif 4199 case VM_PROT_READ|VM_PROT_EXECUTE: 4200 pmap_acquire_page_lock(md); 4201 pmap_clearbit(md, pa, PVF_WRITE); 4202 pmap_release_page_lock(md); 4203 break; 4204 4205 default: 4206 pmap_page_remove(md, pa); 4207 break; 4208 } 4209 } 4210 4211 /* 4212 * pmap_clear_modify: 4213 * 4214 * Clear the "modified" attribute for a page. 4215 */ 4216 bool 4217 pmap_clear_modify(struct vm_page *pg) 4218 { 4219 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4220 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4221 bool rv; 4222 4223 pmap_acquire_page_lock(md); 4224 4225 if (md->pvh_attrs & PVF_MOD) { 4226 rv = true; 4227 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4228 /* 4229 * If we are going to clear the modified bit and there are 4230 * no other modified bits set, flush the page to memory and 4231 * mark it clean. 4232 */ 4233 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD) 4234 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 4235 #endif 4236 pmap_clearbit(md, pa, PVF_MOD); 4237 } else { 4238 rv = false; 4239 } 4240 pmap_release_page_lock(md); 4241 4242 return rv; 4243 } 4244 4245 /* 4246 * pmap_clear_reference: 4247 * 4248 * Clear the "referenced" attribute for a page. 4249 */ 4250 bool 4251 pmap_clear_reference(struct vm_page *pg) 4252 { 4253 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4254 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4255 bool rv; 4256 4257 pmap_acquire_page_lock(md); 4258 4259 if (md->pvh_attrs & PVF_REF) { 4260 rv = true; 4261 pmap_clearbit(md, pa, PVF_REF); 4262 } else { 4263 rv = false; 4264 } 4265 pmap_release_page_lock(md); 4266 4267 return rv; 4268 } 4269 4270 /* 4271 * pmap_is_modified: 4272 * 4273 * Test if a page has the "modified" attribute. 4274 */ 4275 /* See <arm/arm32/pmap.h> */ 4276 4277 /* 4278 * pmap_is_referenced: 4279 * 4280 * Test if a page has the "referenced" attribute. 4281 */ 4282 /* See <arm/arm32/pmap.h> */ 4283 4284 #if defined(ARM_MMU_EXTENDED) && 0 4285 int 4286 pmap_prefetchabt_fixup(void *v) 4287 { 4288 struct trapframe * const tf = v; 4289 vaddr_t va = trunc_page(tf->tf_pc); 4290 int rv = ABORT_FIXUP_FAILED; 4291 4292 if (!TRAP_USERMODE(tf) && va < VM_MAXUSER_ADDRESS) 4293 return rv; 4294 4295 kpreempt_disable(); 4296 pmap_t pm = curcpu()->ci_pmap_cur; 4297 const size_t l1slot = l1pte_index(va); 4298 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 4299 if (l2 == NULL) 4300 goto out; 4301 4302 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4303 if (l2b->l2b_kva == NULL) 4304 goto out; 4305 4306 /* 4307 * Check the PTE itself. 4308 */ 4309 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4310 const pt_entry_t opte = *ptep; 4311 if ((opte & L2_S_PROT_U) == 0 || (opte & L2_XS_XN) == 0) 4312 goto out; 4313 4314 paddr_t pa = l2pte_pa(opte); 4315 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4316 KASSERT(pg != NULL); 4317 4318 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4319 4320 pmap_acquire_page_lock(md); 4321 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4322 KASSERT(pv != NULL); 4323 4324 if (PV_IS_EXEC_P(pv->pv_flags)) { 4325 l2pte_reset(ptep); 4326 PTE_SYNC(ptep); 4327 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4328 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 4329 pmap_syncicache_page(md, pa); 4330 } 4331 rv = ABORT_FIXUP_RETURN; 4332 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4333 PTE_SYNC(ptep); 4334 } 4335 pmap_release_page_lock(md); 4336 4337 out: 4338 kpreempt_enable(); 4339 return rv; 4340 } 4341 #endif 4342 4343 int 4344 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 4345 { 4346 struct l2_dtable *l2; 4347 struct l2_bucket *l2b; 4348 paddr_t pa; 4349 const size_t l1slot = l1pte_index(va); 4350 int rv = 0; 4351 4352 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4353 4354 va = trunc_page(va); 4355 4356 KASSERT(!user || (pm != pmap_kernel())); 4357 4358 UVMHIST_LOG(maphist, " (pm=%#jx, va=%#jx, ftype=%#jx, user=%jd)", 4359 (uintptr_t)pm, va, ftype, user); 4360 #ifdef ARM_MMU_EXTENDED 4361 UVMHIST_LOG(maphist, " ti=%#jx pai=%#jx asid=%#jx", 4362 (uintptr_t)cpu_tlb_info(curcpu()), 4363 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu())), 4364 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu()))->pai_asid, 0); 4365 #endif 4366 4367 pmap_acquire_pmap_lock(pm); 4368 4369 /* 4370 * If there is no l2_dtable for this address, then the process 4371 * has no business accessing it. 4372 * 4373 * Note: This will catch userland processes trying to access 4374 * kernel addresses. 4375 */ 4376 l2 = pm->pm_l2[L2_IDX(l1slot)]; 4377 if (l2 == NULL) { 4378 UVMHIST_LOG(maphist, " no l2 for l1slot %#jx", l1slot, 0, 0, 0); 4379 goto out; 4380 } 4381 4382 /* 4383 * Likewise if there is no L2 descriptor table 4384 */ 4385 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4386 if (l2b->l2b_kva == NULL) { 4387 UVMHIST_LOG(maphist, " <-- done (no ptep for l1slot %#jx)", 4388 l1slot, 0, 0, 0); 4389 goto out; 4390 } 4391 4392 /* 4393 * Check the PTE itself. 4394 */ 4395 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4396 pt_entry_t const opte = *ptep; 4397 if (opte == 0 || (opte & L2_TYPE_MASK) == L2_TYPE_L) { 4398 UVMHIST_LOG(maphist, " <-- done (empty pde for l1slot %#jx)", 4399 l1slot, 0, 0, 0); 4400 goto out; 4401 } 4402 4403 #ifndef ARM_HAS_VBAR 4404 /* 4405 * Catch a userland access to the vector page mapped at 0x0 4406 */ 4407 if (user && (opte & L2_S_PROT_U) == 0) { 4408 UVMHIST_LOG(maphist, " <-- done (vector_page)", 0, 0, 0, 0); 4409 goto out; 4410 } 4411 #endif 4412 4413 pa = l2pte_pa(opte); 4414 4415 if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(opte)) { 4416 /* 4417 * This looks like a good candidate for "page modified" 4418 * emulation... 4419 */ 4420 struct pv_entry *pv; 4421 struct vm_page *pg; 4422 4423 /* Extract the physical address of the page */ 4424 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4425 UVMHIST_LOG(maphist, " <-- done (mod/ref unmanaged page)", 0, 0, 0, 0); 4426 goto out; 4427 } 4428 4429 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4430 4431 /* Get the current flags for this page. */ 4432 pmap_acquire_page_lock(md); 4433 pv = pmap_find_pv(md, pm, va); 4434 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4435 pmap_release_page_lock(md); 4436 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: no PV)", 0, 0, 0, 0); 4437 goto out; 4438 } 4439 4440 /* 4441 * Do the flags say this page is writable? If not then it 4442 * is a genuine write fault. If yes then the write fault is 4443 * our fault as we did not reflect the write access in the 4444 * PTE. Now we know a write has occurred we can correct this 4445 * and also set the modified bit 4446 */ 4447 if ((pv->pv_flags & PVF_WRITE) == 0) { 4448 pmap_release_page_lock(md); 4449 goto out; 4450 } 4451 4452 md->pvh_attrs |= PVF_REF | PVF_MOD; 4453 pv->pv_flags |= PVF_REF | PVF_MOD; 4454 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4455 /* 4456 * If there are cacheable mappings for this page, mark it dirty. 4457 */ 4458 if ((md->pvh_attrs & PVF_NC) == 0) 4459 md->pvh_attrs |= PVF_DIRTY; 4460 #endif 4461 #ifdef ARM_MMU_EXTENDED 4462 if (md->pvh_attrs & PVF_EXEC) { 4463 md->pvh_attrs &= ~PVF_EXEC; 4464 PMAPCOUNT(exec_discarded_modfixup); 4465 } 4466 #endif 4467 pmap_release_page_lock(md); 4468 4469 /* 4470 * Re-enable write permissions for the page. No need to call 4471 * pmap_vac_me_harder(), since this is just a 4472 * modified-emulation fault, and the PVF_WRITE bit isn't 4473 * changing. We've already set the cacheable bits based on 4474 * the assumption that we can write to this page. 4475 */ 4476 const pt_entry_t npte = 4477 l2pte_set_writable((opte & ~L2_TYPE_MASK) | L2_S_PROTO) 4478 #ifdef ARM_MMU_EXTENDED 4479 | (pm != pmap_kernel() ? L2_XS_nG : 0) 4480 #endif 4481 | 0; 4482 l2pte_reset(ptep); 4483 PTE_SYNC(ptep); 4484 pmap_tlb_flush_SE(pm, va, 4485 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4486 l2pte_set(ptep, npte, 0); 4487 PTE_SYNC(ptep); 4488 PMAPCOUNT(fixup_mod); 4489 rv = 1; 4490 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: changed pte " 4491 "from %#jx to %#jx)", opte, npte, 0, 0); 4492 } else if ((opte & L2_TYPE_MASK) == L2_TYPE_INV) { 4493 /* 4494 * This looks like a good candidate for "page referenced" 4495 * emulation. 4496 */ 4497 struct vm_page *pg; 4498 4499 /* Extract the physical address of the page */ 4500 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4501 UVMHIST_LOG(maphist, " <-- done (ref emul: unmanaged page)", 0, 0, 0, 0); 4502 goto out; 4503 } 4504 4505 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4506 4507 /* Get the current flags for this page. */ 4508 pmap_acquire_page_lock(md); 4509 struct pv_entry *pv = pmap_find_pv(md, pm, va); 4510 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4511 pmap_release_page_lock(md); 4512 UVMHIST_LOG(maphist, " <-- done (ref emul no PV)", 0, 0, 0, 0); 4513 goto out; 4514 } 4515 4516 md->pvh_attrs |= PVF_REF; 4517 pv->pv_flags |= PVF_REF; 4518 4519 pt_entry_t npte = 4520 l2pte_set_readonly((opte & ~L2_TYPE_MASK) | L2_S_PROTO); 4521 #ifdef ARM_MMU_EXTENDED 4522 if (pm != pmap_kernel()) { 4523 npte |= L2_XS_nG; 4524 } 4525 /* 4526 * If we got called from prefetch abort, then ftype will have 4527 * VM_PROT_EXECUTE set. Now see if we have no-execute set in 4528 * the PTE. 4529 */ 4530 if (user && (ftype & VM_PROT_EXECUTE) && (npte & L2_XS_XN)) { 4531 /* 4532 * Is this a mapping of an executable page? 4533 */ 4534 if ((pv->pv_flags & PVF_EXEC) == 0) { 4535 pmap_release_page_lock(md); 4536 UVMHIST_LOG(maphist, " <-- done (ref emul: no exec)", 4537 0, 0, 0, 0); 4538 goto out; 4539 } 4540 /* 4541 * If we haven't synced the page, do so now. 4542 */ 4543 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4544 UVMHIST_LOG(maphist, " ref emul: syncicache " 4545 "page #%#jx", pa, 0, 0, 0); 4546 pmap_syncicache_page(md, pa); 4547 PMAPCOUNT(fixup_exec); 4548 } 4549 npte &= ~L2_XS_XN; 4550 } 4551 #endif /* ARM_MMU_EXTENDED */ 4552 pmap_release_page_lock(md); 4553 l2pte_reset(ptep); 4554 PTE_SYNC(ptep); 4555 pmap_tlb_flush_SE(pm, va, 4556 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4557 l2pte_set(ptep, npte, 0); 4558 PTE_SYNC(ptep); 4559 PMAPCOUNT(fixup_ref); 4560 rv = 1; 4561 UVMHIST_LOG(maphist, " <-- done (ref emul: changed pte from " 4562 "%#jx to %#jx)", opte, npte, 0, 0); 4563 #ifdef ARM_MMU_EXTENDED 4564 } else if (user && (ftype & VM_PROT_EXECUTE) && (opte & L2_XS_XN)) { 4565 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4566 if (pg == NULL) { 4567 UVMHIST_LOG(maphist, " <-- done (unmanaged page)", 0, 0, 0, 0); 4568 goto out; 4569 } 4570 4571 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4572 4573 /* Get the current flags for this page. */ 4574 pmap_acquire_page_lock(md); 4575 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4576 if (pv == NULL || (pv->pv_flags & PVF_EXEC) == 0) { 4577 pmap_release_page_lock(md); 4578 UVMHIST_LOG(maphist, " <-- done (no PV or not EXEC)", 0, 0, 0, 0); 4579 goto out; 4580 } 4581 4582 /* 4583 * If we haven't synced the page, do so now. 4584 */ 4585 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4586 UVMHIST_LOG(maphist, "syncicache page #%#jx", 4587 pa, 0, 0, 0); 4588 pmap_syncicache_page(md, pa); 4589 } 4590 pmap_release_page_lock(md); 4591 /* 4592 * Turn off no-execute. 4593 */ 4594 KASSERT(opte & L2_XS_nG); 4595 l2pte_reset(ptep); 4596 PTE_SYNC(ptep); 4597 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4598 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4599 PTE_SYNC(ptep); 4600 rv = 1; 4601 PMAPCOUNT(fixup_exec); 4602 UVMHIST_LOG(maphist, "exec: changed pte from %#jx to %#jx", 4603 opte, opte & ~L2_XS_XN, 0, 0); 4604 #endif 4605 } 4606 4607 #ifndef ARM_MMU_EXTENDED 4608 /* 4609 * We know there is a valid mapping here, so simply 4610 * fix up the L1 if necessary. 4611 */ 4612 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 4613 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa | L1_C_DOM(pmap_domain(pm)); 4614 if (*pdep != pde) { 4615 l1pte_setone(pdep, pde); 4616 PDE_SYNC(pdep); 4617 rv = 1; 4618 PMAPCOUNT(fixup_pdes); 4619 } 4620 #endif 4621 4622 #ifdef CPU_SA110 4623 /* 4624 * There are bugs in the rev K SA110. This is a check for one 4625 * of them. 4626 */ 4627 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 4628 curcpu()->ci_arm_cpurev < 3) { 4629 /* Always current pmap */ 4630 if (l2pte_valid_p(opte)) { 4631 extern int kernel_debug; 4632 if (kernel_debug & 1) { 4633 struct proc *p = curlwp->l_proc; 4634 printf("prefetch_abort: page is already " 4635 "mapped - pte=%p *pte=%08x\n", ptep, opte); 4636 printf("prefetch_abort: pc=%08lx proc=%p " 4637 "process=%s\n", va, p, p->p_comm); 4638 printf("prefetch_abort: far=%08x fs=%x\n", 4639 cpu_faultaddress(), cpu_faultstatus()); 4640 } 4641 #ifdef DDB 4642 if (kernel_debug & 2) 4643 Debugger(); 4644 #endif 4645 rv = 1; 4646 } 4647 } 4648 #endif /* CPU_SA110 */ 4649 4650 #ifndef ARM_MMU_EXTENDED 4651 /* 4652 * If 'rv == 0' at this point, it generally indicates that there is a 4653 * stale TLB entry for the faulting address. That might be due to a 4654 * wrong setting of pmap_needs_pte_sync. So set it and retry. 4655 */ 4656 if (rv == 0 4657 && pm->pm_l1->l1_domain_use_count == 1 4658 && pmap_needs_pte_sync == 0) { 4659 pmap_needs_pte_sync = 1; 4660 PTE_SYNC(ptep); 4661 PMAPCOUNT(fixup_ptesync); 4662 rv = 1; 4663 } 4664 #endif 4665 4666 #ifndef MULTIPROCESSOR 4667 #if defined(DEBUG) || 1 4668 /* 4669 * If 'rv == 0' at this point, it generally indicates that there is a 4670 * stale TLB entry for the faulting address. This happens when two or 4671 * more processes are sharing an L1. Since we don't flush the TLB on 4672 * a context switch between such processes, we can take domain faults 4673 * for mappings which exist at the same VA in both processes. EVEN IF 4674 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 4675 * example. 4676 * 4677 * This is extremely likely to happen if pmap_enter() updated the L1 4678 * entry for a recently entered mapping. In this case, the TLB is 4679 * flushed for the new mapping, but there may still be TLB entries for 4680 * other mappings belonging to other processes in the 1MB range 4681 * covered by the L1 entry. 4682 * 4683 * Since 'rv == 0', we know that the L1 already contains the correct 4684 * value, so the fault must be due to a stale TLB entry. 4685 * 4686 * Since we always need to flush the TLB anyway in the case where we 4687 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 4688 * stale TLB entries dynamically. 4689 * 4690 * However, the above condition can ONLY happen if the current L1 is 4691 * being shared. If it happens when the L1 is unshared, it indicates 4692 * that other parts of the pmap are not doing their job WRT managing 4693 * the TLB. 4694 */ 4695 if (rv == 0 4696 #ifndef ARM_MMU_EXTENDED 4697 && pm->pm_l1->l1_domain_use_count == 1 4698 #endif 4699 && true) { 4700 #ifdef DEBUG 4701 extern int last_fault_code; 4702 #else 4703 int last_fault_code = ftype & VM_PROT_EXECUTE 4704 ? armreg_ifsr_read() 4705 : armreg_dfsr_read(); 4706 #endif 4707 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 4708 pm, va, ftype); 4709 printf("fixup: l2 %p, l2b %p, ptep %p, pte %#x\n", 4710 l2, l2b, ptep, opte); 4711 4712 #ifndef ARM_MMU_EXTENDED 4713 printf("fixup: pdep %p, pde %#x, fsr %#x\n", 4714 pdep, pde, last_fault_code); 4715 #else 4716 printf("fixup: pdep %p, pde %#x, ttbcr %#x\n", 4717 &pmap_l1_kva(pm)[l1slot], pmap_l1_kva(pm)[l1slot], 4718 armreg_ttbcr_read()); 4719 printf("fixup: fsr %#x cpm %p casid %#x contextidr %#x dacr %#x\n", 4720 last_fault_code, curcpu()->ci_pmap_cur, 4721 curcpu()->ci_pmap_asid_cur, 4722 armreg_contextidr_read(), armreg_dacr_read()); 4723 #ifdef _ARM_ARCH_7 4724 if (ftype & VM_PROT_WRITE) 4725 armreg_ats1cuw_write(va); 4726 else 4727 armreg_ats1cur_write(va); 4728 arm_isb(); 4729 printf("fixup: par %#x\n", armreg_par_read()); 4730 #endif 4731 #endif 4732 #ifdef DDB 4733 extern int kernel_debug; 4734 4735 if (kernel_debug & 2) { 4736 pmap_release_pmap_lock(pm); 4737 #ifdef UVMHIST 4738 KERNHIST_DUMP(maphist); 4739 #endif 4740 cpu_Debugger(); 4741 pmap_acquire_pmap_lock(pm); 4742 } 4743 #endif 4744 } 4745 #endif 4746 #endif 4747 4748 #ifndef ARM_MMU_EXTENDED 4749 /* Flush the TLB in the shared L1 case - see comment above */ 4750 pmap_tlb_flush_SE(pm, va, 4751 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4752 #endif 4753 4754 rv = 1; 4755 4756 out: 4757 pmap_release_pmap_lock(pm); 4758 4759 return (rv); 4760 } 4761 4762 /* 4763 * Routine: pmap_procwr 4764 * 4765 * Function: 4766 * Synchronize caches corresponding to [addr, addr+len) in p. 4767 * 4768 */ 4769 void 4770 pmap_procwr(struct proc *p, vaddr_t va, int len) 4771 { 4772 #ifndef ARM_MMU_EXTENDED 4773 4774 /* We only need to do anything if it is the current process. */ 4775 if (p == curproc) 4776 cpu_icache_sync_range(va, len); 4777 #endif 4778 } 4779 4780 /* 4781 * Routine: pmap_unwire 4782 * Function: Clear the wired attribute for a map/virtual-address pair. 4783 * 4784 * In/out conditions: 4785 * The mapping must already exist in the pmap. 4786 */ 4787 void 4788 pmap_unwire(pmap_t pm, vaddr_t va) 4789 { 4790 struct l2_bucket *l2b; 4791 pt_entry_t *ptep, pte; 4792 struct vm_page *pg; 4793 paddr_t pa; 4794 4795 NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va)); 4796 4797 pmap_acquire_pmap_lock(pm); 4798 4799 l2b = pmap_get_l2_bucket(pm, va); 4800 KDASSERT(l2b != NULL); 4801 4802 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4803 pte = *ptep; 4804 4805 /* Extract the physical address of the page */ 4806 pa = l2pte_pa(pte); 4807 4808 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4809 /* Update the wired bit in the pv entry for this page. */ 4810 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4811 4812 pmap_acquire_page_lock(md); 4813 (void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0); 4814 pmap_release_page_lock(md); 4815 } 4816 4817 pmap_release_pmap_lock(pm); 4818 } 4819 4820 #ifdef ARM_MMU_EXTENDED 4821 void 4822 pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) 4823 { 4824 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4825 4826 /* 4827 * Assume that TTBR1 has only global mappings and TTBR0 only 4828 * has non-global mappings. To prevent speculation from doing 4829 * evil things we disable translation table walks using TTBR0 4830 * before setting the CONTEXTIDR (ASID) or new TTBR0 value. 4831 * Once both are set, table walks are reenabled. 4832 */ 4833 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4834 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4835 arm_isb(); 4836 4837 pmap_tlb_asid_acquire(pm, l); 4838 4839 struct cpu_info * const ci = curcpu(); 4840 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); 4841 4842 cpu_setttb(pm->pm_l1_pa, pai->pai_asid); 4843 /* 4844 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 4845 * have been updated. 4846 */ 4847 arm_isb(); 4848 4849 if (pm != pmap_kernel()) { 4850 armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); 4851 } 4852 cpu_cpwait(); 4853 4854 UVMHIST_LOG(maphist, " pm %#jx pm->pm_l1_pa %08jx asid %ju... done", 4855 (uintptr_t)pm, pm->pm_l1_pa, pai->pai_asid, 0); 4856 4857 KASSERTMSG(ci->ci_pmap_asid_cur == pai->pai_asid, "%u vs %u", 4858 ci->ci_pmap_asid_cur, pai->pai_asid); 4859 ci->ci_pmap_cur = pm; 4860 } 4861 4862 void 4863 pmap_md_pdetab_deactivate(pmap_t pm) 4864 { 4865 4866 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4867 4868 kpreempt_disable(); 4869 struct cpu_info * const ci = curcpu(); 4870 /* 4871 * Disable translation table walks from TTBR0 while no pmap has been 4872 * activated. 4873 */ 4874 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4875 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4876 arm_isb(); 4877 pmap_tlb_asid_deactivate(pm); 4878 cpu_setttb(pmap_kernel()->pm_l1_pa, KERNEL_PID); 4879 arm_isb(); 4880 4881 ci->ci_pmap_cur = pmap_kernel(); 4882 KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", 4883 ci->ci_pmap_asid_cur); 4884 kpreempt_enable(); 4885 } 4886 #endif 4887 4888 void 4889 pmap_activate(struct lwp *l) 4890 { 4891 extern int block_userspace_access; 4892 pmap_t npm = l->l_proc->p_vmspace->vm_map.pmap; 4893 4894 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4895 4896 UVMHIST_LOG(maphist, "(l=%#jx) pm=%#jx", (uintptr_t)l, (uintptr_t)npm, 4897 0, 0); 4898 4899 struct cpu_info * const ci = curcpu(); 4900 4901 /* 4902 * If activating a non-current lwp or the current lwp is 4903 * already active, just return. 4904 */ 4905 if (false 4906 || l != curlwp 4907 #ifdef ARM_MMU_EXTENDED 4908 || (ci->ci_pmap_cur == npm && 4909 (npm == pmap_kernel() 4910 /* || PMAP_PAI_ASIDVALID_P(pai, cpu_tlb_info(ci)) */)) 4911 #else 4912 || npm->pm_activated == true 4913 #endif 4914 || false) { 4915 UVMHIST_LOG(maphist, " <-- (same pmap)", (uintptr_t)curlwp, 4916 (uintptr_t)l, 0, 0); 4917 return; 4918 } 4919 4920 #ifndef ARM_MMU_EXTENDED 4921 const uint32_t ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4922 | (DOMAIN_CLIENT << (pmap_domain(npm) * 2)); 4923 4924 /* 4925 * If TTB and DACR are unchanged, short-circuit all the 4926 * TLB/cache management stuff. 4927 */ 4928 pmap_t opm = ci->ci_lastlwp 4929 ? ci->ci_lastlwp->l_proc->p_vmspace->vm_map.pmap 4930 : NULL; 4931 if (opm != NULL) { 4932 uint32_t odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4933 | (DOMAIN_CLIENT << (pmap_domain(opm) * 2)); 4934 4935 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4936 goto all_done; 4937 } 4938 #endif /* !ARM_MMU_EXTENDED */ 4939 4940 PMAPCOUNT(activations); 4941 block_userspace_access = 1; 4942 4943 #ifndef ARM_MMU_EXTENDED 4944 /* 4945 * If switching to a user vmspace which is different to the 4946 * most recent one, and the most recent one is potentially 4947 * live in the cache, we must write-back and invalidate the 4948 * entire cache. 4949 */ 4950 pmap_t rpm = ci->ci_pmap_lastuser; 4951 4952 /* 4953 * XXXSCW: There's a corner case here which can leave turds in the 4954 * cache as reported in kern/41058. They're probably left over during 4955 * tear-down and switching away from an exiting process. Until the root 4956 * cause is identified and fixed, zap the cache when switching pmaps. 4957 * This will result in a few unnecessary cache flushes, but that's 4958 * better than silently corrupting data. 4959 */ 4960 #if 0 4961 if (npm != pmap_kernel() && rpm && npm != rpm && 4962 rpm->pm_cstate.cs_cache) { 4963 rpm->pm_cstate.cs_cache = 0; 4964 #ifdef PMAP_CACHE_VIVT 4965 cpu_idcache_wbinv_all(); 4966 #endif 4967 } 4968 #else 4969 if (rpm) { 4970 rpm->pm_cstate.cs_cache = 0; 4971 if (npm == pmap_kernel()) 4972 ci->ci_pmap_lastuser = NULL; 4973 #ifdef PMAP_CACHE_VIVT 4974 cpu_idcache_wbinv_all(); 4975 #endif 4976 } 4977 #endif 4978 4979 /* No interrupts while we frob the TTB/DACR */ 4980 uint32_t oldirqstate = disable_interrupts(IF32_bits); 4981 #endif /* !ARM_MMU_EXTENDED */ 4982 4983 #ifndef ARM_HAS_VBAR 4984 /* 4985 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 4986 * entry corresponding to 'vector_page' in the incoming L1 table 4987 * before switching to it otherwise subsequent interrupts/exceptions 4988 * (including domain faults!) will jump into hyperspace. 4989 */ 4990 if (npm->pm_pl1vec != NULL) { 4991 cpu_tlb_flushID_SE((u_int)vector_page); 4992 cpu_cpwait(); 4993 *npm->pm_pl1vec = npm->pm_l1vec; 4994 PTE_SYNC(npm->pm_pl1vec); 4995 } 4996 #endif 4997 4998 #ifdef ARM_MMU_EXTENDED 4999 pmap_md_pdetab_activate(npm, l); 5000 #else 5001 cpu_domains(ndacr); 5002 if (npm == pmap_kernel() || npm == rpm) { 5003 /* 5004 * Switching to a kernel thread, or back to the 5005 * same user vmspace as before... Simply update 5006 * the TTB (no TLB flush required) 5007 */ 5008 cpu_setttb(npm->pm_l1->l1_physaddr, false); 5009 cpu_cpwait(); 5010 } else { 5011 /* 5012 * Otherwise, update TTB and flush TLB 5013 */ 5014 cpu_context_switch(npm->pm_l1->l1_physaddr); 5015 if (rpm != NULL) 5016 rpm->pm_cstate.cs_tlb = 0; 5017 } 5018 5019 restore_interrupts(oldirqstate); 5020 #endif /* ARM_MMU_EXTENDED */ 5021 5022 block_userspace_access = 0; 5023 5024 #ifndef ARM_MMU_EXTENDED 5025 all_done: 5026 /* 5027 * The new pmap is resident. Make sure it's marked 5028 * as resident in the cache/TLB. 5029 */ 5030 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5031 if (npm != pmap_kernel()) 5032 ci->ci_pmap_lastuser = npm; 5033 5034 /* The old pmap is not longer active */ 5035 if (opm != npm) { 5036 if (opm != NULL) 5037 opm->pm_activated = false; 5038 5039 /* But the new one is */ 5040 npm->pm_activated = true; 5041 } 5042 ci->ci_pmap_cur = npm; 5043 #endif 5044 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5045 } 5046 5047 void 5048 pmap_deactivate(struct lwp *l) 5049 { 5050 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 5051 5052 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5053 5054 UVMHIST_LOG(maphist, "(l=%#jx) pm=%#jx", (uintptr_t)l, (uintptr_t)pm, 5055 0, 0); 5056 5057 #ifdef ARM_MMU_EXTENDED 5058 pmap_md_pdetab_deactivate(pm); 5059 #else 5060 /* 5061 * If the process is exiting, make sure pmap_activate() does 5062 * a full MMU context-switch and cache flush, which we might 5063 * otherwise skip. See PR port-arm/38950. 5064 */ 5065 if (l->l_proc->p_sflag & PS_WEXIT) 5066 curcpu()->ci_lastlwp = NULL; 5067 5068 pm->pm_activated = false; 5069 #endif 5070 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5071 } 5072 5073 void 5074 pmap_update(pmap_t pm) 5075 { 5076 5077 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5078 5079 UVMHIST_LOG(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5080 pm->pm_remove_all, 0, 0); 5081 5082 #ifndef ARM_MMU_EXTENDED 5083 if (pm->pm_remove_all) { 5084 /* 5085 * Finish up the pmap_remove_all() optimisation by flushing 5086 * the TLB. 5087 */ 5088 pmap_tlb_flushID(pm); 5089 pm->pm_remove_all = false; 5090 } 5091 5092 if (pmap_is_current(pm)) { 5093 /* 5094 * If we're dealing with a current userland pmap, move its L1 5095 * to the end of the LRU. 5096 */ 5097 if (pm != pmap_kernel()) 5098 pmap_use_l1(pm); 5099 5100 /* 5101 * We can assume we're done with frobbing the cache/tlb for 5102 * now. Make sure any future pmap ops don't skip cache/tlb 5103 * flushes. 5104 */ 5105 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5106 } 5107 #else 5108 5109 kpreempt_disable(); 5110 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 5111 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 5112 if (pending && pmap_tlb_shootdown_bystanders(pmap)) { 5113 PMAP_COUNT(shootdown_ipis); 5114 } 5115 #endif 5116 5117 /* 5118 * If pmap_remove_all was called, we deactivated ourselves and released 5119 * our ASID. Now we have to reactivate ourselves. 5120 */ 5121 if (__predict_false(pm->pm_remove_all)) { 5122 pm->pm_remove_all = false; 5123 5124 KASSERT(pm != pmap_kernel()); 5125 pmap_md_pdetab_activate(pm, curlwp); 5126 } 5127 5128 if (arm_has_mpext_p) 5129 armreg_bpiallis_write(0); 5130 else 5131 armreg_bpiall_write(0); 5132 5133 kpreempt_enable(); 5134 5135 KASSERTMSG(pm == pmap_kernel() 5136 || curcpu()->ci_pmap_cur != pm 5137 || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, 5138 "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, 5139 pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, 5140 curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur); 5141 #endif 5142 5143 PMAPCOUNT(updates); 5144 5145 /* 5146 * make sure TLB/cache operations have completed. 5147 */ 5148 cpu_cpwait(); 5149 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5150 } 5151 5152 void 5153 pmap_remove_all(pmap_t pm) 5154 { 5155 5156 /* 5157 * The vmspace described by this pmap is about to be torn down. 5158 * Until pmap_update() is called, UVM will only make calls 5159 * to pmap_remove(). We can make life much simpler by flushing 5160 * the cache now, and deferring TLB invalidation to pmap_update(). 5161 */ 5162 #ifdef PMAP_CACHE_VIVT 5163 pmap_cache_wbinv_all(pm, PVF_EXEC); 5164 #endif 5165 #ifdef ARM_MMU_EXTENDED 5166 #ifdef MULTIPROCESSOR 5167 struct cpu_info * const ci = curcpu(); 5168 // This should be the last CPU with this pmap onproc 5169 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 5170 if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) 5171 #endif 5172 pmap_tlb_asid_deactivate(pm); 5173 #ifdef MULTIPROCESSOR 5174 KASSERT(kcpuset_iszero(pm->pm_onproc)); 5175 #endif 5176 5177 pmap_tlb_asid_release_all(pm); 5178 #endif 5179 pm->pm_remove_all = true; 5180 } 5181 5182 /* 5183 * Retire the given physical map from service. 5184 * Should only be called if the map contains no valid mappings. 5185 */ 5186 void 5187 pmap_destroy(pmap_t pm) 5188 { 5189 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5190 5191 u_int count; 5192 5193 if (pm == NULL) 5194 return; 5195 5196 UVMHIST_LOG(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5197 pm->pm_remove_all, 0, 0); 5198 5199 if (pm->pm_remove_all) { 5200 #ifdef ARM_MMU_EXTENDED 5201 pmap_tlb_asid_release_all(pm); 5202 #else 5203 pmap_tlb_flushID(pm); 5204 #endif 5205 pm->pm_remove_all = false; 5206 } 5207 5208 /* 5209 * Drop reference count 5210 */ 5211 mutex_enter(pm->pm_lock); 5212 count = --pm->pm_obj.uo_refs; 5213 mutex_exit(pm->pm_lock); 5214 if (count > 0) { 5215 #ifndef ARM_MMU_EXTENDED 5216 if (pmap_is_current(pm)) { 5217 if (pm != pmap_kernel()) 5218 pmap_use_l1(pm); 5219 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5220 } 5221 #endif 5222 return; 5223 } 5224 5225 /* 5226 * reference count is zero, free pmap resources and then free pmap. 5227 */ 5228 5229 #ifndef ARM_HAS_VBAR 5230 if (vector_page < KERNEL_BASE) { 5231 KDASSERT(!pmap_is_current(pm)); 5232 5233 /* Remove the vector page mapping */ 5234 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 5235 pmap_update(pm); 5236 } 5237 #endif 5238 5239 pmap_free_l1(pm); 5240 5241 #ifdef ARM_MMU_EXTENDED 5242 #ifdef MULTIPROCESSOR 5243 kcpuset_destroy(pm->pm_active); 5244 kcpuset_destroy(pm->pm_onproc); 5245 #endif 5246 #else 5247 struct cpu_info * const ci = curcpu(); 5248 if (ci->ci_pmap_lastuser == pm) 5249 ci->ci_pmap_lastuser = NULL; 5250 #endif 5251 5252 uvm_obj_destroy(&pm->pm_obj, false); 5253 mutex_destroy(&pm->pm_obj_lock); 5254 pool_cache_put(&pmap_cache, pm); 5255 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5256 } 5257 5258 5259 /* 5260 * void pmap_reference(pmap_t pm) 5261 * 5262 * Add a reference to the specified pmap. 5263 */ 5264 void 5265 pmap_reference(pmap_t pm) 5266 { 5267 5268 if (pm == NULL) 5269 return; 5270 5271 #ifndef ARM_MMU_EXTENDED 5272 pmap_use_l1(pm); 5273 #endif 5274 5275 mutex_enter(pm->pm_lock); 5276 pm->pm_obj.uo_refs++; 5277 mutex_exit(pm->pm_lock); 5278 } 5279 5280 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 5281 5282 static struct evcnt pmap_prefer_nochange_ev = 5283 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 5284 static struct evcnt pmap_prefer_change_ev = 5285 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 5286 5287 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 5288 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 5289 5290 void 5291 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 5292 { 5293 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 5294 vaddr_t va = *vap; 5295 vaddr_t diff = (hint - va) & mask; 5296 if (diff == 0) { 5297 pmap_prefer_nochange_ev.ev_count++; 5298 } else { 5299 pmap_prefer_change_ev.ev_count++; 5300 if (__predict_false(td)) 5301 va -= mask + 1; 5302 *vap = va + diff; 5303 } 5304 } 5305 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */ 5306 5307 /* 5308 * pmap_zero_page() 5309 * 5310 * Zero a given physical page by mapping it at a page hook point. 5311 * In doing the zero page op, the page we zero is mapped cachable, as with 5312 * StrongARM accesses to non-cached pages are non-burst making writing 5313 * _any_ bulk data very slow. 5314 */ 5315 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5316 void 5317 pmap_zero_page_generic(paddr_t pa) 5318 { 5319 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5320 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5321 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5322 #endif 5323 #if defined(PMAP_CACHE_VIPT) 5324 /* Choose the last page color it had, if any */ 5325 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5326 #else 5327 const vsize_t va_offset = 0; 5328 #endif 5329 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5330 /* 5331 * Is this page mapped at its natural color? 5332 * If we have all of memory mapped, then just convert PA to VA. 5333 */ 5334 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5335 || va_offset == (pa & arm_cache_prefer_mask); 5336 const vaddr_t vdstp = okcolor 5337 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5338 : cpu_cdstp(va_offset); 5339 #else 5340 const bool okcolor = false; 5341 const vaddr_t vdstp = cpu_cdstp(va_offset); 5342 #endif 5343 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5344 5345 5346 #ifdef DEBUG 5347 if (!SLIST_EMPTY(&md->pvh_list)) 5348 panic("pmap_zero_page: page has mappings"); 5349 #endif 5350 5351 KDASSERT((pa & PGOFSET) == 0); 5352 5353 if (!okcolor) { 5354 /* 5355 * Hook in the page, zero it, and purge the cache for that 5356 * zeroed page. Invalidate the TLB as needed. 5357 */ 5358 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 5359 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); 5360 l2pte_set(ptep, npte, 0); 5361 PTE_SYNC(ptep); 5362 cpu_tlb_flushD_SE(vdstp); 5363 cpu_cpwait(); 5364 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) \ 5365 && !defined(ARM_MMU_EXTENDED) 5366 /* 5367 * If we are direct-mapped and our color isn't ok, then before 5368 * we bzero the page invalidate its contents from the cache and 5369 * reset the color to its natural color. 5370 */ 5371 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5372 md->pvh_attrs &= ~arm_cache_prefer_mask; 5373 md->pvh_attrs |= (pa & arm_cache_prefer_mask); 5374 #endif 5375 } 5376 bzero_page(vdstp); 5377 if (!okcolor) { 5378 /* 5379 * Unmap the page. 5380 */ 5381 l2pte_reset(ptep); 5382 PTE_SYNC(ptep); 5383 cpu_tlb_flushD_SE(vdstp); 5384 #ifdef PMAP_CACHE_VIVT 5385 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5386 #endif 5387 } 5388 #ifdef PMAP_CACHE_VIPT 5389 /* 5390 * This page is now cache resident so it now has a page color. 5391 * Any contents have been obliterated so clear the EXEC flag. 5392 */ 5393 #ifndef ARM_MMU_EXTENDED 5394 if (!pmap_is_page_colored_p(md)) { 5395 PMAPCOUNT(vac_color_new); 5396 md->pvh_attrs |= PVF_COLORED; 5397 } 5398 md->pvh_attrs |= PVF_DIRTY; 5399 #endif 5400 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5401 md->pvh_attrs &= ~PVF_EXEC; 5402 PMAPCOUNT(exec_discarded_zero); 5403 } 5404 #endif 5405 } 5406 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5407 5408 #if ARM_MMU_XSCALE == 1 5409 void 5410 pmap_zero_page_xscale(paddr_t pa) 5411 { 5412 #ifdef DEBUG 5413 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5414 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5415 5416 if (!SLIST_EMPTY(&md->pvh_list)) 5417 panic("pmap_zero_page: page has mappings"); 5418 #endif 5419 5420 KDASSERT((pa & PGOFSET) == 0); 5421 5422 /* 5423 * Hook in the page, zero it, and purge the cache for that 5424 * zeroed page. Invalidate the TLB as needed. 5425 */ 5426 5427 pt_entry_t npte = L2_S_PROTO | pa | 5428 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 5429 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5430 l2pte_set(cdst_pte, npte, 0); 5431 PTE_SYNC(cdst_pte); 5432 cpu_tlb_flushD_SE(cdstp); 5433 cpu_cpwait(); 5434 bzero_page(cdstp); 5435 xscale_cache_clean_minidata(); 5436 l2pte_reset(cdst_pte); 5437 PTE_SYNC(cdst_pte); 5438 } 5439 #endif /* ARM_MMU_XSCALE == 1 */ 5440 5441 /* pmap_pageidlezero() 5442 * 5443 * The same as above, except that we assume that the page is not 5444 * mapped. This means we never have to flush the cache first. Called 5445 * from the idle loop. 5446 */ 5447 bool 5448 pmap_pageidlezero(paddr_t pa) 5449 { 5450 bool rv = true; 5451 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5452 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 5453 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5454 #endif 5455 #ifdef PMAP_CACHE_VIPT 5456 /* Choose the last page color it had, if any */ 5457 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5458 #else 5459 const vsize_t va_offset = 0; 5460 #endif 5461 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 5462 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5463 || va_offset == (pa & arm_cache_prefer_mask); 5464 const vaddr_t vdstp = okcolor 5465 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5466 : cpu_cdstp(va_offset); 5467 #else 5468 const bool okcolor = false; 5469 const vaddr_t vdstp = cpu_cdstp(va_offset); 5470 #endif 5471 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5472 5473 5474 #ifdef DEBUG 5475 if (!SLIST_EMPTY(&md->pvh_list)) 5476 panic("pmap_pageidlezero: page has mappings"); 5477 #endif 5478 5479 KDASSERT((pa & PGOFSET) == 0); 5480 5481 if (!okcolor) { 5482 /* 5483 * Hook in the page, zero it, and purge the cache for that 5484 * zeroed page. Invalidate the TLB as needed. 5485 */ 5486 const pt_entry_t npte = L2_S_PROTO | pa | 5487 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5488 l2pte_set(ptep, npte, 0); 5489 PTE_SYNC(ptep); 5490 cpu_tlb_flushD_SE(vdstp); 5491 cpu_cpwait(); 5492 } 5493 5494 uint64_t *ptr = (uint64_t *)vdstp; 5495 for (size_t i = 0; i < PAGE_SIZE / sizeof(*ptr); i++) { 5496 if (sched_curcpu_runnable_p() != 0) { 5497 /* 5498 * A process has become ready. Abort now, 5499 * so we don't keep it waiting while we 5500 * do slow memory access to finish this 5501 * page. 5502 */ 5503 rv = false; 5504 break; 5505 } 5506 *ptr++ = 0; 5507 } 5508 5509 #ifdef PMAP_CACHE_VIVT 5510 if (rv) 5511 /* 5512 * if we aborted we'll rezero this page again later so don't 5513 * purge it unless we finished it 5514 */ 5515 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5516 #elif defined(PMAP_CACHE_VIPT) 5517 /* 5518 * This page is now cache resident so it now has a page color. 5519 * Any contents have been obliterated so clear the EXEC flag. 5520 */ 5521 #ifndef ARM_MMU_EXTENDED 5522 if (!pmap_is_page_colored_p(md)) { 5523 PMAPCOUNT(vac_color_new); 5524 md->pvh_attrs |= PVF_COLORED; 5525 } 5526 #endif 5527 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5528 md->pvh_attrs &= ~PVF_EXEC; 5529 PMAPCOUNT(exec_discarded_zero); 5530 } 5531 #endif 5532 /* 5533 * Unmap the page. 5534 */ 5535 if (!okcolor) { 5536 l2pte_reset(ptep); 5537 PTE_SYNC(ptep); 5538 cpu_tlb_flushD_SE(vdstp); 5539 } 5540 5541 return rv; 5542 } 5543 5544 /* 5545 * pmap_copy_page() 5546 * 5547 * Copy one physical page into another, by mapping the pages into 5548 * hook points. The same comment regarding cachability as in 5549 * pmap_zero_page also applies here. 5550 */ 5551 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5552 void 5553 pmap_copy_page_generic(paddr_t src, paddr_t dst) 5554 { 5555 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 5556 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5557 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5558 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 5559 struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg); 5560 #endif 5561 #ifdef PMAP_CACHE_VIPT 5562 const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask; 5563 const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask; 5564 #else 5565 const vsize_t src_va_offset = 0; 5566 const vsize_t dst_va_offset = 0; 5567 #endif 5568 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5569 /* 5570 * Is this page mapped at its natural color? 5571 * If we have all of memory mapped, then just convert PA to VA. 5572 */ 5573 bool src_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5574 || src_va_offset == (src & arm_cache_prefer_mask); 5575 bool dst_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5576 || dst_va_offset == (dst & arm_cache_prefer_mask); 5577 const vaddr_t vsrcp = src_okcolor 5578 ? pmap_direct_mapped_phys(src, &src_okcolor, 5579 cpu_csrcp(src_va_offset)) 5580 : cpu_csrcp(src_va_offset); 5581 const vaddr_t vdstp = pmap_direct_mapped_phys(dst, &dst_okcolor, 5582 cpu_cdstp(dst_va_offset)); 5583 #else 5584 const bool src_okcolor = false; 5585 const bool dst_okcolor = false; 5586 const vaddr_t vsrcp = cpu_csrcp(src_va_offset); 5587 const vaddr_t vdstp = cpu_cdstp(dst_va_offset); 5588 #endif 5589 pt_entry_t * const src_ptep = cpu_csrc_pte(src_va_offset); 5590 pt_entry_t * const dst_ptep = cpu_cdst_pte(dst_va_offset); 5591 5592 #ifdef DEBUG 5593 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5594 panic("pmap_copy_page: dst page has mappings"); 5595 #endif 5596 5597 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5598 KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC)); 5599 #endif 5600 KDASSERT((src & PGOFSET) == 0); 5601 KDASSERT((dst & PGOFSET) == 0); 5602 5603 /* 5604 * Clean the source page. Hold the source page's lock for 5605 * the duration of the copy so that no other mappings can 5606 * be created while we have a potentially aliased mapping. 5607 */ 5608 #ifdef PMAP_CACHE_VIVT 5609 pmap_acquire_page_lock(src_md); 5610 (void) pmap_clean_page(src_md, true); 5611 pmap_release_page_lock(src_md); 5612 #endif 5613 5614 /* 5615 * Map the pages into the page hook points, copy them, and purge 5616 * the cache for the appropriate page. Invalidate the TLB 5617 * as required. 5618 */ 5619 if (!src_okcolor) { 5620 const pt_entry_t nsrc_pte = L2_S_PROTO 5621 | src 5622 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5623 | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 5624 #else // defined(PMAP_CACHE_VIVT) || defined(ARM_MMU_EXTENDED) 5625 | pte_l2_s_cache_mode 5626 #endif 5627 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 5628 l2pte_set(src_ptep, nsrc_pte, 0); 5629 PTE_SYNC(src_ptep); 5630 cpu_tlb_flushD_SE(vsrcp); 5631 cpu_cpwait(); 5632 } 5633 if (!dst_okcolor) { 5634 const pt_entry_t ndst_pte = L2_S_PROTO | dst | 5635 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5636 l2pte_set(dst_ptep, ndst_pte, 0); 5637 PTE_SYNC(dst_ptep); 5638 cpu_tlb_flushD_SE(vdstp); 5639 cpu_cpwait(); 5640 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 5641 /* 5642 * If we are direct-mapped and our color isn't ok, then before 5643 * we bcopy to the new page invalidate its contents from the 5644 * cache and reset its color to its natural color. 5645 */ 5646 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5647 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5648 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5649 #endif 5650 } 5651 bcopy_page(vsrcp, vdstp); 5652 #ifdef PMAP_CACHE_VIVT 5653 cpu_dcache_inv_range(vsrcp, PAGE_SIZE); 5654 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5655 #endif 5656 /* 5657 * Unmap the pages. 5658 */ 5659 if (!src_okcolor) { 5660 l2pte_reset(src_ptep); 5661 PTE_SYNC(src_ptep); 5662 cpu_tlb_flushD_SE(vsrcp); 5663 cpu_cpwait(); 5664 } 5665 if (!dst_okcolor) { 5666 l2pte_reset(dst_ptep); 5667 PTE_SYNC(dst_ptep); 5668 cpu_tlb_flushD_SE(vdstp); 5669 cpu_cpwait(); 5670 } 5671 #ifdef PMAP_CACHE_VIPT 5672 /* 5673 * Now that the destination page is in the cache, mark it as colored. 5674 * If this was an exec page, discard it. 5675 */ 5676 pmap_acquire_page_lock(dst_md); 5677 #ifndef ARM_MMU_EXTENDED 5678 if (arm_pcache.cache_type == CACHE_TYPE_PIPT) { 5679 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5680 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5681 } 5682 if (!pmap_is_page_colored_p(dst_md)) { 5683 PMAPCOUNT(vac_color_new); 5684 dst_md->pvh_attrs |= PVF_COLORED; 5685 } 5686 dst_md->pvh_attrs |= PVF_DIRTY; 5687 #endif 5688 if (PV_IS_EXEC_P(dst_md->pvh_attrs)) { 5689 dst_md->pvh_attrs &= ~PVF_EXEC; 5690 PMAPCOUNT(exec_discarded_copy); 5691 } 5692 pmap_release_page_lock(dst_md); 5693 #endif 5694 } 5695 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5696 5697 #if ARM_MMU_XSCALE == 1 5698 void 5699 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 5700 { 5701 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 5702 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5703 #ifdef DEBUG 5704 struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)); 5705 5706 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5707 panic("pmap_copy_page: dst page has mappings"); 5708 #endif 5709 5710 KDASSERT((src & PGOFSET) == 0); 5711 KDASSERT((dst & PGOFSET) == 0); 5712 5713 /* 5714 * Clean the source page. Hold the source page's lock for 5715 * the duration of the copy so that no other mappings can 5716 * be created while we have a potentially aliased mapping. 5717 */ 5718 #ifdef PMAP_CACHE_VIVT 5719 pmap_acquire_page_lock(src_md); 5720 (void) pmap_clean_page(src_md, true); 5721 pmap_release_page_lock(src_md); 5722 #endif 5723 5724 /* 5725 * Map the pages into the page hook points, copy them, and purge 5726 * the cache for the appropriate page. Invalidate the TLB 5727 * as required. 5728 */ 5729 const pt_entry_t nsrc_pte = L2_S_PROTO | src 5730 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 5731 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5732 l2pte_set(csrc_pte, nsrc_pte, 0); 5733 PTE_SYNC(csrc_pte); 5734 5735 const pt_entry_t ndst_pte = L2_S_PROTO | dst 5736 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) 5737 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5738 l2pte_set(cdst_pte, ndst_pte, 0); 5739 PTE_SYNC(cdst_pte); 5740 5741 cpu_tlb_flushD_SE(csrcp); 5742 cpu_tlb_flushD_SE(cdstp); 5743 cpu_cpwait(); 5744 bcopy_page(csrcp, cdstp); 5745 xscale_cache_clean_minidata(); 5746 l2pte_reset(csrc_pte); 5747 l2pte_reset(cdst_pte); 5748 PTE_SYNC(csrc_pte); 5749 PTE_SYNC(cdst_pte); 5750 } 5751 #endif /* ARM_MMU_XSCALE == 1 */ 5752 5753 /* 5754 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5755 * 5756 * Return the start and end addresses of the kernel's virtual space. 5757 * These values are setup in pmap_bootstrap and are updated as pages 5758 * are allocated. 5759 */ 5760 void 5761 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5762 { 5763 *start = virtual_avail; 5764 *end = virtual_end; 5765 } 5766 5767 /* 5768 * Helper function for pmap_grow_l2_bucket() 5769 */ 5770 static inline int 5771 pmap_grow_map(vaddr_t va, paddr_t *pap) 5772 { 5773 paddr_t pa; 5774 5775 if (uvm.page_init_done == false) { 5776 #ifdef PMAP_STEAL_MEMORY 5777 pv_addr_t pv; 5778 pmap_boot_pagealloc(PAGE_SIZE, 5779 #ifdef PMAP_CACHE_VIPT 5780 arm_cache_prefer_mask, 5781 va & arm_cache_prefer_mask, 5782 #else 5783 0, 0, 5784 #endif 5785 &pv); 5786 pa = pv.pv_pa; 5787 #else 5788 if (uvm_page_physget(&pa) == false) 5789 return (1); 5790 5791 pmap_kenter_pa(va, pa, 5792 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 5793 #endif /* PMAP_STEAL_MEMORY */ 5794 } else { 5795 struct vm_page *pg; 5796 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 5797 if (pg == NULL) 5798 return (1); 5799 pa = VM_PAGE_TO_PHYS(pg); 5800 /* 5801 * This new page must not have any mappings. Enter it via 5802 * pmap_kenter_pa and let that routine do the hard work. 5803 */ 5804 struct vm_page_md *md __diagused = VM_PAGE_TO_MD(pg); 5805 KASSERT(SLIST_EMPTY(&md->pvh_list)); 5806 pmap_kenter_pa(va, pa, 5807 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 5808 } 5809 5810 if (pap) 5811 *pap = pa; 5812 5813 PMAPCOUNT(pt_mappings); 5814 #ifdef DEBUG 5815 struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5816 KDASSERT(l2b != NULL); 5817 5818 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 5819 const pt_entry_t opte = *ptep; 5820 KDASSERT((opte & L2_S_CACHE_MASK) == pte_l2_s_cache_mode_pt); 5821 #endif 5822 memset((void *)va, 0, PAGE_SIZE); 5823 return (0); 5824 } 5825 5826 /* 5827 * This is the same as pmap_alloc_l2_bucket(), except that it is only 5828 * used by pmap_growkernel(). 5829 */ 5830 static inline struct l2_bucket * 5831 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 5832 { 5833 struct l2_dtable *l2; 5834 struct l2_bucket *l2b; 5835 u_short l1slot; 5836 vaddr_t nva; 5837 5838 l1slot = l1pte_index(va); 5839 5840 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 5841 /* 5842 * No mapping at this address, as there is 5843 * no entry in the L1 table. 5844 * Need to allocate a new l2_dtable. 5845 */ 5846 nva = pmap_kernel_l2dtable_kva; 5847 if ((nva & PGOFSET) == 0) { 5848 /* 5849 * Need to allocate a backing page 5850 */ 5851 if (pmap_grow_map(nva, NULL)) 5852 return (NULL); 5853 } 5854 5855 l2 = (struct l2_dtable *)nva; 5856 nva += sizeof(struct l2_dtable); 5857 5858 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 5859 /* 5860 * The new l2_dtable straddles a page boundary. 5861 * Map in another page to cover it. 5862 */ 5863 if (pmap_grow_map(nva, NULL)) 5864 return (NULL); 5865 } 5866 5867 pmap_kernel_l2dtable_kva = nva; 5868 5869 /* 5870 * Link it into the parent pmap 5871 */ 5872 pm->pm_l2[L2_IDX(l1slot)] = l2; 5873 } 5874 5875 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 5876 5877 /* 5878 * Fetch pointer to the L2 page table associated with the address. 5879 */ 5880 if (l2b->l2b_kva == NULL) { 5881 pt_entry_t *ptep; 5882 5883 /* 5884 * No L2 page table has been allocated. Chances are, this 5885 * is because we just allocated the l2_dtable, above. 5886 */ 5887 nva = pmap_kernel_l2ptp_kva; 5888 ptep = (pt_entry_t *)nva; 5889 if ((nva & PGOFSET) == 0) { 5890 /* 5891 * Need to allocate a backing page 5892 */ 5893 if (pmap_grow_map(nva, &pmap_kernel_l2ptp_phys)) 5894 return (NULL); 5895 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 5896 } 5897 5898 l2->l2_occupancy++; 5899 l2b->l2b_kva = ptep; 5900 l2b->l2b_l1slot = l1slot; 5901 l2b->l2b_pa = pmap_kernel_l2ptp_phys; 5902 5903 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 5904 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 5905 } 5906 5907 return (l2b); 5908 } 5909 5910 vaddr_t 5911 pmap_growkernel(vaddr_t maxkvaddr) 5912 { 5913 pmap_t kpm = pmap_kernel(); 5914 #ifndef ARM_MMU_EXTENDED 5915 struct l1_ttable *l1; 5916 #endif 5917 int s; 5918 5919 if (maxkvaddr <= pmap_curmaxkvaddr) 5920 goto out; /* we are OK */ 5921 5922 NPDEBUG(PDB_GROWKERN, 5923 printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n", 5924 pmap_curmaxkvaddr, maxkvaddr)); 5925 5926 KDASSERT(maxkvaddr <= virtual_end); 5927 5928 /* 5929 * whoops! we need to add kernel PTPs 5930 */ 5931 5932 s = splvm(); /* to be safe */ 5933 mutex_enter(&kpm_lock); 5934 5935 /* Map 1MB at a time */ 5936 size_t l1slot = l1pte_index(pmap_curmaxkvaddr); 5937 #ifdef ARM_MMU_EXTENDED 5938 pd_entry_t * const spdep = &kpm->pm_l1[l1slot]; 5939 pd_entry_t *pdep = spdep; 5940 #endif 5941 for (;pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE, 5942 #ifdef ARM_MMU_EXTENDED 5943 pdep++, 5944 #endif 5945 l1slot++) { 5946 struct l2_bucket *l2b = 5947 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 5948 KASSERT(l2b != NULL); 5949 5950 const pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 5951 | L1_C_DOM(PMAP_DOMAIN_KERNEL); 5952 #ifdef ARM_MMU_EXTENDED 5953 l1pte_setone(pdep, npde); 5954 #else 5955 /* Distribute new L1 entry to all other L1s */ 5956 SLIST_FOREACH(l1, &l1_list, l1_link) { 5957 pd_entry_t * const pdep = &l1->l1_kva[l1slot]; 5958 l1pte_setone(pdep, npde); 5959 PDE_SYNC(pdep); 5960 } 5961 #endif 5962 } 5963 #ifdef ARM_MMU_EXTENDED 5964 PDE_SYNC_RANGE(spdep, pdep - spdep); 5965 #endif 5966 5967 #ifdef PMAP_CACHE_VIVT 5968 /* 5969 * flush out the cache, expensive but growkernel will happen so 5970 * rarely 5971 */ 5972 cpu_dcache_wbinv_all(); 5973 cpu_tlb_flushD(); 5974 cpu_cpwait(); 5975 #endif 5976 5977 mutex_exit(&kpm_lock); 5978 splx(s); 5979 5980 out: 5981 return (pmap_curmaxkvaddr); 5982 } 5983 5984 /************************ Utility routines ****************************/ 5985 5986 #ifndef ARM_HAS_VBAR 5987 /* 5988 * vector_page_setprot: 5989 * 5990 * Manipulate the protection of the vector page. 5991 */ 5992 void 5993 vector_page_setprot(int prot) 5994 { 5995 struct l2_bucket *l2b; 5996 pt_entry_t *ptep; 5997 5998 #if defined(CPU_ARMV7) || defined(CPU_ARM11) 5999 /* 6000 * If we are using VBAR to use the vectors in the kernel, then it's 6001 * already mapped in the kernel text so no need to anything here. 6002 */ 6003 if (vector_page != ARM_VECTORS_LOW && vector_page != ARM_VECTORS_HIGH) { 6004 KASSERT((armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0); 6005 return; 6006 } 6007 #endif 6008 6009 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 6010 KASSERT(l2b != NULL); 6011 6012 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 6013 6014 const pt_entry_t opte = *ptep; 6015 #ifdef ARM_MMU_EXTENDED 6016 const pt_entry_t npte = (opte & ~(L2_S_PROT_MASK|L2_XS_XN)) 6017 | L2_S_PROT(PTE_KERNEL, prot); 6018 #else 6019 const pt_entry_t npte = (opte & ~L2_S_PROT_MASK) 6020 | L2_S_PROT(PTE_KERNEL, prot); 6021 #endif 6022 l2pte_set(ptep, npte, opte); 6023 PTE_SYNC(ptep); 6024 cpu_tlb_flushD_SE(vector_page); 6025 cpu_cpwait(); 6026 } 6027 #endif 6028 6029 /* 6030 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 6031 * Returns true if the mapping exists, else false. 6032 * 6033 * NOTE: This function is only used by a couple of arm-specific modules. 6034 * It is not safe to take any pmap locks here, since we could be right 6035 * in the middle of debugging the pmap anyway... 6036 * 6037 * It is possible for this routine to return false even though a valid 6038 * mapping does exist. This is because we don't lock, so the metadata 6039 * state may be inconsistent. 6040 * 6041 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 6042 * a "section" mapping. 6043 */ 6044 bool 6045 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 6046 { 6047 struct l2_dtable *l2; 6048 pd_entry_t *pdep, pde; 6049 pt_entry_t *ptep; 6050 u_short l1slot; 6051 6052 if (pm->pm_l1 == NULL) 6053 return false; 6054 6055 l1slot = l1pte_index(va); 6056 *pdp = pdep = pmap_l1_kva(pm) + l1slot; 6057 pde = *pdep; 6058 6059 if (l1pte_section_p(pde)) { 6060 *ptp = NULL; 6061 return true; 6062 } 6063 6064 l2 = pm->pm_l2[L2_IDX(l1slot)]; 6065 if (l2 == NULL || 6066 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 6067 return false; 6068 } 6069 6070 *ptp = &ptep[l2pte_index(va)]; 6071 return true; 6072 } 6073 6074 bool 6075 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 6076 { 6077 6078 if (pm->pm_l1 == NULL) 6079 return false; 6080 6081 *pdp = pmap_l1_kva(pm) + l1pte_index(va); 6082 6083 return true; 6084 } 6085 6086 /************************ Bootstrapping routines ****************************/ 6087 6088 #ifndef ARM_MMU_EXTENDED 6089 static void 6090 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 6091 { 6092 int i; 6093 6094 l1->l1_kva = l1pt; 6095 l1->l1_domain_use_count = 0; 6096 l1->l1_domain_first = 0; 6097 6098 for (i = 0; i < PMAP_DOMAINS; i++) 6099 l1->l1_domain_free[i] = i + 1; 6100 6101 /* 6102 * Copy the kernel's L1 entries to each new L1. 6103 */ 6104 if (pmap_initialized) 6105 memcpy(l1pt, pmap_l1_kva(pmap_kernel()), L1_TABLE_SIZE); 6106 6107 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 6108 &l1->l1_physaddr) == false) 6109 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 6110 6111 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 6112 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 6113 } 6114 #endif /* !ARM_MMU_EXTENDED */ 6115 6116 /* 6117 * pmap_bootstrap() is called from the board-specific initarm() routine 6118 * once the kernel L1/L2 descriptors tables have been set up. 6119 * 6120 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 6121 * spread over a number of disparate files/functions. 6122 * 6123 * We are passed the following parameters 6124 * - vstart 6125 * 1MB-aligned start of managed kernel virtual memory. 6126 * - vend 6127 * 1MB-aligned end of managed kernel virtual memory. 6128 * 6129 * We use 'kernel_l1pt' to build the metadata (struct l1_ttable and 6130 * struct l2_dtable) necessary to track kernel mappings. 6131 */ 6132 #define PMAP_STATIC_L2_SIZE 16 6133 void 6134 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 6135 { 6136 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 6137 #ifndef ARM_MMU_EXTENDED 6138 static struct l1_ttable static_l1; 6139 struct l1_ttable *l1 = &static_l1; 6140 #endif 6141 struct l2_dtable *l2; 6142 struct l2_bucket *l2b; 6143 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 6144 pmap_t pm = pmap_kernel(); 6145 pt_entry_t *ptep; 6146 paddr_t pa; 6147 vsize_t size; 6148 int nptes, l2idx, l2next = 0; 6149 6150 #ifdef ARM_MMU_EXTENDED 6151 KASSERT(pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt); 6152 KASSERT(pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt); 6153 #endif 6154 6155 VPRINTF("kpm "); 6156 /* 6157 * Initialise the kernel pmap object 6158 */ 6159 curcpu()->ci_pmap_cur = pm; 6160 #ifdef ARM_MMU_EXTENDED 6161 pm->pm_l1 = l1pt; 6162 pm->pm_l1_pa = kernel_l1pt.pv_pa; 6163 VPRINTF("tlb0 "); 6164 pmap_tlb_info_init(&pmap_tlb0_info); 6165 #ifdef MULTIPROCESSOR 6166 VPRINTF("kcpusets "); 6167 pm->pm_onproc = kcpuset_running; 6168 pm->pm_active = kcpuset_running; 6169 #endif 6170 #else 6171 pm->pm_l1 = l1; 6172 #endif 6173 6174 VPRINTF("locks "); 6175 /* 6176 * pmap_kenter_pa() and pmap_kremove() may be called from interrupt 6177 * context, so its locks have to be at IPL_VM 6178 */ 6179 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_VM); 6180 mutex_init(&kpm_lock, MUTEX_DEFAULT, IPL_NONE); 6181 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_VM); 6182 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 6183 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 6184 6185 VPRINTF("l1pt "); 6186 /* 6187 * Scan the L1 translation table created by initarm() and create 6188 * the required metadata for all valid mappings found in it. 6189 */ 6190 for (size_t l1slot = 0; 6191 l1slot < L1_TABLE_SIZE / sizeof(pd_entry_t); 6192 l1slot++) { 6193 pd_entry_t pde = l1pt[l1slot]; 6194 6195 /* 6196 * We're only interested in Coarse mappings. 6197 * pmap_extract() can deal with section mappings without 6198 * recourse to checking L2 metadata. 6199 */ 6200 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 6201 continue; 6202 6203 /* 6204 * Lookup the KVA of this L2 descriptor table 6205 */ 6206 pa = l1pte_pa(pde); 6207 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6208 if (ptep == NULL) { 6209 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 6210 (u_int)l1slot << L1_S_SHIFT, pa); 6211 } 6212 6213 /* 6214 * Fetch the associated L2 metadata structure. 6215 * Allocate a new one if necessary. 6216 */ 6217 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 6218 if (l2next == PMAP_STATIC_L2_SIZE) 6219 panic("pmap_bootstrap: out of static L2s"); 6220 pm->pm_l2[L2_IDX(l1slot)] = l2 = &static_l2[l2next++]; 6221 } 6222 6223 /* 6224 * One more L1 slot tracked... 6225 */ 6226 l2->l2_occupancy++; 6227 6228 /* 6229 * Fill in the details of the L2 descriptor in the 6230 * appropriate bucket. 6231 */ 6232 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 6233 l2b->l2b_kva = ptep; 6234 l2b->l2b_pa = pa; 6235 l2b->l2b_l1slot = l1slot; 6236 6237 /* 6238 * Establish an initial occupancy count for this descriptor 6239 */ 6240 for (l2idx = 0; 6241 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 6242 l2idx++) { 6243 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 6244 l2b->l2b_occupancy++; 6245 } 6246 } 6247 6248 /* 6249 * Make sure the descriptor itself has the correct cache mode. 6250 * If not, fix it, but whine about the problem. Port-meisters 6251 * should consider this a clue to fix up their initarm() 6252 * function. :) 6253 */ 6254 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep, 1)) { 6255 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6256 "L2 pte @ %p\n", ptep); 6257 } 6258 } 6259 6260 VPRINTF("cache(l1pt) "); 6261 /* 6262 * Ensure the primary (kernel) L1 has the correct cache mode for 6263 * a page table. Bitch if it is not correctly set. 6264 */ 6265 if (pmap_set_pt_cache_mode(l1pt, kernel_l1pt.pv_va, 6266 L1_TABLE_SIZE / L2_S_SIZE)) { 6267 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6268 "primary L1 @ 0x%lx\n", kernel_l1pt.pv_va); 6269 } 6270 6271 #ifdef PMAP_CACHE_VIVT 6272 cpu_dcache_wbinv_all(); 6273 cpu_tlb_flushID(); 6274 cpu_cpwait(); 6275 #endif 6276 6277 /* 6278 * now we allocate the "special" VAs which are used for tmp mappings 6279 * by the pmap (and other modules). we allocate the VAs by advancing 6280 * virtual_avail (note that there are no pages mapped at these VAs). 6281 * 6282 * Managed KVM space start from wherever initarm() tells us. 6283 */ 6284 virtual_avail = vstart; 6285 virtual_end = vend; 6286 6287 VPRINTF("specials "); 6288 #ifdef PMAP_CACHE_VIPT 6289 /* 6290 * If we have a VIPT cache, we need one page/pte per possible alias 6291 * page so we won't violate cache aliasing rules. 6292 */ 6293 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 6294 nptes = (arm_cache_prefer_mask >> L2_S_SHIFT) + 1; 6295 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6296 if (arm_pcache.icache_type != CACHE_TYPE_PIPT 6297 && arm_pcache.icache_way_size > nptes * L2_S_SIZE) { 6298 nptes = arm_pcache.icache_way_size >> L2_S_SHIFT; 6299 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6300 } 6301 #else 6302 nptes = PAGE_SIZE / L2_S_SIZE; 6303 #endif 6304 #ifdef MULTIPROCESSOR 6305 cnptes = nptes; 6306 nptes *= arm_cpu_max; 6307 #endif 6308 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 6309 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte, nptes); 6310 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 6311 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte, nptes); 6312 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 6313 if (msgbufaddr == NULL) { 6314 pmap_alloc_specials(&virtual_avail, 6315 round_page(MSGBUFSIZE) / PAGE_SIZE, 6316 (void *)&msgbufaddr, NULL); 6317 } 6318 6319 /* 6320 * Allocate a range of kernel virtual address space to be used 6321 * for L2 descriptor tables and metadata allocation in 6322 * pmap_growkernel(). 6323 */ 6324 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 6325 pmap_alloc_specials(&virtual_avail, 6326 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 6327 &pmap_kernel_l2ptp_kva, NULL); 6328 6329 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 6330 pmap_alloc_specials(&virtual_avail, 6331 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 6332 &pmap_kernel_l2dtable_kva, NULL); 6333 6334 #ifndef ARM_MMU_EXTENDED 6335 /* 6336 * init the static-global locks and global pmap list. 6337 */ 6338 mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM); 6339 6340 /* 6341 * We can now initialise the first L1's metadata. 6342 */ 6343 SLIST_INIT(&l1_list); 6344 TAILQ_INIT(&l1_lru_list); 6345 pmap_init_l1(l1, l1pt); 6346 #endif /* ARM_MMU_EXTENDED */ 6347 6348 #ifndef ARM_HAS_VBAR 6349 /* Set up vector page L1 details, if necessary */ 6350 if (vector_page < KERNEL_BASE) { 6351 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 6352 l2b = pmap_get_l2_bucket(pm, vector_page); 6353 KDASSERT(l2b != NULL); 6354 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 6355 L1_C_DOM(pmap_domain(pm)); 6356 } else 6357 pm->pm_pl1vec = NULL; 6358 #endif 6359 6360 VPRINTF("pools "); 6361 /* 6362 * Initialize the pmap cache 6363 */ 6364 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 6365 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 6366 6367 /* 6368 * Initialize the pv pool. 6369 */ 6370 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 6371 &pmap_bootstrap_pv_allocator, IPL_NONE); 6372 6373 /* 6374 * Initialize the L2 dtable pool and cache. 6375 */ 6376 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 6377 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 6378 6379 /* 6380 * Initialise the L2 descriptor table pool and cache 6381 */ 6382 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 6383 L2_TABLE_SIZE_REAL, 0, 0, "l2ptppl", NULL, IPL_NONE, 6384 pmap_l2ptp_ctor, NULL, NULL); 6385 6386 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 6387 6388 cpu_dcache_wbinv_all(); 6389 } 6390 6391 static bool 6392 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va, size_t nptes) 6393 { 6394 #ifdef ARM_MMU_EXTENDED 6395 return false; 6396 #else 6397 if (pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 6398 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 6399 return false; 6400 6401 const vaddr_t eva = va + nptes * PAGE_SIZE; 6402 int rv = 0; 6403 6404 while (va < eva) { 6405 /* 6406 * Make sure the descriptor itself has the correct cache mode 6407 */ 6408 pd_entry_t * const pdep = &kl1[l1pte_index(va)]; 6409 pd_entry_t pde = *pdep; 6410 6411 if (l1pte_section_p(pde)) { 6412 KASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0); 6413 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 6414 *pdep = (pde & ~L1_S_CACHE_MASK) | 6415 pte_l1_s_cache_mode_pt; 6416 PDE_SYNC(pdep); 6417 cpu_dcache_wbinv_range((vaddr_t)pdep, 6418 sizeof(*pdep)); 6419 rv = 1; 6420 } 6421 return rv; 6422 } 6423 vaddr_t pa = l1pte_pa(pde); 6424 pt_entry_t *ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6425 if (ptep == NULL) 6426 panic("pmap_bootstrap: No PTP for va %#lx\n", va); 6427 6428 ptep += l2pte_index(va); 6429 const pt_entry_t opte = *ptep; 6430 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 6431 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 6432 | pte_l2_s_cache_mode_pt; 6433 l2pte_set(ptep, npte, opte); 6434 PTE_SYNC(ptep); 6435 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 6436 rv = 1; 6437 } 6438 va += PAGE_SIZE; 6439 } 6440 6441 return (rv); 6442 #endif 6443 } 6444 6445 static void 6446 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 6447 { 6448 vaddr_t va = *availp; 6449 struct l2_bucket *l2b; 6450 6451 if (ptep) { 6452 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6453 if (l2b == NULL) 6454 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 6455 6456 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 6457 } 6458 6459 *vap = va; 6460 *availp = va + (PAGE_SIZE * pages); 6461 } 6462 6463 void 6464 pmap_init(void) 6465 { 6466 6467 /* 6468 * Set the available memory vars - These do not map to real memory 6469 * addresses and cannot as the physical memory is fragmented. 6470 * They are used by ps for %mem calculations. 6471 * One could argue whether this should be the entire memory or just 6472 * the memory that is useable in a user process. 6473 */ 6474 avail_start = ptoa(uvm_physseg_get_avail_start(uvm_physseg_get_first())); 6475 avail_end = ptoa(uvm_physseg_get_avail_end(uvm_physseg_get_last())); 6476 6477 /* 6478 * Now we need to free enough pv_entry structures to allow us to get 6479 * the kmem_map/kmem_object allocated and inited (done after this 6480 * function is finished). to do this we allocate one bootstrap page out 6481 * of kernel_map and use it to provide an initial pool of pv_entry 6482 * structures. we never free this page. 6483 */ 6484 pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 6485 6486 #ifdef ARM_MMU_EXTENDED 6487 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 6488 #endif 6489 6490 pmap_initialized = true; 6491 } 6492 6493 static vaddr_t last_bootstrap_page = 0; 6494 static void *free_bootstrap_pages = NULL; 6495 6496 static void * 6497 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 6498 { 6499 extern void *pool_page_alloc(struct pool *, int); 6500 vaddr_t new_page; 6501 void *rv; 6502 6503 if (pmap_initialized) 6504 return (pool_page_alloc(pp, flags)); 6505 6506 if (free_bootstrap_pages) { 6507 rv = free_bootstrap_pages; 6508 free_bootstrap_pages = *((void **)rv); 6509 return (rv); 6510 } 6511 6512 KASSERT(kernel_map != NULL); 6513 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 6514 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 6515 6516 KASSERT(new_page > last_bootstrap_page); 6517 last_bootstrap_page = new_page; 6518 return ((void *)new_page); 6519 } 6520 6521 static void 6522 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 6523 { 6524 extern void pool_page_free(struct pool *, void *); 6525 6526 if ((vaddr_t)v <= last_bootstrap_page) { 6527 *((void **)v) = free_bootstrap_pages; 6528 free_bootstrap_pages = v; 6529 return; 6530 } 6531 6532 if (pmap_initialized) { 6533 pool_page_free(pp, v); 6534 return; 6535 } 6536 } 6537 6538 /* 6539 * pmap_postinit() 6540 * 6541 * This routine is called after the vm and kmem subsystems have been 6542 * initialised. This allows the pmap code to perform any initialisation 6543 * that can only be done once the memory allocation is in place. 6544 */ 6545 void 6546 pmap_postinit(void) 6547 { 6548 #ifndef ARM_MMU_EXTENDED 6549 extern paddr_t physical_start, physical_end; 6550 struct l1_ttable *l1; 6551 struct pglist plist; 6552 struct vm_page *m; 6553 pd_entry_t *pdep; 6554 vaddr_t va, eva; 6555 u_int loop, needed; 6556 int error; 6557 #endif 6558 6559 pool_cache_setlowat(&pmap_l2ptp_cache, (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 6560 pool_cache_setlowat(&pmap_l2dtable_cache, 6561 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 6562 6563 #ifndef ARM_MMU_EXTENDED 6564 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 6565 needed -= 1; 6566 6567 l1 = kmem_alloc(sizeof(*l1) * needed, KM_SLEEP); 6568 6569 for (loop = 0; loop < needed; loop++, l1++) { 6570 /* Allocate a L1 page table */ 6571 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 6572 if (va == 0) 6573 panic("Cannot allocate L1 KVM"); 6574 6575 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 6576 physical_end, L1_TABLE_SIZE, 0, &plist, 1, 1); 6577 if (error) 6578 panic("Cannot allocate L1 physical pages"); 6579 6580 m = TAILQ_FIRST(&plist); 6581 eva = va + L1_TABLE_SIZE; 6582 pdep = (pd_entry_t *)va; 6583 6584 while (m && va < eva) { 6585 paddr_t pa = VM_PAGE_TO_PHYS(m); 6586 6587 pmap_kenter_pa(va, pa, 6588 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6589 6590 va += PAGE_SIZE; 6591 m = TAILQ_NEXT(m, pageq.queue); 6592 } 6593 6594 #ifdef DIAGNOSTIC 6595 if (m) 6596 panic("pmap_alloc_l1pt: pglist not empty"); 6597 #endif /* DIAGNOSTIC */ 6598 6599 pmap_init_l1(l1, pdep); 6600 } 6601 6602 #ifdef DEBUG 6603 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 6604 needed); 6605 #endif 6606 #endif /* !ARM_MMU_EXTENDED */ 6607 } 6608 6609 /* 6610 * Note that the following routines are used by board-specific initialisation 6611 * code to configure the initial kernel page tables. 6612 * 6613 */ 6614 6615 /* 6616 * This list exists for the benefit of pmap_map_chunk(). It keeps track 6617 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 6618 * find them as necessary. 6619 * 6620 * Note that the data on this list MUST remain valid after initarm() returns, 6621 * as pmap_bootstrap() uses it to contruct L2 table metadata. 6622 */ 6623 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 6624 6625 static vaddr_t 6626 kernel_pt_lookup(paddr_t pa) 6627 { 6628 pv_addr_t *pv; 6629 6630 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 6631 if (pv->pv_pa == (pa & ~PGOFSET)) 6632 return (pv->pv_va | (pa & PGOFSET)); 6633 } 6634 return (0); 6635 } 6636 6637 /* 6638 * pmap_map_section: 6639 * 6640 * Create a single section mapping. 6641 */ 6642 void 6643 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6644 { 6645 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6646 const size_t l1slot = l1pte_index(va); 6647 pd_entry_t fl; 6648 6649 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 6650 6651 switch (cache) { 6652 case PTE_NOCACHE: 6653 default: 6654 fl = 0; 6655 break; 6656 6657 case PTE_CACHE: 6658 fl = pte_l1_s_cache_mode; 6659 break; 6660 6661 case PTE_PAGETABLE: 6662 fl = pte_l1_s_cache_mode_pt; 6663 break; 6664 } 6665 6666 const pd_entry_t npde = L1_S_PROTO | pa | 6667 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6668 l1pte_setone(pdep + l1slot, npde); 6669 PDE_SYNC(pdep + l1slot); 6670 } 6671 6672 /* 6673 * pmap_map_entry: 6674 * 6675 * Create a single page mapping. 6676 */ 6677 void 6678 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6679 { 6680 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6681 const size_t l1slot = l1pte_index(va); 6682 pt_entry_t npte; 6683 pt_entry_t *ptep; 6684 6685 KASSERT(((va | pa) & PGOFSET) == 0); 6686 6687 switch (cache) { 6688 case PTE_NOCACHE: 6689 default: 6690 npte = 0; 6691 break; 6692 6693 case PTE_CACHE: 6694 npte = pte_l2_s_cache_mode; 6695 break; 6696 6697 case PTE_PAGETABLE: 6698 npte = pte_l2_s_cache_mode_pt; 6699 break; 6700 } 6701 6702 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6703 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 6704 6705 ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6706 if (ptep == NULL) 6707 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 6708 6709 npte |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 6710 #ifdef ARM_MMU_EXTENDED 6711 if (prot & VM_PROT_EXECUTE) { 6712 npte &= ~L2_XS_XN; 6713 } 6714 #endif 6715 ptep += l2pte_index(va); 6716 l2pte_set(ptep, npte, 0); 6717 PTE_SYNC(ptep); 6718 } 6719 6720 /* 6721 * pmap_link_l2pt: 6722 * 6723 * Link the L2 page table specified by "l2pv" into the L1 6724 * page table at the slot for "va". 6725 */ 6726 void 6727 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 6728 { 6729 pd_entry_t * const pdep = (pd_entry_t *) l1pt + l1pte_index(va); 6730 6731 KASSERT((va & ((L1_S_SIZE * (PAGE_SIZE / L2_T_SIZE)) - 1)) == 0); 6732 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 6733 6734 const pd_entry_t npde = L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO 6735 | l2pv->pv_pa; 6736 6737 l1pte_set(pdep, npde); 6738 PDE_SYNC_RANGE(pdep, PAGE_SIZE / L2_T_SIZE); 6739 6740 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 6741 } 6742 6743 /* 6744 * pmap_map_chunk: 6745 * 6746 * Map a chunk of memory using the most efficient mappings 6747 * possible (section, large page, small page) into the 6748 * provided L1 and L2 tables at the specified virtual address. 6749 */ 6750 vsize_t 6751 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 6752 int prot, int cache) 6753 { 6754 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6755 pt_entry_t f1, f2s, f2l; 6756 vsize_t resid; 6757 6758 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 6759 6760 if (l1pt == 0) 6761 panic("pmap_map_chunk: no L1 table provided"); 6762 6763 // VPRINTF("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 6764 // "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 6765 6766 switch (cache) { 6767 case PTE_NOCACHE: 6768 default: 6769 f1 = 0; 6770 f2l = 0; 6771 f2s = 0; 6772 break; 6773 6774 case PTE_CACHE: 6775 f1 = pte_l1_s_cache_mode; 6776 f2l = pte_l2_l_cache_mode; 6777 f2s = pte_l2_s_cache_mode; 6778 break; 6779 6780 case PTE_PAGETABLE: 6781 f1 = pte_l1_s_cache_mode_pt; 6782 f2l = pte_l2_l_cache_mode_pt; 6783 f2s = pte_l2_s_cache_mode_pt; 6784 break; 6785 } 6786 6787 size = resid; 6788 6789 while (resid > 0) { 6790 const size_t l1slot = l1pte_index(va); 6791 #ifdef ARM_MMU_EXTENDED 6792 /* See if we can use a supersection mapping. */ 6793 if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) { 6794 /* Supersection are always domain 0 */ 6795 const pd_entry_t npde = L1_SS_PROTO | pa 6796 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6797 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6798 | L1_S_PROT(PTE_KERNEL, prot) | f1; 6799 VPRINTF("sS"); 6800 l1pte_set(&pdep[l1slot], npde); 6801 PDE_SYNC_RANGE(&pdep[l1slot], L1_SS_SIZE / L1_S_SIZE); 6802 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6803 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6804 va += L1_SS_SIZE; 6805 pa += L1_SS_SIZE; 6806 resid -= L1_SS_SIZE; 6807 continue; 6808 } 6809 #endif 6810 /* See if we can use a section mapping. */ 6811 if (L1_S_MAPPABLE_P(va, pa, resid)) { 6812 const pd_entry_t npde = L1_S_PROTO | pa 6813 #ifdef ARM_MMU_EXTENDED 6814 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6815 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6816 #endif 6817 | L1_S_PROT(PTE_KERNEL, prot) | f1 6818 | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6819 VPRINTF("S"); 6820 l1pte_set(&pdep[l1slot], npde); 6821 PDE_SYNC(&pdep[l1slot]); 6822 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6823 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6824 va += L1_S_SIZE; 6825 pa += L1_S_SIZE; 6826 resid -= L1_S_SIZE; 6827 continue; 6828 } 6829 6830 /* 6831 * Ok, we're going to use an L2 table. Make sure 6832 * one is actually in the corresponding L1 slot 6833 * for the current VA. 6834 */ 6835 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6836 panic("%s: no L2 table for VA %#lx", __func__, va); 6837 6838 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6839 if (ptep == NULL) 6840 panic("%s: can't find L2 table for VA %#lx", __func__, 6841 va); 6842 6843 ptep += l2pte_index(va); 6844 6845 /* See if we can use a L2 large page mapping. */ 6846 if (L2_L_MAPPABLE_P(va, pa, resid)) { 6847 const pt_entry_t npte = L2_L_PROTO | pa 6848 #ifdef ARM_MMU_EXTENDED 6849 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_L_XN) 6850 | (va & 0x80000000 ? 0 : L2_XS_nG) 6851 #endif 6852 | L2_L_PROT(PTE_KERNEL, prot) | f2l; 6853 VPRINTF("L"); 6854 l2pte_set(ptep, npte, 0); 6855 PTE_SYNC_RANGE(ptep, L2_L_SIZE / L2_S_SIZE); 6856 va += L2_L_SIZE; 6857 pa += L2_L_SIZE; 6858 resid -= L2_L_SIZE; 6859 continue; 6860 } 6861 6862 VPRINTF("P"); 6863 /* Use a small page mapping. */ 6864 pt_entry_t npte = L2_S_PROTO | pa 6865 #ifdef ARM_MMU_EXTENDED 6866 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_XN) 6867 | (va & 0x80000000 ? 0 : L2_XS_nG) 6868 #endif 6869 | L2_S_PROT(PTE_KERNEL, prot) | f2s; 6870 #ifdef ARM_MMU_EXTENDED 6871 npte &= ((prot & VM_PROT_EXECUTE) ? ~L2_XS_XN : ~0); 6872 #endif 6873 l2pte_set(ptep, npte, 0); 6874 PTE_SYNC(ptep); 6875 va += PAGE_SIZE; 6876 pa += PAGE_SIZE; 6877 resid -= PAGE_SIZE; 6878 } 6879 VPRINTF("\n"); 6880 return (size); 6881 } 6882 6883 /* 6884 * pmap_unmap_chunk: 6885 * 6886 * Unmap a chunk of memory that was previously pmap_map_chunk 6887 */ 6888 void 6889 pmap_unmap_chunk(vaddr_t l1pt, vaddr_t va, vsize_t size) 6890 { 6891 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6892 const size_t l1slot = l1pte_index(va); 6893 6894 KASSERT(size == L1_SS_SIZE || size == L1_S_SIZE); 6895 6896 l1pte_set(&pdep[l1slot], 0); 6897 PDE_SYNC_RANGE(&pdep[l1slot], size / L1_S_SIZE); 6898 6899 pmap_tlb_flush_SE(pmap_kernel(), va, PVF_REF); 6900 } 6901 6902 6903 6904 /********************** Static device map routines ***************************/ 6905 6906 static const struct pmap_devmap *pmap_devmap_table; 6907 6908 /* 6909 * Register the devmap table. This is provided in case early console 6910 * initialization needs to register mappings created by bootstrap code 6911 * before pmap_devmap_bootstrap() is called. 6912 */ 6913 void 6914 pmap_devmap_register(const struct pmap_devmap *table) 6915 { 6916 6917 pmap_devmap_table = table; 6918 } 6919 6920 /* 6921 * Map all of the static regions in the devmap table, and remember 6922 * the devmap table so other parts of the kernel can look up entries 6923 * later. 6924 */ 6925 void 6926 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 6927 { 6928 int i; 6929 6930 pmap_devmap_table = table; 6931 6932 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6933 const struct pmap_devmap *pdp = &pmap_devmap_table[i]; 6934 6935 KASSERTMSG(VADDR_MAX - pdp->pd_va >= pdp->pd_size - 1, "va %" PRIxVADDR 6936 " sz %" PRIxPSIZE, pdp->pd_va, pdp->pd_size); 6937 KASSERTMSG(PADDR_MAX - pdp->pd_pa >= pdp->pd_size - 1, "pa %" PRIxPADDR 6938 " sz %" PRIxPSIZE, pdp->pd_pa, pdp->pd_size); 6939 VPRINTF("devmap: %08lx -> %08lx @ %08lx\n", pdp->pd_pa, 6940 pdp->pd_pa + pdp->pd_size - 1, pdp->pd_va); 6941 6942 pmap_map_chunk(l1pt, pdp->pd_va, pdp->pd_pa, pdp->pd_size, 6943 pdp->pd_prot, pdp->pd_cache); 6944 } 6945 } 6946 6947 const struct pmap_devmap * 6948 pmap_devmap_find_pa(paddr_t pa, psize_t size) 6949 { 6950 uint64_t endpa; 6951 int i; 6952 6953 if (pmap_devmap_table == NULL) 6954 return (NULL); 6955 6956 endpa = (uint64_t)pa + (uint64_t)(size - 1); 6957 6958 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6959 if (pa >= pmap_devmap_table[i].pd_pa && 6960 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 6961 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 6962 return (&pmap_devmap_table[i]); 6963 } 6964 6965 return (NULL); 6966 } 6967 6968 const struct pmap_devmap * 6969 pmap_devmap_find_va(vaddr_t va, vsize_t size) 6970 { 6971 int i; 6972 6973 if (pmap_devmap_table == NULL) 6974 return (NULL); 6975 6976 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6977 if (va >= pmap_devmap_table[i].pd_va && 6978 va + size - 1 <= pmap_devmap_table[i].pd_va + 6979 pmap_devmap_table[i].pd_size - 1) 6980 return (&pmap_devmap_table[i]); 6981 } 6982 6983 return (NULL); 6984 } 6985 6986 /********************** PTE initialization routines **************************/ 6987 6988 /* 6989 * These routines are called when the CPU type is identified to set up 6990 * the PTE prototypes, cache modes, etc. 6991 * 6992 * The variables are always here, just in case modules need to reference 6993 * them (though, they shouldn't). 6994 */ 6995 6996 pt_entry_t pte_l1_s_cache_mode; 6997 pt_entry_t pte_l1_s_wc_mode; 6998 pt_entry_t pte_l1_s_cache_mode_pt; 6999 pt_entry_t pte_l1_s_cache_mask; 7000 7001 pt_entry_t pte_l2_l_cache_mode; 7002 pt_entry_t pte_l2_l_wc_mode; 7003 pt_entry_t pte_l2_l_cache_mode_pt; 7004 pt_entry_t pte_l2_l_cache_mask; 7005 7006 pt_entry_t pte_l2_s_cache_mode; 7007 pt_entry_t pte_l2_s_wc_mode; 7008 pt_entry_t pte_l2_s_cache_mode_pt; 7009 pt_entry_t pte_l2_s_cache_mask; 7010 7011 pt_entry_t pte_l1_s_prot_u; 7012 pt_entry_t pte_l1_s_prot_w; 7013 pt_entry_t pte_l1_s_prot_ro; 7014 pt_entry_t pte_l1_s_prot_mask; 7015 7016 pt_entry_t pte_l2_s_prot_u; 7017 pt_entry_t pte_l2_s_prot_w; 7018 pt_entry_t pte_l2_s_prot_ro; 7019 pt_entry_t pte_l2_s_prot_mask; 7020 7021 pt_entry_t pte_l2_l_prot_u; 7022 pt_entry_t pte_l2_l_prot_w; 7023 pt_entry_t pte_l2_l_prot_ro; 7024 pt_entry_t pte_l2_l_prot_mask; 7025 7026 pt_entry_t pte_l1_ss_proto; 7027 pt_entry_t pte_l1_s_proto; 7028 pt_entry_t pte_l1_c_proto; 7029 pt_entry_t pte_l2_s_proto; 7030 7031 void (*pmap_copy_page_func)(paddr_t, paddr_t); 7032 void (*pmap_zero_page_func)(paddr_t); 7033 7034 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 7035 void 7036 pmap_pte_init_generic(void) 7037 { 7038 7039 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7040 pte_l1_s_wc_mode = L1_S_B; 7041 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 7042 7043 pte_l2_l_cache_mode = L2_B|L2_C; 7044 pte_l2_l_wc_mode = L2_B; 7045 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 7046 7047 pte_l2_s_cache_mode = L2_B|L2_C; 7048 pte_l2_s_wc_mode = L2_B; 7049 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 7050 7051 /* 7052 * If we have a write-through cache, set B and C. If 7053 * we have a write-back cache, then we assume setting 7054 * only C will make those pages write-through (except for those 7055 * Cortex CPUs which can read the L1 caches). 7056 */ 7057 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop 7058 #if ARM_MMU_V7 > 0 7059 || CPU_ID_CORTEX_P(curcpu()->ci_arm_cpuid) 7060 #endif 7061 #if ARM_MMU_V6 > 0 7062 || CPU_ID_ARM11_P(curcpu()->ci_arm_cpuid) /* arm116 errata 399234 */ 7063 #endif 7064 || false) { 7065 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7066 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7067 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7068 } else { 7069 pte_l1_s_cache_mode_pt = L1_S_C; /* write through */ 7070 pte_l2_l_cache_mode_pt = L2_C; /* write through */ 7071 pte_l2_s_cache_mode_pt = L2_C; /* write through */ 7072 } 7073 7074 pte_l1_s_prot_u = L1_S_PROT_U_generic; 7075 pte_l1_s_prot_w = L1_S_PROT_W_generic; 7076 pte_l1_s_prot_ro = L1_S_PROT_RO_generic; 7077 pte_l1_s_prot_mask = L1_S_PROT_MASK_generic; 7078 7079 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7080 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7081 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7082 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7083 7084 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7085 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7086 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7087 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7088 7089 pte_l1_ss_proto = L1_SS_PROTO_generic; 7090 pte_l1_s_proto = L1_S_PROTO_generic; 7091 pte_l1_c_proto = L1_C_PROTO_generic; 7092 pte_l2_s_proto = L2_S_PROTO_generic; 7093 7094 pmap_copy_page_func = pmap_copy_page_generic; 7095 pmap_zero_page_func = pmap_zero_page_generic; 7096 } 7097 7098 #if defined(CPU_ARM8) 7099 void 7100 pmap_pte_init_arm8(void) 7101 { 7102 7103 /* 7104 * ARM8 is compatible with generic, but we need to use 7105 * the page tables uncached. 7106 */ 7107 pmap_pte_init_generic(); 7108 7109 pte_l1_s_cache_mode_pt = 0; 7110 pte_l2_l_cache_mode_pt = 0; 7111 pte_l2_s_cache_mode_pt = 0; 7112 } 7113 #endif /* CPU_ARM8 */ 7114 7115 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 7116 void 7117 pmap_pte_init_arm9(void) 7118 { 7119 7120 /* 7121 * ARM9 is compatible with generic, but we want to use 7122 * write-through caching for now. 7123 */ 7124 pmap_pte_init_generic(); 7125 7126 pte_l1_s_cache_mode = L1_S_C; 7127 pte_l2_l_cache_mode = L2_C; 7128 pte_l2_s_cache_mode = L2_C; 7129 7130 pte_l1_s_wc_mode = L1_S_B; 7131 pte_l2_l_wc_mode = L2_B; 7132 pte_l2_s_wc_mode = L2_B; 7133 7134 pte_l1_s_cache_mode_pt = L1_S_C; 7135 pte_l2_l_cache_mode_pt = L2_C; 7136 pte_l2_s_cache_mode_pt = L2_C; 7137 } 7138 #endif /* CPU_ARM9 && ARM9_CACHE_WRITE_THROUGH */ 7139 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 7140 7141 #if defined(CPU_ARM10) 7142 void 7143 pmap_pte_init_arm10(void) 7144 { 7145 7146 /* 7147 * ARM10 is compatible with generic, but we want to use 7148 * write-through caching for now. 7149 */ 7150 pmap_pte_init_generic(); 7151 7152 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 7153 pte_l2_l_cache_mode = L2_B | L2_C; 7154 pte_l2_s_cache_mode = L2_B | L2_C; 7155 7156 pte_l1_s_cache_mode = L1_S_B; 7157 pte_l2_l_cache_mode = L2_B; 7158 pte_l2_s_cache_mode = L2_B; 7159 7160 pte_l1_s_cache_mode_pt = L1_S_C; 7161 pte_l2_l_cache_mode_pt = L2_C; 7162 pte_l2_s_cache_mode_pt = L2_C; 7163 7164 } 7165 #endif /* CPU_ARM10 */ 7166 7167 #if defined(CPU_ARM11) && defined(ARM11_CACHE_WRITE_THROUGH) 7168 void 7169 pmap_pte_init_arm11(void) 7170 { 7171 7172 /* 7173 * ARM11 is compatible with generic, but we want to use 7174 * write-through caching for now. 7175 */ 7176 pmap_pte_init_generic(); 7177 7178 pte_l1_s_cache_mode = L1_S_C; 7179 pte_l2_l_cache_mode = L2_C; 7180 pte_l2_s_cache_mode = L2_C; 7181 7182 pte_l1_s_wc_mode = L1_S_B; 7183 pte_l2_l_wc_mode = L2_B; 7184 pte_l2_s_wc_mode = L2_B; 7185 7186 pte_l1_s_cache_mode_pt = L1_S_C; 7187 pte_l2_l_cache_mode_pt = L2_C; 7188 pte_l2_s_cache_mode_pt = L2_C; 7189 } 7190 #endif /* CPU_ARM11 && ARM11_CACHE_WRITE_THROUGH */ 7191 7192 #if ARM_MMU_SA1 == 1 7193 void 7194 pmap_pte_init_sa1(void) 7195 { 7196 7197 /* 7198 * The StrongARM SA-1 cache does not have a write-through 7199 * mode. So, do the generic initialization, then reset 7200 * the page table cache mode to B=1,C=1, and note that 7201 * the PTEs need to be sync'd. 7202 */ 7203 pmap_pte_init_generic(); 7204 7205 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7206 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7207 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7208 7209 pmap_needs_pte_sync = 1; 7210 } 7211 #endif /* ARM_MMU_SA1 == 1*/ 7212 7213 #if ARM_MMU_XSCALE == 1 7214 #if (ARM_NMMUS > 1) 7215 static u_int xscale_use_minidata; 7216 #endif 7217 7218 void 7219 pmap_pte_init_xscale(void) 7220 { 7221 uint32_t auxctl; 7222 int write_through = 0; 7223 7224 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7225 pte_l1_s_wc_mode = L1_S_B; 7226 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 7227 7228 pte_l2_l_cache_mode = L2_B|L2_C; 7229 pte_l2_l_wc_mode = L2_B; 7230 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 7231 7232 pte_l2_s_cache_mode = L2_B|L2_C; 7233 pte_l2_s_wc_mode = L2_B; 7234 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 7235 7236 pte_l1_s_cache_mode_pt = L1_S_C; 7237 pte_l2_l_cache_mode_pt = L2_C; 7238 pte_l2_s_cache_mode_pt = L2_C; 7239 7240 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 7241 /* 7242 * The XScale core has an enhanced mode where writes that 7243 * miss the cache cause a cache line to be allocated. This 7244 * is significantly faster than the traditional, write-through 7245 * behavior of this case. 7246 */ 7247 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 7248 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 7249 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 7250 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 7251 7252 #ifdef XSCALE_CACHE_WRITE_THROUGH 7253 /* 7254 * Some versions of the XScale core have various bugs in 7255 * their cache units, the work-around for which is to run 7256 * the cache in write-through mode. Unfortunately, this 7257 * has a major (negative) impact on performance. So, we 7258 * go ahead and run fast-and-loose, in the hopes that we 7259 * don't line up the planets in a way that will trip the 7260 * bugs. 7261 * 7262 * However, we give you the option to be slow-but-correct. 7263 */ 7264 write_through = 1; 7265 #elif defined(XSCALE_CACHE_WRITE_BACK) 7266 /* force write back cache mode */ 7267 write_through = 0; 7268 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 7269 /* 7270 * Intel PXA2[15]0 processors are known to have a bug in 7271 * write-back cache on revision 4 and earlier (stepping 7272 * A[01] and B[012]). Fixed for C0 and later. 7273 */ 7274 { 7275 uint32_t id, type; 7276 7277 id = cpufunc_id(); 7278 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 7279 7280 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 7281 if ((id & CPU_ID_REVISION_MASK) < 5) { 7282 /* write through for stepping A0-1 and B0-2 */ 7283 write_through = 1; 7284 } 7285 } 7286 } 7287 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 7288 7289 if (write_through) { 7290 pte_l1_s_cache_mode = L1_S_C; 7291 pte_l2_l_cache_mode = L2_C; 7292 pte_l2_s_cache_mode = L2_C; 7293 } 7294 7295 #if (ARM_NMMUS > 1) 7296 xscale_use_minidata = 1; 7297 #endif 7298 7299 pte_l1_s_prot_u = L1_S_PROT_U_xscale; 7300 pte_l1_s_prot_w = L1_S_PROT_W_xscale; 7301 pte_l1_s_prot_ro = L1_S_PROT_RO_xscale; 7302 pte_l1_s_prot_mask = L1_S_PROT_MASK_xscale; 7303 7304 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 7305 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 7306 pte_l2_s_prot_ro = L2_S_PROT_RO_xscale; 7307 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 7308 7309 pte_l2_l_prot_u = L2_L_PROT_U_xscale; 7310 pte_l2_l_prot_w = L2_L_PROT_W_xscale; 7311 pte_l2_l_prot_ro = L2_L_PROT_RO_xscale; 7312 pte_l2_l_prot_mask = L2_L_PROT_MASK_xscale; 7313 7314 pte_l1_ss_proto = L1_SS_PROTO_xscale; 7315 pte_l1_s_proto = L1_S_PROTO_xscale; 7316 pte_l1_c_proto = L1_C_PROTO_xscale; 7317 pte_l2_s_proto = L2_S_PROTO_xscale; 7318 7319 pmap_copy_page_func = pmap_copy_page_xscale; 7320 pmap_zero_page_func = pmap_zero_page_xscale; 7321 7322 /* 7323 * Disable ECC protection of page table access, for now. 7324 */ 7325 auxctl = armreg_auxctl_read(); 7326 auxctl &= ~XSCALE_AUXCTL_P; 7327 armreg_auxctl_write(auxctl); 7328 } 7329 7330 /* 7331 * xscale_setup_minidata: 7332 * 7333 * Set up the mini-data cache clean area. We require the 7334 * caller to allocate the right amount of physically and 7335 * virtually contiguous space. 7336 */ 7337 void 7338 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 7339 { 7340 extern vaddr_t xscale_minidata_clean_addr; 7341 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 7342 pd_entry_t *pde = (pd_entry_t *) l1pt; 7343 vsize_t size; 7344 uint32_t auxctl; 7345 7346 xscale_minidata_clean_addr = va; 7347 7348 /* Round it to page size. */ 7349 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 7350 7351 for (; size != 0; 7352 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 7353 const size_t l1slot = l1pte_index(va); 7354 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pde[l1slot])); 7355 if (ptep == NULL) 7356 panic("xscale_setup_minidata: can't find L2 table for " 7357 "VA 0x%08lx", va); 7358 7359 ptep += l2pte_index(va); 7360 pt_entry_t opte = *ptep; 7361 l2pte_set(ptep, 7362 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 7363 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X), opte); 7364 } 7365 7366 /* 7367 * Configure the mini-data cache for write-back with 7368 * read/write-allocate. 7369 * 7370 * NOTE: In order to reconfigure the mini-data cache, we must 7371 * make sure it contains no valid data! In order to do that, 7372 * we must issue a global data cache invalidate command! 7373 * 7374 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 7375 * THIS IS VERY IMPORTANT! 7376 */ 7377 7378 /* Invalidate data and mini-data. */ 7379 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 7380 auxctl = armreg_auxctl_read(); 7381 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 7382 armreg_auxctl_write(auxctl); 7383 } 7384 7385 /* 7386 * Change the PTEs for the specified kernel mappings such that they 7387 * will use the mini data cache instead of the main data cache. 7388 */ 7389 void 7390 pmap_uarea(vaddr_t va) 7391 { 7392 vaddr_t next_bucket, eva; 7393 7394 #if (ARM_NMMUS > 1) 7395 if (xscale_use_minidata == 0) 7396 return; 7397 #endif 7398 7399 eva = va + USPACE; 7400 7401 while (va < eva) { 7402 next_bucket = L2_NEXT_BUCKET_VA(va); 7403 if (next_bucket > eva) 7404 next_bucket = eva; 7405 7406 struct l2_bucket *l2b = pmap_get_l2_bucket(pmap_kernel(), va); 7407 KDASSERT(l2b != NULL); 7408 7409 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 7410 pt_entry_t *ptep = sptep; 7411 7412 while (va < next_bucket) { 7413 const pt_entry_t opte = *ptep; 7414 if (!l2pte_minidata_p(opte)) { 7415 cpu_dcache_wbinv_range(va, PAGE_SIZE); 7416 cpu_tlb_flushD_SE(va); 7417 l2pte_set(ptep, opte & ~L2_B, opte); 7418 } 7419 ptep += PAGE_SIZE / L2_S_SIZE; 7420 va += PAGE_SIZE; 7421 } 7422 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 7423 } 7424 cpu_cpwait(); 7425 } 7426 #endif /* ARM_MMU_XSCALE == 1 */ 7427 7428 7429 #if defined(CPU_ARM11MPCORE) 7430 7431 void 7432 pmap_pte_init_arm11mpcore(void) 7433 { 7434 7435 /* cache mode is controlled by 5 bits (B, C, TEX) */ 7436 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6; 7437 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6; 7438 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7439 /* use extended small page (without APn, with TEX) */ 7440 pte_l2_s_cache_mask = L2_XS_CACHE_MASK_armv6; 7441 #else 7442 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6c; 7443 #endif 7444 7445 /* write-back, write-allocate */ 7446 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7447 pte_l2_l_cache_mode = L2_C | L2_B | L2_V6_L_TEX(0x01); 7448 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7449 pte_l2_s_cache_mode = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7450 #else 7451 /* no TEX. read-allocate */ 7452 pte_l2_s_cache_mode = L2_C | L2_B; 7453 #endif 7454 /* 7455 * write-back, write-allocate for page tables. 7456 */ 7457 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7458 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_V6_L_TEX(0x01); 7459 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7460 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7461 #else 7462 pte_l2_s_cache_mode_pt = L2_C | L2_B; 7463 #endif 7464 7465 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7466 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7467 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7468 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7469 7470 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7471 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7472 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7473 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7474 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7475 7476 #else 7477 /* with AP[0..3] */ 7478 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7479 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7480 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7481 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7482 #endif 7483 7484 #ifdef ARM11MPCORE_COMPAT_MMU 7485 /* with AP[0..3] */ 7486 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7487 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7488 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7489 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7490 7491 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7492 pte_l1_s_proto = L1_S_PROTO_armv6; 7493 pte_l1_c_proto = L1_C_PROTO_armv6; 7494 pte_l2_s_proto = L2_S_PROTO_armv6c; 7495 #else 7496 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7497 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7498 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7499 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7500 7501 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7502 pte_l1_s_proto = L1_S_PROTO_armv6; 7503 pte_l1_c_proto = L1_C_PROTO_armv6; 7504 pte_l2_s_proto = L2_S_PROTO_armv6n; 7505 #endif 7506 7507 pmap_copy_page_func = pmap_copy_page_generic; 7508 pmap_zero_page_func = pmap_zero_page_generic; 7509 pmap_needs_pte_sync = 1; 7510 } 7511 #endif /* CPU_ARM11MPCORE */ 7512 7513 7514 #if ARM_MMU_V7 == 1 7515 void 7516 pmap_pte_init_armv7(void) 7517 { 7518 /* 7519 * The ARMv7-A MMU is mostly compatible with generic. If the 7520 * AP field is zero, that now means "no access" rather than 7521 * read-only. The prototypes are a little different because of 7522 * the XN bit. 7523 */ 7524 pmap_pte_init_generic(); 7525 7526 pmap_needs_pte_sync = 1; 7527 7528 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv7; 7529 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv7; 7530 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv7; 7531 7532 /* 7533 * If the core support coherent walk then updates to translation tables 7534 * do not require a clean to the point of unification to ensure 7535 * visibility by subsequent translation table walks. That means we can 7536 * map everything shareable and cached and the right thing will happen. 7537 */ 7538 if (__SHIFTOUT(armreg_mmfr3_read(), __BITS(23,20))) { 7539 pmap_needs_pte_sync = 0; 7540 7541 /* 7542 * write-back, no write-allocate, shareable for normal pages. 7543 */ 7544 pte_l1_s_cache_mode |= L1_S_V6_S; 7545 pte_l2_l_cache_mode |= L2_XS_S; 7546 pte_l2_s_cache_mode |= L2_XS_S; 7547 } 7548 7549 /* 7550 * Page tables are just all other memory. We can use write-back since 7551 * pmap_needs_pte_sync is 1 (or the MMU can read out of cache). 7552 */ 7553 pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode; 7554 pte_l2_l_cache_mode_pt = pte_l2_l_cache_mode; 7555 pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; 7556 7557 /* 7558 * Check the Memory Model Features to see if this CPU supports 7559 * the TLBIASID coproc op. 7560 */ 7561 if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(16,19)) >= 2) { 7562 arm_has_tlbiasid_p = true; 7563 } else if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(12,15)) >= 2) { 7564 arm_has_tlbiasid_p = true; 7565 } 7566 7567 /* 7568 * Check the MPIDR to see if this CPU supports MP extensions. 7569 */ 7570 #ifdef MULTIPROCESSOR 7571 arm_has_mpext_p = (armreg_mpidr_read() & (MPIDR_MP|MPIDR_U)) == MPIDR_MP; 7572 #else 7573 arm_has_mpext_p = false; 7574 #endif 7575 7576 pte_l1_s_prot_u = L1_S_PROT_U_armv7; 7577 pte_l1_s_prot_w = L1_S_PROT_W_armv7; 7578 pte_l1_s_prot_ro = L1_S_PROT_RO_armv7; 7579 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv7; 7580 7581 pte_l2_s_prot_u = L2_S_PROT_U_armv7; 7582 pte_l2_s_prot_w = L2_S_PROT_W_armv7; 7583 pte_l2_s_prot_ro = L2_S_PROT_RO_armv7; 7584 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv7; 7585 7586 pte_l2_l_prot_u = L2_L_PROT_U_armv7; 7587 pte_l2_l_prot_w = L2_L_PROT_W_armv7; 7588 pte_l2_l_prot_ro = L2_L_PROT_RO_armv7; 7589 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv7; 7590 7591 pte_l1_ss_proto = L1_SS_PROTO_armv7; 7592 pte_l1_s_proto = L1_S_PROTO_armv7; 7593 pte_l1_c_proto = L1_C_PROTO_armv7; 7594 pte_l2_s_proto = L2_S_PROTO_armv7; 7595 7596 } 7597 #endif /* ARM_MMU_V7 */ 7598 7599 /* 7600 * return the PA of the current L1 table, for use when handling a crash dump 7601 */ 7602 uint32_t 7603 pmap_kernel_L1_addr(void) 7604 { 7605 #ifdef ARM_MMU_EXTENDED 7606 return pmap_kernel()->pm_l1_pa; 7607 #else 7608 return pmap_kernel()->pm_l1->l1_physaddr; 7609 #endif 7610 } 7611 7612 #if defined(DDB) 7613 /* 7614 * A couple of ddb-callable functions for dumping pmaps 7615 */ 7616 void pmap_dump(pmap_t); 7617 7618 static pt_entry_t ncptes[64]; 7619 static void pmap_dump_ncpg(pmap_t); 7620 7621 void 7622 pmap_dump(pmap_t pm) 7623 { 7624 struct l2_dtable *l2; 7625 struct l2_bucket *l2b; 7626 pt_entry_t *ptep, pte; 7627 vaddr_t l2_va, l2b_va, va; 7628 int i, j, k, occ, rows = 0; 7629 7630 if (pm == pmap_kernel()) 7631 printf("pmap_kernel (%p): ", pm); 7632 else 7633 printf("user pmap (%p): ", pm); 7634 7635 #ifdef ARM_MMU_EXTENDED 7636 printf("l1 at %p\n", pmap_l1_kva(pm)); 7637 #else 7638 printf("domain %d, l1 at %p\n", pmap_domain(pm), pmap_l1_kva(pm)); 7639 #endif 7640 7641 l2_va = 0; 7642 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 7643 l2 = pm->pm_l2[i]; 7644 7645 if (l2 == NULL || l2->l2_occupancy == 0) 7646 continue; 7647 7648 l2b_va = l2_va; 7649 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 7650 l2b = &l2->l2_bucket[j]; 7651 7652 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 7653 continue; 7654 7655 ptep = l2b->l2b_kva; 7656 7657 for (k = 0; k < 256 && ptep[k] == 0; k++) 7658 ; 7659 7660 k &= ~63; 7661 occ = l2b->l2b_occupancy; 7662 va = l2b_va + (k * 4096); 7663 for (; k < 256; k++, va += 0x1000) { 7664 char ch = ' '; 7665 if ((k % 64) == 0) { 7666 if ((rows % 8) == 0) { 7667 printf( 7668 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 7669 } 7670 printf("%08lx: ", va); 7671 } 7672 7673 ncptes[k & 63] = 0; 7674 pte = ptep[k]; 7675 if (pte == 0) { 7676 ch = '.'; 7677 } else { 7678 occ--; 7679 switch (pte & 0x0c) { 7680 case 0x00: 7681 ch = 'D'; /* No cache No buff */ 7682 break; 7683 case 0x04: 7684 ch = 'B'; /* No cache buff */ 7685 break; 7686 case 0x08: 7687 if (pte & 0x40) 7688 ch = 'm'; 7689 else 7690 ch = 'C'; /* Cache No buff */ 7691 break; 7692 case 0x0c: 7693 ch = 'F'; /* Cache Buff */ 7694 break; 7695 } 7696 7697 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 7698 ch += 0x20; 7699 7700 if ((pte & 0xc) == 0) 7701 ncptes[k & 63] = pte; 7702 } 7703 7704 if ((k % 64) == 63) { 7705 rows++; 7706 printf("%c\n", ch); 7707 pmap_dump_ncpg(pm); 7708 if (occ == 0) 7709 break; 7710 } else 7711 printf("%c", ch); 7712 } 7713 } 7714 } 7715 } 7716 7717 static void 7718 pmap_dump_ncpg(pmap_t pm) 7719 { 7720 struct vm_page *pg; 7721 struct vm_page_md *md; 7722 struct pv_entry *pv; 7723 int i; 7724 7725 for (i = 0; i < 63; i++) { 7726 if (ncptes[i] == 0) 7727 continue; 7728 7729 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 7730 if (pg == NULL) 7731 continue; 7732 md = VM_PAGE_TO_MD(pg); 7733 7734 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 7735 VM_PAGE_TO_PHYS(pg), 7736 md->krw_mappings, md->kro_mappings, 7737 md->urw_mappings, md->uro_mappings); 7738 7739 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 7740 printf(" %c va 0x%08lx, flags 0x%x\n", 7741 (pm == pv->pv_pmap) ? '*' : ' ', 7742 pv->pv_va, pv->pv_flags); 7743 } 7744 } 7745 } 7746 #endif 7747 7748 #ifdef PMAP_STEAL_MEMORY 7749 void 7750 pmap_boot_pageadd(pv_addr_t *newpv) 7751 { 7752 pv_addr_t *pv, *npv; 7753 7754 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 7755 if (newpv->pv_pa < pv->pv_va) { 7756 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 7757 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 7758 newpv->pv_size += pv->pv_size; 7759 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 7760 } 7761 pv = NULL; 7762 } else { 7763 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 7764 pv = npv) { 7765 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 7766 KASSERT(pv->pv_pa < newpv->pv_pa); 7767 if (newpv->pv_pa > npv->pv_pa) 7768 continue; 7769 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 7770 pv->pv_size += newpv->pv_size; 7771 return; 7772 } 7773 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 7774 break; 7775 newpv->pv_size += npv->pv_size; 7776 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7777 SLIST_REMOVE_AFTER(newpv, pv_list); 7778 return; 7779 } 7780 } 7781 } 7782 7783 if (pv) { 7784 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7785 } else { 7786 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 7787 } 7788 } 7789 7790 void 7791 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 7792 pv_addr_t *rpv) 7793 { 7794 pv_addr_t *pv, **pvp; 7795 7796 KASSERT(amount & PGOFSET); 7797 KASSERT((mask & PGOFSET) == 0); 7798 KASSERT((match & PGOFSET) == 0); 7799 KASSERT(amount != 0); 7800 7801 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7802 (pv = *pvp) != NULL; 7803 pvp = &SLIST_NEXT(pv, pv_list)) { 7804 pv_addr_t *newpv; 7805 psize_t off; 7806 /* 7807 * If this entry is too small to satisfy the request... 7808 */ 7809 KASSERT(pv->pv_size > 0); 7810 if (pv->pv_size < amount) 7811 continue; 7812 7813 for (off = 0; off <= mask; off += PAGE_SIZE) { 7814 if (((pv->pv_pa + off) & mask) == match 7815 && off + amount <= pv->pv_size) 7816 break; 7817 } 7818 if (off > mask) 7819 continue; 7820 7821 rpv->pv_va = pv->pv_va + off; 7822 rpv->pv_pa = pv->pv_pa + off; 7823 rpv->pv_size = amount; 7824 pv->pv_size -= amount; 7825 if (pv->pv_size == 0) { 7826 KASSERT(off == 0); 7827 KASSERT((vaddr_t) pv == rpv->pv_va); 7828 *pvp = SLIST_NEXT(pv, pv_list); 7829 } else if (off == 0) { 7830 KASSERT((vaddr_t) pv == rpv->pv_va); 7831 newpv = (pv_addr_t *) (rpv->pv_va + amount); 7832 *newpv = *pv; 7833 newpv->pv_pa += amount; 7834 newpv->pv_va += amount; 7835 *pvp = newpv; 7836 } else if (off < pv->pv_size) { 7837 newpv = (pv_addr_t *) (rpv->pv_va + amount); 7838 *newpv = *pv; 7839 newpv->pv_size -= off; 7840 newpv->pv_pa += off + amount; 7841 newpv->pv_va += off + amount; 7842 7843 SLIST_NEXT(pv, pv_list) = newpv; 7844 pv->pv_size = off; 7845 } else { 7846 KASSERT((vaddr_t) pv != rpv->pv_va); 7847 } 7848 memset((void *)rpv->pv_va, 0, amount); 7849 return; 7850 } 7851 7852 if (!uvm_physseg_valid_p(uvm_physseg_get_first())) 7853 panic("pmap_boot_pagealloc: couldn't allocate memory"); 7854 7855 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7856 (pv = *pvp) != NULL; 7857 pvp = &SLIST_NEXT(pv, pv_list)) { 7858 if (SLIST_NEXT(pv, pv_list) == NULL) 7859 break; 7860 } 7861 KASSERT(mask == 0); 7862 7863 for (uvm_physseg_t ups = uvm_physseg_get_first(); 7864 uvm_physseg_valid_p(ups); 7865 ups = uvm_physseg_get_next(ups)) { 7866 7867 paddr_t spn = uvm_physseg_get_start(ups); 7868 paddr_t epn = uvm_physseg_get_end(ups); 7869 if (spn == atop(pv->pv_pa + pv->pv_size) 7870 && pv->pv_va + pv->pv_size <= ptoa(epn)) { 7871 rpv->pv_va = pv->pv_va; 7872 rpv->pv_pa = pv->pv_pa; 7873 rpv->pv_size = amount; 7874 *pvp = NULL; 7875 pmap_map_chunk(kernel_l1pt.pv_va, 7876 ptoa(spn) + (pv->pv_va - pv->pv_pa), 7877 ptoa(spn), 7878 amount - pv->pv_size, 7879 VM_PROT_READ|VM_PROT_WRITE, 7880 PTE_CACHE); 7881 7882 uvm_physseg_unplug(spn, atop(amount - pv->pv_size)); 7883 memset((void *)rpv->pv_va, 0, rpv->pv_size); 7884 return; 7885 } 7886 } 7887 7888 panic("pmap_boot_pagealloc: couldn't allocate memory"); 7889 } 7890 7891 vaddr_t 7892 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 7893 { 7894 pv_addr_t pv; 7895 7896 pmap_boot_pagealloc(size, 0, 0, &pv); 7897 7898 return pv.pv_va; 7899 } 7900 #endif /* PMAP_STEAL_MEMORY */ 7901 7902 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 7903 { 7904 sysctl_createv(clog, 0, NULL, NULL, 7905 CTLFLAG_PERMANENT, 7906 CTLTYPE_NODE, "machdep", NULL, 7907 NULL, 0, NULL, 0, 7908 CTL_MACHDEP, CTL_EOL); 7909 7910 sysctl_createv(clog, 0, NULL, NULL, 7911 CTLFLAG_PERMANENT, 7912 CTLTYPE_INT, "kmpages", 7913 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 7914 NULL, 0, &pmap_kmpages, 0, 7915 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 7916 } 7917 7918 #ifdef PMAP_NEED_ALLOC_POOLPAGE 7919 struct vm_page * 7920 arm_pmap_alloc_poolpage(int flags) 7921 { 7922 /* 7923 * On some systems, only some pages may be "coherent" for dma and we 7924 * want to prefer those for pool pages (think mbufs) but fallback to 7925 * any page if none is available. 7926 */ 7927 if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) { 7928 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 7929 UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist); 7930 } 7931 7932 return uvm_pagealloc(NULL, 0, NULL, flags); 7933 } 7934 #endif 7935 7936 #if defined(ARM_MMU_EXTENDED) && defined(MULTIPROCESSOR) 7937 void 7938 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 7939 { 7940 /* nothing */ 7941 } 7942 7943 int 7944 pic_ipi_shootdown(void *arg) 7945 { 7946 #if PMAP_TLB_NEED_SHOOTDOWN 7947 pmap_tlb_shootdown_process(); 7948 #endif 7949 return 1; 7950 } 7951 #endif /* ARM_MMU_EXTENDED && MULTIPROCESSOR */ 7952 7953 7954 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 7955 vaddr_t 7956 pmap_direct_mapped_phys(paddr_t pa, bool *ok_p, vaddr_t va) 7957 { 7958 bool ok = false; 7959 if (physical_start <= pa && pa < physical_end) { 7960 #ifdef KERNEL_BASE_VOFFSET 7961 const vaddr_t newva = pa + KERNEL_BASE_VOFFSET; 7962 #else 7963 const vaddr_t newva = KERNEL_BASE + pa - physical_start; 7964 #endif 7965 #ifdef ARM_MMU_EXTENDED 7966 if (newva >= KERNEL_BASE && newva < pmap_directlimit) { 7967 #endif 7968 va = newva; 7969 ok = true; 7970 #ifdef ARM_MMU_EXTENDED 7971 } 7972 #endif 7973 } 7974 KASSERT(ok_p); 7975 *ok_p = ok; 7976 return va; 7977 } 7978 7979 vaddr_t 7980 pmap_map_poolpage(paddr_t pa) 7981 { 7982 bool ok __diagused; 7983 vaddr_t va = pmap_direct_mapped_phys(pa, &ok, 0); 7984 KASSERTMSG(ok, "pa %#lx not direct mappable", pa); 7985 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 7986 if (arm_cache_prefer_mask != 0) { 7987 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 7988 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 7989 pmap_acquire_page_lock(md); 7990 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 7991 pmap_release_page_lock(md); 7992 } 7993 #endif 7994 return va; 7995 } 7996 7997 paddr_t 7998 pmap_unmap_poolpage(vaddr_t va) 7999 { 8000 KASSERT(va >= KERNEL_BASE); 8001 #ifdef PMAP_CACHE_VIVT 8002 cpu_idcache_wbinv_range(va, PAGE_SIZE); 8003 #endif 8004 #if defined(KERNEL_BASE_VOFFSET) 8005 return va - KERNEL_BASE_VOFFSET; 8006 #else 8007 return va - KERNEL_BASE + physical_start; 8008 #endif 8009 } 8010 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 8011
Indexes created Sun Sep 28 16:09:52 GMT 2025