pmap.c revision 1.372
1 /* $NetBSD: pmap.c,v 1.372 2019/04/23 11:05:14 bouyer Exp $ */ 2 3 /* 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65 66 /*- 67 * Copyright (c) 1999 The NetBSD Foundation, Inc. 68 * All rights reserved. 69 * 70 * This code is derived from software contributed to The NetBSD Foundation 71 * by Charles M. Hannum. 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 82 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 83 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 84 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 85 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 86 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 87 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 88 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 89 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 90 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 91 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 92 * POSSIBILITY OF SUCH DAMAGE. 93 */ 94 95 /* 96 * Copyright (c) 1994-1998 Mark Brinicombe. 97 * Copyright (c) 1994 Brini. 98 * All rights reserved. 99 * 100 * This code is derived from software written for Brini by Mark Brinicombe 101 * 102 * Redistribution and use in source and binary forms, with or without 103 * modification, are permitted provided that the following conditions 104 * are met: 105 * 1. Redistributions of source code must retain the above copyright 106 * notice, this list of conditions and the following disclaimer. 107 * 2. Redistributions in binary form must reproduce the above copyright 108 * notice, this list of conditions and the following disclaimer in the 109 * documentation and/or other materials provided with the distribution. 110 * 3. All advertising materials mentioning features or use of this software 111 * must display the following acknowledgement: 112 * This product includes software developed by Mark Brinicombe. 113 * 4. The name of the author may not be used to endorse or promote products 114 * derived from this software without specific prior written permission. 115 * 116 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 117 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 118 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 119 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 120 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 121 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 122 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 123 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 124 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 125 * 126 * RiscBSD kernel project 127 * 128 * pmap.c 129 * 130 * Machine dependent vm stuff 131 * 132 * Created : 20/09/94 133 */ 134 135 /* 136 * armv6 and VIPT cache support by 3am Software Foundry, 137 * Copyright (c) 2007 Microsoft 138 */ 139 140 /* 141 * Performance improvements, UVM changes, overhauls and part-rewrites 142 * were contributed by Neil A. Carson <neil (at) causality.com>. 143 */ 144 145 /* 146 * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables 147 * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi 148 * Systems, Inc. 149 * 150 * There are still a few things outstanding at this time: 151 * 152 * - There are some unresolved issues for MP systems: 153 * 154 * o The L1 metadata needs a lock, or more specifically, some places 155 * need to acquire an exclusive lock when modifying L1 translation 156 * table entries. 157 * 158 * o When one cpu modifies an L1 entry, and that L1 table is also 159 * being used by another cpu, then the latter will need to be told 160 * that a tlb invalidation may be necessary. (But only if the old 161 * domain number in the L1 entry being over-written is currently 162 * the active domain on that cpu). I guess there are lots more tlb 163 * shootdown issues too... 164 * 165 * o If the vector_page is at 0x00000000 instead of in kernel VA space, 166 * then MP systems will lose big-time because of the MMU domain hack. 167 * The only way this can be solved (apart from moving the vector 168 * page to 0xffff0000) is to reserve the first 1MB of user address 169 * space for kernel use only. This would require re-linking all 170 * applications so that the text section starts above this 1MB 171 * boundary. 172 * 173 * o Tracking which VM space is resident in the cache/tlb has not yet 174 * been implemented for MP systems. 175 * 176 * o Finally, there is a pathological condition where two cpus running 177 * two separate processes (not lwps) which happen to share an L1 178 * can get into a fight over one or more L1 entries. This will result 179 * in a significant slow-down if both processes are in tight loops. 180 */ 181 182 /* 183 * Special compilation symbols 184 * PMAP_DEBUG - Build in pmap_debug_level code 185 */ 186 187 /* Include header files */ 188 189 #include "opt_arm_debug.h" 190 #include "opt_cpuoptions.h" 191 #include "opt_pmap_debug.h" 192 #include "opt_ddb.h" 193 #include "opt_lockdebug.h" 194 #include "opt_multiprocessor.h" 195 196 #ifdef MULTIPROCESSOR 197 #define _INTR_PRIVATE 198 #endif 199 200 #include <sys/param.h> 201 #include <sys/types.h> 202 #include <sys/kernel.h> 203 #include <sys/systm.h> 204 #include <sys/proc.h> 205 #include <sys/intr.h> 206 #include <sys/pool.h> 207 #include <sys/kmem.h> 208 #include <sys/cdefs.h> 209 #include <sys/cpu.h> 210 #include <sys/sysctl.h> 211 #include <sys/bus.h> 212 #include <sys/atomic.h> 213 #include <sys/kernhist.h> 214 215 #include <uvm/uvm.h> 216 #include <uvm/pmap/pmap_pvt.h> 217 218 #include <arm/locore.h> 219 220 #ifdef DDB 221 #include <arm/db_machdep.h> 222 #endif 223 224 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.372 2019/04/23 11:05:14 bouyer Exp $"); 225 226 //#define PMAP_DEBUG 227 #ifdef PMAP_DEBUG 228 229 /* XXX need to get rid of all refs to this */ 230 int pmap_debug_level = 0; 231 232 /* 233 * for switching to potentially finer grained debugging 234 */ 235 #define PDB_FOLLOW 0x0001 236 #define PDB_INIT 0x0002 237 #define PDB_ENTER 0x0004 238 #define PDB_REMOVE 0x0008 239 #define PDB_CREATE 0x0010 240 #define PDB_PTPAGE 0x0020 241 #define PDB_GROWKERN 0x0040 242 #define PDB_BITS 0x0080 243 #define PDB_COLLECT 0x0100 244 #define PDB_PROTECT 0x0200 245 #define PDB_MAP_L1 0x0400 246 #define PDB_BOOTSTRAP 0x1000 247 #define PDB_PARANOIA 0x2000 248 #define PDB_WIRING 0x4000 249 #define PDB_PVDUMP 0x8000 250 #define PDB_VAC 0x10000 251 #define PDB_KENTER 0x20000 252 #define PDB_KREMOVE 0x40000 253 #define PDB_EXEC 0x80000 254 255 int debugmap = 1; 256 int pmapdebug = 0; 257 #define NPDEBUG(_lev_,_stat_) \ 258 if (pmapdebug & (_lev_)) \ 259 ((_stat_)) 260 261 #else /* PMAP_DEBUG */ 262 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 263 #endif /* PMAP_DEBUG */ 264 265 266 #ifdef VERBOSE_INIT_ARM 267 #define VPRINTF(...) printf(__VA_ARGS__) 268 #else 269 #define VPRINTF(...) __nothing 270 #endif 271 272 /* 273 * pmap_kernel() points here 274 */ 275 static struct pmap kernel_pmap_store = { 276 #ifndef ARM_MMU_EXTENDED 277 .pm_activated = true, 278 .pm_domain = PMAP_DOMAIN_KERNEL, 279 .pm_cstate.cs_all = PMAP_CACHE_STATE_ALL, 280 #endif 281 }; 282 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 283 #undef pmap_kernel 284 #define pmap_kernel() (&kernel_pmap_store) 285 #ifdef PMAP_NEED_ALLOC_POOLPAGE 286 int arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT; 287 #endif 288 289 /* 290 * Pool and cache that pmap structures are allocated from. 291 * We use a cache to avoid clearing the pm_l2[] array (1KB) 292 * in pmap_create(). 293 */ 294 static struct pool_cache pmap_cache; 295 296 /* 297 * Pool of PV structures 298 */ 299 static struct pool pmap_pv_pool; 300 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int); 301 static void pmap_bootstrap_pv_page_free(struct pool *, void *); 302 static struct pool_allocator pmap_bootstrap_pv_allocator = { 303 pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free 304 }; 305 306 /* 307 * Pool and cache of l2_dtable structures. 308 * We use a cache to avoid clearing the structures when they're 309 * allocated. (196 bytes) 310 */ 311 static struct pool_cache pmap_l2dtable_cache; 312 static vaddr_t pmap_kernel_l2dtable_kva; 313 314 /* 315 * Pool and cache of L2 page descriptors. 316 * We use a cache to avoid clearing the descriptor table 317 * when they're allocated. (1KB) 318 */ 319 static struct pool_cache pmap_l2ptp_cache; 320 static vaddr_t pmap_kernel_l2ptp_kva; 321 static paddr_t pmap_kernel_l2ptp_phys; 322 323 #ifdef PMAPCOUNTERS 324 #define PMAP_EVCNT_INITIALIZER(name) \ 325 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name) 326 327 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 328 static struct evcnt pmap_ev_vac_clean_one = 329 PMAP_EVCNT_INITIALIZER("clean page (1 color)"); 330 static struct evcnt pmap_ev_vac_flush_one = 331 PMAP_EVCNT_INITIALIZER("flush page (1 color)"); 332 static struct evcnt pmap_ev_vac_flush_lots = 333 PMAP_EVCNT_INITIALIZER("flush page (2+ colors)"); 334 static struct evcnt pmap_ev_vac_flush_lots2 = 335 PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)"); 336 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one); 337 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one); 338 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots); 339 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2); 340 341 static struct evcnt pmap_ev_vac_color_new = 342 PMAP_EVCNT_INITIALIZER("new page color"); 343 static struct evcnt pmap_ev_vac_color_reuse = 344 PMAP_EVCNT_INITIALIZER("ok first page color"); 345 static struct evcnt pmap_ev_vac_color_ok = 346 PMAP_EVCNT_INITIALIZER("ok page color"); 347 static struct evcnt pmap_ev_vac_color_blind = 348 PMAP_EVCNT_INITIALIZER("blind page color"); 349 static struct evcnt pmap_ev_vac_color_change = 350 PMAP_EVCNT_INITIALIZER("change page color"); 351 static struct evcnt pmap_ev_vac_color_erase = 352 PMAP_EVCNT_INITIALIZER("erase page color"); 353 static struct evcnt pmap_ev_vac_color_none = 354 PMAP_EVCNT_INITIALIZER("no page color"); 355 static struct evcnt pmap_ev_vac_color_restore = 356 PMAP_EVCNT_INITIALIZER("restore page color"); 357 358 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new); 359 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse); 360 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok); 361 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind); 362 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change); 363 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase); 364 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none); 365 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore); 366 #endif 367 368 static struct evcnt pmap_ev_mappings = 369 PMAP_EVCNT_INITIALIZER("pages mapped"); 370 static struct evcnt pmap_ev_unmappings = 371 PMAP_EVCNT_INITIALIZER("pages unmapped"); 372 static struct evcnt pmap_ev_remappings = 373 PMAP_EVCNT_INITIALIZER("pages remapped"); 374 375 EVCNT_ATTACH_STATIC(pmap_ev_mappings); 376 EVCNT_ATTACH_STATIC(pmap_ev_unmappings); 377 EVCNT_ATTACH_STATIC(pmap_ev_remappings); 378 379 static struct evcnt pmap_ev_kernel_mappings = 380 PMAP_EVCNT_INITIALIZER("kernel pages mapped"); 381 static struct evcnt pmap_ev_kernel_unmappings = 382 PMAP_EVCNT_INITIALIZER("kernel pages unmapped"); 383 static struct evcnt pmap_ev_kernel_remappings = 384 PMAP_EVCNT_INITIALIZER("kernel pages remapped"); 385 386 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings); 387 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings); 388 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings); 389 390 static struct evcnt pmap_ev_kenter_mappings = 391 PMAP_EVCNT_INITIALIZER("kenter pages mapped"); 392 static struct evcnt pmap_ev_kenter_unmappings = 393 PMAP_EVCNT_INITIALIZER("kenter pages unmapped"); 394 static struct evcnt pmap_ev_kenter_remappings = 395 PMAP_EVCNT_INITIALIZER("kenter pages remapped"); 396 static struct evcnt pmap_ev_pt_mappings = 397 PMAP_EVCNT_INITIALIZER("page table pages mapped"); 398 399 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings); 400 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings); 401 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings); 402 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings); 403 404 static struct evcnt pmap_ev_fixup_mod = 405 PMAP_EVCNT_INITIALIZER("page modification emulations"); 406 static struct evcnt pmap_ev_fixup_ref = 407 PMAP_EVCNT_INITIALIZER("page reference emulations"); 408 static struct evcnt pmap_ev_fixup_exec = 409 PMAP_EVCNT_INITIALIZER("exec pages fixed up"); 410 static struct evcnt pmap_ev_fixup_pdes = 411 PMAP_EVCNT_INITIALIZER("pdes fixed up"); 412 #ifndef ARM_MMU_EXTENDED 413 static struct evcnt pmap_ev_fixup_ptesync = 414 PMAP_EVCNT_INITIALIZER("ptesync fixed"); 415 #endif 416 417 EVCNT_ATTACH_STATIC(pmap_ev_fixup_mod); 418 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ref); 419 EVCNT_ATTACH_STATIC(pmap_ev_fixup_exec); 420 EVCNT_ATTACH_STATIC(pmap_ev_fixup_pdes); 421 #ifndef ARM_MMU_EXTENDED 422 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ptesync); 423 #endif 424 425 #ifdef PMAP_CACHE_VIPT 426 static struct evcnt pmap_ev_exec_mappings = 427 PMAP_EVCNT_INITIALIZER("exec pages mapped"); 428 static struct evcnt pmap_ev_exec_cached = 429 PMAP_EVCNT_INITIALIZER("exec pages cached"); 430 431 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings); 432 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached); 433 434 static struct evcnt pmap_ev_exec_synced = 435 PMAP_EVCNT_INITIALIZER("exec pages synced"); 436 static struct evcnt pmap_ev_exec_synced_map = 437 PMAP_EVCNT_INITIALIZER("exec pages synced (MP)"); 438 static struct evcnt pmap_ev_exec_synced_unmap = 439 PMAP_EVCNT_INITIALIZER("exec pages synced (UM)"); 440 static struct evcnt pmap_ev_exec_synced_remap = 441 PMAP_EVCNT_INITIALIZER("exec pages synced (RM)"); 442 static struct evcnt pmap_ev_exec_synced_clearbit = 443 PMAP_EVCNT_INITIALIZER("exec pages synced (DG)"); 444 #ifndef ARM_MMU_EXTENDED 445 static struct evcnt pmap_ev_exec_synced_kremove = 446 PMAP_EVCNT_INITIALIZER("exec pages synced (KU)"); 447 #endif 448 449 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced); 450 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map); 451 #ifndef ARM_MMU_EXTENDED 452 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap); 453 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap); 454 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit); 455 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove); 456 #endif 457 458 static struct evcnt pmap_ev_exec_discarded_unmap = 459 PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)"); 460 static struct evcnt pmap_ev_exec_discarded_zero = 461 PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)"); 462 static struct evcnt pmap_ev_exec_discarded_copy = 463 PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)"); 464 static struct evcnt pmap_ev_exec_discarded_page_protect = 465 PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)"); 466 static struct evcnt pmap_ev_exec_discarded_clearbit = 467 PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)"); 468 static struct evcnt pmap_ev_exec_discarded_kremove = 469 PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)"); 470 #ifdef ARM_MMU_EXTENDED 471 static struct evcnt pmap_ev_exec_discarded_modfixup = 472 PMAP_EVCNT_INITIALIZER("exec pages discarded (MF)"); 473 #endif 474 475 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap); 476 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero); 477 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy); 478 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect); 479 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit); 480 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove); 481 #ifdef ARM_MMU_EXTENDED 482 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_modfixup); 483 #endif 484 #endif /* PMAP_CACHE_VIPT */ 485 486 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates"); 487 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects"); 488 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations"); 489 490 EVCNT_ATTACH_STATIC(pmap_ev_updates); 491 EVCNT_ATTACH_STATIC(pmap_ev_collects); 492 EVCNT_ATTACH_STATIC(pmap_ev_activations); 493 494 #define PMAPCOUNT(x) ((void)(pmap_ev_##x.ev_count++)) 495 #else 496 #define PMAPCOUNT(x) ((void)0) 497 #endif 498 499 #ifdef ARM_MMU_EXTENDED 500 void pmap_md_pdetab_activate(pmap_t, struct lwp *); 501 void pmap_md_pdetab_deactivate(pmap_t pm); 502 #endif 503 504 /* 505 * pmap copy/zero page, and mem(5) hook point 506 */ 507 static pt_entry_t *csrc_pte, *cdst_pte; 508 static vaddr_t csrcp, cdstp; 509 #ifdef MULTIPROCESSOR 510 static size_t cnptes; 511 #define cpu_csrc_pte(o) (csrc_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 512 #define cpu_cdst_pte(o) (cdst_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT)) 513 #define cpu_csrcp(o) (csrcp + L2_S_SIZE * cnptes * cpu_number() + (o)) 514 #define cpu_cdstp(o) (cdstp + L2_S_SIZE * cnptes * cpu_number() + (o)) 515 #else 516 #define cpu_csrc_pte(o) (csrc_pte + ((o) >> L2_S_SHIFT)) 517 #define cpu_cdst_pte(o) (cdst_pte + ((o) >> L2_S_SHIFT)) 518 #define cpu_csrcp(o) (csrcp + (o)) 519 #define cpu_cdstp(o) (cdstp + (o)) 520 #endif 521 vaddr_t memhook; /* used by mem.c & others */ 522 kmutex_t memlock __cacheline_aligned; /* used by mem.c & others */ 523 kmutex_t pmap_lock __cacheline_aligned; 524 extern void *msgbufaddr; 525 int pmap_kmpages; 526 /* 527 * Flag to indicate if pmap_init() has done its thing 528 */ 529 bool pmap_initialized; 530 531 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 532 /* 533 * Virtual end of direct-mapped memory 534 */ 535 vaddr_t pmap_directlimit; 536 #endif 537 538 /* 539 * Misc. locking data structures 540 */ 541 542 static inline void 543 pmap_acquire_pmap_lock(pmap_t pm) 544 { 545 #if defined(MULTIPROCESSOR) && defined(DDB) 546 if (db_onproc != NULL) 547 return; 548 #endif 549 550 if (pm == pmap_kernel()) { 551 #ifdef MULTIPROCESSOR 552 KERNEL_LOCK(1, NULL); 553 #endif 554 } else { 555 mutex_enter(pm->pm_lock); 556 } 557 } 558 559 static inline void 560 pmap_release_pmap_lock(pmap_t pm) 561 { 562 #if defined(MULTIPROCESSOR) && defined(DDB) 563 if (db_onproc != NULL) 564 return; 565 #endif 566 if (pm == pmap_kernel()) { 567 #ifdef MULTIPROCESSOR 568 KERNEL_UNLOCK_ONE(NULL); 569 #endif 570 } else { 571 mutex_exit(pm->pm_lock); 572 } 573 } 574 575 static inline void 576 pmap_acquire_page_lock(struct vm_page_md *md) 577 { 578 mutex_enter(&pmap_lock); 579 } 580 581 static inline void 582 pmap_release_page_lock(struct vm_page_md *md) 583 { 584 mutex_exit(&pmap_lock); 585 } 586 587 #ifdef DIAGNOSTIC 588 static inline int 589 pmap_page_locked_p(struct vm_page_md *md) 590 { 591 return mutex_owned(&pmap_lock); 592 } 593 #endif 594 595 596 /* 597 * Metadata for L1 translation tables. 598 */ 599 #ifndef ARM_MMU_EXTENDED 600 struct l1_ttable { 601 /* Entry on the L1 Table list */ 602 SLIST_ENTRY(l1_ttable) l1_link; 603 604 /* Entry on the L1 Least Recently Used list */ 605 TAILQ_ENTRY(l1_ttable) l1_lru; 606 607 /* Track how many domains are allocated from this L1 */ 608 volatile u_int l1_domain_use_count; 609 610 /* 611 * A free-list of domain numbers for this L1. 612 * We avoid using ffs() and a bitmap to track domains since ffs() 613 * is slow on ARM. 614 */ 615 uint8_t l1_domain_first; 616 uint8_t l1_domain_free[PMAP_DOMAINS]; 617 618 /* Physical address of this L1 page table */ 619 paddr_t l1_physaddr; 620 621 /* KVA of this L1 page table */ 622 pd_entry_t *l1_kva; 623 }; 624 625 /* 626 * L1 Page Tables are tracked using a Least Recently Used list. 627 * - New L1s are allocated from the HEAD. 628 * - Freed L1s are added to the TAIl. 629 * - Recently accessed L1s (where an 'access' is some change to one of 630 * the userland pmaps which owns this L1) are moved to the TAIL. 631 */ 632 static TAILQ_HEAD(, l1_ttable) l1_lru_list; 633 static kmutex_t l1_lru_lock __cacheline_aligned; 634 635 /* 636 * A list of all L1 tables 637 */ 638 static SLIST_HEAD(, l1_ttable) l1_list; 639 #endif /* ARM_MMU_EXTENDED */ 640 641 /* 642 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 643 * 644 * This is normally 16MB worth L2 page descriptors for any given pmap. 645 * Reference counts are maintained for L2 descriptors so they can be 646 * freed when empty. 647 */ 648 struct l2_bucket { 649 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 650 paddr_t l2b_pa; /* Physical address of same */ 651 u_short l2b_l1slot; /* This L2 table's L1 index */ 652 u_short l2b_occupancy; /* How many active descriptors */ 653 }; 654 655 struct l2_dtable { 656 /* The number of L2 page descriptors allocated to this l2_dtable */ 657 u_int l2_occupancy; 658 659 /* List of L2 page descriptors */ 660 struct l2_bucket l2_bucket[L2_BUCKET_SIZE]; 661 }; 662 663 /* 664 * Given an L1 table index, calculate the corresponding l2_dtable index 665 * and bucket index within the l2_dtable. 666 */ 667 #define L2_BUCKET_XSHIFT (L2_BUCKET_XLOG2 - L1_S_SHIFT) 668 #define L2_BUCKET_XFRAME (~(vaddr_t)0 << L2_BUCKET_XLOG2) 669 #define L2_BUCKET_IDX(l1slot) ((l1slot) >> L2_BUCKET_XSHIFT) 670 #define L2_IDX(l1slot) (L2_BUCKET_IDX(l1slot) >> L2_BUCKET_LOG2) 671 #define L2_BUCKET(l1slot) (L2_BUCKET_IDX(l1slot) & (L2_BUCKET_SIZE - 1)) 672 673 __CTASSERT(0x100000000ULL == ((uint64_t)L2_SIZE * L2_BUCKET_SIZE * L1_S_SIZE)); 674 __CTASSERT(L2_BUCKET_XFRAME == ~(L2_BUCKET_XSIZE-1)); 675 676 /* 677 * Given a virtual address, this macro returns the 678 * virtual address required to drop into the next L2 bucket. 679 */ 680 #define L2_NEXT_BUCKET_VA(va) (((va) & L2_BUCKET_XFRAME) + L2_BUCKET_XSIZE) 681 682 /* 683 * L2 allocation. 684 */ 685 #define pmap_alloc_l2_dtable() \ 686 pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT) 687 #define pmap_free_l2_dtable(l2) \ 688 pool_cache_put(&pmap_l2dtable_cache, (l2)) 689 #define pmap_alloc_l2_ptp(pap) \ 690 ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\ 691 PR_NOWAIT, (pap))) 692 693 /* 694 * We try to map the page tables write-through, if possible. However, not 695 * all CPUs have a write-through cache mode, so on those we have to sync 696 * the cache when we frob page tables. 697 * 698 * We try to evaluate this at compile time, if possible. However, it's 699 * not always possible to do that, hence this run-time var. 700 */ 701 int pmap_needs_pte_sync; 702 703 /* 704 * Real definition of pv_entry. 705 */ 706 struct pv_entry { 707 SLIST_ENTRY(pv_entry) pv_link; /* next pv_entry */ 708 pmap_t pv_pmap; /* pmap where mapping lies */ 709 vaddr_t pv_va; /* virtual address for mapping */ 710 u_int pv_flags; /* flags */ 711 }; 712 713 /* 714 * Macros to determine if a mapping might be resident in the 715 * instruction/data cache and/or TLB 716 */ 717 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED) 718 /* 719 * Speculative loads by Cortex cores can cause TLB entries to be filled even if 720 * there are no explicit accesses, so there may be always be TLB entries to 721 * flush. If we used ASIDs then this would not be a problem. 722 */ 723 #define PV_BEEN_EXECD(f) (((f) & PVF_EXEC) == PVF_EXEC) 724 #define PV_BEEN_REFD(f) (true) 725 #else 726 #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 727 #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 728 #endif 729 #define PV_IS_EXEC_P(f) (((f) & PVF_EXEC) != 0) 730 #define PV_IS_KENTRY_P(f) (((f) & PVF_KENTRY) != 0) 731 #define PV_IS_WRITE_P(f) (((f) & PVF_WRITE) != 0) 732 733 /* 734 * Local prototypes 735 */ 736 static bool pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t, size_t); 737 static void pmap_alloc_specials(vaddr_t *, int, vaddr_t *, 738 pt_entry_t **); 739 static bool pmap_is_current(pmap_t) __unused; 740 static bool pmap_is_cached(pmap_t); 741 static void pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *, 742 pmap_t, vaddr_t, u_int); 743 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t); 744 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 745 static u_int pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t, 746 u_int, u_int); 747 748 static void pmap_pinit(pmap_t); 749 static int pmap_pmap_ctor(void *, void *, int); 750 751 static void pmap_alloc_l1(pmap_t); 752 static void pmap_free_l1(pmap_t); 753 #ifndef ARM_MMU_EXTENDED 754 static void pmap_use_l1(pmap_t); 755 #endif 756 757 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t); 758 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t); 759 static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 760 static int pmap_l2ptp_ctor(void *, void *, int); 761 static int pmap_l2dtable_ctor(void *, void *, int); 762 763 static void pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 764 #ifdef PMAP_CACHE_VIVT 765 static void pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 766 static void pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t); 767 #endif 768 769 static void pmap_clearbit(struct vm_page_md *, paddr_t, u_int); 770 #ifdef PMAP_CACHE_VIVT 771 static bool pmap_clean_page(struct vm_page_md *, bool); 772 #endif 773 #ifdef PMAP_CACHE_VIPT 774 static void pmap_syncicache_page(struct vm_page_md *, paddr_t); 775 enum pmap_flush_op { 776 PMAP_FLUSH_PRIMARY, 777 PMAP_FLUSH_SECONDARY, 778 PMAP_CLEAN_PRIMARY 779 }; 780 #ifndef ARM_MMU_EXTENDED 781 static void pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op); 782 #endif 783 #endif 784 static void pmap_page_remove(struct vm_page_md *, paddr_t); 785 static void pmap_pv_remove(paddr_t); 786 787 #ifndef ARM_MMU_EXTENDED 788 static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 789 #endif 790 static vaddr_t kernel_pt_lookup(paddr_t); 791 792 793 /* 794 * Misc variables 795 */ 796 vaddr_t virtual_avail; 797 vaddr_t virtual_end; 798 vaddr_t pmap_curmaxkvaddr; 799 800 paddr_t avail_start; 801 paddr_t avail_end; 802 803 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq); 804 pv_addr_t kernelpages; 805 pv_addr_t kernel_l1pt; 806 pv_addr_t systempage; 807 808 /* Function to set the debug level of the pmap code */ 809 810 #ifdef PMAP_DEBUG 811 void 812 pmap_debug(int level) 813 { 814 pmap_debug_level = level; 815 printf("pmap_debug: level=%d\n", pmap_debug_level); 816 } 817 #endif /* PMAP_DEBUG */ 818 819 #ifdef PMAP_CACHE_VIPT 820 #define PMAP_VALIDATE_MD_PAGE(md) \ 821 KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \ 822 "(md) %p: attrs=%#x urw=%u krw=%u", (md), \ 823 (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings); 824 #endif /* PMAP_CACHE_VIPT */ 825 /* 826 * A bunch of routines to conditionally flush the caches/TLB depending 827 * on whether the specified pmap actually needs to be flushed at any 828 * given time. 829 */ 830 static inline void 831 pmap_tlb_flush_SE(pmap_t pm, vaddr_t va, u_int flags) 832 { 833 #ifdef ARM_MMU_EXTENDED 834 pmap_tlb_invalidate_addr(pm, va); 835 #else 836 if (pm->pm_cstate.cs_tlb_id != 0) { 837 if (PV_BEEN_EXECD(flags)) { 838 cpu_tlb_flushID_SE(va); 839 } else if (PV_BEEN_REFD(flags)) { 840 cpu_tlb_flushD_SE(va); 841 } 842 } 843 #endif /* ARM_MMU_EXTENDED */ 844 } 845 846 #ifndef ARM_MMU_EXTENDED 847 static inline void 848 pmap_tlb_flushID(pmap_t pm) 849 { 850 if (pm->pm_cstate.cs_tlb_id) { 851 cpu_tlb_flushID(); 852 #if ARM_MMU_V7 == 0 853 /* 854 * Speculative loads by Cortex cores can cause TLB entries to 855 * be filled even if there are no explicit accesses, so there 856 * may be always be TLB entries to flush. If we used ASIDs 857 * then it would not be a problem. 858 * This is not true for other CPUs. 859 */ 860 pm->pm_cstate.cs_tlb = 0; 861 #endif /* ARM_MMU_V7 */ 862 } 863 } 864 865 static inline void 866 pmap_tlb_flushD(pmap_t pm) 867 { 868 if (pm->pm_cstate.cs_tlb_d) { 869 cpu_tlb_flushD(); 870 #if ARM_MMU_V7 == 0 871 /* 872 * Speculative loads by Cortex cores can cause TLB entries to 873 * be filled even if there are no explicit accesses, so there 874 * may be always be TLB entries to flush. If we used ASIDs 875 * then it would not be a problem. 876 * This is not true for other CPUs. 877 */ 878 pm->pm_cstate.cs_tlb_d = 0; 879 #endif /* ARM_MMU_V7 */ 880 } 881 } 882 #endif /* ARM_MMU_EXTENDED */ 883 884 #ifdef PMAP_CACHE_VIVT 885 static inline void 886 pmap_cache_wbinv_page(pmap_t pm, vaddr_t va, bool do_inv, u_int flags) 887 { 888 if (PV_BEEN_EXECD(flags) && pm->pm_cstate.cs_cache_id) { 889 cpu_idcache_wbinv_range(va, PAGE_SIZE); 890 } else if (PV_BEEN_REFD(flags) && pm->pm_cstate.cs_cache_d) { 891 if (do_inv) { 892 if (flags & PVF_WRITE) 893 cpu_dcache_wbinv_range(va, PAGE_SIZE); 894 else 895 cpu_dcache_inv_range(va, PAGE_SIZE); 896 } else if (flags & PVF_WRITE) { 897 cpu_dcache_wb_range(va, PAGE_SIZE); 898 } 899 } 900 } 901 902 static inline void 903 pmap_cache_wbinv_all(pmap_t pm, u_int flags) 904 { 905 if (PV_BEEN_EXECD(flags)) { 906 if (pm->pm_cstate.cs_cache_id) { 907 cpu_idcache_wbinv_all(); 908 pm->pm_cstate.cs_cache = 0; 909 } 910 } else if (pm->pm_cstate.cs_cache_d) { 911 cpu_dcache_wbinv_all(); 912 pm->pm_cstate.cs_cache_d = 0; 913 } 914 } 915 #endif /* PMAP_CACHE_VIVT */ 916 917 static inline uint8_t 918 pmap_domain(pmap_t pm) 919 { 920 #ifdef ARM_MMU_EXTENDED 921 return pm == pmap_kernel() ? PMAP_DOMAIN_KERNEL : PMAP_DOMAIN_USER; 922 #else 923 return pm->pm_domain; 924 #endif 925 } 926 927 static inline pd_entry_t * 928 pmap_l1_kva(pmap_t pm) 929 { 930 #ifdef ARM_MMU_EXTENDED 931 return pm->pm_l1; 932 #else 933 return pm->pm_l1->l1_kva; 934 #endif 935 } 936 937 static inline bool 938 pmap_is_current(pmap_t pm) 939 { 940 if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm) 941 return true; 942 943 return false; 944 } 945 946 static inline bool 947 pmap_is_cached(pmap_t pm) 948 { 949 #ifdef ARM_MMU_EXTENDED 950 if (pm == pmap_kernel()) 951 return true; 952 #ifdef MULTIPROCESSOR 953 // Is this pmap active on any CPU? 954 if (!kcpuset_iszero(pm->pm_active)) 955 return true; 956 #else 957 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 958 // Is this pmap active? 959 if (PMAP_PAI_ASIDVALID_P(PMAP_PAI(pm, ti), ti)) 960 return true; 961 #endif 962 #else 963 struct cpu_info * const ci = curcpu(); 964 if (pm == pmap_kernel() || ci->ci_pmap_lastuser == NULL 965 || ci->ci_pmap_lastuser == pm) 966 return true; 967 #endif /* ARM_MMU_EXTENDED */ 968 969 return false; 970 } 971 972 /* 973 * PTE_SYNC_CURRENT: 974 * 975 * Make sure the pte is written out to RAM. 976 * We need to do this for one of two cases: 977 * - We're dealing with the kernel pmap 978 * - There is no pmap active in the cache/tlb. 979 * - The specified pmap is 'active' in the cache/tlb. 980 */ 981 982 #ifdef PMAP_INCLUDE_PTE_SYNC 983 static inline void 984 pmap_pte_sync_current(pmap_t pm, pt_entry_t *ptep) 985 { 986 if (PMAP_NEEDS_PTE_SYNC && pmap_is_cached(pm)) 987 PTE_SYNC(ptep); 988 arm_dsb(); 989 } 990 991 # define PTE_SYNC_CURRENT(pm, ptep) pmap_pte_sync_current(pm, ptep) 992 #else 993 # define PTE_SYNC_CURRENT(pm, ptep) __nothing 994 #endif 995 996 /* 997 * main pv_entry manipulation functions: 998 * pmap_enter_pv: enter a mapping onto a vm_page list 999 * pmap_remove_pv: remove a mapping from a vm_page list 1000 * 1001 * NOTE: pmap_enter_pv expects to lock the pvh itself 1002 * pmap_remove_pv expects the caller to lock the pvh before calling 1003 */ 1004 1005 /* 1006 * pmap_enter_pv: enter a mapping onto a vm_page lst 1007 * 1008 * => caller should hold the proper lock on pmap_main_lock 1009 * => caller should have pmap locked 1010 * => we will gain the lock on the vm_page and allocate the new pv_entry 1011 * => caller should adjust ptp's wire_count before calling 1012 * => caller should not adjust pmap's wire_count 1013 */ 1014 static void 1015 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm, 1016 vaddr_t va, u_int flags) 1017 { 1018 struct pv_entry **pvp; 1019 1020 NPDEBUG(PDB_PVDUMP, 1021 printf("pmap_enter_pv: pm %p, md %p, flags 0x%x\n", pm, md, flags)); 1022 1023 pv->pv_pmap = pm; 1024 pv->pv_va = va; 1025 pv->pv_flags = flags; 1026 1027 pvp = &SLIST_FIRST(&md->pvh_list); 1028 #ifdef PMAP_CACHE_VIPT 1029 /* 1030 * Insert unmanaged entries, writeable first, at the head of 1031 * the pv list. 1032 */ 1033 if (__predict_true(!PV_IS_KENTRY_P(flags))) { 1034 while (*pvp != NULL && PV_IS_KENTRY_P((*pvp)->pv_flags)) 1035 pvp = &SLIST_NEXT(*pvp, pv_link); 1036 } 1037 if (!PV_IS_WRITE_P(flags)) { 1038 while (*pvp != NULL && PV_IS_WRITE_P((*pvp)->pv_flags)) 1039 pvp = &SLIST_NEXT(*pvp, pv_link); 1040 } 1041 #endif 1042 SLIST_NEXT(pv, pv_link) = *pvp; /* add to ... */ 1043 *pvp = pv; /* ... locked list */ 1044 md->pvh_attrs |= flags & (PVF_REF | PVF_MOD); 1045 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1046 if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE) 1047 md->pvh_attrs |= PVF_KMOD; 1048 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1049 md->pvh_attrs |= PVF_DIRTY; 1050 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1051 #endif 1052 if (pm == pmap_kernel()) { 1053 PMAPCOUNT(kernel_mappings); 1054 if (flags & PVF_WRITE) 1055 md->krw_mappings++; 1056 else 1057 md->kro_mappings++; 1058 } else { 1059 if (flags & PVF_WRITE) 1060 md->urw_mappings++; 1061 else 1062 md->uro_mappings++; 1063 } 1064 1065 #ifdef PMAP_CACHE_VIPT 1066 #ifndef ARM_MMU_EXTENDED 1067 /* 1068 * Even though pmap_vac_me_harder will set PVF_WRITE for us, 1069 * do it here as well to keep the mappings & KVF_WRITE consistent. 1070 */ 1071 if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) { 1072 md->pvh_attrs |= PVF_WRITE; 1073 } 1074 #endif 1075 /* 1076 * If this is an exec mapping and its the first exec mapping 1077 * for this page, make sure to sync the I-cache. 1078 */ 1079 if (PV_IS_EXEC_P(flags)) { 1080 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 1081 pmap_syncicache_page(md, pa); 1082 PMAPCOUNT(exec_synced_map); 1083 } 1084 PMAPCOUNT(exec_mappings); 1085 } 1086 #endif 1087 1088 PMAPCOUNT(mappings); 1089 1090 if (pv->pv_flags & PVF_WIRED) 1091 ++pm->pm_stats.wired_count; 1092 } 1093 1094 /* 1095 * 1096 * pmap_find_pv: Find a pv entry 1097 * 1098 * => caller should hold lock on vm_page 1099 */ 1100 static inline struct pv_entry * 1101 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va) 1102 { 1103 struct pv_entry *pv; 1104 1105 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1106 if (pm == pv->pv_pmap && va == pv->pv_va) 1107 break; 1108 } 1109 1110 return (pv); 1111 } 1112 1113 /* 1114 * pmap_remove_pv: try to remove a mapping from a pv_list 1115 * 1116 * => caller should hold proper lock on pmap_main_lock 1117 * => pmap should be locked 1118 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1119 * => caller should adjust ptp's wire_count and free PTP if needed 1120 * => caller should NOT adjust pmap's wire_count 1121 * => we return the removed pv 1122 */ 1123 static struct pv_entry * 1124 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1125 { 1126 struct pv_entry *pv, **prevptr; 1127 1128 NPDEBUG(PDB_PVDUMP, 1129 printf("pmap_remove_pv: pm %p, md %p, va 0x%08lx\n", pm, md, va)); 1130 1131 prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */ 1132 pv = *prevptr; 1133 1134 while (pv) { 1135 if (pv->pv_pmap == pm && pv->pv_va == va) { /* match? */ 1136 NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, md " 1137 "%p, flags 0x%x\n", pm, md, pv->pv_flags)); 1138 if (pv->pv_flags & PVF_WIRED) { 1139 --pm->pm_stats.wired_count; 1140 } 1141 *prevptr = SLIST_NEXT(pv, pv_link); /* remove it! */ 1142 if (pm == pmap_kernel()) { 1143 PMAPCOUNT(kernel_unmappings); 1144 if (pv->pv_flags & PVF_WRITE) 1145 md->krw_mappings--; 1146 else 1147 md->kro_mappings--; 1148 } else { 1149 if (pv->pv_flags & PVF_WRITE) 1150 md->urw_mappings--; 1151 else 1152 md->uro_mappings--; 1153 } 1154 1155 PMAPCOUNT(unmappings); 1156 #ifdef PMAP_CACHE_VIPT 1157 /* 1158 * If this page has had an exec mapping, then if 1159 * this was the last mapping, discard the contents, 1160 * otherwise sync the i-cache for this page. 1161 */ 1162 if (PV_IS_EXEC_P(md->pvh_attrs)) { 1163 if (SLIST_EMPTY(&md->pvh_list)) { 1164 md->pvh_attrs &= ~PVF_EXEC; 1165 PMAPCOUNT(exec_discarded_unmap); 1166 } else if (pv->pv_flags & PVF_WRITE) { 1167 pmap_syncicache_page(md, pa); 1168 PMAPCOUNT(exec_synced_unmap); 1169 } 1170 } 1171 #endif /* PMAP_CACHE_VIPT */ 1172 break; 1173 } 1174 prevptr = &SLIST_NEXT(pv, pv_link); /* previous pointer */ 1175 pv = *prevptr; /* advance */ 1176 } 1177 1178 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1179 /* 1180 * If we no longer have a WRITEABLE KENTRY at the head of list, 1181 * clear the KMOD attribute from the page. 1182 */ 1183 if (SLIST_FIRST(&md->pvh_list) == NULL 1184 || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE) 1185 md->pvh_attrs &= ~PVF_KMOD; 1186 1187 /* 1188 * If this was a writeable page and there are no more writeable 1189 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback 1190 * the contents to memory. 1191 */ 1192 if (arm_cache_prefer_mask != 0) { 1193 if (md->krw_mappings + md->urw_mappings == 0) 1194 md->pvh_attrs &= ~PVF_WRITE; 1195 PMAP_VALIDATE_MD_PAGE(md); 1196 } 1197 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1198 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1199 1200 /* return removed pv */ 1201 return pv; 1202 } 1203 1204 /* 1205 * 1206 * pmap_modify_pv: Update pv flags 1207 * 1208 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1209 * => caller should NOT adjust pmap's wire_count 1210 * => caller must call pmap_vac_me_harder() if writable status of a page 1211 * may have changed. 1212 * => we return the old flags 1213 * 1214 * Modify a physical-virtual mapping in the pv table 1215 */ 1216 static u_int 1217 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va, 1218 u_int clr_mask, u_int set_mask) 1219 { 1220 struct pv_entry *npv; 1221 u_int flags, oflags; 1222 1223 KASSERT(!PV_IS_KENTRY_P(clr_mask)); 1224 KASSERT(!PV_IS_KENTRY_P(set_mask)); 1225 1226 if ((npv = pmap_find_pv(md, pm, va)) == NULL) 1227 return (0); 1228 1229 NPDEBUG(PDB_PVDUMP, 1230 printf("pmap_modify_pv: pm %p, md %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, md, clr_mask, set_mask, npv->pv_flags)); 1231 1232 /* 1233 * There is at least one VA mapping this page. 1234 */ 1235 1236 if (clr_mask & (PVF_REF | PVF_MOD)) { 1237 md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1238 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 1239 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC) 1240 md->pvh_attrs |= PVF_DIRTY; 1241 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1242 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 1243 } 1244 1245 oflags = npv->pv_flags; 1246 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1247 1248 if ((flags ^ oflags) & PVF_WIRED) { 1249 if (flags & PVF_WIRED) 1250 ++pm->pm_stats.wired_count; 1251 else 1252 --pm->pm_stats.wired_count; 1253 } 1254 1255 if ((flags ^ oflags) & PVF_WRITE) { 1256 if (pm == pmap_kernel()) { 1257 if (flags & PVF_WRITE) { 1258 md->krw_mappings++; 1259 md->kro_mappings--; 1260 } else { 1261 md->kro_mappings++; 1262 md->krw_mappings--; 1263 } 1264 } else { 1265 if (flags & PVF_WRITE) { 1266 md->urw_mappings++; 1267 md->uro_mappings--; 1268 } else { 1269 md->uro_mappings++; 1270 md->urw_mappings--; 1271 } 1272 } 1273 } 1274 #ifdef PMAP_CACHE_VIPT 1275 if (arm_cache_prefer_mask != 0) { 1276 if (md->urw_mappings + md->krw_mappings == 0) { 1277 md->pvh_attrs &= ~PVF_WRITE; 1278 } else { 1279 md->pvh_attrs |= PVF_WRITE; 1280 } 1281 } 1282 /* 1283 * We have two cases here: the first is from enter_pv (new exec 1284 * page), the second is a combined pmap_remove_pv/pmap_enter_pv. 1285 * Since in latter, pmap_enter_pv won't do anything, we just have 1286 * to do what pmap_remove_pv would do. 1287 */ 1288 if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs)) 1289 || (PV_IS_EXEC_P(md->pvh_attrs) 1290 || (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) { 1291 pmap_syncicache_page(md, pa); 1292 PMAPCOUNT(exec_synced_remap); 1293 } 1294 #ifndef ARM_MMU_EXTENDED 1295 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 1296 #endif /* !ARM_MMU_EXTENDED */ 1297 #endif /* PMAP_CACHE_VIPT */ 1298 1299 PMAPCOUNT(remappings); 1300 1301 return (oflags); 1302 } 1303 1304 /* 1305 * Allocate an L1 translation table for the specified pmap. 1306 * This is called at pmap creation time. 1307 */ 1308 static void 1309 pmap_alloc_l1(pmap_t pm) 1310 { 1311 #ifdef ARM_MMU_EXTENDED 1312 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 1313 struct vm_page *pg; 1314 bool ok __diagused; 1315 for (;;) { 1316 #ifdef PMAP_NEED_ALLOC_POOLPAGE 1317 pg = arm_pmap_alloc_poolpage(UVM_PGA_ZERO); 1318 #else 1319 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1320 #endif 1321 if (pg != NULL) 1322 break; 1323 uvm_wait("pmapl1alloc"); 1324 } 1325 pm->pm_l1_pa = VM_PAGE_TO_PHYS(pg); 1326 vaddr_t va = pmap_direct_mapped_phys(pm->pm_l1_pa, &ok, 0); 1327 KASSERT(ok); 1328 KASSERT(va >= KERNEL_BASE); 1329 1330 #else 1331 KASSERTMSG(kernel_map != NULL, "pm %p", pm); 1332 vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1333 UVM_KMF_WIRED|UVM_KMF_ZERO); 1334 KASSERT(va); 1335 pmap_extract(pmap_kernel(), va, &pm->pm_l1_pa); 1336 #endif 1337 pm->pm_l1 = (pd_entry_t *)va; 1338 PTE_SYNC_RANGE(pm->pm_l1, PAGE_SIZE / sizeof(pt_entry_t)); 1339 #else 1340 struct l1_ttable *l1; 1341 uint8_t domain; 1342 1343 /* 1344 * Remove the L1 at the head of the LRU list 1345 */ 1346 mutex_spin_enter(&l1_lru_lock); 1347 l1 = TAILQ_FIRST(&l1_lru_list); 1348 KDASSERT(l1 != NULL); 1349 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1350 1351 /* 1352 * Pick the first available domain number, and update 1353 * the link to the next number. 1354 */ 1355 domain = l1->l1_domain_first; 1356 l1->l1_domain_first = l1->l1_domain_free[domain]; 1357 1358 /* 1359 * If there are still free domain numbers in this L1, 1360 * put it back on the TAIL of the LRU list. 1361 */ 1362 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 1363 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1364 1365 mutex_spin_exit(&l1_lru_lock); 1366 1367 /* 1368 * Fix up the relevant bits in the pmap structure 1369 */ 1370 pm->pm_l1 = l1; 1371 pm->pm_domain = domain + 1; 1372 #endif 1373 } 1374 1375 /* 1376 * Free an L1 translation table. 1377 * This is called at pmap destruction time. 1378 */ 1379 static void 1380 pmap_free_l1(pmap_t pm) 1381 { 1382 #ifdef ARM_MMU_EXTENDED 1383 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 1384 struct vm_page *pg = PHYS_TO_VM_PAGE(pm->pm_l1_pa); 1385 uvm_pagefree(pg); 1386 #else 1387 uvm_km_free(kernel_map, (vaddr_t)pm->pm_l1, PAGE_SIZE, UVM_KMF_WIRED); 1388 #endif 1389 pm->pm_l1 = NULL; 1390 pm->pm_l1_pa = 0; 1391 #else 1392 struct l1_ttable *l1 = pm->pm_l1; 1393 1394 mutex_spin_enter(&l1_lru_lock); 1395 1396 /* 1397 * If this L1 is currently on the LRU list, remove it. 1398 */ 1399 if (l1->l1_domain_use_count < PMAP_DOMAINS) 1400 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1401 1402 /* 1403 * Free up the domain number which was allocated to the pmap 1404 */ 1405 l1->l1_domain_free[pmap_domain(pm) - 1] = l1->l1_domain_first; 1406 l1->l1_domain_first = pmap_domain(pm) - 1; 1407 l1->l1_domain_use_count--; 1408 1409 /* 1410 * The L1 now must have at least 1 free domain, so add 1411 * it back to the LRU list. If the use count is zero, 1412 * put it at the head of the list, otherwise it goes 1413 * to the tail. 1414 */ 1415 if (l1->l1_domain_use_count == 0) 1416 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 1417 else 1418 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1419 1420 mutex_spin_exit(&l1_lru_lock); 1421 #endif /* ARM_MMU_EXTENDED */ 1422 } 1423 1424 #ifndef ARM_MMU_EXTENDED 1425 static inline void 1426 pmap_use_l1(pmap_t pm) 1427 { 1428 struct l1_ttable *l1; 1429 1430 /* 1431 * Do nothing if we're in interrupt context. 1432 * Access to an L1 by the kernel pmap must not affect 1433 * the LRU list. 1434 */ 1435 if (cpu_intr_p() || pm == pmap_kernel()) 1436 return; 1437 1438 l1 = pm->pm_l1; 1439 1440 /* 1441 * If the L1 is not currently on the LRU list, just return 1442 */ 1443 if (l1->l1_domain_use_count == PMAP_DOMAINS) 1444 return; 1445 1446 mutex_spin_enter(&l1_lru_lock); 1447 1448 /* 1449 * Check the use count again, now that we've acquired the lock 1450 */ 1451 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 1452 mutex_spin_exit(&l1_lru_lock); 1453 return; 1454 } 1455 1456 /* 1457 * Move the L1 to the back of the LRU list 1458 */ 1459 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 1460 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 1461 1462 mutex_spin_exit(&l1_lru_lock); 1463 } 1464 #endif /* !ARM_MMU_EXTENDED */ 1465 1466 /* 1467 * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *) 1468 * 1469 * Free an L2 descriptor table. 1470 */ 1471 static inline void 1472 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1473 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa) 1474 #else 1475 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa) 1476 #endif 1477 { 1478 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1479 /* 1480 * Note: With a write-back cache, we may need to sync this 1481 * L2 table before re-using it. 1482 * This is because it may have belonged to a non-current 1483 * pmap, in which case the cache syncs would have been 1484 * skipped for the pages that were being unmapped. If the 1485 * L2 table were then to be immediately re-allocated to 1486 * the *current* pmap, it may well contain stale mappings 1487 * which have not yet been cleared by a cache write-back 1488 * and so would still be visible to the mmu. 1489 */ 1490 if (need_sync) 1491 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1492 #endif /* PMAP_INCLUDE_PTE_SYNC && PMAP_CACHE_VIVT */ 1493 pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa); 1494 } 1495 1496 /* 1497 * Returns a pointer to the L2 bucket associated with the specified pmap 1498 * and VA, or NULL if no L2 bucket exists for the address. 1499 */ 1500 static inline struct l2_bucket * 1501 pmap_get_l2_bucket(pmap_t pm, vaddr_t va) 1502 { 1503 const size_t l1slot = l1pte_index(va); 1504 struct l2_dtable *l2; 1505 struct l2_bucket *l2b; 1506 1507 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL || 1508 (l2b = &l2->l2_bucket[L2_BUCKET(l1slot)])->l2b_kva == NULL) 1509 return (NULL); 1510 1511 return (l2b); 1512 } 1513 1514 /* 1515 * Returns a pointer to the L2 bucket associated with the specified pmap 1516 * and VA. 1517 * 1518 * If no L2 bucket exists, perform the necessary allocations to put an L2 1519 * bucket/page table in place. 1520 * 1521 * Note that if a new L2 bucket/page was allocated, the caller *must* 1522 * increment the bucket occupancy counter appropriately *before* 1523 * releasing the pmap's lock to ensure no other thread or cpu deallocates 1524 * the bucket/page in the meantime. 1525 */ 1526 static struct l2_bucket * 1527 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va) 1528 { 1529 const size_t l1slot = l1pte_index(va); 1530 struct l2_dtable *l2; 1531 1532 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 1533 /* 1534 * No mapping at this address, as there is 1535 * no entry in the L1 table. 1536 * Need to allocate a new l2_dtable. 1537 */ 1538 if ((l2 = pmap_alloc_l2_dtable()) == NULL) 1539 return (NULL); 1540 1541 /* 1542 * Link it into the parent pmap 1543 */ 1544 pm->pm_l2[L2_IDX(l1slot)] = l2; 1545 } 1546 1547 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 1548 1549 /* 1550 * Fetch pointer to the L2 page table associated with the address. 1551 */ 1552 if (l2b->l2b_kva == NULL) { 1553 pt_entry_t *ptep; 1554 1555 /* 1556 * No L2 page table has been allocated. Chances are, this 1557 * is because we just allocated the l2_dtable, above. 1558 */ 1559 if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_pa)) == NULL) { 1560 /* 1561 * Oops, no more L2 page tables available at this 1562 * time. We may need to deallocate the l2_dtable 1563 * if we allocated a new one above. 1564 */ 1565 if (l2->l2_occupancy == 0) { 1566 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1567 pmap_free_l2_dtable(l2); 1568 } 1569 return (NULL); 1570 } 1571 1572 l2->l2_occupancy++; 1573 l2b->l2b_kva = ptep; 1574 l2b->l2b_l1slot = l1slot; 1575 1576 #ifdef ARM_MMU_EXTENDED 1577 /* 1578 * We know there will be a mapping here, so simply 1579 * enter this PTP into the L1 now. 1580 */ 1581 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1582 pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 1583 | L1_C_DOM(pmap_domain(pm)); 1584 KASSERT(*pdep == 0); 1585 l1pte_setone(pdep, npde); 1586 PDE_SYNC(pdep); 1587 #endif 1588 } 1589 1590 return (l2b); 1591 } 1592 1593 /* 1594 * One or more mappings in the specified L2 descriptor table have just been 1595 * invalidated. 1596 * 1597 * Garbage collect the metadata and descriptor table itself if necessary. 1598 * 1599 * The pmap lock must be acquired when this is called (not necessary 1600 * for the kernel pmap). 1601 */ 1602 static void 1603 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1604 { 1605 KDASSERT(count <= l2b->l2b_occupancy); 1606 1607 /* 1608 * Update the bucket's reference count according to how many 1609 * PTEs the caller has just invalidated. 1610 */ 1611 l2b->l2b_occupancy -= count; 1612 1613 /* 1614 * Note: 1615 * 1616 * Level 2 page tables allocated to the kernel pmap are never freed 1617 * as that would require checking all Level 1 page tables and 1618 * removing any references to the Level 2 page table. See also the 1619 * comment elsewhere about never freeing bootstrap L2 descriptors. 1620 * 1621 * We make do with just invalidating the mapping in the L2 table. 1622 * 1623 * This isn't really a big deal in practice and, in fact, leads 1624 * to a performance win over time as we don't need to continually 1625 * alloc/free. 1626 */ 1627 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1628 return; 1629 1630 /* 1631 * There are no more valid mappings in this level 2 page table. 1632 * Go ahead and NULL-out the pointer in the bucket, then 1633 * free the page table. 1634 */ 1635 const size_t l1slot = l2b->l2b_l1slot; 1636 pt_entry_t * const ptep = l2b->l2b_kva; 1637 l2b->l2b_kva = NULL; 1638 1639 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 1640 pd_entry_t pde __diagused = *pdep; 1641 1642 #ifdef ARM_MMU_EXTENDED 1643 /* 1644 * Invalidate the L1 slot. 1645 */ 1646 KASSERT((pde & L1_TYPE_MASK) == L1_TYPE_C); 1647 #else 1648 /* 1649 * If the L1 slot matches the pmap's domain number, then invalidate it. 1650 */ 1651 if ((pde & (L1_C_DOM_MASK|L1_TYPE_MASK)) 1652 == (L1_C_DOM(pmap_domain(pm))|L1_TYPE_C)) { 1653 #endif 1654 l1pte_setone(pdep, 0); 1655 PDE_SYNC(pdep); 1656 #ifndef ARM_MMU_EXTENDED 1657 } 1658 #endif 1659 1660 /* 1661 * Release the L2 descriptor table back to the pool cache. 1662 */ 1663 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT) 1664 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_pa); 1665 #else 1666 pmap_free_l2_ptp(ptep, l2b->l2b_pa); 1667 #endif 1668 1669 /* 1670 * Update the reference count in the associated l2_dtable 1671 */ 1672 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 1673 if (--l2->l2_occupancy > 0) 1674 return; 1675 1676 /* 1677 * There are no more valid mappings in any of the Level 1 1678 * slots managed by this l2_dtable. Go ahead and NULL-out 1679 * the pointer in the parent pmap and free the l2_dtable. 1680 */ 1681 pm->pm_l2[L2_IDX(l1slot)] = NULL; 1682 pmap_free_l2_dtable(l2); 1683 } 1684 1685 /* 1686 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1687 * structures. 1688 */ 1689 static int 1690 pmap_l2ptp_ctor(void *arg, void *v, int flags) 1691 { 1692 #ifndef PMAP_INCLUDE_PTE_SYNC 1693 vaddr_t va = (vaddr_t)v & ~PGOFSET; 1694 1695 /* 1696 * The mappings for these page tables were initially made using 1697 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1698 * mode will not be right for page table mappings. To avoid 1699 * polluting the pmap_kenter_pa() code with a special case for 1700 * page tables, we simply fix up the cache-mode here if it's not 1701 * correct. 1702 */ 1703 if (pte_l2_s_cache_mode != pte_l2_s_cache_mode_pt) { 1704 const struct l2_bucket * const l2b = 1705 pmap_get_l2_bucket(pmap_kernel(), va); 1706 KASSERTMSG(l2b != NULL, "%#lx", va); 1707 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 1708 const pt_entry_t opte = *ptep; 1709 1710 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1711 /* 1712 * Page tables must have the cache-mode set correctly. 1713 */ 1714 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 1715 | pte_l2_s_cache_mode_pt; 1716 l2pte_set(ptep, npte, opte); 1717 PTE_SYNC(ptep); 1718 cpu_tlb_flushD_SE(va); 1719 cpu_cpwait(); 1720 } 1721 } 1722 #endif 1723 1724 memset(v, 0, L2_TABLE_SIZE_REAL); 1725 PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1726 return (0); 1727 } 1728 1729 static int 1730 pmap_l2dtable_ctor(void *arg, void *v, int flags) 1731 { 1732 1733 memset(v, 0, sizeof(struct l2_dtable)); 1734 return (0); 1735 } 1736 1737 static int 1738 pmap_pmap_ctor(void *arg, void *v, int flags) 1739 { 1740 1741 memset(v, 0, sizeof(struct pmap)); 1742 return (0); 1743 } 1744 1745 static void 1746 pmap_pinit(pmap_t pm) 1747 { 1748 #ifndef ARM_HAS_VBAR 1749 struct l2_bucket *l2b; 1750 1751 if (vector_page < KERNEL_BASE) { 1752 /* 1753 * Map the vector page. 1754 */ 1755 pmap_enter(pm, vector_page, systempage.pv_pa, 1756 VM_PROT_READ | VM_PROT_EXECUTE, 1757 VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED); 1758 pmap_update(pm); 1759 1760 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 1761 l2b = pmap_get_l2_bucket(pm, vector_page); 1762 KASSERTMSG(l2b != NULL, "%#lx", vector_page); 1763 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 1764 L1_C_DOM(pmap_domain(pm)); 1765 } else 1766 pm->pm_pl1vec = NULL; 1767 #endif 1768 } 1769 1770 #ifdef PMAP_CACHE_VIVT 1771 /* 1772 * Since we have a virtually indexed cache, we may need to inhibit caching if 1773 * there is more than one mapping and at least one of them is writable. 1774 * Since we purge the cache on every context switch, we only need to check for 1775 * other mappings within the same pmap, or kernel_pmap. 1776 * This function is also called when a page is unmapped, to possibly reenable 1777 * caching on any remaining mappings. 1778 * 1779 * The code implements the following logic, where: 1780 * 1781 * KW = # of kernel read/write pages 1782 * KR = # of kernel read only pages 1783 * UW = # of user read/write pages 1784 * UR = # of user read only pages 1785 * 1786 * KC = kernel mapping is cacheable 1787 * UC = user mapping is cacheable 1788 * 1789 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1790 * +--------------------------------------------- 1791 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1792 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1793 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1794 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1795 */ 1796 1797 static const int pmap_vac_flags[4][4] = { 1798 {-1, 0, 0, PVF_KNC}, 1799 {0, 0, PVF_NC, PVF_NC}, 1800 {0, PVF_NC, PVF_NC, PVF_NC}, 1801 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1802 }; 1803 1804 static inline int 1805 pmap_get_vac_flags(const struct vm_page_md *md) 1806 { 1807 int kidx, uidx; 1808 1809 kidx = 0; 1810 if (md->kro_mappings || md->krw_mappings > 1) 1811 kidx |= 1; 1812 if (md->krw_mappings) 1813 kidx |= 2; 1814 1815 uidx = 0; 1816 if (md->uro_mappings || md->urw_mappings > 1) 1817 uidx |= 1; 1818 if (md->urw_mappings) 1819 uidx |= 2; 1820 1821 return (pmap_vac_flags[uidx][kidx]); 1822 } 1823 1824 static inline void 1825 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1826 { 1827 int nattr; 1828 1829 nattr = pmap_get_vac_flags(md); 1830 1831 if (nattr < 0) { 1832 md->pvh_attrs &= ~PVF_NC; 1833 return; 1834 } 1835 1836 if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0) 1837 return; 1838 1839 if (pm == pmap_kernel()) 1840 pmap_vac_me_kpmap(md, pa, pm, va); 1841 else 1842 pmap_vac_me_user(md, pa, pm, va); 1843 1844 md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr; 1845 } 1846 1847 static void 1848 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1849 { 1850 u_int u_cacheable, u_entries; 1851 struct pv_entry *pv; 1852 pmap_t last_pmap = pm; 1853 1854 /* 1855 * Pass one, see if there are both kernel and user pmaps for 1856 * this page. Calculate whether there are user-writable or 1857 * kernel-writable pages. 1858 */ 1859 u_cacheable = 0; 1860 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1861 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1862 u_cacheable++; 1863 } 1864 1865 u_entries = md->urw_mappings + md->uro_mappings; 1866 1867 /* 1868 * We know we have just been updating a kernel entry, so if 1869 * all user pages are already cacheable, then there is nothing 1870 * further to do. 1871 */ 1872 if (md->k_mappings == 0 && u_cacheable == u_entries) 1873 return; 1874 1875 if (u_entries) { 1876 /* 1877 * Scan over the list again, for each entry, if it 1878 * might not be set correctly, call pmap_vac_me_user 1879 * to recalculate the settings. 1880 */ 1881 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1882 /* 1883 * We know kernel mappings will get set 1884 * correctly in other calls. We also know 1885 * that if the pmap is the same as last_pmap 1886 * then we've just handled this entry. 1887 */ 1888 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1889 continue; 1890 1891 /* 1892 * If there are kernel entries and this page 1893 * is writable but non-cacheable, then we can 1894 * skip this entry also. 1895 */ 1896 if (md->k_mappings && 1897 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1898 (PVF_NC | PVF_WRITE)) 1899 continue; 1900 1901 /* 1902 * Similarly if there are no kernel-writable 1903 * entries and the page is already 1904 * read-only/cacheable. 1905 */ 1906 if (md->krw_mappings == 0 && 1907 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1908 continue; 1909 1910 /* 1911 * For some of the remaining cases, we know 1912 * that we must recalculate, but for others we 1913 * can't tell if they are correct or not, so 1914 * we recalculate anyway. 1915 */ 1916 pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0); 1917 } 1918 1919 if (md->k_mappings == 0) 1920 return; 1921 } 1922 1923 pmap_vac_me_user(md, pa, pm, va); 1924 } 1925 1926 static void 1927 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 1928 { 1929 pmap_t kpmap = pmap_kernel(); 1930 struct pv_entry *pv, *npv = NULL; 1931 u_int entries = 0; 1932 u_int writable = 0; 1933 u_int cacheable_entries = 0; 1934 u_int kern_cacheable = 0; 1935 u_int other_writable = 0; 1936 1937 /* 1938 * Count mappings and writable mappings in this pmap. 1939 * Include kernel mappings as part of our own. 1940 * Keep a pointer to the first one. 1941 */ 1942 npv = NULL; 1943 KASSERT(pmap_page_locked_p(md)); 1944 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 1945 /* Count mappings in the same pmap */ 1946 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1947 if (entries++ == 0) 1948 npv = pv; 1949 1950 /* Cacheable mappings */ 1951 if ((pv->pv_flags & PVF_NC) == 0) { 1952 cacheable_entries++; 1953 if (kpmap == pv->pv_pmap) 1954 kern_cacheable++; 1955 } 1956 1957 /* Writable mappings */ 1958 if (pv->pv_flags & PVF_WRITE) 1959 ++writable; 1960 } else if (pv->pv_flags & PVF_WRITE) 1961 other_writable = 1; 1962 } 1963 1964 /* 1965 * Enable or disable caching as necessary. 1966 * Note: the first entry might be part of the kernel pmap, 1967 * so we can't assume this is indicative of the state of the 1968 * other (maybe non-kpmap) entries. 1969 */ 1970 if ((entries > 1 && writable) || 1971 (entries > 0 && pm == kpmap && other_writable)) { 1972 if (cacheable_entries == 0) { 1973 return; 1974 } 1975 1976 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 1977 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1978 (pv->pv_flags & PVF_NC)) 1979 continue; 1980 1981 pv->pv_flags |= PVF_NC; 1982 1983 struct l2_bucket * const l2b 1984 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1985 KASSERTMSG(l2b != NULL, "%#lx", va); 1986 pt_entry_t * const ptep 1987 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1988 const pt_entry_t opte = *ptep; 1989 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 1990 1991 if ((va != pv->pv_va || pm != pv->pv_pmap) 1992 && l2pte_valid_p(opte)) { 1993 pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va, 1994 true, pv->pv_flags); 1995 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 1996 pv->pv_flags); 1997 } 1998 1999 l2pte_set(ptep, npte, opte); 2000 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2001 } 2002 cpu_cpwait(); 2003 } else if (entries > cacheable_entries) { 2004 /* 2005 * Turn cacheing back on for some pages. If it is a kernel 2006 * page, only do so if there are no other writable pages. 2007 */ 2008 for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) { 2009 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 2010 (kpmap != pv->pv_pmap || other_writable))) 2011 continue; 2012 2013 pv->pv_flags &= ~PVF_NC; 2014 2015 struct l2_bucket * const l2b 2016 = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 2017 KASSERTMSG(l2b != NULL, "%#lx", va); 2018 pt_entry_t * const ptep 2019 = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2020 const pt_entry_t opte = *ptep; 2021 pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 2022 | pte_l2_s_cache_mode; 2023 2024 if (l2pte_valid_p(opte)) { 2025 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, 2026 pv->pv_flags); 2027 } 2028 2029 l2pte_set(ptep, npte, opte); 2030 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2031 } 2032 } 2033 } 2034 #endif 2035 2036 #ifdef PMAP_CACHE_VIPT 2037 static void 2038 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va) 2039 { 2040 #ifndef ARM_MMU_EXTENDED 2041 struct pv_entry *pv; 2042 vaddr_t tst_mask; 2043 bool bad_alias; 2044 const u_int 2045 rw_mappings = md->urw_mappings + md->krw_mappings, 2046 ro_mappings = md->uro_mappings + md->kro_mappings; 2047 2048 /* do we need to do anything? */ 2049 if (arm_cache_prefer_mask == 0) 2050 return; 2051 2052 NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: md=%p, pmap=%p va=%08lx\n", 2053 md, pm, va)); 2054 2055 KASSERT(!va || pm); 2056 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2057 2058 /* Already a conflict? */ 2059 if (__predict_false(md->pvh_attrs & PVF_NC)) { 2060 /* just an add, things are already non-cached */ 2061 KASSERT(!(md->pvh_attrs & PVF_DIRTY)); 2062 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2063 bad_alias = false; 2064 if (va) { 2065 PMAPCOUNT(vac_color_none); 2066 bad_alias = true; 2067 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2068 goto fixup; 2069 } 2070 pv = SLIST_FIRST(&md->pvh_list); 2071 /* the list can't be empty because it would be cachable */ 2072 if (md->pvh_attrs & PVF_KMPAGE) { 2073 tst_mask = md->pvh_attrs; 2074 } else { 2075 KASSERT(pv); 2076 tst_mask = pv->pv_va; 2077 pv = SLIST_NEXT(pv, pv_link); 2078 } 2079 /* 2080 * Only check for a bad alias if we have writable mappings. 2081 */ 2082 tst_mask &= arm_cache_prefer_mask; 2083 if (rw_mappings > 0) { 2084 for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) { 2085 /* if there's a bad alias, stop checking. */ 2086 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) 2087 bad_alias = true; 2088 } 2089 md->pvh_attrs |= PVF_WRITE; 2090 if (!bad_alias) 2091 md->pvh_attrs |= PVF_DIRTY; 2092 } else { 2093 /* 2094 * We have only read-only mappings. Let's see if there 2095 * are multiple colors in use or if we mapped a KMPAGE. 2096 * If the latter, we have a bad alias. If the former, 2097 * we need to remember that. 2098 */ 2099 for (; pv; pv = SLIST_NEXT(pv, pv_link)) { 2100 if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) { 2101 if (md->pvh_attrs & PVF_KMPAGE) 2102 bad_alias = true; 2103 break; 2104 } 2105 } 2106 md->pvh_attrs &= ~PVF_WRITE; 2107 /* 2108 * No KMPAGE and we exited early, so we must have 2109 * multiple color mappings. 2110 */ 2111 if (!bad_alias && pv != NULL) 2112 md->pvh_attrs |= PVF_MULTCLR; 2113 } 2114 2115 /* If no conflicting colors, set everything back to cached */ 2116 if (!bad_alias) { 2117 #ifdef DEBUG 2118 if ((md->pvh_attrs & PVF_WRITE) 2119 || ro_mappings < 2) { 2120 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2121 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2122 } 2123 #endif 2124 md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC; 2125 md->pvh_attrs |= tst_mask | PVF_COLORED; 2126 /* 2127 * Restore DIRTY bit if page is modified 2128 */ 2129 if (md->pvh_attrs & PVF_DMOD) 2130 md->pvh_attrs |= PVF_DIRTY; 2131 PMAPCOUNT(vac_color_restore); 2132 } else { 2133 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2134 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2135 } 2136 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2137 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2138 } else if (!va) { 2139 KASSERT(pmap_is_page_colored_p(md)); 2140 KASSERT(!(md->pvh_attrs & PVF_WRITE) 2141 || (md->pvh_attrs & PVF_DIRTY)); 2142 if (rw_mappings == 0) { 2143 md->pvh_attrs &= ~PVF_WRITE; 2144 if (ro_mappings == 1 2145 && (md->pvh_attrs & PVF_MULTCLR)) { 2146 /* 2147 * If this is the last readonly mapping 2148 * but it doesn't match the current color 2149 * for the page, change the current color 2150 * to match this last readonly mapping. 2151 */ 2152 pv = SLIST_FIRST(&md->pvh_list); 2153 tst_mask = (md->pvh_attrs ^ pv->pv_va) 2154 & arm_cache_prefer_mask; 2155 if (tst_mask) { 2156 md->pvh_attrs ^= tst_mask; 2157 PMAPCOUNT(vac_color_change); 2158 } 2159 } 2160 } 2161 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2162 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2163 return; 2164 } else if (!pmap_is_page_colored_p(md)) { 2165 /* not colored so we just use its color */ 2166 KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY)); 2167 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2168 PMAPCOUNT(vac_color_new); 2169 md->pvh_attrs &= PAGE_SIZE - 1; 2170 md->pvh_attrs |= PVF_COLORED 2171 | (va & arm_cache_prefer_mask) 2172 | (rw_mappings > 0 ? PVF_WRITE : 0); 2173 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2174 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2175 return; 2176 } else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) { 2177 bad_alias = false; 2178 if (rw_mappings > 0) { 2179 /* 2180 * We now have writeable mappings and if we have 2181 * readonly mappings in more than once color, we have 2182 * an aliasing problem. Regardless mark the page as 2183 * writeable. 2184 */ 2185 if (md->pvh_attrs & PVF_MULTCLR) { 2186 if (ro_mappings < 2) { 2187 /* 2188 * If we only have less than two 2189 * read-only mappings, just flush the 2190 * non-primary colors from the cache. 2191 */ 2192 pmap_flush_page(md, pa, 2193 PMAP_FLUSH_SECONDARY); 2194 } else { 2195 bad_alias = true; 2196 } 2197 } 2198 md->pvh_attrs |= PVF_WRITE; 2199 } 2200 /* If no conflicting colors, set everything back to cached */ 2201 if (!bad_alias) { 2202 #ifdef DEBUG 2203 if (rw_mappings > 0 2204 || (md->pvh_attrs & PMAP_KMPAGE)) { 2205 tst_mask = md->pvh_attrs & arm_cache_prefer_mask; 2206 SLIST_FOREACH(pv, &md->pvh_list, pv_link) 2207 KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0); 2208 } 2209 #endif 2210 if (SLIST_EMPTY(&md->pvh_list)) 2211 PMAPCOUNT(vac_color_reuse); 2212 else 2213 PMAPCOUNT(vac_color_ok); 2214 2215 /* matching color, just return */ 2216 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2217 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2218 return; 2219 } 2220 KASSERT(SLIST_FIRST(&md->pvh_list) != NULL); 2221 KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL); 2222 2223 /* color conflict. evict from cache. */ 2224 2225 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2226 md->pvh_attrs &= ~PVF_COLORED; 2227 md->pvh_attrs |= PVF_NC; 2228 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2229 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2230 PMAPCOUNT(vac_color_erase); 2231 } else if (rw_mappings == 0 2232 && (md->pvh_attrs & PVF_KMPAGE) == 0) { 2233 KASSERT((md->pvh_attrs & PVF_WRITE) == 0); 2234 2235 /* 2236 * If the page has dirty cache lines, clean it. 2237 */ 2238 if (md->pvh_attrs & PVF_DIRTY) 2239 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 2240 2241 /* 2242 * If this is the first remapping (we know that there are no 2243 * writeable mappings), then this is a simple color change. 2244 * Otherwise this is a seconary r/o mapping, which means 2245 * we don't have to do anything. 2246 */ 2247 if (ro_mappings == 1) { 2248 KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0); 2249 md->pvh_attrs &= PAGE_SIZE - 1; 2250 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2251 PMAPCOUNT(vac_color_change); 2252 } else { 2253 PMAPCOUNT(vac_color_blind); 2254 } 2255 md->pvh_attrs |= PVF_MULTCLR; 2256 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2257 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2258 return; 2259 } else { 2260 if (rw_mappings > 0) 2261 md->pvh_attrs |= PVF_WRITE; 2262 2263 /* color conflict. evict from cache. */ 2264 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 2265 2266 /* the list can't be empty because this was a enter/modify */ 2267 pv = SLIST_FIRST(&md->pvh_list); 2268 if ((md->pvh_attrs & PVF_KMPAGE) == 0) { 2269 KASSERT(pv); 2270 /* 2271 * If there's only one mapped page, change color to the 2272 * page's new color and return. Restore the DIRTY bit 2273 * that was erased by pmap_flush_page. 2274 */ 2275 if (SLIST_NEXT(pv, pv_link) == NULL) { 2276 md->pvh_attrs &= PAGE_SIZE - 1; 2277 md->pvh_attrs |= (va & arm_cache_prefer_mask); 2278 if (md->pvh_attrs & PVF_DMOD) 2279 md->pvh_attrs |= PVF_DIRTY; 2280 PMAPCOUNT(vac_color_change); 2281 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2282 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2283 KASSERT(!(md->pvh_attrs & PVF_MULTCLR)); 2284 return; 2285 } 2286 } 2287 bad_alias = true; 2288 md->pvh_attrs &= ~PVF_COLORED; 2289 md->pvh_attrs |= PVF_NC; 2290 PMAPCOUNT(vac_color_erase); 2291 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 2292 } 2293 2294 fixup: 2295 KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE)); 2296 2297 /* 2298 * Turn cacheing on/off for all pages. 2299 */ 2300 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2301 struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap, 2302 pv->pv_va); 2303 KASSERTMSG(l2b != NULL, "%#lx", va); 2304 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2305 const pt_entry_t opte = *ptep; 2306 pt_entry_t npte = opte & ~L2_S_CACHE_MASK; 2307 if (bad_alias) { 2308 pv->pv_flags |= PVF_NC; 2309 } else { 2310 pv->pv_flags &= ~PVF_NC; 2311 npte |= pte_l2_s_cache_mode; 2312 } 2313 2314 if (opte == npte) /* only update is there's a change */ 2315 continue; 2316 2317 if (l2pte_valid_p(opte)) { 2318 pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags); 2319 } 2320 2321 l2pte_set(ptep, npte, opte); 2322 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 2323 } 2324 #endif /* !ARM_MMU_EXTENDED */ 2325 } 2326 #endif /* PMAP_CACHE_VIPT */ 2327 2328 2329 /* 2330 * Modify pte bits for all ptes corresponding to the given physical address. 2331 * We use `maskbits' rather than `clearbits' because we're always passing 2332 * constants and the latter would require an extra inversion at run-time. 2333 */ 2334 static void 2335 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits) 2336 { 2337 struct pv_entry *pv; 2338 #ifdef PMAP_CACHE_VIPT 2339 const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs); 2340 bool need_syncicache = false; 2341 #ifdef ARM_MMU_EXTENDED 2342 const u_int execbits = (maskbits & PVF_EXEC) ? L2_XS_XN : 0; 2343 #else 2344 const u_int execbits = 0; 2345 bool need_vac_me_harder = false; 2346 #endif 2347 #else 2348 const u_int execbits = 0; 2349 #endif 2350 2351 NPDEBUG(PDB_BITS, 2352 printf("pmap_clearbit: md %p mask 0x%x\n", 2353 md, maskbits)); 2354 2355 #ifdef PMAP_CACHE_VIPT 2356 /* 2357 * If we might want to sync the I-cache and we've modified it, 2358 * then we know we definitely need to sync or discard it. 2359 */ 2360 if (want_syncicache) { 2361 if (md->pvh_attrs & PVF_MOD) { 2362 need_syncicache = true; 2363 } 2364 } 2365 #endif 2366 KASSERT(pmap_page_locked_p(md)); 2367 2368 /* 2369 * Clear saved attributes (modify, reference) 2370 */ 2371 md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 2372 2373 if (SLIST_EMPTY(&md->pvh_list)) { 2374 #if defined(PMAP_CACHE_VIPT) 2375 if (need_syncicache) { 2376 /* 2377 * No one has it mapped, so just discard it. The next 2378 * exec remapping will cause it to be synced. 2379 */ 2380 md->pvh_attrs &= ~PVF_EXEC; 2381 PMAPCOUNT(exec_discarded_clearbit); 2382 } 2383 #endif 2384 return; 2385 } 2386 2387 /* 2388 * Loop over all current mappings setting/clearing as appropos 2389 */ 2390 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2391 pmap_t pm = pv->pv_pmap; 2392 const vaddr_t va = pv->pv_va; 2393 const u_int oflags = pv->pv_flags; 2394 #ifndef ARM_MMU_EXTENDED 2395 /* 2396 * Kernel entries are unmanaged and as such not to be changed. 2397 */ 2398 if (PV_IS_KENTRY_P(oflags)) 2399 continue; 2400 #endif 2401 pv->pv_flags &= ~maskbits; 2402 2403 pmap_release_page_lock(md); 2404 pmap_acquire_pmap_lock(pm); 2405 2406 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va); 2407 if (l2b == NULL) { 2408 pmap_release_pmap_lock(pm); 2409 pmap_acquire_page_lock(md); 2410 continue; 2411 } 2412 KASSERTMSG(l2b != NULL, "%#lx", va); 2413 2414 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 2415 const pt_entry_t opte = *ptep; 2416 pt_entry_t npte = opte | execbits; 2417 2418 #ifdef ARM_MMU_EXTENDED 2419 KASSERT((opte & L2_XS_nG) == (pm == pmap_kernel() ? 0 : L2_XS_nG)); 2420 #endif 2421 2422 NPDEBUG(PDB_BITS, 2423 printf( "%s: pv %p, pm %p, va 0x%08lx, flag 0x%x\n", 2424 __func__, pv, pm, va, oflags)); 2425 2426 if (maskbits & (PVF_WRITE|PVF_MOD)) { 2427 #ifdef PMAP_CACHE_VIVT 2428 if ((oflags & PVF_NC)) { 2429 /* 2430 * Entry is not cacheable: 2431 * 2432 * Don't turn caching on again if this is a 2433 * modified emulation. This would be 2434 * inconsitent with the settings created by 2435 * pmap_vac_me_harder(). Otherwise, it's safe 2436 * to re-enable cacheing. 2437 * 2438 * There's no need to call pmap_vac_me_harder() 2439 * here: all pages are losing their write 2440 * permission. 2441 */ 2442 if (maskbits & PVF_WRITE) { 2443 npte |= pte_l2_s_cache_mode; 2444 pv->pv_flags &= ~PVF_NC; 2445 } 2446 } else if (l2pte_writable_p(opte)) { 2447 /* 2448 * Entry is writable/cacheable: check if pmap 2449 * is current if it is flush it, otherwise it 2450 * won't be in the cache 2451 */ 2452 pmap_cache_wbinv_page(pm, va, 2453 (maskbits & PVF_REF) != 0, 2454 oflags|PVF_WRITE); 2455 } 2456 #endif 2457 2458 /* make the pte read only */ 2459 npte = l2pte_set_readonly(npte); 2460 2461 pmap_acquire_page_lock(md); 2462 #ifdef MULTIPROCESSOR 2463 pv = pmap_find_pv(md, pm, va); 2464 #endif 2465 if (pv != NULL && (maskbits & oflags & PVF_WRITE)) { 2466 /* 2467 * Keep alias accounting up to date 2468 */ 2469 if (pm == pmap_kernel()) { 2470 md->krw_mappings--; 2471 md->kro_mappings++; 2472 } else { 2473 md->urw_mappings--; 2474 md->uro_mappings++; 2475 } 2476 #ifdef PMAP_CACHE_VIPT 2477 if (arm_cache_prefer_mask != 0) { 2478 if (md->urw_mappings + md->krw_mappings == 0) { 2479 md->pvh_attrs &= ~PVF_WRITE; 2480 } else { 2481 PMAP_VALIDATE_MD_PAGE(md); 2482 } 2483 } 2484 if (want_syncicache) 2485 need_syncicache = true; 2486 #ifndef ARM_MMU_EXTENDED 2487 need_vac_me_harder = true; 2488 #endif 2489 #endif /* PMAP_CACHE_VIPT */ 2490 } 2491 pmap_release_page_lock(md); 2492 } 2493 2494 if (maskbits & PVF_REF) { 2495 if (true 2496 #ifndef ARM_MMU_EXTENDED 2497 && (oflags & PVF_NC) == 0 2498 #endif 2499 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0 2500 && l2pte_valid_p(npte)) { 2501 #ifdef PMAP_CACHE_VIVT 2502 /* 2503 * Check npte here; we may have already 2504 * done the wbinv above, and the validity 2505 * of the PTE is the same for opte and 2506 * npte. 2507 */ 2508 pmap_cache_wbinv_page(pm, va, true, oflags); 2509 #endif 2510 } 2511 2512 /* 2513 * Make the PTE invalid so that we will take a 2514 * page fault the next time the mapping is 2515 * referenced. 2516 */ 2517 npte &= ~L2_TYPE_MASK; 2518 npte |= L2_TYPE_INV; 2519 } 2520 2521 if (npte != opte) { 2522 l2pte_reset(ptep); 2523 PTE_SYNC(ptep); 2524 2525 /* Flush the TLB entry if a current pmap. */ 2526 pmap_tlb_flush_SE(pm, va, oflags); 2527 2528 l2pte_set(ptep, npte, 0); 2529 PTE_SYNC(ptep); 2530 } 2531 2532 pmap_release_pmap_lock(pm); 2533 pmap_acquire_page_lock(md); 2534 2535 NPDEBUG(PDB_BITS, 2536 printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n", 2537 pm, va, opte, npte)); 2538 } 2539 2540 #if defined(PMAP_CACHE_VIPT) 2541 /* 2542 * If we need to sync the I-cache and we haven't done it yet, do it. 2543 */ 2544 if (need_syncicache) { 2545 pmap_release_page_lock(md); 2546 pmap_syncicache_page(md, pa); 2547 pmap_acquire_page_lock(md); 2548 PMAPCOUNT(exec_synced_clearbit); 2549 } 2550 #ifndef ARM_MMU_EXTENDED 2551 /* 2552 * If we are changing this to read-only, we need to call vac_me_harder 2553 * so we can change all the read-only pages to cacheable. We pretend 2554 * this as a page deletion. 2555 */ 2556 if (need_vac_me_harder) { 2557 if (md->pvh_attrs & PVF_NC) 2558 pmap_vac_me_harder(md, pa, NULL, 0); 2559 } 2560 #endif /* !ARM_MMU_EXTENDED */ 2561 #endif /* PMAP_CACHE_VIPT */ 2562 } 2563 2564 /* 2565 * pmap_clean_page() 2566 * 2567 * This is a local function used to work out the best strategy to clean 2568 * a single page referenced by its entry in the PV table. It's used by 2569 * pmap_copy_page, pmap_zero_page and maybe some others later on. 2570 * 2571 * Its policy is effectively: 2572 * o If there are no mappings, we don't bother doing anything with the cache. 2573 * o If there is one mapping, we clean just that page. 2574 * o If there are multiple mappings, we clean the entire cache. 2575 * 2576 * So that some functions can be further optimised, it returns 0 if it didn't 2577 * clean the entire cache, or 1 if it did. 2578 * 2579 * XXX One bug in this routine is that if the pv_entry has a single page 2580 * mapped at 0x00000000 a whole cache clean will be performed rather than 2581 * just the 1 page. Since this should not occur in everyday use and if it does 2582 * it will just result in not the most efficient clean for the page. 2583 */ 2584 #ifdef PMAP_CACHE_VIVT 2585 static bool 2586 pmap_clean_page(struct vm_page_md *md, bool is_src) 2587 { 2588 struct pv_entry *pv; 2589 pmap_t pm_to_clean = NULL; 2590 bool cache_needs_cleaning = false; 2591 vaddr_t page_to_clean = 0; 2592 u_int flags = 0; 2593 2594 /* 2595 * Since we flush the cache each time we change to a different 2596 * user vmspace, we only need to flush the page if it is in the 2597 * current pmap. 2598 */ 2599 KASSERT(pmap_page_locked_p(md)); 2600 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 2601 if (pmap_is_current(pv->pv_pmap)) { 2602 flags |= pv->pv_flags; 2603 /* 2604 * The page is mapped non-cacheable in 2605 * this map. No need to flush the cache. 2606 */ 2607 if (pv->pv_flags & PVF_NC) { 2608 #ifdef DIAGNOSTIC 2609 KASSERT(!cache_needs_cleaning); 2610 #endif 2611 break; 2612 } else if (is_src && (pv->pv_flags & PVF_WRITE) == 0) 2613 continue; 2614 if (cache_needs_cleaning) { 2615 page_to_clean = 0; 2616 break; 2617 } else { 2618 page_to_clean = pv->pv_va; 2619 pm_to_clean = pv->pv_pmap; 2620 } 2621 cache_needs_cleaning = true; 2622 } 2623 } 2624 2625 if (page_to_clean) { 2626 pmap_cache_wbinv_page(pm_to_clean, page_to_clean, 2627 !is_src, flags | PVF_REF); 2628 } else if (cache_needs_cleaning) { 2629 pmap_t const pm = curproc->p_vmspace->vm_map.pmap; 2630 2631 pmap_cache_wbinv_all(pm, flags); 2632 return true; 2633 } 2634 return false; 2635 } 2636 #endif 2637 2638 #ifdef PMAP_CACHE_VIPT 2639 /* 2640 * Sync a page with the I-cache. Since this is a VIPT, we must pick the 2641 * right cache alias to make sure we flush the right stuff. 2642 */ 2643 void 2644 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa) 2645 { 2646 pmap_t kpm = pmap_kernel(); 2647 const size_t way_size = arm_pcache.icache_type == CACHE_TYPE_PIPT 2648 ? PAGE_SIZE 2649 : arm_pcache.icache_way_size; 2650 2651 NPDEBUG(PDB_EXEC, printf("pmap_syncicache_page: md=%p (attrs=%#x)\n", 2652 md, md->pvh_attrs)); 2653 /* 2654 * No need to clean the page if it's non-cached. 2655 */ 2656 #ifndef ARM_MMU_EXTENDED 2657 if (md->pvh_attrs & PVF_NC) 2658 return; 2659 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED); 2660 #endif 2661 2662 pt_entry_t * const ptep = cpu_cdst_pte(0); 2663 const vaddr_t dstp = cpu_cdstp(0); 2664 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 2665 if (way_size <= PAGE_SIZE) { 2666 bool ok = false; 2667 vaddr_t vdstp = pmap_direct_mapped_phys(pa, &ok, dstp); 2668 if (ok) { 2669 cpu_icache_sync_range(vdstp, way_size); 2670 return; 2671 } 2672 } 2673 #endif 2674 2675 /* 2676 * We don't worry about the color of the exec page, we map the 2677 * same page to pages in the way and then do the icache_sync on 2678 * the entire way making sure we are cleaned. 2679 */ 2680 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 2681 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 2682 2683 for (size_t i = 0, j = 0; i < way_size; 2684 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2685 l2pte_reset(ptep + j); 2686 PTE_SYNC(ptep + j); 2687 2688 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2689 /* 2690 * Set up a PTE with to flush these cache lines. 2691 */ 2692 l2pte_set(ptep + j, npte, 0); 2693 } 2694 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2695 2696 /* 2697 * Flush it. 2698 */ 2699 cpu_icache_sync_range(dstp, way_size); 2700 2701 for (size_t i = 0, j = 0; i < way_size; 2702 i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) { 2703 /* 2704 * Unmap the page(s). 2705 */ 2706 l2pte_reset(ptep + j); 2707 pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC); 2708 } 2709 PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE); 2710 2711 md->pvh_attrs |= PVF_EXEC; 2712 PMAPCOUNT(exec_synced); 2713 } 2714 2715 #ifndef ARM_MMU_EXTENDED 2716 void 2717 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush) 2718 { 2719 vsize_t va_offset, end_va; 2720 bool wbinv_p; 2721 2722 if (arm_cache_prefer_mask == 0) 2723 return; 2724 2725 switch (flush) { 2726 case PMAP_FLUSH_PRIMARY: 2727 if (md->pvh_attrs & PVF_MULTCLR) { 2728 va_offset = 0; 2729 end_va = arm_cache_prefer_mask; 2730 md->pvh_attrs &= ~PVF_MULTCLR; 2731 PMAPCOUNT(vac_flush_lots); 2732 } else { 2733 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2734 end_va = va_offset; 2735 PMAPCOUNT(vac_flush_one); 2736 } 2737 /* 2738 * Mark that the page is no longer dirty. 2739 */ 2740 md->pvh_attrs &= ~PVF_DIRTY; 2741 wbinv_p = true; 2742 break; 2743 case PMAP_FLUSH_SECONDARY: 2744 va_offset = 0; 2745 end_va = arm_cache_prefer_mask; 2746 wbinv_p = true; 2747 md->pvh_attrs &= ~PVF_MULTCLR; 2748 PMAPCOUNT(vac_flush_lots); 2749 break; 2750 case PMAP_CLEAN_PRIMARY: 2751 va_offset = md->pvh_attrs & arm_cache_prefer_mask; 2752 end_va = va_offset; 2753 wbinv_p = false; 2754 /* 2755 * Mark that the page is no longer dirty. 2756 */ 2757 if ((md->pvh_attrs & PVF_DMOD) == 0) 2758 md->pvh_attrs &= ~PVF_DIRTY; 2759 PMAPCOUNT(vac_clean_one); 2760 break; 2761 default: 2762 return; 2763 } 2764 2765 KASSERT(!(md->pvh_attrs & PVF_NC)); 2766 2767 NPDEBUG(PDB_VAC, printf("pmap_flush_page: md=%p (attrs=%#x)\n", 2768 md, md->pvh_attrs)); 2769 2770 const size_t scache_line_size = arm_scache.dcache_line_size; 2771 2772 for (; va_offset <= end_va; va_offset += PAGE_SIZE) { 2773 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 2774 const vaddr_t dstp = cpu_cdstp(va_offset); 2775 const pt_entry_t opte = *ptep; 2776 2777 if (flush == PMAP_FLUSH_SECONDARY 2778 && va_offset == (md->pvh_attrs & arm_cache_prefer_mask)) 2779 continue; 2780 2781 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2782 /* 2783 * Set up a PTE with the right coloring to flush 2784 * existing cache entries. 2785 */ 2786 const pt_entry_t npte = L2_S_PROTO 2787 | pa 2788 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) 2789 | pte_l2_s_cache_mode; 2790 l2pte_set(ptep, npte, opte); 2791 PTE_SYNC(ptep); 2792 2793 /* 2794 * Flush it. Make sure to flush secondary cache too since 2795 * bus_dma will ignore uncached pages. 2796 */ 2797 if (scache_line_size != 0) { 2798 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2799 if (wbinv_p) { 2800 cpu_sdcache_wbinv_range(dstp, pa, PAGE_SIZE); 2801 cpu_dcache_inv_range(dstp, PAGE_SIZE); 2802 } else { 2803 cpu_sdcache_wb_range(dstp, pa, PAGE_SIZE); 2804 } 2805 } else { 2806 if (wbinv_p) { 2807 cpu_dcache_wbinv_range(dstp, PAGE_SIZE); 2808 } else { 2809 cpu_dcache_wb_range(dstp, PAGE_SIZE); 2810 } 2811 } 2812 2813 /* 2814 * Restore the page table entry since we might have interrupted 2815 * pmap_zero_page or pmap_copy_page which was already using 2816 * this pte. 2817 */ 2818 if (opte) { 2819 l2pte_set(ptep, opte, npte); 2820 } else { 2821 l2pte_reset(ptep); 2822 } 2823 PTE_SYNC(ptep); 2824 pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC); 2825 } 2826 } 2827 #endif /* ARM_MMU_EXTENDED */ 2828 #endif /* PMAP_CACHE_VIPT */ 2829 2830 /* 2831 * Routine: pmap_page_remove 2832 * Function: 2833 * Removes this physical page from 2834 * all physical maps in which it resides. 2835 * Reflects back modify bits to the pager. 2836 */ 2837 static void 2838 pmap_page_remove(struct vm_page_md *md, paddr_t pa) 2839 { 2840 struct l2_bucket *l2b; 2841 struct pv_entry *pv; 2842 pt_entry_t *ptep; 2843 #ifndef ARM_MMU_EXTENDED 2844 bool flush = false; 2845 #endif 2846 u_int flags = 0; 2847 2848 NPDEBUG(PDB_FOLLOW, 2849 printf("pmap_page_remove: md %p (0x%08lx)\n", md, 2850 pa)); 2851 2852 struct pv_entry **pvp = &SLIST_FIRST(&md->pvh_list); 2853 pmap_acquire_page_lock(md); 2854 if (*pvp == NULL) { 2855 #ifdef PMAP_CACHE_VIPT 2856 /* 2857 * We *know* the page contents are about to be replaced. 2858 * Discard the exec contents 2859 */ 2860 if (PV_IS_EXEC_P(md->pvh_attrs)) 2861 PMAPCOUNT(exec_discarded_page_protect); 2862 md->pvh_attrs &= ~PVF_EXEC; 2863 PMAP_VALIDATE_MD_PAGE(md); 2864 #endif 2865 pmap_release_page_lock(md); 2866 return; 2867 } 2868 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 2869 KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md)); 2870 #endif 2871 2872 /* 2873 * Clear alias counts 2874 */ 2875 #ifdef PMAP_CACHE_VIVT 2876 md->k_mappings = 0; 2877 #endif 2878 md->urw_mappings = md->uro_mappings = 0; 2879 2880 #ifdef PMAP_CACHE_VIVT 2881 pmap_clean_page(md, false); 2882 #endif 2883 2884 while ((pv = *pvp) != NULL) { 2885 pmap_t pm = pv->pv_pmap; 2886 #ifndef ARM_MMU_EXTENDED 2887 if (flush == false && pmap_is_current(pm)) 2888 flush = true; 2889 #endif 2890 2891 if (pm == pmap_kernel()) { 2892 #ifdef PMAP_CACHE_VIPT 2893 /* 2894 * If this was unmanaged mapping, it must be preserved. 2895 * Move it back on the list and advance the end-of-list 2896 * pointer. 2897 */ 2898 if (PV_IS_KENTRY_P(pv->pv_flags)) { 2899 *pvp = pv; 2900 pvp = &SLIST_NEXT(pv, pv_link); 2901 continue; 2902 } 2903 if (pv->pv_flags & PVF_WRITE) 2904 md->krw_mappings--; 2905 else 2906 md->kro_mappings--; 2907 #endif 2908 PMAPCOUNT(kernel_unmappings); 2909 } 2910 *pvp = SLIST_NEXT(pv, pv_link); /* remove from list */ 2911 PMAPCOUNT(unmappings); 2912 2913 pmap_release_page_lock(md); 2914 pmap_acquire_pmap_lock(pm); 2915 2916 l2b = pmap_get_l2_bucket(pm, pv->pv_va); 2917 KASSERTMSG(l2b != NULL, "%#lx", pv->pv_va); 2918 2919 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2920 2921 /* 2922 * Update statistics 2923 */ 2924 --pm->pm_stats.resident_count; 2925 2926 /* Wired bit */ 2927 if (pv->pv_flags & PVF_WIRED) 2928 --pm->pm_stats.wired_count; 2929 2930 flags |= pv->pv_flags; 2931 2932 /* 2933 * Invalidate the PTEs. 2934 */ 2935 l2pte_reset(ptep); 2936 PTE_SYNC_CURRENT(pm, ptep); 2937 2938 #ifdef ARM_MMU_EXTENDED 2939 pmap_tlb_invalidate_addr(pm, pv->pv_va); 2940 #endif 2941 2942 pmap_free_l2_bucket(pm, l2b, PAGE_SIZE / L2_S_SIZE); 2943 2944 pmap_release_pmap_lock(pm); 2945 2946 pool_put(&pmap_pv_pool, pv); 2947 pmap_acquire_page_lock(md); 2948 #ifdef MULTIPROCESSOR 2949 /* 2950 * Restart of the beginning of the list. 2951 */ 2952 pvp = &SLIST_FIRST(&md->pvh_list); 2953 #endif 2954 } 2955 /* 2956 * if we reach the end of the list and there are still mappings, they 2957 * might be able to be cached now. And they must be kernel mappings. 2958 */ 2959 if (!SLIST_EMPTY(&md->pvh_list)) { 2960 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 2961 } 2962 2963 #ifdef PMAP_CACHE_VIPT 2964 /* 2965 * Its EXEC cache is now gone. 2966 */ 2967 if (PV_IS_EXEC_P(md->pvh_attrs)) 2968 PMAPCOUNT(exec_discarded_page_protect); 2969 md->pvh_attrs &= ~PVF_EXEC; 2970 KASSERT(md->urw_mappings == 0); 2971 KASSERT(md->uro_mappings == 0); 2972 #ifndef ARM_MMU_EXTENDED 2973 if (arm_cache_prefer_mask != 0) { 2974 if (md->krw_mappings == 0) 2975 md->pvh_attrs &= ~PVF_WRITE; 2976 PMAP_VALIDATE_MD_PAGE(md); 2977 } 2978 #endif /* ARM_MMU_EXTENDED */ 2979 #endif /* PMAP_CACHE_VIPT */ 2980 pmap_release_page_lock(md); 2981 2982 #ifndef ARM_MMU_EXTENDED 2983 if (flush) { 2984 /* 2985 * Note: We can't use pmap_tlb_flush{I,D}() here since that 2986 * would need a subsequent call to pmap_update() to ensure 2987 * curpm->pm_cstate.cs_all is reset. Our callers are not 2988 * required to do that (see pmap(9)), so we can't modify 2989 * the current pmap's state. 2990 */ 2991 if (PV_BEEN_EXECD(flags)) 2992 cpu_tlb_flushID(); 2993 else 2994 cpu_tlb_flushD(); 2995 } 2996 cpu_cpwait(); 2997 #endif /* ARM_MMU_EXTENDED */ 2998 } 2999 3000 /* 3001 * pmap_t pmap_create(void) 3002 * 3003 * Create a new pmap structure from scratch. 3004 */ 3005 pmap_t 3006 pmap_create(void) 3007 { 3008 pmap_t pm; 3009 3010 pm = pool_cache_get(&pmap_cache, PR_WAITOK); 3011 3012 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE); 3013 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 3014 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 3015 3016 pm->pm_stats.wired_count = 0; 3017 pm->pm_stats.resident_count = 1; 3018 #ifdef ARM_MMU_EXTENDED 3019 #ifdef MULTIPROCESSOR 3020 kcpuset_create(&pm->pm_active, true); 3021 kcpuset_create(&pm->pm_onproc, true); 3022 #endif 3023 #else 3024 pm->pm_cstate.cs_all = 0; 3025 #endif 3026 pmap_alloc_l1(pm); 3027 3028 /* 3029 * Note: The pool cache ensures that the pm_l2[] array is already 3030 * initialised to zero. 3031 */ 3032 3033 pmap_pinit(pm); 3034 3035 return (pm); 3036 } 3037 3038 u_int 3039 arm32_mmap_flags(paddr_t pa) 3040 { 3041 /* 3042 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff 3043 * and we're using the upper bits in page numbers to pass flags around 3044 * so we might as well use the same bits 3045 */ 3046 return (u_int)pa & PMAP_MD_MASK; 3047 } 3048 /* 3049 * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, 3050 * u_int flags) 3051 * 3052 * Insert the given physical page (p) at 3053 * the specified virtual address (v) in the 3054 * target physical map with the protection requested. 3055 * 3056 * NB: This is the only routine which MAY NOT lazy-evaluate 3057 * or lose information. That is, this routine must actually 3058 * insert this page into the given map NOW. 3059 */ 3060 int 3061 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3062 { 3063 struct l2_bucket *l2b; 3064 struct vm_page *pg, *opg; 3065 u_int nflags; 3066 u_int oflags; 3067 const bool kpm_p = (pm == pmap_kernel()); 3068 #ifdef ARM_HAS_VBAR 3069 const bool vector_page_p = false; 3070 #else 3071 const bool vector_page_p = (va == vector_page); 3072 #endif 3073 3074 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3075 3076 UVMHIST_LOG(maphist, " (pm %#jx va %#jx pa %#jx prot %#jx", 3077 (uintptr_t)pm, va, pa, prot); 3078 UVMHIST_LOG(maphist, " flag %#jx", flags, 0, 0, 0); 3079 3080 KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0); 3081 KDASSERT(((va | pa) & PGOFSET) == 0); 3082 3083 /* 3084 * Get a pointer to the page. Later on in this function, we 3085 * test for a managed page by checking pg != NULL. 3086 */ 3087 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 3088 3089 nflags = 0; 3090 if (prot & VM_PROT_WRITE) 3091 nflags |= PVF_WRITE; 3092 if (prot & VM_PROT_EXECUTE) 3093 nflags |= PVF_EXEC; 3094 if (flags & PMAP_WIRED) 3095 nflags |= PVF_WIRED; 3096 3097 pmap_acquire_pmap_lock(pm); 3098 3099 /* 3100 * Fetch the L2 bucket which maps this page, allocating one if 3101 * necessary for user pmaps. 3102 */ 3103 if (kpm_p) { 3104 l2b = pmap_get_l2_bucket(pm, va); 3105 } else { 3106 l2b = pmap_alloc_l2_bucket(pm, va); 3107 } 3108 if (l2b == NULL) { 3109 if (flags & PMAP_CANFAIL) { 3110 pmap_release_pmap_lock(pm); 3111 return (ENOMEM); 3112 } 3113 panic("pmap_enter: failed to allocate L2 bucket"); 3114 } 3115 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)]; 3116 const pt_entry_t opte = *ptep; 3117 pt_entry_t npte = pa; 3118 oflags = 0; 3119 3120 if (opte) { 3121 /* 3122 * There is already a mapping at this address. 3123 * If the physical address is different, lookup the 3124 * vm_page. 3125 */ 3126 if (l2pte_pa(opte) != pa) { 3127 KASSERT(!pmap_pv_tracked(pa)); 3128 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3129 } else 3130 opg = pg; 3131 } else 3132 opg = NULL; 3133 3134 struct pmap_page *pp = pmap_pv_tracked(pa); 3135 3136 if (pg || pp) { 3137 KASSERT((pg != NULL) != (pp != NULL)); 3138 struct vm_page_md *md = (pg != NULL) ? VM_PAGE_TO_MD(pg) : 3139 PMAP_PAGE_TO_MD(pp); 3140 3141 /* 3142 * This is to be a managed mapping. 3143 */ 3144 pmap_acquire_page_lock(md); 3145 if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) { 3146 /* 3147 * - The access type indicates that we don't need 3148 * to do referenced emulation. 3149 * OR 3150 * - The physical page has already been referenced 3151 * so no need to re-do referenced emulation here. 3152 */ 3153 npte |= l2pte_set_readonly(L2_S_PROTO); 3154 3155 nflags |= PVF_REF; 3156 3157 if ((prot & VM_PROT_WRITE) != 0 && 3158 ((flags & VM_PROT_WRITE) != 0 || 3159 (md->pvh_attrs & PVF_MOD) != 0)) { 3160 /* 3161 * This is a writable mapping, and the 3162 * page's mod state indicates it has 3163 * already been modified. Make it 3164 * writable from the outset. 3165 */ 3166 npte = l2pte_set_writable(npte); 3167 nflags |= PVF_MOD; 3168 } 3169 3170 #ifdef ARM_MMU_EXTENDED 3171 /* 3172 * If the page has been cleaned, then the pvh_attrs 3173 * will have PVF_EXEC set, so mark it execute so we 3174 * don't get an access fault when trying to execute 3175 * from it. 3176 */ 3177 if (md->pvh_attrs & nflags & PVF_EXEC) { 3178 npte &= ~L2_XS_XN; 3179 } 3180 #endif 3181 } else { 3182 /* 3183 * Need to do page referenced emulation. 3184 */ 3185 npte |= L2_TYPE_INV; 3186 } 3187 3188 if (flags & ARM32_MMAP_WRITECOMBINE) { 3189 npte |= pte_l2_s_wc_mode; 3190 } else 3191 npte |= pte_l2_s_cache_mode; 3192 3193 if (pg != NULL && pg == opg) { 3194 /* 3195 * We're changing the attrs of an existing mapping. 3196 */ 3197 oflags = pmap_modify_pv(md, pa, pm, va, 3198 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3199 PVF_MOD | PVF_REF, nflags); 3200 3201 #ifdef PMAP_CACHE_VIVT 3202 /* 3203 * We may need to flush the cache if we're 3204 * doing rw-ro... 3205 */ 3206 if (pm->pm_cstate.cs_cache_d && 3207 (oflags & PVF_NC) == 0 && 3208 l2pte_writable_p(opte) && 3209 (prot & VM_PROT_WRITE) == 0) 3210 cpu_dcache_wb_range(va, PAGE_SIZE); 3211 #endif 3212 } else { 3213 struct pv_entry *pv; 3214 /* 3215 * New mapping, or changing the backing page 3216 * of an existing mapping. 3217 */ 3218 if (opg) { 3219 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3220 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3221 3222 /* 3223 * Replacing an existing mapping with a new one. 3224 * It is part of our managed memory so we 3225 * must remove it from the PV list 3226 */ 3227 pv = pmap_remove_pv(omd, opa, pm, va); 3228 pmap_vac_me_harder(omd, opa, pm, 0); 3229 oflags = pv->pv_flags; 3230 3231 #ifdef PMAP_CACHE_VIVT 3232 /* 3233 * If the old mapping was valid (ref/mod 3234 * emulation creates 'invalid' mappings 3235 * initially) then make sure to frob 3236 * the cache. 3237 */ 3238 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3239 pmap_cache_wbinv_page(pm, va, true, 3240 oflags); 3241 } 3242 #endif 3243 } else { 3244 pmap_release_page_lock(md); 3245 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3246 if (pv == NULL) { 3247 pmap_release_pmap_lock(pm); 3248 if ((flags & PMAP_CANFAIL) == 0) 3249 panic("pmap_enter: " 3250 "no pv entries"); 3251 3252 pmap_free_l2_bucket(pm, l2b, 0); 3253 UVMHIST_LOG(maphist, " <-- done (ENOMEM)", 3254 0, 0, 0, 0); 3255 return (ENOMEM); 3256 } 3257 pmap_acquire_page_lock(md); 3258 } 3259 3260 pmap_enter_pv(md, pa, pv, pm, va, nflags); 3261 } 3262 pmap_release_page_lock(md); 3263 } else { 3264 /* 3265 * We're mapping an unmanaged page. 3266 * These are always readable, and possibly writable, from 3267 * the get go as we don't need to track ref/mod status. 3268 */ 3269 npte |= l2pte_set_readonly(L2_S_PROTO); 3270 if (prot & VM_PROT_WRITE) 3271 npte = l2pte_set_writable(npte); 3272 3273 /* 3274 * Make sure the vector table is mapped cacheable 3275 */ 3276 if ((vector_page_p && !kpm_p) 3277 || (flags & ARM32_MMAP_CACHEABLE)) { 3278 npte |= pte_l2_s_cache_mode; 3279 #ifdef ARM_MMU_EXTENDED 3280 npte &= ~L2_XS_XN; /* and executable */ 3281 #endif 3282 } else if (flags & ARM32_MMAP_WRITECOMBINE) { 3283 npte |= pte_l2_s_wc_mode; 3284 } 3285 if (opg) { 3286 /* 3287 * Looks like there's an existing 'managed' mapping 3288 * at this address. 3289 */ 3290 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3291 paddr_t opa = VM_PAGE_TO_PHYS(opg); 3292 3293 pmap_acquire_page_lock(omd); 3294 struct pv_entry *pv = pmap_remove_pv(omd, opa, pm, va); 3295 pmap_vac_me_harder(omd, opa, pm, 0); 3296 oflags = pv->pv_flags; 3297 pmap_release_page_lock(omd); 3298 3299 #ifdef PMAP_CACHE_VIVT 3300 if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) { 3301 pmap_cache_wbinv_page(pm, va, true, oflags); 3302 } 3303 #endif 3304 pool_put(&pmap_pv_pool, pv); 3305 } 3306 } 3307 3308 /* 3309 * Make sure userland mappings get the right permissions 3310 */ 3311 if (!vector_page_p && !kpm_p) { 3312 npte |= L2_S_PROT_U; 3313 #ifdef ARM_MMU_EXTENDED 3314 npte |= L2_XS_nG; /* user pages are not global */ 3315 #endif 3316 } 3317 3318 /* 3319 * Keep the stats up to date 3320 */ 3321 if (opte == 0) { 3322 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3323 pm->pm_stats.resident_count++; 3324 } 3325 3326 UVMHIST_LOG(maphist, " opte %#jx npte %#jx", opte, npte, 0, 0); 3327 3328 #if defined(ARM_MMU_EXTENDED) 3329 /* 3330 * If exec protection was requested but the page hasn't been synced, 3331 * sync it now and allow execution from it. 3332 */ 3333 if ((nflags & PVF_EXEC) && (npte & L2_XS_XN)) { 3334 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3335 npte &= ~L2_XS_XN; 3336 pmap_syncicache_page(md, pa); 3337 PMAPCOUNT(exec_synced_map); 3338 } 3339 #endif 3340 /* 3341 * If this is just a wiring change, the two PTEs will be 3342 * identical, so there's no need to update the page table. 3343 */ 3344 if (npte != opte) { 3345 l2pte_reset(ptep); 3346 PTE_SYNC(ptep); 3347 if (l2pte_valid_p(opte)) { 3348 pmap_tlb_flush_SE(pm, va, oflags); 3349 } 3350 l2pte_set(ptep, npte, 0); 3351 PTE_SYNC(ptep); 3352 #ifndef ARM_MMU_EXTENDED 3353 bool is_cached = pmap_is_cached(pm); 3354 if (is_cached) { 3355 /* 3356 * We only need to frob the cache/tlb if this pmap 3357 * is current 3358 */ 3359 if (!vector_page_p && l2pte_valid_p(npte)) { 3360 /* 3361 * This mapping is likely to be accessed as 3362 * soon as we return to userland. Fix up the 3363 * L1 entry to avoid taking another 3364 * page/domain fault. 3365 */ 3366 pd_entry_t *pdep = pmap_l1_kva(pm) 3367 + l1pte_index(va); 3368 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa 3369 | L1_C_DOM(pmap_domain(pm)); 3370 if (*pdep != pde) { 3371 l1pte_setone(pdep, pde); 3372 PDE_SYNC(pdep); 3373 } 3374 } 3375 } 3376 #endif /* !ARM_MMU_EXTENDED */ 3377 3378 #ifndef ARM_MMU_EXTENDED 3379 UVMHIST_LOG(maphist, " is_cached %jd cs 0x%08jx", 3380 is_cached, pm->pm_cstate.cs_all, 0, 0); 3381 3382 if (pg != NULL) { 3383 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3384 3385 pmap_acquire_page_lock(md); 3386 pmap_vac_me_harder(md, pa, pm, va); 3387 pmap_release_page_lock(md); 3388 } 3389 #endif 3390 } 3391 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC) 3392 if (pg) { 3393 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3394 3395 pmap_acquire_page_lock(md); 3396 #ifndef ARM_MMU_EXTENDED 3397 KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3398 #endif 3399 PMAP_VALIDATE_MD_PAGE(md); 3400 pmap_release_page_lock(md); 3401 } 3402 #endif 3403 3404 pmap_release_pmap_lock(pm); 3405 3406 return (0); 3407 } 3408 3409 /* 3410 * pmap_remove() 3411 * 3412 * pmap_remove is responsible for nuking a number of mappings for a range 3413 * of virtual address space in the current pmap. To do this efficiently 3414 * is interesting, because in a number of cases a wide virtual address 3415 * range may be supplied that contains few actual mappings. So, the 3416 * optimisations are: 3417 * 1. Skip over hunks of address space for which no L1 or L2 entry exists. 3418 * 2. Build up a list of pages we've hit, up to a maximum, so we can 3419 * maybe do just a partial cache clean. This path of execution is 3420 * complicated by the fact that the cache must be flushed _before_ 3421 * the PTE is nuked, being a VAC :-) 3422 * 3. If we're called after UVM calls pmap_remove_all(), we can defer 3423 * all invalidations until pmap_update(), since pmap_remove_all() has 3424 * already flushed the cache. 3425 * 4. Maybe later fast-case a single page, but I don't think this is 3426 * going to make _that_ much difference overall. 3427 */ 3428 3429 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3430 3431 void 3432 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva) 3433 { 3434 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3435 UVMHIST_LOG(maphist, " (pm=%#jx, sva=%#jx, eva=%#jx)", 3436 (uintptr_t)pm, sva, eva, 0); 3437 3438 /* 3439 * we lock in the pmap => pv_head direction 3440 */ 3441 pmap_acquire_pmap_lock(pm); 3442 3443 #ifndef ARM_MMU_EXTENDED 3444 u_int cleanlist_idx, total, cnt; 3445 struct { 3446 vaddr_t va; 3447 pt_entry_t *ptep; 3448 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3449 3450 if (pm->pm_remove_all || !pmap_is_cached(pm)) { 3451 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3452 if (pm->pm_cstate.cs_tlb == 0) 3453 pm->pm_remove_all = true; 3454 } else 3455 cleanlist_idx = 0; 3456 total = 0; 3457 #endif 3458 3459 while (sva < eva) { 3460 /* 3461 * Do one L2 bucket's worth at a time. 3462 */ 3463 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(sva); 3464 if (next_bucket > eva) 3465 next_bucket = eva; 3466 3467 struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva); 3468 if (l2b == NULL) { 3469 sva = next_bucket; 3470 continue; 3471 } 3472 3473 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3474 u_int mappings = 0; 3475 3476 for (;sva < next_bucket; 3477 sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) { 3478 pt_entry_t opte = *ptep; 3479 3480 if (opte == 0) { 3481 /* Nothing here, move along */ 3482 continue; 3483 } 3484 3485 u_int flags = PVF_REF; 3486 paddr_t pa = l2pte_pa(opte); 3487 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 3488 3489 /* 3490 * Update flags. In a number of circumstances, 3491 * we could cluster a lot of these and do a 3492 * number of sequential pages in one go. 3493 */ 3494 if (pg != NULL) { 3495 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3496 struct pv_entry *pv; 3497 3498 pmap_acquire_page_lock(md); 3499 pv = pmap_remove_pv(md, pa, pm, sva); 3500 pmap_vac_me_harder(md, pa, pm, 0); 3501 pmap_release_page_lock(md); 3502 if (pv != NULL) { 3503 if (pm->pm_remove_all == false) { 3504 flags = pv->pv_flags; 3505 } 3506 pool_put(&pmap_pv_pool, pv); 3507 } 3508 } 3509 mappings += PAGE_SIZE / L2_S_SIZE; 3510 3511 if (!l2pte_valid_p(opte)) { 3512 /* 3513 * Ref/Mod emulation is still active for this 3514 * mapping, therefore it is has not yet been 3515 * accessed. No need to frob the cache/tlb. 3516 */ 3517 l2pte_reset(ptep); 3518 PTE_SYNC_CURRENT(pm, ptep); 3519 continue; 3520 } 3521 3522 #ifdef ARM_MMU_EXTENDED 3523 l2pte_reset(ptep); 3524 PTE_SYNC(ptep); 3525 if (__predict_false(pm->pm_remove_all == false)) { 3526 pmap_tlb_flush_SE(pm, sva, flags); 3527 } 3528 #else 3529 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3530 /* Add to the clean list. */ 3531 cleanlist[cleanlist_idx].ptep = ptep; 3532 cleanlist[cleanlist_idx].va = 3533 sva | (flags & PVF_EXEC); 3534 cleanlist_idx++; 3535 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3536 /* Nuke everything if needed. */ 3537 #ifdef PMAP_CACHE_VIVT 3538 pmap_cache_wbinv_all(pm, PVF_EXEC); 3539 #endif 3540 /* 3541 * Roll back the previous PTE list, 3542 * and zero out the current PTE. 3543 */ 3544 for (cnt = 0; 3545 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3546 l2pte_reset(cleanlist[cnt].ptep); 3547 PTE_SYNC(cleanlist[cnt].ptep); 3548 } 3549 l2pte_reset(ptep); 3550 PTE_SYNC(ptep); 3551 cleanlist_idx++; 3552 pm->pm_remove_all = true; 3553 } else { 3554 l2pte_reset(ptep); 3555 PTE_SYNC(ptep); 3556 if (pm->pm_remove_all == false) { 3557 pmap_tlb_flush_SE(pm, sva, flags); 3558 } 3559 } 3560 #endif 3561 } 3562 3563 #ifndef ARM_MMU_EXTENDED 3564 /* 3565 * Deal with any left overs 3566 */ 3567 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3568 total += cleanlist_idx; 3569 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3570 l2pte_reset(cleanlist[cnt].ptep); 3571 PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep); 3572 vaddr_t va = cleanlist[cnt].va; 3573 if (pm->pm_cstate.cs_all != 0) { 3574 vaddr_t clva = va & ~PAGE_MASK; 3575 u_int flags = va & PVF_EXEC; 3576 #ifdef PMAP_CACHE_VIVT 3577 pmap_cache_wbinv_page(pm, clva, true, 3578 PVF_REF | PVF_WRITE | flags); 3579 #endif 3580 pmap_tlb_flush_SE(pm, clva, 3581 PVF_REF | flags); 3582 } 3583 } 3584 3585 /* 3586 * If it looks like we're removing a whole bunch 3587 * of mappings, it's faster to just write-back 3588 * the whole cache now and defer TLB flushes until 3589 * pmap_update() is called. 3590 */ 3591 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3592 cleanlist_idx = 0; 3593 else { 3594 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3595 #ifdef PMAP_CACHE_VIVT 3596 pmap_cache_wbinv_all(pm, PVF_EXEC); 3597 #endif 3598 pm->pm_remove_all = true; 3599 } 3600 } 3601 #endif /* ARM_MMU_EXTENDED */ 3602 3603 pmap_free_l2_bucket(pm, l2b, mappings); 3604 pm->pm_stats.resident_count -= mappings / (PAGE_SIZE/L2_S_SIZE); 3605 } 3606 3607 pmap_release_pmap_lock(pm); 3608 } 3609 3610 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3611 static struct pv_entry * 3612 pmap_kremove_pg(struct vm_page *pg, vaddr_t va) 3613 { 3614 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 3615 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3616 struct pv_entry *pv; 3617 3618 KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC)); 3619 KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0); 3620 KASSERT(pmap_page_locked_p(md)); 3621 3622 pv = pmap_remove_pv(md, pa, pmap_kernel(), va); 3623 KASSERTMSG(pv, "pg %p (pa #%lx) va %#lx", pg, pa, va); 3624 KASSERT(PV_IS_KENTRY_P(pv->pv_flags)); 3625 3626 /* 3627 * If we are removing a writeable mapping to a cached exec page, 3628 * if it's the last mapping then clear it execness other sync 3629 * the page to the icache. 3630 */ 3631 if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC 3632 && (pv->pv_flags & PVF_WRITE) != 0) { 3633 if (SLIST_EMPTY(&md->pvh_list)) { 3634 md->pvh_attrs &= ~PVF_EXEC; 3635 PMAPCOUNT(exec_discarded_kremove); 3636 } else { 3637 pmap_syncicache_page(md, pa); 3638 PMAPCOUNT(exec_synced_kremove); 3639 } 3640 } 3641 pmap_vac_me_harder(md, pa, pmap_kernel(), 0); 3642 3643 return pv; 3644 } 3645 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */ 3646 3647 /* 3648 * pmap_kenter_pa: enter an unmanaged, wired kernel mapping 3649 * 3650 * We assume there is already sufficient KVM space available 3651 * to do this, as we can't allocate L2 descriptor tables/metadata 3652 * from here. 3653 */ 3654 void 3655 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 3656 { 3657 #ifdef PMAP_CACHE_VIVT 3658 struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL; 3659 #endif 3660 #ifdef PMAP_CACHE_VIPT 3661 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 3662 struct vm_page *opg; 3663 #ifndef ARM_MMU_EXTENDED 3664 struct pv_entry *pv = NULL; 3665 #endif 3666 #endif 3667 struct vm_page_md *md = pg != NULL ? VM_PAGE_TO_MD(pg) : NULL; 3668 3669 UVMHIST_FUNC(__func__); 3670 3671 if (pmap_initialized) { 3672 UVMHIST_CALLED(maphist); 3673 UVMHIST_LOG(maphist, " (va=%#jx, pa=%#jx, prot=%#jx, flags=%#jx", 3674 va, pa, prot, flags); 3675 } 3676 3677 pmap_t kpm = pmap_kernel(); 3678 pmap_acquire_pmap_lock(kpm); 3679 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3680 const size_t l1slot __diagused = l1pte_index(va); 3681 KASSERTMSG(l2b != NULL, 3682 "va %#lx pa %#lx prot %d maxkvaddr %#lx: l2 %p l2b %p kva %p", 3683 va, pa, prot, pmap_curmaxkvaddr, kpm->pm_l2[L2_IDX(l1slot)], 3684 kpm->pm_l2[L2_IDX(l1slot)] 3685 ? &kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)] 3686 : NULL, 3687 kpm->pm_l2[L2_IDX(l1slot)] 3688 ? kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)].l2b_kva 3689 : NULL); 3690 KASSERT(l2b->l2b_kva != NULL); 3691 3692 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 3693 const pt_entry_t opte = *ptep; 3694 3695 if (opte == 0) { 3696 PMAPCOUNT(kenter_mappings); 3697 l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE; 3698 } else { 3699 PMAPCOUNT(kenter_remappings); 3700 #ifdef PMAP_CACHE_VIPT 3701 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3702 #if !defined(ARM_MMU_EXTENDED) || defined(DIAGNOSTIC) 3703 struct vm_page_md *omd __diagused = VM_PAGE_TO_MD(opg); 3704 #endif 3705 if (opg && arm_cache_prefer_mask != 0) { 3706 KASSERT(opg != pg); 3707 KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0); 3708 KASSERT((flags & PMAP_KMPAGE) == 0); 3709 #ifndef ARM_MMU_EXTENDED 3710 pmap_acquire_page_lock(omd); 3711 pv = pmap_kremove_pg(opg, va); 3712 pmap_release_page_lock(omd); 3713 #endif 3714 } 3715 #endif 3716 if (l2pte_valid_p(opte)) { 3717 l2pte_reset(ptep); 3718 PTE_SYNC(ptep); 3719 #ifdef PMAP_CACHE_VIVT 3720 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3721 #endif 3722 cpu_tlb_flushD_SE(va); 3723 cpu_cpwait(); 3724 } 3725 } 3726 pmap_release_pmap_lock(kpm); 3727 pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 3728 3729 if (flags & PMAP_PTE) { 3730 KASSERT((flags & PMAP_CACHE_MASK) == 0); 3731 if (!(flags & PMAP_NOCACHE)) 3732 npte |= pte_l2_s_cache_mode_pt; 3733 } else { 3734 switch (flags & PMAP_CACHE_MASK) { 3735 case PMAP_NOCACHE: 3736 break; 3737 case PMAP_WRITE_COMBINE: 3738 npte |= pte_l2_s_wc_mode; 3739 break; 3740 default: 3741 npte |= pte_l2_s_cache_mode; 3742 break; 3743 } 3744 } 3745 #ifdef ARM_MMU_EXTENDED 3746 if (prot & VM_PROT_EXECUTE) 3747 npte &= ~L2_XS_XN; 3748 #endif 3749 l2pte_set(ptep, npte, 0); 3750 PTE_SYNC(ptep); 3751 3752 if (pg) { 3753 if (flags & PMAP_KMPAGE) { 3754 KASSERT(md->urw_mappings == 0); 3755 KASSERT(md->uro_mappings == 0); 3756 KASSERT(md->krw_mappings == 0); 3757 KASSERT(md->kro_mappings == 0); 3758 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3759 KASSERT(pv == NULL); 3760 KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0); 3761 KASSERT((md->pvh_attrs & PVF_NC) == 0); 3762 /* if there is a color conflict, evict from cache. */ 3763 if (pmap_is_page_colored_p(md) 3764 && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) { 3765 PMAPCOUNT(vac_color_change); 3766 pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY); 3767 } else if (md->pvh_attrs & PVF_MULTCLR) { 3768 /* 3769 * If this page has multiple colors, expunge 3770 * them. 3771 */ 3772 PMAPCOUNT(vac_flush_lots2); 3773 pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY); 3774 } 3775 /* 3776 * Since this is a KMPAGE, there can be no contention 3777 * for this page so don't lock it. 3778 */ 3779 md->pvh_attrs &= PAGE_SIZE - 1; 3780 md->pvh_attrs |= PVF_KMPAGE | PVF_COLORED | PVF_DIRTY 3781 | (va & arm_cache_prefer_mask); 3782 #else /* !PMAP_CACHE_VIPT || ARM_MMU_EXTENDED */ 3783 md->pvh_attrs |= PVF_KMPAGE; 3784 #endif 3785 atomic_inc_32(&pmap_kmpages); 3786 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3787 } else if (arm_cache_prefer_mask != 0) { 3788 if (pv == NULL) { 3789 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 3790 KASSERT(pv != NULL); 3791 } 3792 pmap_acquire_page_lock(md); 3793 pmap_enter_pv(md, pa, pv, pmap_kernel(), va, 3794 PVF_WIRED | PVF_KENTRY 3795 | (prot & VM_PROT_WRITE ? PVF_WRITE : 0)); 3796 if ((prot & VM_PROT_WRITE) 3797 && !(md->pvh_attrs & PVF_NC)) 3798 md->pvh_attrs |= PVF_DIRTY; 3799 KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC))); 3800 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 3801 pmap_release_page_lock(md); 3802 #endif 3803 } 3804 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3805 } else { 3806 if (pv != NULL) 3807 pool_put(&pmap_pv_pool, pv); 3808 #endif 3809 } 3810 if (pmap_initialized) { 3811 UVMHIST_LOG(maphist, " <-- done (ptep %#jx: %#jx -> %#jx)", 3812 (uintptr_t)ptep, opte, npte, 0); 3813 } 3814 3815 } 3816 3817 void 3818 pmap_kremove(vaddr_t va, vsize_t len) 3819 { 3820 #ifdef UVMHIST 3821 u_int total_mappings = 0; 3822 #endif 3823 3824 PMAPCOUNT(kenter_unmappings); 3825 3826 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 3827 3828 UVMHIST_LOG(maphist, " (va=%#jx, len=%#jx)", va, len, 0, 0); 3829 3830 const vaddr_t eva = va + len; 3831 3832 pmap_acquire_pmap_lock(pmap_kernel()); 3833 3834 while (va < eva) { 3835 vaddr_t next_bucket = L2_NEXT_BUCKET_VA(va); 3836 if (next_bucket > eva) 3837 next_bucket = eva; 3838 3839 pmap_t kpm = pmap_kernel(); 3840 struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va); 3841 KDASSERT(l2b != NULL); 3842 3843 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 3844 pt_entry_t *ptep = sptep; 3845 u_int mappings = 0; 3846 3847 while (va < next_bucket) { 3848 const pt_entry_t opte = *ptep; 3849 struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3850 if (opg != NULL) { 3851 struct vm_page_md *omd = VM_PAGE_TO_MD(opg); 3852 3853 if (omd->pvh_attrs & PVF_KMPAGE) { 3854 KASSERT(omd->urw_mappings == 0); 3855 KASSERT(omd->uro_mappings == 0); 3856 KASSERT(omd->krw_mappings == 0); 3857 KASSERT(omd->kro_mappings == 0); 3858 omd->pvh_attrs &= ~PVF_KMPAGE; 3859 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3860 if (arm_cache_prefer_mask != 0) { 3861 omd->pvh_attrs &= ~PVF_WRITE; 3862 } 3863 #endif 3864 atomic_dec_32(&pmap_kmpages); 3865 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 3866 } else if (arm_cache_prefer_mask != 0) { 3867 pmap_acquire_page_lock(omd); 3868 pool_put(&pmap_pv_pool, 3869 pmap_kremove_pg(opg, va)); 3870 pmap_release_page_lock(omd); 3871 #endif 3872 } 3873 } 3874 if (l2pte_valid_p(opte)) { 3875 l2pte_reset(ptep); 3876 PTE_SYNC(ptep); 3877 #ifdef PMAP_CACHE_VIVT 3878 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3879 #endif 3880 cpu_tlb_flushD_SE(va); 3881 3882 mappings += PAGE_SIZE / L2_S_SIZE; 3883 } 3884 va += PAGE_SIZE; 3885 ptep += PAGE_SIZE / L2_S_SIZE; 3886 } 3887 KDASSERTMSG(mappings <= l2b->l2b_occupancy, "%u %u", 3888 mappings, l2b->l2b_occupancy); 3889 l2b->l2b_occupancy -= mappings; 3890 //PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3891 #ifdef UVMHIST 3892 total_mappings += mappings; 3893 #endif 3894 } 3895 pmap_release_pmap_lock(pmap_kernel()); 3896 cpu_cpwait(); 3897 UVMHIST_LOG(maphist, " <--- done (%ju mappings removed)", 3898 total_mappings, 0, 0, 0); 3899 } 3900 3901 bool 3902 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 3903 { 3904 3905 return pmap_extract_coherency(pm, va, pap, NULL); 3906 } 3907 3908 bool 3909 pmap_extract_coherency(pmap_t pm, vaddr_t va, paddr_t *pap, bool *coherentp) 3910 { 3911 struct l2_dtable *l2; 3912 pd_entry_t *pdep, pde; 3913 pt_entry_t *ptep, pte; 3914 paddr_t pa; 3915 u_int l1slot; 3916 bool coherent; 3917 3918 pmap_acquire_pmap_lock(pm); 3919 3920 l1slot = l1pte_index(va); 3921 pdep = pmap_l1_kva(pm) + l1slot; 3922 pde = *pdep; 3923 3924 if (l1pte_section_p(pde)) { 3925 /* 3926 * These should only happen for pmap_kernel() 3927 */ 3928 KDASSERT(pm == pmap_kernel()); 3929 pmap_release_pmap_lock(pm); 3930 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 3931 if (l1pte_supersection_p(pde)) { 3932 pa = (pde & L1_SS_FRAME) | (va & L1_SS_OFFSET); 3933 } else 3934 #endif 3935 pa = (pde & L1_S_FRAME) | (va & L1_S_OFFSET); 3936 coherent = (pde & L1_S_CACHE_MASK) == 0; 3937 } else { 3938 /* 3939 * Note that we can't rely on the validity of the L1 3940 * descriptor as an indication that a mapping exists. 3941 * We have to look it up in the L2 dtable. 3942 */ 3943 l2 = pm->pm_l2[L2_IDX(l1slot)]; 3944 3945 if (l2 == NULL || 3946 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 3947 pmap_release_pmap_lock(pm); 3948 return false; 3949 } 3950 3951 pte = ptep[l2pte_index(va)]; 3952 pmap_release_pmap_lock(pm); 3953 3954 if (pte == 0) 3955 return false; 3956 3957 switch (pte & L2_TYPE_MASK) { 3958 case L2_TYPE_L: 3959 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3960 coherent = (pte & L2_L_CACHE_MASK) == 0; 3961 break; 3962 3963 default: 3964 pa = (pte & ~PAGE_MASK) | (va & PAGE_MASK); 3965 coherent = (pte & L2_S_CACHE_MASK) == 0; 3966 break; 3967 } 3968 } 3969 3970 if (pap != NULL) 3971 *pap = pa; 3972 3973 if (coherentp != NULL) 3974 *coherentp = (pm == pmap_kernel() && coherent); 3975 3976 return true; 3977 } 3978 3979 /* 3980 * pmap_pv_remove: remove an unmanaged pv-tracked page from all pmaps 3981 * that map it 3982 */ 3983 3984 static void 3985 pmap_pv_remove(paddr_t pa) 3986 { 3987 struct pmap_page *pp; 3988 3989 pp = pmap_pv_tracked(pa); 3990 if (pp == NULL) 3991 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 3992 pa); 3993 3994 struct vm_page_md *md = PMAP_PAGE_TO_MD(pp); 3995 pmap_page_remove(md, pa); 3996 } 3997 3998 void 3999 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 4000 { 4001 4002 /* the only case is remove at the moment */ 4003 KASSERT(prot == VM_PROT_NONE); 4004 pmap_pv_remove(pa); 4005 } 4006 4007 void 4008 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 4009 { 4010 struct l2_bucket *l2b; 4011 vaddr_t next_bucket; 4012 4013 NPDEBUG(PDB_PROTECT, 4014 printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n", 4015 pm, sva, eva, prot)); 4016 4017 if ((prot & VM_PROT_READ) == 0) { 4018 pmap_remove(pm, sva, eva); 4019 return; 4020 } 4021 4022 if (prot & VM_PROT_WRITE) { 4023 /* 4024 * If this is a read->write transition, just ignore it and let 4025 * uvm_fault() take care of it later. 4026 */ 4027 return; 4028 } 4029 4030 pmap_acquire_pmap_lock(pm); 4031 4032 #ifndef ARM_MMU_EXTENDED 4033 const bool flush = eva - sva >= PAGE_SIZE * 4; 4034 u_int flags = 0; 4035 #endif 4036 u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC); 4037 4038 while (sva < eva) { 4039 next_bucket = L2_NEXT_BUCKET_VA(sva); 4040 if (next_bucket > eva) 4041 next_bucket = eva; 4042 4043 l2b = pmap_get_l2_bucket(pm, sva); 4044 if (l2b == NULL) { 4045 sva = next_bucket; 4046 continue; 4047 } 4048 4049 pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4050 4051 while (sva < next_bucket) { 4052 const pt_entry_t opte = *ptep; 4053 if (l2pte_valid_p(opte) && l2pte_writable_p(opte)) { 4054 struct vm_page *pg; 4055 #ifndef ARM_MMU_EXTENDED 4056 u_int f; 4057 #endif 4058 4059 #ifdef PMAP_CACHE_VIVT 4060 /* 4061 * OK, at this point, we know we're doing 4062 * write-protect operation. If the pmap is 4063 * active, write-back the page. 4064 */ 4065 pmap_cache_wbinv_page(pm, sva, false, 4066 PVF_REF | PVF_WRITE); 4067 #endif 4068 4069 pg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 4070 pt_entry_t npte = l2pte_set_readonly(opte); 4071 l2pte_reset(ptep); 4072 PTE_SYNC(ptep); 4073 #ifdef ARM_MMU_EXTENDED 4074 pmap_tlb_flush_SE(pm, sva, PVF_REF); 4075 #endif 4076 l2pte_set(ptep, npte, 0); 4077 PTE_SYNC(ptep); 4078 4079 if (pg != NULL) { 4080 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4081 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4082 4083 pmap_acquire_page_lock(md); 4084 #ifndef ARM_MMU_EXTENDED 4085 f = 4086 #endif 4087 pmap_modify_pv(md, pa, pm, sva, 4088 clr_mask, 0); 4089 pmap_vac_me_harder(md, pa, pm, sva); 4090 pmap_release_page_lock(md); 4091 #ifndef ARM_MMU_EXTENDED 4092 } else { 4093 f = PVF_REF | PVF_EXEC; 4094 } 4095 4096 if (flush) { 4097 flags |= f; 4098 } else { 4099 pmap_tlb_flush_SE(pm, sva, f); 4100 #endif 4101 } 4102 } 4103 4104 sva += PAGE_SIZE; 4105 ptep += PAGE_SIZE / L2_S_SIZE; 4106 } 4107 } 4108 4109 #ifndef ARM_MMU_EXTENDED 4110 if (flush) { 4111 if (PV_BEEN_EXECD(flags)) { 4112 pmap_tlb_flushID(pm); 4113 } else if (PV_BEEN_REFD(flags)) { 4114 pmap_tlb_flushD(pm); 4115 } 4116 } 4117 #endif 4118 4119 pmap_release_pmap_lock(pm); 4120 } 4121 4122 void 4123 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva) 4124 { 4125 struct l2_bucket *l2b; 4126 pt_entry_t *ptep; 4127 vaddr_t next_bucket; 4128 vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva; 4129 4130 NPDEBUG(PDB_EXEC, 4131 printf("pmap_icache_sync_range: pm %p sva 0x%lx eva 0x%lx\n", 4132 pm, sva, eva)); 4133 4134 pmap_acquire_pmap_lock(pm); 4135 4136 while (sva < eva) { 4137 next_bucket = L2_NEXT_BUCKET_VA(sva); 4138 if (next_bucket > eva) 4139 next_bucket = eva; 4140 4141 l2b = pmap_get_l2_bucket(pm, sva); 4142 if (l2b == NULL) { 4143 sva = next_bucket; 4144 continue; 4145 } 4146 4147 for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; 4148 sva < next_bucket; 4149 sva += page_size, 4150 ptep += PAGE_SIZE / L2_S_SIZE, 4151 page_size = PAGE_SIZE) { 4152 if (l2pte_valid_p(*ptep)) { 4153 cpu_icache_sync_range(sva, 4154 uimin(page_size, eva - sva)); 4155 } 4156 } 4157 } 4158 4159 pmap_release_pmap_lock(pm); 4160 } 4161 4162 void 4163 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 4164 { 4165 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4166 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4167 4168 NPDEBUG(PDB_PROTECT, 4169 printf("pmap_page_protect: md %p (0x%08lx), prot 0x%x\n", 4170 md, pa, prot)); 4171 4172 switch(prot) { 4173 case VM_PROT_READ|VM_PROT_WRITE: 4174 #if defined(ARM_MMU_EXTENDED) 4175 pmap_acquire_page_lock(md); 4176 pmap_clearbit(md, pa, PVF_EXEC); 4177 pmap_release_page_lock(md); 4178 break; 4179 #endif 4180 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 4181 break; 4182 4183 case VM_PROT_READ: 4184 #if defined(ARM_MMU_EXTENDED) 4185 pmap_acquire_page_lock(md); 4186 pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC); 4187 pmap_release_page_lock(md); 4188 break; 4189 #endif 4190 case VM_PROT_READ|VM_PROT_EXECUTE: 4191 pmap_acquire_page_lock(md); 4192 pmap_clearbit(md, pa, PVF_WRITE); 4193 pmap_release_page_lock(md); 4194 break; 4195 4196 default: 4197 pmap_page_remove(md, pa); 4198 break; 4199 } 4200 } 4201 4202 /* 4203 * pmap_clear_modify: 4204 * 4205 * Clear the "modified" attribute for a page. 4206 */ 4207 bool 4208 pmap_clear_modify(struct vm_page *pg) 4209 { 4210 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4211 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4212 bool rv; 4213 4214 pmap_acquire_page_lock(md); 4215 4216 if (md->pvh_attrs & PVF_MOD) { 4217 rv = true; 4218 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4219 /* 4220 * If we are going to clear the modified bit and there are 4221 * no other modified bits set, flush the page to memory and 4222 * mark it clean. 4223 */ 4224 if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD) 4225 pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY); 4226 #endif 4227 pmap_clearbit(md, pa, PVF_MOD); 4228 } else { 4229 rv = false; 4230 } 4231 pmap_release_page_lock(md); 4232 4233 return rv; 4234 } 4235 4236 /* 4237 * pmap_clear_reference: 4238 * 4239 * Clear the "referenced" attribute for a page. 4240 */ 4241 bool 4242 pmap_clear_reference(struct vm_page *pg) 4243 { 4244 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4245 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4246 bool rv; 4247 4248 pmap_acquire_page_lock(md); 4249 4250 if (md->pvh_attrs & PVF_REF) { 4251 rv = true; 4252 pmap_clearbit(md, pa, PVF_REF); 4253 } else { 4254 rv = false; 4255 } 4256 pmap_release_page_lock(md); 4257 4258 return rv; 4259 } 4260 4261 /* 4262 * pmap_is_modified: 4263 * 4264 * Test if a page has the "modified" attribute. 4265 */ 4266 /* See <arm/arm32/pmap.h> */ 4267 4268 /* 4269 * pmap_is_referenced: 4270 * 4271 * Test if a page has the "referenced" attribute. 4272 */ 4273 /* See <arm/arm32/pmap.h> */ 4274 4275 #if defined(ARM_MMU_EXTENDED) && 0 4276 int 4277 pmap_prefetchabt_fixup(void *v) 4278 { 4279 struct trapframe * const tf = v; 4280 vaddr_t va = trunc_page(tf->tf_pc); 4281 int rv = ABORT_FIXUP_FAILED; 4282 4283 if (!TRAP_USERMODE(tf) && va < VM_MAXUSER_ADDRESS) 4284 return rv; 4285 4286 kpreempt_disable(); 4287 pmap_t pm = curcpu()->ci_pmap_cur; 4288 const size_t l1slot = l1pte_index(va); 4289 struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)]; 4290 if (l2 == NULL) 4291 goto out; 4292 4293 struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4294 if (l2b->l2b_kva == NULL) 4295 goto out; 4296 4297 /* 4298 * Check the PTE itself. 4299 */ 4300 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4301 const pt_entry_t opte = *ptep; 4302 if ((opte & L2_S_PROT_U) == 0 || (opte & L2_XS_XN) == 0) 4303 goto out; 4304 4305 paddr_t pa = l2pte_pa(opte); 4306 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4307 KASSERT(pg != NULL); 4308 4309 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4310 4311 pmap_acquire_page_lock(md); 4312 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4313 KASSERT(pv != NULL); 4314 4315 if (PV_IS_EXEC_P(pv->pv_flags)) { 4316 l2pte_reset(ptep); 4317 PTE_SYNC(ptep); 4318 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4319 if (!PV_IS_EXEC_P(md->pvh_attrs)) { 4320 pmap_syncicache_page(md, pa); 4321 } 4322 rv = ABORT_FIXUP_RETURN; 4323 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4324 PTE_SYNC(ptep); 4325 } 4326 pmap_release_page_lock(md); 4327 4328 out: 4329 kpreempt_enable(); 4330 return rv; 4331 } 4332 #endif 4333 4334 int 4335 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user) 4336 { 4337 struct l2_dtable *l2; 4338 struct l2_bucket *l2b; 4339 paddr_t pa; 4340 const size_t l1slot = l1pte_index(va); 4341 int rv = 0; 4342 4343 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4344 4345 va = trunc_page(va); 4346 4347 KASSERT(!user || (pm != pmap_kernel())); 4348 4349 UVMHIST_LOG(maphist, " (pm=%#jx, va=%#jx, ftype=%#jx, user=%jd)", 4350 (uintptr_t)pm, va, ftype, user); 4351 #ifdef ARM_MMU_EXTENDED 4352 UVMHIST_LOG(maphist, " ti=%#jx pai=%#jx asid=%#jx", 4353 (uintptr_t)cpu_tlb_info(curcpu()), 4354 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu())), 4355 (uintptr_t)PMAP_PAI(pm, cpu_tlb_info(curcpu()))->pai_asid, 0); 4356 #endif 4357 4358 pmap_acquire_pmap_lock(pm); 4359 4360 /* 4361 * If there is no l2_dtable for this address, then the process 4362 * has no business accessing it. 4363 * 4364 * Note: This will catch userland processes trying to access 4365 * kernel addresses. 4366 */ 4367 l2 = pm->pm_l2[L2_IDX(l1slot)]; 4368 if (l2 == NULL) { 4369 UVMHIST_LOG(maphist, " no l2 for l1slot %#jx", l1slot, 0, 0, 0); 4370 goto out; 4371 } 4372 4373 /* 4374 * Likewise if there is no L2 descriptor table 4375 */ 4376 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 4377 if (l2b->l2b_kva == NULL) { 4378 UVMHIST_LOG(maphist, " <-- done (no ptep for l1slot %#jx)", 4379 l1slot, 0, 0, 0); 4380 goto out; 4381 } 4382 4383 /* 4384 * Check the PTE itself. 4385 */ 4386 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 4387 pt_entry_t const opte = *ptep; 4388 if (opte == 0 || (opte & L2_TYPE_MASK) == L2_TYPE_L) { 4389 UVMHIST_LOG(maphist, " <-- done (empty pde for l1slot %#jx)", 4390 l1slot, 0, 0, 0); 4391 goto out; 4392 } 4393 4394 #ifndef ARM_HAS_VBAR 4395 /* 4396 * Catch a userland access to the vector page mapped at 0x0 4397 */ 4398 if (user && (opte & L2_S_PROT_U) == 0) { 4399 UVMHIST_LOG(maphist, " <-- done (vector_page)", 0, 0, 0, 0); 4400 goto out; 4401 } 4402 #endif 4403 4404 pa = l2pte_pa(opte); 4405 4406 if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(opte)) { 4407 /* 4408 * This looks like a good candidate for "page modified" 4409 * emulation... 4410 */ 4411 struct pv_entry *pv; 4412 struct vm_page *pg; 4413 4414 /* Extract the physical address of the page */ 4415 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4416 UVMHIST_LOG(maphist, " <-- done (mod/ref unmanaged page)", 0, 0, 0, 0); 4417 goto out; 4418 } 4419 4420 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4421 4422 /* Get the current flags for this page. */ 4423 pmap_acquire_page_lock(md); 4424 pv = pmap_find_pv(md, pm, va); 4425 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4426 pmap_release_page_lock(md); 4427 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: no PV)", 0, 0, 0, 0); 4428 goto out; 4429 } 4430 4431 /* 4432 * Do the flags say this page is writable? If not then it 4433 * is a genuine write fault. If yes then the write fault is 4434 * our fault as we did not reflect the write access in the 4435 * PTE. Now we know a write has occurred we can correct this 4436 * and also set the modified bit 4437 */ 4438 if ((pv->pv_flags & PVF_WRITE) == 0) { 4439 pmap_release_page_lock(md); 4440 goto out; 4441 } 4442 4443 md->pvh_attrs |= PVF_REF | PVF_MOD; 4444 pv->pv_flags |= PVF_REF | PVF_MOD; 4445 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 4446 /* 4447 * If there are cacheable mappings for this page, mark it dirty. 4448 */ 4449 if ((md->pvh_attrs & PVF_NC) == 0) 4450 md->pvh_attrs |= PVF_DIRTY; 4451 #endif 4452 #ifdef ARM_MMU_EXTENDED 4453 if (md->pvh_attrs & PVF_EXEC) { 4454 md->pvh_attrs &= ~PVF_EXEC; 4455 PMAPCOUNT(exec_discarded_modfixup); 4456 } 4457 #endif 4458 pmap_release_page_lock(md); 4459 4460 /* 4461 * Re-enable write permissions for the page. No need to call 4462 * pmap_vac_me_harder(), since this is just a 4463 * modified-emulation fault, and the PVF_WRITE bit isn't 4464 * changing. We've already set the cacheable bits based on 4465 * the assumption that we can write to this page. 4466 */ 4467 const pt_entry_t npte = 4468 l2pte_set_writable((opte & ~L2_TYPE_MASK) | L2_S_PROTO) 4469 #ifdef ARM_MMU_EXTENDED 4470 | (pm != pmap_kernel() ? L2_XS_nG : 0) 4471 #endif 4472 | 0; 4473 l2pte_reset(ptep); 4474 PTE_SYNC(ptep); 4475 pmap_tlb_flush_SE(pm, va, 4476 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4477 l2pte_set(ptep, npte, 0); 4478 PTE_SYNC(ptep); 4479 PMAPCOUNT(fixup_mod); 4480 rv = 1; 4481 UVMHIST_LOG(maphist, " <-- done (mod/ref emul: changed pte " 4482 "from %#jx to %#jx)", opte, npte, 0, 0); 4483 } else if ((opte & L2_TYPE_MASK) == L2_TYPE_INV) { 4484 /* 4485 * This looks like a good candidate for "page referenced" 4486 * emulation. 4487 */ 4488 struct vm_page *pg; 4489 4490 /* Extract the physical address of the page */ 4491 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 4492 UVMHIST_LOG(maphist, " <-- done (ref emul: unmanaged page)", 0, 0, 0, 0); 4493 goto out; 4494 } 4495 4496 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4497 4498 /* Get the current flags for this page. */ 4499 pmap_acquire_page_lock(md); 4500 struct pv_entry *pv = pmap_find_pv(md, pm, va); 4501 if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) { 4502 pmap_release_page_lock(md); 4503 UVMHIST_LOG(maphist, " <-- done (ref emul no PV)", 0, 0, 0, 0); 4504 goto out; 4505 } 4506 4507 md->pvh_attrs |= PVF_REF; 4508 pv->pv_flags |= PVF_REF; 4509 4510 pt_entry_t npte = 4511 l2pte_set_readonly((opte & ~L2_TYPE_MASK) | L2_S_PROTO); 4512 #ifdef ARM_MMU_EXTENDED 4513 if (pm != pmap_kernel()) { 4514 npte |= L2_XS_nG; 4515 } 4516 /* 4517 * If we got called from prefetch abort, then ftype will have 4518 * VM_PROT_EXECUTE set. Now see if we have no-execute set in 4519 * the PTE. 4520 */ 4521 if (user && (ftype & VM_PROT_EXECUTE) && (npte & L2_XS_XN)) { 4522 /* 4523 * Is this a mapping of an executable page? 4524 */ 4525 if ((pv->pv_flags & PVF_EXEC) == 0) { 4526 pmap_release_page_lock(md); 4527 UVMHIST_LOG(maphist, " <-- done (ref emul: no exec)", 4528 0, 0, 0, 0); 4529 goto out; 4530 } 4531 /* 4532 * If we haven't synced the page, do so now. 4533 */ 4534 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4535 UVMHIST_LOG(maphist, " ref emul: syncicache " 4536 "page #%#jx", pa, 0, 0, 0); 4537 pmap_syncicache_page(md, pa); 4538 PMAPCOUNT(fixup_exec); 4539 } 4540 npte &= ~L2_XS_XN; 4541 } 4542 #endif /* ARM_MMU_EXTENDED */ 4543 pmap_release_page_lock(md); 4544 l2pte_reset(ptep); 4545 PTE_SYNC(ptep); 4546 pmap_tlb_flush_SE(pm, va, 4547 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4548 l2pte_set(ptep, npte, 0); 4549 PTE_SYNC(ptep); 4550 PMAPCOUNT(fixup_ref); 4551 rv = 1; 4552 UVMHIST_LOG(maphist, " <-- done (ref emul: changed pte from " 4553 "%#jx to %#jx)", opte, npte, 0, 0); 4554 #ifdef ARM_MMU_EXTENDED 4555 } else if (user && (ftype & VM_PROT_EXECUTE) && (opte & L2_XS_XN)) { 4556 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 4557 if (pg == NULL) { 4558 UVMHIST_LOG(maphist, " <-- done (unmanaged page)", 0, 0, 0, 0); 4559 goto out; 4560 } 4561 4562 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 4563 4564 /* Get the current flags for this page. */ 4565 pmap_acquire_page_lock(md); 4566 struct pv_entry * const pv = pmap_find_pv(md, pm, va); 4567 if (pv == NULL || (pv->pv_flags & PVF_EXEC) == 0) { 4568 pmap_release_page_lock(md); 4569 UVMHIST_LOG(maphist, " <-- done (no PV or not EXEC)", 0, 0, 0, 0); 4570 goto out; 4571 } 4572 4573 /* 4574 * If we haven't synced the page, do so now. 4575 */ 4576 if ((md->pvh_attrs & PVF_EXEC) == 0) { 4577 UVMHIST_LOG(maphist, "syncicache page #%#jx", 4578 pa, 0, 0, 0); 4579 pmap_syncicache_page(md, pa); 4580 } 4581 pmap_release_page_lock(md); 4582 /* 4583 * Turn off no-execute. 4584 */ 4585 KASSERT(opte & L2_XS_nG); 4586 l2pte_reset(ptep); 4587 PTE_SYNC(ptep); 4588 pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF); 4589 l2pte_set(ptep, opte & ~L2_XS_XN, 0); 4590 PTE_SYNC(ptep); 4591 rv = 1; 4592 PMAPCOUNT(fixup_exec); 4593 UVMHIST_LOG(maphist, "exec: changed pte from %#jx to %#jx", 4594 opte, opte & ~L2_XS_XN, 0, 0); 4595 #endif 4596 } 4597 4598 #ifndef ARM_MMU_EXTENDED 4599 /* 4600 * We know there is a valid mapping here, so simply 4601 * fix up the L1 if necessary. 4602 */ 4603 pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot; 4604 pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa | L1_C_DOM(pmap_domain(pm)); 4605 if (*pdep != pde) { 4606 l1pte_setone(pdep, pde); 4607 PDE_SYNC(pdep); 4608 rv = 1; 4609 PMAPCOUNT(fixup_pdes); 4610 } 4611 #endif 4612 4613 #ifdef CPU_SA110 4614 /* 4615 * There are bugs in the rev K SA110. This is a check for one 4616 * of them. 4617 */ 4618 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 4619 curcpu()->ci_arm_cpurev < 3) { 4620 /* Always current pmap */ 4621 if (l2pte_valid_p(opte)) { 4622 extern int kernel_debug; 4623 if (kernel_debug & 1) { 4624 struct proc *p = curlwp->l_proc; 4625 printf("prefetch_abort: page is already " 4626 "mapped - pte=%p *pte=%08x\n", ptep, opte); 4627 printf("prefetch_abort: pc=%08lx proc=%p " 4628 "process=%s\n", va, p, p->p_comm); 4629 printf("prefetch_abort: far=%08x fs=%x\n", 4630 cpu_faultaddress(), cpu_faultstatus()); 4631 } 4632 #ifdef DDB 4633 if (kernel_debug & 2) 4634 Debugger(); 4635 #endif 4636 rv = 1; 4637 } 4638 } 4639 #endif /* CPU_SA110 */ 4640 4641 #ifndef ARM_MMU_EXTENDED 4642 /* 4643 * If 'rv == 0' at this point, it generally indicates that there is a 4644 * stale TLB entry for the faulting address. That might be due to a 4645 * wrong setting of pmap_needs_pte_sync. So set it and retry. 4646 */ 4647 if (rv == 0 4648 && pm->pm_l1->l1_domain_use_count == 1 4649 && pmap_needs_pte_sync == 0) { 4650 pmap_needs_pte_sync = 1; 4651 PTE_SYNC(ptep); 4652 PMAPCOUNT(fixup_ptesync); 4653 rv = 1; 4654 } 4655 #endif 4656 4657 #ifndef MULTIPROCESSOR 4658 #if defined(DEBUG) || 1 4659 /* 4660 * If 'rv == 0' at this point, it generally indicates that there is a 4661 * stale TLB entry for the faulting address. This happens when two or 4662 * more processes are sharing an L1. Since we don't flush the TLB on 4663 * a context switch between such processes, we can take domain faults 4664 * for mappings which exist at the same VA in both processes. EVEN IF 4665 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 4666 * example. 4667 * 4668 * This is extremely likely to happen if pmap_enter() updated the L1 4669 * entry for a recently entered mapping. In this case, the TLB is 4670 * flushed for the new mapping, but there may still be TLB entries for 4671 * other mappings belonging to other processes in the 1MB range 4672 * covered by the L1 entry. 4673 * 4674 * Since 'rv == 0', we know that the L1 already contains the correct 4675 * value, so the fault must be due to a stale TLB entry. 4676 * 4677 * Since we always need to flush the TLB anyway in the case where we 4678 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 4679 * stale TLB entries dynamically. 4680 * 4681 * However, the above condition can ONLY happen if the current L1 is 4682 * being shared. If it happens when the L1 is unshared, it indicates 4683 * that other parts of the pmap are not doing their job WRT managing 4684 * the TLB. 4685 */ 4686 if (rv == 0 4687 #ifndef ARM_MMU_EXTENDED 4688 && pm->pm_l1->l1_domain_use_count == 1 4689 #endif 4690 && true) { 4691 #ifdef DEBUG 4692 extern int last_fault_code; 4693 #else 4694 int last_fault_code = ftype & VM_PROT_EXECUTE 4695 ? armreg_ifsr_read() 4696 : armreg_dfsr_read(); 4697 #endif 4698 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 4699 pm, va, ftype); 4700 printf("fixup: l2 %p, l2b %p, ptep %p, pte %#x\n", 4701 l2, l2b, ptep, opte); 4702 4703 #ifndef ARM_MMU_EXTENDED 4704 printf("fixup: pdep %p, pde %#x, fsr %#x\n", 4705 pdep, pde, last_fault_code); 4706 #else 4707 printf("fixup: pdep %p, pde %#x, ttbcr %#x\n", 4708 &pmap_l1_kva(pm)[l1slot], pmap_l1_kva(pm)[l1slot], 4709 armreg_ttbcr_read()); 4710 printf("fixup: fsr %#x cpm %p casid %#x contextidr %#x dacr %#x\n", 4711 last_fault_code, curcpu()->ci_pmap_cur, 4712 curcpu()->ci_pmap_asid_cur, 4713 armreg_contextidr_read(), armreg_dacr_read()); 4714 #ifdef _ARM_ARCH_7 4715 if (ftype & VM_PROT_WRITE) 4716 armreg_ats1cuw_write(va); 4717 else 4718 armreg_ats1cur_write(va); 4719 arm_isb(); 4720 printf("fixup: par %#x\n", armreg_par_read()); 4721 #endif 4722 #endif 4723 #ifdef DDB 4724 extern int kernel_debug; 4725 4726 if (kernel_debug & 2) { 4727 pmap_release_pmap_lock(pm); 4728 #ifdef UVMHIST 4729 KERNHIST_DUMP(maphist); 4730 #endif 4731 cpu_Debugger(); 4732 pmap_acquire_pmap_lock(pm); 4733 } 4734 #endif 4735 } 4736 #endif 4737 #endif 4738 4739 #ifndef ARM_MMU_EXTENDED 4740 /* Flush the TLB in the shared L1 case - see comment above */ 4741 pmap_tlb_flush_SE(pm, va, 4742 (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF); 4743 #endif 4744 4745 rv = 1; 4746 4747 out: 4748 pmap_release_pmap_lock(pm); 4749 4750 return (rv); 4751 } 4752 4753 /* 4754 * Routine: pmap_procwr 4755 * 4756 * Function: 4757 * Synchronize caches corresponding to [addr, addr+len) in p. 4758 * 4759 */ 4760 void 4761 pmap_procwr(struct proc *p, vaddr_t va, int len) 4762 { 4763 #ifndef ARM_MMU_EXTENDED 4764 4765 /* We only need to do anything if it is the current process. */ 4766 if (p == curproc) 4767 cpu_icache_sync_range(va, len); 4768 #endif 4769 } 4770 4771 /* 4772 * Routine: pmap_unwire 4773 * Function: Clear the wired attribute for a map/virtual-address pair. 4774 * 4775 * In/out conditions: 4776 * The mapping must already exist in the pmap. 4777 */ 4778 void 4779 pmap_unwire(pmap_t pm, vaddr_t va) 4780 { 4781 struct l2_bucket *l2b; 4782 pt_entry_t *ptep, pte; 4783 struct vm_page *pg; 4784 paddr_t pa; 4785 4786 NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va)); 4787 4788 pmap_acquire_pmap_lock(pm); 4789 4790 l2b = pmap_get_l2_bucket(pm, va); 4791 KDASSERT(l2b != NULL); 4792 4793 ptep = &l2b->l2b_kva[l2pte_index(va)]; 4794 pte = *ptep; 4795 4796 /* Extract the physical address of the page */ 4797 pa = l2pte_pa(pte); 4798 4799 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 4800 /* Update the wired bit in the pv entry for this page. */ 4801 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 4802 4803 pmap_acquire_page_lock(md); 4804 (void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0); 4805 pmap_release_page_lock(md); 4806 } 4807 4808 pmap_release_pmap_lock(pm); 4809 } 4810 4811 #ifdef ARM_MMU_EXTENDED 4812 void 4813 pmap_md_pdetab_activate(pmap_t pm, struct lwp *l) 4814 { 4815 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4816 4817 /* 4818 * Assume that TTBR1 has only global mappings and TTBR0 only 4819 * has non-global mappings. To prevent speculation from doing 4820 * evil things we disable translation table walks using TTBR0 4821 * before setting the CONTEXTIDR (ASID) or new TTBR0 value. 4822 * Once both are set, table walks are reenabled. 4823 */ 4824 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4825 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4826 arm_isb(); 4827 4828 pmap_tlb_asid_acquire(pm, l); 4829 4830 struct cpu_info * const ci = curcpu(); 4831 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci)); 4832 4833 cpu_setttb(pm->pm_l1_pa, pai->pai_asid); 4834 /* 4835 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 4836 * have been updated. 4837 */ 4838 arm_isb(); 4839 4840 if (pm != pmap_kernel()) { 4841 armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); 4842 } 4843 cpu_cpwait(); 4844 4845 UVMHIST_LOG(maphist, " pm %#jx pm->pm_l1_pa %08jx asid %ju... done", 4846 (uintptr_t)pm, pm->pm_l1_pa, pai->pai_asid, 0); 4847 4848 KASSERTMSG(ci->ci_pmap_asid_cur == pai->pai_asid, "%u vs %u", 4849 ci->ci_pmap_asid_cur, pai->pai_asid); 4850 ci->ci_pmap_cur = pm; 4851 } 4852 4853 void 4854 pmap_md_pdetab_deactivate(pmap_t pm) 4855 { 4856 4857 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4858 4859 kpreempt_disable(); 4860 struct cpu_info * const ci = curcpu(); 4861 /* 4862 * Disable translation table walks from TTBR0 while no pmap has been 4863 * activated. 4864 */ 4865 const uint32_t old_ttbcr = armreg_ttbcr_read(); 4866 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 4867 arm_isb(); 4868 pmap_tlb_asid_deactivate(pm); 4869 cpu_setttb(pmap_kernel()->pm_l1_pa, KERNEL_PID); 4870 arm_isb(); 4871 4872 ci->ci_pmap_cur = pmap_kernel(); 4873 KASSERTMSG(ci->ci_pmap_asid_cur == KERNEL_PID, "ci_pmap_asid_cur %u", 4874 ci->ci_pmap_asid_cur); 4875 kpreempt_enable(); 4876 } 4877 #endif 4878 4879 void 4880 pmap_activate(struct lwp *l) 4881 { 4882 extern int block_userspace_access; 4883 pmap_t npm = l->l_proc->p_vmspace->vm_map.pmap; 4884 4885 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 4886 4887 UVMHIST_LOG(maphist, "(l=%#jx) pm=%#jx", (uintptr_t)l, (uintptr_t)npm, 4888 0, 0); 4889 4890 struct cpu_info * const ci = curcpu(); 4891 4892 /* 4893 * If activating a non-current lwp or the current lwp is 4894 * already active, just return. 4895 */ 4896 if (false 4897 || l != curlwp 4898 #ifdef ARM_MMU_EXTENDED 4899 || (ci->ci_pmap_cur == npm && 4900 (npm == pmap_kernel() 4901 /* || PMAP_PAI_ASIDVALID_P(pai, cpu_tlb_info(ci)) */)) 4902 #else 4903 || npm->pm_activated == true 4904 #endif 4905 || false) { 4906 UVMHIST_LOG(maphist, " <-- (same pmap)", (uintptr_t)curlwp, 4907 (uintptr_t)l, 0, 0); 4908 return; 4909 } 4910 4911 #ifndef ARM_MMU_EXTENDED 4912 const uint32_t ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4913 | (DOMAIN_CLIENT << (pmap_domain(npm) * 2)); 4914 4915 /* 4916 * If TTB and DACR are unchanged, short-circuit all the 4917 * TLB/cache management stuff. 4918 */ 4919 pmap_t opm = ci->ci_lastlwp 4920 ? ci->ci_lastlwp->l_proc->p_vmspace->vm_map.pmap 4921 : NULL; 4922 if (opm != NULL) { 4923 uint32_t odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) 4924 | (DOMAIN_CLIENT << (pmap_domain(opm) * 2)); 4925 4926 if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr) 4927 goto all_done; 4928 } 4929 #endif /* !ARM_MMU_EXTENDED */ 4930 4931 PMAPCOUNT(activations); 4932 block_userspace_access = 1; 4933 4934 #ifndef ARM_MMU_EXTENDED 4935 /* 4936 * If switching to a user vmspace which is different to the 4937 * most recent one, and the most recent one is potentially 4938 * live in the cache, we must write-back and invalidate the 4939 * entire cache. 4940 */ 4941 pmap_t rpm = ci->ci_pmap_lastuser; 4942 4943 /* 4944 * XXXSCW: There's a corner case here which can leave turds in the 4945 * cache as reported in kern/41058. They're probably left over during 4946 * tear-down and switching away from an exiting process. Until the root 4947 * cause is identified and fixed, zap the cache when switching pmaps. 4948 * This will result in a few unnecessary cache flushes, but that's 4949 * better than silently corrupting data. 4950 */ 4951 #if 0 4952 if (npm != pmap_kernel() && rpm && npm != rpm && 4953 rpm->pm_cstate.cs_cache) { 4954 rpm->pm_cstate.cs_cache = 0; 4955 #ifdef PMAP_CACHE_VIVT 4956 cpu_idcache_wbinv_all(); 4957 #endif 4958 } 4959 #else 4960 if (rpm) { 4961 rpm->pm_cstate.cs_cache = 0; 4962 if (npm == pmap_kernel()) 4963 ci->ci_pmap_lastuser = NULL; 4964 #ifdef PMAP_CACHE_VIVT 4965 cpu_idcache_wbinv_all(); 4966 #endif 4967 } 4968 #endif 4969 4970 /* No interrupts while we frob the TTB/DACR */ 4971 uint32_t oldirqstate = disable_interrupts(IF32_bits); 4972 #endif /* !ARM_MMU_EXTENDED */ 4973 4974 #ifndef ARM_HAS_VBAR 4975 /* 4976 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1 4977 * entry corresponding to 'vector_page' in the incoming L1 table 4978 * before switching to it otherwise subsequent interrupts/exceptions 4979 * (including domain faults!) will jump into hyperspace. 4980 */ 4981 if (npm->pm_pl1vec != NULL) { 4982 cpu_tlb_flushID_SE((u_int)vector_page); 4983 cpu_cpwait(); 4984 *npm->pm_pl1vec = npm->pm_l1vec; 4985 PTE_SYNC(npm->pm_pl1vec); 4986 } 4987 #endif 4988 4989 #ifdef ARM_MMU_EXTENDED 4990 pmap_md_pdetab_activate(npm, l); 4991 #else 4992 cpu_domains(ndacr); 4993 if (npm == pmap_kernel() || npm == rpm) { 4994 /* 4995 * Switching to a kernel thread, or back to the 4996 * same user vmspace as before... Simply update 4997 * the TTB (no TLB flush required) 4998 */ 4999 cpu_setttb(npm->pm_l1->l1_physaddr, false); 5000 cpu_cpwait(); 5001 } else { 5002 /* 5003 * Otherwise, update TTB and flush TLB 5004 */ 5005 cpu_context_switch(npm->pm_l1->l1_physaddr); 5006 if (rpm != NULL) 5007 rpm->pm_cstate.cs_tlb = 0; 5008 } 5009 5010 restore_interrupts(oldirqstate); 5011 #endif /* ARM_MMU_EXTENDED */ 5012 5013 block_userspace_access = 0; 5014 5015 #ifndef ARM_MMU_EXTENDED 5016 all_done: 5017 /* 5018 * The new pmap is resident. Make sure it's marked 5019 * as resident in the cache/TLB. 5020 */ 5021 npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5022 if (npm != pmap_kernel()) 5023 ci->ci_pmap_lastuser = npm; 5024 5025 /* The old pmap is not longer active */ 5026 if (opm != npm) { 5027 if (opm != NULL) 5028 opm->pm_activated = false; 5029 5030 /* But the new one is */ 5031 npm->pm_activated = true; 5032 } 5033 ci->ci_pmap_cur = npm; 5034 #endif 5035 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5036 } 5037 5038 void 5039 pmap_deactivate(struct lwp *l) 5040 { 5041 pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap; 5042 5043 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5044 5045 UVMHIST_LOG(maphist, "(l=%#jx) pm=%#jx", (uintptr_t)l, (uintptr_t)pm, 5046 0, 0); 5047 5048 #ifdef ARM_MMU_EXTENDED 5049 pmap_md_pdetab_deactivate(pm); 5050 #else 5051 /* 5052 * If the process is exiting, make sure pmap_activate() does 5053 * a full MMU context-switch and cache flush, which we might 5054 * otherwise skip. See PR port-arm/38950. 5055 */ 5056 if (l->l_proc->p_sflag & PS_WEXIT) 5057 curcpu()->ci_lastlwp = NULL; 5058 5059 pm->pm_activated = false; 5060 #endif 5061 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5062 } 5063 5064 void 5065 pmap_update(pmap_t pm) 5066 { 5067 5068 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5069 5070 UVMHIST_LOG(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5071 pm->pm_remove_all, 0, 0); 5072 5073 #ifndef ARM_MMU_EXTENDED 5074 if (pm->pm_remove_all) { 5075 /* 5076 * Finish up the pmap_remove_all() optimisation by flushing 5077 * the TLB. 5078 */ 5079 pmap_tlb_flushID(pm); 5080 pm->pm_remove_all = false; 5081 } 5082 5083 if (pmap_is_current(pm)) { 5084 /* 5085 * If we're dealing with a current userland pmap, move its L1 5086 * to the end of the LRU. 5087 */ 5088 if (pm != pmap_kernel()) 5089 pmap_use_l1(pm); 5090 5091 /* 5092 * We can assume we're done with frobbing the cache/tlb for 5093 * now. Make sure any future pmap ops don't skip cache/tlb 5094 * flushes. 5095 */ 5096 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5097 } 5098 #else 5099 5100 kpreempt_disable(); 5101 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 5102 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 5103 if (pending && pmap_tlb_shootdown_bystanders(pmap)) { 5104 PMAP_COUNT(shootdown_ipis); 5105 } 5106 #endif 5107 5108 /* 5109 * If pmap_remove_all was called, we deactivated ourselves and released 5110 * our ASID. Now we have to reactivate ourselves. 5111 */ 5112 if (__predict_false(pm->pm_remove_all)) { 5113 pm->pm_remove_all = false; 5114 5115 KASSERT(pm != pmap_kernel()); 5116 pmap_md_pdetab_activate(pm, curlwp); 5117 } 5118 5119 if (arm_has_mpext_p) 5120 armreg_bpiallis_write(0); 5121 else 5122 armreg_bpiall_write(0); 5123 5124 kpreempt_enable(); 5125 5126 KASSERTMSG(pm == pmap_kernel() 5127 || curcpu()->ci_pmap_cur != pm 5128 || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, 5129 "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, 5130 pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, 5131 curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur); 5132 #endif 5133 5134 PMAPCOUNT(updates); 5135 5136 /* 5137 * make sure TLB/cache operations have completed. 5138 */ 5139 cpu_cpwait(); 5140 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5141 } 5142 5143 void 5144 pmap_remove_all(pmap_t pm) 5145 { 5146 5147 /* 5148 * The vmspace described by this pmap is about to be torn down. 5149 * Until pmap_update() is called, UVM will only make calls 5150 * to pmap_remove(). We can make life much simpler by flushing 5151 * the cache now, and deferring TLB invalidation to pmap_update(). 5152 */ 5153 #ifdef PMAP_CACHE_VIVT 5154 pmap_cache_wbinv_all(pm, PVF_EXEC); 5155 #endif 5156 #ifdef ARM_MMU_EXTENDED 5157 #ifdef MULTIPROCESSOR 5158 struct cpu_info * const ci = curcpu(); 5159 // This should be the last CPU with this pmap onproc 5160 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 5161 if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) 5162 #endif 5163 pmap_tlb_asid_deactivate(pm); 5164 #ifdef MULTIPROCESSOR 5165 KASSERT(kcpuset_iszero(pm->pm_onproc)); 5166 #endif 5167 5168 pmap_tlb_asid_release_all(pm); 5169 #endif 5170 pm->pm_remove_all = true; 5171 } 5172 5173 /* 5174 * Retire the given physical map from service. 5175 * Should only be called if the map contains no valid mappings. 5176 */ 5177 void 5178 pmap_destroy(pmap_t pm) 5179 { 5180 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 5181 5182 u_int count; 5183 5184 if (pm == NULL) 5185 return; 5186 5187 UVMHIST_LOG(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm, 5188 pm->pm_remove_all, 0, 0); 5189 5190 if (pm->pm_remove_all) { 5191 #ifdef ARM_MMU_EXTENDED 5192 pmap_tlb_asid_release_all(pm); 5193 #else 5194 pmap_tlb_flushID(pm); 5195 #endif 5196 pm->pm_remove_all = false; 5197 } 5198 5199 /* 5200 * Drop reference count 5201 */ 5202 mutex_enter(pm->pm_lock); 5203 count = --pm->pm_obj.uo_refs; 5204 mutex_exit(pm->pm_lock); 5205 if (count > 0) { 5206 #ifndef ARM_MMU_EXTENDED 5207 if (pmap_is_current(pm)) { 5208 if (pm != pmap_kernel()) 5209 pmap_use_l1(pm); 5210 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 5211 } 5212 #endif 5213 return; 5214 } 5215 5216 /* 5217 * reference count is zero, free pmap resources and then free pmap. 5218 */ 5219 5220 #ifndef ARM_HAS_VBAR 5221 if (vector_page < KERNEL_BASE) { 5222 KDASSERT(!pmap_is_current(pm)); 5223 5224 /* Remove the vector page mapping */ 5225 pmap_remove(pm, vector_page, vector_page + PAGE_SIZE); 5226 pmap_update(pm); 5227 } 5228 #endif 5229 5230 pmap_free_l1(pm); 5231 5232 #ifdef ARM_MMU_EXTENDED 5233 #ifdef MULTIPROCESSOR 5234 kcpuset_destroy(pm->pm_active); 5235 kcpuset_destroy(pm->pm_onproc); 5236 #endif 5237 #else 5238 struct cpu_info * const ci = curcpu(); 5239 if (ci->ci_pmap_lastuser == pm) 5240 ci->ci_pmap_lastuser = NULL; 5241 #endif 5242 5243 uvm_obj_destroy(&pm->pm_obj, false); 5244 mutex_destroy(&pm->pm_obj_lock); 5245 pool_cache_put(&pmap_cache, pm); 5246 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 5247 } 5248 5249 5250 /* 5251 * void pmap_reference(pmap_t pm) 5252 * 5253 * Add a reference to the specified pmap. 5254 */ 5255 void 5256 pmap_reference(pmap_t pm) 5257 { 5258 5259 if (pm == NULL) 5260 return; 5261 5262 #ifndef ARM_MMU_EXTENDED 5263 pmap_use_l1(pm); 5264 #endif 5265 5266 mutex_enter(pm->pm_lock); 5267 pm->pm_obj.uo_refs++; 5268 mutex_exit(pm->pm_lock); 5269 } 5270 5271 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 5272 5273 static struct evcnt pmap_prefer_nochange_ev = 5274 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 5275 static struct evcnt pmap_prefer_change_ev = 5276 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 5277 5278 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 5279 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 5280 5281 void 5282 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 5283 { 5284 vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1); 5285 vaddr_t va = *vap; 5286 vaddr_t diff = (hint - va) & mask; 5287 if (diff == 0) { 5288 pmap_prefer_nochange_ev.ev_count++; 5289 } else { 5290 pmap_prefer_change_ev.ev_count++; 5291 if (__predict_false(td)) 5292 va -= mask + 1; 5293 *vap = va + diff; 5294 } 5295 } 5296 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */ 5297 5298 /* 5299 * pmap_zero_page() 5300 * 5301 * Zero a given physical page by mapping it at a page hook point. 5302 * In doing the zero page op, the page we zero is mapped cachable, as with 5303 * StrongARM accesses to non-cached pages are non-burst making writing 5304 * _any_ bulk data very slow. 5305 */ 5306 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5307 void 5308 pmap_zero_page_generic(paddr_t pa) 5309 { 5310 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5311 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5312 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5313 #endif 5314 #if defined(PMAP_CACHE_VIPT) 5315 /* Choose the last page color it had, if any */ 5316 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5317 #else 5318 const vsize_t va_offset = 0; 5319 #endif 5320 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5321 /* 5322 * Is this page mapped at its natural color? 5323 * If we have all of memory mapped, then just convert PA to VA. 5324 */ 5325 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5326 || va_offset == (pa & arm_cache_prefer_mask); 5327 const vaddr_t vdstp = okcolor 5328 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5329 : cpu_cdstp(va_offset); 5330 #else 5331 const bool okcolor = false; 5332 const vaddr_t vdstp = cpu_cdstp(va_offset); 5333 #endif 5334 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5335 5336 5337 #ifdef DEBUG 5338 if (!SLIST_EMPTY(&md->pvh_list)) 5339 panic("pmap_zero_page: page has mappings"); 5340 #endif 5341 5342 KDASSERT((pa & PGOFSET) == 0); 5343 5344 if (!okcolor) { 5345 /* 5346 * Hook in the page, zero it, and purge the cache for that 5347 * zeroed page. Invalidate the TLB as needed. 5348 */ 5349 const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode 5350 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); 5351 l2pte_set(ptep, npte, 0); 5352 PTE_SYNC(ptep); 5353 cpu_tlb_flushD_SE(vdstp); 5354 cpu_cpwait(); 5355 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) \ 5356 && !defined(ARM_MMU_EXTENDED) 5357 /* 5358 * If we are direct-mapped and our color isn't ok, then before 5359 * we bzero the page invalidate its contents from the cache and 5360 * reset the color to its natural color. 5361 */ 5362 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5363 md->pvh_attrs &= ~arm_cache_prefer_mask; 5364 md->pvh_attrs |= (pa & arm_cache_prefer_mask); 5365 #endif 5366 } 5367 bzero_page(vdstp); 5368 if (!okcolor) { 5369 /* 5370 * Unmap the page. 5371 */ 5372 l2pte_reset(ptep); 5373 PTE_SYNC(ptep); 5374 cpu_tlb_flushD_SE(vdstp); 5375 #ifdef PMAP_CACHE_VIVT 5376 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5377 #endif 5378 } 5379 #ifdef PMAP_CACHE_VIPT 5380 /* 5381 * This page is now cache resident so it now has a page color. 5382 * Any contents have been obliterated so clear the EXEC flag. 5383 */ 5384 #ifndef ARM_MMU_EXTENDED 5385 if (!pmap_is_page_colored_p(md)) { 5386 PMAPCOUNT(vac_color_new); 5387 md->pvh_attrs |= PVF_COLORED; 5388 } 5389 md->pvh_attrs |= PVF_DIRTY; 5390 #endif 5391 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5392 md->pvh_attrs &= ~PVF_EXEC; 5393 PMAPCOUNT(exec_discarded_zero); 5394 } 5395 #endif 5396 } 5397 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5398 5399 #if ARM_MMU_XSCALE == 1 5400 void 5401 pmap_zero_page_xscale(paddr_t pa) 5402 { 5403 #ifdef DEBUG 5404 struct vm_page *pg = PHYS_TO_VM_PAGE(pa); 5405 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5406 5407 if (!SLIST_EMPTY(&md->pvh_list)) 5408 panic("pmap_zero_page: page has mappings"); 5409 #endif 5410 5411 KDASSERT((pa & PGOFSET) == 0); 5412 5413 /* 5414 * Hook in the page, zero it, and purge the cache for that 5415 * zeroed page. Invalidate the TLB as needed. 5416 */ 5417 5418 pt_entry_t npte = L2_S_PROTO | pa | 5419 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 5420 L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5421 l2pte_set(cdst_pte, npte, 0); 5422 PTE_SYNC(cdst_pte); 5423 cpu_tlb_flushD_SE(cdstp); 5424 cpu_cpwait(); 5425 bzero_page(cdstp); 5426 xscale_cache_clean_minidata(); 5427 l2pte_reset(cdst_pte); 5428 PTE_SYNC(cdst_pte); 5429 } 5430 #endif /* ARM_MMU_XSCALE == 1 */ 5431 5432 /* pmap_pageidlezero() 5433 * 5434 * The same as above, except that we assume that the page is not 5435 * mapped. This means we never have to flush the cache first. Called 5436 * from the idle loop. 5437 */ 5438 bool 5439 pmap_pageidlezero(paddr_t pa) 5440 { 5441 bool rv = true; 5442 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5443 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 5444 struct vm_page_md *md = VM_PAGE_TO_MD(pg); 5445 #endif 5446 #ifdef PMAP_CACHE_VIPT 5447 /* Choose the last page color it had, if any */ 5448 const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; 5449 #else 5450 const vsize_t va_offset = 0; 5451 #endif 5452 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 5453 bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5454 || va_offset == (pa & arm_cache_prefer_mask); 5455 const vaddr_t vdstp = okcolor 5456 ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset)) 5457 : cpu_cdstp(va_offset); 5458 #else 5459 const bool okcolor = false; 5460 const vaddr_t vdstp = cpu_cdstp(va_offset); 5461 #endif 5462 pt_entry_t * const ptep = cpu_cdst_pte(va_offset); 5463 5464 5465 #ifdef DEBUG 5466 if (!SLIST_EMPTY(&md->pvh_list)) 5467 panic("pmap_pageidlezero: page has mappings"); 5468 #endif 5469 5470 KDASSERT((pa & PGOFSET) == 0); 5471 5472 if (!okcolor) { 5473 /* 5474 * Hook in the page, zero it, and purge the cache for that 5475 * zeroed page. Invalidate the TLB as needed. 5476 */ 5477 const pt_entry_t npte = L2_S_PROTO | pa | 5478 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5479 l2pte_set(ptep, npte, 0); 5480 PTE_SYNC(ptep); 5481 cpu_tlb_flushD_SE(vdstp); 5482 cpu_cpwait(); 5483 } 5484 5485 uint64_t *ptr = (uint64_t *)vdstp; 5486 for (size_t i = 0; i < PAGE_SIZE / sizeof(*ptr); i++) { 5487 if (sched_curcpu_runnable_p() != 0) { 5488 /* 5489 * A process has become ready. Abort now, 5490 * so we don't keep it waiting while we 5491 * do slow memory access to finish this 5492 * page. 5493 */ 5494 rv = false; 5495 break; 5496 } 5497 *ptr++ = 0; 5498 } 5499 5500 #ifdef PMAP_CACHE_VIVT 5501 if (rv) 5502 /* 5503 * if we aborted we'll rezero this page again later so don't 5504 * purge it unless we finished it 5505 */ 5506 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5507 #elif defined(PMAP_CACHE_VIPT) 5508 /* 5509 * This page is now cache resident so it now has a page color. 5510 * Any contents have been obliterated so clear the EXEC flag. 5511 */ 5512 #ifndef ARM_MMU_EXTENDED 5513 if (!pmap_is_page_colored_p(md)) { 5514 PMAPCOUNT(vac_color_new); 5515 md->pvh_attrs |= PVF_COLORED; 5516 } 5517 #endif 5518 if (PV_IS_EXEC_P(md->pvh_attrs)) { 5519 md->pvh_attrs &= ~PVF_EXEC; 5520 PMAPCOUNT(exec_discarded_zero); 5521 } 5522 #endif 5523 /* 5524 * Unmap the page. 5525 */ 5526 if (!okcolor) { 5527 l2pte_reset(ptep); 5528 PTE_SYNC(ptep); 5529 cpu_tlb_flushD_SE(vdstp); 5530 } 5531 5532 return rv; 5533 } 5534 5535 /* 5536 * pmap_copy_page() 5537 * 5538 * Copy one physical page into another, by mapping the pages into 5539 * hook points. The same comment regarding cachability as in 5540 * pmap_zero_page also applies here. 5541 */ 5542 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 5543 void 5544 pmap_copy_page_generic(paddr_t src, paddr_t dst) 5545 { 5546 struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src); 5547 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5548 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG) 5549 struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst); 5550 struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg); 5551 #endif 5552 #ifdef PMAP_CACHE_VIPT 5553 const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask; 5554 const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask; 5555 #else 5556 const vsize_t src_va_offset = 0; 5557 const vsize_t dst_va_offset = 0; 5558 #endif 5559 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 5560 /* 5561 * Is this page mapped at its natural color? 5562 * If we have all of memory mapped, then just convert PA to VA. 5563 */ 5564 bool src_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5565 || src_va_offset == (src & arm_cache_prefer_mask); 5566 bool dst_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT 5567 || dst_va_offset == (dst & arm_cache_prefer_mask); 5568 const vaddr_t vsrcp = src_okcolor 5569 ? pmap_direct_mapped_phys(src, &src_okcolor, 5570 cpu_csrcp(src_va_offset)) 5571 : cpu_csrcp(src_va_offset); 5572 const vaddr_t vdstp = pmap_direct_mapped_phys(dst, &dst_okcolor, 5573 cpu_cdstp(dst_va_offset)); 5574 #else 5575 const bool src_okcolor = false; 5576 const bool dst_okcolor = false; 5577 const vaddr_t vsrcp = cpu_csrcp(src_va_offset); 5578 const vaddr_t vdstp = cpu_cdstp(dst_va_offset); 5579 #endif 5580 pt_entry_t * const src_ptep = cpu_csrc_pte(src_va_offset); 5581 pt_entry_t * const dst_ptep = cpu_cdst_pte(dst_va_offset); 5582 5583 #ifdef DEBUG 5584 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5585 panic("pmap_copy_page: dst page has mappings"); 5586 #endif 5587 5588 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5589 KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC)); 5590 #endif 5591 KDASSERT((src & PGOFSET) == 0); 5592 KDASSERT((dst & PGOFSET) == 0); 5593 5594 /* 5595 * Clean the source page. Hold the source page's lock for 5596 * the duration of the copy so that no other mappings can 5597 * be created while we have a potentially aliased mapping. 5598 */ 5599 #ifdef PMAP_CACHE_VIVT 5600 pmap_acquire_page_lock(src_md); 5601 (void) pmap_clean_page(src_md, true); 5602 pmap_release_page_lock(src_md); 5603 #endif 5604 5605 /* 5606 * Map the pages into the page hook points, copy them, and purge 5607 * the cache for the appropriate page. Invalidate the TLB 5608 * as required. 5609 */ 5610 if (!src_okcolor) { 5611 const pt_entry_t nsrc_pte = L2_S_PROTO 5612 | src 5613 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 5614 | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) 5615 #else // defined(PMAP_CACHE_VIVT) || defined(ARM_MMU_EXTENDED) 5616 | pte_l2_s_cache_mode 5617 #endif 5618 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); 5619 l2pte_set(src_ptep, nsrc_pte, 0); 5620 PTE_SYNC(src_ptep); 5621 cpu_tlb_flushD_SE(vsrcp); 5622 cpu_cpwait(); 5623 } 5624 if (!dst_okcolor) { 5625 const pt_entry_t ndst_pte = L2_S_PROTO | dst | 5626 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 5627 l2pte_set(dst_ptep, ndst_pte, 0); 5628 PTE_SYNC(dst_ptep); 5629 cpu_tlb_flushD_SE(vdstp); 5630 cpu_cpwait(); 5631 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) 5632 /* 5633 * If we are direct-mapped and our color isn't ok, then before 5634 * we bcopy to the new page invalidate its contents from the 5635 * cache and reset its color to its natural color. 5636 */ 5637 cpu_dcache_inv_range(vdstp, PAGE_SIZE); 5638 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5639 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5640 #endif 5641 } 5642 bcopy_page(vsrcp, vdstp); 5643 #ifdef PMAP_CACHE_VIVT 5644 cpu_dcache_inv_range(vsrcp, PAGE_SIZE); 5645 cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); 5646 #endif 5647 /* 5648 * Unmap the pages. 5649 */ 5650 if (!src_okcolor) { 5651 l2pte_reset(src_ptep); 5652 PTE_SYNC(src_ptep); 5653 cpu_tlb_flushD_SE(vsrcp); 5654 cpu_cpwait(); 5655 } 5656 if (!dst_okcolor) { 5657 l2pte_reset(dst_ptep); 5658 PTE_SYNC(dst_ptep); 5659 cpu_tlb_flushD_SE(vdstp); 5660 cpu_cpwait(); 5661 } 5662 #ifdef PMAP_CACHE_VIPT 5663 /* 5664 * Now that the destination page is in the cache, mark it as colored. 5665 * If this was an exec page, discard it. 5666 */ 5667 pmap_acquire_page_lock(dst_md); 5668 #ifndef ARM_MMU_EXTENDED 5669 if (arm_pcache.cache_type == CACHE_TYPE_PIPT) { 5670 dst_md->pvh_attrs &= ~arm_cache_prefer_mask; 5671 dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); 5672 } 5673 if (!pmap_is_page_colored_p(dst_md)) { 5674 PMAPCOUNT(vac_color_new); 5675 dst_md->pvh_attrs |= PVF_COLORED; 5676 } 5677 dst_md->pvh_attrs |= PVF_DIRTY; 5678 #endif 5679 if (PV_IS_EXEC_P(dst_md->pvh_attrs)) { 5680 dst_md->pvh_attrs &= ~PVF_EXEC; 5681 PMAPCOUNT(exec_discarded_copy); 5682 } 5683 pmap_release_page_lock(dst_md); 5684 #endif 5685 } 5686 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 5687 5688 #if ARM_MMU_XSCALE == 1 5689 void 5690 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 5691 { 5692 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 5693 struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg); 5694 #ifdef DEBUG 5695 struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst)); 5696 5697 if (!SLIST_EMPTY(&dst_md->pvh_list)) 5698 panic("pmap_copy_page: dst page has mappings"); 5699 #endif 5700 5701 KDASSERT((src & PGOFSET) == 0); 5702 KDASSERT((dst & PGOFSET) == 0); 5703 5704 /* 5705 * Clean the source page. Hold the source page's lock for 5706 * the duration of the copy so that no other mappings can 5707 * be created while we have a potentially aliased mapping. 5708 */ 5709 #ifdef PMAP_CACHE_VIVT 5710 pmap_acquire_page_lock(src_md); 5711 (void) pmap_clean_page(src_md, true); 5712 pmap_release_page_lock(src_md); 5713 #endif 5714 5715 /* 5716 * Map the pages into the page hook points, copy them, and purge 5717 * the cache for the appropriate page. Invalidate the TLB 5718 * as required. 5719 */ 5720 const pt_entry_t nsrc_pte = L2_S_PROTO | src 5721 | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 5722 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5723 l2pte_set(csrc_pte, nsrc_pte, 0); 5724 PTE_SYNC(csrc_pte); 5725 5726 const pt_entry_t ndst_pte = L2_S_PROTO | dst 5727 | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) 5728 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X); /* mini-data */ 5729 l2pte_set(cdst_pte, ndst_pte, 0); 5730 PTE_SYNC(cdst_pte); 5731 5732 cpu_tlb_flushD_SE(csrcp); 5733 cpu_tlb_flushD_SE(cdstp); 5734 cpu_cpwait(); 5735 bcopy_page(csrcp, cdstp); 5736 xscale_cache_clean_minidata(); 5737 l2pte_reset(csrc_pte); 5738 l2pte_reset(cdst_pte); 5739 PTE_SYNC(csrc_pte); 5740 PTE_SYNC(cdst_pte); 5741 } 5742 #endif /* ARM_MMU_XSCALE == 1 */ 5743 5744 /* 5745 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5746 * 5747 * Return the start and end addresses of the kernel's virtual space. 5748 * These values are setup in pmap_bootstrap and are updated as pages 5749 * are allocated. 5750 */ 5751 void 5752 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 5753 { 5754 *start = virtual_avail; 5755 *end = virtual_end; 5756 } 5757 5758 /* 5759 * Helper function for pmap_grow_l2_bucket() 5760 */ 5761 static inline int 5762 pmap_grow_map(vaddr_t va, paddr_t *pap) 5763 { 5764 paddr_t pa; 5765 5766 if (uvm.page_init_done == false) { 5767 #ifdef PMAP_STEAL_MEMORY 5768 pv_addr_t pv; 5769 pmap_boot_pagealloc(PAGE_SIZE, 5770 #ifdef PMAP_CACHE_VIPT 5771 arm_cache_prefer_mask, 5772 va & arm_cache_prefer_mask, 5773 #else 5774 0, 0, 5775 #endif 5776 &pv); 5777 pa = pv.pv_pa; 5778 #else 5779 if (uvm_page_physget(&pa) == false) 5780 return (1); 5781 #endif /* PMAP_STEAL_MEMORY */ 5782 } else { 5783 struct vm_page *pg; 5784 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE); 5785 if (pg == NULL) 5786 return (1); 5787 pa = VM_PAGE_TO_PHYS(pg); 5788 /* 5789 * This new page must not have any mappings. Enter it via 5790 * pmap_kenter_pa and let that routine do the hard work. 5791 */ 5792 struct vm_page_md *md __diagused = VM_PAGE_TO_MD(pg); 5793 KASSERT(SLIST_EMPTY(&md->pvh_list)); 5794 pmap_kenter_pa(va, pa, 5795 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 5796 } 5797 5798 if (pap) 5799 *pap = pa; 5800 5801 PMAPCOUNT(pt_mappings); 5802 #ifdef DEBUG 5803 struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va); 5804 KDASSERT(l2b != NULL); 5805 5806 pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)]; 5807 const pt_entry_t opte = *ptep; 5808 KDASSERT((opte & L2_S_CACHE_MASK) == pte_l2_s_cache_mode_pt); 5809 #endif 5810 memset((void *)va, 0, PAGE_SIZE); 5811 return (0); 5812 } 5813 5814 /* 5815 * This is the same as pmap_alloc_l2_bucket(), except that it is only 5816 * used by pmap_growkernel(). 5817 */ 5818 static inline struct l2_bucket * 5819 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va) 5820 { 5821 struct l2_dtable *l2; 5822 struct l2_bucket *l2b; 5823 u_short l1slot; 5824 vaddr_t nva; 5825 5826 l1slot = l1pte_index(va); 5827 5828 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 5829 /* 5830 * No mapping at this address, as there is 5831 * no entry in the L1 table. 5832 * Need to allocate a new l2_dtable. 5833 */ 5834 nva = pmap_kernel_l2dtable_kva; 5835 if ((nva & PGOFSET) == 0) { 5836 /* 5837 * Need to allocate a backing page 5838 */ 5839 if (pmap_grow_map(nva, NULL)) 5840 return (NULL); 5841 } 5842 5843 l2 = (struct l2_dtable *)nva; 5844 nva += sizeof(struct l2_dtable); 5845 5846 if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) { 5847 /* 5848 * The new l2_dtable straddles a page boundary. 5849 * Map in another page to cover it. 5850 */ 5851 if (pmap_grow_map(nva, NULL)) 5852 return (NULL); 5853 } 5854 5855 pmap_kernel_l2dtable_kva = nva; 5856 5857 /* 5858 * Link it into the parent pmap 5859 */ 5860 pm->pm_l2[L2_IDX(l1slot)] = l2; 5861 } 5862 5863 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 5864 5865 /* 5866 * Fetch pointer to the L2 page table associated with the address. 5867 */ 5868 if (l2b->l2b_kva == NULL) { 5869 pt_entry_t *ptep; 5870 5871 /* 5872 * No L2 page table has been allocated. Chances are, this 5873 * is because we just allocated the l2_dtable, above. 5874 */ 5875 nva = pmap_kernel_l2ptp_kva; 5876 ptep = (pt_entry_t *)nva; 5877 if ((nva & PGOFSET) == 0) { 5878 /* 5879 * Need to allocate a backing page 5880 */ 5881 if (pmap_grow_map(nva, &pmap_kernel_l2ptp_phys)) 5882 return (NULL); 5883 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 5884 } 5885 5886 l2->l2_occupancy++; 5887 l2b->l2b_kva = ptep; 5888 l2b->l2b_l1slot = l1slot; 5889 l2b->l2b_pa = pmap_kernel_l2ptp_phys; 5890 5891 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 5892 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 5893 } 5894 5895 return (l2b); 5896 } 5897 5898 vaddr_t 5899 pmap_growkernel(vaddr_t maxkvaddr) 5900 { 5901 pmap_t kpm = pmap_kernel(); 5902 #ifndef ARM_MMU_EXTENDED 5903 struct l1_ttable *l1; 5904 #endif 5905 int s; 5906 5907 if (maxkvaddr <= pmap_curmaxkvaddr) 5908 goto out; /* we are OK */ 5909 5910 NPDEBUG(PDB_GROWKERN, 5911 printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n", 5912 pmap_curmaxkvaddr, maxkvaddr)); 5913 5914 KDASSERT(maxkvaddr <= virtual_end); 5915 5916 /* 5917 * whoops! we need to add kernel PTPs 5918 */ 5919 5920 s = splhigh(); /* to be safe */ 5921 mutex_enter(kpm->pm_lock); 5922 5923 /* Map 1MB at a time */ 5924 size_t l1slot = l1pte_index(pmap_curmaxkvaddr); 5925 #ifdef ARM_MMU_EXTENDED 5926 pd_entry_t * const spdep = &kpm->pm_l1[l1slot]; 5927 pd_entry_t *pdep = spdep; 5928 #endif 5929 for (;pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE, 5930 #ifdef ARM_MMU_EXTENDED 5931 pdep++, 5932 #endif 5933 l1slot++) { 5934 struct l2_bucket *l2b = 5935 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 5936 KASSERT(l2b != NULL); 5937 5938 const pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa 5939 | L1_C_DOM(PMAP_DOMAIN_KERNEL); 5940 #ifdef ARM_MMU_EXTENDED 5941 l1pte_setone(pdep, npde); 5942 #else 5943 /* Distribute new L1 entry to all other L1s */ 5944 SLIST_FOREACH(l1, &l1_list, l1_link) { 5945 pd_entry_t * const pdep = &l1->l1_kva[l1slot]; 5946 l1pte_setone(pdep, npde); 5947 PDE_SYNC(pdep); 5948 } 5949 #endif 5950 } 5951 #ifdef ARM_MMU_EXTENDED 5952 PDE_SYNC_RANGE(spdep, pdep - spdep); 5953 #endif 5954 5955 #ifdef PMAP_CACHE_VIVT 5956 /* 5957 * flush out the cache, expensive but growkernel will happen so 5958 * rarely 5959 */ 5960 cpu_dcache_wbinv_all(); 5961 cpu_tlb_flushD(); 5962 cpu_cpwait(); 5963 #endif 5964 5965 mutex_exit(kpm->pm_lock); 5966 splx(s); 5967 5968 out: 5969 return (pmap_curmaxkvaddr); 5970 } 5971 5972 /************************ Utility routines ****************************/ 5973 5974 #ifndef ARM_HAS_VBAR 5975 /* 5976 * vector_page_setprot: 5977 * 5978 * Manipulate the protection of the vector page. 5979 */ 5980 void 5981 vector_page_setprot(int prot) 5982 { 5983 struct l2_bucket *l2b; 5984 pt_entry_t *ptep; 5985 5986 #if defined(CPU_ARMV7) || defined(CPU_ARM11) 5987 /* 5988 * If we are using VBAR to use the vectors in the kernel, then it's 5989 * already mapped in the kernel text so no need to anything here. 5990 */ 5991 if (vector_page != ARM_VECTORS_LOW && vector_page != ARM_VECTORS_HIGH) { 5992 KASSERT((armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0); 5993 return; 5994 } 5995 #endif 5996 5997 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 5998 KASSERT(l2b != NULL); 5999 6000 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 6001 6002 const pt_entry_t opte = *ptep; 6003 #ifdef ARM_MMU_EXTENDED 6004 const pt_entry_t npte = (opte & ~(L2_S_PROT_MASK|L2_XS_XN)) 6005 | L2_S_PROT(PTE_KERNEL, prot); 6006 #else 6007 const pt_entry_t npte = (opte & ~L2_S_PROT_MASK) 6008 | L2_S_PROT(PTE_KERNEL, prot); 6009 #endif 6010 l2pte_set(ptep, npte, opte); 6011 PTE_SYNC(ptep); 6012 cpu_tlb_flushD_SE(vector_page); 6013 cpu_cpwait(); 6014 } 6015 #endif 6016 6017 /* 6018 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 6019 * Returns true if the mapping exists, else false. 6020 * 6021 * NOTE: This function is only used by a couple of arm-specific modules. 6022 * It is not safe to take any pmap locks here, since we could be right 6023 * in the middle of debugging the pmap anyway... 6024 * 6025 * It is possible for this routine to return false even though a valid 6026 * mapping does exist. This is because we don't lock, so the metadata 6027 * state may be inconsistent. 6028 * 6029 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 6030 * a "section" mapping. 6031 */ 6032 bool 6033 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp) 6034 { 6035 struct l2_dtable *l2; 6036 pd_entry_t *pdep, pde; 6037 pt_entry_t *ptep; 6038 u_short l1slot; 6039 6040 if (pm->pm_l1 == NULL) 6041 return false; 6042 6043 l1slot = l1pte_index(va); 6044 *pdp = pdep = pmap_l1_kva(pm) + l1slot; 6045 pde = *pdep; 6046 6047 if (l1pte_section_p(pde)) { 6048 *ptp = NULL; 6049 return true; 6050 } 6051 6052 l2 = pm->pm_l2[L2_IDX(l1slot)]; 6053 if (l2 == NULL || 6054 (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) { 6055 return false; 6056 } 6057 6058 *ptp = &ptep[l2pte_index(va)]; 6059 return true; 6060 } 6061 6062 bool 6063 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp) 6064 { 6065 6066 if (pm->pm_l1 == NULL) 6067 return false; 6068 6069 *pdp = pmap_l1_kva(pm) + l1pte_index(va); 6070 6071 return true; 6072 } 6073 6074 /************************ Bootstrapping routines ****************************/ 6075 6076 #ifndef ARM_MMU_EXTENDED 6077 static void 6078 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 6079 { 6080 int i; 6081 6082 l1->l1_kva = l1pt; 6083 l1->l1_domain_use_count = 0; 6084 l1->l1_domain_first = 0; 6085 6086 for (i = 0; i < PMAP_DOMAINS; i++) 6087 l1->l1_domain_free[i] = i + 1; 6088 6089 /* 6090 * Copy the kernel's L1 entries to each new L1. 6091 */ 6092 if (pmap_initialized) 6093 memcpy(l1pt, pmap_l1_kva(pmap_kernel()), L1_TABLE_SIZE); 6094 6095 if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt, 6096 &l1->l1_physaddr) == false) 6097 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 6098 6099 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 6100 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 6101 } 6102 #endif /* !ARM_MMU_EXTENDED */ 6103 6104 /* 6105 * pmap_bootstrap() is called from the board-specific initarm() routine 6106 * once the kernel L1/L2 descriptors tables have been set up. 6107 * 6108 * This is a somewhat convoluted process since pmap bootstrap is, effectively, 6109 * spread over a number of disparate files/functions. 6110 * 6111 * We are passed the following parameters 6112 * - vstart 6113 * 1MB-aligned start of managed kernel virtual memory. 6114 * - vend 6115 * 1MB-aligned end of managed kernel virtual memory. 6116 * 6117 * We use 'kernel_l1pt' to build the metadata (struct l1_ttable and 6118 * struct l2_dtable) necessary to track kernel mappings. 6119 */ 6120 #define PMAP_STATIC_L2_SIZE 16 6121 void 6122 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 6123 { 6124 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 6125 #ifndef ARM_MMU_EXTENDED 6126 static struct l1_ttable static_l1; 6127 struct l1_ttable *l1 = &static_l1; 6128 #endif 6129 struct l2_dtable *l2; 6130 struct l2_bucket *l2b; 6131 pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va; 6132 pmap_t pm = pmap_kernel(); 6133 pt_entry_t *ptep; 6134 paddr_t pa; 6135 vsize_t size; 6136 int nptes, l2idx, l2next = 0; 6137 6138 #ifdef ARM_MMU_EXTENDED 6139 KASSERT(pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt); 6140 KASSERT(pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt); 6141 #endif 6142 6143 VPRINTF("kpm "); 6144 /* 6145 * Initialise the kernel pmap object 6146 */ 6147 curcpu()->ci_pmap_cur = pm; 6148 #ifdef ARM_MMU_EXTENDED 6149 pm->pm_l1 = l1pt; 6150 pm->pm_l1_pa = kernel_l1pt.pv_pa; 6151 VPRINTF("tlb0 "); 6152 pmap_tlb_info_init(&pmap_tlb0_info); 6153 #ifdef MULTIPROCESSOR 6154 VPRINTF("kcpusets "); 6155 pm->pm_onproc = kcpuset_running; 6156 pm->pm_active = kcpuset_running; 6157 #endif 6158 #else 6159 pm->pm_l1 = l1; 6160 #endif 6161 6162 VPRINTF("locks "); 6163 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 6164 if (arm_cache_prefer_mask != 0) { 6165 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_VM); 6166 } else { 6167 #endif 6168 mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE); 6169 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 6170 } 6171 #endif 6172 mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE); 6173 uvm_obj_init(&pm->pm_obj, NULL, false, 1); 6174 uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock); 6175 6176 VPRINTF("l1pt "); 6177 /* 6178 * Scan the L1 translation table created by initarm() and create 6179 * the required metadata for all valid mappings found in it. 6180 */ 6181 for (size_t l1slot = 0; 6182 l1slot < L1_TABLE_SIZE / sizeof(pd_entry_t); 6183 l1slot++) { 6184 pd_entry_t pde = l1pt[l1slot]; 6185 6186 /* 6187 * We're only interested in Coarse mappings. 6188 * pmap_extract() can deal with section mappings without 6189 * recourse to checking L2 metadata. 6190 */ 6191 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 6192 continue; 6193 6194 /* 6195 * Lookup the KVA of this L2 descriptor table 6196 */ 6197 pa = l1pte_pa(pde); 6198 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6199 if (ptep == NULL) { 6200 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 6201 (u_int)l1slot << L1_S_SHIFT, pa); 6202 } 6203 6204 /* 6205 * Fetch the associated L2 metadata structure. 6206 * Allocate a new one if necessary. 6207 */ 6208 if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) { 6209 if (l2next == PMAP_STATIC_L2_SIZE) 6210 panic("pmap_bootstrap: out of static L2s"); 6211 pm->pm_l2[L2_IDX(l1slot)] = l2 = &static_l2[l2next++]; 6212 } 6213 6214 /* 6215 * One more L1 slot tracked... 6216 */ 6217 l2->l2_occupancy++; 6218 6219 /* 6220 * Fill in the details of the L2 descriptor in the 6221 * appropriate bucket. 6222 */ 6223 l2b = &l2->l2_bucket[L2_BUCKET(l1slot)]; 6224 l2b->l2b_kva = ptep; 6225 l2b->l2b_pa = pa; 6226 l2b->l2b_l1slot = l1slot; 6227 6228 /* 6229 * Establish an initial occupancy count for this descriptor 6230 */ 6231 for (l2idx = 0; 6232 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 6233 l2idx++) { 6234 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 6235 l2b->l2b_occupancy++; 6236 } 6237 } 6238 6239 /* 6240 * Make sure the descriptor itself has the correct cache mode. 6241 * If not, fix it, but whine about the problem. Port-meisters 6242 * should consider this a clue to fix up their initarm() 6243 * function. :) 6244 */ 6245 if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep, 1)) { 6246 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6247 "L2 pte @ %p\n", ptep); 6248 } 6249 } 6250 6251 VPRINTF("cache(l1pt) "); 6252 /* 6253 * Ensure the primary (kernel) L1 has the correct cache mode for 6254 * a page table. Bitch if it is not correctly set. 6255 */ 6256 if (pmap_set_pt_cache_mode(l1pt, kernel_l1pt.pv_va, 6257 L1_TABLE_SIZE / L2_S_SIZE)) { 6258 printf("pmap_bootstrap: WARNING! wrong cache mode for " 6259 "primary L1 @ 0x%lx\n", kernel_l1pt.pv_va); 6260 } 6261 6262 #ifdef PMAP_CACHE_VIVT 6263 cpu_dcache_wbinv_all(); 6264 cpu_tlb_flushID(); 6265 cpu_cpwait(); 6266 #endif 6267 6268 /* 6269 * now we allocate the "special" VAs which are used for tmp mappings 6270 * by the pmap (and other modules). we allocate the VAs by advancing 6271 * virtual_avail (note that there are no pages mapped at these VAs). 6272 * 6273 * Managed KVM space start from wherever initarm() tells us. 6274 */ 6275 virtual_avail = vstart; 6276 virtual_end = vend; 6277 6278 VPRINTF("specials "); 6279 #ifdef PMAP_CACHE_VIPT 6280 /* 6281 * If we have a VIPT cache, we need one page/pte per possible alias 6282 * page so we won't violate cache aliasing rules. 6283 */ 6284 virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask; 6285 nptes = (arm_cache_prefer_mask >> L2_S_SHIFT) + 1; 6286 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6287 if (arm_pcache.icache_type != CACHE_TYPE_PIPT 6288 && arm_pcache.icache_way_size > nptes * L2_S_SIZE) { 6289 nptes = arm_pcache.icache_way_size >> L2_S_SHIFT; 6290 nptes = roundup(nptes, PAGE_SIZE / L2_S_SIZE); 6291 } 6292 #else 6293 nptes = PAGE_SIZE / L2_S_SIZE; 6294 #endif 6295 #ifdef MULTIPROCESSOR 6296 cnptes = nptes; 6297 nptes *= arm_cpu_max; 6298 #endif 6299 pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte); 6300 pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte, nptes); 6301 pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte); 6302 pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte, nptes); 6303 pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL); 6304 if (msgbufaddr == NULL) { 6305 pmap_alloc_specials(&virtual_avail, 6306 round_page(MSGBUFSIZE) / PAGE_SIZE, 6307 (void *)&msgbufaddr, NULL); 6308 } 6309 6310 /* 6311 * Allocate a range of kernel virtual address space to be used 6312 * for L2 descriptor tables and metadata allocation in 6313 * pmap_growkernel(). 6314 */ 6315 size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 6316 pmap_alloc_specials(&virtual_avail, 6317 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 6318 &pmap_kernel_l2ptp_kva, NULL); 6319 6320 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 6321 pmap_alloc_specials(&virtual_avail, 6322 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 6323 &pmap_kernel_l2dtable_kva, NULL); 6324 6325 #ifndef ARM_MMU_EXTENDED 6326 /* 6327 * init the static-global locks and global pmap list. 6328 */ 6329 mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM); 6330 6331 /* 6332 * We can now initialise the first L1's metadata. 6333 */ 6334 SLIST_INIT(&l1_list); 6335 TAILQ_INIT(&l1_lru_list); 6336 pmap_init_l1(l1, l1pt); 6337 #endif /* ARM_MMU_EXTENDED */ 6338 6339 #ifndef ARM_HAS_VBAR 6340 /* Set up vector page L1 details, if necessary */ 6341 if (vector_page < KERNEL_BASE) { 6342 pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page); 6343 l2b = pmap_get_l2_bucket(pm, vector_page); 6344 KDASSERT(l2b != NULL); 6345 pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO | 6346 L1_C_DOM(pmap_domain(pm)); 6347 } else 6348 pm->pm_pl1vec = NULL; 6349 #endif 6350 6351 VPRINTF("pools "); 6352 /* 6353 * Initialize the pmap cache 6354 */ 6355 pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, 6356 "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL); 6357 6358 /* 6359 * Initialize the pv pool. 6360 */ 6361 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl", 6362 &pmap_bootstrap_pv_allocator, IPL_NONE); 6363 6364 /* 6365 * Initialize the L2 dtable pool and cache. 6366 */ 6367 pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0, 6368 0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL); 6369 6370 /* 6371 * Initialise the L2 descriptor table pool and cache 6372 */ 6373 pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 6374 L2_TABLE_SIZE_REAL, 0, 0, "l2ptppl", NULL, IPL_NONE, 6375 pmap_l2ptp_ctor, NULL, NULL); 6376 6377 mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE); 6378 6379 cpu_dcache_wbinv_all(); 6380 } 6381 6382 static bool 6383 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va, size_t nptes) 6384 { 6385 #ifdef ARM_MMU_EXTENDED 6386 return false; 6387 #else 6388 if (pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 6389 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 6390 return false; 6391 6392 const vaddr_t eva = va + nptes * PAGE_SIZE; 6393 int rv = 0; 6394 6395 while (va < eva) { 6396 /* 6397 * Make sure the descriptor itself has the correct cache mode 6398 */ 6399 pd_entry_t * const pdep = &kl1[l1pte_index(va)]; 6400 pd_entry_t pde = *pdep; 6401 6402 if (l1pte_section_p(pde)) { 6403 __CTASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0); 6404 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 6405 *pdep = (pde & ~L1_S_CACHE_MASK) | 6406 pte_l1_s_cache_mode_pt; 6407 PDE_SYNC(pdep); 6408 cpu_dcache_wbinv_range((vaddr_t)pdep, 6409 sizeof(*pdep)); 6410 rv = 1; 6411 } 6412 return rv; 6413 } 6414 vaddr_t pa = l1pte_pa(pde); 6415 pt_entry_t *ptep = (pt_entry_t *)kernel_pt_lookup(pa); 6416 if (ptep == NULL) 6417 panic("pmap_bootstrap: No PTP for va %#lx\n", va); 6418 6419 ptep += l2pte_index(va); 6420 const pt_entry_t opte = *ptep; 6421 if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 6422 const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK) 6423 | pte_l2_s_cache_mode_pt; 6424 l2pte_set(ptep, npte, opte); 6425 PTE_SYNC(ptep); 6426 cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep)); 6427 rv = 1; 6428 } 6429 va += PAGE_SIZE; 6430 } 6431 6432 return (rv); 6433 #endif 6434 } 6435 6436 static void 6437 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep) 6438 { 6439 vaddr_t va = *availp; 6440 struct l2_bucket *l2b; 6441 6442 if (ptep) { 6443 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 6444 if (l2b == NULL) 6445 panic("pmap_alloc_specials: no l2b for 0x%lx", va); 6446 6447 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 6448 } 6449 6450 *vap = va; 6451 *availp = va + (PAGE_SIZE * pages); 6452 } 6453 6454 void 6455 pmap_init(void) 6456 { 6457 6458 /* 6459 * Set the available memory vars - These do not map to real memory 6460 * addresses and cannot as the physical memory is fragmented. 6461 * They are used by ps for %mem calculations. 6462 * One could argue whether this should be the entire memory or just 6463 * the memory that is useable in a user process. 6464 */ 6465 avail_start = ptoa(uvm_physseg_get_avail_start(uvm_physseg_get_first())); 6466 avail_end = ptoa(uvm_physseg_get_avail_end(uvm_physseg_get_last())); 6467 6468 /* 6469 * Now we need to free enough pv_entry structures to allow us to get 6470 * the kmem_map/kmem_object allocated and inited (done after this 6471 * function is finished). to do this we allocate one bootstrap page out 6472 * of kernel_map and use it to provide an initial pool of pv_entry 6473 * structures. we never free this page. 6474 */ 6475 pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2); 6476 6477 #ifdef ARM_MMU_EXTENDED 6478 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 6479 #endif 6480 6481 pmap_initialized = true; 6482 } 6483 6484 static vaddr_t last_bootstrap_page = 0; 6485 static void *free_bootstrap_pages = NULL; 6486 6487 static void * 6488 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags) 6489 { 6490 extern void *pool_page_alloc(struct pool *, int); 6491 vaddr_t new_page; 6492 void *rv; 6493 6494 if (pmap_initialized) 6495 return (pool_page_alloc(pp, flags)); 6496 6497 if (free_bootstrap_pages) { 6498 rv = free_bootstrap_pages; 6499 free_bootstrap_pages = *((void **)rv); 6500 return (rv); 6501 } 6502 6503 KASSERT(kernel_map != NULL); 6504 new_page = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 6505 UVM_KMF_WIRED | ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT)); 6506 6507 KASSERT(new_page > last_bootstrap_page); 6508 last_bootstrap_page = new_page; 6509 return ((void *)new_page); 6510 } 6511 6512 static void 6513 pmap_bootstrap_pv_page_free(struct pool *pp, void *v) 6514 { 6515 extern void pool_page_free(struct pool *, void *); 6516 6517 if ((vaddr_t)v <= last_bootstrap_page) { 6518 *((void **)v) = free_bootstrap_pages; 6519 free_bootstrap_pages = v; 6520 return; 6521 } 6522 6523 if (pmap_initialized) { 6524 pool_page_free(pp, v); 6525 return; 6526 } 6527 } 6528 6529 /* 6530 * pmap_postinit() 6531 * 6532 * This routine is called after the vm and kmem subsystems have been 6533 * initialised. This allows the pmap code to perform any initialisation 6534 * that can only be done once the memory allocation is in place. 6535 */ 6536 void 6537 pmap_postinit(void) 6538 { 6539 #ifndef ARM_MMU_EXTENDED 6540 extern paddr_t physical_start, physical_end; 6541 struct l1_ttable *l1; 6542 struct pglist plist; 6543 struct vm_page *m; 6544 pd_entry_t *pdep; 6545 vaddr_t va, eva; 6546 u_int loop, needed; 6547 int error; 6548 #endif 6549 6550 pool_cache_setlowat(&pmap_l2ptp_cache, (PAGE_SIZE / L2_TABLE_SIZE_REAL) * 4); 6551 pool_cache_setlowat(&pmap_l2dtable_cache, 6552 (PAGE_SIZE / sizeof(struct l2_dtable)) * 2); 6553 6554 #ifndef ARM_MMU_EXTENDED 6555 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 6556 needed -= 1; 6557 6558 l1 = kmem_alloc(sizeof(*l1) * needed, KM_SLEEP); 6559 6560 for (loop = 0; loop < needed; loop++, l1++) { 6561 /* Allocate a L1 page table */ 6562 va = uvm_km_alloc(kernel_map, L1_TABLE_SIZE, 0, UVM_KMF_VAONLY); 6563 if (va == 0) 6564 panic("Cannot allocate L1 KVM"); 6565 6566 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, 6567 physical_end, L1_TABLE_SIZE, 0, &plist, 1, 1); 6568 if (error) 6569 panic("Cannot allocate L1 physical pages"); 6570 6571 m = TAILQ_FIRST(&plist); 6572 eva = va + L1_TABLE_SIZE; 6573 pdep = (pd_entry_t *)va; 6574 6575 while (m && va < eva) { 6576 paddr_t pa = VM_PAGE_TO_PHYS(m); 6577 6578 pmap_kenter_pa(va, pa, 6579 VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE); 6580 6581 va += PAGE_SIZE; 6582 m = TAILQ_NEXT(m, pageq.queue); 6583 } 6584 6585 #ifdef DIAGNOSTIC 6586 if (m) 6587 panic("pmap_alloc_l1pt: pglist not empty"); 6588 #endif /* DIAGNOSTIC */ 6589 6590 pmap_init_l1(l1, pdep); 6591 } 6592 6593 #ifdef DEBUG 6594 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 6595 needed); 6596 #endif 6597 #endif /* !ARM_MMU_EXTENDED */ 6598 } 6599 6600 /* 6601 * Note that the following routines are used by board-specific initialisation 6602 * code to configure the initial kernel page tables. 6603 * 6604 */ 6605 6606 /* 6607 * This list exists for the benefit of pmap_map_chunk(). It keeps track 6608 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 6609 * find them as necessary. 6610 * 6611 * Note that the data on this list MUST remain valid after initarm() returns, 6612 * as pmap_bootstrap() uses it to contruct L2 table metadata. 6613 */ 6614 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 6615 6616 static vaddr_t 6617 kernel_pt_lookup(paddr_t pa) 6618 { 6619 pv_addr_t *pv; 6620 6621 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 6622 if (pv->pv_pa == (pa & ~PGOFSET)) 6623 return (pv->pv_va | (pa & PGOFSET)); 6624 } 6625 return (0); 6626 } 6627 6628 /* 6629 * pmap_map_section: 6630 * 6631 * Create a single section mapping. 6632 */ 6633 void 6634 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6635 { 6636 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6637 const size_t l1slot = l1pte_index(va); 6638 pd_entry_t fl; 6639 6640 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 6641 6642 switch (cache) { 6643 case PTE_NOCACHE: 6644 default: 6645 fl = 0; 6646 break; 6647 6648 case PTE_CACHE: 6649 fl = pte_l1_s_cache_mode; 6650 break; 6651 6652 case PTE_PAGETABLE: 6653 fl = pte_l1_s_cache_mode_pt; 6654 break; 6655 } 6656 6657 const pd_entry_t npde = L1_S_PROTO | pa | 6658 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6659 l1pte_setone(pdep + l1slot, npde); 6660 PDE_SYNC(pdep + l1slot); 6661 } 6662 6663 /* 6664 * pmap_map_entry: 6665 * 6666 * Create a single page mapping. 6667 */ 6668 void 6669 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 6670 { 6671 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6672 const size_t l1slot = l1pte_index(va); 6673 pt_entry_t npte; 6674 pt_entry_t *ptep; 6675 6676 KASSERT(((va | pa) & PGOFSET) == 0); 6677 6678 switch (cache) { 6679 case PTE_NOCACHE: 6680 default: 6681 npte = 0; 6682 break; 6683 6684 case PTE_CACHE: 6685 npte = pte_l2_s_cache_mode; 6686 break; 6687 6688 case PTE_PAGETABLE: 6689 npte = pte_l2_s_cache_mode_pt; 6690 break; 6691 } 6692 6693 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6694 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 6695 6696 ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6697 if (ptep == NULL) 6698 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 6699 6700 npte |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot); 6701 #ifdef ARM_MMU_EXTENDED 6702 if (prot & VM_PROT_EXECUTE) { 6703 npte &= ~L2_XS_XN; 6704 } 6705 #endif 6706 ptep += l2pte_index(va); 6707 l2pte_set(ptep, npte, 0); 6708 PTE_SYNC(ptep); 6709 } 6710 6711 /* 6712 * pmap_link_l2pt: 6713 * 6714 * Link the L2 page table specified by "l2pv" into the L1 6715 * page table at the slot for "va". 6716 */ 6717 void 6718 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 6719 { 6720 pd_entry_t * const pdep = (pd_entry_t *) l1pt + l1pte_index(va); 6721 6722 KASSERT((va & ((L1_S_SIZE * (PAGE_SIZE / L2_T_SIZE)) - 1)) == 0); 6723 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 6724 6725 const pd_entry_t npde = L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO 6726 | l2pv->pv_pa; 6727 6728 l1pte_set(pdep, npde); 6729 PDE_SYNC_RANGE(pdep, PAGE_SIZE / L2_T_SIZE); 6730 6731 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 6732 } 6733 6734 /* 6735 * pmap_map_chunk: 6736 * 6737 * Map a chunk of memory using the most efficient mappings 6738 * possible (section, large page, small page) into the 6739 * provided L1 and L2 tables at the specified virtual address. 6740 */ 6741 vsize_t 6742 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 6743 int prot, int cache) 6744 { 6745 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6746 pt_entry_t f1, f2s, f2l; 6747 vsize_t resid; 6748 6749 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 6750 6751 if (l1pt == 0) 6752 panic("pmap_map_chunk: no L1 table provided"); 6753 6754 // VPRINTF("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 6755 // "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 6756 6757 switch (cache) { 6758 case PTE_NOCACHE: 6759 default: 6760 f1 = 0; 6761 f2l = 0; 6762 f2s = 0; 6763 break; 6764 6765 case PTE_CACHE: 6766 f1 = pte_l1_s_cache_mode; 6767 f2l = pte_l2_l_cache_mode; 6768 f2s = pte_l2_s_cache_mode; 6769 break; 6770 6771 case PTE_PAGETABLE: 6772 f1 = pte_l1_s_cache_mode_pt; 6773 f2l = pte_l2_l_cache_mode_pt; 6774 f2s = pte_l2_s_cache_mode_pt; 6775 break; 6776 } 6777 6778 size = resid; 6779 6780 while (resid > 0) { 6781 const size_t l1slot = l1pte_index(va); 6782 #ifdef ARM_MMU_EXTENDED 6783 /* See if we can use a supersection mapping. */ 6784 if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) { 6785 /* Supersection are always domain 0 */ 6786 const pd_entry_t npde = L1_SS_PROTO | pa 6787 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6788 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6789 | L1_S_PROT(PTE_KERNEL, prot) | f1; 6790 VPRINTF("sS"); 6791 l1pte_set(&pdep[l1slot], npde); 6792 PDE_SYNC_RANGE(&pdep[l1slot], L1_SS_SIZE / L1_S_SIZE); 6793 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6794 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6795 va += L1_SS_SIZE; 6796 pa += L1_SS_SIZE; 6797 resid -= L1_SS_SIZE; 6798 continue; 6799 } 6800 #endif 6801 /* See if we can use a section mapping. */ 6802 if (L1_S_MAPPABLE_P(va, pa, resid)) { 6803 const pd_entry_t npde = L1_S_PROTO | pa 6804 #ifdef ARM_MMU_EXTENDED 6805 | ((prot & VM_PROT_EXECUTE) ? 0 : L1_S_V6_XN) 6806 | (va & 0x80000000 ? 0 : L1_S_V6_nG) 6807 #endif 6808 | L1_S_PROT(PTE_KERNEL, prot) | f1 6809 | L1_S_DOM(PMAP_DOMAIN_KERNEL); 6810 VPRINTF("S"); 6811 l1pte_set(&pdep[l1slot], npde); 6812 PDE_SYNC(&pdep[l1slot]); 6813 // VPRINTF("\npmap_map_chunk: pa=0x%lx va=0x%lx resid=0x%08lx " 6814 // "npdep=%p pde=0x%x\n", pa, va, resid, &pdep[l1slot], npde); 6815 va += L1_S_SIZE; 6816 pa += L1_S_SIZE; 6817 resid -= L1_S_SIZE; 6818 continue; 6819 } 6820 6821 /* 6822 * Ok, we're going to use an L2 table. Make sure 6823 * one is actually in the corresponding L1 slot 6824 * for the current VA. 6825 */ 6826 if ((pdep[l1slot] & L1_TYPE_MASK) != L1_TYPE_C) 6827 panic("%s: no L2 table for VA %#lx", __func__, va); 6828 6829 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pdep[l1slot])); 6830 if (ptep == NULL) 6831 panic("%s: can't find L2 table for VA %#lx", __func__, 6832 va); 6833 6834 ptep += l2pte_index(va); 6835 6836 /* See if we can use a L2 large page mapping. */ 6837 if (L2_L_MAPPABLE_P(va, pa, resid)) { 6838 const pt_entry_t npte = L2_L_PROTO | pa 6839 #ifdef ARM_MMU_EXTENDED 6840 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_L_XN) 6841 | (va & 0x80000000 ? 0 : L2_XS_nG) 6842 #endif 6843 | L2_L_PROT(PTE_KERNEL, prot) | f2l; 6844 VPRINTF("L"); 6845 l2pte_set(ptep, npte, 0); 6846 PTE_SYNC_RANGE(ptep, L2_L_SIZE / L2_S_SIZE); 6847 va += L2_L_SIZE; 6848 pa += L2_L_SIZE; 6849 resid -= L2_L_SIZE; 6850 continue; 6851 } 6852 6853 VPRINTF("P"); 6854 /* Use a small page mapping. */ 6855 pt_entry_t npte = L2_S_PROTO | pa 6856 #ifdef ARM_MMU_EXTENDED 6857 | ((prot & VM_PROT_EXECUTE) ? 0 : L2_XS_XN) 6858 | (va & 0x80000000 ? 0 : L2_XS_nG) 6859 #endif 6860 | L2_S_PROT(PTE_KERNEL, prot) | f2s; 6861 #ifdef ARM_MMU_EXTENDED 6862 npte &= ((prot & VM_PROT_EXECUTE) ? ~L2_XS_XN : ~0); 6863 #endif 6864 l2pte_set(ptep, npte, 0); 6865 PTE_SYNC(ptep); 6866 va += PAGE_SIZE; 6867 pa += PAGE_SIZE; 6868 resid -= PAGE_SIZE; 6869 } 6870 VPRINTF("\n"); 6871 return (size); 6872 } 6873 6874 /* 6875 * pmap_unmap_chunk: 6876 * 6877 * Unmap a chunk of memory that was previously pmap_map_chunk 6878 */ 6879 void 6880 pmap_unmap_chunk(vaddr_t l1pt, vaddr_t va, vsize_t size) 6881 { 6882 pd_entry_t * const pdep = (pd_entry_t *) l1pt; 6883 const size_t l1slot = l1pte_index(va); 6884 6885 KASSERT(size == L1_SS_SIZE || size == L1_S_SIZE); 6886 6887 l1pte_set(&pdep[l1slot], 0); 6888 PDE_SYNC_RANGE(&pdep[l1slot], size / L1_S_SIZE); 6889 6890 pmap_tlb_flush_SE(pmap_kernel(), va, PVF_REF); 6891 } 6892 6893 6894 6895 /********************** Static device map routines ***************************/ 6896 6897 static const struct pmap_devmap *pmap_devmap_table; 6898 6899 /* 6900 * Register the devmap table. This is provided in case early console 6901 * initialization needs to register mappings created by bootstrap code 6902 * before pmap_devmap_bootstrap() is called. 6903 */ 6904 void 6905 pmap_devmap_register(const struct pmap_devmap *table) 6906 { 6907 6908 pmap_devmap_table = table; 6909 } 6910 6911 /* 6912 * Map all of the static regions in the devmap table, and remember 6913 * the devmap table so other parts of the kernel can look up entries 6914 * later. 6915 */ 6916 void 6917 pmap_devmap_bootstrap(vaddr_t l1pt, const struct pmap_devmap *table) 6918 { 6919 int i; 6920 6921 pmap_devmap_table = table; 6922 6923 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6924 const struct pmap_devmap *pdp = &pmap_devmap_table[i]; 6925 6926 KASSERTMSG(VADDR_MAX - pdp->pd_va >= pdp->pd_size - 1, "va %" PRIxVADDR 6927 " sz %" PRIxPSIZE, pdp->pd_va, pdp->pd_size); 6928 KASSERTMSG(PADDR_MAX - pdp->pd_pa >= pdp->pd_size - 1, "pa %" PRIxPADDR 6929 " sz %" PRIxPSIZE, pdp->pd_pa, pdp->pd_size); 6930 VPRINTF("devmap: %08lx -> %08lx @ %08lx\n", pdp->pd_pa, 6931 pdp->pd_pa + pdp->pd_size - 1, pdp->pd_va); 6932 6933 pmap_map_chunk(l1pt, pdp->pd_va, pdp->pd_pa, pdp->pd_size, 6934 pdp->pd_prot, pdp->pd_cache); 6935 } 6936 } 6937 6938 const struct pmap_devmap * 6939 pmap_devmap_find_pa(paddr_t pa, psize_t size) 6940 { 6941 uint64_t endpa; 6942 int i; 6943 6944 if (pmap_devmap_table == NULL) 6945 return (NULL); 6946 6947 endpa = (uint64_t)pa + (uint64_t)(size - 1); 6948 6949 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6950 if (pa >= pmap_devmap_table[i].pd_pa && 6951 endpa <= (uint64_t)pmap_devmap_table[i].pd_pa + 6952 (uint64_t)(pmap_devmap_table[i].pd_size - 1)) 6953 return (&pmap_devmap_table[i]); 6954 } 6955 6956 return (NULL); 6957 } 6958 6959 const struct pmap_devmap * 6960 pmap_devmap_find_va(vaddr_t va, vsize_t size) 6961 { 6962 int i; 6963 6964 if (pmap_devmap_table == NULL) 6965 return (NULL); 6966 6967 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 6968 if (va >= pmap_devmap_table[i].pd_va && 6969 va + size - 1 <= pmap_devmap_table[i].pd_va + 6970 pmap_devmap_table[i].pd_size - 1) 6971 return (&pmap_devmap_table[i]); 6972 } 6973 6974 return (NULL); 6975 } 6976 6977 /********************** PTE initialization routines **************************/ 6978 6979 /* 6980 * These routines are called when the CPU type is identified to set up 6981 * the PTE prototypes, cache modes, etc. 6982 * 6983 * The variables are always here, just in case modules need to reference 6984 * them (though, they shouldn't). 6985 */ 6986 6987 pt_entry_t pte_l1_s_cache_mode; 6988 pt_entry_t pte_l1_s_wc_mode; 6989 pt_entry_t pte_l1_s_cache_mode_pt; 6990 pt_entry_t pte_l1_s_cache_mask; 6991 6992 pt_entry_t pte_l2_l_cache_mode; 6993 pt_entry_t pte_l2_l_wc_mode; 6994 pt_entry_t pte_l2_l_cache_mode_pt; 6995 pt_entry_t pte_l2_l_cache_mask; 6996 6997 pt_entry_t pte_l2_s_cache_mode; 6998 pt_entry_t pte_l2_s_wc_mode; 6999 pt_entry_t pte_l2_s_cache_mode_pt; 7000 pt_entry_t pte_l2_s_cache_mask; 7001 7002 pt_entry_t pte_l1_s_prot_u; 7003 pt_entry_t pte_l1_s_prot_w; 7004 pt_entry_t pte_l1_s_prot_ro; 7005 pt_entry_t pte_l1_s_prot_mask; 7006 7007 pt_entry_t pte_l2_s_prot_u; 7008 pt_entry_t pte_l2_s_prot_w; 7009 pt_entry_t pte_l2_s_prot_ro; 7010 pt_entry_t pte_l2_s_prot_mask; 7011 7012 pt_entry_t pte_l2_l_prot_u; 7013 pt_entry_t pte_l2_l_prot_w; 7014 pt_entry_t pte_l2_l_prot_ro; 7015 pt_entry_t pte_l2_l_prot_mask; 7016 7017 pt_entry_t pte_l1_ss_proto; 7018 pt_entry_t pte_l1_s_proto; 7019 pt_entry_t pte_l1_c_proto; 7020 pt_entry_t pte_l2_s_proto; 7021 7022 void (*pmap_copy_page_func)(paddr_t, paddr_t); 7023 void (*pmap_zero_page_func)(paddr_t); 7024 7025 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 7026 void 7027 pmap_pte_init_generic(void) 7028 { 7029 7030 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7031 pte_l1_s_wc_mode = L1_S_B; 7032 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 7033 7034 pte_l2_l_cache_mode = L2_B|L2_C; 7035 pte_l2_l_wc_mode = L2_B; 7036 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 7037 7038 pte_l2_s_cache_mode = L2_B|L2_C; 7039 pte_l2_s_wc_mode = L2_B; 7040 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 7041 7042 /* 7043 * If we have a write-through cache, set B and C. If 7044 * we have a write-back cache, then we assume setting 7045 * only C will make those pages write-through (except for those 7046 * Cortex CPUs which can read the L1 caches). 7047 */ 7048 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop 7049 #if ARM_MMU_V7 > 0 7050 || CPU_ID_CORTEX_P(curcpu()->ci_arm_cpuid) 7051 #endif 7052 #if ARM_MMU_V6 > 0 7053 || CPU_ID_ARM11_P(curcpu()->ci_arm_cpuid) /* arm116 errata 399234 */ 7054 #endif 7055 || false) { 7056 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7057 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7058 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7059 } else { 7060 pte_l1_s_cache_mode_pt = L1_S_C; /* write through */ 7061 pte_l2_l_cache_mode_pt = L2_C; /* write through */ 7062 pte_l2_s_cache_mode_pt = L2_C; /* write through */ 7063 } 7064 7065 pte_l1_s_prot_u = L1_S_PROT_U_generic; 7066 pte_l1_s_prot_w = L1_S_PROT_W_generic; 7067 pte_l1_s_prot_ro = L1_S_PROT_RO_generic; 7068 pte_l1_s_prot_mask = L1_S_PROT_MASK_generic; 7069 7070 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7071 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7072 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7073 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7074 7075 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7076 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7077 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7078 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7079 7080 pte_l1_ss_proto = L1_SS_PROTO_generic; 7081 pte_l1_s_proto = L1_S_PROTO_generic; 7082 pte_l1_c_proto = L1_C_PROTO_generic; 7083 pte_l2_s_proto = L2_S_PROTO_generic; 7084 7085 pmap_copy_page_func = pmap_copy_page_generic; 7086 pmap_zero_page_func = pmap_zero_page_generic; 7087 } 7088 7089 #if defined(CPU_ARM8) 7090 void 7091 pmap_pte_init_arm8(void) 7092 { 7093 7094 /* 7095 * ARM8 is compatible with generic, but we need to use 7096 * the page tables uncached. 7097 */ 7098 pmap_pte_init_generic(); 7099 7100 pte_l1_s_cache_mode_pt = 0; 7101 pte_l2_l_cache_mode_pt = 0; 7102 pte_l2_s_cache_mode_pt = 0; 7103 } 7104 #endif /* CPU_ARM8 */ 7105 7106 #if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 7107 void 7108 pmap_pte_init_arm9(void) 7109 { 7110 7111 /* 7112 * ARM9 is compatible with generic, but we want to use 7113 * write-through caching for now. 7114 */ 7115 pmap_pte_init_generic(); 7116 7117 pte_l1_s_cache_mode = L1_S_C; 7118 pte_l2_l_cache_mode = L2_C; 7119 pte_l2_s_cache_mode = L2_C; 7120 7121 pte_l1_s_wc_mode = L1_S_B; 7122 pte_l2_l_wc_mode = L2_B; 7123 pte_l2_s_wc_mode = L2_B; 7124 7125 pte_l1_s_cache_mode_pt = L1_S_C; 7126 pte_l2_l_cache_mode_pt = L2_C; 7127 pte_l2_s_cache_mode_pt = L2_C; 7128 } 7129 #endif /* CPU_ARM9 && ARM9_CACHE_WRITE_THROUGH */ 7130 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */ 7131 7132 #if defined(CPU_ARM10) 7133 void 7134 pmap_pte_init_arm10(void) 7135 { 7136 7137 /* 7138 * ARM10 is compatible with generic, but we want to use 7139 * write-through caching for now. 7140 */ 7141 pmap_pte_init_generic(); 7142 7143 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 7144 pte_l2_l_cache_mode = L2_B | L2_C; 7145 pte_l2_s_cache_mode = L2_B | L2_C; 7146 7147 pte_l1_s_cache_mode = L1_S_B; 7148 pte_l2_l_cache_mode = L2_B; 7149 pte_l2_s_cache_mode = L2_B; 7150 7151 pte_l1_s_cache_mode_pt = L1_S_C; 7152 pte_l2_l_cache_mode_pt = L2_C; 7153 pte_l2_s_cache_mode_pt = L2_C; 7154 7155 } 7156 #endif /* CPU_ARM10 */ 7157 7158 #if defined(CPU_ARM11) && defined(ARM11_CACHE_WRITE_THROUGH) 7159 void 7160 pmap_pte_init_arm11(void) 7161 { 7162 7163 /* 7164 * ARM11 is compatible with generic, but we want to use 7165 * write-through caching for now. 7166 */ 7167 pmap_pte_init_generic(); 7168 7169 pte_l1_s_cache_mode = L1_S_C; 7170 pte_l2_l_cache_mode = L2_C; 7171 pte_l2_s_cache_mode = L2_C; 7172 7173 pte_l1_s_wc_mode = L1_S_B; 7174 pte_l2_l_wc_mode = L2_B; 7175 pte_l2_s_wc_mode = L2_B; 7176 7177 pte_l1_s_cache_mode_pt = L1_S_C; 7178 pte_l2_l_cache_mode_pt = L2_C; 7179 pte_l2_s_cache_mode_pt = L2_C; 7180 } 7181 #endif /* CPU_ARM11 && ARM11_CACHE_WRITE_THROUGH */ 7182 7183 #if ARM_MMU_SA1 == 1 7184 void 7185 pmap_pte_init_sa1(void) 7186 { 7187 7188 /* 7189 * The StrongARM SA-1 cache does not have a write-through 7190 * mode. So, do the generic initialization, then reset 7191 * the page table cache mode to B=1,C=1, and note that 7192 * the PTEs need to be sync'd. 7193 */ 7194 pmap_pte_init_generic(); 7195 7196 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 7197 pte_l2_l_cache_mode_pt = L2_B|L2_C; 7198 pte_l2_s_cache_mode_pt = L2_B|L2_C; 7199 7200 pmap_needs_pte_sync = 1; 7201 } 7202 #endif /* ARM_MMU_SA1 == 1*/ 7203 7204 #if ARM_MMU_XSCALE == 1 7205 #if (ARM_NMMUS > 1) 7206 static u_int xscale_use_minidata; 7207 #endif 7208 7209 void 7210 pmap_pte_init_xscale(void) 7211 { 7212 uint32_t auxctl; 7213 int write_through = 0; 7214 7215 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 7216 pte_l1_s_wc_mode = L1_S_B; 7217 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 7218 7219 pte_l2_l_cache_mode = L2_B|L2_C; 7220 pte_l2_l_wc_mode = L2_B; 7221 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 7222 7223 pte_l2_s_cache_mode = L2_B|L2_C; 7224 pte_l2_s_wc_mode = L2_B; 7225 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 7226 7227 pte_l1_s_cache_mode_pt = L1_S_C; 7228 pte_l2_l_cache_mode_pt = L2_C; 7229 pte_l2_s_cache_mode_pt = L2_C; 7230 7231 #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 7232 /* 7233 * The XScale core has an enhanced mode where writes that 7234 * miss the cache cause a cache line to be allocated. This 7235 * is significantly faster than the traditional, write-through 7236 * behavior of this case. 7237 */ 7238 pte_l1_s_cache_mode |= L1_S_XS_TEX(TEX_XSCALE_X); 7239 pte_l2_l_cache_mode |= L2_XS_L_TEX(TEX_XSCALE_X); 7240 pte_l2_s_cache_mode |= L2_XS_T_TEX(TEX_XSCALE_X); 7241 #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 7242 7243 #ifdef XSCALE_CACHE_WRITE_THROUGH 7244 /* 7245 * Some versions of the XScale core have various bugs in 7246 * their cache units, the work-around for which is to run 7247 * the cache in write-through mode. Unfortunately, this 7248 * has a major (negative) impact on performance. So, we 7249 * go ahead and run fast-and-loose, in the hopes that we 7250 * don't line up the planets in a way that will trip the 7251 * bugs. 7252 * 7253 * However, we give you the option to be slow-but-correct. 7254 */ 7255 write_through = 1; 7256 #elif defined(XSCALE_CACHE_WRITE_BACK) 7257 /* force write back cache mode */ 7258 write_through = 0; 7259 #elif defined(CPU_XSCALE_PXA250) || defined(CPU_XSCALE_PXA270) 7260 /* 7261 * Intel PXA2[15]0 processors are known to have a bug in 7262 * write-back cache on revision 4 and earlier (stepping 7263 * A[01] and B[012]). Fixed for C0 and later. 7264 */ 7265 { 7266 uint32_t id, type; 7267 7268 id = cpufunc_id(); 7269 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 7270 7271 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 7272 if ((id & CPU_ID_REVISION_MASK) < 5) { 7273 /* write through for stepping A0-1 and B0-2 */ 7274 write_through = 1; 7275 } 7276 } 7277 } 7278 #endif /* XSCALE_CACHE_WRITE_THROUGH */ 7279 7280 if (write_through) { 7281 pte_l1_s_cache_mode = L1_S_C; 7282 pte_l2_l_cache_mode = L2_C; 7283 pte_l2_s_cache_mode = L2_C; 7284 } 7285 7286 #if (ARM_NMMUS > 1) 7287 xscale_use_minidata = 1; 7288 #endif 7289 7290 pte_l1_s_prot_u = L1_S_PROT_U_xscale; 7291 pte_l1_s_prot_w = L1_S_PROT_W_xscale; 7292 pte_l1_s_prot_ro = L1_S_PROT_RO_xscale; 7293 pte_l1_s_prot_mask = L1_S_PROT_MASK_xscale; 7294 7295 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 7296 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 7297 pte_l2_s_prot_ro = L2_S_PROT_RO_xscale; 7298 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 7299 7300 pte_l2_l_prot_u = L2_L_PROT_U_xscale; 7301 pte_l2_l_prot_w = L2_L_PROT_W_xscale; 7302 pte_l2_l_prot_ro = L2_L_PROT_RO_xscale; 7303 pte_l2_l_prot_mask = L2_L_PROT_MASK_xscale; 7304 7305 pte_l1_ss_proto = L1_SS_PROTO_xscale; 7306 pte_l1_s_proto = L1_S_PROTO_xscale; 7307 pte_l1_c_proto = L1_C_PROTO_xscale; 7308 pte_l2_s_proto = L2_S_PROTO_xscale; 7309 7310 pmap_copy_page_func = pmap_copy_page_xscale; 7311 pmap_zero_page_func = pmap_zero_page_xscale; 7312 7313 /* 7314 * Disable ECC protection of page table access, for now. 7315 */ 7316 auxctl = armreg_auxctl_read(); 7317 auxctl &= ~XSCALE_AUXCTL_P; 7318 armreg_auxctl_write(auxctl); 7319 } 7320 7321 /* 7322 * xscale_setup_minidata: 7323 * 7324 * Set up the mini-data cache clean area. We require the 7325 * caller to allocate the right amount of physically and 7326 * virtually contiguous space. 7327 */ 7328 void 7329 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 7330 { 7331 extern vaddr_t xscale_minidata_clean_addr; 7332 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 7333 pd_entry_t *pde = (pd_entry_t *) l1pt; 7334 vsize_t size; 7335 uint32_t auxctl; 7336 7337 xscale_minidata_clean_addr = va; 7338 7339 /* Round it to page size. */ 7340 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 7341 7342 for (; size != 0; 7343 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 7344 const size_t l1slot = l1pte_index(va); 7345 pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(l1pte_pa(pde[l1slot])); 7346 if (ptep == NULL) 7347 panic("xscale_setup_minidata: can't find L2 table for " 7348 "VA 0x%08lx", va); 7349 7350 ptep += l2pte_index(va); 7351 pt_entry_t opte = *ptep; 7352 l2pte_set(ptep, 7353 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) 7354 | L2_C | L2_XS_T_TEX(TEX_XSCALE_X), opte); 7355 } 7356 7357 /* 7358 * Configure the mini-data cache for write-back with 7359 * read/write-allocate. 7360 * 7361 * NOTE: In order to reconfigure the mini-data cache, we must 7362 * make sure it contains no valid data! In order to do that, 7363 * we must issue a global data cache invalidate command! 7364 * 7365 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 7366 * THIS IS VERY IMPORTANT! 7367 */ 7368 7369 /* Invalidate data and mini-data. */ 7370 __asm volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 7371 auxctl = armreg_auxctl_read(); 7372 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 7373 armreg_auxctl_write(auxctl); 7374 } 7375 7376 /* 7377 * Change the PTEs for the specified kernel mappings such that they 7378 * will use the mini data cache instead of the main data cache. 7379 */ 7380 void 7381 pmap_uarea(vaddr_t va) 7382 { 7383 vaddr_t next_bucket, eva; 7384 7385 #if (ARM_NMMUS > 1) 7386 if (xscale_use_minidata == 0) 7387 return; 7388 #endif 7389 7390 eva = va + USPACE; 7391 7392 while (va < eva) { 7393 next_bucket = L2_NEXT_BUCKET_VA(va); 7394 if (next_bucket > eva) 7395 next_bucket = eva; 7396 7397 struct l2_bucket *l2b = pmap_get_l2_bucket(pmap_kernel(), va); 7398 KDASSERT(l2b != NULL); 7399 7400 pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)]; 7401 pt_entry_t *ptep = sptep; 7402 7403 while (va < next_bucket) { 7404 const pt_entry_t opte = *ptep; 7405 if (!l2pte_minidata_p(opte)) { 7406 cpu_dcache_wbinv_range(va, PAGE_SIZE); 7407 cpu_tlb_flushD_SE(va); 7408 l2pte_set(ptep, opte & ~L2_B, opte); 7409 } 7410 ptep += PAGE_SIZE / L2_S_SIZE; 7411 va += PAGE_SIZE; 7412 } 7413 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 7414 } 7415 cpu_cpwait(); 7416 } 7417 #endif /* ARM_MMU_XSCALE == 1 */ 7418 7419 7420 #if defined(CPU_ARM11MPCORE) 7421 7422 void 7423 pmap_pte_init_arm11mpcore(void) 7424 { 7425 7426 /* cache mode is controlled by 5 bits (B, C, TEX) */ 7427 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv6; 7428 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv6; 7429 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7430 /* use extended small page (without APn, with TEX) */ 7431 pte_l2_s_cache_mask = L2_XS_CACHE_MASK_armv6; 7432 #else 7433 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv6c; 7434 #endif 7435 7436 /* write-back, write-allocate */ 7437 pte_l1_s_cache_mode = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7438 pte_l2_l_cache_mode = L2_C | L2_B | L2_V6_L_TEX(0x01); 7439 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7440 pte_l2_s_cache_mode = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7441 #else 7442 /* no TEX. read-allocate */ 7443 pte_l2_s_cache_mode = L2_C | L2_B; 7444 #endif 7445 /* 7446 * write-back, write-allocate for page tables. 7447 */ 7448 pte_l1_s_cache_mode_pt = L1_S_C | L1_S_B | L1_S_V6_TEX(0x01); 7449 pte_l2_l_cache_mode_pt = L2_C | L2_B | L2_V6_L_TEX(0x01); 7450 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7451 pte_l2_s_cache_mode_pt = L2_C | L2_B | L2_V6_XS_TEX(0x01); 7452 #else 7453 pte_l2_s_cache_mode_pt = L2_C | L2_B; 7454 #endif 7455 7456 pte_l1_s_prot_u = L1_S_PROT_U_armv6; 7457 pte_l1_s_prot_w = L1_S_PROT_W_armv6; 7458 pte_l1_s_prot_ro = L1_S_PROT_RO_armv6; 7459 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv6; 7460 7461 #if defined(ARM11MPCORE_COMPAT_MMU) || defined(ARMV6_EXTENDED_SMALL_PAGE) 7462 pte_l2_s_prot_u = L2_S_PROT_U_armv6n; 7463 pte_l2_s_prot_w = L2_S_PROT_W_armv6n; 7464 pte_l2_s_prot_ro = L2_S_PROT_RO_armv6n; 7465 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv6n; 7466 7467 #else 7468 /* with AP[0..3] */ 7469 pte_l2_s_prot_u = L2_S_PROT_U_generic; 7470 pte_l2_s_prot_w = L2_S_PROT_W_generic; 7471 pte_l2_s_prot_ro = L2_S_PROT_RO_generic; 7472 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 7473 #endif 7474 7475 #ifdef ARM11MPCORE_COMPAT_MMU 7476 /* with AP[0..3] */ 7477 pte_l2_l_prot_u = L2_L_PROT_U_generic; 7478 pte_l2_l_prot_w = L2_L_PROT_W_generic; 7479 pte_l2_l_prot_ro = L2_L_PROT_RO_generic; 7480 pte_l2_l_prot_mask = L2_L_PROT_MASK_generic; 7481 7482 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7483 pte_l1_s_proto = L1_S_PROTO_armv6; 7484 pte_l1_c_proto = L1_C_PROTO_armv6; 7485 pte_l2_s_proto = L2_S_PROTO_armv6c; 7486 #else 7487 pte_l2_l_prot_u = L2_L_PROT_U_armv6n; 7488 pte_l2_l_prot_w = L2_L_PROT_W_armv6n; 7489 pte_l2_l_prot_ro = L2_L_PROT_RO_armv6n; 7490 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv6n; 7491 7492 pte_l1_ss_proto = L1_SS_PROTO_armv6; 7493 pte_l1_s_proto = L1_S_PROTO_armv6; 7494 pte_l1_c_proto = L1_C_PROTO_armv6; 7495 pte_l2_s_proto = L2_S_PROTO_armv6n; 7496 #endif 7497 7498 pmap_copy_page_func = pmap_copy_page_generic; 7499 pmap_zero_page_func = pmap_zero_page_generic; 7500 pmap_needs_pte_sync = 1; 7501 } 7502 #endif /* CPU_ARM11MPCORE */ 7503 7504 7505 #if ARM_MMU_V7 == 1 7506 void 7507 pmap_pte_init_armv7(void) 7508 { 7509 /* 7510 * The ARMv7-A MMU is mostly compatible with generic. If the 7511 * AP field is zero, that now means "no access" rather than 7512 * read-only. The prototypes are a little different because of 7513 * the XN bit. 7514 */ 7515 pmap_pte_init_generic(); 7516 7517 pmap_needs_pte_sync = 1; 7518 7519 pte_l1_s_cache_mask = L1_S_CACHE_MASK_armv7; 7520 pte_l2_l_cache_mask = L2_L_CACHE_MASK_armv7; 7521 pte_l2_s_cache_mask = L2_S_CACHE_MASK_armv7; 7522 7523 /* 7524 * If the core support coherent walk then updates to translation tables 7525 * do not require a clean to the point of unification to ensure 7526 * visibility by subsequent translation table walks. That means we can 7527 * map everything shareable and cached and the right thing will happen. 7528 */ 7529 if (__SHIFTOUT(armreg_mmfr3_read(), __BITS(23,20))) { 7530 pmap_needs_pte_sync = 0; 7531 7532 /* 7533 * write-back, no write-allocate, shareable for normal pages. 7534 */ 7535 pte_l1_s_cache_mode |= L1_S_V6_S; 7536 pte_l2_l_cache_mode |= L2_XS_S; 7537 pte_l2_s_cache_mode |= L2_XS_S; 7538 } 7539 7540 /* 7541 * Page tables are just all other memory. We can use write-back since 7542 * pmap_needs_pte_sync is 1 (or the MMU can read out of cache). 7543 */ 7544 pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode; 7545 pte_l2_l_cache_mode_pt = pte_l2_l_cache_mode; 7546 pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; 7547 7548 /* 7549 * Check the Memory Model Features to see if this CPU supports 7550 * the TLBIASID coproc op. 7551 */ 7552 if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(16,19)) >= 2) { 7553 arm_has_tlbiasid_p = true; 7554 } else if (__SHIFTOUT(armreg_mmfr2_read(), __BITS(12,15)) >= 2) { 7555 arm_has_tlbiasid_p = true; 7556 } 7557 7558 /* 7559 * Check the MPIDR to see if this CPU supports MP extensions. 7560 */ 7561 #ifdef MULTIPROCESSOR 7562 arm_has_mpext_p = (armreg_mpidr_read() & (MPIDR_MP|MPIDR_U)) == MPIDR_MP; 7563 #else 7564 arm_has_mpext_p = false; 7565 #endif 7566 7567 pte_l1_s_prot_u = L1_S_PROT_U_armv7; 7568 pte_l1_s_prot_w = L1_S_PROT_W_armv7; 7569 pte_l1_s_prot_ro = L1_S_PROT_RO_armv7; 7570 pte_l1_s_prot_mask = L1_S_PROT_MASK_armv7; 7571 7572 pte_l2_s_prot_u = L2_S_PROT_U_armv7; 7573 pte_l2_s_prot_w = L2_S_PROT_W_armv7; 7574 pte_l2_s_prot_ro = L2_S_PROT_RO_armv7; 7575 pte_l2_s_prot_mask = L2_S_PROT_MASK_armv7; 7576 7577 pte_l2_l_prot_u = L2_L_PROT_U_armv7; 7578 pte_l2_l_prot_w = L2_L_PROT_W_armv7; 7579 pte_l2_l_prot_ro = L2_L_PROT_RO_armv7; 7580 pte_l2_l_prot_mask = L2_L_PROT_MASK_armv7; 7581 7582 pte_l1_ss_proto = L1_SS_PROTO_armv7; 7583 pte_l1_s_proto = L1_S_PROTO_armv7; 7584 pte_l1_c_proto = L1_C_PROTO_armv7; 7585 pte_l2_s_proto = L2_S_PROTO_armv7; 7586 7587 } 7588 #endif /* ARM_MMU_V7 */ 7589 7590 /* 7591 * return the PA of the current L1 table, for use when handling a crash dump 7592 */ 7593 uint32_t 7594 pmap_kernel_L1_addr(void) 7595 { 7596 #ifdef ARM_MMU_EXTENDED 7597 return pmap_kernel()->pm_l1_pa; 7598 #else 7599 return pmap_kernel()->pm_l1->l1_physaddr; 7600 #endif 7601 } 7602 7603 #if defined(DDB) 7604 /* 7605 * A couple of ddb-callable functions for dumping pmaps 7606 */ 7607 void pmap_dump(pmap_t); 7608 7609 static pt_entry_t ncptes[64]; 7610 static void pmap_dump_ncpg(pmap_t); 7611 7612 void 7613 pmap_dump(pmap_t pm) 7614 { 7615 struct l2_dtable *l2; 7616 struct l2_bucket *l2b; 7617 pt_entry_t *ptep, pte; 7618 vaddr_t l2_va, l2b_va, va; 7619 int i, j, k, occ, rows = 0; 7620 7621 if (pm == pmap_kernel()) 7622 printf("pmap_kernel (%p): ", pm); 7623 else 7624 printf("user pmap (%p): ", pm); 7625 7626 #ifdef ARM_MMU_EXTENDED 7627 printf("l1 at %p\n", pmap_l1_kva(pm)); 7628 #else 7629 printf("domain %d, l1 at %p\n", pmap_domain(pm), pmap_l1_kva(pm)); 7630 #endif 7631 7632 l2_va = 0; 7633 for (i = 0; i < L2_SIZE; i++, l2_va += 0x01000000) { 7634 l2 = pm->pm_l2[i]; 7635 7636 if (l2 == NULL || l2->l2_occupancy == 0) 7637 continue; 7638 7639 l2b_va = l2_va; 7640 for (j = 0; j < L2_BUCKET_SIZE; j++, l2b_va += 0x00100000) { 7641 l2b = &l2->l2_bucket[j]; 7642 7643 if (l2b->l2b_occupancy == 0 || l2b->l2b_kva == NULL) 7644 continue; 7645 7646 ptep = l2b->l2b_kva; 7647 7648 for (k = 0; k < 256 && ptep[k] == 0; k++) 7649 ; 7650 7651 k &= ~63; 7652 occ = l2b->l2b_occupancy; 7653 va = l2b_va + (k * 4096); 7654 for (; k < 256; k++, va += 0x1000) { 7655 char ch = ' '; 7656 if ((k % 64) == 0) { 7657 if ((rows % 8) == 0) { 7658 printf( 7659 " |0000 |8000 |10000 |18000 |20000 |28000 |30000 |38000\n"); 7660 } 7661 printf("%08lx: ", va); 7662 } 7663 7664 ncptes[k & 63] = 0; 7665 pte = ptep[k]; 7666 if (pte == 0) { 7667 ch = '.'; 7668 } else { 7669 occ--; 7670 switch (pte & 0x0c) { 7671 case 0x00: 7672 ch = 'D'; /* No cache No buff */ 7673 break; 7674 case 0x04: 7675 ch = 'B'; /* No cache buff */ 7676 break; 7677 case 0x08: 7678 if (pte & 0x40) 7679 ch = 'm'; 7680 else 7681 ch = 'C'; /* Cache No buff */ 7682 break; 7683 case 0x0c: 7684 ch = 'F'; /* Cache Buff */ 7685 break; 7686 } 7687 7688 if ((pte & L2_S_PROT_U) == L2_S_PROT_U) 7689 ch += 0x20; 7690 7691 if ((pte & 0xc) == 0) 7692 ncptes[k & 63] = pte; 7693 } 7694 7695 if ((k % 64) == 63) { 7696 rows++; 7697 printf("%c\n", ch); 7698 pmap_dump_ncpg(pm); 7699 if (occ == 0) 7700 break; 7701 } else 7702 printf("%c", ch); 7703 } 7704 } 7705 } 7706 } 7707 7708 static void 7709 pmap_dump_ncpg(pmap_t pm) 7710 { 7711 struct vm_page *pg; 7712 struct vm_page_md *md; 7713 struct pv_entry *pv; 7714 int i; 7715 7716 for (i = 0; i < 63; i++) { 7717 if (ncptes[i] == 0) 7718 continue; 7719 7720 pg = PHYS_TO_VM_PAGE(l2pte_pa(ncptes[i])); 7721 if (pg == NULL) 7722 continue; 7723 md = VM_PAGE_TO_MD(pg); 7724 7725 printf(" pa 0x%08lx: krw %d kro %d urw %d uro %d\n", 7726 VM_PAGE_TO_PHYS(pg), 7727 md->krw_mappings, md->kro_mappings, 7728 md->urw_mappings, md->uro_mappings); 7729 7730 SLIST_FOREACH(pv, &md->pvh_list, pv_link) { 7731 printf(" %c va 0x%08lx, flags 0x%x\n", 7732 (pm == pv->pv_pmap) ? '*' : ' ', 7733 pv->pv_va, pv->pv_flags); 7734 } 7735 } 7736 } 7737 #endif 7738 7739 #ifdef PMAP_STEAL_MEMORY 7740 void 7741 pmap_boot_pageadd(pv_addr_t *newpv) 7742 { 7743 pv_addr_t *pv, *npv; 7744 7745 if ((pv = SLIST_FIRST(&pmap_boot_freeq)) != NULL) { 7746 if (newpv->pv_pa < pv->pv_va) { 7747 KASSERT(newpv->pv_pa + newpv->pv_size <= pv->pv_pa); 7748 if (newpv->pv_pa + newpv->pv_size == pv->pv_pa) { 7749 newpv->pv_size += pv->pv_size; 7750 SLIST_REMOVE_HEAD(&pmap_boot_freeq, pv_list); 7751 } 7752 pv = NULL; 7753 } else { 7754 for (; (npv = SLIST_NEXT(pv, pv_list)) != NULL; 7755 pv = npv) { 7756 KASSERT(pv->pv_pa + pv->pv_size < npv->pv_pa); 7757 KASSERT(pv->pv_pa < newpv->pv_pa); 7758 if (newpv->pv_pa > npv->pv_pa) 7759 continue; 7760 if (pv->pv_pa + pv->pv_size == newpv->pv_pa) { 7761 pv->pv_size += newpv->pv_size; 7762 return; 7763 } 7764 if (newpv->pv_pa + newpv->pv_size < npv->pv_pa) 7765 break; 7766 newpv->pv_size += npv->pv_size; 7767 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7768 SLIST_REMOVE_AFTER(newpv, pv_list); 7769 return; 7770 } 7771 } 7772 } 7773 7774 if (pv) { 7775 SLIST_INSERT_AFTER(pv, newpv, pv_list); 7776 } else { 7777 SLIST_INSERT_HEAD(&pmap_boot_freeq, newpv, pv_list); 7778 } 7779 } 7780 7781 void 7782 pmap_boot_pagealloc(psize_t amount, psize_t mask, psize_t match, 7783 pv_addr_t *rpv) 7784 { 7785 pv_addr_t *pv, **pvp; 7786 struct vm_physseg *ps; 7787 size_t i; 7788 7789 KASSERT(amount & PGOFSET); 7790 KASSERT((mask & PGOFSET) == 0); 7791 KASSERT((match & PGOFSET) == 0); 7792 KASSERT(amount != 0); 7793 7794 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7795 (pv = *pvp) != NULL; 7796 pvp = &SLIST_NEXT(pv, pv_list)) { 7797 pv_addr_t *newpv; 7798 psize_t off; 7799 /* 7800 * If this entry is too small to satify the request... 7801 */ 7802 KASSERT(pv->pv_size > 0); 7803 if (pv->pv_size < amount) 7804 continue; 7805 7806 for (off = 0; off <= mask; off += PAGE_SIZE) { 7807 if (((pv->pv_pa + off) & mask) == match 7808 && off + amount <= pv->pv_size) 7809 break; 7810 } 7811 if (off > mask) 7812 continue; 7813 7814 rpv->pv_va = pv->pv_va + off; 7815 rpv->pv_pa = pv->pv_pa + off; 7816 rpv->pv_size = amount; 7817 pv->pv_size -= amount; 7818 if (pv->pv_size == 0) { 7819 KASSERT(off == 0); 7820 KASSERT((vaddr_t) pv == rpv->pv_va); 7821 *pvp = SLIST_NEXT(pv, pv_list); 7822 } else if (off == 0) { 7823 KASSERT((vaddr_t) pv == rpv->pv_va); 7824 newpv = (pv_addr_t *) (rpv->pv_va + amount); 7825 *newpv = *pv; 7826 newpv->pv_pa += amount; 7827 newpv->pv_va += amount; 7828 *pvp = newpv; 7829 } else if (off < pv->pv_size) { 7830 newpv = (pv_addr_t *) (rpv->pv_va + amount); 7831 *newpv = *pv; 7832 newpv->pv_size -= off; 7833 newpv->pv_pa += off + amount; 7834 newpv->pv_va += off + amount; 7835 7836 SLIST_NEXT(pv, pv_list) = newpv; 7837 pv->pv_size = off; 7838 } else { 7839 KASSERT((vaddr_t) pv != rpv->pv_va); 7840 } 7841 memset((void *)rpv->pv_va, 0, amount); 7842 return; 7843 } 7844 7845 if (vm_nphysseg == 0) 7846 panic("pmap_boot_pagealloc: couldn't allocate memory"); 7847 7848 for (pvp = &SLIST_FIRST(&pmap_boot_freeq); 7849 (pv = *pvp) != NULL; 7850 pvp = &SLIST_NEXT(pv, pv_list)) { 7851 if (SLIST_NEXT(pv, pv_list) == NULL) 7852 break; 7853 } 7854 KASSERT(mask == 0); 7855 for (i = 0; i < vm_nphysseg; i++) { 7856 ps = VM_PHYSMEM_PTR(i); 7857 if (ps->avail_start == atop(pv->pv_pa + pv->pv_size) 7858 && pv->pv_va + pv->pv_size <= ptoa(ps->avail_end)) { 7859 rpv->pv_va = pv->pv_va; 7860 rpv->pv_pa = pv->pv_pa; 7861 rpv->pv_size = amount; 7862 *pvp = NULL; 7863 pmap_map_chunk(kernel_l1pt.pv_va, 7864 ptoa(ps->avail_start) + (pv->pv_va - pv->pv_pa), 7865 ptoa(ps->avail_start), 7866 amount - pv->pv_size, 7867 VM_PROT_READ|VM_PROT_WRITE, 7868 PTE_CACHE); 7869 ps->avail_start += atop(amount - pv->pv_size); 7870 /* 7871 * If we consumed the entire physseg, remove it. 7872 */ 7873 if (ps->avail_start == ps->avail_end) { 7874 for (--vm_nphysseg; i < vm_nphysseg; i++) 7875 VM_PHYSMEM_PTR_SWAP(i, i + 1); 7876 } 7877 memset((void *)rpv->pv_va, 0, rpv->pv_size); 7878 return; 7879 } 7880 } 7881 7882 panic("pmap_boot_pagealloc: couldn't allocate memory"); 7883 } 7884 7885 vaddr_t 7886 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 7887 { 7888 pv_addr_t pv; 7889 7890 pmap_boot_pagealloc(size, 0, 0, &pv); 7891 7892 return pv.pv_va; 7893 } 7894 #endif /* PMAP_STEAL_MEMORY */ 7895 7896 SYSCTL_SETUP(sysctl_machdep_pmap_setup, "sysctl machdep.kmpages setup") 7897 { 7898 sysctl_createv(clog, 0, NULL, NULL, 7899 CTLFLAG_PERMANENT, 7900 CTLTYPE_NODE, "machdep", NULL, 7901 NULL, 0, NULL, 0, 7902 CTL_MACHDEP, CTL_EOL); 7903 7904 sysctl_createv(clog, 0, NULL, NULL, 7905 CTLFLAG_PERMANENT, 7906 CTLTYPE_INT, "kmpages", 7907 SYSCTL_DESCR("count of pages allocated to kernel memory allocators"), 7908 NULL, 0, &pmap_kmpages, 0, 7909 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 7910 } 7911 7912 #ifdef PMAP_NEED_ALLOC_POOLPAGE 7913 struct vm_page * 7914 arm_pmap_alloc_poolpage(int flags) 7915 { 7916 /* 7917 * On some systems, only some pages may be "coherent" for dma and we 7918 * want to prefer those for pool pages (think mbufs) but fallback to 7919 * any page if none is available. 7920 */ 7921 if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) { 7922 return uvm_pagealloc_strat(NULL, 0, NULL, flags, 7923 UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist); 7924 } 7925 7926 return uvm_pagealloc(NULL, 0, NULL, flags); 7927 } 7928 #endif 7929 7930 #if defined(ARM_MMU_EXTENDED) && defined(MULTIPROCESSOR) 7931 void 7932 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 7933 { 7934 /* nothing */ 7935 } 7936 7937 int 7938 pic_ipi_shootdown(void *arg) 7939 { 7940 #if PMAP_TLB_NEED_SHOOTDOWN 7941 pmap_tlb_shootdown_process(); 7942 #endif 7943 return 1; 7944 } 7945 #endif /* ARM_MMU_EXTENDED && MULTIPROCESSOR */ 7946 7947 7948 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 7949 vaddr_t 7950 pmap_direct_mapped_phys(paddr_t pa, bool *ok_p, vaddr_t va) 7951 { 7952 bool ok = false; 7953 if (physical_start <= pa && pa < physical_end) { 7954 #ifdef KERNEL_BASE_VOFFSET 7955 const vaddr_t newva = pa + KERNEL_BASE_VOFFSET; 7956 #else 7957 const vaddr_t newva = KERNEL_BASE + pa - physical_start; 7958 #endif 7959 #ifdef ARM_MMU_EXTENDED 7960 if (newva >= KERNEL_BASE && newva < pmap_directlimit) { 7961 #endif 7962 va = newva; 7963 ok = true; 7964 #ifdef ARM_MMU_EXTENDED 7965 } 7966 #endif 7967 } 7968 KASSERT(ok_p); 7969 *ok_p = ok; 7970 return va; 7971 } 7972 7973 vaddr_t 7974 pmap_map_poolpage(paddr_t pa) 7975 { 7976 bool ok __diagused; 7977 vaddr_t va = pmap_direct_mapped_phys(pa, &ok, 0); 7978 KASSERTMSG(ok, "pa %#lx not direct mappable", pa); 7979 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED) 7980 if (arm_cache_prefer_mask != 0) { 7981 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 7982 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 7983 pmap_acquire_page_lock(md); 7984 pmap_vac_me_harder(md, pa, pmap_kernel(), va); 7985 pmap_release_page_lock(md); 7986 } 7987 #endif 7988 return va; 7989 } 7990 7991 paddr_t 7992 pmap_unmap_poolpage(vaddr_t va) 7993 { 7994 KASSERT(va >= KERNEL_BASE); 7995 #ifdef PMAP_CACHE_VIVT 7996 cpu_idcache_wbinv_range(va, PAGE_SIZE); 7997 #endif 7998 #if defined(KERNEL_BASE_VOFFSET) 7999 return va - KERNEL_BASE_VOFFSET; 8000 #else 8001 return va - KERNEL_BASE + physical_start; 8002 #endif 8003 } 8004 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 8005
Indexes created Mon Sep 29 15:09:53 GMT 2025