Home | History | Annotate | Line # | Download | only in arm32
pmap.c revision 1.297
      1 /*	$NetBSD: pmap.c,v 1.296 2014/08/13 05:56:03 matt Exp $	*/
      2 
      3 /*
      4  * Copyright 2003 Wasabi Systems, Inc.
      5  * All rights reserved.
      6  *
      7  * Written by Steve C. Woodford for Wasabi Systems, Inc.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  * 3. All advertising materials mentioning features or use of this software
     18  *    must display the following acknowledgement:
     19  *      This product includes software developed for the NetBSD Project by
     20  *      Wasabi Systems, Inc.
     21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
     22  *    or promote products derived from this software without specific prior
     23  *    written permission.
     24  *
     25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
     26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
     29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     35  * POSSIBILITY OF SUCH DAMAGE.
     36  */
     37 
     38 /*
     39  * Copyright (c) 2002-2003 Wasabi Systems, Inc.
     40  * Copyright (c) 2001 Richard Earnshaw
     41  * Copyright (c) 2001-2002 Christopher Gilbert
     42  * All rights reserved.
     43  *
     44  * 1. Redistributions of source code must retain the above copyright
     45  *    notice, this list of conditions and the following disclaimer.
     46  * 2. Redistributions in binary form must reproduce the above copyright
     47  *    notice, this list of conditions and the following disclaimer in the
     48  *    documentation and/or other materials provided with the distribution.
     49  * 3. The name of the company nor the name of the author may be used to
     50  *    endorse or promote products derived from this software without specific
     51  *    prior written permission.
     52  *
     53  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
     54  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     55  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     56  * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
     57  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     58  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     59  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     63  * SUCH DAMAGE.
     64  */
     65 
     66 /*-
     67  * Copyright (c) 1999 The NetBSD Foundation, Inc.
     68  * All rights reserved.
     69  *
     70  * This code is derived from software contributed to The NetBSD Foundation
     71  * by Charles M. Hannum.
     72  *
     73  * Redistribution and use in source and binary forms, with or without
     74  * modification, are permitted provided that the following conditions
     75  * are met:
     76  * 1. Redistributions of source code must retain the above copyright
     77  *    notice, this list of conditions and the following disclaimer.
     78  * 2. Redistributions in binary form must reproduce the above copyright
     79  *    notice, this list of conditions and the following disclaimer in the
     80  *    documentation and/or other materials provided with the distribution.
     81  *
     82  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     83  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     84  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     85  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     86  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     87  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     88  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     89  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     90  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     91  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     92  * POSSIBILITY OF SUCH DAMAGE.
     93  */
     94 
     95 /*
     96  * Copyright (c) 1994-1998 Mark Brinicombe.
     97  * Copyright (c) 1994 Brini.
     98  * All rights reserved.
     99  *
    100  * This code is derived from software written for Brini by Mark Brinicombe
    101  *
    102  * Redistribution and use in source and binary forms, with or without
    103  * modification, are permitted provided that the following conditions
    104  * are met:
    105  * 1. Redistributions of source code must retain the above copyright
    106  *    notice, this list of conditions and the following disclaimer.
    107  * 2. Redistributions in binary form must reproduce the above copyright
    108  *    notice, this list of conditions and the following disclaimer in the
    109  *    documentation and/or other materials provided with the distribution.
    110  * 3. All advertising materials mentioning features or use of this software
    111  *    must display the following acknowledgement:
    112  *	This product includes software developed by Mark Brinicombe.
    113  * 4. The name of the author may not be used to endorse or promote products
    114  *    derived from this software without specific prior written permission.
    115  *
    116  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
    117  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
    118  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
    119  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
    120  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
    121  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    122  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    123  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    124  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
    125  *
    126  * RiscBSD kernel project
    127  *
    128  * pmap.c
    129  *
    130  * Machine dependent vm stuff
    131  *
    132  * Created      : 20/09/94
    133  */
    134 
    135 /*
    136  * armv6 and VIPT cache support by 3am Software Foundry,
    137  * Copyright (c) 2007 Microsoft
    138  */
    139 
    140 /*
    141  * Performance improvements, UVM changes, overhauls and part-rewrites
    142  * were contributed by Neil A. Carson <neil (at) causality.com>.
    143  */
    144 
    145 /*
    146  * Overhauled again to speedup the pmap, use MMU Domains so that L1 tables
    147  * can be shared, and re-work the KVM layout, by Steve Woodford of Wasabi
    148  * Systems, Inc.
    149  *
    150  * There are still a few things outstanding at this time:
    151  *
    152  *   - There are some unresolved issues for MP systems:
    153  *
    154  *     o The L1 metadata needs a lock, or more specifically, some places
    155  *       need to acquire an exclusive lock when modifying L1 translation
    156  *       table entries.
    157  *
    158  *     o When one cpu modifies an L1 entry, and that L1 table is also
    159  *       being used by another cpu, then the latter will need to be told
    160  *       that a tlb invalidation may be necessary. (But only if the old
    161  *       domain number in the L1 entry being over-written is currently
    162  *       the active domain on that cpu). I guess there are lots more tlb
    163  *       shootdown issues too...
    164  *
    165  *     o If the vector_page is at 0x00000000 instead of in kernel VA space,
    166  *       then MP systems will lose big-time because of the MMU domain hack.
    167  *       The only way this can be solved (apart from moving the vector
    168  *       page to 0xffff0000) is to reserve the first 1MB of user address
    169  *       space for kernel use only. This would require re-linking all
    170  *       applications so that the text section starts above this 1MB
    171  *       boundary.
    172  *
    173  *     o Tracking which VM space is resident in the cache/tlb has not yet
    174  *       been implemented for MP systems.
    175  *
    176  *     o Finally, there is a pathological condition where two cpus running
    177  *       two separate processes (not lwps) which happen to share an L1
    178  *       can get into a fight over one or more L1 entries. This will result
    179  *       in a significant slow-down if both processes are in tight loops.
    180  */
    181 
    182 /*
    183  * Special compilation symbols
    184  * PMAP_DEBUG		- Build in pmap_debug_level code
    185  */
    186 
    187 /* Include header files */
    188 
    189 #include "opt_cpuoptions.h"
    190 #include "opt_pmap_debug.h"
    191 #include "opt_ddb.h"
    192 #include "opt_lockdebug.h"
    193 #include "opt_multiprocessor.h"
    194 
    195 #ifdef MULTIPROCESSOR
    196 #define _INTR_PRIVATE
    197 #endif
    198 
    199 #include <sys/param.h>
    200 #include <sys/types.h>
    201 #include <sys/kernel.h>
    202 #include <sys/systm.h>
    203 #include <sys/proc.h>
    204 #include <sys/intr.h>
    205 #include <sys/pool.h>
    206 #include <sys/kmem.h>
    207 #include <sys/cdefs.h>
    208 #include <sys/cpu.h>
    209 #include <sys/sysctl.h>
    210 #include <sys/bus.h>
    211 #include <sys/atomic.h>
    212 #include <sys/kernhist.h>
    213 
    214 #include <uvm/uvm.h>
    215 
    216 #include <arm/locore.h>
    217 //#include <arm/arm32/katelib.h>
    218 
    219 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.296 2014/08/13 05:56:03 matt Exp $");
    220 
    221 //#define PMAP_DEBUG
    222 #ifdef PMAP_DEBUG
    223 
    224 /* XXX need to get rid of all refs to this */
    225 int pmap_debug_level = 0;
    226 
    227 /*
    228  * for switching to potentially finer grained debugging
    229  */
    230 #define	PDB_FOLLOW	0x0001
    231 #define	PDB_INIT	0x0002
    232 #define	PDB_ENTER	0x0004
    233 #define	PDB_REMOVE	0x0008
    234 #define	PDB_CREATE	0x0010
    235 #define	PDB_PTPAGE	0x0020
    236 #define	PDB_GROWKERN	0x0040
    237 #define	PDB_BITS	0x0080
    238 #define	PDB_COLLECT	0x0100
    239 #define	PDB_PROTECT	0x0200
    240 #define	PDB_MAP_L1	0x0400
    241 #define	PDB_BOOTSTRAP	0x1000
    242 #define	PDB_PARANOIA	0x2000
    243 #define	PDB_WIRING	0x4000
    244 #define	PDB_PVDUMP	0x8000
    245 #define	PDB_VAC		0x10000
    246 #define	PDB_KENTER	0x20000
    247 #define	PDB_KREMOVE	0x40000
    248 #define	PDB_EXEC	0x80000
    249 
    250 int debugmap = 1;
    251 int pmapdebug = 0;
    252 #define	NPDEBUG(_lev_,_stat_) \
    253 	if (pmapdebug & (_lev_)) \
    254         	((_stat_))
    255 
    256 #else	/* PMAP_DEBUG */
    257 #define NPDEBUG(_lev_,_stat_) /* Nothing */
    258 #endif	/* PMAP_DEBUG */
    259 
    260 /*
    261  * pmap_kernel() points here
    262  */
    263 static struct pmap	kernel_pmap_store = {
    264 #ifndef ARM_MMU_EXTENDED
    265 	.pm_activated = true,
    266 	.pm_domain = PMAP_DOMAIN_KERNEL,
    267 	.pm_cstate.cs_all = PMAP_CACHE_STATE_ALL,
    268 #endif
    269 };
    270 struct pmap * const	kernel_pmap_ptr = &kernel_pmap_store;
    271 #undef pmap_kernel
    272 #define pmap_kernel()	(&kernel_pmap_store)
    273 #ifdef PMAP_NEED_ALLOC_POOLPAGE
    274 int			arm_poolpage_vmfreelist = VM_FREELIST_DEFAULT;
    275 #endif
    276 
    277 /*
    278  * Pool and cache that pmap structures are allocated from.
    279  * We use a cache to avoid clearing the pm_l2[] array (1KB)
    280  * in pmap_create().
    281  */
    282 static struct pool_cache pmap_cache;
    283 static LIST_HEAD(, pmap) pmap_pmaps;
    284 
    285 /*
    286  * Pool of PV structures
    287  */
    288 static struct pool pmap_pv_pool;
    289 static void *pmap_bootstrap_pv_page_alloc(struct pool *, int);
    290 static void pmap_bootstrap_pv_page_free(struct pool *, void *);
    291 static struct pool_allocator pmap_bootstrap_pv_allocator = {
    292 	pmap_bootstrap_pv_page_alloc, pmap_bootstrap_pv_page_free
    293 };
    294 
    295 /*
    296  * Pool and cache of l2_dtable structures.
    297  * We use a cache to avoid clearing the structures when they're
    298  * allocated. (196 bytes)
    299  */
    300 static struct pool_cache pmap_l2dtable_cache;
    301 static vaddr_t pmap_kernel_l2dtable_kva;
    302 
    303 /*
    304  * Pool and cache of L2 page descriptors.
    305  * We use a cache to avoid clearing the descriptor table
    306  * when they're allocated. (1KB)
    307  */
    308 static struct pool_cache pmap_l2ptp_cache;
    309 static vaddr_t pmap_kernel_l2ptp_kva;
    310 static paddr_t pmap_kernel_l2ptp_phys;
    311 
    312 #ifdef PMAPCOUNTERS
    313 #define	PMAP_EVCNT_INITIALIZER(name) \
    314 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name)
    315 
    316 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
    317 static struct evcnt pmap_ev_vac_clean_one =
    318    PMAP_EVCNT_INITIALIZER("clean page (1 color)");
    319 static struct evcnt pmap_ev_vac_flush_one =
    320    PMAP_EVCNT_INITIALIZER("flush page (1 color)");
    321 static struct evcnt pmap_ev_vac_flush_lots =
    322    PMAP_EVCNT_INITIALIZER("flush page (2+ colors)");
    323 static struct evcnt pmap_ev_vac_flush_lots2 =
    324    PMAP_EVCNT_INITIALIZER("flush page (2+ colors, kmpage)");
    325 EVCNT_ATTACH_STATIC(pmap_ev_vac_clean_one);
    326 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_one);
    327 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots);
    328 EVCNT_ATTACH_STATIC(pmap_ev_vac_flush_lots2);
    329 
    330 static struct evcnt pmap_ev_vac_color_new =
    331    PMAP_EVCNT_INITIALIZER("new page color");
    332 static struct evcnt pmap_ev_vac_color_reuse =
    333    PMAP_EVCNT_INITIALIZER("ok first page color");
    334 static struct evcnt pmap_ev_vac_color_ok =
    335    PMAP_EVCNT_INITIALIZER("ok page color");
    336 static struct evcnt pmap_ev_vac_color_blind =
    337    PMAP_EVCNT_INITIALIZER("blind page color");
    338 static struct evcnt pmap_ev_vac_color_change =
    339    PMAP_EVCNT_INITIALIZER("change page color");
    340 static struct evcnt pmap_ev_vac_color_erase =
    341    PMAP_EVCNT_INITIALIZER("erase page color");
    342 static struct evcnt pmap_ev_vac_color_none =
    343    PMAP_EVCNT_INITIALIZER("no page color");
    344 static struct evcnt pmap_ev_vac_color_restore =
    345    PMAP_EVCNT_INITIALIZER("restore page color");
    346 
    347 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new);
    348 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse);
    349 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok);
    350 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind);
    351 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change);
    352 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase);
    353 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none);
    354 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore);
    355 #endif
    356 
    357 static struct evcnt pmap_ev_mappings =
    358    PMAP_EVCNT_INITIALIZER("pages mapped");
    359 static struct evcnt pmap_ev_unmappings =
    360    PMAP_EVCNT_INITIALIZER("pages unmapped");
    361 static struct evcnt pmap_ev_remappings =
    362    PMAP_EVCNT_INITIALIZER("pages remapped");
    363 
    364 EVCNT_ATTACH_STATIC(pmap_ev_mappings);
    365 EVCNT_ATTACH_STATIC(pmap_ev_unmappings);
    366 EVCNT_ATTACH_STATIC(pmap_ev_remappings);
    367 
    368 static struct evcnt pmap_ev_kernel_mappings =
    369    PMAP_EVCNT_INITIALIZER("kernel pages mapped");
    370 static struct evcnt pmap_ev_kernel_unmappings =
    371    PMAP_EVCNT_INITIALIZER("kernel pages unmapped");
    372 static struct evcnt pmap_ev_kernel_remappings =
    373    PMAP_EVCNT_INITIALIZER("kernel pages remapped");
    374 
    375 EVCNT_ATTACH_STATIC(pmap_ev_kernel_mappings);
    376 EVCNT_ATTACH_STATIC(pmap_ev_kernel_unmappings);
    377 EVCNT_ATTACH_STATIC(pmap_ev_kernel_remappings);
    378 
    379 static struct evcnt pmap_ev_kenter_mappings =
    380    PMAP_EVCNT_INITIALIZER("kenter pages mapped");
    381 static struct evcnt pmap_ev_kenter_unmappings =
    382    PMAP_EVCNT_INITIALIZER("kenter pages unmapped");
    383 static struct evcnt pmap_ev_kenter_remappings =
    384    PMAP_EVCNT_INITIALIZER("kenter pages remapped");
    385 static struct evcnt pmap_ev_pt_mappings =
    386    PMAP_EVCNT_INITIALIZER("page table pages mapped");
    387 
    388 EVCNT_ATTACH_STATIC(pmap_ev_kenter_mappings);
    389 EVCNT_ATTACH_STATIC(pmap_ev_kenter_unmappings);
    390 EVCNT_ATTACH_STATIC(pmap_ev_kenter_remappings);
    391 EVCNT_ATTACH_STATIC(pmap_ev_pt_mappings);
    392 
    393 static struct evcnt pmap_ev_fixup_mod =
    394    PMAP_EVCNT_INITIALIZER("page modification emulations");
    395 static struct evcnt pmap_ev_fixup_ref =
    396    PMAP_EVCNT_INITIALIZER("page reference emulations");
    397 static struct evcnt pmap_ev_fixup_exec =
    398    PMAP_EVCNT_INITIALIZER("exec pages fixed up");
    399 static struct evcnt pmap_ev_fixup_pdes =
    400    PMAP_EVCNT_INITIALIZER("pdes fixed up");
    401 #ifndef ARM_MMU_EXTENDED
    402 static struct evcnt pmap_ev_fixup_ptesync =
    403    PMAP_EVCNT_INITIALIZER("ptesync fixed");
    404 #endif
    405 
    406 EVCNT_ATTACH_STATIC(pmap_ev_fixup_mod);
    407 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ref);
    408 EVCNT_ATTACH_STATIC(pmap_ev_fixup_exec);
    409 EVCNT_ATTACH_STATIC(pmap_ev_fixup_pdes);
    410 #ifndef ARM_MMU_EXTENDED
    411 EVCNT_ATTACH_STATIC(pmap_ev_fixup_ptesync);
    412 #endif
    413 
    414 #ifdef PMAP_CACHE_VIPT
    415 static struct evcnt pmap_ev_exec_mappings =
    416    PMAP_EVCNT_INITIALIZER("exec pages mapped");
    417 static struct evcnt pmap_ev_exec_cached =
    418    PMAP_EVCNT_INITIALIZER("exec pages cached");
    419 
    420 EVCNT_ATTACH_STATIC(pmap_ev_exec_mappings);
    421 EVCNT_ATTACH_STATIC(pmap_ev_exec_cached);
    422 
    423 static struct evcnt pmap_ev_exec_synced =
    424    PMAP_EVCNT_INITIALIZER("exec pages synced");
    425 static struct evcnt pmap_ev_exec_synced_map =
    426    PMAP_EVCNT_INITIALIZER("exec pages synced (MP)");
    427 #ifndef ARM_MMU_EXTENDED
    428 static struct evcnt pmap_ev_exec_synced_unmap =
    429    PMAP_EVCNT_INITIALIZER("exec pages synced (UM)");
    430 static struct evcnt pmap_ev_exec_synced_remap =
    431    PMAP_EVCNT_INITIALIZER("exec pages synced (RM)");
    432 static struct evcnt pmap_ev_exec_synced_clearbit =
    433    PMAP_EVCNT_INITIALIZER("exec pages synced (DG)");
    434 static struct evcnt pmap_ev_exec_synced_kremove =
    435    PMAP_EVCNT_INITIALIZER("exec pages synced (KU)");
    436 #endif
    437 
    438 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced);
    439 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_map);
    440 #ifndef ARM_MMU_EXTENDED
    441 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_unmap);
    442 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_remap);
    443 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_clearbit);
    444 EVCNT_ATTACH_STATIC(pmap_ev_exec_synced_kremove);
    445 #endif
    446 
    447 static struct evcnt pmap_ev_exec_discarded_unmap =
    448    PMAP_EVCNT_INITIALIZER("exec pages discarded (UM)");
    449 static struct evcnt pmap_ev_exec_discarded_zero =
    450    PMAP_EVCNT_INITIALIZER("exec pages discarded (ZP)");
    451 static struct evcnt pmap_ev_exec_discarded_copy =
    452    PMAP_EVCNT_INITIALIZER("exec pages discarded (CP)");
    453 static struct evcnt pmap_ev_exec_discarded_page_protect =
    454    PMAP_EVCNT_INITIALIZER("exec pages discarded (PP)");
    455 static struct evcnt pmap_ev_exec_discarded_clearbit =
    456    PMAP_EVCNT_INITIALIZER("exec pages discarded (DG)");
    457 static struct evcnt pmap_ev_exec_discarded_kremove =
    458    PMAP_EVCNT_INITIALIZER("exec pages discarded (KU)");
    459 #ifdef ARM_MMU_EXTENDED
    460 static struct evcnt pmap_ev_exec_discarded_modfixup =
    461    PMAP_EVCNT_INITIALIZER("exec pages discarded (MF)");
    462 #endif
    463 
    464 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_unmap);
    465 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_zero);
    466 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_copy);
    467 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_page_protect);
    468 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_clearbit);
    469 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_kremove);
    470 #ifdef ARM_MMU_EXTENDED
    471 EVCNT_ATTACH_STATIC(pmap_ev_exec_discarded_modfixup);
    472 #endif
    473 #endif /* PMAP_CACHE_VIPT */
    474 
    475 static struct evcnt pmap_ev_updates = PMAP_EVCNT_INITIALIZER("updates");
    476 static struct evcnt pmap_ev_collects = PMAP_EVCNT_INITIALIZER("collects");
    477 static struct evcnt pmap_ev_activations = PMAP_EVCNT_INITIALIZER("activations");
    478 
    479 EVCNT_ATTACH_STATIC(pmap_ev_updates);
    480 EVCNT_ATTACH_STATIC(pmap_ev_collects);
    481 EVCNT_ATTACH_STATIC(pmap_ev_activations);
    482 
    483 #define	PMAPCOUNT(x)	((void)(pmap_ev_##x.ev_count++))
    484 #else
    485 #define	PMAPCOUNT(x)	((void)0)
    486 #endif
    487 
    488 /*
    489  * pmap copy/zero page, and mem(5) hook point
    490  */
    491 static pt_entry_t *csrc_pte, *cdst_pte;
    492 static vaddr_t csrcp, cdstp;
    493 #ifdef MULTIPROCESSOR
    494 static size_t cnptes;
    495 #define	cpu_csrc_pte(o)	(csrc_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT))
    496 #define	cpu_cdst_pte(o)	(cdst_pte + cnptes * cpu_number() + ((o) >> L2_S_SHIFT))
    497 #define	cpu_csrcp(o)	(csrcp + L2_S_SIZE * cnptes * cpu_number() + (o))
    498 #define	cpu_cdstp(o)	(cdstp + L2_S_SIZE * cnptes * cpu_number() + (o))
    499 #else
    500 #define	cpu_csrc_pte(o)	(csrc_pte + ((o) >> L2_S_SHIFT))
    501 #define	cpu_cdst_pte(o)	(cdst_pte + ((o) >> L2_S_SHIFT))
    502 #define	cpu_csrcp(o)	(csrcp + (o))
    503 #define	cpu_cdstp(o)	(cdstp + (o))
    504 #endif
    505 vaddr_t memhook;			/* used by mem.c & others */
    506 kmutex_t memlock __cacheline_aligned;	/* used by mem.c & others */
    507 kmutex_t pmap_lock __cacheline_aligned;
    508 extern void *msgbufaddr;
    509 int pmap_kmpages;
    510 /*
    511  * Flag to indicate if pmap_init() has done its thing
    512  */
    513 bool pmap_initialized;
    514 
    515 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
    516 /*
    517  * Start of direct-mapped memory
    518  */
    519 vaddr_t pmap_directbase = KERNEL_BASE;
    520 #endif
    521 
    522 /*
    523  * Misc. locking data structures
    524  */
    525 
    526 static inline void
    527 pmap_acquire_pmap_lock(pmap_t pm)
    528 {
    529 	if (pm == pmap_kernel()) {
    530 #ifdef MULTIPROCESSOR
    531 		KERNEL_LOCK(1, NULL);
    532 #endif
    533 	} else {
    534 		mutex_enter(pm->pm_lock);
    535 	}
    536 }
    537 
    538 static inline void
    539 pmap_release_pmap_lock(pmap_t pm)
    540 {
    541 	if (pm == pmap_kernel()) {
    542 #ifdef MULTIPROCESSOR
    543 		KERNEL_UNLOCK_ONE(NULL);
    544 #endif
    545 	} else {
    546 		mutex_exit(pm->pm_lock);
    547 	}
    548 }
    549 
    550 static inline void
    551 pmap_acquire_page_lock(struct vm_page_md *md)
    552 {
    553 	mutex_enter(&pmap_lock);
    554 }
    555 
    556 static inline void
    557 pmap_release_page_lock(struct vm_page_md *md)
    558 {
    559 	mutex_exit(&pmap_lock);
    560 }
    561 
    562 #ifdef DIAGNOSTIC
    563 static inline int
    564 pmap_page_locked_p(struct vm_page_md *md)
    565 {
    566 	return mutex_owned(&pmap_lock);
    567 }
    568 #endif
    569 
    570 
    571 /*
    572  * Metadata for L1 translation tables.
    573  */
    574 #ifndef ARM_MMU_EXTENDED
    575 struct l1_ttable {
    576 	/* Entry on the L1 Table list */
    577 	SLIST_ENTRY(l1_ttable) l1_link;
    578 
    579 	/* Entry on the L1 Least Recently Used list */
    580 	TAILQ_ENTRY(l1_ttable) l1_lru;
    581 
    582 	/* Track how many domains are allocated from this L1 */
    583 	volatile u_int l1_domain_use_count;
    584 
    585 	/*
    586 	 * A free-list of domain numbers for this L1.
    587 	 * We avoid using ffs() and a bitmap to track domains since ffs()
    588 	 * is slow on ARM.
    589 	 */
    590 	uint8_t l1_domain_first;
    591 	uint8_t l1_domain_free[PMAP_DOMAINS];
    592 
    593 	/* Physical address of this L1 page table */
    594 	paddr_t l1_physaddr;
    595 
    596 	/* KVA of this L1 page table */
    597 	pd_entry_t *l1_kva;
    598 };
    599 
    600 /*
    601  * L1 Page Tables are tracked using a Least Recently Used list.
    602  *  - New L1s are allocated from the HEAD.
    603  *  - Freed L1s are added to the TAIl.
    604  *  - Recently accessed L1s (where an 'access' is some change to one of
    605  *    the userland pmaps which owns this L1) are moved to the TAIL.
    606  */
    607 static TAILQ_HEAD(, l1_ttable) l1_lru_list;
    608 static kmutex_t l1_lru_lock __cacheline_aligned;
    609 
    610 /*
    611  * A list of all L1 tables
    612  */
    613 static SLIST_HEAD(, l1_ttable) l1_list;
    614 #endif /* ARM_MMU_EXTENDED */
    615 
    616 /*
    617  * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots.
    618  *
    619  * This is normally 16MB worth L2 page descriptors for any given pmap.
    620  * Reference counts are maintained for L2 descriptors so they can be
    621  * freed when empty.
    622  */
    623 struct l2_dtable {
    624 	/* The number of L2 page descriptors allocated to this l2_dtable */
    625 	u_int l2_occupancy;
    626 
    627 	/* List of L2 page descriptors */
    628 	struct l2_bucket {
    629 		pt_entry_t *l2b_kva;	/* KVA of L2 Descriptor Table */
    630 		paddr_t l2b_pa;		/* Physical address of same */
    631 		u_short l2b_l1slot;	/* This L2 table's L1 index */
    632 		u_short l2b_occupancy;	/* How many active descriptors */
    633 	} l2_bucket[L2_BUCKET_SIZE];
    634 };
    635 
    636 /*
    637  * Given an L1 table index, calculate the corresponding l2_dtable index
    638  * and bucket index within the l2_dtable.
    639  */
    640 #define L2_BUCKET_XSHIFT	(L2_BUCKET_XLOG2 - L1_S_SHIFT)
    641 #define L2_BUCKET_XFRAME	(~(vaddr_t)0 << L2_BUCKET_XLOG2)
    642 #define L2_BUCKET_IDX(l1slot)	((l1slot) >> L2_BUCKET_XSHIFT)
    643 #define L2_IDX(l1slot)		(L2_BUCKET_IDX(l1slot) >> L2_BUCKET_LOG2)
    644 #define L2_BUCKET(l1slot)	(L2_BUCKET_IDX(l1slot) & (L2_BUCKET_SIZE - 1))
    645 
    646 __CTASSERT(0x100000000ULL == ((uint64_t)L2_SIZE * L2_BUCKET_SIZE * L1_S_SIZE));
    647 __CTASSERT(L2_BUCKET_XFRAME == ~(L2_BUCKET_XSIZE-1));
    648 
    649 /*
    650  * Given a virtual address, this macro returns the
    651  * virtual address required to drop into the next L2 bucket.
    652  */
    653 #define	L2_NEXT_BUCKET_VA(va)	(((va) & L2_BUCKET_XFRAME) + L2_BUCKET_XSIZE)
    654 
    655 /*
    656  * L2 allocation.
    657  */
    658 #define	pmap_alloc_l2_dtable()		\
    659 	    pool_cache_get(&pmap_l2dtable_cache, PR_NOWAIT)
    660 #define	pmap_free_l2_dtable(l2)		\
    661 	    pool_cache_put(&pmap_l2dtable_cache, (l2))
    662 #define pmap_alloc_l2_ptp(pap)		\
    663 	    ((pt_entry_t *)pool_cache_get_paddr(&pmap_l2ptp_cache,\
    664 	    PR_NOWAIT, (pap)))
    665 
    666 /*
    667  * We try to map the page tables write-through, if possible.  However, not
    668  * all CPUs have a write-through cache mode, so on those we have to sync
    669  * the cache when we frob page tables.
    670  *
    671  * We try to evaluate this at compile time, if possible.  However, it's
    672  * not always possible to do that, hence this run-time var.
    673  */
    674 int	pmap_needs_pte_sync;
    675 
    676 /*
    677  * Real definition of pv_entry.
    678  */
    679 struct pv_entry {
    680 	SLIST_ENTRY(pv_entry) pv_link;	/* next pv_entry */
    681 	pmap_t		pv_pmap;        /* pmap where mapping lies */
    682 	vaddr_t		pv_va;          /* virtual address for mapping */
    683 	u_int		pv_flags;       /* flags */
    684 };
    685 
    686 /*
    687  * Macro to determine if a mapping might be resident in the
    688  * instruction cache and/or TLB
    689  */
    690 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED)
    691 /*
    692  * Speculative loads by Cortex cores can cause TLB entries to be filled even if
    693  * there are no explicit accesses, so there may be always be TLB entries to
    694  * flush.  If we used ASIDs then this would not be a problem.
    695  */
    696 #define	PV_BEEN_EXECD(f)  (((f) & PVF_EXEC) == PVF_EXEC)
    697 #else
    698 #define	PV_BEEN_EXECD(f)  (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC))
    699 #endif
    700 #define	PV_IS_EXEC_P(f)   (((f) & PVF_EXEC) != 0)
    701 #define	PV_IS_KENTRY_P(f) (((f) & PVF_KENTRY) != 0)
    702 #define	PV_IS_WRITE_P(f)  (((f) & PVF_WRITE) != 0)
    703 
    704 /*
    705  * Macro to determine if a mapping might be resident in the
    706  * data cache and/or TLB
    707  */
    708 #if ARM_MMU_V7 > 0 && !defined(ARM_MMU_EXTENDED)
    709 /*
    710  * Speculative loads by Cortex cores can cause TLB entries to be filled even if
    711  * there are no explicit accesses, so there may be always be TLB entries to
    712  * flush.  If we used ASIDs then this would not be a problem.
    713  */
    714 #define	PV_BEEN_REFD(f)   (1)
    715 #else
    716 #define	PV_BEEN_REFD(f)   (((f) & PVF_REF) != 0)
    717 #endif
    718 
    719 /*
    720  * Local prototypes
    721  */
    722 static bool		pmap_set_pt_cache_mode(pd_entry_t *, vaddr_t, size_t);
    723 static void		pmap_alloc_specials(vaddr_t *, int, vaddr_t *,
    724 			    pt_entry_t **);
    725 static bool		pmap_is_current(pmap_t) __unused;
    726 static bool		pmap_is_cached(pmap_t);
    727 static void		pmap_enter_pv(struct vm_page_md *, paddr_t, struct pv_entry *,
    728 			    pmap_t, vaddr_t, u_int);
    729 static struct pv_entry *pmap_find_pv(struct vm_page_md *, pmap_t, vaddr_t);
    730 static struct pv_entry *pmap_remove_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t);
    731 static u_int		pmap_modify_pv(struct vm_page_md *, paddr_t, pmap_t, vaddr_t,
    732 			    u_int, u_int);
    733 
    734 static void		pmap_pinit(pmap_t);
    735 static int		pmap_pmap_ctor(void *, void *, int);
    736 
    737 static void		pmap_alloc_l1(pmap_t);
    738 static void		pmap_free_l1(pmap_t);
    739 #ifndef ARM_MMU_EXTENDED
    740 static void		pmap_use_l1(pmap_t);
    741 #endif
    742 
    743 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vaddr_t);
    744 static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vaddr_t);
    745 static void		pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
    746 static int		pmap_l2ptp_ctor(void *, void *, int);
    747 static int		pmap_l2dtable_ctor(void *, void *, int);
    748 
    749 static void		pmap_vac_me_harder(struct vm_page_md *, paddr_t, pmap_t, vaddr_t);
    750 #ifdef PMAP_CACHE_VIVT
    751 static void		pmap_vac_me_kpmap(struct vm_page_md *, paddr_t, pmap_t, vaddr_t);
    752 static void		pmap_vac_me_user(struct vm_page_md *, paddr_t, pmap_t, vaddr_t);
    753 #endif
    754 
    755 static void		pmap_clearbit(struct vm_page_md *, paddr_t, u_int);
    756 #ifdef PMAP_CACHE_VIVT
    757 static bool		pmap_clean_page(struct vm_page_md *, bool);
    758 #endif
    759 #ifdef PMAP_CACHE_VIPT
    760 static void		pmap_syncicache_page(struct vm_page_md *, paddr_t);
    761 enum pmap_flush_op {
    762 	PMAP_FLUSH_PRIMARY,
    763 	PMAP_FLUSH_SECONDARY,
    764 	PMAP_CLEAN_PRIMARY
    765 };
    766 #ifndef ARM_MMU_EXTENDED
    767 static void		pmap_flush_page(struct vm_page_md *, paddr_t, enum pmap_flush_op);
    768 #endif
    769 #endif
    770 static void		pmap_page_remove(struct vm_page_md *, paddr_t);
    771 
    772 #ifndef ARM_MMU_EXTENDED
    773 static void		pmap_init_l1(struct l1_ttable *, pd_entry_t *);
    774 #endif
    775 static vaddr_t		kernel_pt_lookup(paddr_t);
    776 
    777 
    778 /*
    779  * Misc variables
    780  */
    781 vaddr_t virtual_avail;
    782 vaddr_t virtual_end;
    783 vaddr_t pmap_curmaxkvaddr;
    784 
    785 paddr_t avail_start;
    786 paddr_t avail_end;
    787 
    788 pv_addrqh_t pmap_boot_freeq = SLIST_HEAD_INITIALIZER(&pmap_boot_freeq);
    789 pv_addr_t kernelpages;
    790 pv_addr_t kernel_l1pt;
    791 pv_addr_t systempage;
    792 
    793 /* Function to set the debug level of the pmap code */
    794 
    795 #ifdef PMAP_DEBUG
    796 void
    797 pmap_debug(int level)
    798 {
    799 	pmap_debug_level = level;
    800 	printf("pmap_debug: level=%d\n", pmap_debug_level);
    801 }
    802 #endif	/* PMAP_DEBUG */
    803 
    804 #ifdef PMAP_CACHE_VIPT
    805 #define PMAP_VALIDATE_MD_PAGE(md)	\
    806 	KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \
    807 	    "(md) %p: attrs=%#x urw=%u krw=%u", (md), \
    808 	    (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings);
    809 #endif /* PMAP_CACHE_VIPT */
    810 /*
    811  * A bunch of routines to conditionally flush the caches/TLB depending
    812  * on whether the specified pmap actually needs to be flushed at any
    813  * given time.
    814  */
    815 static inline void
    816 pmap_tlb_flush_SE(pmap_t pm, vaddr_t va, u_int flags)
    817 {
    818 #ifdef ARM_MMU_EXTENDED
    819 	pmap_tlb_invalidate_addr(pm, va);
    820 #else
    821 	if (pm->pm_cstate.cs_tlb_id != 0) {
    822 		if (PV_BEEN_EXECD(flags)) {
    823 			cpu_tlb_flushID_SE(va);
    824 		} else if (PV_BEEN_REFD(flags)) {
    825 			cpu_tlb_flushD_SE(va);
    826 		}
    827 	}
    828 #endif /* ARM_MMU_EXTENDED */
    829 }
    830 
    831 static inline void
    832 pmap_tlb_flushID(pmap_t pm)
    833 {
    834 #ifdef ARM_MMU_EXTENDED
    835 	pmap_tlb_asid_release_all(pm);
    836 #else
    837 	if (pm->pm_cstate.cs_tlb_id) {
    838 		cpu_tlb_flushID();
    839 #if ARM_MMU_V7 == 0
    840 		/*
    841 		 * Speculative loads by Cortex cores can cause TLB entries to
    842 		 * be filled even if there are no explicit accesses, so there
    843 		 * may be always be TLB entries to flush.  If we used ASIDs
    844 		 * then it would not be a problem.
    845 		 * This is not true for other CPUs.
    846 		 */
    847 		pm->pm_cstate.cs_tlb = 0;
    848 #endif /* ARM_MMU_V7 */
    849 	}
    850 #endif /* ARM_MMU_EXTENDED */
    851 }
    852 
    853 static inline void
    854 pmap_tlb_flushD(pmap_t pm)
    855 {
    856 #ifdef ARM_MMU_EXTENDED
    857 	pmap_tlb_asid_release_all(pm);
    858 #else
    859 	if (pm->pm_cstate.cs_tlb_d) {
    860 		cpu_tlb_flushD();
    861 #if ARM_MMU_V7 == 0
    862 		/*
    863 		 * Speculative loads by Cortex cores can cause TLB entries to
    864 		 * be filled even if there are no explicit accesses, so there
    865 		 * may be always be TLB entries to flush.  If we used ASIDs
    866 		 * then it would not be a problem.
    867 		 * This is not true for other CPUs.
    868 		 */
    869 		pm->pm_cstate.cs_tlb_d = 0;
    870 #endif /* ARM_MMU_V7 */
    871 	}
    872 #endif /* ARM_MMU_EXTENDED */
    873 }
    874 
    875 #ifdef PMAP_CACHE_VIVT
    876 static inline void
    877 pmap_cache_wbinv_page(pmap_t pm, vaddr_t va, bool do_inv, u_int flags)
    878 {
    879 	if (PV_BEEN_EXECD(flags) && pm->pm_cstate.cs_cache_id) {
    880 		cpu_idcache_wbinv_range(va, PAGE_SIZE);
    881 	} else if (PV_BEEN_REFD(flags) && pm->pm_cstate.cs_cache_d) {
    882 		if (do_inv) {
    883 			if (flags & PVF_WRITE)
    884 				cpu_dcache_wbinv_range(va, PAGE_SIZE);
    885 			else
    886 				cpu_dcache_inv_range(va, PAGE_SIZE);
    887 		} else if (flags & PVF_WRITE) {
    888 			cpu_dcache_wb_range(va, PAGE_SIZE);
    889 		}
    890 	}
    891 }
    892 
    893 static inline void
    894 pmap_cache_wbinv_all(pmap_t pm, u_int flags)
    895 {
    896 	if (PV_BEEN_EXECD(flags)) {
    897 		if (pm->pm_cstate.cs_cache_id) {
    898 			cpu_idcache_wbinv_all();
    899 			pm->pm_cstate.cs_cache = 0;
    900 		}
    901 	} else if (pm->pm_cstate.cs_cache_d) {
    902 		cpu_dcache_wbinv_all();
    903 		pm->pm_cstate.cs_cache_d = 0;
    904 	}
    905 }
    906 #endif /* PMAP_CACHE_VIVT */
    907 
    908 static inline uint8_t
    909 pmap_domain(pmap_t pm)
    910 {
    911 #ifdef ARM_MMU_EXTENDED
    912 	return pm == pmap_kernel() ? PMAP_DOMAIN_KERNEL : PMAP_DOMAIN_USER;
    913 #else
    914 	return pm->pm_domain;
    915 #endif
    916 }
    917 
    918 static inline pd_entry_t *
    919 pmap_l1_kva(pmap_t pm)
    920 {
    921 #ifdef ARM_MMU_EXTENDED
    922 	return pm->pm_l1;
    923 #else
    924 	return pm->pm_l1->l1_kva;
    925 #endif
    926 }
    927 
    928 static inline bool
    929 pmap_is_current(pmap_t pm)
    930 {
    931 	if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm)
    932 		return true;
    933 
    934 	return false;
    935 }
    936 
    937 static inline bool
    938 pmap_is_cached(pmap_t pm)
    939 {
    940 #ifdef ARM_MMU_EXTENDED
    941 	struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
    942 	if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(PMAP_PAI(pm, ti), ti))
    943 		return true;
    944 #else
    945 	struct cpu_info * const ci = curcpu();
    946 	if (pm == pmap_kernel() || ci->ci_pmap_lastuser == NULL
    947 	    || ci->ci_pmap_lastuser == pm)
    948 		return true;
    949 #endif /* ARM_MMU_EXTENDED */
    950 
    951 	return false;
    952 }
    953 
    954 /*
    955  * PTE_SYNC_CURRENT:
    956  *
    957  *     Make sure the pte is written out to RAM.
    958  *     We need to do this for one of two cases:
    959  *       - We're dealing with the kernel pmap
    960  *       - There is no pmap active in the cache/tlb.
    961  *       - The specified pmap is 'active' in the cache/tlb.
    962  */
    963 #ifdef PMAP_INCLUDE_PTE_SYNC
    964 #define	PTE_SYNC_CURRENT(pm, ptep)	\
    965 do {					\
    966 	if (PMAP_NEEDS_PTE_SYNC && 	\
    967 	    pmap_is_cached(pm))		\
    968 		PTE_SYNC(ptep);		\
    969 } while (/*CONSTCOND*/0)
    970 #else
    971 #define	PTE_SYNC_CURRENT(pm, ptep)	/* nothing */
    972 #endif
    973 
    974 /*
    975  * main pv_entry manipulation functions:
    976  *   pmap_enter_pv: enter a mapping onto a vm_page list
    977  *   pmap_remove_pv: remove a mapping from a vm_page list
    978  *
    979  * NOTE: pmap_enter_pv expects to lock the pvh itself
    980  *       pmap_remove_pv expects the caller to lock the pvh before calling
    981  */
    982 
    983 /*
    984  * pmap_enter_pv: enter a mapping onto a vm_page lst
    985  *
    986  * => caller should hold the proper lock on pmap_main_lock
    987  * => caller should have pmap locked
    988  * => we will gain the lock on the vm_page and allocate the new pv_entry
    989  * => caller should adjust ptp's wire_count before calling
    990  * => caller should not adjust pmap's wire_count
    991  */
    992 static void
    993 pmap_enter_pv(struct vm_page_md *md, paddr_t pa, struct pv_entry *pv, pmap_t pm,
    994     vaddr_t va, u_int flags)
    995 {
    996 	struct pv_entry **pvp;
    997 
    998 	NPDEBUG(PDB_PVDUMP,
    999 	    printf("pmap_enter_pv: pm %p, md %p, flags 0x%x\n", pm, md, flags));
   1000 
   1001 	pv->pv_pmap = pm;
   1002 	pv->pv_va = va;
   1003 	pv->pv_flags = flags;
   1004 
   1005 	pvp = &SLIST_FIRST(&md->pvh_list);
   1006 #ifdef PMAP_CACHE_VIPT
   1007 	/*
   1008 	 * Insert unmanaged entries, writeable first, at the head of
   1009 	 * the pv list.
   1010 	 */
   1011 	if (__predict_true(!PV_IS_KENTRY_P(flags))) {
   1012 		while (*pvp != NULL && PV_IS_KENTRY_P((*pvp)->pv_flags))
   1013 			pvp = &SLIST_NEXT(*pvp, pv_link);
   1014 	}
   1015 	if (!PV_IS_WRITE_P(flags)) {
   1016 		while (*pvp != NULL && PV_IS_WRITE_P((*pvp)->pv_flags))
   1017 			pvp = &SLIST_NEXT(*pvp, pv_link);
   1018 	}
   1019 #endif
   1020 	SLIST_NEXT(pv, pv_link) = *pvp;		/* add to ... */
   1021 	*pvp = pv;				/* ... locked list */
   1022 	md->pvh_attrs |= flags & (PVF_REF | PVF_MOD);
   1023 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   1024 	if ((pv->pv_flags & PVF_KWRITE) == PVF_KWRITE)
   1025 		md->pvh_attrs |= PVF_KMOD;
   1026 	if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC)
   1027 		md->pvh_attrs |= PVF_DIRTY;
   1028 	KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   1029 #endif
   1030 	if (pm == pmap_kernel()) {
   1031 		PMAPCOUNT(kernel_mappings);
   1032 		if (flags & PVF_WRITE)
   1033 			md->krw_mappings++;
   1034 		else
   1035 			md->kro_mappings++;
   1036 	} else {
   1037 		if (flags & PVF_WRITE)
   1038 			md->urw_mappings++;
   1039 		else
   1040 			md->uro_mappings++;
   1041 	}
   1042 
   1043 #ifdef PMAP_CACHE_VIPT
   1044 #ifndef ARM_MMU_EXTENDED
   1045 	/*
   1046 	 * Even though pmap_vac_me_harder will set PVF_WRITE for us,
   1047 	 * do it here as well to keep the mappings & KVF_WRITE consistent.
   1048 	 */
   1049 	if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) {
   1050 		md->pvh_attrs |= PVF_WRITE;
   1051 	}
   1052 #endif
   1053 	/*
   1054 	 * If this is an exec mapping and its the first exec mapping
   1055 	 * for this page, make sure to sync the I-cache.
   1056 	 */
   1057 	if (PV_IS_EXEC_P(flags)) {
   1058 #ifndef ARM_MMU_EXTENDED
   1059 		if (!PV_IS_EXEC_P(md->pvh_attrs)) {
   1060 			pmap_syncicache_page(md, pa);
   1061 			PMAPCOUNT(exec_synced_map);
   1062 		}
   1063 #endif
   1064 		PMAPCOUNT(exec_mappings);
   1065 	}
   1066 #endif
   1067 
   1068 	PMAPCOUNT(mappings);
   1069 
   1070 	if (pv->pv_flags & PVF_WIRED)
   1071 		++pm->pm_stats.wired_count;
   1072 }
   1073 
   1074 /*
   1075  *
   1076  * pmap_find_pv: Find a pv entry
   1077  *
   1078  * => caller should hold lock on vm_page
   1079  */
   1080 static inline struct pv_entry *
   1081 pmap_find_pv(struct vm_page_md *md, pmap_t pm, vaddr_t va)
   1082 {
   1083 	struct pv_entry *pv;
   1084 
   1085 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   1086 		if (pm == pv->pv_pmap && va == pv->pv_va)
   1087 			break;
   1088 	}
   1089 
   1090 	return (pv);
   1091 }
   1092 
   1093 /*
   1094  * pmap_remove_pv: try to remove a mapping from a pv_list
   1095  *
   1096  * => caller should hold proper lock on pmap_main_lock
   1097  * => pmap should be locked
   1098  * => caller should hold lock on vm_page [so that attrs can be adjusted]
   1099  * => caller should adjust ptp's wire_count and free PTP if needed
   1100  * => caller should NOT adjust pmap's wire_count
   1101  * => we return the removed pv
   1102  */
   1103 static struct pv_entry *
   1104 pmap_remove_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va)
   1105 {
   1106 	struct pv_entry *pv, **prevptr;
   1107 
   1108 	NPDEBUG(PDB_PVDUMP,
   1109 	    printf("pmap_remove_pv: pm %p, md %p, va 0x%08lx\n", pm, md, va));
   1110 
   1111 	prevptr = &SLIST_FIRST(&md->pvh_list); /* prev pv_entry ptr */
   1112 	pv = *prevptr;
   1113 
   1114 	while (pv) {
   1115 		if (pv->pv_pmap == pm && pv->pv_va == va) {	/* match? */
   1116 			NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, md "
   1117 			    "%p, flags 0x%x\n", pm, md, pv->pv_flags));
   1118 			if (pv->pv_flags & PVF_WIRED) {
   1119 				--pm->pm_stats.wired_count;
   1120 			}
   1121 			*prevptr = SLIST_NEXT(pv, pv_link);	/* remove it! */
   1122 			if (pm == pmap_kernel()) {
   1123 				PMAPCOUNT(kernel_unmappings);
   1124 				if (pv->pv_flags & PVF_WRITE)
   1125 					md->krw_mappings--;
   1126 				else
   1127 					md->kro_mappings--;
   1128 			} else {
   1129 				if (pv->pv_flags & PVF_WRITE)
   1130 					md->urw_mappings--;
   1131 				else
   1132 					md->uro_mappings--;
   1133 			}
   1134 
   1135 			PMAPCOUNT(unmappings);
   1136 #ifdef PMAP_CACHE_VIPT
   1137 			if (!(pv->pv_flags & PVF_WRITE))
   1138 				break;
   1139 			/*
   1140 			 * If this page has had an exec mapping, then if
   1141 			 * this was the last mapping, discard the contents,
   1142 			 * otherwise sync the i-cache for this page.
   1143 			 */
   1144 			if (PV_IS_EXEC_P(md->pvh_attrs)) {
   1145 #ifdef ARM_MMU_EXTENDED
   1146 				md->pvh_attrs &= ~PVF_EXEC;
   1147 				PMAPCOUNT(exec_discarded_unmap);
   1148 #else
   1149 				if (SLIST_EMPTY(&md->pvh_list)) {
   1150 					md->pvh_attrs &= ~PVF_EXEC;
   1151 					PMAPCOUNT(exec_discarded_unmap);
   1152 				} else {
   1153 					pmap_syncicache_page(md, pa);
   1154 					PMAPCOUNT(exec_synced_unmap);
   1155 				}
   1156 #endif /* ARM_MMU_EXTENDED */
   1157 			}
   1158 #endif /* PMAP_CACHE_VIPT */
   1159 			break;
   1160 		}
   1161 		prevptr = &SLIST_NEXT(pv, pv_link);	/* previous pointer */
   1162 		pv = *prevptr;				/* advance */
   1163 	}
   1164 
   1165 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   1166 	/*
   1167 	 * If we no longer have a WRITEABLE KENTRY at the head of list,
   1168 	 * clear the KMOD attribute from the page.
   1169 	 */
   1170 	if (SLIST_FIRST(&md->pvh_list) == NULL
   1171 	    || (SLIST_FIRST(&md->pvh_list)->pv_flags & PVF_KWRITE) != PVF_KWRITE)
   1172 		md->pvh_attrs &= ~PVF_KMOD;
   1173 
   1174 	/*
   1175 	 * If this was a writeable page and there are no more writeable
   1176 	 * mappings (ignoring KMPAGE), clear the WRITE flag and writeback
   1177 	 * the contents to memory.
   1178 	 */
   1179 	if (arm_cache_prefer_mask != 0) {
   1180 		if (md->krw_mappings + md->urw_mappings == 0)
   1181 			md->pvh_attrs &= ~PVF_WRITE;
   1182 		PMAP_VALIDATE_MD_PAGE(md);
   1183 	}
   1184 	KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   1185 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */
   1186 
   1187 	return(pv);				/* return removed pv */
   1188 }
   1189 
   1190 /*
   1191  *
   1192  * pmap_modify_pv: Update pv flags
   1193  *
   1194  * => caller should hold lock on vm_page [so that attrs can be adjusted]
   1195  * => caller should NOT adjust pmap's wire_count
   1196  * => caller must call pmap_vac_me_harder() if writable status of a page
   1197  *    may have changed.
   1198  * => we return the old flags
   1199  *
   1200  * Modify a physical-virtual mapping in the pv table
   1201  */
   1202 static u_int
   1203 pmap_modify_pv(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va,
   1204     u_int clr_mask, u_int set_mask)
   1205 {
   1206 	struct pv_entry *npv;
   1207 	u_int flags, oflags;
   1208 
   1209 	KASSERT(!PV_IS_KENTRY_P(clr_mask));
   1210 	KASSERT(!PV_IS_KENTRY_P(set_mask));
   1211 
   1212 	if ((npv = pmap_find_pv(md, pm, va)) == NULL)
   1213 		return (0);
   1214 
   1215 	NPDEBUG(PDB_PVDUMP,
   1216 	    printf("pmap_modify_pv: pm %p, md %p, clr 0x%x, set 0x%x, flags 0x%x\n", pm, md, clr_mask, set_mask, npv->pv_flags));
   1217 
   1218 	/*
   1219 	 * There is at least one VA mapping this page.
   1220 	 */
   1221 
   1222 	if (clr_mask & (PVF_REF | PVF_MOD)) {
   1223 		md->pvh_attrs |= set_mask & (PVF_REF | PVF_MOD);
   1224 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   1225 		if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) != PVF_NC)
   1226 			md->pvh_attrs |= PVF_DIRTY;
   1227 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   1228 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */
   1229 	}
   1230 
   1231 	oflags = npv->pv_flags;
   1232 	npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask;
   1233 
   1234 	if ((flags ^ oflags) & PVF_WIRED) {
   1235 		if (flags & PVF_WIRED)
   1236 			++pm->pm_stats.wired_count;
   1237 		else
   1238 			--pm->pm_stats.wired_count;
   1239 	}
   1240 
   1241 	if ((flags ^ oflags) & PVF_WRITE) {
   1242 		if (pm == pmap_kernel()) {
   1243 			if (flags & PVF_WRITE) {
   1244 				md->krw_mappings++;
   1245 				md->kro_mappings--;
   1246 			} else {
   1247 				md->kro_mappings++;
   1248 				md->krw_mappings--;
   1249 			}
   1250 		} else {
   1251 			if (flags & PVF_WRITE) {
   1252 				md->urw_mappings++;
   1253 				md->uro_mappings--;
   1254 			} else {
   1255 				md->uro_mappings++;
   1256 				md->urw_mappings--;
   1257 			}
   1258 		}
   1259 	}
   1260 #ifdef PMAP_CACHE_VIPT
   1261 	if (arm_cache_prefer_mask != 0) {
   1262 		if (md->urw_mappings + md->krw_mappings == 0) {
   1263 			md->pvh_attrs &= ~PVF_WRITE;
   1264 		} else {
   1265 			md->pvh_attrs |= PVF_WRITE;
   1266 		}
   1267 	}
   1268 #ifndef ARM_MMU_EXTENDED
   1269 	/*
   1270 	 * We have two cases here: the first is from enter_pv (new exec
   1271 	 * page), the second is a combined pmap_remove_pv/pmap_enter_pv.
   1272 	 * Since in latter, pmap_enter_pv won't do anything, we just have
   1273 	 * to do what pmap_remove_pv would do.
   1274 	 */
   1275 	if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(md->pvh_attrs))
   1276 	    || (PV_IS_EXEC_P(md->pvh_attrs)
   1277 		|| (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) {
   1278 		pmap_syncicache_page(md, pa);
   1279 		PMAPCOUNT(exec_synced_remap);
   1280 	}
   1281 	KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   1282 #endif /* !ARM_MMU_EXTENDED */
   1283 #endif /* PMAP_CACHE_VIPT */
   1284 
   1285 	PMAPCOUNT(remappings);
   1286 
   1287 	return (oflags);
   1288 }
   1289 
   1290 /*
   1291  * Allocate an L1 translation table for the specified pmap.
   1292  * This is called at pmap creation time.
   1293  */
   1294 static void
   1295 pmap_alloc_l1(pmap_t pm)
   1296 {
   1297 #ifdef ARM_MMU_EXTENDED
   1298 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
   1299 #ifdef PMAP_NEED_ALLOC_POOLPAGE
   1300 	struct vm_page *pg = arm_pmap_alloc_poolpage(UVM_PGA_ZERO);
   1301 #else
   1302 	struct vm_page *pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
   1303 #endif
   1304 	bool ok __diagused;
   1305 	KASSERT(pg != NULL);
   1306 	pm->pm_l1_pa = VM_PAGE_TO_PHYS(pg);
   1307 	vaddr_t va = pmap_direct_mapped_phys(pm->pm_l1_pa, &ok, 0);
   1308 	KASSERT(ok);
   1309 	KASSERT(va >= KERNEL_BASE);
   1310 
   1311 #else
   1312 	KASSERTMSG(kernel_map != NULL, "pm %p", pm);
   1313 	vaddr_t va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1314 	    UVM_KMF_WIRED|UVM_KMF_ZERO);
   1315 	KASSERT(!va);
   1316 	pmap_extract(pmap_kernel(), va, &pm->pm_l1_pa);
   1317 #endif
   1318 	pm->pm_l1 = (pd_entry_t *)va;
   1319 	PTE_SYNC_RANGE(pm->pm_l1, PAGE_SIZE / sizeof(pt_entry_t));
   1320 #else
   1321 	struct l1_ttable *l1;
   1322 	uint8_t domain;
   1323 
   1324 	/*
   1325 	 * Remove the L1 at the head of the LRU list
   1326 	 */
   1327 	mutex_spin_enter(&l1_lru_lock);
   1328 	l1 = TAILQ_FIRST(&l1_lru_list);
   1329 	KDASSERT(l1 != NULL);
   1330 	TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
   1331 
   1332 	/*
   1333 	 * Pick the first available domain number, and update
   1334 	 * the link to the next number.
   1335 	 */
   1336 	domain = l1->l1_domain_first;
   1337 	l1->l1_domain_first = l1->l1_domain_free[domain];
   1338 
   1339 	/*
   1340 	 * If there are still free domain numbers in this L1,
   1341 	 * put it back on the TAIL of the LRU list.
   1342 	 */
   1343 	if (++l1->l1_domain_use_count < PMAP_DOMAINS)
   1344 		TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
   1345 
   1346 	mutex_spin_exit(&l1_lru_lock);
   1347 
   1348 	/*
   1349 	 * Fix up the relevant bits in the pmap structure
   1350 	 */
   1351 	pm->pm_l1 = l1;
   1352 	pm->pm_domain = domain + 1;
   1353 #endif
   1354 }
   1355 
   1356 /*
   1357  * Free an L1 translation table.
   1358  * This is called at pmap destruction time.
   1359  */
   1360 static void
   1361 pmap_free_l1(pmap_t pm)
   1362 {
   1363 #ifdef ARM_MMU_EXTENDED
   1364 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
   1365 	struct vm_page *pg = PHYS_TO_VM_PAGE(pm->pm_l1_pa);
   1366 	uvm_pagefree(pg);
   1367 #else
   1368 	uvm_km_free(kernel_map, (vaddr_t)pm->pm_l1, PAGE_SIZE, UVM_KMF_WIRED);
   1369 #endif
   1370 	pm->pm_l1 = NULL;
   1371 	pm->pm_l1_pa = 0;
   1372 #else
   1373 	struct l1_ttable *l1 = pm->pm_l1;
   1374 
   1375 	mutex_spin_enter(&l1_lru_lock);
   1376 
   1377 	/*
   1378 	 * If this L1 is currently on the LRU list, remove it.
   1379 	 */
   1380 	if (l1->l1_domain_use_count < PMAP_DOMAINS)
   1381 		TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
   1382 
   1383 	/*
   1384 	 * Free up the domain number which was allocated to the pmap
   1385 	 */
   1386 	l1->l1_domain_free[pmap_domain(pm) - 1] = l1->l1_domain_first;
   1387 	l1->l1_domain_first = pmap_domain(pm) - 1;
   1388 	l1->l1_domain_use_count--;
   1389 
   1390 	/*
   1391 	 * The L1 now must have at least 1 free domain, so add
   1392 	 * it back to the LRU list. If the use count is zero,
   1393 	 * put it at the head of the list, otherwise it goes
   1394 	 * to the tail.
   1395 	 */
   1396 	if (l1->l1_domain_use_count == 0)
   1397 		TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru);
   1398 	else
   1399 		TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
   1400 
   1401 	mutex_spin_exit(&l1_lru_lock);
   1402 #endif /* ARM_MMU_EXTENDED */
   1403 }
   1404 
   1405 #ifndef ARM_MMU_EXTENDED
   1406 static inline void
   1407 pmap_use_l1(pmap_t pm)
   1408 {
   1409 	struct l1_ttable *l1;
   1410 
   1411 	/*
   1412 	 * Do nothing if we're in interrupt context.
   1413 	 * Access to an L1 by the kernel pmap must not affect
   1414 	 * the LRU list.
   1415 	 */
   1416 	if (cpu_intr_p() || pm == pmap_kernel())
   1417 		return;
   1418 
   1419 	l1 = pm->pm_l1;
   1420 
   1421 	/*
   1422 	 * If the L1 is not currently on the LRU list, just return
   1423 	 */
   1424 	if (l1->l1_domain_use_count == PMAP_DOMAINS)
   1425 		return;
   1426 
   1427 	mutex_spin_enter(&l1_lru_lock);
   1428 
   1429 	/*
   1430 	 * Check the use count again, now that we've acquired the lock
   1431 	 */
   1432 	if (l1->l1_domain_use_count == PMAP_DOMAINS) {
   1433 		mutex_spin_exit(&l1_lru_lock);
   1434 		return;
   1435 	}
   1436 
   1437 	/*
   1438 	 * Move the L1 to the back of the LRU list
   1439 	 */
   1440 	TAILQ_REMOVE(&l1_lru_list, l1, l1_lru);
   1441 	TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
   1442 
   1443 	mutex_spin_exit(&l1_lru_lock);
   1444 }
   1445 #endif /* !ARM_MMU_EXTENDED */
   1446 
   1447 /*
   1448  * void pmap_free_l2_ptp(pt_entry_t *, paddr_t *)
   1449  *
   1450  * Free an L2 descriptor table.
   1451  */
   1452 static inline void
   1453 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT)
   1454 pmap_free_l2_ptp(bool need_sync, pt_entry_t *l2, paddr_t pa)
   1455 #else
   1456 pmap_free_l2_ptp(pt_entry_t *l2, paddr_t pa)
   1457 #endif
   1458 {
   1459 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT)
   1460 	/*
   1461 	 * Note: With a write-back cache, we may need to sync this
   1462 	 * L2 table before re-using it.
   1463 	 * This is because it may have belonged to a non-current
   1464 	 * pmap, in which case the cache syncs would have been
   1465 	 * skipped for the pages that were being unmapped. If the
   1466 	 * L2 table were then to be immediately re-allocated to
   1467 	 * the *current* pmap, it may well contain stale mappings
   1468 	 * which have not yet been cleared by a cache write-back
   1469 	 * and so would still be visible to the mmu.
   1470 	 */
   1471 	if (need_sync)
   1472 		PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
   1473 #endif /* PMAP_INCLUDE_PTE_SYNC && PMAP_CACHE_VIVT */
   1474 	pool_cache_put_paddr(&pmap_l2ptp_cache, (void *)l2, pa);
   1475 }
   1476 
   1477 /*
   1478  * Returns a pointer to the L2 bucket associated with the specified pmap
   1479  * and VA, or NULL if no L2 bucket exists for the address.
   1480  */
   1481 static inline struct l2_bucket *
   1482 pmap_get_l2_bucket(pmap_t pm, vaddr_t va)
   1483 {
   1484 	const size_t l1slot = l1pte_index(va);
   1485 	struct l2_dtable *l2;
   1486 	struct l2_bucket *l2b;
   1487 
   1488 	if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL ||
   1489 	    (l2b = &l2->l2_bucket[L2_BUCKET(l1slot)])->l2b_kva == NULL)
   1490 		return (NULL);
   1491 
   1492 	return (l2b);
   1493 }
   1494 
   1495 /*
   1496  * Returns a pointer to the L2 bucket associated with the specified pmap
   1497  * and VA.
   1498  *
   1499  * If no L2 bucket exists, perform the necessary allocations to put an L2
   1500  * bucket/page table in place.
   1501  *
   1502  * Note that if a new L2 bucket/page was allocated, the caller *must*
   1503  * increment the bucket occupancy counter appropriately *before*
   1504  * releasing the pmap's lock to ensure no other thread or cpu deallocates
   1505  * the bucket/page in the meantime.
   1506  */
   1507 static struct l2_bucket *
   1508 pmap_alloc_l2_bucket(pmap_t pm, vaddr_t va)
   1509 {
   1510 	const size_t l1slot = l1pte_index(va);
   1511 	struct l2_dtable *l2;
   1512 
   1513 	if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) {
   1514 		/*
   1515 		 * No mapping at this address, as there is
   1516 		 * no entry in the L1 table.
   1517 		 * Need to allocate a new l2_dtable.
   1518 		 */
   1519 		if ((l2 = pmap_alloc_l2_dtable()) == NULL)
   1520 			return (NULL);
   1521 
   1522 		/*
   1523 		 * Link it into the parent pmap
   1524 		 */
   1525 		pm->pm_l2[L2_IDX(l1slot)] = l2;
   1526 	}
   1527 
   1528 	struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)];
   1529 
   1530 	/*
   1531 	 * Fetch pointer to the L2 page table associated with the address.
   1532 	 */
   1533 	if (l2b->l2b_kva == NULL) {
   1534 		pt_entry_t *ptep;
   1535 
   1536 		/*
   1537 		 * No L2 page table has been allocated. Chances are, this
   1538 		 * is because we just allocated the l2_dtable, above.
   1539 		 */
   1540 		if ((ptep = pmap_alloc_l2_ptp(&l2b->l2b_pa)) == NULL) {
   1541 			/*
   1542 			 * Oops, no more L2 page tables available at this
   1543 			 * time. We may need to deallocate the l2_dtable
   1544 			 * if we allocated a new one above.
   1545 			 */
   1546 			if (l2->l2_occupancy == 0) {
   1547 				pm->pm_l2[L2_IDX(l1slot)] = NULL;
   1548 				pmap_free_l2_dtable(l2);
   1549 			}
   1550 			return (NULL);
   1551 		}
   1552 
   1553 		l2->l2_occupancy++;
   1554 		l2b->l2b_kva = ptep;
   1555 		l2b->l2b_l1slot = l1slot;
   1556 
   1557 #ifdef ARM_MMU_EXTENDED
   1558 		/*
   1559 		 * We know there will be a mapping here, so simply
   1560 		 * enter this PTP into the L1 now.
   1561 		 */
   1562 		pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot;
   1563 		pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa
   1564 		    | L1_C_DOM(pmap_domain(pm));
   1565 		KASSERT(*pdep == 0);
   1566 		l1pte_setone(pdep, npde);
   1567 		PTE_SYNC(pdep);
   1568 #endif
   1569 	}
   1570 
   1571 	return (l2b);
   1572 }
   1573 
   1574 /*
   1575  * One or more mappings in the specified L2 descriptor table have just been
   1576  * invalidated.
   1577  *
   1578  * Garbage collect the metadata and descriptor table itself if necessary.
   1579  *
   1580  * The pmap lock must be acquired when this is called (not necessary
   1581  * for the kernel pmap).
   1582  */
   1583 static void
   1584 pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count)
   1585 {
   1586 	KDASSERT(count <= l2b->l2b_occupancy);
   1587 
   1588 	/*
   1589 	 * Update the bucket's reference count according to how many
   1590 	 * PTEs the caller has just invalidated.
   1591 	 */
   1592 	l2b->l2b_occupancy -= count;
   1593 
   1594 	/*
   1595 	 * Note:
   1596 	 *
   1597 	 * Level 2 page tables allocated to the kernel pmap are never freed
   1598 	 * as that would require checking all Level 1 page tables and
   1599 	 * removing any references to the Level 2 page table. See also the
   1600 	 * comment elsewhere about never freeing bootstrap L2 descriptors.
   1601 	 *
   1602 	 * We make do with just invalidating the mapping in the L2 table.
   1603 	 *
   1604 	 * This isn't really a big deal in practice and, in fact, leads
   1605 	 * to a performance win over time as we don't need to continually
   1606 	 * alloc/free.
   1607 	 */
   1608 	if (l2b->l2b_occupancy > 0 || pm == pmap_kernel())
   1609 		return;
   1610 
   1611 	/*
   1612 	 * There are no more valid mappings in this level 2 page table.
   1613 	 * Go ahead and NULL-out the pointer in the bucket, then
   1614 	 * free the page table.
   1615 	 */
   1616 	const size_t l1slot = l2b->l2b_l1slot;
   1617 	pt_entry_t * const ptep = l2b->l2b_kva;
   1618 	l2b->l2b_kva = NULL;
   1619 
   1620 	pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot;
   1621 	pd_entry_t pde __diagused = *pdep;
   1622 
   1623 #ifdef ARM_MMU_EXTENDED
   1624 	/*
   1625 	 * Invalidate the L1 slot.
   1626 	 */
   1627 	KASSERT((pde & L1_TYPE_MASK) == L1_TYPE_C);
   1628 #else
   1629 	/*
   1630 	 * If the L1 slot matches the pmap's domain number, then invalidate it.
   1631 	 */
   1632 	if ((pde & (L1_C_DOM_MASK|L1_TYPE_MASK))
   1633 	    == (L1_C_DOM(pmap_domain(pm))|L1_TYPE_C)) {
   1634 #endif
   1635 		l1pte_setone(pdep, 0);
   1636 		PDE_SYNC(pdep);
   1637 #ifndef ARM_MMU_EXTENDED
   1638 	}
   1639 #endif
   1640 
   1641 	/*
   1642 	 * Release the L2 descriptor table back to the pool cache.
   1643 	 */
   1644 #if defined(PMAP_INCLUDE_PTE_SYNC) && defined(PMAP_CACHE_VIVT)
   1645 	pmap_free_l2_ptp(!pmap_is_cached(pm), ptep, l2b->l2b_pa);
   1646 #else
   1647 	pmap_free_l2_ptp(ptep, l2b->l2b_pa);
   1648 #endif
   1649 
   1650 	/*
   1651 	 * Update the reference count in the associated l2_dtable
   1652 	 */
   1653 	struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)];
   1654 	if (--l2->l2_occupancy > 0)
   1655 		return;
   1656 
   1657 	/*
   1658 	 * There are no more valid mappings in any of the Level 1
   1659 	 * slots managed by this l2_dtable. Go ahead and NULL-out
   1660 	 * the pointer in the parent pmap and free the l2_dtable.
   1661 	 */
   1662 	pm->pm_l2[L2_IDX(l1slot)] = NULL;
   1663 	pmap_free_l2_dtable(l2);
   1664 }
   1665 
   1666 /*
   1667  * Pool cache constructors for L2 descriptor tables, metadata and pmap
   1668  * structures.
   1669  */
   1670 static int
   1671 pmap_l2ptp_ctor(void *arg, void *v, int flags)
   1672 {
   1673 #ifndef PMAP_INCLUDE_PTE_SYNC
   1674 	vaddr_t va = (vaddr_t)v & ~PGOFSET;
   1675 
   1676 	/*
   1677 	 * The mappings for these page tables were initially made using
   1678 	 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache-
   1679 	 * mode will not be right for page table mappings. To avoid
   1680 	 * polluting the pmap_kenter_pa() code with a special case for
   1681 	 * page tables, we simply fix up the cache-mode here if it's not
   1682 	 * correct.
   1683 	 */
   1684 	if (pte_l2_s_cache_mode != pte_l2_s_cache_mode_pt) {
   1685 		const struct l2_bucket * const l2b =
   1686 		    pmap_get_l2_bucket(pmap_kernel(), va);
   1687 		KASSERTMSG(l2b != NULL, "%#lx", va);
   1688 		pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   1689 		const pt_entry_t opte = *ptep;
   1690 
   1691 		if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
   1692 			/*
   1693 			 * Page tables must have the cache-mode set correctly.
   1694 			 */
   1695 			const pt_entry_t npte = (pte & ~L2_S_CACHE_MASK)
   1696 			    | pte_l2_s_cache_mode_pt;
   1697 			l2pte_set(ptep, npte, opte);
   1698 			PTE_SYNC(ptep);
   1699 			cpu_tlb_flushD_SE(va);
   1700 			cpu_cpwait();
   1701 		}
   1702 	}
   1703 #endif
   1704 
   1705 	memset(v, 0, L2_TABLE_SIZE_REAL);
   1706 	PTE_SYNC_RANGE(v, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
   1707 	return (0);
   1708 }
   1709 
   1710 static int
   1711 pmap_l2dtable_ctor(void *arg, void *v, int flags)
   1712 {
   1713 
   1714 	memset(v, 0, sizeof(struct l2_dtable));
   1715 	return (0);
   1716 }
   1717 
   1718 static int
   1719 pmap_pmap_ctor(void *arg, void *v, int flags)
   1720 {
   1721 
   1722 	memset(v, 0, sizeof(struct pmap));
   1723 	return (0);
   1724 }
   1725 
   1726 static void
   1727 pmap_pinit(pmap_t pm)
   1728 {
   1729 #ifndef ARM_HAS_VBAR
   1730 	struct l2_bucket *l2b;
   1731 
   1732 	if (vector_page < KERNEL_BASE) {
   1733 		/*
   1734 		 * Map the vector page.
   1735 		 */
   1736 		pmap_enter(pm, vector_page, systempage.pv_pa,
   1737 		    VM_PROT_READ | VM_PROT_EXECUTE,
   1738 		    VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED);
   1739 		pmap_update(pm);
   1740 
   1741 		pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page);
   1742 		l2b = pmap_get_l2_bucket(pm, vector_page);
   1743 		KASSERTMSG(l2b != NULL, "%#lx", vector_page);
   1744 		pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO |
   1745 		    L1_C_DOM(pmap_domain(pm));
   1746 	} else
   1747 		pm->pm_pl1vec = NULL;
   1748 #endif
   1749 }
   1750 
   1751 #ifdef PMAP_CACHE_VIVT
   1752 /*
   1753  * Since we have a virtually indexed cache, we may need to inhibit caching if
   1754  * there is more than one mapping and at least one of them is writable.
   1755  * Since we purge the cache on every context switch, we only need to check for
   1756  * other mappings within the same pmap, or kernel_pmap.
   1757  * This function is also called when a page is unmapped, to possibly reenable
   1758  * caching on any remaining mappings.
   1759  *
   1760  * The code implements the following logic, where:
   1761  *
   1762  * KW = # of kernel read/write pages
   1763  * KR = # of kernel read only pages
   1764  * UW = # of user read/write pages
   1765  * UR = # of user read only pages
   1766  *
   1767  * KC = kernel mapping is cacheable
   1768  * UC = user mapping is cacheable
   1769  *
   1770  *               KW=0,KR=0  KW=0,KR>0  KW=1,KR=0  KW>1,KR>=0
   1771  *             +---------------------------------------------
   1772  * UW=0,UR=0   | ---        KC=1       KC=1       KC=0
   1773  * UW=0,UR>0   | UC=1       KC=1,UC=1  KC=0,UC=0  KC=0,UC=0
   1774  * UW=1,UR=0   | UC=1       KC=0,UC=0  KC=0,UC=0  KC=0,UC=0
   1775  * UW>1,UR>=0  | UC=0       KC=0,UC=0  KC=0,UC=0  KC=0,UC=0
   1776  */
   1777 
   1778 static const int pmap_vac_flags[4][4] = {
   1779 	{-1,		0,		0,		PVF_KNC},
   1780 	{0,		0,		PVF_NC,		PVF_NC},
   1781 	{0,		PVF_NC,		PVF_NC,		PVF_NC},
   1782 	{PVF_UNC,	PVF_NC,		PVF_NC,		PVF_NC}
   1783 };
   1784 
   1785 static inline int
   1786 pmap_get_vac_flags(const struct vm_page_md *md)
   1787 {
   1788 	int kidx, uidx;
   1789 
   1790 	kidx = 0;
   1791 	if (md->kro_mappings || md->krw_mappings > 1)
   1792 		kidx |= 1;
   1793 	if (md->krw_mappings)
   1794 		kidx |= 2;
   1795 
   1796 	uidx = 0;
   1797 	if (md->uro_mappings || md->urw_mappings > 1)
   1798 		uidx |= 1;
   1799 	if (md->urw_mappings)
   1800 		uidx |= 2;
   1801 
   1802 	return (pmap_vac_flags[uidx][kidx]);
   1803 }
   1804 
   1805 static inline void
   1806 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va)
   1807 {
   1808 	int nattr;
   1809 
   1810 	nattr = pmap_get_vac_flags(md);
   1811 
   1812 	if (nattr < 0) {
   1813 		md->pvh_attrs &= ~PVF_NC;
   1814 		return;
   1815 	}
   1816 
   1817 	if (nattr == 0 && (md->pvh_attrs & PVF_NC) == 0)
   1818 		return;
   1819 
   1820 	if (pm == pmap_kernel())
   1821 		pmap_vac_me_kpmap(md, pa, pm, va);
   1822 	else
   1823 		pmap_vac_me_user(md, pa, pm, va);
   1824 
   1825 	md->pvh_attrs = (md->pvh_attrs & ~PVF_NC) | nattr;
   1826 }
   1827 
   1828 static void
   1829 pmap_vac_me_kpmap(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va)
   1830 {
   1831 	u_int u_cacheable, u_entries;
   1832 	struct pv_entry *pv;
   1833 	pmap_t last_pmap = pm;
   1834 
   1835 	/*
   1836 	 * Pass one, see if there are both kernel and user pmaps for
   1837 	 * this page.  Calculate whether there are user-writable or
   1838 	 * kernel-writable pages.
   1839 	 */
   1840 	u_cacheable = 0;
   1841 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   1842 		if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0)
   1843 			u_cacheable++;
   1844 	}
   1845 
   1846 	u_entries = md->urw_mappings + md->uro_mappings;
   1847 
   1848 	/*
   1849 	 * We know we have just been updating a kernel entry, so if
   1850 	 * all user pages are already cacheable, then there is nothing
   1851 	 * further to do.
   1852 	 */
   1853 	if (md->k_mappings == 0 && u_cacheable == u_entries)
   1854 		return;
   1855 
   1856 	if (u_entries) {
   1857 		/*
   1858 		 * Scan over the list again, for each entry, if it
   1859 		 * might not be set correctly, call pmap_vac_me_user
   1860 		 * to recalculate the settings.
   1861 		 */
   1862 		SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   1863 			/*
   1864 			 * We know kernel mappings will get set
   1865 			 * correctly in other calls.  We also know
   1866 			 * that if the pmap is the same as last_pmap
   1867 			 * then we've just handled this entry.
   1868 			 */
   1869 			if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap)
   1870 				continue;
   1871 
   1872 			/*
   1873 			 * If there are kernel entries and this page
   1874 			 * is writable but non-cacheable, then we can
   1875 			 * skip this entry also.
   1876 			 */
   1877 			if (md->k_mappings &&
   1878 			    (pv->pv_flags & (PVF_NC | PVF_WRITE)) ==
   1879 			    (PVF_NC | PVF_WRITE))
   1880 				continue;
   1881 
   1882 			/*
   1883 			 * Similarly if there are no kernel-writable
   1884 			 * entries and the page is already
   1885 			 * read-only/cacheable.
   1886 			 */
   1887 			if (md->krw_mappings == 0 &&
   1888 			    (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0)
   1889 				continue;
   1890 
   1891 			/*
   1892 			 * For some of the remaining cases, we know
   1893 			 * that we must recalculate, but for others we
   1894 			 * can't tell if they are correct or not, so
   1895 			 * we recalculate anyway.
   1896 			 */
   1897 			pmap_vac_me_user(md, pa, (last_pmap = pv->pv_pmap), 0);
   1898 		}
   1899 
   1900 		if (md->k_mappings == 0)
   1901 			return;
   1902 	}
   1903 
   1904 	pmap_vac_me_user(md, pa, pm, va);
   1905 }
   1906 
   1907 static void
   1908 pmap_vac_me_user(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va)
   1909 {
   1910 	pmap_t kpmap = pmap_kernel();
   1911 	struct pv_entry *pv, *npv = NULL;
   1912 	u_int entries = 0;
   1913 	u_int writable = 0;
   1914 	u_int cacheable_entries = 0;
   1915 	u_int kern_cacheable = 0;
   1916 	u_int other_writable = 0;
   1917 
   1918 	/*
   1919 	 * Count mappings and writable mappings in this pmap.
   1920 	 * Include kernel mappings as part of our own.
   1921 	 * Keep a pointer to the first one.
   1922 	 */
   1923 	npv = NULL;
   1924 	KASSERT(pmap_page_locked_p(md));
   1925 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   1926 		/* Count mappings in the same pmap */
   1927 		if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) {
   1928 			if (entries++ == 0)
   1929 				npv = pv;
   1930 
   1931 			/* Cacheable mappings */
   1932 			if ((pv->pv_flags & PVF_NC) == 0) {
   1933 				cacheable_entries++;
   1934 				if (kpmap == pv->pv_pmap)
   1935 					kern_cacheable++;
   1936 			}
   1937 
   1938 			/* Writable mappings */
   1939 			if (pv->pv_flags & PVF_WRITE)
   1940 				++writable;
   1941 		} else
   1942 		if (pv->pv_flags & PVF_WRITE)
   1943 			other_writable = 1;
   1944 	}
   1945 
   1946 	/*
   1947 	 * Enable or disable caching as necessary.
   1948 	 * Note: the first entry might be part of the kernel pmap,
   1949 	 * so we can't assume this is indicative of the state of the
   1950 	 * other (maybe non-kpmap) entries.
   1951 	 */
   1952 	if ((entries > 1 && writable) ||
   1953 	    (entries > 0 && pm == kpmap && other_writable)) {
   1954 		if (cacheable_entries == 0) {
   1955 			return;
   1956 		}
   1957 
   1958 		for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) {
   1959 			if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) ||
   1960 			    (pv->pv_flags & PVF_NC))
   1961 				continue;
   1962 
   1963 			pv->pv_flags |= PVF_NC;
   1964 
   1965 			struct l2_bucket * const l2b
   1966 			    = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
   1967 			KASSERTMSG(l2b != NULL, "%#lx", va);
   1968 			pt_entry_t * const ptep
   1969 			    = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
   1970 			const pt_entry_t opte = *ptep;
   1971 			pt_entry_t npte = opte & ~L2_S_CACHE_MASK;
   1972 
   1973 			if ((va != pv->pv_va || pm != pv->pv_pmap)
   1974 			    && l2pte_valid_p(npte)) {
   1975 #ifdef PMAP_CACHE_VIVT
   1976 				pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va,
   1977 				    true, pv->pv_flags);
   1978 #endif
   1979 				pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va,
   1980 				    pv->pv_flags);
   1981 			}
   1982 
   1983 			l2pte_set(ptep, npte, opte);
   1984 			PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
   1985 		}
   1986 		cpu_cpwait();
   1987 	} else
   1988 	if (entries > cacheable_entries) {
   1989 		/*
   1990 		 * Turn cacheing back on for some pages.  If it is a kernel
   1991 		 * page, only do so if there are no other writable pages.
   1992 		 */
   1993 		for (pv = npv; pv; pv = SLIST_NEXT(pv, pv_link)) {
   1994 			if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap &&
   1995 			    (kpmap != pv->pv_pmap || other_writable)))
   1996 				continue;
   1997 
   1998 			pv->pv_flags &= ~PVF_NC;
   1999 
   2000 			struct l2_bucket * const l2b
   2001 			    = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
   2002 			KASSERTMSG(l2b != NULL, "%#lx", va);
   2003 			pt_entry_t * const ptep
   2004 			    = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
   2005 			const pt_entry_t opte = *ptep;
   2006 			pt_entry_t npte = (opte & ~L2_S_CACHE_MASK)
   2007 			    | pte_l2_s_cache_mode;
   2008 
   2009 			if (l2pte_valid_p(opte)) {
   2010 				pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va,
   2011 				    pv->pv_flags);
   2012 			}
   2013 
   2014 			l2pte_set(ptep, npte, opte);
   2015 			PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
   2016 		}
   2017 	}
   2018 }
   2019 #endif
   2020 
   2021 #ifdef PMAP_CACHE_VIPT
   2022 static void
   2023 pmap_vac_me_harder(struct vm_page_md *md, paddr_t pa, pmap_t pm, vaddr_t va)
   2024 {
   2025 #ifndef ARM_MMU_EXTENDED
   2026 	struct pv_entry *pv;
   2027 	vaddr_t tst_mask;
   2028 	bool bad_alias;
   2029 	const u_int
   2030 	    rw_mappings = md->urw_mappings + md->krw_mappings,
   2031 	    ro_mappings = md->uro_mappings + md->kro_mappings;
   2032 
   2033 	/* do we need to do anything? */
   2034 	if (arm_cache_prefer_mask == 0)
   2035 		return;
   2036 
   2037 	NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: md=%p, pmap=%p va=%08lx\n",
   2038 	    md, pm, va));
   2039 
   2040 	KASSERT(!va || pm);
   2041 	KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2042 
   2043 	/* Already a conflict? */
   2044 	if (__predict_false(md->pvh_attrs & PVF_NC)) {
   2045 		/* just an add, things are already non-cached */
   2046 		KASSERT(!(md->pvh_attrs & PVF_DIRTY));
   2047 		KASSERT(!(md->pvh_attrs & PVF_MULTCLR));
   2048 		bad_alias = false;
   2049 		if (va) {
   2050 			PMAPCOUNT(vac_color_none);
   2051 			bad_alias = true;
   2052 			KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2053 			goto fixup;
   2054 		}
   2055 		pv = SLIST_FIRST(&md->pvh_list);
   2056 		/* the list can't be empty because it would be cachable */
   2057 		if (md->pvh_attrs & PVF_KMPAGE) {
   2058 			tst_mask = md->pvh_attrs;
   2059 		} else {
   2060 			KASSERT(pv);
   2061 			tst_mask = pv->pv_va;
   2062 			pv = SLIST_NEXT(pv, pv_link);
   2063 		}
   2064 		/*
   2065 		 * Only check for a bad alias if we have writable mappings.
   2066 		 */
   2067 		tst_mask &= arm_cache_prefer_mask;
   2068 		if (rw_mappings > 0) {
   2069 			for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) {
   2070 				/* if there's a bad alias, stop checking. */
   2071 				if (tst_mask != (pv->pv_va & arm_cache_prefer_mask))
   2072 					bad_alias = true;
   2073 			}
   2074 			md->pvh_attrs |= PVF_WRITE;
   2075 			if (!bad_alias)
   2076 				md->pvh_attrs |= PVF_DIRTY;
   2077 		} else {
   2078 			/*
   2079 			 * We have only read-only mappings.  Let's see if there
   2080 			 * are multiple colors in use or if we mapped a KMPAGE.
   2081 			 * If the latter, we have a bad alias.  If the former,
   2082 			 * we need to remember that.
   2083 			 */
   2084 			for (; pv; pv = SLIST_NEXT(pv, pv_link)) {
   2085 				if (tst_mask != (pv->pv_va & arm_cache_prefer_mask)) {
   2086 					if (md->pvh_attrs & PVF_KMPAGE)
   2087 						bad_alias = true;
   2088 					break;
   2089 				}
   2090 			}
   2091 			md->pvh_attrs &= ~PVF_WRITE;
   2092 			/*
   2093 			 * No KMPAGE and we exited early, so we must have
   2094 			 * multiple color mappings.
   2095 			 */
   2096 			if (!bad_alias && pv != NULL)
   2097 				md->pvh_attrs |= PVF_MULTCLR;
   2098 		}
   2099 
   2100 		/* If no conflicting colors, set everything back to cached */
   2101 		if (!bad_alias) {
   2102 #ifdef DEBUG
   2103 			if ((md->pvh_attrs & PVF_WRITE)
   2104 			    || ro_mappings < 2) {
   2105 				SLIST_FOREACH(pv, &md->pvh_list, pv_link)
   2106 					KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0);
   2107 			}
   2108 #endif
   2109 			md->pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC;
   2110 			md->pvh_attrs |= tst_mask | PVF_COLORED;
   2111 			/*
   2112 			 * Restore DIRTY bit if page is modified
   2113 			 */
   2114 			if (md->pvh_attrs & PVF_DMOD)
   2115 				md->pvh_attrs |= PVF_DIRTY;
   2116 			PMAPCOUNT(vac_color_restore);
   2117 		} else {
   2118 			KASSERT(SLIST_FIRST(&md->pvh_list) != NULL);
   2119 			KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL);
   2120 		}
   2121 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2122 		KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2123 	} else if (!va) {
   2124 		KASSERT(pmap_is_page_colored_p(md));
   2125 		KASSERT(!(md->pvh_attrs & PVF_WRITE)
   2126 		    || (md->pvh_attrs & PVF_DIRTY));
   2127 		if (rw_mappings == 0) {
   2128 			md->pvh_attrs &= ~PVF_WRITE;
   2129 			if (ro_mappings == 1
   2130 			    && (md->pvh_attrs & PVF_MULTCLR)) {
   2131 				/*
   2132 				 * If this is the last readonly mapping
   2133 				 * but it doesn't match the current color
   2134 				 * for the page, change the current color
   2135 				 * to match this last readonly mapping.
   2136 				 */
   2137 				pv = SLIST_FIRST(&md->pvh_list);
   2138 				tst_mask = (md->pvh_attrs ^ pv->pv_va)
   2139 				    & arm_cache_prefer_mask;
   2140 				if (tst_mask) {
   2141 					md->pvh_attrs ^= tst_mask;
   2142 					PMAPCOUNT(vac_color_change);
   2143 				}
   2144 			}
   2145 		}
   2146 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2147 		KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2148 		return;
   2149 	} else if (!pmap_is_page_colored_p(md)) {
   2150 		/* not colored so we just use its color */
   2151 		KASSERT(md->pvh_attrs & (PVF_WRITE|PVF_DIRTY));
   2152 		KASSERT(!(md->pvh_attrs & PVF_MULTCLR));
   2153 		PMAPCOUNT(vac_color_new);
   2154 		md->pvh_attrs &= PAGE_SIZE - 1;
   2155 		md->pvh_attrs |= PVF_COLORED
   2156 		    | (va & arm_cache_prefer_mask)
   2157 		    | (rw_mappings > 0 ? PVF_WRITE : 0);
   2158 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2159 		KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2160 		return;
   2161 	} else if (((md->pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) {
   2162 		bad_alias = false;
   2163 		if (rw_mappings > 0) {
   2164 			/*
   2165 			 * We now have writeable mappings and if we have
   2166 			 * readonly mappings in more than once color, we have
   2167 			 * an aliasing problem.  Regardless mark the page as
   2168 			 * writeable.
   2169 			 */
   2170 			if (md->pvh_attrs & PVF_MULTCLR) {
   2171 				if (ro_mappings < 2) {
   2172 					/*
   2173 					 * If we only have less than two
   2174 					 * read-only mappings, just flush the
   2175 					 * non-primary colors from the cache.
   2176 					 */
   2177 					pmap_flush_page(md, pa,
   2178 					    PMAP_FLUSH_SECONDARY);
   2179 				} else {
   2180 					bad_alias = true;
   2181 				}
   2182 			}
   2183 			md->pvh_attrs |= PVF_WRITE;
   2184 		}
   2185 		/* If no conflicting colors, set everything back to cached */
   2186 		if (!bad_alias) {
   2187 #ifdef DEBUG
   2188 			if (rw_mappings > 0
   2189 			    || (md->pvh_attrs & PMAP_KMPAGE)) {
   2190 				tst_mask = md->pvh_attrs & arm_cache_prefer_mask;
   2191 				SLIST_FOREACH(pv, &md->pvh_list, pv_link)
   2192 					KDASSERT(((tst_mask ^ pv->pv_va) & arm_cache_prefer_mask) == 0);
   2193 			}
   2194 #endif
   2195 			if (SLIST_EMPTY(&md->pvh_list))
   2196 				PMAPCOUNT(vac_color_reuse);
   2197 			else
   2198 				PMAPCOUNT(vac_color_ok);
   2199 
   2200 			/* matching color, just return */
   2201 			KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2202 			KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2203 			return;
   2204 		}
   2205 		KASSERT(SLIST_FIRST(&md->pvh_list) != NULL);
   2206 		KASSERT(SLIST_NEXT(SLIST_FIRST(&md->pvh_list), pv_link) != NULL);
   2207 
   2208 		/* color conflict.  evict from cache. */
   2209 
   2210 		pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY);
   2211 		md->pvh_attrs &= ~PVF_COLORED;
   2212 		md->pvh_attrs |= PVF_NC;
   2213 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2214 		KASSERT(!(md->pvh_attrs & PVF_MULTCLR));
   2215 		PMAPCOUNT(vac_color_erase);
   2216 	} else if (rw_mappings == 0
   2217 		   && (md->pvh_attrs & PVF_KMPAGE) == 0) {
   2218 		KASSERT((md->pvh_attrs & PVF_WRITE) == 0);
   2219 
   2220 		/*
   2221 		 * If the page has dirty cache lines, clean it.
   2222 		 */
   2223 		if (md->pvh_attrs & PVF_DIRTY)
   2224 			pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY);
   2225 
   2226 		/*
   2227 		 * If this is the first remapping (we know that there are no
   2228 		 * writeable mappings), then this is a simple color change.
   2229 		 * Otherwise this is a seconary r/o mapping, which means
   2230 		 * we don't have to do anything.
   2231 		 */
   2232 		if (ro_mappings == 1) {
   2233 			KASSERT(((md->pvh_attrs ^ va) & arm_cache_prefer_mask) != 0);
   2234 			md->pvh_attrs &= PAGE_SIZE - 1;
   2235 			md->pvh_attrs |= (va & arm_cache_prefer_mask);
   2236 			PMAPCOUNT(vac_color_change);
   2237 		} else {
   2238 			PMAPCOUNT(vac_color_blind);
   2239 		}
   2240 		md->pvh_attrs |= PVF_MULTCLR;
   2241 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2242 		KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2243 		return;
   2244 	} else {
   2245 		if (rw_mappings > 0)
   2246 			md->pvh_attrs |= PVF_WRITE;
   2247 
   2248 		/* color conflict.  evict from cache. */
   2249 		pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY);
   2250 
   2251 		/* the list can't be empty because this was a enter/modify */
   2252 		pv = SLIST_FIRST(&md->pvh_list);
   2253 		if ((md->pvh_attrs & PVF_KMPAGE) == 0) {
   2254 			KASSERT(pv);
   2255 			/*
   2256 			 * If there's only one mapped page, change color to the
   2257 			 * page's new color and return.  Restore the DIRTY bit
   2258 			 * that was erased by pmap_flush_page.
   2259 			 */
   2260 			if (SLIST_NEXT(pv, pv_link) == NULL) {
   2261 				md->pvh_attrs &= PAGE_SIZE - 1;
   2262 				md->pvh_attrs |= (va & arm_cache_prefer_mask);
   2263 				if (md->pvh_attrs & PVF_DMOD)
   2264 					md->pvh_attrs |= PVF_DIRTY;
   2265 				PMAPCOUNT(vac_color_change);
   2266 				KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2267 				KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2268 				KASSERT(!(md->pvh_attrs & PVF_MULTCLR));
   2269 				return;
   2270 			}
   2271 		}
   2272 		bad_alias = true;
   2273 		md->pvh_attrs &= ~PVF_COLORED;
   2274 		md->pvh_attrs |= PVF_NC;
   2275 		PMAPCOUNT(vac_color_erase);
   2276 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   2277 	}
   2278 
   2279   fixup:
   2280 	KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
   2281 
   2282 	/*
   2283 	 * Turn cacheing on/off for all pages.
   2284 	 */
   2285 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   2286 		struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap,
   2287 		    pv->pv_va);
   2288 		KASSERTMSG(l2b != NULL, "%#lx", va);
   2289 		pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
   2290 		const pt_entry_t opte = *ptep;
   2291 		pt_entry_t npte = opte & ~L2_S_CACHE_MASK;
   2292 		if (bad_alias) {
   2293 			pv->pv_flags |= PVF_NC;
   2294 		} else {
   2295 			pv->pv_flags &= ~PVF_NC;
   2296 			npte |= pte_l2_s_cache_mode;
   2297 		}
   2298 
   2299 		if (opte == npte)	/* only update is there's a change */
   2300 			continue;
   2301 
   2302 		if (l2pte_valid_p(npte)) {
   2303 			pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags);
   2304 		}
   2305 
   2306 		l2pte_set(ptep, npte, opte);
   2307 		PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
   2308 	}
   2309 #endif /* !ARM_MMU_EXTENDED */
   2310 }
   2311 #endif	/* PMAP_CACHE_VIPT */
   2312 
   2313 
   2314 /*
   2315  * Modify pte bits for all ptes corresponding to the given physical address.
   2316  * We use `maskbits' rather than `clearbits' because we're always passing
   2317  * constants and the latter would require an extra inversion at run-time.
   2318  */
   2319 static void
   2320 pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits)
   2321 {
   2322 	struct pv_entry *pv;
   2323 #ifdef PMAP_CACHE_VIPT
   2324 	const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs);
   2325 #ifdef ARM_MMU_EXTENDED
   2326 	const u_int execbits = (maskbits & PVF_EXEC) ? L2_XS_XN : 0;
   2327 #else
   2328 	const u_int execbits = 0;
   2329 	bool need_vac_me_harder = false;
   2330 	bool need_syncicache = false;
   2331 #endif
   2332 #else
   2333 	const u_int execbits = 0;
   2334 #endif
   2335 
   2336 	NPDEBUG(PDB_BITS,
   2337 	    printf("pmap_clearbit: md %p mask 0x%x\n",
   2338 	    md, maskbits));
   2339 
   2340 #ifdef PMAP_CACHE_VIPT
   2341 	/*
   2342 	 * If we might want to sync the I-cache and we've modified it,
   2343 	 * then we know we definitely need to sync or discard it.
   2344 	 */
   2345 	if (want_syncicache) {
   2346 #ifdef ARM_MMU_EXTENDED
   2347 		if (md->pvh_attrs & PVF_MOD)
   2348 			md->pvh_attrs &= ~PVF_EXEC;
   2349 #else
   2350 		need_syncicache = md->pvh_attrs & PVF_MOD;
   2351 #endif
   2352 	}
   2353 #endif
   2354 	KASSERT(pmap_page_locked_p(md));
   2355 
   2356 	/*
   2357 	 * Clear saved attributes (modify, reference)
   2358 	 */
   2359 	md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF));
   2360 
   2361 	if (SLIST_EMPTY(&md->pvh_list)) {
   2362 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   2363 		if (need_syncicache) {
   2364 			/*
   2365 			 * No one has it mapped, so just discard it.  The next
   2366 			 * exec remapping will cause it to be synced.
   2367 			 */
   2368 			md->pvh_attrs &= ~PVF_EXEC;
   2369 			PMAPCOUNT(exec_discarded_clearbit);
   2370 		}
   2371 #endif
   2372 		return;
   2373 	}
   2374 
   2375 	/*
   2376 	 * Loop over all current mappings setting/clearing as appropos
   2377 	 */
   2378 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   2379 		pmap_t pm = pv->pv_pmap;
   2380 		const vaddr_t va = pv->pv_va;
   2381 		const u_int oflags = pv->pv_flags;
   2382 #ifndef ARM_MMU_EXTENDED
   2383 		/*
   2384 		 * Kernel entries are unmanaged and as such not to be changed.
   2385 		 */
   2386 		if (PV_IS_KENTRY_P(oflags))
   2387 			continue;
   2388 #endif
   2389 		pv->pv_flags &= ~maskbits;
   2390 
   2391 		pmap_release_page_lock(md);
   2392 		pmap_acquire_pmap_lock(pm);
   2393 
   2394 		struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va);
   2395 		if (l2b == NULL) {
   2396 			pmap_release_pmap_lock(pm);
   2397 			pmap_acquire_page_lock(md);
   2398 			continue;
   2399 		}
   2400 		KASSERTMSG(l2b != NULL, "%#lx", va);
   2401 
   2402 		pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   2403 		const pt_entry_t opte = *ptep;
   2404 		pt_entry_t npte = opte | execbits;
   2405 
   2406 		KASSERT((opte & L2_XS_nG) == (pm == pmap_kernel() ? 0 : L2_XS_nG));
   2407 
   2408 		NPDEBUG(PDB_BITS,
   2409 		    printf( "%s: pv %p, pm %p, va 0x%08lx, flag 0x%x\n",
   2410 			__func__, pv, pm, va, oflags));
   2411 
   2412 		if (maskbits & (PVF_WRITE|PVF_MOD)) {
   2413 #ifdef PMAP_CACHE_VIVT
   2414 			if ((oflags & PVF_NC)) {
   2415 				/*
   2416 				 * Entry is not cacheable:
   2417 				 *
   2418 				 * Don't turn caching on again if this is a
   2419 				 * modified emulation. This would be
   2420 				 * inconsitent with the settings created by
   2421 				 * pmap_vac_me_harder(). Otherwise, it's safe
   2422 				 * to re-enable cacheing.
   2423 				 *
   2424 				 * There's no need to call pmap_vac_me_harder()
   2425 				 * here: all pages are losing their write
   2426 				 * permission.
   2427 				 */
   2428 				if (maskbits & PVF_WRITE) {
   2429 					npte |= pte_l2_s_cache_mode;
   2430 					pv->pv_flags &= ~PVF_NC;
   2431 				}
   2432 			} else
   2433 			if (l2pte_writable_p(opte)) {
   2434 				/*
   2435 				 * Entry is writable/cacheable: check if pmap
   2436 				 * is current if it is flush it, otherwise it
   2437 				 * won't be in the cache
   2438 				 */
   2439 				pmap_cache_wbinv_page(pm, va,
   2440 				    (maskbits & PVF_REF) != 0,
   2441 				    oflags|PVF_WRITE);
   2442 			}
   2443 #endif
   2444 
   2445 			/* make the pte read only */
   2446 			npte = l2pte_set_readonly(npte);
   2447 
   2448 			pmap_acquire_page_lock(md);
   2449 #ifdef MULTIPROCESSOR
   2450 			pv = pmap_find_pv(md, pm, va);
   2451 #endif
   2452 			if (pv != NULL && (maskbits & oflags & PVF_WRITE)) {
   2453 				/*
   2454 				 * Keep alias accounting up to date
   2455 				 */
   2456 				if (pm == pmap_kernel()) {
   2457 					md->krw_mappings--;
   2458 					md->kro_mappings++;
   2459 				} else {
   2460 					md->urw_mappings--;
   2461 					md->uro_mappings++;
   2462 				}
   2463 #ifdef PMAP_CACHE_VIPT
   2464 				if (arm_cache_prefer_mask != 0) {
   2465 					if (md->urw_mappings + md->krw_mappings == 0) {
   2466 						md->pvh_attrs &= ~PVF_WRITE;
   2467 					} else {
   2468 						PMAP_VALIDATE_MD_PAGE(md);
   2469 					}
   2470 				}
   2471 #ifndef ARM_MMU_EXTENDED
   2472 				if (want_syncicache)
   2473 					need_syncicache = true;
   2474 				need_vac_me_harder = true;
   2475 #endif
   2476 #endif /* PMAP_CACHE_VIPT */
   2477 			}
   2478 			pmap_release_page_lock(md);
   2479 		}
   2480 
   2481 		if (maskbits & PVF_REF) {
   2482 			if (true
   2483 #ifndef ARM_MMU_EXTENDED
   2484 			    && (oflags & PVF_NC) == 0
   2485 #endif
   2486 			    && (maskbits & (PVF_WRITE|PVF_MOD)) == 0
   2487 			    && l2pte_valid_p(npte)) {
   2488 #ifdef PMAP_CACHE_VIVT
   2489 				/*
   2490 				 * Check npte here; we may have already
   2491 				 * done the wbinv above, and the validity
   2492 				 * of the PTE is the same for opte and
   2493 				 * npte.
   2494 				 */
   2495 				pmap_cache_wbinv_page(pm, va, true, oflags);
   2496 #endif
   2497 			}
   2498 
   2499 			/*
   2500 			 * Make the PTE invalid so that we will take a
   2501 			 * page fault the next time the mapping is
   2502 			 * referenced.
   2503 			 */
   2504 			npte &= ~L2_TYPE_MASK;
   2505 			npte |= L2_TYPE_INV;
   2506 		}
   2507 
   2508 		if (npte != opte) {
   2509 			l2pte_set(ptep, npte, opte);
   2510 			PTE_SYNC(ptep);
   2511 
   2512 			/* Flush the TLB entry if a current pmap. */
   2513 			pmap_tlb_flush_SE(pm, va, oflags);
   2514 		}
   2515 
   2516 		pmap_release_pmap_lock(pm);
   2517 		pmap_acquire_page_lock(md);
   2518 
   2519 		NPDEBUG(PDB_BITS,
   2520 		    printf("pmap_clearbit: pm %p va 0x%lx opte 0x%08x npte 0x%08x\n",
   2521 		    pm, va, opte, npte));
   2522 	}
   2523 
   2524 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   2525 	/*
   2526 	 * If we need to sync the I-cache and we haven't done it yet, do it.
   2527 	 */
   2528 	if (need_syncicache) {
   2529 		pmap_release_page_lock(md);
   2530 		pmap_syncicache_page(md, pa);
   2531 		pmap_acquire_page_lock(md);
   2532 		PMAPCOUNT(exec_synced_clearbit);
   2533 	}
   2534 
   2535 	/*
   2536 	 * If we are changing this to read-only, we need to call vac_me_harder
   2537 	 * so we can change all the read-only pages to cacheable.  We pretend
   2538 	 * this as a page deletion.
   2539 	 */
   2540 	if (need_vac_me_harder) {
   2541 		if (md->pvh_attrs & PVF_NC)
   2542 			pmap_vac_me_harder(md, pa, NULL, 0);
   2543 	}
   2544 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */
   2545 }
   2546 
   2547 /*
   2548  * pmap_clean_page()
   2549  *
   2550  * This is a local function used to work out the best strategy to clean
   2551  * a single page referenced by its entry in the PV table. It's used by
   2552  * pmap_copy_page, pmap_zero page and maybe some others later on.
   2553  *
   2554  * Its policy is effectively:
   2555  *  o If there are no mappings, we don't bother doing anything with the cache.
   2556  *  o If there is one mapping, we clean just that page.
   2557  *  o If there are multiple mappings, we clean the entire cache.
   2558  *
   2559  * So that some functions can be further optimised, it returns 0 if it didn't
   2560  * clean the entire cache, or 1 if it did.
   2561  *
   2562  * XXX One bug in this routine is that if the pv_entry has a single page
   2563  * mapped at 0x00000000 a whole cache clean will be performed rather than
   2564  * just the 1 page. Since this should not occur in everyday use and if it does
   2565  * it will just result in not the most efficient clean for the page.
   2566  */
   2567 #ifdef PMAP_CACHE_VIVT
   2568 static bool
   2569 pmap_clean_page(struct vm_page_md *md, bool is_src)
   2570 {
   2571 	struct pv_entry *pv;
   2572 	pmap_t pm_to_clean = NULL;
   2573 	bool cache_needs_cleaning = false;
   2574 	vaddr_t page_to_clean = 0;
   2575 	u_int flags = 0;
   2576 
   2577 	/*
   2578 	 * Since we flush the cache each time we change to a different
   2579 	 * user vmspace, we only need to flush the page if it is in the
   2580 	 * current pmap.
   2581 	 */
   2582 	KASSERT(pmap_page_locked_p(md));
   2583 	SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
   2584 		if (pmap_is_current(pv->pv_pmap)) {
   2585 			flags |= pv->pv_flags;
   2586 			/*
   2587 			 * The page is mapped non-cacheable in
   2588 			 * this map.  No need to flush the cache.
   2589 			 */
   2590 			if (pv->pv_flags & PVF_NC) {
   2591 #ifdef DIAGNOSTIC
   2592 				KASSERT(!cache_needs_cleaning);
   2593 #endif
   2594 				break;
   2595 			} else if (is_src && (pv->pv_flags & PVF_WRITE) == 0)
   2596 				continue;
   2597 			if (cache_needs_cleaning) {
   2598 				page_to_clean = 0;
   2599 				break;
   2600 			} else {
   2601 				page_to_clean = pv->pv_va;
   2602 				pm_to_clean = pv->pv_pmap;
   2603 			}
   2604 			cache_needs_cleaning = true;
   2605 		}
   2606 	}
   2607 
   2608 	if (page_to_clean) {
   2609 		pmap_cache_wbinv_page(pm_to_clean, page_to_clean,
   2610 		    !is_src, flags | PVF_REF);
   2611 	} else if (cache_needs_cleaning) {
   2612 		pmap_t const pm = curproc->p_vmspace->vm_map.pmap;
   2613 
   2614 		pmap_cache_wbinv_all(pm, flags);
   2615 		return true;
   2616 	}
   2617 	return false;
   2618 }
   2619 #endif
   2620 
   2621 #ifdef PMAP_CACHE_VIPT
   2622 /*
   2623  * Sync a page with the I-cache.  Since this is a VIPT, we must pick the
   2624  * right cache alias to make sure we flush the right stuff.
   2625  */
   2626 void
   2627 pmap_syncicache_page(struct vm_page_md *md, paddr_t pa)
   2628 {
   2629 	pmap_t kpm = pmap_kernel();
   2630 	const size_t way_size = arm_pcache.icache_type == CACHE_TYPE_PIPT
   2631 	    ? PAGE_SIZE
   2632 	    : arm_pcache.icache_way_size;
   2633 
   2634 	NPDEBUG(PDB_EXEC, printf("pmap_syncicache_page: md=%p (attrs=%#x)\n",
   2635 	    md, md->pvh_attrs));
   2636 	/*
   2637 	 * No need to clean the page if it's non-cached.
   2638 	 */
   2639 #ifndef ARM_MMU_EXTENDED
   2640 	if (md->pvh_attrs & PVF_NC)
   2641 		return;
   2642 	KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & PVF_COLORED);
   2643 #endif
   2644 
   2645 	pt_entry_t * const ptep = cpu_cdst_pte(0);
   2646 	const vaddr_t dstp = cpu_cdstp(0);
   2647 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
   2648 	if (way_size <= PAGE_SIZE) {
   2649 		bool ok = false;
   2650 		vaddr_t vdstp = pmap_direct_mapped_phys(pa, &ok, dstp);
   2651 		if (ok) {
   2652 			cpu_icache_sync_range(vdstp, way_size);
   2653 			return;
   2654 		}
   2655 	}
   2656 #endif
   2657 
   2658 	/*
   2659 	 * We don't worry about the color of the exec page, we map the
   2660 	 * same page to pages in the way and then do the icache_sync on
   2661 	 * the entire way making sure we are cleaned.
   2662 	 */
   2663 	const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode
   2664 	    | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE);
   2665 
   2666 	for (size_t i = 0, j = 0; i < way_size;
   2667 	     i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) {
   2668 		pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC);
   2669 		/*
   2670 		 * Set up a PTE with to flush these cache lines.
   2671 		 */
   2672 		l2pte_set(ptep + j, npte, 0);
   2673 	}
   2674 	PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE);
   2675 
   2676 	/*
   2677 	 * Flush it.
   2678 	 */
   2679 	cpu_icache_sync_range(dstp, way_size);
   2680 
   2681 	for (size_t i = 0, j = 0; i < way_size;
   2682 	     i += PAGE_SIZE, j += PAGE_SIZE / L2_S_SIZE) {
   2683 		/*
   2684 		 * Unmap the page(s).
   2685 		 */
   2686 		l2pte_reset(ptep + j);
   2687 		pmap_tlb_flush_SE(kpm, dstp + i, PVF_REF | PVF_EXEC);
   2688 	}
   2689 	PTE_SYNC_RANGE(ptep, way_size / L2_S_SIZE);
   2690 
   2691 	md->pvh_attrs |= PVF_EXEC;
   2692 	PMAPCOUNT(exec_synced);
   2693 }
   2694 
   2695 #ifndef ARM_MMU_EXTENDED
   2696 void
   2697 pmap_flush_page(struct vm_page_md *md, paddr_t pa, enum pmap_flush_op flush)
   2698 {
   2699 	vsize_t va_offset, end_va;
   2700 	bool wbinv_p;
   2701 
   2702 	if (arm_cache_prefer_mask == 0)
   2703 		return;
   2704 
   2705 	switch (flush) {
   2706 	case PMAP_FLUSH_PRIMARY:
   2707 		if (md->pvh_attrs & PVF_MULTCLR) {
   2708 			va_offset = 0;
   2709 			end_va = arm_cache_prefer_mask;
   2710 			md->pvh_attrs &= ~PVF_MULTCLR;
   2711 			PMAPCOUNT(vac_flush_lots);
   2712 		} else {
   2713 			va_offset = md->pvh_attrs & arm_cache_prefer_mask;
   2714 			end_va = va_offset;
   2715 			PMAPCOUNT(vac_flush_one);
   2716 		}
   2717 		/*
   2718 		 * Mark that the page is no longer dirty.
   2719 		 */
   2720 		md->pvh_attrs &= ~PVF_DIRTY;
   2721 		wbinv_p = true;
   2722 		break;
   2723 	case PMAP_FLUSH_SECONDARY:
   2724 		va_offset = 0;
   2725 		end_va = arm_cache_prefer_mask;
   2726 		wbinv_p = true;
   2727 		md->pvh_attrs &= ~PVF_MULTCLR;
   2728 		PMAPCOUNT(vac_flush_lots);
   2729 		break;
   2730 	case PMAP_CLEAN_PRIMARY:
   2731 		va_offset = md->pvh_attrs & arm_cache_prefer_mask;
   2732 		end_va = va_offset;
   2733 		wbinv_p = false;
   2734 		/*
   2735 		 * Mark that the page is no longer dirty.
   2736 		 */
   2737 		if ((md->pvh_attrs & PVF_DMOD) == 0)
   2738 			md->pvh_attrs &= ~PVF_DIRTY;
   2739 		PMAPCOUNT(vac_clean_one);
   2740 		break;
   2741 	default:
   2742 		return;
   2743 	}
   2744 
   2745 	KASSERT(!(md->pvh_attrs & PVF_NC));
   2746 
   2747 	NPDEBUG(PDB_VAC, printf("pmap_flush_page: md=%p (attrs=%#x)\n",
   2748 	    md, md->pvh_attrs));
   2749 
   2750 	const size_t scache_line_size = arm_scache.dcache_line_size;
   2751 
   2752 	for (; va_offset <= end_va; va_offset += PAGE_SIZE) {
   2753 		pt_entry_t * const ptep = cpu_cdst_pte(va_offset);
   2754 		const vaddr_t dstp = cpu_cdstp(va_offset);
   2755 		const pt_entry_t opte = *ptep;
   2756 
   2757 		if (flush == PMAP_FLUSH_SECONDARY
   2758 		    && va_offset == (md->pvh_attrs & arm_cache_prefer_mask))
   2759 			continue;
   2760 
   2761 		pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC);
   2762 		/*
   2763 		 * Set up a PTE with the right coloring to flush
   2764 		 * existing cache entries.
   2765 		 */
   2766 		const pt_entry_t npte = L2_S_PROTO
   2767 		    | pa
   2768 		    | L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE)
   2769 		    | pte_l2_s_cache_mode;
   2770 		l2pte_set(ptep, npte, opte);
   2771 		PTE_SYNC(ptep);
   2772 
   2773 		/*
   2774 		 * Flush it.  Make sure to flush secondary cache too since
   2775 		 * bus_dma will ignore uncached pages.
   2776 		 */
   2777 		if (scache_line_size != 0) {
   2778 			cpu_dcache_wb_range(dstp, PAGE_SIZE);
   2779 			if (wbinv_p) {
   2780 				cpu_sdcache_wbinv_range(dstp, pa, PAGE_SIZE);
   2781 				cpu_dcache_inv_range(dstp, PAGE_SIZE);
   2782 			} else {
   2783 				cpu_sdcache_wb_range(dstp, pa, PAGE_SIZE);
   2784 			}
   2785 		} else {
   2786 			if (wbinv_p) {
   2787 				cpu_dcache_wbinv_range(dstp, PAGE_SIZE);
   2788 			} else {
   2789 				cpu_dcache_wb_range(dstp, PAGE_SIZE);
   2790 			}
   2791 		}
   2792 
   2793 		/*
   2794 		 * Restore the page table entry since we might have interrupted
   2795 		 * pmap_zero_page or pmap_copy_page which was already using
   2796 		 * this pte.
   2797 		 */
   2798 		if (opte) {
   2799 			l2pte_set(ptep, opte, npte);
   2800 		} else {
   2801 			l2pte_reset(ptep);
   2802 		}
   2803 		PTE_SYNC(ptep);
   2804 		pmap_tlb_flush_SE(pmap_kernel(), dstp, PVF_REF | PVF_EXEC);
   2805 	}
   2806 }
   2807 #endif /* ARM_MMU_EXTENDED */
   2808 #endif /* PMAP_CACHE_VIPT */
   2809 
   2810 /*
   2811  * Routine:	pmap_page_remove
   2812  * Function:
   2813  *		Removes this physical page from
   2814  *		all physical maps in which it resides.
   2815  *		Reflects back modify bits to the pager.
   2816  */
   2817 static void
   2818 pmap_page_remove(struct vm_page_md *md, paddr_t pa)
   2819 {
   2820 	struct l2_bucket *l2b;
   2821 	struct pv_entry *pv;
   2822 	pt_entry_t *ptep;
   2823 #ifndef ARM_MMU_EXTENDED
   2824 	bool flush = false;
   2825 #endif
   2826 	u_int flags = 0;
   2827 
   2828 	NPDEBUG(PDB_FOLLOW,
   2829 	    printf("pmap_page_remove: md %p (0x%08lx)\n", md,
   2830 	    pa));
   2831 
   2832 	struct pv_entry **pvp = &SLIST_FIRST(&md->pvh_list);
   2833 	pmap_acquire_page_lock(md);
   2834 	if (*pvp == NULL) {
   2835 #ifdef PMAP_CACHE_VIPT
   2836 		/*
   2837 		 * We *know* the page contents are about to be replaced.
   2838 		 * Discard the exec contents
   2839 		 */
   2840 		if (PV_IS_EXEC_P(md->pvh_attrs))
   2841 			PMAPCOUNT(exec_discarded_page_protect);
   2842 		md->pvh_attrs &= ~PVF_EXEC;
   2843 		PMAP_VALIDATE_MD_PAGE(md);
   2844 #endif
   2845 		pmap_release_page_lock(md);
   2846 		return;
   2847 	}
   2848 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   2849 	KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md));
   2850 #endif
   2851 
   2852 	/*
   2853 	 * Clear alias counts
   2854 	 */
   2855 #ifdef PMAP_CACHE_VIVT
   2856 	md->k_mappings = 0;
   2857 #endif
   2858 	md->urw_mappings = md->uro_mappings = 0;
   2859 
   2860 #ifdef PMAP_CACHE_VIVT
   2861 	pmap_clean_page(md, false);
   2862 #endif
   2863 
   2864 	while ((pv = *pvp) != NULL) {
   2865 		pmap_t pm = pv->pv_pmap;
   2866 #ifndef ARM_MMU_EXTENDED
   2867 		if (flush == false && pmap_is_current(pm))
   2868 			flush = true;
   2869 #endif
   2870 
   2871 		if (pm == pmap_kernel()) {
   2872 #ifdef PMAP_CACHE_VIPT
   2873 			/*
   2874 			 * If this was unmanaged mapping, it must be preserved.
   2875 			 * Move it back on the list and advance the end-of-list
   2876 			 * pointer.
   2877 			 */
   2878 			if (PV_IS_KENTRY_P(pv->pv_flags)) {
   2879 				*pvp = pv;
   2880 				pvp = &SLIST_NEXT(pv, pv_link);
   2881 				continue;
   2882 			}
   2883 			if (pv->pv_flags & PVF_WRITE)
   2884 				md->krw_mappings--;
   2885 			else
   2886 				md->kro_mappings--;
   2887 #endif
   2888 			PMAPCOUNT(kernel_unmappings);
   2889 		}
   2890 		*pvp = SLIST_NEXT(pv, pv_link); /* remove from list */
   2891 		PMAPCOUNT(unmappings);
   2892 
   2893 		pmap_release_page_lock(md);
   2894 		pmap_acquire_pmap_lock(pm);
   2895 
   2896 #ifdef ARM_MMU_EXTENDED
   2897 		pmap_tlb_invalidate_addr(pm, pv->pv_va);
   2898 #endif
   2899 
   2900 		l2b = pmap_get_l2_bucket(pm, pv->pv_va);
   2901 		KASSERTMSG(l2b != NULL, "%#lx", pv->pv_va);
   2902 
   2903 		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
   2904 
   2905 		/*
   2906 		 * Update statistics
   2907 		 */
   2908 		--pm->pm_stats.resident_count;
   2909 
   2910 		/* Wired bit */
   2911 		if (pv->pv_flags & PVF_WIRED)
   2912 			--pm->pm_stats.wired_count;
   2913 
   2914 		flags |= pv->pv_flags;
   2915 
   2916 		/*
   2917 		 * Invalidate the PTEs.
   2918 		 */
   2919 		l2pte_reset(ptep);
   2920 		PTE_SYNC_CURRENT(pm, ptep);
   2921 		pmap_free_l2_bucket(pm, l2b, PAGE_SIZE / L2_S_SIZE);
   2922 		pmap_release_pmap_lock(pm);
   2923 
   2924 		pool_put(&pmap_pv_pool, pv);
   2925 		pmap_acquire_page_lock(md);
   2926 #ifdef MULTIPROCESSOR
   2927 		/*
   2928 		 * Restart of the beginning of the list.
   2929 		 */
   2930 		pvp = &SLIST_FIRST(&md->pvh_list);
   2931 #endif
   2932 	}
   2933 	/*
   2934 	 * if we reach the end of the list and there are still mappings, they
   2935 	 * might be able to be cached now.  And they must be kernel mappings.
   2936 	 */
   2937 	if (!SLIST_EMPTY(&md->pvh_list)) {
   2938 		pmap_vac_me_harder(md, pa, pmap_kernel(), 0);
   2939 	}
   2940 
   2941 #ifdef PMAP_CACHE_VIPT
   2942 	/*
   2943 	 * Its EXEC cache is now gone.
   2944 	 */
   2945 	if (PV_IS_EXEC_P(md->pvh_attrs))
   2946 		PMAPCOUNT(exec_discarded_page_protect);
   2947 	md->pvh_attrs &= ~PVF_EXEC;
   2948 	KASSERT(md->urw_mappings == 0);
   2949 	KASSERT(md->uro_mappings == 0);
   2950 #ifndef ARM_MMU_EXTENDED
   2951 	if (arm_cache_prefer_mask != 0) {
   2952 		if (md->krw_mappings == 0)
   2953 			md->pvh_attrs &= ~PVF_WRITE;
   2954 		PMAP_VALIDATE_MD_PAGE(md);
   2955 	}
   2956 #endif /* ARM_MMU_EXTENDED */
   2957 #endif /* PMAP_CACHE_VIPT */
   2958 	pmap_release_page_lock(md);
   2959 
   2960 #ifndef ARM_MMU_EXTENDED
   2961 	if (flush) {
   2962 		/*
   2963 		 * Note: We can't use pmap_tlb_flush{I,D}() here since that
   2964 		 * would need a subsequent call to pmap_update() to ensure
   2965 		 * curpm->pm_cstate.cs_all is reset. Our callers are not
   2966 		 * required to do that (see pmap(9)), so we can't modify
   2967 		 * the current pmap's state.
   2968 		 */
   2969 		if (PV_BEEN_EXECD(flags))
   2970 			cpu_tlb_flushID();
   2971 		else
   2972 			cpu_tlb_flushD();
   2973 	}
   2974 	cpu_cpwait();
   2975 #endif /* ARM_MMU_EXTENDED */
   2976 }
   2977 
   2978 /*
   2979  * pmap_t pmap_create(void)
   2980  *
   2981  *      Create a new pmap structure from scratch.
   2982  */
   2983 pmap_t
   2984 pmap_create(void)
   2985 {
   2986 	pmap_t pm;
   2987 
   2988 	pm = pool_cache_get(&pmap_cache, PR_WAITOK);
   2989 
   2990 	mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE);
   2991 	uvm_obj_init(&pm->pm_obj, NULL, false, 1);
   2992 	uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock);
   2993 
   2994 	pm->pm_stats.wired_count = 0;
   2995 	pm->pm_stats.resident_count = 1;
   2996 #ifdef ARM_MMU_EXTENDED
   2997 #ifdef MULTIPROCESSOR
   2998 	kcpuset_create(&pm->pm_active, true);
   2999 	kcpuset_create(&pm->pm_onproc, true);
   3000 #endif
   3001 #else
   3002 	pm->pm_cstate.cs_all = 0;
   3003 #endif
   3004 	pmap_alloc_l1(pm);
   3005 
   3006 	/*
   3007 	 * Note: The pool cache ensures that the pm_l2[] array is already
   3008 	 * initialised to zero.
   3009 	 */
   3010 
   3011 	pmap_pinit(pm);
   3012 
   3013 	LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list);
   3014 
   3015 	return (pm);
   3016 }
   3017 
   3018 u_int
   3019 arm32_mmap_flags(paddr_t pa)
   3020 {
   3021 	/*
   3022 	 * the upper 8 bits in pmap_enter()'s flags are reserved for MD stuff
   3023 	 * and we're using the upper bits in page numbers to pass flags around
   3024 	 * so we might as well use the same bits
   3025 	 */
   3026 	return (u_int)pa & PMAP_MD_MASK;
   3027 }
   3028 /*
   3029  * int pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot,
   3030  *      u_int flags)
   3031  *
   3032  *      Insert the given physical page (p) at
   3033  *      the specified virtual address (v) in the
   3034  *      target physical map with the protection requested.
   3035  *
   3036  *      NB:  This is the only routine which MAY NOT lazy-evaluate
   3037  *      or lose information.  That is, this routine must actually
   3038  *      insert this page into the given map NOW.
   3039  */
   3040 int
   3041 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
   3042 {
   3043 	struct l2_bucket *l2b;
   3044 	struct vm_page *pg, *opg;
   3045 	u_int nflags;
   3046 	u_int oflags;
   3047 	const bool kpm_p = (pm == pmap_kernel());
   3048 #ifdef ARM_HAS_VBAR
   3049 	const bool vector_page_p = false;
   3050 #else
   3051 	const bool vector_page_p = (va == vector_page);
   3052 #endif
   3053 
   3054 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   3055 
   3056 	UVMHIST_LOG(maphist, " (pm %p va %#x pa %#x prot %#x",
   3057 	    pm, va, pa, prot);
   3058 	UVMHIST_LOG(maphist, "  flag %#x", flags, 0, 0, 0);
   3059 
   3060 	KDASSERT((flags & PMAP_WIRED) == 0 || (flags & VM_PROT_ALL) != 0);
   3061 	KDASSERT(((va | pa) & PGOFSET) == 0);
   3062 
   3063 	/*
   3064 	 * Get a pointer to the page.  Later on in this function, we
   3065 	 * test for a managed page by checking pg != NULL.
   3066 	 */
   3067 	pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL;
   3068 
   3069 	nflags = 0;
   3070 	if (prot & VM_PROT_WRITE)
   3071 		nflags |= PVF_WRITE;
   3072 	if (prot & VM_PROT_EXECUTE)
   3073 		nflags |= PVF_EXEC;
   3074 	if (flags & PMAP_WIRED)
   3075 		nflags |= PVF_WIRED;
   3076 
   3077 	pmap_acquire_pmap_lock(pm);
   3078 
   3079 	/*
   3080 	 * Fetch the L2 bucket which maps this page, allocating one if
   3081 	 * necessary for user pmaps.
   3082 	 */
   3083 	if (kpm_p) {
   3084 		l2b = pmap_get_l2_bucket(pm, va);
   3085 	} else {
   3086 		l2b = pmap_alloc_l2_bucket(pm, va);
   3087 	}
   3088 	if (l2b == NULL) {
   3089 		if (flags & PMAP_CANFAIL) {
   3090 			pmap_release_pmap_lock(pm);
   3091 			return (ENOMEM);
   3092 		}
   3093 		panic("pmap_enter: failed to allocate L2 bucket");
   3094 	}
   3095 	pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)];
   3096 	const pt_entry_t opte = *ptep;
   3097 	pt_entry_t npte = pa;
   3098 	oflags = 0;
   3099 
   3100 	if (opte) {
   3101 		/*
   3102 		 * There is already a mapping at this address.
   3103 		 * If the physical address is different, lookup the
   3104 		 * vm_page.
   3105 		 */
   3106 		if (l2pte_pa(opte) != pa)
   3107 			opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
   3108 		else
   3109 			opg = pg;
   3110 	} else
   3111 		opg = NULL;
   3112 
   3113 	if (pg) {
   3114 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3115 
   3116 		/*
   3117 		 * This is to be a managed mapping.
   3118 		 */
   3119 		pmap_acquire_page_lock(md);
   3120 		if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) {
   3121 			/*
   3122 			 * - The access type indicates that we don't need
   3123 			 *   to do referenced emulation.
   3124 			 * OR
   3125 			 * - The physical page has already been referenced
   3126 			 *   so no need to re-do referenced emulation here.
   3127 			 */
   3128 			npte |= l2pte_set_readonly(L2_S_PROTO);
   3129 
   3130 			nflags |= PVF_REF;
   3131 
   3132 			if ((prot & VM_PROT_WRITE) != 0 &&
   3133 			    ((flags & VM_PROT_WRITE) != 0 ||
   3134 			     (md->pvh_attrs & PVF_MOD) != 0)) {
   3135 				/*
   3136 				 * This is a writable mapping, and the
   3137 				 * page's mod state indicates it has
   3138 				 * already been modified. Make it
   3139 				 * writable from the outset.
   3140 				 */
   3141 				npte = l2pte_set_writable(npte);
   3142 				nflags |= PVF_MOD;
   3143 			}
   3144 
   3145 #ifdef ARM_MMU_EXTENDED
   3146 			/*
   3147 			 * If the page has been cleaned, then the pvh_attrs
   3148 			 * will have PVF_EXEC set, so mark it execute so we
   3149 			 * don't get an access fault when trying to execute
   3150 			 * from it.
   3151 			 */
   3152 			if (md->pvh_attrs & nflags & PVF_EXEC) {
   3153 				npte &= ~L2_XS_XN;
   3154 			}
   3155 #endif
   3156 		} else {
   3157 			/*
   3158 			 * Need to do page referenced emulation.
   3159 			 */
   3160 			npte |= L2_TYPE_INV;
   3161 		}
   3162 
   3163 		if (flags & ARM32_MMAP_WRITECOMBINE) {
   3164 			npte |= pte_l2_s_wc_mode;
   3165 		} else
   3166 			npte |= pte_l2_s_cache_mode;
   3167 
   3168 		if (pg == opg) {
   3169 			/*
   3170 			 * We're changing the attrs of an existing mapping.
   3171 			 */
   3172 			oflags = pmap_modify_pv(md, pa, pm, va,
   3173 			    PVF_WRITE | PVF_EXEC | PVF_WIRED |
   3174 			    PVF_MOD | PVF_REF, nflags);
   3175 
   3176 #ifdef PMAP_CACHE_VIVT
   3177 			/*
   3178 			 * We may need to flush the cache if we're
   3179 			 * doing rw-ro...
   3180 			 */
   3181 			if (pm->pm_cstate.cs_cache_d &&
   3182 			    (oflags & PVF_NC) == 0 &&
   3183 			    l2pte_writable_p(opte) &&
   3184 			    (prot & VM_PROT_WRITE) == 0)
   3185 				cpu_dcache_wb_range(va, PAGE_SIZE);
   3186 #endif
   3187 		} else {
   3188 			struct pv_entry *pv;
   3189 			/*
   3190 			 * New mapping, or changing the backing page
   3191 			 * of an existing mapping.
   3192 			 */
   3193 			if (opg) {
   3194 				struct vm_page_md *omd = VM_PAGE_TO_MD(opg);
   3195 				paddr_t opa = VM_PAGE_TO_PHYS(opg);
   3196 
   3197 				/*
   3198 				 * Replacing an existing mapping with a new one.
   3199 				 * It is part of our managed memory so we
   3200 				 * must remove it from the PV list
   3201 				 */
   3202 				pv = pmap_remove_pv(omd, opa, pm, va);
   3203 				pmap_vac_me_harder(omd, opa, pm, 0);
   3204 				oflags = pv->pv_flags;
   3205 
   3206 #ifdef PMAP_CACHE_VIVT
   3207 				/*
   3208 				 * If the old mapping was valid (ref/mod
   3209 				 * emulation creates 'invalid' mappings
   3210 				 * initially) then make sure to frob
   3211 				 * the cache.
   3212 				 */
   3213 				if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) {
   3214 					pmap_cache_wbinv_page(pm, va, true,
   3215 					    oflags);
   3216 				}
   3217 #endif
   3218 			} else {
   3219 				pmap_release_page_lock(md);
   3220 				pv = pool_get(&pmap_pv_pool, PR_NOWAIT);
   3221 				if (pv == NULL) {
   3222 					pmap_release_pmap_lock(pm);
   3223 					if ((flags & PMAP_CANFAIL) == 0)
   3224 						panic("pmap_enter: "
   3225 						    "no pv entries");
   3226 
   3227 					pmap_free_l2_bucket(pm, l2b, 0);
   3228 					UVMHIST_LOG(maphist, "  <-- done (ENOMEM)",
   3229 					    0, 0, 0, 0);
   3230 					return (ENOMEM);
   3231 				}
   3232 				pmap_acquire_page_lock(md);
   3233 			}
   3234 
   3235 			pmap_enter_pv(md, pa, pv, pm, va, nflags);
   3236 		}
   3237 		pmap_release_page_lock(md);
   3238 	} else {
   3239 		/*
   3240 		 * We're mapping an unmanaged page.
   3241 		 * These are always readable, and possibly writable, from
   3242 		 * the get go as we don't need to track ref/mod status.
   3243 		 */
   3244 		npte |= l2pte_set_readonly(L2_S_PROTO);
   3245 		if (prot & VM_PROT_WRITE)
   3246 			npte = l2pte_set_writable(npte);
   3247 
   3248 		/*
   3249 		 * Make sure the vector table is mapped cacheable
   3250 		 */
   3251 		if ((vector_page_p && !kpm_p)
   3252 		    || (flags & ARM32_MMAP_CACHEABLE)) {
   3253 			npte |= pte_l2_s_cache_mode;
   3254 #ifdef ARM_MMU_EXTENDED
   3255 			npte &= ~L2_XS_XN;	/* and executable */
   3256 #endif
   3257 		} else if (flags & ARM32_MMAP_WRITECOMBINE) {
   3258 			npte |= pte_l2_s_wc_mode;
   3259 		}
   3260 		if (opg) {
   3261 			/*
   3262 			 * Looks like there's an existing 'managed' mapping
   3263 			 * at this address.
   3264 			 */
   3265 			struct vm_page_md *omd = VM_PAGE_TO_MD(opg);
   3266 			paddr_t opa = VM_PAGE_TO_PHYS(opg);
   3267 
   3268 			pmap_acquire_page_lock(omd);
   3269 			struct pv_entry *pv = pmap_remove_pv(omd, opa, pm, va);
   3270 			pmap_vac_me_harder(omd, opa, pm, 0);
   3271 			oflags = pv->pv_flags;
   3272 			pmap_release_page_lock(omd);
   3273 
   3274 #ifdef PMAP_CACHE_VIVT
   3275 			if (!(oflags & PVF_NC) && l2pte_valid_p(opte)) {
   3276 				pmap_cache_wbinv_page(pm, va, true, oflags);
   3277 			}
   3278 #endif
   3279 			pool_put(&pmap_pv_pool, pv);
   3280 		}
   3281 	}
   3282 
   3283 	/*
   3284 	 * Make sure userland mappings get the right permissions
   3285 	 */
   3286 	if (!vector_page_p && !kpm_p) {
   3287 		npte |= L2_S_PROT_U;
   3288 #ifdef ARM_MMU_EXTENDED
   3289 		npte |= L2_XS_nG;	/* user pages are not global */
   3290 #endif
   3291 	}
   3292 
   3293 	/*
   3294 	 * Keep the stats up to date
   3295 	 */
   3296 	if (opte == 0) {
   3297 		l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE;
   3298 		pm->pm_stats.resident_count++;
   3299 	}
   3300 
   3301 	UVMHIST_LOG(maphist, " opte %#x npte %#x", opte, npte, 0, 0);
   3302 
   3303 #if defined(ARM_MMU_EXTENDED)
   3304 	/*
   3305 	 * If exec protection was requested but the page hasn't been synced,
   3306 	 * sync it now and allow execution from it.
   3307 	 */
   3308 	if ((nflags & PVF_EXEC) && (npte & L2_XS_XN)) {
   3309 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3310 		npte &= ~L2_XS_XN;
   3311 		pmap_syncicache_page(md, pa);
   3312 		PMAPCOUNT(exec_synced_map);
   3313 	}
   3314 #endif
   3315 	/*
   3316 	 * If this is just a wiring change, the two PTEs will be
   3317 	 * identical, so there's no need to update the page table.
   3318 	 */
   3319 	if (npte != opte) {
   3320 
   3321 		l2pte_set(ptep, npte, opte);
   3322 		PTE_SYNC(ptep);
   3323 #ifndef ARM_MMU_EXTENDED
   3324 		bool is_cached = pmap_is_cached(pm);
   3325 		if (is_cached) {
   3326 			/*
   3327 			 * We only need to frob the cache/tlb if this pmap
   3328 			 * is current
   3329 			 */
   3330 			if (!vector_page_p && l2pte_valid_p(npte)) {
   3331 				/*
   3332 				 * This mapping is likely to be accessed as
   3333 				 * soon as we return to userland. Fix up the
   3334 				 * L1 entry to avoid taking another
   3335 				 * page/domain fault.
   3336 				 */
   3337 				pd_entry_t *pdep = pmap_l1_kva(pm)
   3338 				     + l1pte_index(va);
   3339 				pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa
   3340 				    | L1_C_DOM(pmap_domain(pm));
   3341 				if (*pdep != pde) {
   3342 					l1pte_setone(pdep, pde);
   3343 					PTE_SYNC(pdep);
   3344 				}
   3345 			}
   3346 		}
   3347 #endif /* !ARMM_MMU_EXTENDED */
   3348 
   3349 		pmap_tlb_flush_SE(pm, va, oflags);
   3350 
   3351 #ifndef ARM_MMU_EXTENDED
   3352 		UVMHIST_LOG(maphist, "  is_cached %d cs 0x%08x\n",
   3353 		    is_cached, pm->pm_cstate.cs_all, 0, 0);
   3354 
   3355 		if (pg != NULL) {
   3356 			struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3357 
   3358 			pmap_acquire_page_lock(md);
   3359 			pmap_vac_me_harder(md, pa, pm, va);
   3360 			pmap_release_page_lock(md);
   3361 		}
   3362 #endif
   3363 	}
   3364 #if defined(PMAP_CACHE_VIPT) && defined(DIAGNOSTIC)
   3365 	if (pg) {
   3366 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3367 
   3368 		pmap_acquire_page_lock(md);
   3369 #ifndef ARM_MMU_EXTENDED
   3370 		KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   3371 #endif
   3372 		PMAP_VALIDATE_MD_PAGE(md);
   3373 		pmap_release_page_lock(md);
   3374 	}
   3375 #endif
   3376 
   3377 	pmap_release_pmap_lock(pm);
   3378 
   3379 	return (0);
   3380 }
   3381 
   3382 /*
   3383  * pmap_remove()
   3384  *
   3385  * pmap_remove is responsible for nuking a number of mappings for a range
   3386  * of virtual address space in the current pmap. To do this efficiently
   3387  * is interesting, because in a number of cases a wide virtual address
   3388  * range may be supplied that contains few actual mappings. So, the
   3389  * optimisations are:
   3390  *  1. Skip over hunks of address space for which no L1 or L2 entry exists.
   3391  *  2. Build up a list of pages we've hit, up to a maximum, so we can
   3392  *     maybe do just a partial cache clean. This path of execution is
   3393  *     complicated by the fact that the cache must be flushed _before_
   3394  *     the PTE is nuked, being a VAC :-)
   3395  *  3. If we're called after UVM calls pmap_remove_all(), we can defer
   3396  *     all invalidations until pmap_update(), since pmap_remove_all() has
   3397  *     already flushed the cache.
   3398  *  4. Maybe later fast-case a single page, but I don't think this is
   3399  *     going to make _that_ much difference overall.
   3400  */
   3401 
   3402 #define	PMAP_REMOVE_CLEAN_LIST_SIZE	3
   3403 
   3404 void
   3405 pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva)
   3406 {
   3407 	vaddr_t next_bucket;
   3408 	u_int cleanlist_idx, total, cnt;
   3409 	struct {
   3410 		vaddr_t va;
   3411 		pt_entry_t *ptep;
   3412 	} cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE];
   3413 	u_int mappings;
   3414 
   3415 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   3416 	UVMHIST_LOG(maphist, " (pm=%p, sva=%#x, eva=%#x)", pm, sva, eva, 0);
   3417 
   3418 	/*
   3419 	 * we lock in the pmap => pv_head direction
   3420 	 */
   3421 	pmap_acquire_pmap_lock(pm);
   3422 
   3423 	if (pm->pm_remove_all || !pmap_is_cached(pm)) {
   3424 		cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
   3425 #ifndef ARM_MMU_EXTENDED
   3426 		if (pm->pm_cstate.cs_tlb == 0)
   3427 			pm->pm_remove_all = true;
   3428 #endif
   3429 	} else
   3430 		cleanlist_idx = 0;
   3431 
   3432 	total = 0;
   3433 
   3434 	while (sva < eva) {
   3435 		/*
   3436 		 * Do one L2 bucket's worth at a time.
   3437 		 */
   3438 		next_bucket = L2_NEXT_BUCKET_VA(sva);
   3439 		if (next_bucket > eva)
   3440 			next_bucket = eva;
   3441 
   3442 		struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva);
   3443 		if (l2b == NULL) {
   3444 			sva = next_bucket;
   3445 			continue;
   3446 		}
   3447 
   3448 		pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)];
   3449 
   3450 		for (mappings = 0;
   3451 		     sva < next_bucket;
   3452 		     sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) {
   3453 			pt_entry_t opte = *ptep;
   3454 
   3455 			if (opte == 0) {
   3456 				/* Nothing here, move along */
   3457 				continue;
   3458 			}
   3459 
   3460 			u_int flags = PVF_REF;
   3461 			paddr_t pa = l2pte_pa(opte);
   3462 			struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
   3463 
   3464 			/*
   3465 			 * Update flags. In a number of circumstances,
   3466 			 * we could cluster a lot of these and do a
   3467 			 * number of sequential pages in one go.
   3468 			 */
   3469 			if (pg != NULL) {
   3470 				struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3471 				struct pv_entry *pv;
   3472 
   3473 				pmap_acquire_page_lock(md);
   3474 				pv = pmap_remove_pv(md, pa, pm, sva);
   3475 				pmap_vac_me_harder(md, pa, pm, 0);
   3476 				pmap_release_page_lock(md);
   3477 				if (pv != NULL) {
   3478 					if (pm->pm_remove_all == false) {
   3479 						flags = pv->pv_flags;
   3480 					}
   3481 					pool_put(&pmap_pv_pool, pv);
   3482 				}
   3483 			}
   3484 			mappings += PAGE_SIZE / L2_S_SIZE;
   3485 
   3486 			if (!l2pte_valid_p(opte)) {
   3487 				/*
   3488 				 * Ref/Mod emulation is still active for this
   3489 				 * mapping, therefore it is has not yet been
   3490 				 * accessed. No need to frob the cache/tlb.
   3491 				 */
   3492 				l2pte_reset(ptep);
   3493 				PTE_SYNC_CURRENT(pm, ptep);
   3494 				continue;
   3495 			}
   3496 
   3497 #ifdef ARM_MMU_EXTENDED
   3498 			if (pm == pmap_kernel()) {
   3499 				l2pte_reset(ptep);
   3500 				PTE_SYNC(ptep);
   3501 				pmap_tlb_flush_SE(pm, sva, flags);
   3502 				continue;
   3503 			}
   3504 #endif
   3505 			if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) {
   3506 				/* Add to the clean list. */
   3507 				cleanlist[cleanlist_idx].ptep = ptep;
   3508 				cleanlist[cleanlist_idx].va =
   3509 				    sva | (flags & PVF_EXEC);
   3510 				cleanlist_idx++;
   3511 			} else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) {
   3512 				/* Nuke everything if needed. */
   3513 #ifdef PMAP_CACHE_VIVT
   3514 				pmap_cache_wbinv_all(pm, PVF_EXEC);
   3515 #endif
   3516 				/*
   3517 				 * Roll back the previous PTE list,
   3518 				 * and zero out the current PTE.
   3519 				 */
   3520 				for (cnt = 0;
   3521 				     cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) {
   3522 					l2pte_reset(cleanlist[cnt].ptep);
   3523 					PTE_SYNC(cleanlist[cnt].ptep);
   3524 				}
   3525 				l2pte_reset(ptep);
   3526 				PTE_SYNC(ptep);
   3527 				cleanlist_idx++;
   3528 				pm->pm_remove_all = true;
   3529 			} else {
   3530 				l2pte_reset(ptep);
   3531 				PTE_SYNC(ptep);
   3532 				if (pm->pm_remove_all == false) {
   3533 					pmap_tlb_flush_SE(pm, sva, flags);
   3534 				}
   3535 			}
   3536 		}
   3537 
   3538 		/*
   3539 		 * Deal with any left overs
   3540 		 */
   3541 		if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) {
   3542 			total += cleanlist_idx;
   3543 			for (cnt = 0; cnt < cleanlist_idx; cnt++) {
   3544 #ifdef ARM_MMU_EXTENDED
   3545 				vaddr_t clva = cleanlist[cnt].va;
   3546 				pmap_tlb_flush_SE(pm, clva, PVF_REF);
   3547 #else
   3548 				vaddr_t va = cleanlist[cnt].va;
   3549 				if (pm->pm_cstate.cs_all != 0) {
   3550 					vaddr_t clva = va & ~PAGE_MASK;
   3551 					u_int flags = va & PVF_EXEC;
   3552 #ifdef PMAP_CACHE_VIVT
   3553 					pmap_cache_wbinv_page(pm, clva, true,
   3554 					    PVF_REF | PVF_WRITE | flags);
   3555 #endif
   3556 					pmap_tlb_flush_SE(pm, clva,
   3557 					    PVF_REF | flags);
   3558 				}
   3559 #endif /* ARM_MMU_EXTENDED */
   3560 				l2pte_reset(cleanlist[cnt].ptep);
   3561 				PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep);
   3562 			}
   3563 
   3564 			/*
   3565 			 * If it looks like we're removing a whole bunch
   3566 			 * of mappings, it's faster to just write-back
   3567 			 * the whole cache now and defer TLB flushes until
   3568 			 * pmap_update() is called.
   3569 			 */
   3570 			if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE)
   3571 				cleanlist_idx = 0;
   3572 			else {
   3573 				cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1;
   3574 #ifdef PMAP_CACHE_VIVT
   3575 				pmap_cache_wbinv_all(pm, PVF_EXEC);
   3576 #endif
   3577 				pm->pm_remove_all = true;
   3578 			}
   3579 		}
   3580 
   3581 
   3582 		pmap_free_l2_bucket(pm, l2b, mappings);
   3583 		pm->pm_stats.resident_count -= mappings / (PAGE_SIZE/L2_S_SIZE);
   3584 	}
   3585 
   3586 	pmap_release_pmap_lock(pm);
   3587 }
   3588 
   3589 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3590 static struct pv_entry *
   3591 pmap_kremove_pg(struct vm_page *pg, vaddr_t va)
   3592 {
   3593 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3594 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
   3595 	struct pv_entry *pv;
   3596 
   3597 	KASSERT(arm_cache_prefer_mask == 0 || md->pvh_attrs & (PVF_COLORED|PVF_NC));
   3598 	KASSERT((md->pvh_attrs & PVF_KMPAGE) == 0);
   3599 	KASSERT(pmap_page_locked_p(md));
   3600 
   3601 	pv = pmap_remove_pv(md, pa, pmap_kernel(), va);
   3602 	KASSERTMSG(pv, "pg %p (pa #%lx) va %#lx", pg, pa, va);
   3603 	KASSERT(PV_IS_KENTRY_P(pv->pv_flags));
   3604 
   3605 	/*
   3606 	 * If we are removing a writeable mapping to a cached exec page,
   3607 	 * if it's the last mapping then clear it execness other sync
   3608 	 * the page to the icache.
   3609 	 */
   3610 	if ((md->pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC
   3611 	    && (pv->pv_flags & PVF_WRITE) != 0) {
   3612 		if (SLIST_EMPTY(&md->pvh_list)) {
   3613 			md->pvh_attrs &= ~PVF_EXEC;
   3614 			PMAPCOUNT(exec_discarded_kremove);
   3615 		} else {
   3616 			pmap_syncicache_page(md, pa);
   3617 			PMAPCOUNT(exec_synced_kremove);
   3618 		}
   3619 	}
   3620 	pmap_vac_me_harder(md, pa, pmap_kernel(), 0);
   3621 
   3622 	return pv;
   3623 }
   3624 #endif /* PMAP_CACHE_VIPT && !ARM_MMU_EXTENDED */
   3625 
   3626 /*
   3627  * pmap_kenter_pa: enter an unmanaged, wired kernel mapping
   3628  *
   3629  * We assume there is already sufficient KVM space available
   3630  * to do this, as we can't allocate L2 descriptor tables/metadata
   3631  * from here.
   3632  */
   3633 void
   3634 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
   3635 {
   3636 #ifdef PMAP_CACHE_VIVT
   3637 	struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL;
   3638 #endif
   3639 #ifdef PMAP_CACHE_VIPT
   3640 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
   3641 	struct vm_page *opg;
   3642 #ifndef ARM_MMU_EXTENDED
   3643 	struct pv_entry *pv = NULL;
   3644 #endif
   3645 #endif
   3646 	struct vm_page_md *md = pg != NULL ? VM_PAGE_TO_MD(pg) : NULL;
   3647 
   3648 	UVMHIST_FUNC(__func__);
   3649 
   3650 	if (pmap_initialized) {
   3651 		UVMHIST_CALLED(maphist);
   3652 		UVMHIST_LOG(maphist, " (va=%#x, pa=%#x, prot=%#x, flags=%#x",
   3653 		    va, pa, prot, flags);
   3654 	}
   3655 
   3656 	pmap_t kpm = pmap_kernel();
   3657 	struct l2_bucket * const l2b = pmap_get_l2_bucket(kpm, va);
   3658 	const size_t l1slot __diagused = l1pte_index(va);
   3659 	KASSERTMSG(l2b != NULL,
   3660 	    "va %#lx pa %#lx prot %d maxkvaddr %#lx: l2 %p l2b %p kva %p",
   3661 	    va, pa, prot, pmap_curmaxkvaddr, kpm->pm_l2[L2_IDX(l1slot)],
   3662 	    kpm->pm_l2[L2_IDX(l1slot)]
   3663 		? &kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)]
   3664 		: NULL,
   3665 	    kpm->pm_l2[L2_IDX(l1slot)]
   3666 		? kpm->pm_l2[L2_IDX(l1slot)]->l2_bucket[L2_BUCKET(l1slot)].l2b_kva
   3667 		: NULL);
   3668 	KASSERT(l2b->l2b_kva != NULL);
   3669 
   3670 	pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   3671 	const pt_entry_t opte = *ptep;
   3672 
   3673 	if (opte == 0) {
   3674 		PMAPCOUNT(kenter_mappings);
   3675 		l2b->l2b_occupancy += PAGE_SIZE / L2_S_SIZE;
   3676 	} else {
   3677 		PMAPCOUNT(kenter_remappings);
   3678 #ifdef PMAP_CACHE_VIPT
   3679 		opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
   3680 #if !defined(ARM_MMU_EXTENDED) || defined(DIAGNOSTIC)
   3681 		struct vm_page_md *omd __diagused = VM_PAGE_TO_MD(opg);
   3682 #endif
   3683 		if (opg && arm_cache_prefer_mask != 0) {
   3684 			KASSERT(opg != pg);
   3685 			KASSERT((omd->pvh_attrs & PVF_KMPAGE) == 0);
   3686 			KASSERT((flags & PMAP_KMPAGE) == 0);
   3687 #ifndef ARM_MMU_EXTENDED
   3688 			pmap_acquire_page_lock(omd);
   3689 			pv = pmap_kremove_pg(opg, va);
   3690 			pmap_release_page_lock(omd);
   3691 #endif
   3692 		}
   3693 #endif
   3694 		if (l2pte_valid_p(opte)) {
   3695 #ifdef PMAP_CACHE_VIVT
   3696 			cpu_dcache_wbinv_range(va, PAGE_SIZE);
   3697 #endif
   3698 			cpu_tlb_flushD_SE(va);
   3699 			cpu_cpwait();
   3700 		}
   3701 	}
   3702 
   3703 	pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot)
   3704 	    | ((flags & PMAP_NOCACHE)
   3705 		? 0
   3706 		: ((flags & PMAP_PTE)
   3707 		    ? pte_l2_s_cache_mode_pt : pte_l2_s_cache_mode));
   3708 #ifdef ARM_MMU_EXTENDED
   3709 	if (prot & VM_PROT_EXECUTE)
   3710 		npte &= ~L2_XS_XN;
   3711 #endif
   3712 	l2pte_set(ptep, npte, opte);
   3713 	PTE_SYNC(ptep);
   3714 
   3715 	if (pg) {
   3716 		if (flags & PMAP_KMPAGE) {
   3717 			KASSERT(md->urw_mappings == 0);
   3718 			KASSERT(md->uro_mappings == 0);
   3719 			KASSERT(md->krw_mappings == 0);
   3720 			KASSERT(md->kro_mappings == 0);
   3721 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3722 			KASSERT(pv == NULL);
   3723 			KASSERT(arm_cache_prefer_mask == 0 || (va & PVF_COLORED) == 0);
   3724 			KASSERT((md->pvh_attrs & PVF_NC) == 0);
   3725 			/* if there is a color conflict, evict from cache. */
   3726 			if (pmap_is_page_colored_p(md)
   3727 			    && ((va ^ md->pvh_attrs) & arm_cache_prefer_mask)) {
   3728 				PMAPCOUNT(vac_color_change);
   3729 				pmap_flush_page(md, pa, PMAP_FLUSH_PRIMARY);
   3730 			} else if (md->pvh_attrs & PVF_MULTCLR) {
   3731 				/*
   3732 				 * If this page has multiple colors, expunge
   3733 				 * them.
   3734 				 */
   3735 				PMAPCOUNT(vac_flush_lots2);
   3736 				pmap_flush_page(md, pa, PMAP_FLUSH_SECONDARY);
   3737 			}
   3738 			/*
   3739 			 * Since this is a KMPAGE, there can be no contention
   3740 			 * for this page so don't lock it.
   3741 			 */
   3742 			md->pvh_attrs &= PAGE_SIZE - 1;
   3743 			md->pvh_attrs |= PVF_KMPAGE | PVF_COLORED | PVF_DIRTY
   3744 			    | (va & arm_cache_prefer_mask);
   3745 #else /* !PMAP_CACHE_VIPT || ARM_MMU_EXTENDED */
   3746 			md->pvh_attrs |= PVF_KMPAGE;
   3747 #endif
   3748 			atomic_inc_32(&pmap_kmpages);
   3749 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3750 		} else if (arm_cache_prefer_mask != 0) {
   3751 			if (pv == NULL) {
   3752 				pv = pool_get(&pmap_pv_pool, PR_NOWAIT);
   3753 				KASSERT(pv != NULL);
   3754 			}
   3755 			pmap_acquire_page_lock(md);
   3756 			pmap_enter_pv(md, pa, pv, pmap_kernel(), va,
   3757 			    PVF_WIRED | PVF_KENTRY
   3758 			    | (prot & VM_PROT_WRITE ? PVF_WRITE : 0));
   3759 			if ((prot & VM_PROT_WRITE)
   3760 			    && !(md->pvh_attrs & PVF_NC))
   3761 				md->pvh_attrs |= PVF_DIRTY;
   3762 			KASSERT((prot & VM_PROT_WRITE) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
   3763 			pmap_vac_me_harder(md, pa, pmap_kernel(), va);
   3764 			pmap_release_page_lock(md);
   3765 #endif
   3766 		}
   3767 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3768 	} else {
   3769 		if (pv != NULL)
   3770 			pool_put(&pmap_pv_pool, pv);
   3771 #endif
   3772 	}
   3773 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3774 	KASSERT(md == NULL || !pmap_page_locked_p(md));
   3775 #endif
   3776 	if (pmap_initialized) {
   3777 		UVMHIST_LOG(maphist, "  <-- done (ptep %p: %#x -> %#x)",
   3778 		    ptep, opte, npte, 0);
   3779 	}
   3780 
   3781 }
   3782 
   3783 void
   3784 pmap_kremove(vaddr_t va, vsize_t len)
   3785 {
   3786 #ifdef UVMHIST
   3787 	u_int total_mappings = 0;
   3788 #endif
   3789 
   3790 	PMAPCOUNT(kenter_unmappings);
   3791 
   3792 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   3793 
   3794 	UVMHIST_LOG(maphist, " (va=%#x, len=%#x)", va, len, 0, 0);
   3795 
   3796 	const vaddr_t eva = va + len;
   3797 
   3798 	while (va < eva) {
   3799 		vaddr_t next_bucket = L2_NEXT_BUCKET_VA(va);
   3800 		if (next_bucket > eva)
   3801 			next_bucket = eva;
   3802 
   3803 		struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va);
   3804 		KDASSERT(l2b != NULL);
   3805 
   3806 		pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)];
   3807 		pt_entry_t *ptep = sptep;
   3808 		u_int mappings = 0;
   3809 
   3810 		while (va < next_bucket) {
   3811 			const pt_entry_t opte = *ptep;
   3812 			struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
   3813 			if (opg != NULL) {
   3814 				struct vm_page_md *omd = VM_PAGE_TO_MD(opg);
   3815 
   3816 				if (omd->pvh_attrs & PVF_KMPAGE) {
   3817 					KASSERT(omd->urw_mappings == 0);
   3818 					KASSERT(omd->uro_mappings == 0);
   3819 					KASSERT(omd->krw_mappings == 0);
   3820 					KASSERT(omd->kro_mappings == 0);
   3821 					omd->pvh_attrs &= ~PVF_KMPAGE;
   3822 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3823 					if (arm_cache_prefer_mask != 0) {
   3824 						omd->pvh_attrs &= ~PVF_WRITE;
   3825 					}
   3826 #endif
   3827 					atomic_dec_32(&pmap_kmpages);
   3828 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   3829 				} else if (arm_cache_prefer_mask != 0) {
   3830 					pmap_acquire_page_lock(omd);
   3831 					pool_put(&pmap_pv_pool,
   3832 					    pmap_kremove_pg(opg, va));
   3833 					pmap_release_page_lock(omd);
   3834 #endif
   3835 				}
   3836 			}
   3837 			if (l2pte_valid_p(opte)) {
   3838 #ifdef PMAP_CACHE_VIVT
   3839 				cpu_dcache_wbinv_range(va, PAGE_SIZE);
   3840 #endif
   3841 				cpu_tlb_flushD_SE(va);
   3842 			}
   3843 			if (opte) {
   3844 				l2pte_reset(ptep);
   3845 				mappings += PAGE_SIZE / L2_S_SIZE;
   3846 			}
   3847 			va += PAGE_SIZE;
   3848 			ptep += PAGE_SIZE / L2_S_SIZE;
   3849 		}
   3850 		KDASSERTMSG(mappings <= l2b->l2b_occupancy, "%u %u",
   3851 		    mappings, l2b->l2b_occupancy);
   3852 		l2b->l2b_occupancy -= mappings;
   3853 		PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep));
   3854 #ifdef UVMHIST
   3855 		total_mappings += mappings;
   3856 #endif
   3857 	}
   3858 	cpu_cpwait();
   3859 	UVMHIST_LOG(maphist, "  <--- done (%u mappings removed)",
   3860 	    total_mappings, 0, 0, 0);
   3861 }
   3862 
   3863 bool
   3864 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
   3865 {
   3866 	struct l2_dtable *l2;
   3867 	pd_entry_t *pdep, pde;
   3868 	pt_entry_t *ptep, pte;
   3869 	paddr_t pa;
   3870 	u_int l1slot;
   3871 
   3872 	pmap_acquire_pmap_lock(pm);
   3873 
   3874 	l1slot = l1pte_index(va);
   3875 	pdep = pmap_l1_kva(pm) + l1slot;
   3876 	pde = *pdep;
   3877 
   3878 	if (l1pte_section_p(pde)) {
   3879 		/*
   3880 		 * These should only happen for pmap_kernel()
   3881 		 */
   3882 		KDASSERT(pm == pmap_kernel());
   3883 		pmap_release_pmap_lock(pm);
   3884 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0
   3885 		if (l1pte_supersection_p(pde)) {
   3886 			pa = (pde & L1_SS_FRAME) | (va & L1_SS_OFFSET);
   3887 		} else
   3888 #endif
   3889 			pa = (pde & L1_S_FRAME) | (va & L1_S_OFFSET);
   3890 	} else {
   3891 		/*
   3892 		 * Note that we can't rely on the validity of the L1
   3893 		 * descriptor as an indication that a mapping exists.
   3894 		 * We have to look it up in the L2 dtable.
   3895 		 */
   3896 		l2 = pm->pm_l2[L2_IDX(l1slot)];
   3897 
   3898 		if (l2 == NULL ||
   3899 		    (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) {
   3900 			pmap_release_pmap_lock(pm);
   3901 			return false;
   3902 		}
   3903 
   3904 		pte = ptep[l2pte_index(va)];
   3905 		pmap_release_pmap_lock(pm);
   3906 
   3907 		if (pte == 0)
   3908 			return false;
   3909 
   3910 		switch (pte & L2_TYPE_MASK) {
   3911 		case L2_TYPE_L:
   3912 			pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET);
   3913 			break;
   3914 
   3915 		default:
   3916 			pa = (pte & ~PAGE_MASK) | (va & PAGE_MASK);
   3917 			break;
   3918 		}
   3919 	}
   3920 
   3921 	if (pap != NULL)
   3922 		*pap = pa;
   3923 
   3924 	return true;
   3925 }
   3926 
   3927 void
   3928 pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
   3929 {
   3930 	struct l2_bucket *l2b;
   3931 	vaddr_t next_bucket;
   3932 
   3933 	NPDEBUG(PDB_PROTECT,
   3934 	    printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n",
   3935 	    pm, sva, eva, prot));
   3936 
   3937 	if ((prot & VM_PROT_READ) == 0) {
   3938 		pmap_remove(pm, sva, eva);
   3939 		return;
   3940 	}
   3941 
   3942 	if (prot & VM_PROT_WRITE) {
   3943 		/*
   3944 		 * If this is a read->write transition, just ignore it and let
   3945 		 * uvm_fault() take care of it later.
   3946 		 */
   3947 		return;
   3948 	}
   3949 
   3950 	pmap_acquire_pmap_lock(pm);
   3951 
   3952 	const bool flush = eva - sva >= PAGE_SIZE * 4;
   3953 	u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC);
   3954 	u_int flags = 0;
   3955 
   3956 	while (sva < eva) {
   3957 		next_bucket = L2_NEXT_BUCKET_VA(sva);
   3958 		if (next_bucket > eva)
   3959 			next_bucket = eva;
   3960 
   3961 		l2b = pmap_get_l2_bucket(pm, sva);
   3962 		if (l2b == NULL) {
   3963 			sva = next_bucket;
   3964 			continue;
   3965 		}
   3966 
   3967 		pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)];
   3968 
   3969 		while (sva < next_bucket) {
   3970 			const pt_entry_t opte = *ptep;
   3971 			if (l2pte_valid_p(opte) && l2pte_writable_p(opte)) {
   3972 				struct vm_page *pg;
   3973 				u_int f;
   3974 
   3975 #ifdef PMAP_CACHE_VIVT
   3976 				/*
   3977 				 * OK, at this point, we know we're doing
   3978 				 * write-protect operation.  If the pmap is
   3979 				 * active, write-back the page.
   3980 				 */
   3981 				pmap_cache_wbinv_page(pm, sva, false,
   3982 				    PVF_REF | PVF_WRITE);
   3983 #endif
   3984 
   3985 				pg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
   3986 				pt_entry_t npte = l2pte_set_readonly(opte);
   3987 				l2pte_set(ptep, npte, opte);
   3988 				PTE_SYNC(ptep);
   3989 
   3990 				if (pg != NULL) {
   3991 					struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   3992 					paddr_t pa = VM_PAGE_TO_PHYS(pg);
   3993 
   3994 					pmap_acquire_page_lock(md);
   3995 					f = pmap_modify_pv(md, pa, pm, sva,
   3996 					    clr_mask, 0);
   3997 					pmap_vac_me_harder(md, pa, pm, sva);
   3998 					pmap_release_page_lock(md);
   3999 				} else {
   4000 					f = PVF_REF | PVF_EXEC;
   4001 				}
   4002 
   4003 				if (flush) {
   4004 					flags |= f;
   4005 				} else {
   4006 					pmap_tlb_flush_SE(pm, sva, f);
   4007 				}
   4008 			}
   4009 
   4010 			sva += PAGE_SIZE;
   4011 			ptep += PAGE_SIZE / L2_S_SIZE;
   4012 		}
   4013 	}
   4014 
   4015 	if (flush) {
   4016 		if (PV_BEEN_EXECD(flags)) {
   4017 			pmap_tlb_flushID(pm);
   4018 		} else if (PV_BEEN_REFD(flags)) {
   4019 			pmap_tlb_flushD(pm);
   4020 		}
   4021 	}
   4022 
   4023 	pmap_release_pmap_lock(pm);
   4024 }
   4025 
   4026 void
   4027 pmap_icache_sync_range(pmap_t pm, vaddr_t sva, vaddr_t eva)
   4028 {
   4029 	struct l2_bucket *l2b;
   4030 	pt_entry_t *ptep;
   4031 	vaddr_t next_bucket;
   4032 	vsize_t page_size = trunc_page(sva) + PAGE_SIZE - sva;
   4033 
   4034 	NPDEBUG(PDB_EXEC,
   4035 	    printf("pmap_icache_sync_range: pm %p sva 0x%lx eva 0x%lx\n",
   4036 	    pm, sva, eva));
   4037 
   4038 	pmap_acquire_pmap_lock(pm);
   4039 
   4040 	while (sva < eva) {
   4041 		next_bucket = L2_NEXT_BUCKET_VA(sva);
   4042 		if (next_bucket > eva)
   4043 			next_bucket = eva;
   4044 
   4045 		l2b = pmap_get_l2_bucket(pm, sva);
   4046 		if (l2b == NULL) {
   4047 			sva = next_bucket;
   4048 			continue;
   4049 		}
   4050 
   4051 		for (ptep = &l2b->l2b_kva[l2pte_index(sva)];
   4052 		     sva < next_bucket;
   4053 		     sva += page_size,
   4054 		     ptep += PAGE_SIZE / L2_S_SIZE,
   4055 		     page_size = PAGE_SIZE) {
   4056 			if (l2pte_valid_p(*ptep)) {
   4057 				cpu_icache_sync_range(sva,
   4058 				    min(page_size, eva - sva));
   4059 			}
   4060 		}
   4061 	}
   4062 
   4063 	pmap_release_pmap_lock(pm);
   4064 }
   4065 
   4066 void
   4067 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
   4068 {
   4069 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4070 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
   4071 
   4072 	NPDEBUG(PDB_PROTECT,
   4073 	    printf("pmap_page_protect: md %p (0x%08lx), prot 0x%x\n",
   4074 	    md, pa, prot));
   4075 
   4076 	switch(prot) {
   4077 	case VM_PROT_READ|VM_PROT_WRITE:
   4078 #if defined(ARM_MMU_EXTENDED)
   4079 		pmap_acquire_page_lock(md);
   4080 		pmap_clearbit(md, pa, PVF_EXEC);
   4081 		pmap_release_page_lock(md);
   4082 		break;
   4083 #endif
   4084 	case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE:
   4085 		break;
   4086 
   4087 	case VM_PROT_READ:
   4088 #if defined(ARM_MMU_EXTENDED)
   4089 		pmap_acquire_page_lock(md);
   4090 		pmap_clearbit(md, pa, PVF_WRITE|PVF_EXEC);
   4091 		pmap_release_page_lock(md);
   4092 		break;
   4093 #endif
   4094 	case VM_PROT_READ|VM_PROT_EXECUTE:
   4095 		pmap_acquire_page_lock(md);
   4096 		pmap_clearbit(md, pa, PVF_WRITE);
   4097 		pmap_release_page_lock(md);
   4098 		break;
   4099 
   4100 	default:
   4101 		pmap_page_remove(md, pa);
   4102 		break;
   4103 	}
   4104 }
   4105 
   4106 /*
   4107  * pmap_clear_modify:
   4108  *
   4109  *	Clear the "modified" attribute for a page.
   4110  */
   4111 bool
   4112 pmap_clear_modify(struct vm_page *pg)
   4113 {
   4114 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4115 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
   4116 	bool rv;
   4117 
   4118 	pmap_acquire_page_lock(md);
   4119 
   4120 	if (md->pvh_attrs & PVF_MOD) {
   4121 		rv = true;
   4122 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   4123 		/*
   4124 		 * If we are going to clear the modified bit and there are
   4125 		 * no other modified bits set, flush the page to memory and
   4126 		 * mark it clean.
   4127 		 */
   4128 		if ((md->pvh_attrs & (PVF_DMOD|PVF_NC)) == PVF_MOD)
   4129 			pmap_flush_page(md, pa, PMAP_CLEAN_PRIMARY);
   4130 #endif
   4131 		pmap_clearbit(md, pa, PVF_MOD);
   4132 	} else {
   4133 		rv = false;
   4134 	}
   4135 	pmap_release_page_lock(md);
   4136 
   4137 	return rv;
   4138 }
   4139 
   4140 /*
   4141  * pmap_clear_reference:
   4142  *
   4143  *	Clear the "referenced" attribute for a page.
   4144  */
   4145 bool
   4146 pmap_clear_reference(struct vm_page *pg)
   4147 {
   4148 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4149 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
   4150 	bool rv;
   4151 
   4152 	pmap_acquire_page_lock(md);
   4153 
   4154 	if (md->pvh_attrs & PVF_REF) {
   4155 		rv = true;
   4156 		pmap_clearbit(md, pa, PVF_REF);
   4157 	} else {
   4158 		rv = false;
   4159 	}
   4160 	pmap_release_page_lock(md);
   4161 
   4162 	return rv;
   4163 }
   4164 
   4165 /*
   4166  * pmap_is_modified:
   4167  *
   4168  *	Test if a page has the "modified" attribute.
   4169  */
   4170 /* See <arm/arm32/pmap.h> */
   4171 
   4172 /*
   4173  * pmap_is_referenced:
   4174  *
   4175  *	Test if a page has the "referenced" attribute.
   4176  */
   4177 /* See <arm/arm32/pmap.h> */
   4178 
   4179 #if defined(ARM_MMU_EXTENDED) && 0
   4180 int
   4181 pmap_prefetchabt_fixup(void *v)
   4182 {
   4183 	struct trapframe * const tf = v;
   4184 	vaddr_t va = trunc_page(tf->tf_pc);
   4185 	int rv = ABORT_FIXUP_FAILED;
   4186 
   4187 	if (!TRAP_USERMODE(tf) && va < VM_MAXUSER_ADDRESS)
   4188 		return rv;
   4189 
   4190 	kpreempt_disable();
   4191 	pmap_t pm = curcpu()->ci_pmap_cur;
   4192 	const size_t l1slot = l1pte_index(va);
   4193 	struct l2_dtable * const l2 = pm->pm_l2[L2_IDX(l1slot)];
   4194 	if (l2 == NULL)
   4195 		goto out;
   4196 
   4197 	struct l2_bucket * const l2b = &l2->l2_bucket[L2_BUCKET(l1slot)];
   4198 	if (l2b->l2b_kva == NULL)
   4199 		goto out;
   4200 
   4201 	/*
   4202 	 * Check the PTE itself.
   4203 	 */
   4204 	pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   4205 	const pt_entry_t opte = *ptep;
   4206 	if ((opte & L2_S_PROT_U) == 0 || (opte & L2_XS_XN) == 0)
   4207 		goto out;
   4208 
   4209 	paddr_t pa = l2pte_pa(pte);
   4210 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
   4211 	KASSERT(pg != NULL);
   4212 
   4213 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
   4214 
   4215 	pmap_acquire_page_lock(md);
   4216 	struct pv_entry * const pv = pmap_find_pv(md, pm, va);
   4217 	KASSERT(pv != NULL);
   4218 
   4219 	if (PV_IS_EXEC_P(pv->pv_flags)) {
   4220 		if (!PV_IS_EXEC_P(md->pvh_attrs)) {
   4221 			pmap_syncicache_page(md, pa);
   4222 		}
   4223 		rv = ABORT_FIXUP_RETURN;
   4224 		l2pte_set(ptep, opte & ~L2_XS_XN, opte);
   4225 		pmap_tlb_flush_SE(pm, va, PVF_EXEC | PVF_REF);
   4226 	}
   4227 	pmap_release_page_lock(md);
   4228 
   4229   out:
   4230 	kpreempt_enable();
   4231 	return rv;
   4232 }
   4233 #endif
   4234 
   4235 int
   4236 pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype, int user)
   4237 {
   4238 	struct l2_dtable *l2;
   4239 	struct l2_bucket *l2b;
   4240 	paddr_t pa;
   4241 	const size_t l1slot = l1pte_index(va);
   4242 	int rv = 0;
   4243 
   4244 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   4245 
   4246 	va = trunc_page(va);
   4247 
   4248 	KASSERT(!user || (pm != pmap_kernel()));
   4249 
   4250 	UVMHIST_LOG(maphist, " (pm=%#x, va=%#x, ftype=%#x, user=%d)",
   4251 	    pm, va, ftype, user);
   4252 #ifdef ARM_MMU_EXTENDED
   4253 	UVMHIST_LOG(maphist, " ti=%#x pai=%#x asid=%#x",
   4254 	    cpu_tlb_info(curcpu()), PMAP_PAI(pm, cpu_tlb_info(curcpu())),
   4255 	    PMAP_PAI(pm, cpu_tlb_info(curcpu()))->pai_asid, 0);
   4256 #endif
   4257 
   4258 	pmap_acquire_pmap_lock(pm);
   4259 
   4260 	/*
   4261 	 * If there is no l2_dtable for this address, then the process
   4262 	 * has no business accessing it.
   4263 	 *
   4264 	 * Note: This will catch userland processes trying to access
   4265 	 * kernel addresses.
   4266 	 */
   4267 	l2 = pm->pm_l2[L2_IDX(l1slot)];
   4268 	if (l2 == NULL) {
   4269 		UVMHIST_LOG(maphist, " no l2 for l1slot %#x", l1slot, 0, 0, 0);
   4270 		goto out;
   4271 	}
   4272 
   4273 	/*
   4274 	 * Likewise if there is no L2 descriptor table
   4275 	 */
   4276 	l2b = &l2->l2_bucket[L2_BUCKET(l1slot)];
   4277 	if (l2b->l2b_kva == NULL) {
   4278 		UVMHIST_LOG(maphist, " <-- done (no ptep for l1slot %#x)", l1slot, 0, 0, 0);
   4279 		goto out;
   4280 	}
   4281 
   4282 	/*
   4283 	 * Check the PTE itself.
   4284 	 */
   4285 	pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   4286 	pt_entry_t const opte = *ptep;
   4287 	if (opte == 0 || (opte & L2_TYPE_MASK) == L2_TYPE_L) {
   4288 		UVMHIST_LOG(maphist, " <-- done (empty pde for l1slot %#x)", l1slot, 0, 0, 0);
   4289 		goto out;
   4290 	}
   4291 
   4292 #ifndef ARM_HAS_VBAR
   4293 	/*
   4294 	 * Catch a userland access to the vector page mapped at 0x0
   4295 	 */
   4296 	if (user && (opte & L2_S_PROT_U) == 0) {
   4297 		UVMHIST_LOG(maphist, " <-- done (vector_page)", 0, 0, 0, 0);
   4298 		goto out;
   4299 	}
   4300 #endif
   4301 
   4302 	pa = l2pte_pa(opte);
   4303 
   4304 	if ((ftype & VM_PROT_WRITE) && !l2pte_writable_p(opte)) {
   4305 		/*
   4306 		 * This looks like a good candidate for "page modified"
   4307 		 * emulation...
   4308 		 */
   4309 		struct pv_entry *pv;
   4310 		struct vm_page *pg;
   4311 
   4312 		/* Extract the physical address of the page */
   4313 		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) {
   4314 			UVMHIST_LOG(maphist, " <-- done (mod/ref unmanaged page)", 0, 0, 0, 0);
   4315 			goto out;
   4316 		}
   4317 
   4318 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4319 
   4320 		/* Get the current flags for this page. */
   4321 		pmap_acquire_page_lock(md);
   4322 		pv = pmap_find_pv(md, pm, va);
   4323 		if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) {
   4324 			pmap_release_page_lock(md);
   4325 			UVMHIST_LOG(maphist, " <-- done (mod/ref emul: no PV)", 0, 0, 0, 0);
   4326 			goto out;
   4327 		}
   4328 
   4329 		/*
   4330 		 * Do the flags say this page is writable? If not then it
   4331 		 * is a genuine write fault. If yes then the write fault is
   4332 		 * our fault as we did not reflect the write access in the
   4333 		 * PTE. Now we know a write has occurred we can correct this
   4334 		 * and also set the modified bit
   4335 		 */
   4336 		if ((pv->pv_flags & PVF_WRITE) == 0) {
   4337 			pmap_release_page_lock(md);
   4338 			goto out;
   4339 		}
   4340 
   4341 		md->pvh_attrs |= PVF_REF | PVF_MOD;
   4342 		pv->pv_flags |= PVF_REF | PVF_MOD;
   4343 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   4344 		/*
   4345 		 * If there are cacheable mappings for this page, mark it dirty.
   4346 		 */
   4347 		if ((md->pvh_attrs & PVF_NC) == 0)
   4348 			md->pvh_attrs |= PVF_DIRTY;
   4349 #endif
   4350 #ifdef ARM_MMU_EXTENDED
   4351 		if (md->pvh_attrs & PVF_EXEC) {
   4352 			md->pvh_attrs &= ~PVF_EXEC;
   4353 			PMAPCOUNT(exec_discarded_modfixup);
   4354 		}
   4355 #endif
   4356 		pmap_release_page_lock(md);
   4357 
   4358 		/*
   4359 		 * Re-enable write permissions for the page.  No need to call
   4360 		 * pmap_vac_me_harder(), since this is just a
   4361 		 * modified-emulation fault, and the PVF_WRITE bit isn't
   4362 		 * changing. We've already set the cacheable bits based on
   4363 		 * the assumption that we can write to this page.
   4364 		 */
   4365 		const pt_entry_t npte =
   4366 		    l2pte_set_writable((opte & ~L2_TYPE_MASK) | L2_S_PROTO)
   4367 #ifdef ARM_MMU_EXTENDED
   4368 		    | (pm != pmap_kernel() ? L2_XS_nG : 0)
   4369 #endif
   4370 		    | 0;
   4371 		l2pte_set(ptep, npte, opte);
   4372 		PTE_SYNC(ptep);
   4373 		PMAPCOUNT(fixup_mod);
   4374 		rv = 1;
   4375 		UVMHIST_LOG(maphist, " <-- done (mod/ref emul: changed pte from %#x to %#x)",
   4376 		    opte, npte, 0, 0);
   4377 	} else if ((opte & L2_TYPE_MASK) == L2_TYPE_INV) {
   4378 		/*
   4379 		 * This looks like a good candidate for "page referenced"
   4380 		 * emulation.
   4381 		 */
   4382 		struct vm_page *pg;
   4383 
   4384 		/* Extract the physical address of the page */
   4385 		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) {
   4386 			UVMHIST_LOG(maphist, " <-- done (ref emul: unmanaged page)", 0, 0, 0, 0);
   4387 			goto out;
   4388 		}
   4389 
   4390 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4391 
   4392 		/* Get the current flags for this page. */
   4393 		pmap_acquire_page_lock(md);
   4394 		struct pv_entry *pv = pmap_find_pv(md, pm, va);
   4395 		if (pv == NULL || PV_IS_KENTRY_P(pv->pv_flags)) {
   4396 			pmap_release_page_lock(md);
   4397 			UVMHIST_LOG(maphist, " <-- done (ref emul no PV)", 0, 0, 0, 0);
   4398 			goto out;
   4399 		}
   4400 
   4401 		md->pvh_attrs |= PVF_REF;
   4402 		pv->pv_flags |= PVF_REF;
   4403 
   4404 		pt_entry_t npte =
   4405 		    l2pte_set_readonly((opte & ~L2_TYPE_MASK) | L2_S_PROTO);
   4406 #ifdef ARM_MMU_EXTENDED
   4407 		if (pm != pmap_kernel()) {
   4408 			npte |= L2_XS_nG;
   4409 		}
   4410 		/*
   4411 		 * If we got called from prefetch abort, then ftype will have
   4412 		 * VM_PROT_EXECUTE set.  Now see if we have no-execute set in
   4413 		 * the PTE.
   4414 		 */
   4415 		if (user && (ftype & VM_PROT_EXECUTE) && (npte & L2_XS_XN)) {
   4416 			/*
   4417 			 * Is this a mapping of an executable page?
   4418 			 */
   4419 			if ((pv->pv_flags & PVF_EXEC) == 0) {
   4420 				pmap_release_page_lock(md);
   4421 				UVMHIST_LOG(maphist, " <-- done (ref emul: no exec)",
   4422 				    0, 0, 0, 0);
   4423 				goto out;
   4424 			}
   4425 			/*
   4426 			 * If we haven't synced the page, do so now.
   4427 			 */
   4428 			if ((md->pvh_attrs & PVF_EXEC) == 0) {
   4429 				UVMHIST_LOG(maphist, " ref emul: syncicache page #%#x",
   4430 				    pa, 0, 0, 0);
   4431 				pmap_syncicache_page(md, pa);
   4432 				PMAPCOUNT(fixup_exec);
   4433 			}
   4434 			npte &= ~L2_XS_XN;
   4435 		}
   4436 #endif /* ARM_MMU_EXTENDED */
   4437 		pmap_release_page_lock(md);
   4438 		l2pte_set(ptep, npte, opte);
   4439 		PTE_SYNC(ptep);
   4440 		PMAPCOUNT(fixup_ref);
   4441 		rv = 1;
   4442 		UVMHIST_LOG(maphist, " <-- done (ref emul: changed pte from %#x to %#x)",
   4443 		    opte, npte, 0, 0);
   4444 #ifdef ARM_MMU_EXTENDED
   4445 	} else if (user && (ftype & VM_PROT_EXECUTE) && (opte & L2_XS_XN)) {
   4446 		struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
   4447 		if (pg == NULL) {
   4448 			UVMHIST_LOG(maphist, " <-- done (unmanaged page)", 0, 0, 0, 0);
   4449 			goto out;
   4450 		}
   4451 
   4452 		struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
   4453 
   4454 		/* Get the current flags for this page. */
   4455 		pmap_acquire_page_lock(md);
   4456 		struct pv_entry * const pv = pmap_find_pv(md, pm, va);
   4457 		if (pv == NULL || (pv->pv_flags & PVF_EXEC) == 0) {
   4458 			pmap_release_page_lock(md);
   4459 			UVMHIST_LOG(maphist, " <-- done (no PV or not EXEC)", 0, 0, 0, 0);
   4460 			goto out;
   4461 		}
   4462 
   4463 		/*
   4464 		 * If we haven't synced the page, do so now.
   4465 		 */
   4466 		if ((md->pvh_attrs & PVF_EXEC) == 0) {
   4467 			UVMHIST_LOG(maphist, "syncicache page #%#x",
   4468 			    pa, 0, 0, 0);
   4469 			pmap_syncicache_page(md, pa);
   4470 		}
   4471 		pmap_release_page_lock(md);
   4472 		/*
   4473 		 * Turn off no-execute.
   4474 		 */
   4475 		KASSERT(opte & L2_XS_nG);
   4476 		l2pte_set(ptep, opte & ~L2_XS_XN, opte);
   4477 		PTE_SYNC(ptep);
   4478 		rv = 1;
   4479 		PMAPCOUNT(fixup_exec);
   4480 		UVMHIST_LOG(maphist, "exec: changed pte from %#x to %#x",
   4481 		    opte, opte & ~L2_XS_XN, 0, 0);
   4482 #endif
   4483 	}
   4484 
   4485 #ifndef ARM_MMU_EXTENDED
   4486 	/*
   4487 	 * We know there is a valid mapping here, so simply
   4488 	 * fix up the L1 if necessary.
   4489 	 */
   4490 	pd_entry_t * const pdep = pmap_l1_kva(pm) + l1slot;
   4491 	pd_entry_t pde = L1_C_PROTO | l2b->l2b_pa | L1_C_DOM(pmap_domain(pm));
   4492 	if (*pdep != pde) {
   4493 		l1pte_setone(pdep, pde);
   4494 		PTE_SYNC(pdep);
   4495 		rv = 1;
   4496 		PMAPCOUNT(fixup_pdes);
   4497 	}
   4498 #endif
   4499 
   4500 #ifdef CPU_SA110
   4501 	/*
   4502 	 * There are bugs in the rev K SA110.  This is a check for one
   4503 	 * of them.
   4504 	 */
   4505 	if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 &&
   4506 	    curcpu()->ci_arm_cpurev < 3) {
   4507 		/* Always current pmap */
   4508 		if (l2pte_valid_p(opte)) {
   4509 			extern int kernel_debug;
   4510 			if (kernel_debug & 1) {
   4511 				struct proc *p = curlwp->l_proc;
   4512 				printf("prefetch_abort: page is already "
   4513 				    "mapped - pte=%p *pte=%08x\n", ptep, opte);
   4514 				printf("prefetch_abort: pc=%08lx proc=%p "
   4515 				    "process=%s\n", va, p, p->p_comm);
   4516 				printf("prefetch_abort: far=%08x fs=%x\n",
   4517 				    cpu_faultaddress(), cpu_faultstatus());
   4518 			}
   4519 #ifdef DDB
   4520 			if (kernel_debug & 2)
   4521 				Debugger();
   4522 #endif
   4523 			rv = 1;
   4524 		}
   4525 	}
   4526 #endif /* CPU_SA110 */
   4527 
   4528 #ifndef ARM_MMU_EXTENDED
   4529 	/*
   4530 	 * If 'rv == 0' at this point, it generally indicates that there is a
   4531 	 * stale TLB entry for the faulting address.  That might be due to a
   4532 	 * wrong setting of pmap_needs_pte_sync.  So set it and retry.
   4533 	 */
   4534 	if (rv == 0
   4535 	    && pm->pm_l1->l1_domain_use_count == 1
   4536 	    && pmap_needs_pte_sync == 0) {
   4537 		pmap_needs_pte_sync = 1;
   4538 		PTE_SYNC(ptep);
   4539 		PMAPCOUNT(fixup_ptesync);
   4540 		rv = 1;
   4541 	}
   4542 #endif
   4543 
   4544 #if defined(DEBUG) || 1
   4545 	/*
   4546 	 * If 'rv == 0' at this point, it generally indicates that there is a
   4547 	 * stale TLB entry for the faulting address. This happens when two or
   4548 	 * more processes are sharing an L1. Since we don't flush the TLB on
   4549 	 * a context switch between such processes, we can take domain faults
   4550 	 * for mappings which exist at the same VA in both processes. EVEN IF
   4551 	 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for
   4552 	 * example.
   4553 	 *
   4554 	 * This is extremely likely to happen if pmap_enter() updated the L1
   4555 	 * entry for a recently entered mapping. In this case, the TLB is
   4556 	 * flushed for the new mapping, but there may still be TLB entries for
   4557 	 * other mappings belonging to other processes in the 1MB range
   4558 	 * covered by the L1 entry.
   4559 	 *
   4560 	 * Since 'rv == 0', we know that the L1 already contains the correct
   4561 	 * value, so the fault must be due to a stale TLB entry.
   4562 	 *
   4563 	 * Since we always need to flush the TLB anyway in the case where we
   4564 	 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with
   4565 	 * stale TLB entries dynamically.
   4566 	 *
   4567 	 * However, the above condition can ONLY happen if the current L1 is
   4568 	 * being shared. If it happens when the L1 is unshared, it indicates
   4569 	 * that other parts of the pmap are not doing their job WRT managing
   4570 	 * the TLB.
   4571 	 */
   4572 	if (rv == 0
   4573 #ifndef ARM_MMU_EXTENDED
   4574 	    && pm->pm_l1->l1_domain_use_count == 1
   4575 #endif
   4576 	    && true) {
   4577 #ifdef DEBUG
   4578 		extern int last_fault_code;
   4579 #else
   4580 		int last_fault_code = ftype & VM_PROT_EXECUTE
   4581 		    ? armreg_ifsr_read()
   4582 		    : armreg_dfsr_read();
   4583 #endif
   4584 		printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n",
   4585 		    pm, va, ftype);
   4586 		printf("fixup: l2 %p, l2b %p, ptep %p, pte %#x\n",
   4587 		    l2, l2b, ptep, opte);
   4588 
   4589 #ifndef ARM_MMU_EXTENDED
   4590 		printf("fixup: pdep %p, pde %#x, fsr %#x\n",
   4591 		    pdep, pde, last_fault_code);
   4592 #else
   4593 		printf("fixup: pdep %p, pde %#x, ttbcr %#x\n",
   4594 		    &pmap_l1_kva(pm)[l1slot], pmap_l1_kva(pm)[l1slot],
   4595 		   armreg_ttbcr_read());
   4596 		printf("fixup: fsr %#x cpm %p casid %#x contextidr %#x dacr %#x\n",
   4597 		    last_fault_code, curcpu()->ci_pmap_cur,
   4598 		    curcpu()->ci_pmap_asid_cur,
   4599 		    armreg_contextidr_read(), armreg_dacr_read());
   4600 #ifdef _ARM_ARCH_7
   4601 		if (ftype & VM_PROT_WRITE)
   4602 			armreg_ats1cuw_write(va);
   4603 		else
   4604 			armreg_ats1cur_write(va);
   4605 		arm_isb();
   4606 		printf("fixup: par %#x\n", armreg_par_read());
   4607 #endif
   4608 #endif
   4609 #ifdef DDB
   4610 		extern int kernel_debug;
   4611 
   4612 		if (kernel_debug & 2) {
   4613 			pmap_release_pmap_lock(pm);
   4614 #ifdef UVMHIST
   4615 			KERNHIST_DUMP(maphist);
   4616 #endif
   4617 			cpu_Debugger();
   4618 			pmap_acquire_pmap_lock(pm);
   4619 		}
   4620 #endif
   4621 	}
   4622 #endif
   4623 
   4624 	pmap_tlb_flush_SE(pm, va,
   4625 	    (ftype & VM_PROT_EXECUTE) ? PVF_EXEC | PVF_REF : PVF_REF);
   4626 
   4627 	rv = 1;
   4628 
   4629 out:
   4630 	pmap_release_pmap_lock(pm);
   4631 
   4632 	return (rv);
   4633 }
   4634 
   4635 /*
   4636  * Routine:	pmap_procwr
   4637  *
   4638  * Function:
   4639  *	Synchronize caches corresponding to [addr, addr+len) in p.
   4640  *
   4641  */
   4642 void
   4643 pmap_procwr(struct proc *p, vaddr_t va, int len)
   4644 {
   4645 	/* We only need to do anything if it is the current process. */
   4646 	if (p == curproc)
   4647 		cpu_icache_sync_range(va, len);
   4648 }
   4649 
   4650 /*
   4651  * Routine:	pmap_unwire
   4652  * Function:	Clear the wired attribute for a map/virtual-address pair.
   4653  *
   4654  * In/out conditions:
   4655  *		The mapping must already exist in the pmap.
   4656  */
   4657 void
   4658 pmap_unwire(pmap_t pm, vaddr_t va)
   4659 {
   4660 	struct l2_bucket *l2b;
   4661 	pt_entry_t *ptep, pte;
   4662 	struct vm_page *pg;
   4663 	paddr_t pa;
   4664 
   4665 	NPDEBUG(PDB_WIRING, printf("pmap_unwire: pm %p, va 0x%08lx\n", pm, va));
   4666 
   4667 	pmap_acquire_pmap_lock(pm);
   4668 
   4669 	l2b = pmap_get_l2_bucket(pm, va);
   4670 	KDASSERT(l2b != NULL);
   4671 
   4672 	ptep = &l2b->l2b_kva[l2pte_index(va)];
   4673 	pte = *ptep;
   4674 
   4675 	/* Extract the physical address of the page */
   4676 	pa = l2pte_pa(pte);
   4677 
   4678 	if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
   4679 		/* Update the wired bit in the pv entry for this page. */
   4680 		struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   4681 
   4682 		pmap_acquire_page_lock(md);
   4683 		(void) pmap_modify_pv(md, pa, pm, va, PVF_WIRED, 0);
   4684 		pmap_release_page_lock(md);
   4685 	}
   4686 
   4687 	pmap_release_pmap_lock(pm);
   4688 }
   4689 
   4690 void
   4691 pmap_activate(struct lwp *l)
   4692 {
   4693 	struct cpu_info * const ci = curcpu();
   4694 	extern int block_userspace_access;
   4695 	pmap_t npm = l->l_proc->p_vmspace->vm_map.pmap;
   4696 #ifdef ARM_MMU_EXTENDED
   4697 	struct pmap_asid_info * const pai = PMAP_PAI(npm, cpu_tlb_info(ci));
   4698 #endif
   4699 
   4700 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   4701 
   4702 	UVMHIST_LOG(maphist, "(l=%#x) pm=%#x", l, npm, 0, 0);
   4703 
   4704 	/*
   4705 	 * If activating a non-current lwp or the current lwp is
   4706 	 * already active, just return.
   4707 	 */
   4708 	if (false
   4709 	    || l != curlwp
   4710 #ifdef ARM_MMU_EXTENDED
   4711 	    || (ci->ci_pmap_cur == npm &&
   4712 		(npm == pmap_kernel()
   4713 		 /* || PMAP_PAI_ASIDVALID_P(pai, cpu_tlb_info(ci)) */))
   4714 #else
   4715 	    || npm->pm_activated == true
   4716 #endif
   4717 	    || false) {
   4718 		UVMHIST_LOG(maphist, " <-- (same pmap)", curlwp, l, 0, 0);
   4719 		return;
   4720 	}
   4721 
   4722 #ifndef ARM_MMU_EXTENDED
   4723 	const uint32_t ndacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2))
   4724 	    | (DOMAIN_CLIENT << (pmap_domain(npm) * 2));
   4725 
   4726 	/*
   4727 	 * If TTB and DACR are unchanged, short-circuit all the
   4728 	 * TLB/cache management stuff.
   4729 	 */
   4730 	pmap_t opm = ci->ci_lastlwp
   4731 	    ? ci->ci_lastlwp->l_proc->p_vmspace->vm_map.pmap
   4732 	    : NULL;
   4733 	if (opm != NULL) {
   4734 		uint32_t odacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2))
   4735 		    | (DOMAIN_CLIENT << (pmap_domain(opm) * 2));
   4736 
   4737 		if (opm->pm_l1 == npm->pm_l1 && odacr == ndacr)
   4738 			goto all_done;
   4739 	}
   4740 #endif /* !ARM_MMU_EXTENDED */
   4741 
   4742 	PMAPCOUNT(activations);
   4743 	block_userspace_access = 1;
   4744 
   4745 #ifndef ARM_MMU_EXTENDED
   4746 	/*
   4747 	 * If switching to a user vmspace which is different to the
   4748 	 * most recent one, and the most recent one is potentially
   4749 	 * live in the cache, we must write-back and invalidate the
   4750 	 * entire cache.
   4751 	 */
   4752 	pmap_t rpm = ci->ci_pmap_lastuser;
   4753 #endif
   4754 
   4755 /*
   4756  * XXXSCW: There's a corner case here which can leave turds in the cache as
   4757  * reported in kern/41058. They're probably left over during tear-down and
   4758  * switching away from an exiting process. Until the root cause is identified
   4759  * and fixed, zap the cache when switching pmaps. This will result in a few
   4760  * unnecessary cache flushes, but that's better than silently corrupting data.
   4761  */
   4762 #ifndef ARM_MMU_EXTENDED
   4763 #if 0
   4764 	if (npm != pmap_kernel() && rpm && npm != rpm &&
   4765 	    rpm->pm_cstate.cs_cache) {
   4766 		rpm->pm_cstate.cs_cache = 0;
   4767 #ifdef PMAP_CACHE_VIVT
   4768 		cpu_idcache_wbinv_all();
   4769 #endif
   4770 	}
   4771 #else
   4772 	if (rpm) {
   4773 		rpm->pm_cstate.cs_cache = 0;
   4774 		if (npm == pmap_kernel())
   4775 			ci->ci_pmap_lastuser = NULL;
   4776 #ifdef PMAP_CACHE_VIVT
   4777 		cpu_idcache_wbinv_all();
   4778 #endif
   4779 	}
   4780 #endif
   4781 
   4782 	/* No interrupts while we frob the TTB/DACR */
   4783 	uint32_t oldirqstate = disable_interrupts(IF32_bits);
   4784 #endif /* !ARM_MMU_EXTENDED */
   4785 
   4786 #ifndef ARM_HAS_VBAR
   4787 	/*
   4788 	 * For ARM_VECTORS_LOW, we MUST, I repeat, MUST fix up the L1
   4789 	 * entry corresponding to 'vector_page' in the incoming L1 table
   4790 	 * before switching to it otherwise subsequent interrupts/exceptions
   4791 	 * (including domain faults!) will jump into hyperspace.
   4792 	 */
   4793 	if (npm->pm_pl1vec != NULL) {
   4794 		cpu_tlb_flushID_SE((u_int)vector_page);
   4795 		cpu_cpwait();
   4796 		*npm->pm_pl1vec = npm->pm_l1vec;
   4797 		PTE_SYNC(npm->pm_pl1vec);
   4798 	}
   4799 #endif
   4800 
   4801 #ifdef ARM_MMU_EXTENDED
   4802 	/*
   4803 	 * Assume that TTBR1 has only global mappings and TTBR0 only has
   4804 	 * non-global mappings.  To prevent speculation from doing evil things
   4805 	 * we disable translation table walks using TTBR0 before setting the
   4806 	 * CONTEXTIDR (ASID) or new TTBR0 value.  Once both are set, table
   4807 	 * walks are reenabled.
   4808 	 */
   4809 	UVMHIST_LOG(maphist, " acquiring asid", 0, 0, 0, 0);
   4810 	const uint32_t old_ttbcr = armreg_ttbcr_read();
   4811 	armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0);
   4812 	arm_isb();
   4813 	pmap_tlb_asid_acquire(npm, l);
   4814 	UVMHIST_LOG(maphist, " setting ttbr pa=%#x asid=%#x", npm->pm_l1_pa, pai->pai_asid, 0, 0);
   4815 	cpu_setttb(npm->pm_l1_pa, pai->pai_asid);
   4816 	/*
   4817 	 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 have
   4818 	 * been updated.
   4819 	 */
   4820 	arm_isb();
   4821 	if (npm != pmap_kernel()) {
   4822 		armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0);
   4823 	}
   4824 	cpu_cpwait();
   4825 	ci->ci_pmap_asid_cur = pai->pai_asid;
   4826 #else
   4827 	cpu_domains(ndacr);
   4828 	if (npm == pmap_kernel() || npm == rpm) {
   4829 		/*
   4830 		 * Switching to a kernel thread, or back to the
   4831 		 * same user vmspace as before... Simply update
   4832 		 * the TTB (no TLB flush required)
   4833 		 */
   4834 		cpu_setttb(npm->pm_l1->l1_physaddr, false);
   4835 		cpu_cpwait();
   4836 	} else {
   4837 		/*
   4838 		 * Otherwise, update TTB and flush TLB
   4839 		 */
   4840 		cpu_context_switch(npm->pm_l1->l1_physaddr);
   4841 		if (rpm != NULL)
   4842 			rpm->pm_cstate.cs_tlb = 0;
   4843 	}
   4844 
   4845 	restore_interrupts(oldirqstate);
   4846 #endif /* ARM_MMU_EXTENDED */
   4847 
   4848 	block_userspace_access = 0;
   4849 
   4850 #ifndef ARM_MMU_EXTENDED
   4851  all_done:
   4852 	/*
   4853 	 * The new pmap is resident. Make sure it's marked
   4854 	 * as resident in the cache/TLB.
   4855 	 */
   4856 	npm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
   4857 	if (npm != pmap_kernel())
   4858 		ci->ci_pmap_lastuser = npm;
   4859 
   4860 	/* The old pmap is not longer active */
   4861 	if (opm != npm) {
   4862 		if (opm != NULL)
   4863 			opm->pm_activated = false;
   4864 
   4865 		/* But the new one is */
   4866 		npm->pm_activated = true;
   4867 	}
   4868 #endif
   4869 	ci->ci_pmap_cur = npm;
   4870 	UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
   4871 }
   4872 
   4873 void
   4874 pmap_deactivate(struct lwp *l)
   4875 {
   4876 	pmap_t pm = l->l_proc->p_vmspace->vm_map.pmap;
   4877 
   4878 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   4879 
   4880 	UVMHIST_LOG(maphist, "(l=%#x) pm=%#x", l, pm, 0, 0);
   4881 
   4882 #ifdef ARM_MMU_EXTENDED
   4883 	kpreempt_disable();
   4884 	struct cpu_info * const ci = curcpu();
   4885 	struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
   4886 	/*
   4887 	 * Disable translation table walks from TTBR0 while no pmap has been
   4888 	 * activated.
   4889 	 */
   4890 	const uint32_t old_ttbcr = armreg_ttbcr_read();
   4891 	armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0);
   4892 	arm_isb();
   4893 	pmap_tlb_asid_deactivate(pm);
   4894 	cpu_setttb(pmap_kernel()->pm_l1_pa, pai->pai_asid);
   4895 	ci->ci_pmap_cur = pmap_kernel();
   4896 	kpreempt_enable();
   4897 #else
   4898 	/*
   4899 	 * If the process is exiting, make sure pmap_activate() does
   4900 	 * a full MMU context-switch and cache flush, which we might
   4901 	 * otherwise skip. See PR port-arm/38950.
   4902 	 */
   4903 	if (l->l_proc->p_sflag & PS_WEXIT)
   4904 		curcpu()->ci_lastlwp = NULL;
   4905 
   4906 	pm->pm_activated = false;
   4907 #endif
   4908 	UVMHIST_LOG(maphist, "  <-- done", 0, 0, 0, 0);
   4909 }
   4910 
   4911 void
   4912 pmap_update(pmap_t pm)
   4913 {
   4914 
   4915 	if (pm->pm_remove_all) {
   4916 #ifdef ARM_MMU_EXTENDED
   4917 		KASSERTMSG(curcpu()->ci_pmap_cur != pm || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur);
   4918 		/*
   4919 		 * Finish up the pmap_remove_all() optimisation by flushing
   4920 		 * all our ASIDs.
   4921 		 */
   4922 		pmap_tlb_asid_release_all(pm);
   4923 #else
   4924 		/*
   4925 		 * Finish up the pmap_remove_all() optimisation by flushing
   4926 		 * the TLB.
   4927 		 */
   4928 		pmap_tlb_flushID(pm);
   4929 #endif
   4930 		pm->pm_remove_all = false;
   4931 	}
   4932 
   4933 #ifdef ARM_MMU_EXTENDED
   4934 
   4935 #if defined(MULTIPROCESSOR) && PMAP_MAX_TLB > 1
   4936 	u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0);
   4937 	if (pending && pmap_tlb_shootdown_bystanders(pmap)) {
   4938 		PMAP_COUNT(shootdown_ipis);
   4939 	}
   4940 #endif
   4941 	KASSERTMSG(curcpu()->ci_pmap_cur != pm || pm->pm_pai[0].pai_asid == curcpu()->ci_pmap_asid_cur, "pmap/asid %p/%#x != %s cur pmap/asid %p/%#x", pm, pm->pm_pai[0].pai_asid, curcpu()->ci_data.cpu_name, curcpu()->ci_pmap_cur, curcpu()->ci_pmap_asid_cur);
   4942 #else
   4943 	if (pmap_is_current(pm)) {
   4944 		/*
   4945 		 * If we're dealing with a current userland pmap, move its L1
   4946 		 * to the end of the LRU.
   4947 		 */
   4948 		if (pm != pmap_kernel())
   4949 			pmap_use_l1(pm);
   4950 
   4951 		/*
   4952 		 * We can assume we're done with frobbing the cache/tlb for
   4953 		 * now. Make sure any future pmap ops don't skip cache/tlb
   4954 		 * flushes.
   4955 		 */
   4956 		pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
   4957 	}
   4958 #endif
   4959 
   4960 	PMAPCOUNT(updates);
   4961 
   4962 	/*
   4963 	 * make sure TLB/cache operations have completed.
   4964 	 */
   4965 	cpu_cpwait();
   4966 }
   4967 
   4968 void
   4969 pmap_remove_all(pmap_t pm)
   4970 {
   4971 
   4972 	/*
   4973 	 * The vmspace described by this pmap is about to be torn down.
   4974 	 * Until pmap_update() is called, UVM will only make calls
   4975 	 * to pmap_remove(). We can make life much simpler by flushing
   4976 	 * the cache now, and deferring TLB invalidation to pmap_update().
   4977 	 */
   4978 #ifdef PMAP_CACHE_VIVT
   4979 	pmap_cache_wbinv_all(pm, PVF_EXEC);
   4980 #endif
   4981 	pm->pm_remove_all = true;
   4982 }
   4983 
   4984 /*
   4985  * Retire the given physical map from service.
   4986  * Should only be called if the map contains no valid mappings.
   4987  */
   4988 void
   4989 pmap_destroy(pmap_t pm)
   4990 {
   4991 	u_int count;
   4992 
   4993 	if (pm == NULL)
   4994 		return;
   4995 
   4996 	if (pm->pm_remove_all) {
   4997 		pmap_tlb_flushID(pm);
   4998 		pm->pm_remove_all = false;
   4999 	}
   5000 
   5001 	/*
   5002 	 * Drop reference count
   5003 	 */
   5004 	mutex_enter(pm->pm_lock);
   5005 	count = --pm->pm_obj.uo_refs;
   5006 	mutex_exit(pm->pm_lock);
   5007 	if (count > 0) {
   5008 #ifndef ARM_MMU_EXTENDED
   5009 		if (pmap_is_current(pm)) {
   5010 			if (pm != pmap_kernel())
   5011 				pmap_use_l1(pm);
   5012 			pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL;
   5013 		}
   5014 #endif
   5015 		return;
   5016 	}
   5017 
   5018 	/*
   5019 	 * reference count is zero, free pmap resources and then free pmap.
   5020 	 */
   5021 
   5022 #ifndef ARM_HAS_VBAR
   5023 	if (vector_page < KERNEL_BASE) {
   5024 		KDASSERT(!pmap_is_current(pm));
   5025 
   5026 		/* Remove the vector page mapping */
   5027 		pmap_remove(pm, vector_page, vector_page + PAGE_SIZE);
   5028 		pmap_update(pm);
   5029 	}
   5030 #endif
   5031 
   5032 	LIST_REMOVE(pm, pm_list);
   5033 
   5034 	pmap_free_l1(pm);
   5035 
   5036 #ifdef ARM_MMU_EXTENDED
   5037 #ifdef MULTIPROCESSOR
   5038 	kcpuset_destroy(pm->pm_active);
   5039 	kcpuset_destroy(pm->pm_onproc);
   5040 #endif
   5041 #else
   5042 	struct cpu_info * const ci = curcpu();
   5043 	if (ci->ci_pmap_lastuser == pm)
   5044 		ci->ci_pmap_lastuser = NULL;
   5045 #endif
   5046 
   5047 	uvm_obj_destroy(&pm->pm_obj, false);
   5048 	mutex_destroy(&pm->pm_obj_lock);
   5049 	pool_cache_put(&pmap_cache, pm);
   5050 }
   5051 
   5052 
   5053 /*
   5054  * void pmap_reference(pmap_t pm)
   5055  *
   5056  * Add a reference to the specified pmap.
   5057  */
   5058 void
   5059 pmap_reference(pmap_t pm)
   5060 {
   5061 
   5062 	if (pm == NULL)
   5063 		return;
   5064 
   5065 #ifndef ARM_MMU_EXTENDED
   5066 	pmap_use_l1(pm);
   5067 #endif
   5068 
   5069 	mutex_enter(pm->pm_lock);
   5070 	pm->pm_obj.uo_refs++;
   5071 	mutex_exit(pm->pm_lock);
   5072 }
   5073 
   5074 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0
   5075 
   5076 static struct evcnt pmap_prefer_nochange_ev =
   5077     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange");
   5078 static struct evcnt pmap_prefer_change_ev =
   5079     EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change");
   5080 
   5081 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev);
   5082 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev);
   5083 
   5084 void
   5085 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td)
   5086 {
   5087 	vsize_t mask = arm_cache_prefer_mask | (PAGE_SIZE - 1);
   5088 	vaddr_t va = *vap;
   5089 	vaddr_t diff = (hint - va) & mask;
   5090 	if (diff == 0) {
   5091 		pmap_prefer_nochange_ev.ev_count++;
   5092 	} else {
   5093 		pmap_prefer_change_ev.ev_count++;
   5094 		if (__predict_false(td))
   5095 			va -= mask + 1;
   5096 		*vap = va + diff;
   5097 	}
   5098 }
   5099 #endif /* ARM_MMU_V6 | ARM_MMU_V7 */
   5100 
   5101 /*
   5102  * pmap_zero_page()
   5103  *
   5104  * Zero a given physical page by mapping it at a page hook point.
   5105  * In doing the zero page op, the page we zero is mapped cachable, as with
   5106  * StrongARM accesses to non-cached pages are non-burst making writing
   5107  * _any_ bulk data very slow.
   5108  */
   5109 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0
   5110 void
   5111 pmap_zero_page_generic(paddr_t pa)
   5112 {
   5113 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG)
   5114 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
   5115 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   5116 #endif
   5117 #if defined(PMAP_CACHE_VIPT)
   5118 	/* Choose the last page color it had, if any */
   5119 	const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask;
   5120 #else
   5121 	const vsize_t va_offset = 0;
   5122 #endif
   5123 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
   5124 	/*
   5125 	 * Is this page mapped at its natural color?
   5126 	 * If we have all of memory mapped, then just convert PA to VA.
   5127 	 */
   5128 	bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT
   5129 	   || va_offset == (pa & arm_cache_prefer_mask);
   5130 	const vaddr_t vdstp = okcolor
   5131 	    ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset))
   5132 	    : cpu_cdstp(va_offset);
   5133 #else
   5134 	const bool okcolor = false;
   5135 	const vaddr_t vdstp = cpu_cdstp(va_offset);
   5136 #endif
   5137 	pt_entry_t * const ptep = cpu_cdst_pte(va_offset);
   5138 
   5139 
   5140 #ifdef DEBUG
   5141 	if (!SLIST_EMPTY(&md->pvh_list))
   5142 		panic("pmap_zero_page: page has mappings");
   5143 #endif
   5144 
   5145 	KDASSERT((pa & PGOFSET) == 0);
   5146 
   5147 	if (!okcolor) {
   5148 		/*
   5149 		 * Hook in the page, zero it, and purge the cache for that
   5150 		 * zeroed page. Invalidate the TLB as needed.
   5151 		 */
   5152 		const pt_entry_t npte = L2_S_PROTO | pa | pte_l2_s_cache_mode
   5153 		    | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE);
   5154 		l2pte_set(ptep, npte, 0);
   5155 		PTE_SYNC(ptep);
   5156 		cpu_tlb_flushD_SE(vdstp);
   5157 		cpu_cpwait();
   5158 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) \
   5159     && !defined(ARM_MMU_EXTENDED)
   5160 		/*
   5161 		 * If we are direct-mapped and our color isn't ok, then before
   5162 		 * we bzero the page invalidate its contents from the cache and
   5163 		 * reset the color to its natural color.
   5164 		 */
   5165 		cpu_dcache_inv_range(vdstp, PAGE_SIZE);
   5166 		md->pvh_attrs &= ~arm_cache_prefer_mask;
   5167 		md->pvh_attrs |= (pa & arm_cache_prefer_mask);
   5168 #endif
   5169 	}
   5170 	bzero_page(vdstp);
   5171 	if (!okcolor) {
   5172 		/*
   5173 		 * Unmap the page.
   5174 		 */
   5175 		l2pte_reset(ptep);
   5176 		PTE_SYNC(ptep);
   5177 		cpu_tlb_flushD_SE(vdstp);
   5178 #ifdef PMAP_CACHE_VIVT
   5179 		cpu_dcache_wbinv_range(vdstp, PAGE_SIZE);
   5180 #endif
   5181 	}
   5182 #ifdef PMAP_CACHE_VIPT
   5183 	/*
   5184 	 * This page is now cache resident so it now has a page color.
   5185 	 * Any contents have been obliterated so clear the EXEC flag.
   5186 	 */
   5187 #ifndef ARM_MMU_EXTENDED
   5188 	if (!pmap_is_page_colored_p(md)) {
   5189 		PMAPCOUNT(vac_color_new);
   5190 		md->pvh_attrs |= PVF_COLORED;
   5191 	}
   5192 	md->pvh_attrs |= PVF_DIRTY;
   5193 #endif
   5194 	if (PV_IS_EXEC_P(md->pvh_attrs)) {
   5195 		md->pvh_attrs &= ~PVF_EXEC;
   5196 		PMAPCOUNT(exec_discarded_zero);
   5197 	}
   5198 #endif
   5199 }
   5200 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */
   5201 
   5202 #if ARM_MMU_XSCALE == 1
   5203 void
   5204 pmap_zero_page_xscale(paddr_t pa)
   5205 {
   5206 #ifdef DEBUG
   5207 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
   5208 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   5209 
   5210 	if (!SLIST_EMPTY(&md->pvh_list))
   5211 		panic("pmap_zero_page: page has mappings");
   5212 #endif
   5213 
   5214 	KDASSERT((pa & PGOFSET) == 0);
   5215 
   5216 	/*
   5217 	 * Hook in the page, zero it, and purge the cache for that
   5218 	 * zeroed page. Invalidate the TLB as needed.
   5219 	 */
   5220 
   5221 	pt_entry_t npte = L2_S_PROTO | pa |
   5222 	    L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) |
   5223 	    L2_C | L2_XS_T_TEX(TEX_XSCALE_X);	/* mini-data */
   5224 	l2pte_set(cdst_pte, npte, 0);
   5225 	PTE_SYNC(cdst_pte);
   5226 	cpu_tlb_flushD_SE(cdstp);
   5227 	cpu_cpwait();
   5228 	bzero_page(cdstp);
   5229 	xscale_cache_clean_minidata();
   5230 	l2pte_reset(cdst_pte);
   5231 	PTE_SYNC(cdst_pte);
   5232 }
   5233 #endif /* ARM_MMU_XSCALE == 1 */
   5234 
   5235 /* pmap_pageidlezero()
   5236  *
   5237  * The same as above, except that we assume that the page is not
   5238  * mapped.  This means we never have to flush the cache first.  Called
   5239  * from the idle loop.
   5240  */
   5241 bool
   5242 pmap_pageidlezero(paddr_t pa)
   5243 {
   5244 	bool rv = true;
   5245 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG)
   5246 	struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
   5247 	struct vm_page_md *md = VM_PAGE_TO_MD(pg);
   5248 #endif
   5249 #ifdef PMAP_CACHE_VIPT
   5250 	/* Choose the last page color it had, if any */
   5251 	const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask;
   5252 #else
   5253 	const vsize_t va_offset = 0;
   5254 #endif
   5255 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
   5256 	bool okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT
   5257 	   || va_offset == (pa & arm_cache_prefer_mask);
   5258 	const vaddr_t vdstp = okcolor
   5259 	    ? pmap_direct_mapped_phys(pa, &okcolor, cpu_cdstp(va_offset))
   5260 	    : cpu_cdstp(va_offset);
   5261 #else
   5262 	const bool okcolor = false;
   5263 	const vaddr_t vdstp = cpu_cdstp(va_offset);
   5264 #endif
   5265 	pt_entry_t * const ptep = cpu_cdst_pte(va_offset);
   5266 
   5267 
   5268 #ifdef DEBUG
   5269 	if (!SLIST_EMPTY(&md->pvh_list))
   5270 		panic("pmap_pageidlezero: page has mappings");
   5271 #endif
   5272 
   5273 	KDASSERT((pa & PGOFSET) == 0);
   5274 
   5275 	if (!okcolor) {
   5276 		/*
   5277 		 * Hook in the page, zero it, and purge the cache for that
   5278 		 * zeroed page. Invalidate the TLB as needed.
   5279 		 */
   5280 		const pt_entry_t npte = L2_S_PROTO | pa |
   5281 		    L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
   5282 		l2pte_set(ptep, npte, 0);
   5283 		PTE_SYNC(ptep);
   5284 		cpu_tlb_flushD_SE(vdstp);
   5285 		cpu_cpwait();
   5286 	}
   5287 
   5288 	uint64_t *ptr = (uint64_t *)vdstp;
   5289 	for (size_t i = 0; i < PAGE_SIZE / sizeof(*ptr); i++) {
   5290 		if (sched_curcpu_runnable_p() != 0) {
   5291 			/*
   5292 			 * A process has become ready.  Abort now,
   5293 			 * so we don't keep it waiting while we
   5294 			 * do slow memory access to finish this
   5295 			 * page.
   5296 			 */
   5297 			rv = false;
   5298 			break;
   5299 		}
   5300 		*ptr++ = 0;
   5301 	}
   5302 
   5303 #ifdef PMAP_CACHE_VIVT
   5304 	if (rv)
   5305 		/*
   5306 		 * if we aborted we'll rezero this page again later so don't
   5307 		 * purge it unless we finished it
   5308 		 */
   5309 		cpu_dcache_wbinv_range(vdstp, PAGE_SIZE);
   5310 #elif defined(PMAP_CACHE_VIPT)
   5311 	/*
   5312 	 * This page is now cache resident so it now has a page color.
   5313 	 * Any contents have been obliterated so clear the EXEC flag.
   5314 	 */
   5315 #ifndef ARM_MMU_EXTENDED
   5316 	if (!pmap_is_page_colored_p(md)) {
   5317 		PMAPCOUNT(vac_color_new);
   5318 		md->pvh_attrs |= PVF_COLORED;
   5319 	}
   5320 #endif
   5321 	if (PV_IS_EXEC_P(md->pvh_attrs)) {
   5322 		md->pvh_attrs &= ~PVF_EXEC;
   5323 		PMAPCOUNT(exec_discarded_zero);
   5324 	}
   5325 #endif
   5326 	/*
   5327 	 * Unmap the page.
   5328 	 */
   5329 	if (!okcolor) {
   5330 		l2pte_reset(ptep);
   5331 		PTE_SYNC(ptep);
   5332 		cpu_tlb_flushD_SE(vdstp);
   5333 	}
   5334 
   5335 	return rv;
   5336 }
   5337 
   5338 /*
   5339  * pmap_copy_page()
   5340  *
   5341  * Copy one physical page into another, by mapping the pages into
   5342  * hook points. The same comment regarding cachability as in
   5343  * pmap_zero_page also applies here.
   5344  */
   5345 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0
   5346 void
   5347 pmap_copy_page_generic(paddr_t src, paddr_t dst)
   5348 {
   5349 	struct vm_page * const src_pg = PHYS_TO_VM_PAGE(src);
   5350 	struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg);
   5351 #if defined(PMAP_CACHE_VIPT) || defined(DEBUG)
   5352 	struct vm_page * const dst_pg = PHYS_TO_VM_PAGE(dst);
   5353 	struct vm_page_md *dst_md = VM_PAGE_TO_MD(dst_pg);
   5354 #endif
   5355 #ifdef PMAP_CACHE_VIPT
   5356 	const vsize_t src_va_offset = src_md->pvh_attrs & arm_cache_prefer_mask;
   5357 	const vsize_t dst_va_offset = dst_md->pvh_attrs & arm_cache_prefer_mask;
   5358 #else
   5359 	const vsize_t src_va_offset = 0;
   5360 	const vsize_t dst_va_offset = 0;
   5361 #endif
   5362 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
   5363 	/*
   5364 	 * Is this page mapped at its natural color?
   5365 	 * If we have all of memory mapped, then just convert PA to VA.
   5366 	 */
   5367 	bool src_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT
   5368 	    || src_va_offset == (src & arm_cache_prefer_mask);
   5369 	bool dst_okcolor = arm_pcache.dcache_type == CACHE_TYPE_PIPT
   5370 	    || dst_va_offset == (dst & arm_cache_prefer_mask);
   5371 	const vaddr_t vsrcp = src_okcolor
   5372 	    ? pmap_direct_mapped_phys(src, &src_okcolor,
   5373 		cpu_csrcp(src_va_offset))
   5374 	    : cpu_csrcp(src_va_offset);
   5375 	const vaddr_t vdstp = pmap_direct_mapped_phys(dst, &dst_okcolor,
   5376 	    cpu_cdstp(dst_va_offset));
   5377 #else
   5378 	const bool src_okcolor = false;
   5379 	const bool dst_okcolor = false;
   5380 	const vaddr_t vsrcp = cpu_csrcp(src_va_offset);
   5381 	const vaddr_t vdstp = cpu_cdstp(dst_va_offset);
   5382 #endif
   5383 	pt_entry_t * const src_ptep = cpu_csrc_pte(src_va_offset);
   5384 	pt_entry_t * const dst_ptep = cpu_cdst_pte(dst_va_offset);
   5385 
   5386 #ifdef DEBUG
   5387 	if (!SLIST_EMPTY(&dst_md->pvh_list))
   5388 		panic("pmap_copy_page: dst page has mappings");
   5389 #endif
   5390 
   5391 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   5392 	KASSERT(arm_cache_prefer_mask == 0 || src_md->pvh_attrs & (PVF_COLORED|PVF_NC));
   5393 #endif
   5394 	KDASSERT((src & PGOFSET) == 0);
   5395 	KDASSERT((dst & PGOFSET) == 0);
   5396 
   5397 	/*
   5398 	 * Clean the source page.  Hold the source page's lock for
   5399 	 * the duration of the copy so that no other mappings can
   5400 	 * be created while we have a potentially aliased mapping.
   5401 	 */
   5402 #ifdef PMAP_CACHE_VIVT
   5403 	pmap_acquire_page_lock(src_md);
   5404 	(void) pmap_clean_page(src_md, true);
   5405 	pmap_release_page_lock(src_md);
   5406 #endif
   5407 
   5408 	/*
   5409 	 * Map the pages into the page hook points, copy them, and purge
   5410 	 * the cache for the appropriate page. Invalidate the TLB
   5411 	 * as required.
   5412 	 */
   5413 	if (!src_okcolor) {
   5414 		const pt_entry_t nsrc_pte = L2_S_PROTO
   5415 		    | src
   5416 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   5417 		    | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode)
   5418 #else // defined(PMAP_CACHE_VIVT) || defined(ARM_MMU_EXTENDED)
   5419 		    | pte_l2_s_cache_mode
   5420 #endif
   5421 		    | L2_S_PROT(PTE_KERNEL, VM_PROT_READ);
   5422 		l2pte_set(src_ptep, nsrc_pte, 0);
   5423 		PTE_SYNC(src_ptep);
   5424 		cpu_tlb_flushD_SE(vsrcp);
   5425 		cpu_cpwait();
   5426 	}
   5427 	if (!dst_okcolor) {
   5428 		const pt_entry_t ndst_pte = L2_S_PROTO | dst |
   5429 		    L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
   5430 		l2pte_set(dst_ptep, ndst_pte, 0);
   5431 		PTE_SYNC(dst_ptep);
   5432 		cpu_tlb_flushD_SE(vdstp);
   5433 		cpu_cpwait();
   5434 #if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT)
   5435 		/*
   5436 		 * If we are direct-mapped and our color isn't ok, then before
   5437 		 * we bcopy to the new page invalidate its contents from the
   5438 		 * cache and reset its color to its natural color.
   5439 		 */
   5440 		cpu_dcache_inv_range(vdstp, PAGE_SIZE);
   5441 		dst_md->pvh_attrs &= ~arm_cache_prefer_mask;
   5442 		dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask);
   5443 #endif
   5444 	}
   5445 	bcopy_page(vsrcp, vdstp);
   5446 #ifdef PMAP_CACHE_VIVT
   5447 	cpu_dcache_inv_range(vsrcp, PAGE_SIZE);
   5448 	cpu_dcache_wbinv_range(vdstp, PAGE_SIZE);
   5449 #endif
   5450 	/*
   5451 	 * Unmap the pages.
   5452 	 */
   5453 	if (!src_okcolor) {
   5454 		l2pte_reset(src_ptep);
   5455 		PTE_SYNC(src_ptep);
   5456 		cpu_tlb_flushD_SE(vsrcp);
   5457 		cpu_cpwait();
   5458 	}
   5459 	if (!dst_okcolor) {
   5460 		l2pte_reset(dst_ptep);
   5461 		PTE_SYNC(dst_ptep);
   5462 		cpu_tlb_flushD_SE(vdstp);
   5463 		cpu_cpwait();
   5464 	}
   5465 #ifdef PMAP_CACHE_VIPT
   5466 	/*
   5467 	 * Now that the destination page is in the cache, mark it as colored.
   5468 	 * If this was an exec page, discard it.
   5469 	 */
   5470 	pmap_acquire_page_lock(dst_md);
   5471 #ifndef ARM_MMU_EXTENDED
   5472 	if (arm_pcache.cache_type == CACHE_TYPE_PIPT) {
   5473 		dst_md->pvh_attrs &= ~arm_cache_prefer_mask;
   5474 		dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask);
   5475 	}
   5476 	if (!pmap_is_page_colored_p(dst_md)) {
   5477 		PMAPCOUNT(vac_color_new);
   5478 		dst_md->pvh_attrs |= PVF_COLORED;
   5479 	}
   5480 	dst_md->pvh_attrs |= PVF_DIRTY;
   5481 #endif
   5482 	if (PV_IS_EXEC_P(dst_md->pvh_attrs)) {
   5483 		dst_md->pvh_attrs &= ~PVF_EXEC;
   5484 		PMAPCOUNT(exec_discarded_copy);
   5485 	}
   5486 	pmap_release_page_lock(dst_md);
   5487 #endif
   5488 }
   5489 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6) != 0 */
   5490 
   5491 #if ARM_MMU_XSCALE == 1
   5492 void
   5493 pmap_copy_page_xscale(paddr_t src, paddr_t dst)
   5494 {
   5495 	struct vm_page *src_pg = PHYS_TO_VM_PAGE(src);
   5496 	struct vm_page_md *src_md = VM_PAGE_TO_MD(src_pg);
   5497 #ifdef DEBUG
   5498 	struct vm_page_md *dst_md = VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst));
   5499 
   5500 	if (!SLIST_EMPTY(&dst_md->pvh_list))
   5501 		panic("pmap_copy_page: dst page has mappings");
   5502 #endif
   5503 
   5504 	KDASSERT((src & PGOFSET) == 0);
   5505 	KDASSERT((dst & PGOFSET) == 0);
   5506 
   5507 	/*
   5508 	 * Clean the source page.  Hold the source page's lock for
   5509 	 * the duration of the copy so that no other mappings can
   5510 	 * be created while we have a potentially aliased mapping.
   5511 	 */
   5512 #ifdef PMAP_CACHE_VIVT
   5513 	pmap_acquire_page_lock(src_md);
   5514 	(void) pmap_clean_page(src_md, true);
   5515 	pmap_release_page_lock(src_md);
   5516 #endif
   5517 
   5518 	/*
   5519 	 * Map the pages into the page hook points, copy them, and purge
   5520 	 * the cache for the appropriate page. Invalidate the TLB
   5521 	 * as required.
   5522 	 */
   5523 	const pt_entry_t nsrc_pte = L2_S_PROTO | src
   5524 	    | L2_S_PROT(PTE_KERNEL, VM_PROT_READ)
   5525 	    | L2_C | L2_XS_T_TEX(TEX_XSCALE_X);	/* mini-data */
   5526 	l2pte_set(csrc_pte, nsrc_pte, 0);
   5527 	PTE_SYNC(csrc_pte);
   5528 
   5529 	const pt_entry_t ndst_pte = L2_S_PROTO | dst
   5530 	    | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE)
   5531 	    | L2_C | L2_XS_T_TEX(TEX_XSCALE_X);	/* mini-data */
   5532 	l2pte_set(cdst_pte, ndst_pte, 0);
   5533 	PTE_SYNC(cdst_pte);
   5534 
   5535 	cpu_tlb_flushD_SE(csrcp);
   5536 	cpu_tlb_flushD_SE(cdstp);
   5537 	cpu_cpwait();
   5538 	bcopy_page(csrcp, cdstp);
   5539 	xscale_cache_clean_minidata();
   5540 	l2pte_reset(csrc_pte);
   5541 	l2pte_reset(cdst_pte);
   5542 	PTE_SYNC(csrc_pte);
   5543 	PTE_SYNC(cdst_pte);
   5544 }
   5545 #endif /* ARM_MMU_XSCALE == 1 */
   5546 
   5547 /*
   5548  * void pmap_virtual_space(vaddr_t *start, vaddr_t *end)
   5549  *
   5550  * Return the start and end addresses of the kernel's virtual space.
   5551  * These values are setup in pmap_bootstrap and are updated as pages
   5552  * are allocated.
   5553  */
   5554 void
   5555 pmap_virtual_space(vaddr_t *start, vaddr_t *end)
   5556 {
   5557 	*start = virtual_avail;
   5558 	*end = virtual_end;
   5559 }
   5560 
   5561 /*
   5562  * Helper function for pmap_grow_l2_bucket()
   5563  */
   5564 static inline int
   5565 pmap_grow_map(vaddr_t va, paddr_t *pap)
   5566 {
   5567 	paddr_t pa;
   5568 
   5569 	if (uvm.page_init_done == false) {
   5570 #ifdef PMAP_STEAL_MEMORY
   5571 		pv_addr_t pv;
   5572 		pmap_boot_pagealloc(PAGE_SIZE,
   5573 #ifdef PMAP_CACHE_VIPT
   5574 		    arm_cache_prefer_mask,
   5575 		    va & arm_cache_prefer_mask,
   5576 #else
   5577 		    0, 0,
   5578 #endif
   5579 		    &pv);
   5580 		pa = pv.pv_pa;
   5581 #else
   5582 		if (uvm_page_physget(&pa) == false)
   5583 			return (1);
   5584 #endif	/* PMAP_STEAL_MEMORY */
   5585 	} else {
   5586 		struct vm_page *pg;
   5587 		pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
   5588 		if (pg == NULL)
   5589 			return (1);
   5590 		pa = VM_PAGE_TO_PHYS(pg);
   5591 		/*
   5592 		 * This new page must not have any mappings.  Enter it via
   5593 		 * pmap_kenter_pa and let that routine do the hard work.
   5594 		 */
   5595 		struct vm_page_md *md __diagused = VM_PAGE_TO_MD(pg);
   5596 		KASSERT(SLIST_EMPTY(&md->pvh_list));
   5597 		pmap_kenter_pa(va, pa,
   5598 		    VM_PROT_READ|VM_PROT_WRITE, PMAP_KMPAGE|PMAP_PTE);
   5599 	}
   5600 
   5601 	if (pap)
   5602 		*pap = pa;
   5603 
   5604 	PMAPCOUNT(pt_mappings);
   5605 #ifdef DEBUG
   5606 	struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va);
   5607 	KDASSERT(l2b != NULL);
   5608 
   5609 	pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
   5610 	const pt_entry_t opte = *ptep;
   5611 	KDASSERT((opte & L2_S_CACHE_MASK) == pte_l2_s_cache_mode_pt);
   5612 #endif
   5613 	memset((void *)va, 0, PAGE_SIZE);
   5614 	return (0);
   5615 }
   5616 
   5617 /*
   5618  * This is the same as pmap_alloc_l2_bucket(), except that it is only
   5619  * used by pmap_growkernel().
   5620  */
   5621 static inline struct l2_bucket *
   5622 pmap_grow_l2_bucket(pmap_t pm, vaddr_t va)
   5623 {
   5624 	struct l2_dtable *l2;
   5625 	struct l2_bucket *l2b;
   5626 	u_short l1slot;
   5627 	vaddr_t nva;
   5628 
   5629 	l1slot = l1pte_index(va);
   5630 
   5631 	if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) {
   5632 		/*
   5633 		 * No mapping at this address, as there is
   5634 		 * no entry in the L1 table.
   5635 		 * Need to allocate a new l2_dtable.
   5636 		 */
   5637 		nva = pmap_kernel_l2dtable_kva;
   5638 		if ((nva & PGOFSET) == 0) {
   5639 			/*
   5640 			 * Need to allocate a backing page
   5641 			 */
   5642 			if (pmap_grow_map(nva, NULL))
   5643 				return (NULL);
   5644 		}
   5645 
   5646 		l2 = (struct l2_dtable *)nva;
   5647 		nva += sizeof(struct l2_dtable);
   5648 
   5649 		if ((nva & PGOFSET) < (pmap_kernel_l2dtable_kva & PGOFSET)) {
   5650 			/*
   5651 			 * The new l2_dtable straddles a page boundary.
   5652 			 * Map in another page to cover it.
   5653 			 */
   5654 			if (pmap_grow_map(nva, NULL))
   5655 				return (NULL);
   5656 		}
   5657 
   5658 		pmap_kernel_l2dtable_kva = nva;
   5659 
   5660 		/*
   5661 		 * Link it into the parent pmap
   5662 		 */
   5663 		pm->pm_l2[L2_IDX(l1slot)] = l2;
   5664 	}
   5665 
   5666 	l2b = &l2->l2_bucket[L2_BUCKET(l1slot)];
   5667 
   5668 	/*
   5669 	 * Fetch pointer to the L2 page table associated with the address.
   5670 	 */
   5671 	if (l2b->l2b_kva == NULL) {
   5672 		pt_entry_t *ptep;
   5673 
   5674 		/*
   5675 		 * No L2 page table has been allocated. Chances are, this
   5676 		 * is because we just allocated the l2_dtable, above.
   5677 		 */
   5678 		nva = pmap_kernel_l2ptp_kva;
   5679 		ptep = (pt_entry_t *)nva;
   5680 		if ((nva & PGOFSET) == 0) {
   5681 			/*
   5682 			 * Need to allocate a backing page
   5683 			 */
   5684 			if (pmap_grow_map(nva, &pmap_kernel_l2ptp_phys))
   5685 				return (NULL);
   5686 			PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t));
   5687 		}
   5688 
   5689 		l2->l2_occupancy++;
   5690 		l2b->l2b_kva = ptep;
   5691 		l2b->l2b_l1slot = l1slot;
   5692 		l2b->l2b_pa = pmap_kernel_l2ptp_phys;
   5693 
   5694 		pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL;
   5695 		pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL;
   5696 	}
   5697 
   5698 	return (l2b);
   5699 }
   5700 
   5701 vaddr_t
   5702 pmap_growkernel(vaddr_t maxkvaddr)
   5703 {
   5704 	pmap_t kpm = pmap_kernel();
   5705 #ifndef ARM_MMU_EXTENDED
   5706 	struct l1_ttable *l1;
   5707 #endif
   5708 	int s;
   5709 
   5710 	if (maxkvaddr <= pmap_curmaxkvaddr)
   5711 		goto out;		/* we are OK */
   5712 
   5713 	NPDEBUG(PDB_GROWKERN,
   5714 	    printf("pmap_growkernel: growing kernel from 0x%lx to 0x%lx\n",
   5715 	    pmap_curmaxkvaddr, maxkvaddr));
   5716 
   5717 	KDASSERT(maxkvaddr <= virtual_end);
   5718 
   5719 	/*
   5720 	 * whoops!   we need to add kernel PTPs
   5721 	 */
   5722 
   5723 	s = splhigh();	/* to be safe */
   5724 	mutex_enter(kpm->pm_lock);
   5725 
   5726 	/* Map 1MB at a time */
   5727 	size_t l1slot = l1pte_index(pmap_curmaxkvaddr);
   5728 #ifdef ARM_MMU_EXTENDED
   5729 	pd_entry_t * const spdep = &kpm->pm_l1[l1slot];
   5730 	pd_entry_t *pdep = spdep;
   5731 #endif
   5732 	for (;pmap_curmaxkvaddr < maxkvaddr; pmap_curmaxkvaddr += L1_S_SIZE,
   5733 #ifdef ARM_MMU_EXTENDED
   5734 	     pdep++,
   5735 #endif
   5736 	     l1slot++) {
   5737 		struct l2_bucket *l2b =
   5738 		    pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr);
   5739 		KASSERT(l2b != NULL);
   5740 
   5741 		const pd_entry_t npde = L1_C_PROTO | l2b->l2b_pa
   5742 		    | L1_C_DOM(PMAP_DOMAIN_KERNEL);
   5743 #ifdef ARM_MMU_EXTENDED
   5744 		l1pte_setone(pdep, npde);
   5745 #else
   5746 		/* Distribute new L1 entry to all other L1s */
   5747 		SLIST_FOREACH(l1, &l1_list, l1_link) {
   5748 			pd_entry_t * const pdep = &l1->l1_kva[l1slot];
   5749 			l1pte_setone(pdep, npde);
   5750 			PDE_SYNC(pdep);
   5751 		}
   5752 #endif
   5753 	}
   5754 #ifdef ARM_MMU_EXTENDED
   5755 	PDE_SYNC_RANGE(spdep, pdep - spdep);
   5756 #endif
   5757 
   5758 #ifdef PMAP_CACHE_VIVT
   5759 	/*
   5760 	 * flush out the cache, expensive but growkernel will happen so
   5761 	 * rarely
   5762 	 */
   5763 	cpu_dcache_wbinv_all();
   5764 	cpu_tlb_flushD();
   5765 	cpu_cpwait();
   5766 #endif
   5767 
   5768 	mutex_exit(kpm->pm_lock);
   5769 	splx(s);
   5770 
   5771 out:
   5772 	return (pmap_curmaxkvaddr);
   5773 }
   5774 
   5775 /************************ Utility routines ****************************/
   5776 
   5777 #ifndef ARM_HAS_VBAR
   5778 /*
   5779  * vector_page_setprot:
   5780  *
   5781  *	Manipulate the protection of the vector page.
   5782  */
   5783 void
   5784 vector_page_setprot(int prot)
   5785 {
   5786 	struct l2_bucket *l2b;
   5787 	pt_entry_t *ptep;
   5788 
   5789 #if defined(CPU_ARMV7) || defined(CPU_ARM11)
   5790 	/*
   5791 	 * If we are using VBAR to use the vectors in the kernel, then it's
   5792 	 * already mapped in the kernel text so no need to anything here.
   5793 	 */
   5794 	if (vector_page != ARM_VECTORS_LOW && vector_page != ARM_VECTORS_HIGH) {
   5795 		KASSERT((armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0);
   5796 		return;
   5797 	}
   5798 #endif
   5799 
   5800 	l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page);
   5801 	KASSERT(l2b != NULL);
   5802 
   5803 	ptep = &l2b->l2b_kva[l2pte_index(vector_page)];
   5804 
   5805 	const pt_entry_t opte = *ptep;
   5806 #ifdef ARM_MMU_EXTENDED
   5807 	const pt_entry_t npte = (opte & ~(L2_S_PROT_MASK|L2_XS_XN))
   5808 	    | L2_S_PROT(PTE_KERNEL, prot);
   5809 #else
   5810 	const pt_entry_t npte = (opte & ~L2_S_PROT_MASK)
   5811 	    | L2_S_PROT(PTE_KERNEL, prot);
   5812 #endif
   5813 	l2pte_set(ptep, npte, opte);
   5814 	PTE_SYNC(ptep);
   5815 	cpu_tlb_flushD_SE(vector_page);
   5816 	cpu_cpwait();
   5817 }
   5818 #endif
   5819 
   5820 /*
   5821  * Fetch pointers to the PDE/PTE for the given pmap/VA pair.
   5822  * Returns true if the mapping exists, else false.
   5823  *
   5824  * NOTE: This function is only used by a couple of arm-specific modules.
   5825  * It is not safe to take any pmap locks here, since we could be right
   5826  * in the middle of debugging the pmap anyway...
   5827  *
   5828  * It is possible for this routine to return false even though a valid
   5829  * mapping does exist. This is because we don't lock, so the metadata
   5830  * state may be inconsistent.
   5831  *
   5832  * NOTE: We can return a NULL *ptp in the case where the L1 pde is
   5833  * a "section" mapping.
   5834  */
   5835 bool
   5836 pmap_get_pde_pte(pmap_t pm, vaddr_t va, pd_entry_t **pdp, pt_entry_t **ptp)
   5837 {
   5838 	struct l2_dtable *l2;
   5839 	pd_entry_t *pdep, pde;
   5840 	pt_entry_t *ptep;
   5841 	u_short l1slot;
   5842 
   5843 	if (pm->pm_l1 == NULL)
   5844 		return false;
   5845 
   5846 	l1slot = l1pte_index(va);
   5847 	*pdp = pdep = pmap_l1_kva(pm) + l1slot;
   5848 	pde = *pdep;
   5849 
   5850 	if (l1pte_section_p(pde)) {
   5851 		*ptp = NULL;
   5852 		return true;
   5853 	}
   5854 
   5855 	l2 = pm->pm_l2[L2_IDX(l1slot)];
   5856 	if (l2 == NULL ||
   5857 	    (ptep = l2->l2_bucket[L2_BUCKET(l1slot)].l2b_kva) == NULL) {
   5858 		return false;
   5859 	}
   5860 
   5861 	*ptp = &ptep[l2pte_index(va)];
   5862 	return true;
   5863 }
   5864 
   5865 bool
   5866 pmap_get_pde(pmap_t pm, vaddr_t va, pd_entry_t **pdp)
   5867 {
   5868 
   5869 	if (pm->pm_l1 == NULL)
   5870 		return false;
   5871 
   5872 	*pdp = pmap_l1_kva(pm) + l1pte_index(va);
   5873 
   5874 	return true;
   5875 }
   5876 
   5877 /************************ Bootstrapping routines ****************************/
   5878 
   5879 #ifndef ARM_MMU_EXTENDED
   5880 static void
   5881 pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt)
   5882 {
   5883 	int i;
   5884 
   5885 	l1->l1_kva = l1pt;
   5886 	l1->l1_domain_use_count = 0;
   5887 	l1->l1_domain_first = 0;
   5888 
   5889 	for (i = 0; i < PMAP_DOMAINS; i++)
   5890 		l1->l1_domain_free[i] = i + 1;
   5891 
   5892 	/*
   5893 	 * Copy the kernel's L1 entries to each new L1.
   5894 	 */
   5895 	if (pmap_initialized)
   5896 		memcpy(l1pt, pmap_l1_kva(pmap_kernel()), L1_TABLE_SIZE);
   5897 
   5898 	if (pmap_extract(pmap_kernel(), (vaddr_t)l1pt,
   5899 	    &l1->l1_physaddr) == false)
   5900 		panic("pmap_init_l1: can't get PA of L1 at %p", l1pt);
   5901 
   5902 	SLIST_INSERT_HEAD(&l1_list, l1, l1_link);
   5903 	TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru);
   5904 }
   5905 #endif /* !ARM_MMU_EXTENDED */
   5906 
   5907 /*
   5908  * pmap_bootstrap() is called from the board-specific initarm() routine
   5909  * once the kernel L1/L2 descriptors tables have been set up.
   5910  *
   5911  * This is a somewhat convoluted process since pmap bootstrap is, effectively,
   5912  * spread over a number of disparate files/functions.
   5913  *
   5914  * We are passed the following parameters
   5915  *  - kernel_l1pt
   5916  *    This is a pointer to the base of the kernel's L1 translation table.
   5917  *  - vstart
   5918  *    1MB-aligned start of managed kernel virtual memory.
   5919  *  - vend
   5920  *    1MB-aligned end of managed kernel virtual memory.
   5921  *
   5922  * We use the first parameter to build the metadata (struct l1_ttable and
   5923  * struct l2_dtable) necessary to track kernel mappings.
   5924  */
   5925 #define	PMAP_STATIC_L2_SIZE 16
   5926 void
   5927 pmap_bootstrap(vaddr_t vstart, vaddr_t vend)
   5928 {
   5929 	static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE];
   5930 #ifndef ARM_MMU_EXTENDED
   5931 	static struct l1_ttable static_l1;
   5932 	struct l1_ttable *l1 = &static_l1;
   5933 #endif
   5934 	struct l2_dtable *l2;
   5935 	struct l2_bucket *l2b;
   5936 	pd_entry_t *l1pt = (pd_entry_t *) kernel_l1pt.pv_va;
   5937 	pmap_t pm = pmap_kernel();
   5938 	pt_entry_t *ptep;
   5939 	paddr_t pa;
   5940 	vsize_t size;
   5941 	int nptes, l2idx, l2next = 0;
   5942 
   5943 #ifdef ARM_MMU_EXTENDED
   5944 	KASSERT(pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt);
   5945 	KASSERT(pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt);
   5946 #endif
   5947 
   5948 #ifdef VERBOSE_INIT_ARM
   5949 	printf("kpm ");
   5950 #endif
   5951 	/*
   5952 	 * Initialise the kernel pmap object
   5953 	 */
   5954 	curcpu()->ci_pmap_cur = pm;
   5955 #ifdef ARM_MMU_EXTENDED
   5956 	pm->pm_l1 = l1pt;
   5957 	pm->pm_l1_pa = kernel_l1pt.pv_pa;
   5958 #ifdef VERBOSE_INIT_ARM
   5959 	printf("tlb0 ");
   5960 #endif
   5961 	pmap_tlb_info_init(&pmap_tlb0_info);
   5962 #ifdef MULTIPROCESSOR
   5963 #ifdef VERBOSE_INIT_ARM
   5964 	printf("kcpusets ");
   5965 #endif
   5966 	pm->pm_onproc = kcpuset_running;
   5967 	pm->pm_active = kcpuset_running;
   5968 #endif
   5969 #else
   5970 	pm->pm_l1 = l1;
   5971 #endif
   5972 
   5973 #ifdef VERBOSE_INIT_ARM
   5974 	printf("locks ");
   5975 #endif
   5976 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   5977 	if (arm_cache_prefer_mask != 0) {
   5978 		mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_VM);
   5979 	} else {
   5980 #endif
   5981 		mutex_init(&pmap_lock, MUTEX_DEFAULT, IPL_NONE);
   5982 #if defined(PMAP_CACHE_VIPT) && !defined(ARM_MMU_EXTENDED)
   5983 	}
   5984 #endif
   5985 	mutex_init(&pm->pm_obj_lock, MUTEX_DEFAULT, IPL_NONE);
   5986 	uvm_obj_init(&pm->pm_obj, NULL, false, 1);
   5987 	uvm_obj_setlock(&pm->pm_obj, &pm->pm_obj_lock);
   5988 
   5989 #ifdef VERBOSE_INIT_ARM
   5990 	printf("l1pt ");
   5991 #endif
   5992 	/*
   5993 	 * Scan the L1 translation table created by initarm() and create
   5994 	 * the required metadata for all valid mappings found in it.
   5995 	 */
   5996 	for (size_t l1slot = 0;
   5997 	     l1slot < L1_TABLE_SIZE / sizeof(pd_entry_t);
   5998 	     l1slot++) {
   5999 		pd_entry_t pde = l1pt[l1slot];
   6000 
   6001 		/*
   6002 		 * We're only interested in Coarse mappings.
   6003 		 * pmap_extract() can deal with section mappings without
   6004 		 * recourse to checking L2 metadata.
   6005 		 */
   6006 		if ((pde & L1_TYPE_MASK) != L1_TYPE_C)
   6007 			continue;
   6008 
   6009 		/*
   6010 		 * Lookup the KVA of this L2 descriptor table
   6011 		 */
   6012 		pa = l1pte_pa(pde);
   6013 		ptep = (pt_entry_t *)kernel_pt_lookup(pa);
   6014 		if (ptep == NULL) {
   6015 			panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx",
   6016 			    (u_int)l1slot << L1_S_SHIFT, pa);
   6017 		}
   6018 
   6019 		/*
   6020 		 * Fetch the associated L2 metadata structure.
   6021 		 * Allocate a new one if necessary.
   6022 		 */
   6023 		if ((l2 = pm->pm_l2[L2_IDX(l1slot)]) == NULL) {
   6024 			if (l2next == PMAP_STATIC_L2_SIZE)
   6025 				panic("pmap_bootstrap: out of static L2s");
   6026 			pm->pm_l2[L2_IDX(l1slot)] = l2 = &static_l2[l2next++];
   6027 		}
   6028 
   6029 		/*
   6030 		 * One more L1 slot tracked...
   6031 		 */
   6032 		l2->l2_occupancy++;
   6033 
   6034 		/*
   6035 		 * Fill in the details of the L2 descriptor in the
   6036 		 * appropriate bucket.
   6037 		 */
   6038 		l2b = &l2->l2_bucket[L2_BUCKET(l1slot)];
   6039 		l2b->l2b_kva = ptep;
   6040 		l2b->l2b_pa = pa;
   6041 		l2b->l2b_l1slot = l1slot;
   6042 
   6043 		/*
   6044 		 * Establish an initial occupancy count for this descriptor
   6045 		 */
   6046 		for (l2idx = 0;
   6047 		    l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t));
   6048 		    l2idx++) {
   6049 			if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) {
   6050 				l2b->l2b_occupancy++;
   6051 			}
   6052 		}
   6053 
   6054 		/*
   6055 		 * Make sure the descriptor itself has the correct cache mode.
   6056 		 * If not, fix it, but whine about the problem. Port-meisters
   6057 		 * should consider this a clue to fix up their initarm()
   6058 		 * function. :)
   6059 		 */
   6060 		if (pmap_set_pt_cache_mode(l1pt, (vaddr_t)ptep, 1)) {
   6061 			printf("pmap_bootstrap: WARNING! wrong cache mode for "
   6062 			    "L2 pte @ %p\n", ptep);
   6063 		}
   6064 	}
   6065 
   6066 #ifdef VERBOSE_INIT_ARM
   6067 	printf("cache(l1pt) ");
   6068 #endif
   6069 	/*
   6070 	 * Ensure the primary (kernel) L1 has the correct cache mode for
   6071 	 * a page table. Bitch if it is not correctly set.
   6072 	 */
   6073 	if (pmap_set_pt_cache_mode(l1pt, kernel_l1pt.pv_va,
   6074 		    L1_TABLE_SIZE / L2_S_SIZE)) {
   6075 		printf("pmap_bootstrap: WARNING! wrong cache mode for "
   6076 		    "primary L1 @ 0x%lx\n", kernel_l1pt.pv_va);
   6077 	}
   6078 
   6079 #ifdef PMAP_CACHE_VIVT
   6080 	cpu_dcache_wbinv_all();
   6081 	cpu_tlb_flushID();
   6082 	cpu_cpwait();
   6083 #endif
   6084 
   6085 	/*
   6086 	 * now we allocate the "special" VAs which are used for tmp mappings
   6087 	 * by the pmap (and other modules).  we allocate the VAs by advancing
   6088 	 * virtual_avail (note that there are no pages mapped at these VAs).
   6089 	 *
   6090 	 * Managed KVM space start from wherever initarm() tells us.
   6091 	 */
   6092 	virtual_avail = vstart;
   6093 	virtual_end = vend;
   6094 
   6095 #ifdef VERBOSE_INIT_ARM
   6096 	printf("specials ");
   6097 #endif
   6098 #ifdef PMAP_CACHE_VIPT
   6099 	/*
   6100 	 * If we have a VIPT cache, we need one page/pte per possible alias
   6101 	 * page so we won't violate cache aliasing rules.
   6102 	 */
   6103 	virtual_avail = (virtual_avail + arm_cache_prefer_mask) & ~arm_cache_prefer_mask;
   6104 	nptes = (arm_cache_prefer_mask >> L2_S_SHIFT) + 1;
   6105 	if (arm_pcache.icache_type != CACHE_TYPE_PIPT
   6106 	    && arm_pcache.icache_way_size > nptes * L2_S_SIZE) {
   6107 		nptes = arm_pcache.icache_way_size >> L2_S_SHIFT;
   6108 	}
   6109 #else
   6110 	nptes = PAGE_SIZE / L2_S_SIZE;
   6111 #endif
   6112 #ifdef MULTIPROCESSOR
   6113 	cnptes = nptes;
   6114 	nptes *= arm_cpu_max;
   6115 #endif
   6116 	pmap_alloc_specials(&virtual_avail, nptes, &csrcp, &csrc_pte);
   6117 	pmap_set_pt_cache_mode(l1pt, (vaddr_t)csrc_pte, nptes);
   6118 	pmap_alloc_specials(&virtual_avail, nptes, &cdstp, &cdst_pte);
   6119 	pmap_set_pt_cache_mode(l1pt, (vaddr_t)cdst_pte, nptes);
   6120 	pmap_alloc_specials(&virtual_avail, nptes, &memhook, NULL);
   6121 	if (msgbufaddr == NULL) {
   6122 		pmap_alloc_specials(&virtual_avail,
   6123 		    round_page(MSGBUFSIZE) / PAGE_SIZE,
   6124 		    (void *)&msgbufaddr, NULL);
   6125 	}
   6126 
   6127 	/*
   6128 	 * Allocate a range of kernel virtual address space to be used
   6129 	 * for L2 descriptor tables and metadata allocation in
   6130 	 * pmap_growkernel().
   6131 	 */
   6132 	size = ((virtual_end - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE;
   6133 	pmap_alloc_specials(&virtual_avail,
   6134 	    round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE,
   6135 	    &pmap_kernel_l2ptp_kva, NULL);
   6136 
   6137 	size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE;
   6138 	pmap_alloc_specials(&virtual_avail,
   6139 	    round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE,
   6140 	    &pmap_kernel_l2dtable_kva, NULL);
   6141 
   6142 #ifndef ARM_MMU_EXTENDED
   6143 	/*
   6144 	 * init the static-global locks and global pmap list.
   6145 	 */
   6146 	mutex_init(&l1_lru_lock, MUTEX_DEFAULT, IPL_VM);
   6147 
   6148 	/*
   6149 	 * We can now initialise the first L1's metadata.
   6150 	 */
   6151 	SLIST_INIT(&l1_list);
   6152 	TAILQ_INIT(&l1_lru_list);
   6153 	pmap_init_l1(l1, l1pt);
   6154 #endif /* ARM_MMU_EXTENDED */
   6155 
   6156 #ifndef ARM_HAS_VBAR
   6157 	/* Set up vector page L1 details, if necessary */
   6158 	if (vector_page < KERNEL_BASE) {
   6159 		pm->pm_pl1vec = pmap_l1_kva(pm) + l1pte_index(vector_page);
   6160 		l2b = pmap_get_l2_bucket(pm, vector_page);
   6161 		KDASSERT(l2b != NULL);
   6162 		pm->pm_l1vec = l2b->l2b_pa | L1_C_PROTO |
   6163 		    L1_C_DOM(pmap_domain(pm));
   6164 	} else
   6165 		pm->pm_pl1vec = NULL;
   6166 #endif
   6167 
   6168 #ifdef VERBOSE_INIT_ARM
   6169 	printf("pools ");
   6170 #endif
   6171 	/*
   6172 	 * Initialize the pmap cache
   6173 	 */
   6174 	pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
   6175 	    "pmappl", NULL, IPL_NONE, pmap_pmap_ctor, NULL, NULL);
   6176 	LIST_INIT(&pmap_pmaps);
   6177 	LIST_INSERT_HEAD(&pmap_pmaps, pm, pm_list);
   6178 
   6179 	/*
   6180 	 * Initialize the pv pool.
   6181 	 */
   6182 	pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvepl",
   6183 	    &pmap_bootstrap_pv_allocator, IPL_NONE);
   6184 
   6185 	/*
   6186 	 * Initialize the L2 dtable pool and cache.
   6187 	 */
   6188 	pool_cache_bootstrap(&pmap_l2dtable_cache, sizeof(struct l2_dtable), 0,
   6189 	    0, 0, "l2dtblpl", NULL, IPL_NONE, pmap_l2dtable_ctor, NULL, NULL);
   6190 
   6191 	/*
   6192 	 * Initialise the L2 descriptor table pool and cache
   6193 	 */
   6194 	pool_cache_bootstrap(&pmap_l2ptp_cache, L2_TABLE_SIZE_REAL, 0,
   6195 	    L2_TABLE_SIZE_REAL, 0, "l2ptppl", NULL, IPL_NONE,
   6196 	    pmap_l2ptp_ctor, NULL, NULL);
   6197 
   6198 	mutex_init(&memlock, MUTEX_DEFAULT, IPL_NONE);
   6199 
   6200 	cpu_dcache_wbinv_all();
   6201 }
   6202 
   6203 static bool
   6204 pmap_set_pt_cache_mode(pd_entry_t *kl1, vaddr_t va, size_t nptes)
   6205 {
   6206 #ifdef ARM_MMU_EXTENDED
   6207 	return false;
   6208 #else
   6209 	if (pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt
   6210 	    && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt)
   6211 		return false;
   6212 
   6213 	const vaddr_t eva = va + nptes * PAGE_SIZE;
   6214 	int rv = 0;
   6215 
   6216 	while (va < eva) {
   6217 		/*
   6218 		 * Make sure the descriptor itself has the correct cache mode
   6219 		 */
   6220 		pd_entry_t * const pdep = &kl1[l1pte_index(va)];
   6221 		pd_entry_t pde = *pdep;
   6222 
   6223 		if (l1pte_section_p(pde)) {
   6224 			__CTASSERT((L1_S_CACHE_MASK & L1_S_V6_SUPER) == 0);
   6225 			if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) {
   6226 				*pdep = (pde & ~L1_S_CACHE_MASK) |
   6227 				    pte_l1_s_cache_mode_pt;
   6228 				PDE_SYNC(pdep);
   6229 				cpu_dcache_wbinv_range((vaddr_t)pdep,
   6230 				    sizeof(*pdep));
   6231 				rv = 1;
   6232 			}
   6233 			return rv;
   6234 			if (pde & L1_S_V6_SUPER) {
   6235 				va = (va & -L1_SS_SIZE) + L1_SS_SIZE;
   6236 			} else {
   6237 				va = (va & -L1_S_SIZE) + L1_S_SIZE;
   6238 			}
   6239 			continue;
   6240 		}
   6241 		vaddr_t pa = l1pte_pa(pde);
   6242 		pt_entry_t *ptep = (pt_entry_t *)kernel_pt_lookup(pa);
   6243 		if (ptep == NULL)
   6244 			panic("pmap_bootstrap: No PTP for va %#lx\n", va);
   6245 
   6246 		ptep += l2pte_index(va);
   6247 		const pt_entry_t opte = *ptep;
   6248 		if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
   6249 			const pt_entry_t npte = (opte & ~L2_S_CACHE_MASK)
   6250 			    | pte_l2_s_cache_mode_pt;
   6251 			l2pte_set(ptep, npte, opte);
   6252 			PTE_SYNC(ptep);
   6253 			cpu_dcache_wbinv_range((vaddr_t)ptep, sizeof(*ptep));
   6254 			rv = 1;
   6255 		}
   6256 		va += PAGE_SIZE;
   6257 	}
   6258 
   6259 	return (rv);
   6260 #endif
   6261 }
   6262 
   6263 static void
   6264 pmap_alloc_specials(vaddr_t *availp, int pages, vaddr_t *vap, pt_entry_t **ptep)
   6265 {
   6266 	vaddr_t va = *availp;
   6267 	struct l2_bucket *l2b;
   6268 
   6269 	if (ptep) {
   6270 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
   6271 		if (l2b == NULL)
   6272 			panic("pmap_alloc_specials: no l2b for 0x%lx", va);
   6273 
   6274 		if (ptep)
   6275 			*ptep = &l2b->l2b_kva[l2pte_index(va)];
   6276 	}
   6277 
   6278 	*vap = va;
   6279 	*availp = va + (PAGE_SIZE * pages);
   6280 }
   6281 
   6282 void
   6283 pmap_init(void)
   6284 {
   6285 
   6286 	/*
   6287 	 * Set the available memory vars - These do not map to real memory
   6288 	 * addresses and cannot as the physical memory is fragmented.
   6289 	 * They are used by ps for %mem calculations.
   6290 	 * One could argue whether this should be the entire memory or just
   6291 	 * the memory that is useable in a user process.
   6292 	 */
   6293 	avail_start = ptoa(VM_PHYSMEM_PTR(0)->start);
   6294 	avail_end = ptoa(VM_PHYSMEM_PTR(vm_nphysseg - 1)->end);
   6295 
   6296 	/*
   6297 	 * Now we need to free enough pv_entry structures to allow us to get
   6298 	 * the kmem_map/kmem_object allocated and inited (done after this
   6299 	 * function is finished).  to do this we allocate one bootstrap page out
   6300 	 * of kernel_map and use it to provide an initial pool of pv_entry
   6301 	 * structures.   we never free this page.
   6302 	 */
   6303 	pool_setlowat(&pmap_pv_pool, (PAGE_SIZE / sizeof(struct pv_entry)) * 2);
   6304 
   6305 #ifdef ARM_MMU_EXTENDED
   6306 	pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
   6307 #endif
   6308 
   6309 	pmap_initialized = true;
   6310 }
   6311 
   6312 static vaddr_t last_bootstrap_page = 0;
   6313 static void *free_bootstrap_pages = NULL;
   6314 
   6315 static void *
   6316 pmap_bootstrap_pv_page_alloc(struct pool *pp, int flags)
   6317 {
   6318 	extern void *pool_page_alloc(struct pool *, int);
   6319 	vaddr_t new_page;
   6320 	void *rv;
   6321 
   6322 	if (pmap_initialized)
   6323 		return (pool_page_alloc(pp, flags));
   6324 
   6325 	if (free_bootstrap_pages) {
   6326 		rv = free_bootstrap_pages;
   6327