1 /* $NetBSD: pmap_tlb.c,v 1.63 2025/09/02 07:54:25 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas at 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 __KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.63 2025/09/02 07:54:25 skrll Exp $"); 35 36 /* 37 * Manages address spaces in a TLB. 38 * 39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some 40 * implementations may share a TLB between multiple CPUs (really CPU thread 41 * contexts). This requires the TLB abstraction to be separated from the 42 * CPU abstraction. It also requires that the TLB be locked while doing 43 * TLB activities. 44 * 45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps 46 * that have a valid ASID. 47 * 48 * We allocate ASIDs in increasing order until we have exhausted the supply, 49 * then reinitialize the ASID space, and start allocating again at 1. When 50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding 51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill 52 * and, when completely filled, a reinitialization of the ASID space. 53 * 54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs 55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries 56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated, 57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise, 58 * (the normal case), any ASID present in the TLB (even those which are no 59 * longer used by a pmap) will remain active (allocated) and all other ASIDs 60 * will be freed. If the size of the TLB is much smaller than the ASID space, 61 * this algorithm completely avoids TLB invalidation. 62 * 63 * For multiprocessors, we also have to deal TLB invalidation requests from 64 * other CPUs, some of which are dealt with the reinitialization of the ASID 65 * space. Whereas above we keep the ASIDs of those pmaps which have active 66 * TLB entries, this type of reinitialization preserves the ASIDs of any 67 * "onproc" user pmap and all other ASIDs will be freed. We must do this 68 * since we can't change the current ASID. 69 * 70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active 71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in 72 * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its 73 * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for 74 * MIPS, or the ASID field of TTBR0 for AA64. The bit number used in these 75 * bitmaps comes from the CPU's cpu_index(). Even though these bitmaps contain 76 * the bits for all CPUs, the bits that correspond to the bits belonging to 77 * the CPUs sharing a TLB can only be manipulated while holding that TLB's 78 * lock. Atomic ops must be used to update them since multiple CPUs may be 79 * changing different sets of bits at same time but these sets never overlap. 80 * 81 * When a change to the local TLB may require a change in the TLB's of other 82 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if 83 * we are updating a PTE and that PTE previously was invalid and therefore 84 * couldn't support an active mapping, there's no need for an IPI since there 85 * can't be a TLB entry to invalidate. The other case is when we change a PTE 86 * to be modified we just update the local TLB. If another TLB has a stale 87 * entry, a TLB MOD exception will be raised and that will cause the local TLB 88 * to be updated. 89 * 90 * We never need to update a non-local TLB if the pmap doesn't have a valid 91 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc" 92 * we simply reset its ASID for that TLB and then when it goes "onproc" it 93 * will allocate a new ASID and any existing TLB entries will be orphaned. 94 * Only in the case that pmap has an "onproc" ASID do we actually have to send 95 * an IPI. 96 * 97 * Once we determined we must send an IPI to shootdown a TLB, we need to send 98 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU 99 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that 100 * TLB would do, but interrupting an active CPU seems best. 101 * 102 * A TLB might have multiple shootdowns active concurrently. The shootdown 103 * logic compresses these into a few cases: 104 * 0) nobody needs to have its TLB entries invalidated 105 * 1) one ASID needs to have its TLB entries invalidated 106 * 2) more than one ASID needs to have its TLB entries invalidated 107 * 3) the kernel needs to have its TLB entries invalidated 108 * 4) the kernel and one or more ASID need their TLB entries invalidated. 109 * 110 * And for each case we do: 111 * 0) nothing, 112 * 1) if that ASID is still "onproc", we invalidate the TLB entries for 113 * that single ASID. If not, just reset the pmap's ASID to invalidate 114 * and let it allocate a new ASID the next time it goes "onproc", 115 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and 116 * invalidate all non-wired non-global TLB entries, 117 * 3) we invalidate all of the non-wired global TLB entries, 118 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs) 119 * invalidate all non-wired TLB entries. 120 * 121 * As you can see, shootdowns are not concerned with addresses, just address 122 * spaces. Since the number of TLB entries is usually quite small, this avoids 123 * a lot of overhead for not much gain. 124 */ 125 126 #define __PMAP_PRIVATE 127 128 #include "opt_multiprocessor.h" 129 130 #include <sys/param.h> 131 132 #include <sys/atomic.h> 133 #include <sys/cpu.h> 134 #include <sys/kernel.h> /* for cold */ 135 #include <sys/mutex.h> 136 #include <sys/proc.h> 137 #include <sys/systm.h> 138 139 #include <uvm/uvm.h> 140 141 static kmutex_t pmap_tlb0_lock __cacheline_aligned; 142 143 #define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0) 144 145 #if KERNEL_PID > 31 146 #error "KERNEL_PID expected in range 0-31" 147 #endif 148 149 #define TLBINFO_ASID_MARK_UNUSED(ti, asid) \ 150 __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap) 151 #define TLBINFO_ASID_MARK_USED(ti, asid) \ 152 __BITMAP_SET((asid), &(ti)->ti_asid_bitmap) 153 #define TLBINFO_ASID_INUSE_P(ti, asid) \ 154 __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap) 155 #define TLBINFO_ASID_RESET(ti) \ 156 do { \ 157 __BITMAP_ZERO(&ti->ti_asid_bitmap); \ 158 for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) \ 159 TLBINFO_ASID_MARK_USED(ti, asid); \ 160 } while (0) 161 #define TLBINFO_ASID_INITIAL_FREE(asid_max) \ 162 (asid_max + 1 /* 0 */ - (1 + KERNEL_PID)) 163 164 struct pmap_tlb_info pmap_tlb0_info = { 165 .ti_name = "tlb0", 166 .ti_asid_hint = KERNEL_PID + 1, 167 #ifdef PMAP_TLB_NUM_PIDS 168 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1), 169 .ti_asids_free = IFCONSTANT( 170 TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)), 171 #endif 172 .ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID), 173 #ifdef PMAP_TLB_WIRED_UPAGES 174 .ti_wired = PMAP_TLB_WIRED_UPAGES, 175 #endif 176 .ti_lock = &pmap_tlb0_lock, 177 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais), 178 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 179 .ti_tlbinvop = TLBINV_NOBODY, 180 #endif 181 }; 182 183 #undef IFCONSTANT 184 185 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 186 struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = { 187 [0] = &pmap_tlb0_info, 188 }; 189 u_int pmap_ntlbs = 1; 190 #endif 191 192 #ifdef MULTIPROCESSOR 193 __unused static inline bool 194 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) 195 { 196 #if PMAP_TLB_MAX == 1 197 return !kcpuset_iszero(pm->pm_active); 198 #else 199 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset); 200 #endif 201 } 202 203 static inline bool 204 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti) 205 { 206 #if PMAP_TLB_MAX == 1 207 return !kcpuset_iszero(pm->pm_onproc); 208 #else 209 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset); 210 #endif 211 } 212 #endif 213 214 static void 215 pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p) 216 { 217 #ifdef DIAGNOSTIC 218 UVMHIST_FUNC(__func__); 219 UVMHIST_CALLARGS(maphist, "(ti=%#jx)", (uintptr_t)ti, 0, 0, 0); 220 221 struct pmap_asid_info *pai; 222 if (!locked_p) 223 TLBINFO_LOCK(ti); 224 LIST_FOREACH(pai, &ti->ti_pais, pai_link) { 225 KASSERT(pai != NULL); 226 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); 227 KASSERTMSG(pai->pai_asid > KERNEL_PID, 228 "pm %p asid %#x (%d)", PAI_PMAP(pai, ti), pai->pai_asid, 229 KERNEL_PID); 230 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max, 231 "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid); 232 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid), 233 "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid); 234 #ifdef MULTIPROCESSOR 235 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti)); 236 #endif 237 } 238 if (!locked_p) 239 TLBINFO_UNLOCK(ti); 240 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 241 #endif 242 } 243 244 static void 245 pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, 246 struct pmap *pm) 247 { 248 UVMHIST_FUNC(__func__); 249 UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u", 250 (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid); 251 252 /* 253 * We must have an ASID but it must not be onproc (on a processor). 254 */ 255 KASSERT(pai->pai_asid > KERNEL_PID); 256 KASSERT(pai->pai_asid <= ti->ti_asid_max); 257 #if defined(MULTIPROCESSOR) 258 KASSERT(pmap_tlb_intersecting_active_p(pm, ti)); 259 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 260 #endif 261 LIST_REMOVE(pai, pai_link); 262 #ifdef DIAGNOSTIC 263 pai->pai_link.le_prev = NULL; /* tagged as unlinked */ 264 #endif 265 /* 266 * If the platform has a cheap way to flush ASIDs then free the ASID 267 * back into the pool. On multiprocessor systems, we will flush the 268 * ASID from the TLB when it's allocated. That way we know the flush 269 * was always done in the correct TLB space. On uniprocessor systems, 270 * just do the flush now since we know that it has been used. This has 271 * a bit less overhead. Either way, this will mean that we will only 272 * need to flush all ASIDs if all ASIDs are in use and we need to 273 * allocate a new one. 274 */ 275 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 276 #ifndef MULTIPROCESSOR 277 UVMHIST_LOG(maphist, " ... asid %u flushed", pai->pai_asid, 0, 278 0, 0); 279 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 280 #endif 281 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 282 UVMHIST_LOG(maphist, " ... asid marked unused", 283 pai->pai_asid, 0, 0, 0); 284 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid); 285 ti->ti_asids_free++; 286 } 287 } 288 /* 289 * Note that we don't mark the ASID as not in use in the TLB's ASID 290 * bitmap (thus it can't be allocated until the ASID space is exhausted 291 * and therefore reinitialized). We don't want to flush the TLB for 292 * entries belonging to this ASID so we will let natural TLB entry 293 * replacement flush them out of the TLB. Any new entries for this 294 * pmap will need a new ASID allocated. 295 */ 296 pai->pai_asid = 0; 297 298 #if defined(MULTIPROCESSOR) 299 /* 300 * The bits in pm_active belonging to this TLB can only be changed 301 * while this TLB's lock is held. 302 */ 303 #if PMAP_TLB_MAX == 1 304 kcpuset_zero(pm->pm_active); 305 #else 306 kcpuset_remove(pm->pm_active, ti->ti_kcpuset); 307 #endif 308 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 309 #endif /* MULTIPROCESSOR */ 310 311 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 312 } 313 314 void 315 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti) 316 { 317 #if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT) 318 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired, 319 EVCNT_TYPE_MISC, NULL, 320 ti->ti_name, "icache syncs desired"); 321 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts, 322 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 323 ti->ti_name, "icache sync asts"); 324 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all, 325 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 326 ti->ti_name, "icache full syncs"); 327 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages, 328 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 329 ti->ti_name, "icache pages synced"); 330 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate, 331 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 332 ti->ti_name, "icache dup pages skipped"); 333 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred, 334 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 335 ti->ti_name, "icache pages deferred"); 336 #endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */ 337 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits, 338 EVCNT_TYPE_MISC, NULL, 339 ti->ti_name, "asid pool reinit"); 340 } 341 342 void 343 pmap_tlb_info_init(struct pmap_tlb_info *ti) 344 { 345 #if defined(MULTIPROCESSOR) 346 #if PMAP_TLB_MAX == 1 347 KASSERT(ti == &pmap_tlb0_info); 348 #else 349 if (ti != &pmap_tlb0_info) { 350 KASSERT(pmap_ntlbs < PMAP_TLB_MAX); 351 352 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL); 353 354 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 355 TLBINFO_ASID_RESET(ti); 356 ti->ti_asid_hint = KERNEL_PID + 1; 357 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; 358 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 359 ti->ti_tlbinvop = TLBINV_NOBODY; 360 ti->ti_victim = NULL; 361 kcpuset_create(&ti->ti_kcpuset, true); 362 ti->ti_index = pmap_ntlbs++; 363 ti->ti_wired = 0; 364 pmap_tlbs[ti->ti_index] = ti; 365 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u", 366 ti->ti_index); 367 pmap_tlb_info_evcnt_attach(ti); 368 369 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 370 return; 371 } 372 #endif 373 #endif /* MULTIPROCESSOR */ 374 KASSERT(ti == &pmap_tlb0_info); 375 KASSERT(ti->ti_lock == &pmap_tlb0_lock); 376 377 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED); 378 #if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 379 kcpuset_create(&ti->ti_kcpuset, true); 380 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu())); 381 #endif 382 383 const tlb_asid_t asid_max = pmap_md_tlb_asid_max(); 384 if (ti->ti_asid_max == 0 || asid_max < ti->ti_asid_max) { 385 ti->ti_asid_max = asid_max; 386 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 387 } 388 389 KASSERT(__type_fit(tlb_asid_t, ti->ti_asid_max + 1)); 390 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 391 } 392 393 #if defined(MULTIPROCESSOR) 394 void 395 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 396 { 397 KASSERT(!CPU_IS_PRIMARY(ci)); 398 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 399 KASSERT(cold); 400 401 TLBINFO_LOCK(ti); 402 #if PMAP_TLB_MAX > 1 403 kcpuset_set(ti->ti_kcpuset, cpu_index(ci)); 404 cpu_set_tlb_info(ci, ti); 405 #endif 406 407 /* 408 * Do any MD tlb info init. 409 */ 410 pmap_md_tlb_info_attach(ti, ci); 411 412 /* 413 * The kernel pmap uses the kcpuset_running set so it's always 414 * up-to-date. 415 */ 416 TLBINFO_UNLOCK(ti); 417 } 418 #endif /* MULTIPROCESSOR */ 419 420 #ifdef DIAGNOSTIC 421 static size_t 422 pmap_tlb_asid_count(struct pmap_tlb_info *ti) 423 { 424 size_t count = 0; 425 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) { 426 if (TLBINFO_ASID_INUSE_P(ti, asid)) 427 count++; 428 } 429 return count; 430 } 431 #endif 432 433 static void 434 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op) 435 { 436 UVMHIST_FUNC(__func__); 437 UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0); 438 439 pmap_tlb_pai_check(ti, true); 440 441 ti->ti_evcnt_asid_reinits.ev_count++; 442 443 /* 444 * First, clear the ASID bitmap (except for ASID 0 which belongs 445 * to the kernel). 446 */ 447 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 448 ti->ti_asid_hint = KERNEL_PID + 1; 449 TLBINFO_ASID_RESET(ti); 450 451 switch (op) { 452 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 453 case TLBINV_ALL: 454 tlb_invalidate_all(); 455 break; 456 case TLBINV_ALLUSER: 457 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 458 break; 459 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 460 case TLBINV_NOBODY: { 461 /* 462 * If we are just reclaiming ASIDs in the TLB, let's go find 463 * what ASIDs are in use in the TLB. Since this is a 464 * semi-expensive operation, we don't want to do it too often. 465 * So if more half of the ASIDs are in use, we don't have 466 * enough free ASIDs so invalidate the TLB entries with ASIDs 467 * and clear the ASID bitmap. That will force everyone to 468 * allocate a new ASID. 469 */ 470 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 471 pmap_tlb_asid_check(); 472 const u_int asids_found = tlb_record_asids( 473 ti->ti_asid_bitmap._b, ti->ti_asid_max); 474 pmap_tlb_asid_check(); 475 #ifdef DIAGNOSTIC 476 const u_int asids_count = pmap_tlb_asid_count(ti); 477 KASSERTMSG(asids_found == asids_count, 478 "found %u != count %u", asids_found, asids_count); 479 #endif 480 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { 481 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 482 #else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 483 /* 484 * For those systems (PowerPC) that don't require 485 * cross cpu TLB shootdowns, we have to invalidate the 486 * entire TLB because we can't record the ASIDs in use 487 * on the other CPUs. This is hopefully cheaper than 488 * than trying to use an IPI to record all the ASIDs 489 * on all the CPUs (which would be a synchronization 490 * nightmare). 491 */ 492 tlb_invalidate_all(); 493 #endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 494 TLBINFO_ASID_RESET(ti); 495 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE( 496 ti->ti_asid_max); 497 #if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 498 } else { 499 ti->ti_asids_free -= asids_found; 500 } 501 #endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */ 502 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", 503 ti->ti_asids_free); 504 break; 505 } 506 default: 507 panic("%s: unexpected op %d", __func__, op); 508 } 509 510 /* 511 * Now go through the active ASIDs. If the ASID is on a processor or 512 * we aren't invalidating all ASIDs and the TLB has an entry owned by 513 * that ASID, mark it as in use. Otherwise release the ASID. 514 */ 515 struct pmap_asid_info *pai, *next; 516 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) { 517 struct pmap * const pm = PAI_PMAP(pai, ti); 518 next = LIST_NEXT(pai, pai_link); 519 KASSERT(pm != pmap_kernel()); 520 KASSERT(pai->pai_asid > KERNEL_PID); 521 #if defined(MULTIPROCESSOR) 522 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 523 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 524 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 525 ti->ti_asids_free--; 526 } 527 continue; 528 } 529 #endif /* MULTIPROCESSOR */ 530 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 531 KASSERT(op == TLBINV_NOBODY); 532 } else { 533 pmap_tlb_pai_reset(ti, pai, pm); 534 } 535 } 536 #ifdef DIAGNOSTIC 537 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti); 538 KASSERTMSG(free_count == ti->ti_asids_free, 539 "bitmap error: %zu != %u", free_count, ti->ti_asids_free); 540 #endif 541 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 542 } 543 544 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 545 #if PMAP_TLB_MAX == 1 546 #error shootdown not required for single TLB systems 547 #endif 548 void 549 pmap_tlb_shootdown_process(void) 550 { 551 struct cpu_info * const ci = curcpu(); 552 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 553 554 UVMHIST_FUNC(__func__); 555 UVMHIST_CALLED(maphist); 556 557 KASSERT(cpu_intr_p()); 558 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, "%s: cpl (%d) < IPL_SCHED (%d)", 559 __func__, ci->ci_cpl, IPL_SCHED); 560 561 TLBINFO_LOCK(ti); 562 UVMHIST_LOG(maphist, "ti %#jx", (uintptr_t)ti, 0, 0, 0); 563 564 switch (ti->ti_tlbinvop) { 565 case TLBINV_ONE: { 566 /* 567 * We only need to invalidate one user ASID. 568 */ 569 UVMHIST_LOG(maphist, "TLBINV_ONE ti->ti_victim %#jx", (uintptr_t)ti->ti_victim, 0, 0, 0); 570 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); 571 KASSERT(ti->ti_victim != pmap_kernel()); 572 if (pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) { 573 UVMHIST_LOG(maphist, "... onproc asid %jd", pai->pai_asid, 0, 0, 0); 574 /* 575 * The victim is an active pmap so we will just 576 * invalidate its TLB entries. 577 */ 578 KASSERT(pai->pai_asid > KERNEL_PID); 579 pmap_tlb_asid_check(); 580 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 581 pmap_tlb_asid_check(); 582 } else if (pai->pai_asid) { 583 UVMHIST_LOG(maphist, "... not active asid %jd", pai->pai_asid, 0, 0, 0); 584 /* 585 * The victim is no longer an active pmap for this TLB. 586 * So simply clear its ASID and when pmap_activate is 587 * next called for this pmap, it will allocate a new 588 * ASID. 589 */ 590 pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti)); 591 } 592 break; 593 } 594 case TLBINV_ALLUSER: 595 /* 596 * Flush all user TLB entries. 597 */ 598 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER); 599 break; 600 case TLBINV_ALLKERNEL: 601 /* 602 * We need to invalidate all global TLB entries. 603 */ 604 pmap_tlb_asid_check(); 605 tlb_invalidate_globals(); 606 pmap_tlb_asid_check(); 607 break; 608 case TLBINV_ALL: 609 /* 610 * Flush all the TLB entries (user and kernel). 611 */ 612 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL); 613 break; 614 case TLBINV_NOBODY: 615 /* 616 * Might be spurious or another SMT CPU sharing this TLB 617 * could have already done the work. 618 */ 619 break; 620 } 621 622 /* 623 * Indicate we are done with shutdown event. 624 */ 625 ti->ti_victim = NULL; 626 ti->ti_tlbinvop = TLBINV_NOBODY; 627 TLBINFO_UNLOCK(ti); 628 } 629 630 /* 631 * This state machine could be encoded into an array of integers but since all 632 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which 633 * can be loaded in a single instruction. 634 */ 635 #define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \ 636 (((( (nobody) << 3 * TLBINV_NOBODY) \ 637 | ( (one) << 3 * TLBINV_ONE) \ 638 | ( (alluser) << 3 * TLBINV_ALLUSER) \ 639 | ((allkernel) << 3 * TLBINV_ALLKERNEL) \ 640 | ( (all) << 3 * TLBINV_ALL)) >> 3 * (op)) & 7) 641 642 #define TLBINV_USER_MAP(op) \ 643 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \ 644 TLBINV_ALL, TLBINV_ALL) 645 646 #define TLBINV_KERNEL_MAP(op) \ 647 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \ 648 TLBINV_ALLKERNEL, TLBINV_ALL) 649 650 bool 651 pmap_tlb_shootdown_bystanders(pmap_t pm) 652 { 653 /* 654 * We don't need to deal with our own TLB. 655 */ 656 657 UVMHIST_FUNC(__func__); 658 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 659 660 KASSERT(kpreempt_disabled()); 661 662 const struct cpu_info * const ci = curcpu(); 663 664 kcpuset_t *pm_active = ci->ci_shootdowncpus; 665 kcpuset_copy(pm_active, pm->pm_active); 666 kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset); 667 const bool kernel_p = (pm == pmap_kernel()); 668 bool ipi_sent = false; 669 670 /* 671 * If pm_active gets more bits set, then it's after all our changes 672 * have been made so they will already be cognizant of them. 673 */ 674 675 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) { 676 KASSERT(i < pmap_ntlbs); 677 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 678 KASSERT(tlbinfo_index(ti) == i); 679 UVMHIST_LOG(maphist, "ti %#jx", (uintptr_t)ti, 0, 0, 0); 680 /* 681 * Skip this TLB if there are no active mappings for it. 682 */ 683 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset)) 684 continue; 685 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 686 kcpuset_remove(pm_active, ti->ti_kcpuset); 687 TLBINFO_LOCK(ti); 688 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, 689 ti->ti_kcpuset); 690 // post decrement since ffs returns bit + 1 or 0 if no bit 691 if (j-- > 0) { 692 if (kernel_p) { 693 ti->ti_tlbinvop = 694 TLBINV_KERNEL_MAP(ti->ti_tlbinvop); 695 ti->ti_victim = NULL; 696 } else { 697 KASSERT(pai->pai_asid); 698 if (__predict_false(ti->ti_victim == pm)) { 699 KASSERT(ti->ti_tlbinvop == TLBINV_ONE); 700 /* 701 * We still need to invalidate this one 702 * ASID so there's nothing to change. 703 */ 704 } else { 705 ti->ti_tlbinvop = 706 TLBINV_USER_MAP(ti->ti_tlbinvop); 707 if (ti->ti_tlbinvop == TLBINV_ONE) 708 ti->ti_victim = pm; 709 else 710 ti->ti_victim = NULL; 711 } 712 } 713 UVMHIST_LOG(maphist, "tlbinvop %jx victim %#jx", ti->ti_tlbinvop, 714 (uintptr_t)ti->ti_victim, 0, 0); 715 TLBINFO_UNLOCK(ti); 716 /* 717 * Now we can send out the shootdown IPIs to a CPU 718 * that shares this TLB and is currently using this 719 * pmap. That CPU will process the IPI and do the 720 * all the work. Any other CPUs sharing that TLB 721 * will take advantage of that work. pm_onproc might 722 * change now that we have released the lock but we 723 * can tolerate spurious shootdowns. 724 */ 725 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN); 726 ipi_sent = true; 727 continue; 728 } 729 if (!pmap_tlb_intersecting_active_p(pm, ti)) { 730 UVMHIST_LOG(maphist, "pm %#jx not active", (uintptr_t)pm, 0, 0, 0); 731 /* 732 * If this pmap has an ASID assigned but it's not 733 * currently running, nuke its ASID. Next time the 734 * pmap is activated, it will allocate a new ASID. 735 * And best of all, we avoid an IPI. 736 */ 737 KASSERT(!kernel_p); 738 pmap_tlb_pai_reset(ti, pai, pm); 739 //ti->ti_evcnt_lazy_shots.ev_count++; 740 } 741 TLBINFO_UNLOCK(ti); 742 } 743 744 UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0); 745 746 return ipi_sent; 747 } 748 #endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 749 750 int 751 pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags) 752 { 753 KASSERT(kpreempt_disabled()); 754 755 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 756 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 757 int rv = -1; 758 759 UVMHIST_FUNC(__func__); 760 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)", 761 (uintptr_t)pm, va, pte_value(pte), flags); 762 763 KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE, 764 va, pte_value(pte)); 765 766 TLBINFO_LOCK(ti); 767 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 768 pmap_tlb_asid_check(); 769 rv = tlb_update_addr(va, pai->pai_asid, pte, 770 (flags & PMAP_TLB_INSERT) != 0); 771 pmap_tlb_asid_check(); 772 UVMHIST_LOG(maphist, 773 " %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)", 774 rv, va, pai->pai_asid, pte_value(pte)); 775 KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1, 776 "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d", 777 pm, pai->pai_asid, va, pte_value(pte), rv); 778 } 779 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 780 if (flags & PMAP_TLB_NEED_IPI) 781 pm->pm_shootdown_pending = 1; 782 #endif 783 TLBINFO_UNLOCK(ti); 784 785 UVMHIST_LOG(maphist, " <-- done (rv=%jd)", rv, 0, 0, 0); 786 787 return rv; 788 } 789 790 void 791 pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) 792 { 793 KASSERT(kpreempt_disabled()); 794 795 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 796 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 797 798 UVMHIST_FUNC(__func__); 799 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx", 800 (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid); 801 802 TLBINFO_LOCK(ti); 803 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 804 pmap_tlb_asid_check(); 805 UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx", 806 va, pai->pai_asid, 0, 0); 807 tlb_invalidate_addr(va, pai->pai_asid); 808 pmap_tlb_asid_check(); 809 } 810 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 811 pm->pm_shootdown_pending = 1; 812 #endif 813 TLBINFO_UNLOCK(ti); 814 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 815 } 816 817 static inline void 818 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm, 819 struct pmap_asid_info *pai) 820 { 821 /* 822 * We shouldn't have an ASID assigned, and thusly must not be onproc 823 * nor active. 824 */ 825 KASSERT(pm != pmap_kernel()); 826 KASSERT(pai->pai_asid == 0); 827 KASSERT(pai->pai_link.le_prev == NULL); 828 #if defined(MULTIPROCESSOR) 829 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 830 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 831 #endif 832 KASSERT(ti->ti_asids_free > 0); 833 KASSERT(ti->ti_asid_hint > KERNEL_PID); 834 835 /* 836 * If the last ASID allocated was the maximum ASID, then the 837 * hint will be out of range. Reset the hint to first 838 * available ASID. 839 */ 840 if (PMAP_TLB_FLUSH_ASID_ON_RESET 841 && ti->ti_asid_hint > ti->ti_asid_max) { 842 ti->ti_asid_hint = KERNEL_PID + 1; 843 } 844 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u", 845 ti->ti_asid_hint); 846 847 /* 848 * Let's see if the hinted ASID is free. If not search for 849 * a new one. 850 */ 851 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { 852 const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]); 853 size_t i; 854 u_long bits; 855 for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) { 856 KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1); 857 } 858 /* 859 * ffs wants to find the first bit set while we want 860 * to find the first bit cleared. 861 */ 862 const u_int n = __builtin_ffsl(bits) - 1; 863 KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)), 864 "n %u bits %#lx", n, bits); 865 KASSERT(n < nbpw); 866 ti->ti_asid_hint = n + i * nbpw; 867 } 868 869 KASSERT(ti->ti_asid_hint > KERNEL_PID); 870 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max); 871 KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET 872 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1), 873 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 874 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint), 875 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 876 877 /* 878 * The hint contains our next ASID so take it and advance the hint. 879 * Mark it as used and insert the pai into the list of active asids. 880 * There is also one less asid free in this TLB. 881 */ 882 pai->pai_asid = ti->ti_asid_hint++; 883 #ifdef MULTIPROCESSOR 884 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 885 /* 886 * Clean the new ASID from the TLB. 887 */ 888 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 889 } 890 #endif 891 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 892 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link); 893 ti->ti_asids_free--; 894 895 #if defined(MULTIPROCESSOR) 896 /* 897 * Mark that we now have an active ASID for all CPUs sharing this TLB. 898 * The bits in pm_active belonging to this TLB can only be changed 899 * while this TLBs lock is held. 900 */ 901 #if PMAP_TLB_MAX == 1 902 kcpuset_copy(pm->pm_active, kcpuset_running); 903 #else 904 kcpuset_merge(pm->pm_active, ti->ti_kcpuset); 905 #endif 906 #endif 907 } 908 909 /* 910 * Acquire a TLB address space tag (called ASID or TLBPID) and return it. 911 * ASID might have already been previously acquired. 912 */ 913 void 914 pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) 915 { 916 KASSERT(kpreempt_disabled()); 917 918 struct cpu_info * const ci = l->l_cpu; 919 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 920 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 921 922 UVMHIST_FUNC(__func__); 923 UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm, 924 (uintptr_t)l, (uintptr_t)ti, 0); 925 926 /* 927 * Kernels use a fixed ASID and thus doesn't need to acquire one. 928 */ 929 if (pm == pmap_kernel()) { 930 UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0); 931 return; 932 } 933 934 TLBINFO_LOCK(ti); 935 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); 936 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); 937 pmap_tlb_pai_check(ti, true); 938 939 if (__predict_false(!tlbinfo_asids_p(ti))) { 940 #if defined(MULTIPROCESSOR) 941 /* 942 * Mark that we are active for all CPUs sharing this TLB. 943 * The bits in pm_active belonging to this TLB can only 944 * be changed while this TLBs lock is held. 945 */ 946 #if PMAP_TLB_MAX == 1 947 kcpuset_copy(pm->pm_active, kcpuset_running); 948 #else 949 kcpuset_merge(pm->pm_active, ti->ti_kcpuset); 950 #endif 951 #endif 952 } else if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { 953 /* 954 * If we've run out ASIDs, reinitialize the ASID space. 955 */ 956 if (__predict_false(tlbinfo_noasids_p(ti))) { 957 KASSERT(l == curlwp); 958 UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0); 959 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY); 960 KASSERT(!tlbinfo_noasids_p(ti)); 961 } 962 963 /* 964 * Get an ASID. 965 */ 966 pmap_tlb_asid_alloc(ti, pm, pai); 967 UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid, 968 0, 0, 0); 969 } 970 pmap_tlb_pai_check(ti, true); 971 #if defined(MULTIPROCESSOR) 972 KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci))); 973 #endif 974 975 if (l == curlwp) { 976 #if defined(MULTIPROCESSOR) 977 /* 978 * The bits in pm_onproc belonging to this TLB can only 979 * be changed while this TLBs lock is held unless atomic 980 * operations are used. 981 */ 982 KASSERT(pm != pmap_kernel()); 983 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci)); 984 #endif 985 ci->ci_pmap_asid_cur = pai->pai_asid; 986 UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid, 987 0, 0, 0); 988 tlb_set_asid(pai->pai_asid, pm); 989 pmap_tlb_asid_check(); 990 } else { 991 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp); 992 } 993 TLBINFO_UNLOCK(ti); 994 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 995 } 996 997 void 998 pmap_tlb_asid_deactivate(pmap_t pm) 999 { 1000 UVMHIST_FUNC(__func__); 1001 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 1002 1003 KASSERT(kpreempt_disabled()); 1004 #if defined(MULTIPROCESSOR) 1005 /* 1006 * The kernel pmap is aways onproc and active and must never have 1007 * those bits cleared. If pmap_remove_all was called, it has already 1008 * deactivated the pmap and thusly onproc will be 0 so there's nothing 1009 * to do. 1010 */ 1011 if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) { 1012 struct cpu_info * const ci = curcpu(); 1013 KASSERT(!cpu_intr_p()); 1014 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)), 1015 "%s: pmap %p onproc %p doesn't include cpu %d (%p)", 1016 __func__, pm, pm->pm_onproc, cpu_index(ci), ci); 1017 /* 1018 * The bits in pm_onproc that belong to this TLB can 1019 * be changed while this TLBs lock is not held as long 1020 * as we use atomic ops. 1021 */ 1022 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); 1023 } 1024 #endif 1025 curcpu()->ci_pmap_asid_cur = KERNEL_PID; 1026 tlb_set_asid(KERNEL_PID, pmap_kernel()); 1027 1028 pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false); 1029 #if defined(DEBUG) 1030 pmap_tlb_asid_check(); 1031 #endif 1032 UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 1033 } 1034 1035 void 1036 pmap_tlb_asid_release_all(struct pmap *pm) 1037 { 1038 UVMHIST_FUNC(__func__); 1039 UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 1040 1041 KASSERT(pm != pmap_kernel()); 1042 #if defined(MULTIPROCESSOR) 1043 //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX 1044 struct cpu_info * const ci __diagused = curcpu(); 1045 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 1046 #if PMAP_TLB_MAX > 1 1047 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { 1048 KASSERT(i < pmap_ntlbs); 1049 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 1050 #else 1051 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1052 #endif 1053 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1054 TLBINFO_LOCK(ti); 1055 if (PMAP_PAI_ASIDVALID_P(pai, ti)) { 1056 /* 1057 * This pmap should not be in use by any other cpu so 1058 * we can just reset and be happy. 1059 */ 1060 if (ti->ti_victim == pm) 1061 ti->ti_victim = NULL; 1062 if (__predict_true(tlbinfo_asids_p(ti))) 1063 pmap_tlb_pai_reset(ti, pai, pm); 1064 } 1065 KASSERT(pai->pai_link.le_prev == NULL); 1066 TLBINFO_UNLOCK(ti); 1067 #if PMAP_TLB_MAX > 1 1068 } 1069 #endif 1070 #ifdef DIAGNOSTIC 1071 for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) { 1072 KASSERTMSG(pm->pm_pai[i].pai_asid == 0, 1073 "pm %p i %zu asid %u", 1074 pm, i, pm->pm_pai[i].pai_asid); 1075 } 1076 #endif 1077 #else 1078 /* 1079 * Handle the case of an UP kernel which only has, at most, one TLB. 1080 * If the pmap has an ASID allocated, free it. 1081 */ 1082 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1083 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1084 TLBINFO_LOCK(ti); 1085 if (pai->pai_asid > KERNEL_PID) { 1086 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { 1087 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 1088 } else { 1089 pmap_tlb_pai_reset(ti, pai, pm); 1090 } 1091 } 1092 TLBINFO_UNLOCK(ti); 1093 #endif /* MULTIPROCESSOR */ 1094 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 1095 } 1096 1097 void 1098 pmap_tlb_asid_check(void) 1099 { 1100 UVMHIST_FUNC(__func__); 1101 UVMHIST_CALLED(maphist); 1102 1103 #ifdef DEBUG 1104 kpreempt_disable(); 1105 const tlb_asid_t asid __debugused = tlb_get_asid(); 1106 UVMHIST_LOG(maphist, " asid %u vs pmap_cur_asid %u", asid, 1107 curcpu()->ci_pmap_asid_cur, 0, 0); 1108 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur, 1109 "%s: asid (%#x) != current asid (%#x)", 1110 __func__, asid, curcpu()->ci_pmap_asid_cur); 1111 kpreempt_enable(); 1112 #endif 1113 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 1114 } 1115 1116 #ifdef DEBUG 1117 void 1118 pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) 1119 { 1120 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 1121 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1122 TLBINFO_LOCK(ti); 1123 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID) 1124 tlb_walk(pm, func); 1125 TLBINFO_UNLOCK(ti); 1126 } 1127 #endif /* DEBUG */ 1128 1129 #ifdef DDB 1130 void 1131 pmap_db_tlb_print(struct pmap *pm, 1132 void (*pr)(const char *, ...) __printflike(1, 2)) 1133 { 1134 #if !defined(MULTIPROCESSOR) || PMAP_TLB_MAX == 1 1135 pr(" asid %5u\n", pm->pm_pai[0].pai_asid); 1136 #else 1137 for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) { 1138 pr(" tlb %zu asid %5u\n", i, pm->pm_pai[i].pai_asid); 1139 } 1140 #endif 1141 } 1142 #endif /* DDB */ 1143
Indexes created Sun Sep 21 18:09:52 GMT 2025