Lines Matching defs:ti
149 #define TLBINFO_ASID_MARK_UNUSED(ti, asid) \
150 __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap)
151 #define TLBINFO_ASID_MARK_USED(ti, asid) \
152 __BITMAP_SET((asid), &(ti)->ti_asid_bitmap)
153 #define TLBINFO_ASID_INUSE_P(ti, asid) \
154 __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap)
155 #define TLBINFO_ASID_RESET(ti) \
157 __BITMAP_ZERO(&ti->ti_asid_bitmap); \
159 TLBINFO_ASID_MARK_USED(ti, asid); \
194 pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
199 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
204 pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
209 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
215 pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p)
219 UVMHIST_CALLARGS(maphist, "(ti=%#jx)", (uintptr_t)ti, 0, 0, 0);
223 TLBINFO_LOCK(ti);
224 LIST_FOREACH(pai, &ti->ti_pais, pai_link) {
226 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());
228 "pm %p asid %#x (%d)", PAI_PMAP(pai, ti), pai->pai_asid,
230 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max,
231 "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid);
232 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid),
233 "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid);
235 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti));
239 TLBINFO_UNLOCK(ti);
245 pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
249 UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u",
250 (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid);
256 KASSERT(pai->pai_asid <= ti->ti_asid_max);
258 KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
259 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
281 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
284 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid);
285 ti->ti_asids_free++;
306 kcpuset_remove(pm->pm_active, ti->ti_kcpuset);
308 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
315 pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti)
318 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired,
320 ti->ti_name, "icache syncs desired");
321 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts,
322 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
323 ti->ti_name, "icache sync asts");
324 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all,
325 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
326 ti->ti_name, "icache full syncs");
327 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages,
328 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
329 ti->ti_name, "icache pages synced");
330 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate,
331 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
332 ti->ti_name, "icache dup pages skipped");
333 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred,
334 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
335 ti->ti_name, "icache pages deferred");
337 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits,
339 ti->ti_name, "asid pool reinit");
343 pmap_tlb_info_init(struct pmap_tlb_info *ti)
347 KASSERT(ti == &pmap_tlb0_info);
349 if (ti != &pmap_tlb0_info) {
354 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
355 TLBINFO_ASID_RESET(ti);
356 ti->ti_asid_hint = KERNEL_PID + 1;
357 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;
358 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
359 ti->ti_tlbinvop = TLBINV_NOBODY;
360 ti->ti_victim = NULL;
361 kcpuset_create(&ti->ti_kcpuset, true);
362 ti->ti_index = pmap_ntlbs++;
363 ti->ti_wired = 0;
364 pmap_tlbs[ti->ti_index] = ti;
365 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u",
366 ti->ti_index);
367 pmap_tlb_info_evcnt_attach(ti);
369 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
374 KASSERT(ti == &pmap_tlb0_info);
375 KASSERT(ti->ti_lock == &pmap_tlb0_lock);
377 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
379 kcpuset_create(&ti->ti_kcpuset, true);
380 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu()));
384 if (ti->ti_asid_max == 0 || asid_max < ti->ti_asid_max) {
385 ti->ti_asid_max = asid_max;
386 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
389 KASSERT(__type_fit(tlb_asid_t, ti->ti_asid_max + 1));
390 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
395 pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
401 TLBINFO_LOCK(ti);
403 kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
404 cpu_set_tlb_info(ci, ti);
410 pmap_md_tlb_info_attach(ti, ci);
416 TLBINFO_UNLOCK(ti);
422 pmap_tlb_asid_count(struct pmap_tlb_info *ti)
425 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) {
426 if (TLBINFO_ASID_INUSE_P(ti, asid))
434 pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op)
437 UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0);
439 pmap_tlb_pai_check(ti, true);
441 ti->ti_evcnt_asid_reinits.ev_count++;
447 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
448 ti->ti_asid_hint = KERNEL_PID + 1;
449 TLBINFO_ASID_RESET(ti);
457 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
473 ti->ti_asid_bitmap._b, ti->ti_asid_max);
476 const u_int asids_count = pmap_tlb_asid_count(ti);
480 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {
481 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
494 TLBINFO_ASID_RESET(ti);
495 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(
496 ti->ti_asid_max);
499 ti->ti_asids_free -= asids_found;
502 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
503 ti->ti_asids_free);
516 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) {
517 struct pmap * const pm = PAI_PMAP(pai, ti);
522 if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
523 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
524 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
525 ti->ti_asids_free--;
530 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
533 pmap_tlb_pai_reset(ti, pai, pm);
537 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti);
538 KASSERTMSG(free_count == ti->ti_asids_free,
539 "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
552 struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
561 TLBINFO_LOCK(ti);
562 UVMHIST_LOG(maphist, "ti %#jx", (uintptr_t)ti, 0, 0, 0);
564 switch (ti->ti_tlbinvop) {
569 UVMHIST_LOG(maphist, "TLBINV_ONE ti->ti_victim %#jx", (uintptr_t)ti->ti_victim, 0, 0, 0);
570 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
571 KASSERT(ti->ti_victim != pmap_kernel());
572 if (pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) {
590 pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti));
598 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER);
612 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL);
625 ti->ti_victim = NULL;
626 ti->ti_tlbinvop = TLBINV_NOBODY;
627 TLBINFO_UNLOCK(ti);
677 struct pmap_tlb_info * const ti = pmap_tlbs[i];
678 KASSERT(tlbinfo_index(ti) == i);
679 UVMHIST_LOG(maphist, "ti %#jx", (uintptr_t)ti, 0, 0, 0);
683 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset))
685 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
686 kcpuset_remove(pm_active, ti->ti_kcpuset);
687 TLBINFO_LOCK(ti);
689 ti->ti_kcpuset);
693 ti->ti_tlbinvop =
694 TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
695 ti->ti_victim = NULL;
698 if (__predict_false(ti->ti_victim == pm)) {
699 KASSERT(ti->ti_tlbinvop == TLBINV_ONE);
705 ti->ti_tlbinvop =
706 TLBINV_USER_MAP(ti->ti_tlbinvop);
707 if (ti->ti_tlbinvop == TLBINV_ONE)
708 ti->ti_victim = pm;
710 ti->ti_victim = NULL;
713 UVMHIST_LOG(maphist, "tlbinvop %jx victim %#jx", ti->ti_tlbinvop,
714 (uintptr_t)ti->ti_victim, 0, 0);
715 TLBINFO_UNLOCK(ti);
729 if (!pmap_tlb_intersecting_active_p(pm, ti)) {
738 pmap_tlb_pai_reset(ti, pai, pm);
739 //ti->ti_evcnt_lazy_shots.ev_count++;
741 TLBINFO_UNLOCK(ti);
755 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
756 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
766 TLBINFO_LOCK(ti);
767 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
783 TLBINFO_UNLOCK(ti);
795 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
796 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
799 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx",
800 (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid);
802 TLBINFO_LOCK(ti);
803 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
813 TLBINFO_UNLOCK(ti);
818 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
829 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
830 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
832 KASSERT(ti->ti_asids_free > 0);
833 KASSERT(ti->ti_asid_hint > KERNEL_PID);
841 && ti->ti_asid_hint > ti->ti_asid_max) {
842 ti->ti_asid_hint = KERNEL_PID + 1;
844 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u",
845 ti->ti_asid_hint);
851 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
852 const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]);
855 for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) {
856 KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1);
866 ti->ti_asid_hint = n + i * nbpw;
869 KASSERT(ti->ti_asid_hint > KERNEL_PID);
870 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
872 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1),
873 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
874 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint),
875 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
882 pai->pai_asid = ti->ti_asid_hint++;
891 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
892 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link);
893 ti->ti_asids_free--;
904 kcpuset_merge(pm->pm_active, ti->ti_kcpuset);
919 struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
920 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
923 UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm,
924 (uintptr_t)l, (uintptr_t)ti, 0);
934 TLBINFO_LOCK(ti);
937 pmap_tlb_pai_check(ti, true);
939 if (__predict_false(!tlbinfo_asids_p(ti))) {
949 kcpuset_merge(pm->pm_active, ti->ti_kcpuset);
952 } else if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {
956 if (__predict_false(tlbinfo_noasids_p(ti))) {
959 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY);
960 KASSERT(!tlbinfo_noasids_p(ti));
966 pmap_tlb_asid_alloc(ti, pm, pai);
970 pmap_tlb_pai_check(ti, true);
993 TLBINFO_UNLOCK(ti);
1049 struct pmap_tlb_info * const ti = pmap_tlbs[i];
1051 struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1053 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1054 TLBINFO_LOCK(ti);
1055 if (PMAP_PAI_ASIDVALID_P(pai, ti)) {
1060 if (ti->ti_victim == pm)
1061 ti->ti_victim = NULL;
1062 if (__predict_true(tlbinfo_asids_p(ti)))
1063 pmap_tlb_pai_reset(ti, pai, pm);
1066 TLBINFO_UNLOCK(ti);
1082 struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1083 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1084 TLBINFO_LOCK(ti);
1089 pmap_tlb_pai_reset(ti, pai, pm);
1092 TLBINFO_UNLOCK(ti);
1120 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1121 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1122 TLBINFO_LOCK(ti);
1125 TLBINFO_UNLOCK(ti);
Indexes created Sun Sep 28 16:09:52 GMT 2025