pmap_synci.c revision 1.1.2.3
1 1.1.2.2 yamt /*- 2 1.1.2.2 yamt * Copyright (c) 2011 The NetBSD Foundation, Inc. 3 1.1.2.2 yamt * All rights reserved. 4 1.1.2.2 yamt * 5 1.1.2.2 yamt * This code is derived from software contributed to The NetBSD Foundation 6 1.1.2.2 yamt * by Matt Thomas of 3am Software Foundry. 7 1.1.2.2 yamt * 8 1.1.2.2 yamt * Redistribution and use in source and binary forms, with or without 9 1.1.2.2 yamt * modification, are permitted provided that the following conditions 10 1.1.2.2 yamt * are met: 11 1.1.2.2 yamt * 1. Redistributions of source code must retain the above copyright 12 1.1.2.2 yamt * notice, this list of conditions and the following disclaimer. 13 1.1.2.2 yamt * 2. Redistributions in binary form must reproduce the above copyright 14 1.1.2.2 yamt * notice, this list of conditions and the following disclaimer in the 15 1.1.2.2 yamt * documentation and/or other materials provided with the distribution. 16 1.1.2.2 yamt * 17 1.1.2.2 yamt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 1.1.2.2 yamt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 1.1.2.2 yamt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 1.1.2.2 yamt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 1.1.2.2 yamt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 1.1.2.2 yamt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 1.1.2.2 yamt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 1.1.2.2 yamt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 1.1.2.2 yamt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 1.1.2.2 yamt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 1.1.2.2 yamt * POSSIBILITY OF SUCH DAMAGE. 28 1.1.2.2 yamt */ 29 1.1.2.2 yamt 30 1.1.2.2 yamt #include <sys/cdefs.h> 31 1.1.2.2 yamt 32 1.1.2.3 yamt __KERNEL_RCSID(0, "$NetBSD: pmap_synci.c,v 1.1.2.3 2014/05/22 11:41:19 yamt Exp $"); 33 1.1.2.2 yamt 34 1.1.2.2 yamt #define __PMAP_PRIVATE 35 1.1.2.2 yamt 36 1.1.2.2 yamt #include "opt_multiprocessor.h" 37 1.1.2.2 yamt 38 1.1.2.2 yamt #include <sys/param.h> 39 1.1.2.2 yamt #include <sys/systm.h> 40 1.1.2.2 yamt #include <sys/mutex.h> 41 1.1.2.2 yamt #include <sys/atomic.h> 42 1.1.2.2 yamt #include <sys/cpu.h> 43 1.1.2.2 yamt 44 1.1.2.2 yamt #include <uvm/uvm.h> 45 1.1.2.2 yamt 46 1.1.2.2 yamt #if defined(MULTIPROCESSOR) 47 1.1.2.2 yamt void 48 1.1.2.2 yamt pmap_syncicache_ast(struct cpu_info *ci) 49 1.1.2.2 yamt { 50 1.1.2.3 yamt struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 51 1.1.2.2 yamt 52 1.1.2.2 yamt KASSERT(kpreempt_disabled()); 53 1.1.2.2 yamt 54 1.1.2.2 yamt uint32_t page_bitmap = atomic_swap_32(&ti->ti_synci_page_bitmap, 0); 55 1.1.2.2 yamt #if 0 56 1.1.2.2 yamt printf("%s: need to sync %#x\n", __func__, page_bitmap); 57 1.1.2.2 yamt #endif 58 1.1.2.2 yamt ti->ti_evcnt_synci_asts.ev_count++; 59 1.1.2.2 yamt /* 60 1.1.2.2 yamt * If every bit is set in the bitmap, sync the entire icache. 61 1.1.2.2 yamt */ 62 1.1.2.2 yamt if (page_bitmap == pmap_tlb_synci_map_mask) { 63 1.1.2.2 yamt pmap_md_icache_sync_all(); 64 1.1.2.2 yamt ti->ti_evcnt_synci_all.ev_count++; 65 1.1.2.2 yamt ti->ti_evcnt_synci_pages.ev_count += pmap_tlb_synci_page_mask+1; 66 1.1.2.2 yamt kpreempt_enable(); 67 1.1.2.2 yamt return; 68 1.1.2.2 yamt } 69 1.1.2.2 yamt 70 1.1.2.2 yamt /* 71 1.1.2.2 yamt * Loop through the bitmap clearing each set of indices for each page. 72 1.1.2.2 yamt */ 73 1.1.2.2 yamt for (vaddr_t va = 0; 74 1.1.2.2 yamt page_bitmap != 0; 75 1.1.2.2 yamt page_bitmap >>= 1, va += PAGE_SIZE) { 76 1.1.2.2 yamt if (page_bitmap & 1) { 77 1.1.2.2 yamt /* 78 1.1.2.2 yamt * Each bit set represents a page index to be synced. 79 1.1.2.2 yamt */ 80 1.1.2.2 yamt pmap_md_icache_sync_range_index(va, PAGE_SIZE); 81 1.1.2.2 yamt ti->ti_evcnt_synci_pages.ev_count++; 82 1.1.2.2 yamt } 83 1.1.2.2 yamt } 84 1.1.2.2 yamt 85 1.1.2.2 yamt kpreempt_enable(); 86 1.1.2.2 yamt } 87 1.1.2.2 yamt 88 1.1.2.2 yamt void 89 1.1.2.2 yamt pmap_tlb_syncicache(vaddr_t va, uint32_t page_onproc) 90 1.1.2.2 yamt { 91 1.1.2.2 yamt KASSERT(kpreempt_disabled()); 92 1.1.2.2 yamt /* 93 1.1.2.2 yamt * We don't sync the icache here but let ast do it for us just before 94 1.1.2.2 yamt * returning to userspace. We do this because we don't really know 95 1.1.2.2 yamt * on which CPU we will return to userspace and if we synch the icache 96 1.1.2.2 yamt * now it might not be on the CPU we need it on. In addition, others 97 1.1.2.2 yamt * threads might sync the icache before we get to return to userland 98 1.1.2.2 yamt * so there's no reason for us to do it. 99 1.1.2.2 yamt * 100 1.1.2.2 yamt * Each TLB/cache keeps a synci sequence number which gets advanced 101 1.1.2.2 yamt * each time that TLB/cache performs a pmap_md_sync_icache_all. When 102 1.1.2.2 yamt * we return to userland, we check the pmap's corresponding synci 103 1.1.2.2 yamt * sequence number for that TLB/cache. If they match, it means that 104 1.1.2.2 yamt * no one has yet synched the icache so we much do it ourselves. If 105 1.1.2.2 yamt * they don't match someone has already synced the icache for us. 106 1.1.2.2 yamt * 107 1.1.2.2 yamt * There is a small chance that the generation numbers will wrap and 108 1.1.2.2 yamt * then become equal but that's a one in 4 billion cache and will 109 1.1.2.2 yamt * just cause an extra sync of the icache. 110 1.1.2.2 yamt */ 111 1.1.2.2 yamt const uint32_t cpu_mask = 1L << cpu_index(curcpu()); 112 1.1.2.2 yamt const uint32_t page_mask = 113 1.1.2.2 yamt 1L << ((va >> PGSHIFT) & pmap_tlb_synci_page_mask); 114 1.1.2.2 yamt uint32_t onproc = 0; 115 1.1.2.2 yamt for (size_t i = 0; i < pmap_ntlbs; i++) { 116 1.1.2.2 yamt struct pmap_tlb_info * const ti = pmap_tlbs[i]; 117 1.1.2.2 yamt TLBINFO_LOCK(ti); 118 1.1.2.2 yamt for (;;) { 119 1.1.2.2 yamt uint32_t old_page_bitmap = ti->ti_synci_page_bitmap; 120 1.1.2.2 yamt if (old_page_bitmap & page_mask) { 121 1.1.2.2 yamt ti->ti_evcnt_synci_duplicate.ev_count++; 122 1.1.2.2 yamt break; 123 1.1.2.2 yamt } 124 1.1.2.2 yamt 125 1.1.2.2 yamt uint32_t orig_page_bitmap = atomic_cas_32( 126 1.1.2.2 yamt &ti->ti_synci_page_bitmap, old_page_bitmap, 127 1.1.2.2 yamt old_page_bitmap | page_mask); 128 1.1.2.2 yamt 129 1.1.2.2 yamt if (orig_page_bitmap == old_page_bitmap) { 130 1.1.2.2 yamt if (old_page_bitmap == 0) { 131 1.1.2.2 yamt onproc |= ti->ti_cpu_mask; 132 1.1.2.2 yamt } else { 133 1.1.2.2 yamt ti->ti_evcnt_synci_deferred.ev_count++; 134 1.1.2.2 yamt } 135 1.1.2.2 yamt ti->ti_evcnt_synci_desired.ev_count++; 136 1.1.2.2 yamt break; 137 1.1.2.2 yamt } 138 1.1.2.2 yamt } 139 1.1.2.2 yamt #if 0 140 1.1.2.2 yamt printf("%s: %s: %x to %x on cpus %#x\n", __func__, 141 1.1.2.2 yamt ti->ti_name, page_mask, ti->ti_synci_page_bitmap, 142 1.1.2.2 yamt onproc & page_onproc & ti->ti_cpu_mask); 143 1.1.2.2 yamt #endif 144 1.1.2.2 yamt TLBINFO_UNLOCK(ti); 145 1.1.2.2 yamt } 146 1.1.2.2 yamt onproc &= page_onproc; 147 1.1.2.2 yamt if (__predict_false(onproc != 0)) { 148 1.1.2.2 yamt /* 149 1.1.2.2 yamt * If the cpu need to sync this page, tell the current lwp 150 1.1.2.2 yamt * to sync the icache before it returns to userspace. 151 1.1.2.2 yamt */ 152 1.1.2.2 yamt if (onproc & cpu_mask) { 153 1.1.2.2 yamt if (curcpu()->ci_flags & CPUF_USERPMAP) { 154 1.1.2.2 yamt curlwp->l_md.md_astpending = 1; /* force call to ast() */ 155 1.1.2.2 yamt curcpu()->ci_evcnt_synci_onproc_rqst.ev_count++; 156 1.1.2.2 yamt } else { 157 1.1.2.2 yamt curcpu()->ci_evcnt_synci_deferred_rqst.ev_count++; 158 1.1.2.2 yamt } 159 1.1.2.2 yamt onproc ^= cpu_mask; 160 1.1.2.2 yamt } 161 1.1.2.2 yamt 162 1.1.2.2 yamt /* 163 1.1.2.2 yamt * For each cpu that is affect, send an IPI telling 164 1.1.2.2 yamt * that CPU that the current thread needs to sync its icache. 165 1.1.2.2 yamt * We might cause some spurious icache syncs but that's not 166 1.1.2.2 yamt * going to break anything. 167 1.1.2.2 yamt */ 168 1.1.2.2 yamt for (u_int n = ffs(onproc); 169 1.1.2.2 yamt onproc != 0; 170 1.1.2.2 yamt onproc >>= n, onproc <<= n, n = ffs(onproc)) { 171 1.1.2.2 yamt cpu_send_ipi(cpu_lookup(n-1), IPI_SYNCICACHE); 172 1.1.2.2 yamt } 173 1.1.2.2 yamt } 174 1.1.2.2 yamt } 175 1.1.2.2 yamt 176 1.1.2.2 yamt void 177 1.1.2.2 yamt pmap_tlb_syncicache_wanted(struct cpu_info *ci) 178 1.1.2.2 yamt { 179 1.1.2.3 yamt struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 180 1.1.2.2 yamt 181 1.1.2.2 yamt KASSERT(cpu_intr_p()); 182 1.1.2.2 yamt 183 1.1.2.2 yamt TLBINFO_LOCK(ti); 184 1.1.2.2 yamt 185 1.1.2.2 yamt /* 186 1.1.2.2 yamt * We might have been notified because another CPU changed an exec 187 1.1.2.2 yamt * page and now needs us to sync the icache so tell the current lwp 188 1.1.2.2 yamt * to do the next time it returns to userland (which should be very 189 1.1.2.2 yamt * soon). 190 1.1.2.2 yamt */ 191 1.1.2.2 yamt if (ti->ti_synci_page_bitmap && (ci->ci_flags & CPUF_USERPMAP)) { 192 1.1.2.2 yamt curlwp->l_md.md_astpending = 1; /* force call to ast() */ 193 1.1.2.2 yamt ci->ci_evcnt_synci_ipi_rqst.ev_count++; 194 1.1.2.2 yamt } 195 1.1.2.2 yamt 196 1.1.2.2 yamt TLBINFO_UNLOCK(ti); 197 1.1.2.2 yamt 198 1.1.2.2 yamt } 199 1.1.2.2 yamt #endif /* MULTIPROCESSOR */ 200
Indexes created Mon Oct 27 20:09:55 GMT 2025