cpufunc.h revision 1.27
1 /* $NetBSD: cpufunc.h,v 1.27 2024/02/07 04:20:26 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #ifndef _AARCH64_CPUFUNC_H_ 30 #define _AARCH64_CPUFUNC_H_ 31 32 #ifdef _KERNEL 33 34 #include <arm/armreg.h> 35 #include <sys/device_if.h> 36 37 #include <sys/cpu.h> 38 39 #include <uvm/uvm_extern.h> 40 #include <uvm/pmap/pmap_devmap.h> 41 42 extern u_int aarch64_cache_vindexsize; /* cachesize/way (VIVT/VIPT) */ 43 extern u_int aarch64_cache_prefer_mask; 44 extern u_int cputype; /* compat arm */ 45 46 extern int aarch64_bti_enabled; 47 extern int aarch64_hafdbs_enabled; 48 extern int aarch64_pan_enabled; 49 extern int aarch64_pac_enabled; 50 51 void aarch64_hafdbs_init(int); 52 void aarch64_pan_init(int); 53 int aarch64_pac_init(int); 54 55 int set_cpufuncs(void); 56 int aarch64_setcpufuncs(struct cpu_info *); 57 void aarch64_getcacheinfo(struct cpu_info *); 58 void aarch64_parsecacheinfo(struct cpu_info *); 59 void aarch64_printcacheinfo(device_t, struct cpu_info *); 60 61 void aarch64_dcache_wbinv_all(void); 62 void aarch64_dcache_inv_all(void); 63 void aarch64_dcache_wb_all(void); 64 void aarch64_icache_inv_all(void); 65 66 /* cache op in cpufunc_asm_armv8.S */ 67 void aarch64_nullop(void); 68 uint32_t aarch64_cpuid(void); 69 void aarch64_icache_sync_range(vaddr_t, vsize_t); 70 void aarch64_icache_inv_range(vaddr_t, vsize_t); 71 void aarch64_icache_barrier_range(vaddr_t, vsize_t); 72 void aarch64_idcache_wbinv_range(vaddr_t, vsize_t); 73 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t); 74 void aarch64_dcache_inv_range(vaddr_t, vsize_t); 75 void aarch64_dcache_wb_range(vaddr_t, vsize_t); 76 void aarch64_icache_inv_all(void); 77 void aarch64_drain_writebuf(void); 78 79 /* tlb op in cpufunc_asm_armv8.S */ 80 #define cpu_set_ttbr0(t) curcpu()->ci_cpufuncs.cf_set_ttbr0((t)) 81 void aarch64_set_ttbr0(uint64_t); 82 void aarch64_set_ttbr0_thunderx(uint64_t); 83 void aarch64_tlbi_all(void); /* all ASID, all VA */ 84 void aarch64_tlbi_by_asid(int); /* an ASID, all VA */ 85 void aarch64_tlbi_by_va(vaddr_t); /* all ASID, a VA */ 86 void aarch64_tlbi_by_va_ll(vaddr_t); /* all ASID, a VA, lastlevel */ 87 void aarch64_tlbi_by_asid_va(int, vaddr_t); /* an ASID, a VA */ 88 void aarch64_tlbi_by_asid_va_ll(int, vaddr_t); /* an ASID, a VA, lastlevel */ 89 90 /* misc */ 91 #define cpu_idnum() aarch64_cpuid() 92 93 /* cache op */ 94 #define cpu_dcache_wbinv_all() aarch64_dcache_wbinv_all() 95 #define cpu_dcache_inv_all() aarch64_dcache_inv_all() 96 #define cpu_dcache_wb_all() aarch64_dcache_wb_all() 97 #define cpu_idcache_wbinv_all() \ 98 (aarch64_dcache_wbinv_all(), aarch64_icache_inv_all()) 99 #define cpu_icache_sync_all() \ 100 (aarch64_dcache_wb_all(), aarch64_icache_inv_all()) 101 #define cpu_icache_inv_all() aarch64_icache_inv_all() 102 103 #define cpu_dcache_wbinv_range(v,s) aarch64_dcache_wbinv_range((v),(s)) 104 #define cpu_dcache_inv_range(v,s) aarch64_dcache_inv_range((v),(s)) 105 #define cpu_dcache_wb_range(v,s) aarch64_dcache_wb_range((v),(s)) 106 #define cpu_idcache_wbinv_range(v,s) aarch64_idcache_wbinv_range((v),(s)) 107 #define cpu_icache_sync_range(v,s) \ 108 curcpu()->ci_cpufuncs.cf_icache_sync_range((v),(s)) 109 110 #define cpu_sdcache_wbinv_range(v,p,s) ((void)0) 111 #define cpu_sdcache_inv_range(v,p,s) ((void)0) 112 #define cpu_sdcache_wb_range(v,p,s) ((void)0) 113 114 /* others */ 115 #define cpu_drain_writebuf() aarch64_drain_writebuf() 116 117 extern u_int arm_dcache_align; 118 extern u_int arm_dcache_align_mask; 119 120 static inline bool 121 cpu_gtmr_exists_p(void) 122 { 123 124 return true; 125 } 126 127 static inline u_int 128 cpu_clusterid(void) 129 { 130 131 return __SHIFTOUT(reg_mpidr_el1_read(), MPIDR_AFF1); 132 } 133 134 static inline bool 135 cpu_earlydevice_va_p(void) 136 { 137 /* This function may be called before enabling MMU, or mapping KVA */ 138 if ((reg_sctlr_el1_read() & SCTLR_M) == 0) 139 return false; 140 141 /* device mapping will be available after pmap_devmap_bootstrap() */ 142 if (!pmap_devmap_bootstrapped_p()) 143 return false; 144 145 return true; 146 } 147 148 #endif /* _KERNEL */ 149 150 /* definitions of TAG and PAC in pointers */ 151 #define AARCH64_ADDRTOP_TAG_BIT 55 152 #define AARCH64_ADDRTOP_TAG __BIT(55) /* ECR_EL1.TBI[01]=1 */ 153 #define AARCH64_ADDRTOP_MSB __BIT(63) /* ECR_EL1.TBI[01]=0 */ 154 #define AARCH64_ADDRESS_TAG_MASK __BITS(63,56) /* if TCR.TBI[01]=1 */ 155 #define AARCH64_ADDRESS_PAC_MASK __BITS(54,48) /* depend on VIRT_BIT */ 156 #define AARCH64_ADDRESS_TAGPAC_MASK \ 157 (AARCH64_ADDRESS_TAG_MASK|AARCH64_ADDRESS_PAC_MASK) 158 159 #ifdef _KERNEL 160 /* 161 * Which is the address space of this VA? 162 * return the space considering TBI. (PAC is not yet) 163 * 164 * return value: AARCH64_ADDRSPACE_{LOWER,UPPER}{_OUTOFRANGE}? 165 */ 166 #define AARCH64_ADDRSPACE_LOWER 0 /* -> TTBR0 */ 167 #define AARCH64_ADDRSPACE_UPPER 1 /* -> TTBR1 */ 168 #define AARCH64_ADDRSPACE_LOWER_OUTOFRANGE -1 /* certainly fault */ 169 #define AARCH64_ADDRSPACE_UPPER_OUTOFRANGE -2 /* certainly fault */ 170 static inline int 171 aarch64_addressspace(vaddr_t va) 172 { 173 uint64_t addrtop, tbi; 174 175 addrtop = va & AARCH64_ADDRTOP_TAG; 176 tbi = addrtop ? TCR_TBI1 : TCR_TBI0; 177 if (reg_tcr_el1_read() & tbi) { 178 if (addrtop == 0) { 179 /* lower address, and TBI0 enabled */ 180 if ((va & AARCH64_ADDRESS_PAC_MASK) != 0) 181 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE; 182 return AARCH64_ADDRSPACE_LOWER; 183 } 184 /* upper address, and TBI1 enabled */ 185 if ((va & AARCH64_ADDRESS_PAC_MASK) != AARCH64_ADDRESS_PAC_MASK) 186 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE; 187 return AARCH64_ADDRSPACE_UPPER; 188 } 189 190 addrtop = va & AARCH64_ADDRTOP_MSB; 191 if (addrtop == 0) { 192 /* lower address, and TBI0 disabled */ 193 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != 0) 194 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE; 195 return AARCH64_ADDRSPACE_LOWER; 196 } 197 /* upper address, and TBI1 disabled */ 198 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != AARCH64_ADDRESS_TAGPAC_MASK) 199 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE; 200 return AARCH64_ADDRSPACE_UPPER; 201 } 202 203 static inline vaddr_t 204 aarch64_untag_address(vaddr_t va) 205 { 206 uint64_t addrtop, tbi; 207 208 addrtop = va & AARCH64_ADDRTOP_TAG; 209 tbi = addrtop ? TCR_TBI1 : TCR_TBI0; 210 if (reg_tcr_el1_read() & tbi) { 211 if (addrtop == 0) { 212 /* lower address, and TBI0 enabled */ 213 return va & ~AARCH64_ADDRESS_TAG_MASK; 214 } 215 /* upper address, and TBI1 enabled */ 216 return va | AARCH64_ADDRESS_TAG_MASK; 217 } 218 219 /* TBI[01] is disabled, nothing to do */ 220 return va; 221 } 222 223 #endif /* _KERNEL */ 224 225 static __inline uint64_t 226 aarch64_strip_pac(uint64_t __val) 227 { 228 if (__val & AARCH64_ADDRTOP_TAG) 229 return __val | AARCH64_ADDRESS_TAGPAC_MASK; 230 return __val & ~AARCH64_ADDRESS_TAGPAC_MASK; 231 } 232 233 #endif /* _AARCH64_CPUFUNC_H_ */ 234
Indexes created Thu Oct 02 01:09:59 GMT 2025