cpufunc.h revision 1.19.4.1
1 /* $NetBSD: cpufunc.h,v 1.19.4.1 2021/06/17 04:46:16 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu <ryo (at) nerv.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #ifndef _AARCH64_CPUFUNC_H_ 30 #define _AARCH64_CPUFUNC_H_ 31 32 #ifdef _KERNEL 33 34 #include <arm/armreg.h> 35 #include <sys/device_if.h> 36 37 struct aarch64_cache_unit { 38 u_int cache_type; 39 #define CACHE_TYPE_VPIPT 0 /* VMID-aware PIPT */ 40 #define CACHE_TYPE_VIVT 1 /* ASID-tagged VIVT */ 41 #define CACHE_TYPE_VIPT 2 42 #define CACHE_TYPE_PIPT 3 43 u_int cache_line_size; 44 u_int cache_ways; 45 u_int cache_sets; 46 u_int cache_way_size; 47 u_int cache_size; 48 }; 49 50 struct aarch64_cache_info { 51 u_int cacheable; 52 #define CACHE_CACHEABLE_NONE 0 53 #define CACHE_CACHEABLE_ICACHE 1 /* instruction cache only */ 54 #define CACHE_CACHEABLE_DCACHE 2 /* data cache only */ 55 #define CACHE_CACHEABLE_IDCACHE 3 /* instruction and data caches */ 56 #define CACHE_CACHEABLE_UNIFIED 4 /* unified cache */ 57 struct aarch64_cache_unit icache; 58 struct aarch64_cache_unit dcache; 59 }; 60 61 #define MAX_CACHE_LEVEL 8 /* ARMv8 has maximum 8 level cache */ 62 extern u_int aarch64_cache_vindexsize; /* cachesize/way (VIVT/VIPT) */ 63 extern u_int aarch64_cache_prefer_mask; 64 extern u_int cputype; /* compat arm */ 65 66 extern int aarch64_bti_enabled; 67 extern int aarch64_pan_enabled; 68 extern int aarch64_pac_enabled; 69 70 void aarch64_pan_init(int); 71 int aarch64_pac_init(int); 72 73 int set_cpufuncs(void); 74 void aarch64_getcacheinfo(int); 75 void aarch64_printcacheinfo(device_t); 76 77 void aarch64_dcache_wbinv_all(void); 78 void aarch64_dcache_inv_all(void); 79 void aarch64_dcache_wb_all(void); 80 void aarch64_icache_inv_all(void); 81 82 /* cache op in cpufunc_asm_armv8.S */ 83 void aarch64_nullop(void); 84 uint32_t aarch64_cpuid(void); 85 void aarch64_icache_sync_range(vaddr_t, vsize_t); 86 void aarch64_icache_inv_range(vaddr_t, vsize_t); 87 void aarch64_icache_barrier_range(vaddr_t, vsize_t); 88 void aarch64_idcache_wbinv_range(vaddr_t, vsize_t); 89 void aarch64_dcache_wbinv_range(vaddr_t, vsize_t); 90 void aarch64_dcache_inv_range(vaddr_t, vsize_t); 91 void aarch64_dcache_wb_range(vaddr_t, vsize_t); 92 void aarch64_icache_inv_all(void); 93 void aarch64_drain_writebuf(void); 94 95 /* tlb op in cpufunc_asm_armv8.S */ 96 #define cpu_set_ttbr0(t) curcpu()->ci_cpufuncs.cf_set_ttbr0((t)) 97 void aarch64_set_ttbr0(uint64_t); 98 void aarch64_set_ttbr0_thunderx(uint64_t); 99 void aarch64_tlbi_all(void); /* all ASID, all VA */ 100 void aarch64_tlbi_by_asid(int); /* an ASID, all VA */ 101 void aarch64_tlbi_by_va(vaddr_t); /* all ASID, a VA */ 102 void aarch64_tlbi_by_va_ll(vaddr_t); /* all ASID, a VA, lastlevel */ 103 void aarch64_tlbi_by_asid_va(int, vaddr_t); /* an ASID, a VA */ 104 void aarch64_tlbi_by_asid_va_ll(int, vaddr_t); /* an ASID, a VA, lastlevel */ 105 106 107 /* misc */ 108 #define cpu_idnum() aarch64_cpuid() 109 110 /* cache op */ 111 112 #define cpu_dcache_wbinv_all() aarch64_dcache_wbinv_all() 113 #define cpu_dcache_inv_all() aarch64_dcache_inv_all() 114 #define cpu_dcache_wb_all() aarch64_dcache_wb_all() 115 #define cpu_idcache_wbinv_all() \ 116 (aarch64_dcache_wbinv_all(), aarch64_icache_inv_all()) 117 #define cpu_icache_sync_all() \ 118 (aarch64_dcache_wb_all(), aarch64_icache_inv_all()) 119 #define cpu_icache_inv_all() aarch64_icache_inv_all() 120 121 #define cpu_dcache_wbinv_range(v,s) aarch64_dcache_wbinv_range((v),(s)) 122 #define cpu_dcache_inv_range(v,s) aarch64_dcache_inv_range((v),(s)) 123 #define cpu_dcache_wb_range(v,s) aarch64_dcache_wb_range((v),(s)) 124 #define cpu_idcache_wbinv_range(v,s) aarch64_idcache_wbinv_range((v),(s)) 125 #define cpu_icache_sync_range(v,s) \ 126 curcpu()->ci_cpufuncs.cf_icache_sync_range((v),(s)) 127 128 #define cpu_sdcache_wbinv_range(v,p,s) ((void)0) 129 #define cpu_sdcache_inv_range(v,p,s) ((void)0) 130 #define cpu_sdcache_wb_range(v,p,s) ((void)0) 131 132 /* others */ 133 #define cpu_drain_writebuf() aarch64_drain_writebuf() 134 135 extern u_int arm_dcache_align; 136 extern u_int arm_dcache_align_mask; 137 138 static inline bool 139 cpu_gtmr_exists_p(void) 140 { 141 142 return true; 143 } 144 145 static inline u_int 146 cpu_clusterid(void) 147 { 148 149 return __SHIFTOUT(reg_mpidr_el1_read(), MPIDR_AFF1); 150 } 151 152 static inline bool 153 cpu_earlydevice_va_p(void) 154 { 155 extern bool pmap_devmap_bootstrap_done; /* in pmap.c */ 156 157 /* This function may be called before enabling MMU, or mapping KVA */ 158 if ((reg_sctlr_el1_read() & SCTLR_M) == 0) 159 return false; 160 161 /* device mapping will be availabled after pmap_devmap_bootstrap() */ 162 if (!pmap_devmap_bootstrap_done) 163 return false; 164 165 return true; 166 } 167 168 #endif /* _KERNEL */ 169 170 /* definitions of TAG and PAC in pointers */ 171 #define AARCH64_ADDRTOP_TAG_BIT 55 172 #define AARCH64_ADDRTOP_TAG __BIT(55) /* ECR_EL1.TBI[01]=1 */ 173 #define AARCH64_ADDRTOP_MSB __BIT(63) /* ECR_EL1.TBI[01]=0 */ 174 #define AARCH64_ADDRESS_TAG_MASK __BITS(63,56) /* if TCR.TBI[01]=1 */ 175 #define AARCH64_ADDRESS_PAC_MASK __BITS(54,48) /* depend on VIRT_BIT */ 176 #define AARCH64_ADDRESS_TAGPAC_MASK \ 177 (AARCH64_ADDRESS_TAG_MASK|AARCH64_ADDRESS_PAC_MASK) 178 179 #ifdef _KERNEL 180 /* 181 * Which is the address space of this VA? 182 * return the space considering TBI. (PAC is not yet) 183 * 184 * return value: AARCH64_ADDRSPACE_{LOWER,UPPER}{_OUTOFRANGE}? 185 */ 186 #define AARCH64_ADDRSPACE_LOWER 0 /* -> TTBR0 */ 187 #define AARCH64_ADDRSPACE_UPPER 1 /* -> TTBR1 */ 188 #define AARCH64_ADDRSPACE_LOWER_OUTOFRANGE -1 /* certainly fault */ 189 #define AARCH64_ADDRSPACE_UPPER_OUTOFRANGE -2 /* certainly fault */ 190 static inline int 191 aarch64_addressspace(vaddr_t va) 192 { 193 uint64_t addrtop, tbi; 194 195 addrtop = va & AARCH64_ADDRTOP_TAG; 196 tbi = addrtop ? TCR_TBI1 : TCR_TBI0; 197 if (reg_tcr_el1_read() & tbi) { 198 if (addrtop == 0) { 199 /* lower address, and TBI0 enabled */ 200 if ((va & AARCH64_ADDRESS_PAC_MASK) != 0) 201 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE; 202 return AARCH64_ADDRSPACE_LOWER; 203 } 204 /* upper address, and TBI1 enabled */ 205 if ((va & AARCH64_ADDRESS_PAC_MASK) != AARCH64_ADDRESS_PAC_MASK) 206 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE; 207 return AARCH64_ADDRSPACE_UPPER; 208 } 209 210 addrtop = va & AARCH64_ADDRTOP_MSB; 211 if (addrtop == 0) { 212 /* lower address, and TBI0 disabled */ 213 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != 0) 214 return AARCH64_ADDRSPACE_LOWER_OUTOFRANGE; 215 return AARCH64_ADDRSPACE_LOWER; 216 } 217 /* upper address, and TBI1 disabled */ 218 if ((va & AARCH64_ADDRESS_TAGPAC_MASK) != AARCH64_ADDRESS_TAGPAC_MASK) 219 return AARCH64_ADDRSPACE_UPPER_OUTOFRANGE; 220 return AARCH64_ADDRSPACE_UPPER; 221 } 222 223 static inline vaddr_t 224 aarch64_untag_address(vaddr_t va) 225 { 226 uint64_t addrtop, tbi; 227 228 addrtop = va & AARCH64_ADDRTOP_TAG; 229 tbi = addrtop ? TCR_TBI1 : TCR_TBI0; 230 if (reg_tcr_el1_read() & tbi) { 231 if (addrtop == 0) { 232 /* lower address, and TBI0 enabled */ 233 return va & ~AARCH64_ADDRESS_TAG_MASK; 234 } 235 /* upper address, and TBI1 enabled */ 236 return va | AARCH64_ADDRESS_TAG_MASK; 237 } 238 239 /* TBI[01] is disabled, nothing to do */ 240 return va; 241 } 242 243 #endif /* _KERNEL */ 244 245 static __inline uint64_t 246 aarch64_strip_pac(uint64_t __val) 247 { 248 if (__val & AARCH64_ADDRTOP_TAG) 249 return __val | AARCH64_ADDRESS_TAGPAC_MASK; 250 return __val & ~AARCH64_ADDRESS_TAGPAC_MASK; 251 } 252 253 #endif /* _AARCH64_CPUFUNC_H_ */ 254
Indexes created Wed Oct 22 13:09:56 GMT 2025