1 /* $NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $ */ 2 3 /*- 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 9 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 10 * 11 * This material is based upon work supported by the Defense Advanced Research 12 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 13 * Contract No. N66001-09-C-2073. 14 * Approved for Public Release, Distribution Unlimited 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 #define __UFETCHSTORE_PRIVATE 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: copyin.c,v 1.9 2020/07/06 09:34:16 rin Exp $"); 42 43 #include <sys/param.h> 44 #include <sys/lwp.h> 45 #include <sys/systm.h> 46 47 #include <powerpc/pcb.h> 48 49 #include <powerpc/booke/cpuvar.h> 50 51 static inline uint8_t 52 copyin_byte(const uint8_t * const usaddr8, register_t ds_msr) 53 { 54 register_t msr; 55 uint8_t data; 56 __asm volatile( 57 "mfmsr %[msr]; " /* Save MSR */ 58 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 59 "lbz %[data],0(%[usaddr8]); " /* fetch user byte */ 60 "mtmsr %[msr]; sync; isync; " /* DS off */ 61 : [msr] "=&r" (msr), [data] "=r" (data) 62 : [ds_msr] "r" (ds_msr), [usaddr8] "b" (usaddr8)); 63 return data; 64 } 65 66 static inline uint16_t 67 copyin_halfword(const uint16_t * const usaddr16, register_t ds_msr) 68 { 69 register_t msr; 70 uint16_t data; 71 __asm volatile( 72 "mfmsr %[msr]; " /* Save MSR */ 73 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 74 "lhz %[data],0(%[usaddr16]); " /* fetch user byte */ 75 "mtmsr %[msr]; sync; isync; " /* DS off */ 76 : [msr] "=&r" (msr), [data] "=r" (data) 77 : [ds_msr] "r" (ds_msr), [usaddr16] "b" (usaddr16)); 78 return data; 79 } 80 81 static inline uint32_t 82 copyin_word(const uint32_t * const usaddr32, register_t ds_msr) 83 { 84 register_t msr; 85 uint32_t data; 86 __asm volatile( 87 "mfmsr %[msr]; " /* Save MSR */ 88 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 89 "lwz %[data],0(%[usaddr32]); " /* load user byte */ 90 "mtmsr %[msr]; sync; isync; " /* DS off */ 91 : [msr] "=&r" (msr), [data] "=r" (data) 92 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 93 return data; 94 } 95 96 static inline uint32_t 97 copyin_word_bswap(const uint32_t * const usaddr32, register_t ds_msr) 98 { 99 register_t msr; 100 uint32_t data; 101 __asm volatile( 102 "mfmsr %[msr]; " /* Save MSR */ 103 "mtmsr %[ds_msr]; sync; isync; " /* DS on */ 104 "lwbrx %[data],0,%[usaddr32]; " /* load user LE word */ 105 "mtmsr %[msr]; sync; isync; " /* DS off */ 106 : [msr] "=&r" (msr), [data] "=r" (data) 107 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 108 return data; 109 } 110 111 static inline void 112 copyin_8words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 113 { 114 register_t msr; 115 //uint32_t data[8]; 116 __asm volatile( 117 "mfmsr %[msr]" /* Save MSR */ 118 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 119 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 120 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 121 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 122 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 123 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 124 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 125 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 126 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 127 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 128 : [msr] "=&r" (msr), 129 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 130 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 131 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 132 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]) 133 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 134 } 135 136 static inline void 137 copyin_16words(const uint32_t *usaddr32, uint32_t *kdaddr32, register_t ds_msr) 138 { 139 register_t msr; 140 __asm volatile( 141 "mfmsr %[msr]" /* Save MSR */ 142 "\n\t" "mtmsr %[ds_msr]; sync; isync" /* DS on */ 143 "\n\t" "lwz %[data0],0(%[usaddr32])" /* fetch user data */ 144 "\n\t" "lwz %[data1],4(%[usaddr32])" /* fetch user data */ 145 "\n\t" "lwz %[data2],8(%[usaddr32])" /* fetch user data */ 146 "\n\t" "lwz %[data3],12(%[usaddr32])" /* fetch user data */ 147 "\n\t" "lwz %[data4],16(%[usaddr32])" /* fetch user data */ 148 "\n\t" "lwz %[data5],20(%[usaddr32])" /* fetch user data */ 149 "\n\t" "lwz %[data6],24(%[usaddr32])" /* fetch user data */ 150 "\n\t" "lwz %[data7],28(%[usaddr32])" /* fetch user data */ 151 "\n\t" "lwz %[data8],32(%[usaddr32])" /* fetch user data */ 152 "\n\t" "lwz %[data9],36(%[usaddr32])" /* fetch user data */ 153 "\n\t" "lwz %[data10],40(%[usaddr32])" /* fetch user data */ 154 "\n\t" "lwz %[data11],44(%[usaddr32])" /* fetch user data */ 155 "\n\t" "lwz %[data12],48(%[usaddr32])" /* fetch user data */ 156 "\n\t" "lwz %[data13],52(%[usaddr32])" /* fetch user data */ 157 "\n\t" "lwz %[data14],56(%[usaddr32])" /* fetch user data */ 158 "\n\t" "lwz %[data15],60(%[usaddr32])" /* fetch user data */ 159 "\n\t" "mtmsr %[msr]; sync; isync" /* DS off */ 160 : [msr] "=&r" (msr), 161 [data0] "=&r" (kdaddr32[0]), [data1] "=&r" (kdaddr32[1]), 162 [data2] "=&r" (kdaddr32[2]), [data3] "=&r" (kdaddr32[3]), 163 [data4] "=&r" (kdaddr32[4]), [data5] "=&r" (kdaddr32[5]), 164 [data6] "=&r" (kdaddr32[6]), [data7] "=&r" (kdaddr32[7]), 165 [data8] "=&r" (kdaddr32[8]), [data9] "=&r" (kdaddr32[9]), 166 [data10] "=&r" (kdaddr32[10]), [data11] "=&r" (kdaddr32[11]), 167 [data12] "=&r" (kdaddr32[12]), [data13] "=&r" (kdaddr32[13]), 168 [data14] "=&r" (kdaddr32[14]), [data15] "=&r" (kdaddr32[15]) 169 : [ds_msr] "r" (ds_msr), [usaddr32] "b" (usaddr32)); 170 } 171 static inline void 172 copyin_bytes(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 173 { 174 const uint8_t *usaddr8 = (void *)usaddr; 175 uint8_t *kdaddr8 = (void *)kdaddr; 176 while (len-- > 0) { 177 *kdaddr8++ = copyin_byte(usaddr8++, ds_msr); 178 } 179 } 180 181 static inline void 182 copyin_words(vaddr_t usaddr, vaddr_t kdaddr, size_t len, register_t ds_msr) 183 { 184 KASSERT((kdaddr & 3) == 0); 185 KASSERT((usaddr & 3) == 0); 186 const uint32_t *usaddr32 = (void *)usaddr; 187 uint32_t *kdaddr32 = (void *)kdaddr; 188 len >>= 2; 189 while (len >= 16) { 190 copyin_16words(usaddr32, kdaddr32, ds_msr); 191 usaddr32 += 16, kdaddr32 += 16, len -= 16; 192 } 193 KASSERT(len < 16); 194 if (len >= 8) { 195 copyin_8words(usaddr32, kdaddr32, ds_msr); 196 usaddr32 += 8, kdaddr32 += 8, len -= 8; 197 } 198 while (len-- > 0) { 199 *kdaddr32++ = copyin_word(usaddr32++, ds_msr); 200 } 201 } 202 203 int 204 _ufetch_8(const uint8_t *vusaddr, uint8_t *valp) 205 { 206 struct pcb * const pcb = lwp_getpcb(curlwp); 207 struct faultbuf env; 208 209 if (setfault(&env) != 0) { 210 pcb->pcb_onfault = NULL; 211 return EFAULT; 212 } 213 214 *valp = copyin_byte(vusaddr, mfmsr() | PSL_DS); 215 216 pcb->pcb_onfault = NULL; 217 218 return 0; 219 } 220 221 int 222 _ufetch_16(const uint16_t *vusaddr, uint16_t *valp) 223 { 224 struct pcb * const pcb = lwp_getpcb(curlwp); 225 struct faultbuf env; 226 227 if (setfault(&env) != 0) { 228 pcb->pcb_onfault = NULL; 229 return EFAULT; 230 } 231 232 *valp = copyin_halfword(vusaddr, mfmsr() | PSL_DS); 233 234 pcb->pcb_onfault = NULL; 235 236 return 0; 237 } 238 239 int 240 _ufetch_32(const uint32_t *vusaddr, uint32_t *valp) 241 { 242 struct pcb * const pcb = lwp_getpcb(curlwp); 243 struct faultbuf env; 244 245 if (setfault(&env) != 0) { 246 pcb->pcb_onfault = NULL; 247 return EFAULT; 248 } 249 250 *valp = copyin_word(vusaddr, mfmsr() | PSL_DS); 251 252 pcb->pcb_onfault = NULL; 253 254 return 0; 255 } 256 257 int 258 copyin(const void *vusaddr, void *vkdaddr, size_t len) 259 { 260 struct pcb * const pcb = lwp_getpcb(curlwp); 261 struct faultbuf env; 262 vaddr_t usaddr = (vaddr_t) vusaddr; 263 vaddr_t kdaddr = (vaddr_t) vkdaddr; 264 265 if (__predict_false(len == 0)) { 266 return 0; 267 } 268 269 const register_t ds_msr = mfmsr() | PSL_DS; 270 271 int rv = setfault(&env); 272 if (rv != 0) { 273 pcb->pcb_onfault = NULL; 274 return rv; 275 } 276 277 if (__predict_false(len < 4)) { 278 copyin_bytes(usaddr, kdaddr, len, ds_msr); 279 pcb->pcb_onfault = NULL; 280 return 0; 281 } 282 283 const size_t alignment = (usaddr ^ kdaddr) & 3; 284 if (__predict_true(alignment == 0)) { 285 size_t slen; 286 if (__predict_false(kdaddr & 3)) { 287 slen = 4 - (kdaddr & 3); 288 copyin_bytes(usaddr, kdaddr, slen, ds_msr); 289 usaddr += slen, kdaddr += slen, len -= slen; 290 } 291 slen = len & ~3; 292 if (__predict_true(slen >= 4)) { 293 copyin_words(usaddr, kdaddr, slen, ds_msr); 294 usaddr += slen, kdaddr += slen, len -= slen; 295 } 296 } 297 if (len > 0) { 298 copyin_bytes(usaddr, kdaddr, len, ds_msr); 299 } 300 pcb->pcb_onfault = NULL; 301 return 0; 302 } 303 304 int 305 copyinstr(const void *usaddr, void *kdaddr, size_t len, size_t *done) 306 { 307 struct pcb * const pcb = lwp_getpcb(curlwp); 308 struct faultbuf env; 309 int rv; 310 311 if (__predict_false(len == 0)) { 312 if (done) 313 *done = 0; 314 return 0; 315 } 316 317 rv = setfault(&env); 318 if (rv != 0) { 319 pcb->pcb_onfault = NULL; 320 if (done) 321 *done = 0; 322 return rv; 323 } 324 325 const register_t ds_msr = mfmsr() | PSL_DS; 326 const uint32_t *usaddr32 = (const void *)((uintptr_t)usaddr & ~3); 327 uint8_t *kdaddr8 = kdaddr; 328 size_t copylen, wlen; 329 uint32_t data; 330 size_t uoff = (uintptr_t)usaddr & 3; 331 wlen = 4 - uoff; 332 /* 333 * We need discard any leading bytes if the address was 334 * unaligned. We read the words byteswapped so that the LSB 335 * contains the lowest address byte. 336 */ 337 data = copyin_word_bswap(usaddr32++, ds_msr) >> (8 * uoff); 338 for (copylen = 0; copylen < len; copylen++, wlen--, data >>= 8) { 339 if (wlen == 0) { 340 /* 341 * If we've depleted the data in the word, fetch the 342 * next one. 343 */ 344 data = copyin_word_bswap(usaddr32++, ds_msr); 345 wlen = 4; 346 } 347 *kdaddr8++ = data; 348 if ((uint8_t) data == 0) { 349 copylen++; 350 goto out; 351 } 352 } 353 rv = ENAMETOOLONG; 354 355 out: 356 pcb->pcb_onfault = NULL; 357 if (done) 358 *done = copylen; 359 return rv; 360 } 361
Indexes created Tue Sep 30 20:09:53 GMT 2025