altivec.c revision 1.17
1 /* $NetBSD: altivec.c,v 1.17 2011/01/14 02:06:30 rmind Exp $ */ 2 3 /* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: altivec.c,v 1.17 2011/01/14 02:06:30 rmind Exp $"); 36 37 #include "opt_multiprocessor.h" 38 39 #include <sys/param.h> 40 #include <sys/proc.h> 41 #include <sys/systm.h> 42 #include <sys/malloc.h> 43 #include <sys/pool.h> 44 45 #include <uvm/uvm_extern.h> 46 47 #include <powerpc/pcb.h> 48 #include <powerpc/altivec.h> 49 #include <powerpc/psl.h> 50 #include <powerpc/spr.h> 51 #include <powerpc/oea/spr.h> 52 53 #ifdef MULTIPROCESSOR 54 #include <arch/powerpc/pic/picvar.h> 55 #include <arch/powerpc/pic/ipivar.h> 56 static void mp_save_vec_lwp(struct lwp *); 57 #endif 58 59 void 60 enable_vec(void) 61 { 62 struct cpu_info *ci = curcpu(); 63 struct lwp *l = curlwp; 64 struct pcb *pcb = lwp_getpcb(l); 65 struct trapframe *tf = trapframe(l); 66 struct vreg *vr = &pcb->pcb_vr; 67 register_t msr; 68 69 KASSERT(pcb->pcb_veccpu == NULL); 70 71 pcb->pcb_flags |= PCB_ALTIVEC; 72 73 /* 74 * Enable AltiVec temporarily (and disable interrupts). 75 */ 76 msr = mfmsr(); 77 mtmsr((msr & ~PSL_EE) | PSL_VEC); 78 __asm volatile ("isync"); 79 if (ci->ci_veclwp) { 80 save_vec_cpu(); 81 } 82 KASSERT(curcpu()->ci_veclwp == NULL); 83 84 /* 85 * Restore VSCR by first loading it into a vector and then into VSCR. 86 * (this needs to done before loading the user's vector registers 87 * since we need to use a scratch vector register) 88 */ 89 __asm volatile("vxor %2,%2,%2; lvewx %2,%0,%1; mtvscr %2" \ 90 :: "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0)); 91 92 /* 93 * VRSAVE will be restored when trap frame returns 94 */ 95 tf->tf_xtra[TF_VRSAVE] = vr->vrsave; 96 97 #define LVX(n,vr) __asm /*volatile*/("lvx %2,%0,%1" \ 98 :: "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n)); 99 100 /* 101 * Load all 32 vector registers 102 */ 103 LVX( 0,vr); LVX( 1,vr); LVX( 2,vr); LVX( 3,vr); 104 LVX( 4,vr); LVX( 5,vr); LVX( 6,vr); LVX( 7,vr); 105 LVX( 8,vr); LVX( 9,vr); LVX(10,vr); LVX(11,vr); 106 LVX(12,vr); LVX(13,vr); LVX(14,vr); LVX(15,vr); 107 108 LVX(16,vr); LVX(17,vr); LVX(18,vr); LVX(19,vr); 109 LVX(20,vr); LVX(21,vr); LVX(22,vr); LVX(23,vr); 110 LVX(24,vr); LVX(25,vr); LVX(26,vr); LVX(27,vr); 111 LVX(28,vr); LVX(29,vr); LVX(30,vr); LVX(31,vr); 112 __asm volatile ("isync"); 113 114 /* 115 * Enable AltiVec when we return to user-mode. 116 * Record the new ownership of the AltiVec unit. 117 */ 118 curcpu()->ci_veclwp = l; 119 pcb->pcb_veccpu = curcpu(); 120 pcb->pcb_flags |= PCB_OWNALTIVEC; 121 __asm volatile ("sync"); 122 123 /* 124 * Restore MSR (turn off AltiVec) 125 */ 126 mtmsr(msr); 127 } 128 129 void 130 save_vec_cpu(void) 131 { 132 struct cpu_info *ci = curcpu(); 133 struct lwp *l; 134 struct pcb *pcb; 135 struct vreg *vr; 136 struct trapframe *tf; 137 register_t msr; 138 139 /* 140 * Turn on AltiVEC, turn off interrupts. 141 */ 142 msr = mfmsr(); 143 mtmsr((msr & ~PSL_EE) | PSL_VEC); 144 __asm volatile ("isync"); 145 l = ci->ci_veclwp; 146 if (l == NULL) 147 goto out; 148 pcb = lwp_getpcb(l); 149 vr = &pcb->pcb_vr; 150 tf = trapframe(l); 151 152 #define STVX(n,vr) __asm /*volatile*/("stvx %2,%0,%1" \ 153 :: "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n)); 154 155 /* 156 * Save the vector registers. 157 */ 158 STVX( 0,vr); STVX( 1,vr); STVX( 2,vr); STVX( 3,vr); 159 STVX( 4,vr); STVX( 5,vr); STVX( 6,vr); STVX( 7,vr); 160 STVX( 8,vr); STVX( 9,vr); STVX(10,vr); STVX(11,vr); 161 STVX(12,vr); STVX(13,vr); STVX(14,vr); STVX(15,vr); 162 163 STVX(16,vr); STVX(17,vr); STVX(18,vr); STVX(19,vr); 164 STVX(20,vr); STVX(21,vr); STVX(22,vr); STVX(23,vr); 165 STVX(24,vr); STVX(25,vr); STVX(26,vr); STVX(27,vr); 166 STVX(28,vr); STVX(29,vr); STVX(30,vr); STVX(31,vr); 167 168 /* 169 * Save VSCR (this needs to be done after save the vector registers 170 * since we need to use one as scratch). 171 */ 172 __asm volatile("mfvscr %2; stvewx %2,%0,%1" \ 173 :: "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0)); 174 175 /* 176 * Save VRSAVE 177 */ 178 vr->vrsave = tf->tf_xtra[TF_VRSAVE]; 179 180 /* 181 * Note that we aren't using any CPU resources and stop any 182 * data streams. 183 */ 184 pcb->pcb_veccpu = NULL; 185 ci->ci_veclwp = NULL; 186 __asm volatile ("dssall; sync"); 187 188 out: 189 190 /* 191 * Restore MSR (turn off AltiVec) 192 */ 193 mtmsr(msr); 194 } 195 196 #ifdef MULTIPROCESSOR 197 /* 198 * Save a process's AltiVEC state to its PCB. The state may be in any CPU. 199 * The process must either be curproc or traced by curproc (and stopped). 200 * (The point being that the process must not run on another CPU during 201 * this function). 202 */ 203 static void 204 mp_save_vec_lwp(struct lwp *l) 205 { 206 struct pcb *pcb = lwp_getpcb(l); 207 struct cpu_info *veccpu; 208 int i; 209 210 /* 211 * Send an IPI to the other CPU with the data and wait for that CPU 212 * to flush the data. Note that the other CPU might have switched 213 * to a different proc's AltiVEC state by the time it receives the IPI, 214 * but that will only result in an unnecessary reload. 215 */ 216 217 veccpu = pcb->pcb_veccpu; 218 if (veccpu == NULL) 219 return; 220 221 ppc_send_ipi(veccpu->ci_index, PPC_IPI_FLUSH_VEC); 222 223 /* Wait for flush. */ 224 for (i = 0; i < 0x3fffffff; i++) 225 if (pcb->pcb_veccpu == NULL) 226 return; 227 228 aprint_error("mp_save_vec_lwp{%d} pid = %d.%d, veccpu->ci_cpuid = %d\n", 229 cpu_number(), l->l_proc->p_pid, l->l_lid, veccpu->ci_cpuid); 230 panic("mp_save_vec_lwp: timed out"); 231 } 232 #endif /*MULTIPROCESSOR*/ 233 234 /* 235 * Save a process's AltiVEC state to its PCB. The state may be in any CPU. 236 * The process must either be curproc or traced by curproc (and stopped). 237 * (The point being that the process must not run on another CPU during 238 * this function). 239 */ 240 void 241 save_vec_lwp(struct lwp *l, int discard) 242 { 243 struct pcb * const pcb = lwp_getpcb(l); 244 struct cpu_info * const ci = curcpu(); 245 246 /* 247 * If it's already in the PCB, there's nothing to do. 248 */ 249 if (pcb->pcb_veccpu == NULL) 250 return; 251 252 /* 253 * If we simply need to discard the information, then don't 254 * to save anything. 255 */ 256 if (discard) { 257 #ifndef MULTIPROCESSOR 258 KASSERT(ci == pcb->pcb_veccpu); 259 #endif 260 KASSERT(l == pcb->pcb_veccpu->ci_veclwp); 261 pcb->pcb_veccpu->ci_veclwp = NULL; 262 pcb->pcb_veccpu = NULL; 263 pcb->pcb_flags &= ~PCB_OWNALTIVEC; 264 return; 265 } 266 267 /* 268 * If the state is in the current CPU, just flush the current CPU's 269 * state. 270 */ 271 if (l == ci->ci_veclwp) { 272 save_vec_cpu(); 273 return; 274 } 275 276 277 #ifdef MULTIPROCESSOR 278 /* 279 * It must be on another CPU, flush it from there. 280 */ 281 282 mp_save_vec_lwp(l); 283 #endif 284 } 285 286 #define ZERO_VEC 19 287 288 void 289 vzeropage(paddr_t pa) 290 { 291 const paddr_t ea = pa + PAGE_SIZE; 292 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 293 register_t omsr, msr; 294 295 __asm volatile("mfmsr %0" : "=r"(omsr) :); 296 297 /* 298 * Turn on AltiVec, turn off interrupts. 299 */ 300 msr = (omsr & ~PSL_EE) | PSL_VEC; 301 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 302 303 /* 304 * Save the VEC register we are going to use before we disable 305 * relocation. 306 */ 307 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 308 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 309 310 /* 311 * Zero the page using a single cache line. 312 */ 313 __asm volatile( 314 " sync ;" 315 " mfmsr %[msr];" 316 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 317 " mtmsr %[msr];" /* Turn off DMMU */ 318 " isync;" 319 "1: stvx %[zv], %[pa], %[off0];" 320 " stvxl %[zv], %[pa], %[off16];" 321 " stvx %[zv], %[pa], %[off32];" 322 " stvxl %[zv], %[pa], %[off48];" 323 " addi %[pa], %[pa], 64;" 324 " cmplw %[pa], %[ea];" 325 " blt+ 1b;" 326 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 327 " sync;" 328 " mtmsr %[msr];" /* Turn on DMMU */ 329 " isync;" 330 :: [msr] "r"(msr), [pa] "b"(pa), [ea] "b"(ea), 331 [off0] "r"(0), [off16] "r"(16), [off32] "r"(32), [off48] "r"(48), 332 [zv] "n"(ZERO_VEC)); 333 334 /* 335 * Restore VEC register (now that we can access the stack again). 336 */ 337 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 338 339 /* 340 * Restore old MSR (AltiVec OFF). 341 */ 342 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 343 } 344 345 #define LO_VEC 16 346 #define HI_VEC 17 347 348 void 349 vcopypage(paddr_t dst, paddr_t src) 350 { 351 const paddr_t edst = dst + PAGE_SIZE; 352 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 353 register_t omsr, msr; 354 355 __asm volatile("mfmsr %0" : "=r"(omsr) :); 356 357 /* 358 * Turn on AltiVec, turn off interrupts. 359 */ 360 msr = (omsr & ~PSL_EE) | PSL_VEC; 361 __asm volatile("sync; mtmsr %0; isync" :: "r"(msr)); 362 363 /* 364 * Save the VEC registers we will be using before we disable 365 * relocation. 366 */ 367 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 368 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 369 370 /* 371 * Copy the page using a single cache line, with DMMU 372 * disabled. On most PPCs, two vector registers occupy one 373 * cache line. 374 */ 375 __asm volatile( 376 " sync ;" 377 " mfmsr %[msr];" 378 " rlwinm %[msr],%[msr],0,28,26;" /* Clear PSL_DR */ 379 " mtmsr %[msr];" /* Turn off DMMU */ 380 " isync;" 381 "1: lvx %[lv], %[src], %[off0];" 382 " stvx %[lv], %[dst], %[off0];" 383 " lvxl %[hv], %[src], %[off16];" 384 " stvxl %[hv], %[dst], %[off16];" 385 " addi %[src], %[src], 32;" 386 " addi %[dst], %[dst], 32;" 387 " cmplw %[dst], %[edst];" 388 " blt+ 1b;" 389 " ori %[msr], %[msr], 0x10;" /* Set PSL_DR */ 390 " sync;" 391 " mtmsr %[msr];" /* Turn on DMMU */ 392 " isync;" 393 :: [msr] "r"(msr), [src] "b"(src), [dst] "b"(dst), 394 [edst] "b"(edst), [off0] "r"(0), [off16] "r"(16), 395 [lv] "n"(LO_VEC), [hv] "n"(HI_VEC)); 396 397 /* 398 * Restore VEC registers (now that we can access the stack again). 399 */ 400 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 401 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 402 403 /* 404 * Restore old MSR (AltiVec OFF). 405 */ 406 __asm volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 407 } 408
Indexes created Sun Sep 28 07:09:56 GMT 2025