altivec.c revision 1.3
1 /* $NetBSD: altivec.c,v 1.3 2003/04/02 02:47:19 thorpej Exp $ */ 2 3 /* 4 * Copyright (C) 1996 Wolfgang Solfrank. 5 * Copyright (C) 1996 TooLs GmbH. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by TooLs GmbH. 19 * 4. The name of TooLs GmbH may not be used to endorse or promote products 20 * derived from this software without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 25 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 27 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 28 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 29 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 30 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 31 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 #include <sys/param.h> 34 #include <sys/proc.h> 35 #include <sys/sa.h> 36 #include <sys/systm.h> 37 #include <sys/user.h> 38 #include <sys/malloc.h> 39 #include <sys/pool.h> 40 41 #include <uvm/uvm_extern.h> 42 43 #include <powerpc/altivec.h> 44 #include <powerpc/spr.h> 45 #include <powerpc/psl.h> 46 47 void 48 enable_vec() 49 { 50 struct cpu_info *ci = curcpu(); 51 struct lwp *l = curlwp; 52 struct pcb *pcb = &l->l_addr->u_pcb; 53 struct trapframe *tf = trapframe(l); 54 struct vreg *vr = &pcb->pcb_vr; 55 register_t msr; 56 57 KASSERT(pcb->pcb_veccpu == NULL); 58 59 pcb->pcb_flags |= PCB_ALTIVEC; 60 61 /* 62 * Enable AltiVec temporarily (and disable interrupts). 63 */ 64 msr = mfmsr(); 65 mtmsr((msr & ~PSL_EE) | PSL_VEC); 66 __asm __volatile ("isync"); 67 if (ci->ci_veclwp) { 68 save_vec_cpu(); 69 } 70 KASSERT(curcpu()->ci_veclwp == NULL); 71 72 /* 73 * Restore VSCR by first loading it into a vector and then into VSCR. 74 * (this needs to done before loading the user's vector registers 75 * since we need to use a scratch vector register) 76 */ 77 __asm __volatile("vxor %2,%2,%2; lvewx %2,%0,%1; mtvscr %2" \ 78 :: "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0)); 79 80 /* 81 * VRSAVE will be restored when trap frame returns 82 */ 83 tf->tf_xtra[TF_VRSAVE] = vr->vrsave; 84 85 #define LVX(n,vr) __asm /*__volatile*/("lvx %2,%0,%1" \ 86 :: "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n)); 87 88 /* 89 * Load all 32 vector registers 90 */ 91 LVX( 0,vr); LVX( 1,vr); LVX( 2,vr); LVX( 3,vr); 92 LVX( 4,vr); LVX( 5,vr); LVX( 6,vr); LVX( 7,vr); 93 LVX( 8,vr); LVX( 9,vr); LVX(10,vr); LVX(11,vr); 94 LVX(12,vr); LVX(13,vr); LVX(14,vr); LVX(15,vr); 95 96 LVX(16,vr); LVX(17,vr); LVX(18,vr); LVX(19,vr); 97 LVX(20,vr); LVX(21,vr); LVX(22,vr); LVX(23,vr); 98 LVX(24,vr); LVX(25,vr); LVX(26,vr); LVX(27,vr); 99 LVX(28,vr); LVX(29,vr); LVX(30,vr); LVX(31,vr); 100 __asm __volatile ("isync"); 101 102 /* 103 * Enable AltiVec when we return to user-mode. 104 * Record the new ownership of the AltiVec unit. 105 */ 106 tf->srr1 |= PSL_VEC; 107 curcpu()->ci_veclwp = l; 108 pcb->pcb_veccpu = curcpu(); 109 __asm __volatile ("sync"); 110 111 /* 112 * Restore MSR (turn off AltiVec) 113 */ 114 mtmsr(msr); 115 } 116 117 void 118 save_vec_cpu(void) 119 { 120 struct cpu_info *ci = curcpu(); 121 struct lwp *l; 122 struct pcb *pcb; 123 struct vreg *vr; 124 struct trapframe *tf; 125 register_t msr; 126 127 /* 128 * Turn on AltiVEC, turn off interrupts. 129 */ 130 msr = mfmsr(); 131 mtmsr((msr & ~PSL_EE) | PSL_VEC); 132 __asm __volatile ("isync"); 133 l = ci->ci_veclwp; 134 if (l == NULL) { 135 goto out; 136 } 137 pcb = &l->l_addr->u_pcb; 138 vr = &pcb->pcb_vr; 139 tf = trapframe(l); 140 141 #define STVX(n,vr) __asm /*__volatile*/("stvx %2,%0,%1" \ 142 :: "b"(vr), "r"(offsetof(struct vreg, vreg[n])), "n"(n)); 143 144 /* 145 * Save the vector registers. 146 */ 147 STVX( 0,vr); STVX( 1,vr); STVX( 2,vr); STVX( 3,vr); 148 STVX( 4,vr); STVX( 5,vr); STVX( 6,vr); STVX( 7,vr); 149 STVX( 8,vr); STVX( 9,vr); STVX(10,vr); STVX(11,vr); 150 STVX(12,vr); STVX(13,vr); STVX(14,vr); STVX(15,vr); 151 152 STVX(16,vr); STVX(17,vr); STVX(18,vr); STVX(19,vr); 153 STVX(20,vr); STVX(21,vr); STVX(22,vr); STVX(23,vr); 154 STVX(24,vr); STVX(25,vr); STVX(26,vr); STVX(27,vr); 155 STVX(28,vr); STVX(29,vr); STVX(30,vr); STVX(31,vr); 156 157 /* 158 * Save VSCR (this needs to be done after save the vector registers 159 * since we need to use one as scratch). 160 */ 161 __asm __volatile("mfvscr %2; stvewx %2,%0,%1" \ 162 :: "b"(vr), "r"(offsetof(struct vreg, vscr)), "n"(0)); 163 164 /* 165 * Save VRSAVE 166 */ 167 vr->vrsave = tf->tf_xtra[TF_VRSAVE]; 168 169 /* 170 * Note that we aren't using any CPU resources and stop any 171 * data streams. 172 */ 173 tf->srr1 &= ~PSL_VEC; 174 pcb->pcb_veccpu = NULL; 175 ci->ci_veclwp = NULL; 176 __asm __volatile ("dssall; sync"); 177 178 out: 179 180 /* 181 * Restore MSR (turn off AltiVec) 182 */ 183 mtmsr(msr); 184 } 185 186 /* 187 * Save a process's AltiVEC state to its PCB. The state may be in any CPU. 188 * The process must either be curproc or traced by curproc (and stopped). 189 * (The point being that the process must not run on another CPU during 190 * this function). 191 */ 192 void 193 save_vec_lwp(l) 194 struct lwp *l; 195 { 196 struct pcb *pcb = &l->l_addr->u_pcb; 197 struct cpu_info *ci = curcpu(); 198 199 /* 200 * If it's already in the PCB, there's nothing to do. 201 */ 202 203 if (pcb->pcb_veccpu == NULL) { 204 return; 205 } 206 207 /* 208 * If the state is in the current CPU, just flush the current CPU's 209 * state. 210 */ 211 212 if (l == ci->ci_veclwp) { 213 save_vec_cpu(); 214 return; 215 } 216 217 #ifdef MULTIPROCESSOR 218 219 /* 220 * It must be on another CPU, flush it from there. 221 */ 222 223 mp_save_vec_lwp(l); 224 #endif 225 } 226 227 #define ZERO_VEC 19 228 229 void 230 vzeropage(paddr_t pa) 231 { 232 const paddr_t ea = pa + PAGE_SIZE; 233 uint32_t vec[7], *vp = (void *) roundup((uintptr_t) vec, 16); 234 register_t omsr, msr; 235 236 __asm __volatile("mfmsr %0" : "=r"(omsr) :); 237 238 /* 239 * Turn on AltiVec, turn off interrupts. 240 */ 241 msr = (omsr & ~PSL_EE) | PSL_VEC; 242 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 243 244 /* 245 * Save the VEC register we are going to use before we disable 246 * relocation. 247 */ 248 __asm("stvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 249 __asm("vxor %0,%0,%0" :: "n"(ZERO_VEC)); 250 251 /* 252 * Turn off data relocation (DMMU off). 253 */ 254 msr &= ~PSL_DR; 255 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 256 257 /* 258 * Zero the page using a single cache line. 259 */ 260 do { 261 __asm("stvx %2,%0,%1" :: "b"(pa), "r"( 0), "n"(ZERO_VEC)); 262 __asm("stvxl %2,%0,%1" :: "b"(pa), "r"(16), "n"(ZERO_VEC)); 263 __asm("stvx %2,%0,%1" :: "b"(pa), "r"(32), "n"(ZERO_VEC)); 264 __asm("stvxl %2,%0,%1" :: "b"(pa), "r"(48), "n"(ZERO_VEC)); 265 pa += 64; 266 } while (pa < ea); 267 268 /* 269 * Restore data relocation (DMMU on); 270 */ 271 msr |= PSL_DR; 272 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 273 274 /* 275 * Restore VEC register (now that we can access the stack again). 276 */ 277 __asm("lvx %1,0,%0" :: "r"(vp), "n"(ZERO_VEC)); 278 279 /* 280 * Restore old MSR (AltiVec OFF). 281 */ 282 __asm __volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 283 } 284 285 #define LO_VEC 16 286 #define HI_VEC 17 287 288 void 289 vcopypage(paddr_t dst, paddr_t src) 290 { 291 const paddr_t edst = dst + PAGE_SIZE; 292 uint32_t vec[11], *vp = (void *) roundup((uintptr_t) vec, 16); 293 register_t omsr, msr; 294 295 __asm __volatile("mfmsr %0" : "=r"(omsr) :); 296 297 /* 298 * Turn on AltiVec, turn off interrupts. 299 */ 300 msr = (omsr & ~PSL_EE) | PSL_VEC; 301 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 302 303 /* 304 * Save the VEC registers we will be using before we disable 305 * relocation. 306 */ 307 __asm("stvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 308 __asm("stvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 309 310 /* 311 * Turn off data relocation (DMMU off). 312 */ 313 msr &= ~PSL_DR; 314 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 315 316 /* 317 * Copy the page using a single cache line. On most PPCs, two 318 * vector registers occupy one cache line. 319 */ 320 do { 321 __asm("lvx %2,%0,%1" :: "b"(src), "r"( 0), "n"(LO_VEC)); 322 __asm("stvx %2,%0,%1" :: "b"(dst), "r"( 0), "n"(LO_VEC)); 323 __asm("lvxl %2,%0,%1" :: "b"(src), "r"(16), "n"(HI_VEC)); 324 __asm("stvxl %2,%0,%1" :: "b"(dst), "r"(16), "n"(HI_VEC)); 325 src += 32; 326 dst += 32; 327 } while (dst < edst); 328 329 /* 330 * Restore data relocation (DMMU on); 331 */ 332 msr |= PSL_DR; 333 __asm __volatile("sync; mtmsr %0; isync" :: "r"(msr)); 334 335 /* 336 * Restore VEC registers (now that we can access the stack again). 337 */ 338 __asm("lvx %2,%1,%0" :: "b"(vp), "r"( 0), "n"(LO_VEC)); 339 __asm("lvx %2,%1,%0" :: "b"(vp), "r"(16), "n"(HI_VEC)); 340 341 /* 342 * Restore old MSR (AltiVec OFF). 343 */ 344 __asm __volatile("sync; mtmsr %0; isync" :: "r"(omsr)); 345 } 346
Indexes created Fri Sep 26 20:09:58 GMT 2025