1 /* $NetBSD: octeon_cop2var.h,v 1.2 2020/06/18 13:52:08 simonb Exp $ */ 2 3 /* 4 * TODO: 5 * 6 * - Utilize prefetch. 7 * 8 * - Implement loop in CBC operations. Take an argument of the number of 9 * blocks. Better if prefetch is used too. 10 * 11 * - In AES and DES buffer block loop, merge encrypt / decrypt. Take a 12 * direction argument (int dir, 0 => encrypt, 1 => decrypt) then branch. 13 */ 14 15 #ifndef _OCTEON_COP2VAR_H_ 16 #define _OCTEON_COP2VAR_H_ 17 18 #ifdef __OCTEON_USEUN__ 19 #define CNASM_ULD(r, o, b) "uld %["#r"], "#o"(%["#b"]) \n\t" 20 #define CNASM_USD(r, o, b) "usd %["#r"], "#o"(%["#b"]) \n\t" 21 #define CNASM_ULW(r, o, b) "ulw %["#r"], "#o"(%["#b"]) \n\t" 22 #define CNASM_USW(r, o, b) "usw %["#r"], "#o"(%["#b"]) \n\t" 23 #else 24 #define __CNASM_ULH(i, r, o, x, b) i" %["#r"], ("#o" + "#x")(%["#b"]) \n\t" 25 #define __CNASM_ULS(p, r, o, l, h, b) __CNASM_ULH(#p"l", r, o, l, b) \ 26 __CNASM_ULH(#p"r", r, o, h, b) 27 #define CNASM_ULD(r, o, b) __CNASM_ULS(ld, r, o, 0, 7, b) 28 #define CNASM_USD(r, o, b) __CNASM_ULS(sd, r, o, 0, 7, b) 29 #define CNASM_ULW(r, o, b) __CNASM_ULS(lw, r, o, 0, 3, b) 30 #define CNASM_USW(r, o, b) __CNASM_ULS(sw, r, o, 0, 3, b) 31 #endif 32 33 #define CNASM_ALD(r, o, b) "ld %["#r"], "#o"(%["#b"]) \n\t" 34 #define CNASM_ASD(r, o, b) "sd %["#r"], "#o"(%["#b"]) \n\t" 35 36 #undef __s 37 #define __s(s) #s /* stringify */ 38 #define CNASM_MT2(r, n, o) "dmtc2 %["#r"], ("__s(n)" + "#o") \n\t" 39 #define CNASM_MF2(r, n, o) "dmfc2 %["#r"], ("__s(n)" + "#o") \n\t" 40 #define CNASM_MT2ZERO(n, o) "dmtc2 $0, ("__s(n)" + "#o") \n\t" 41 #define CNASM_MT2ZERO(n, o) "dmtc2 $0, ("__s(n)" + "#o") \n\t" 42 43 #define CNASM_START() ".set push \n\t" \ 44 ".set mips64 \n\t" \ 45 ".set arch=octeon \n\t" \ 46 ".set noreorder \n\t" 47 #define CNASM_END() ".set pop" 48 49 #define __aligned_t uint64_t 50 #define __unaligned_t uint8_t 51 52 /* -------------------------------------------------------------------------- */ 53 54 /* AES */ 55 56 #define __octeon_cop2_aes_set_key_au_vaddr64(au, AU) \ 57 static inline void \ 58 octeon_cop2_aes_set_key_##au##_vaddr64(uint64_t key, uint32_t klen) \ 59 { \ 60 uint64_t tmp0, tmp1, tmp2, tmp3; \ 61 \ 62 asm volatile ( \ 63 CNASM_START() \ 64 /* %[cnt] is either 4 (256), 3 (192), or 2 (128) */ \ 65 /* Each operation set AESKEYLEN of cop2 also */ \ 66 /* >= 64 */ \ 67 CNASM_##AU##LD(tmp0, 0, key) \ 68 " subu %[cnt], %[cnt], 1 \n" \ 69 " beqz %[cnt], 1f \n" \ 70 CNASM_MT2(tmp0, CVM_MT_AES_KEY, 0) /* delay slot */ \ 71 /* >= 128 */ \ 72 CNASM_##AU##LD(tmp1, 8, key) \ 73 " subu %[cnt], %[cnt], 1 \n" \ 74 " beqz %[cnt], 1f \n" \ 75 CNASM_MT2(tmp1, CVM_MT_AES_KEY, 1) /* delay slot */ \ 76 /* >= 192 */ \ 77 CNASM_##AU##LD(tmp2, 16, key) \ 78 " subu %[cnt], %[cnt], 1 \n" \ 79 " beqz %[cnt], 1f \n" \ 80 CNASM_MT2(tmp2, CVM_MT_AES_KEY, 2) /* delay slot */ \ 81 /* >= 256 */ \ 82 CNASM_##AU##LD(tmp3, 24, key) \ 83 CNASM_MT2(tmp3, CVM_MT_AES_KEY, 3) \ 84 /* done */ \ 85 "1: \n" \ 86 CNASM_END() \ 87 : [tmp0] "=&r" (tmp0), \ 88 [tmp1] "=&r" (tmp1), \ 89 [tmp2] "=&r" (tmp2), \ 90 [tmp3] "=&r" (tmp3) \ 91 : [key] "d" (key), \ 92 [cnt] "d" (klen >> 6)); \ 93 } 94 95 #define __octeon_cop2_aes_set_key_au_ptr(au, AU, ptr) \ 96 static inline void \ 97 octeon_cop2_aes_set_key_##au(ptr key, uint32_t klen) \ 98 { \ 99 octeon_cop2_aes_set_key_##au##_vaddr64((intptr_t)key, klen); \ 100 } 101 102 #define __octeon_cop2_aes_set_key_au(au, AU) \ 103 __octeon_cop2_aes_set_key_au_vaddr64(au, AU) \ 104 __octeon_cop2_aes_set_key_au_ptr(au, AU, __##au##_t *) 105 106 #define __octeon_cop2_aes_set_key \ 107 __octeon_cop2_aes_set_key_au(aligned, A) \ 108 __octeon_cop2_aes_set_key_au(unaligned, U) 109 110 __octeon_cop2_aes_set_key 111 112 static inline void 113 octeon_cop2_aes_set_iv_unaligned_vaddr64(uint64_t iv) 114 { 115 uint64_t tmp0, tmp1; 116 117 asm volatile ( 118 CNASM_START() 119 /* Store the IV to cop2 */ 120 CNASM_ULD(tmp0, 0, iv) 121 CNASM_ULD(tmp1, 8, iv) 122 CNASM_MT2(tmp0, CVM_MT_AES_IV, 0) 123 CNASM_MT2(tmp1, CVM_MT_AES_IV, 1) 124 CNASM_END() 125 : [tmp0] "=&r" (tmp0), 126 [tmp1] "=&r" (tmp1) 127 : [iv] "d" (iv)); 128 } 129 130 static inline void 131 octeon_cop2_aes_set_iv_unaligned(uint8_t *iv) 132 { 133 octeon_cop2_aes_set_iv_unaligned_vaddr64((intptr_t)iv); 134 } 135 136 #define __octeon_cop2_aes_ed_16_au_vaddr64(ed, ED, au, AU) \ 137 static inline void \ 138 octeon_cop2_aes_##ed##_16_##au##_vaddr64(uint64_t d, uint64_t s) \ 139 { \ 140 uint64_t tmp0, tmp1; \ 141 \ 142 asm volatile ( \ 143 CNASM_START() \ 144 CNASM_##AU##LD(tmp0, 0, s) \ 145 CNASM_##AU##LD(tmp1, 8, s) \ 146 CNASM_MT2(tmp0, CVM_MT_AES_##ED##0, 0) \ 147 CNASM_MT2(tmp1, CVM_MT_AES_##ED##1, 0) \ 148 CNASM_MF2(tmp0, CVM_MF_AES_RESINP, 0) \ 149 CNASM_MF2(tmp1, CVM_MF_AES_RESINP, 1) \ 150 CNASM_##AU##SD(tmp0, 0, d) \ 151 CNASM_##AU##SD(tmp1, 8, d) \ 152 CNASM_END() \ 153 : [tmp0] "=&r" (tmp0), \ 154 [tmp1] "=&r" (tmp1) \ 155 : [d] "d" (d), \ 156 [s] "d" (s)); \ 157 } 158 159 #define __octeon_cop2_aes_ed_16_au_ptr(ed, ED, au, AU, ptr) \ 160 static inline void \ 161 octeon_cop2_aes_##ed##_16_##au(ptr d, ptr s) \ 162 { \ 163 octeon_cop2_aes_##ed##_16_##au##_vaddr64((intptr_t)d, (intptr_t)s); \ 164 } 165 166 #define __octeon_cop2_aes_ed_16_au(ed, ED, au, AU) \ 167 __octeon_cop2_aes_ed_16_au_vaddr64(ed, ED, au, AU) \ 168 __octeon_cop2_aes_ed_16_au_ptr(ed, ED, au, AU, __##au##_t *) 169 170 #define __octeon_cop2_aes_ed_16(ed, ED) \ 171 __octeon_cop2_aes_ed_16_au(ed, ED, aligned, A) \ 172 __octeon_cop2_aes_ed_16_au(ed, ED, unaligned, U) 173 174 #define __octeon_cop2_aes_16 \ 175 __octeon_cop2_aes_ed_16(encrypt, ENC) \ 176 __octeon_cop2_aes_ed_16(decrypt, DEC) \ 177 __octeon_cop2_aes_ed_16(cbc_encrypt, ENC_CBC) \ 178 __octeon_cop2_aes_ed_16(cbc_decrypt, DEC_CBC) 179 180 __octeon_cop2_aes_16 181 182 #define __octeon_cop2_aes_ed_block_au_vaddr64(ed, ED, au, AU) \ 183 static inline void \ 184 octeon_cop2_aes_##ed##_block_##au##_vaddr64(uint64_t d, uint64_t s, int n) \ 185 { \ 186 uint64_t tmp0, tmp1; \ 187 uint64_t x = d + 16 * n; \ 188 \ 189 asm volatile ( \ 190 CNASM_START() \ 191 "1: \n" \ 192 CNASM_##AU##LD(tmp0, 0, s) \ 193 CNASM_##AU##LD(tmp1, 8, s) \ 194 CNASM_MT2(tmp0, CVM_MT_AES_##ED##0, 0) \ 195 CNASM_MT2(tmp1, CVM_MT_AES_##ED##1, 0) \ 196 CNASM_MF2(tmp0, CVM_MF_AES_RESINP, 0) \ 197 CNASM_MF2(tmp1, CVM_MF_AES_RESINP, 1) \ 198 CNASM_##AU##SD(tmp0, 0, d) \ 199 CNASM_##AU##SD(tmp1, 8, d) \ 200 " daddu %[d], %[d], 16 \n" \ 201 " bne %[d], %[x], 1b \n" \ 202 " daddu %[s], %[s], 16 \n" /* delay slot */ \ 203 CNASM_END() \ 204 : [d] "=d" (d), \ 205 [s] "=d" (s), \ 206 [tmp0] "=&r" (tmp0), \ 207 [tmp1] "=&r" (tmp1) \ 208 : "0" (d), \ 209 "1" (s), \ 210 [x] "d" (x)); \ 211 } 212 213 #define __octeon_cop2_aes_ed_block_au_ptr(ed, ED, au, AU, ptr) \ 214 static inline void \ 215 octeon_cop2_aes_##ed##_block_##au(ptr d, ptr s, int n) \ 216 { \ 217 octeon_cop2_aes_##ed##_block_##au##_vaddr64((intptr_t)d, (intptr_t)s, n); \ 218 } 219 220 #define __octeon_cop2_aes_ed_block_au(ed, ED, au, AU) \ 221 __octeon_cop2_aes_ed_block_au_vaddr64(ed, ED, au, AU) \ 222 __octeon_cop2_aes_ed_block_au_ptr(ed, ED, au, AU, __##au##_t *) 223 224 #define __octeon_cop2_aes_ed_block(ed, ED) \ 225 __octeon_cop2_aes_ed_block_au(ed, ED, aligned, A) \ 226 __octeon_cop2_aes_ed_block_au(ed, ED, unaligned, U) 227 228 #define __octeon_cop2_aes_block \ 229 /* __octeon_cop2_aes_ed_block(encrypt, ENC) */ \ 230 /* __octeon_cop2_aes_ed_block(decrypt, DEC) */ \ 231 __octeon_cop2_aes_ed_block(cbc_encrypt, ENC_CBC) \ 232 __octeon_cop2_aes_ed_block(cbc_decrypt, DEC_CBC) 233 234 __octeon_cop2_aes_block 235 236 #define __octeon_cop2_aes_ed_64_au_vaddr64(ed, ED, au, AU) \ 237 static inline void \ 238 octeon_cop2_aes_##ed##_64_##au##_vaddr64(uint64_t d, uint64_t s) \ 239 { \ 240 uint64_t tmp0, tmp1, tmp2, tmp3; \ 241 \ 242 asm volatile ( \ 243 CNASM_START() \ 244 CNASM_##AU##LD(tmp0, 0, s) \ 245 CNASM_##AU##LD(tmp1, 8, s) \ 246 CNASM_MT2(tmp0, CVM_MT_AES_##ED##0, 0) \ 247 CNASM_MT2(tmp1, CVM_MT_AES_##ED##1, 0) \ 248 CNASM_##AU##LD(tmp2, 16, s) \ 249 CNASM_##AU##LD(tmp3, 24, s) \ 250 CNASM_MF2(tmp0, CVM_MF_AES_RESINP, 0) \ 251 CNASM_MF2(tmp1, CVM_MF_AES_RESINP, 1) \ 252 CNASM_MT2(tmp2, CVM_MT_AES_##ED##0, 0) \ 253 CNASM_MT2(tmp3, CVM_MT_AES_##ED##1, 0) \ 254 CNASM_##AU##SD(tmp0, 0, d) \ 255 CNASM_##AU##SD(tmp1, 8, d) \ 256 CNASM_MF2(tmp2, CVM_MF_AES_RESINP, 0) \ 257 CNASM_MF2(tmp3, CVM_MF_AES_RESINP, 1) \ 258 CNASM_##AU##SD(tmp2, 16, d) \ 259 CNASM_##AU##SD(tmp3, 24, d) \ 260 CNASM_##AU##LD(tmp0, 32, s) \ 261 CNASM_##AU##LD(tmp1, 40, s) \ 262 CNASM_MT2(tmp0, CVM_MT_AES_##ED##0, 0) \ 263 CNASM_MT2(tmp1, CVM_MT_AES_##ED##1, 0) \ 264 CNASM_##AU##LD(tmp2, 48, s) \ 265 CNASM_##AU##LD(tmp3, 56, s) \ 266 CNASM_MF2(tmp0, CVM_MF_AES_RESINP, 0) \ 267 CNASM_MF2(tmp1, CVM_MF_AES_RESINP, 1) \ 268 CNASM_MT2(tmp2, CVM_MT_AES_##ED##0, 0) \ 269 CNASM_MT2(tmp3, CVM_MT_AES_##ED##1, 0) \ 270 CNASM_##AU##SD(tmp0, 32, d) \ 271 CNASM_##AU##SD(tmp1, 40, d) \ 272 CNASM_MF2(tmp2, CVM_MF_AES_RESINP, 0) \ 273 CNASM_MF2(tmp3, CVM_MF_AES_RESINP, 1) \ 274 CNASM_##AU##SD(tmp2, 48, d) \ 275 CNASM_##AU##SD(tmp3, 56, d) \ 276 CNASM_END() \ 277 : [tmp0] "=&r" (tmp0), \ 278 [tmp1] "=&r" (tmp1), \ 279 [tmp2] "=&r" (tmp2), \ 280 [tmp3] "=&r" (tmp3) \ 281 : [d] "d" (d), \ 282 [s] "d" (s)); \ 283 } 284 285 #define __octeon_cop2_aes_ed_64_au_ptr(ed, ED, au, AU, ptr) \ 286 static inline void \ 287 octeon_cop2_aes_##ed##_64_##au(ptr d, ptr s) \ 288 { \ 289 octeon_cop2_aes_##ed##_64_##au##_vaddr64((intptr_t)d, (intptr_t)s); \ 290 } 291 292 #define __octeon_cop2_aes_ed_64_au(ed, ED, au, AU) \ 293 __octeon_cop2_aes_ed_64_au_vaddr64(ed, ED, au, AU) \ 294 __octeon_cop2_aes_ed_64_au_ptr(ed, ED, au, AU, __##au##_t *) 295 296 #define __octeon_cop2_aes_ed_64(ed, ED) \ 297 __octeon_cop2_aes_ed_64_au(ed, ED, aligned, A) \ 298 __octeon_cop2_aes_ed_64_au(ed, ED, unaligned, U) 299 300 #define __octeon_cop2_aes_64 \ 301 /* __octeon_cop2_aes_ed_64(encrypt, ENC) */ \ 302 /* __octeon_cop2_aes_ed_64(decrypt, DEC) */ \ 303 __octeon_cop2_aes_ed_64(cbc_encrypt, ENC_CBC) \ 304 __octeon_cop2_aes_ed_64(cbc_decrypt, DEC_CBC) 305 306 __octeon_cop2_aes_64 307 308 /* -------------------------------------------------------------------------- */ 309 310 /* DES */ 311 312 static inline void 313 octeon_cop2_des_set_key_unaligned_vaddr64(uint64_t k1, uint64_t k2, uint64_t k3) 314 { 315 uint64_t tmp0, tmp1, tmp2; 316 317 asm volatile ( 318 CNASM_START() 319 /* Set key */ 320 CNASM_ULD(tmp0, 0, k1) 321 CNASM_ULD(tmp1, 0, k2) 322 CNASM_ULD(tmp2, 0, k3) 323 CNASM_MT2(tmp0, CVM_MT_3DES_KEY, 0) 324 CNASM_MT2(tmp1, CVM_MT_3DES_KEY, 1) 325 CNASM_MT2(tmp2, CVM_MT_3DES_KEY, 2) 326 CNASM_END() 327 : [tmp0] "=&r" (tmp0), 328 [tmp1] "=&r" (tmp1), 329 [tmp2] "=&r" (tmp2) 330 : [k1] "d" (k1), 331 [k2] "d" (k2), 332 [k3] "d" (k3)); 333 } 334 335 static inline void 336 octeon_cop2_des_set_key_unaligned(uint64_t *k1, uint64_t *k2, uint64_t *k3) 337 { 338 octeon_cop2_des_set_key_unaligned_vaddr64((intptr_t)k1, (intptr_t)k2, (intptr_t)k3); 339 } 340 341 static inline void 342 octeon_cop2_des_set_iv_unaligned_vaddr64(uint64_t iv) 343 { 344 uint64_t tmp0; 345 346 asm volatile ( 347 CNASM_START() 348 /* Load IV to a register */ 349 CNASM_ULD(tmp0, 0, iv) 350 /* Store the IV to cop2 */ 351 CNASM_MT2(tmp0, CVM_MT_3DES_IV, 0) 352 CNASM_END() 353 : [tmp0] "=&r" (tmp0) 354 : [iv] "d" (iv)); 355 } 356 357 static inline void 358 octeon_cop2_des_set_iv_unaligned(uint8_t *iv) 359 { 360 octeon_cop2_des_set_iv_unaligned_vaddr64((intptr_t)iv); 361 } 362 363 #define __octeon_cop2_des_ed_8_au_vaddr64(ed, ED, au, AU) \ 364 static inline void \ 365 octeon_cop2_des_##ed##_8_##au##_vaddr64(uint64_t d, uint64_t s) \ 366 { \ 367 uint64_t tmp0; \ 368 \ 369 asm volatile ( \ 370 CNASM_START() \ 371 CNASM_##AU##LD(tmp0, 0, s) \ 372 CNASM_MT2(tmp0, CVM_MT_3DES_##ED, 0) \ 373 CNASM_MF2(tmp0, CVM_MF_3DES_RESULT, 0) \ 374 CNASM_##AU##SD(tmp0, 0, s) \ 375 CNASM_END() \ 376 : [tmp0] "=&r" (tmp0) \ 377 : [d] "d" (d), \ 378 [s] "d" (s)); \ 379 } 380 381 #define __octeon_cop2_des_ed_8_au_ptr(ed, ED, au, AU, ptr) \ 382 static inline void \ 383 octeon_cop2_des_##ed##_8_##au(ptr d, ptr s) \ 384 { \ 385 octeon_cop2_des_##ed##_8_##au##_vaddr64((intptr_t)d, (intptr_t)s); \ 386 } 387 388 #define __octeon_cop2_des_ed_8_au(ed, ED, au, AU) \ 389 __octeon_cop2_des_ed_8_au_vaddr64(ed, ED, au, AU) \ 390 __octeon_cop2_des_ed_8_au_ptr(ed, ED, au, AU, __##au##_t *) 391 392 #define __octeon_cop2_des_ed_8(ed, ED) \ 393 __octeon_cop2_des_ed_8_au(ed, ED, aligned, A) \ 394 __octeon_cop2_des_ed_8_au(ed, ED, unaligned, U) 395 396 #define __octeon_cop2_des_8 \ 397 __octeon_cop2_des_ed_8(encrypt, ENC) \ 398 __octeon_cop2_des_ed_8(decrypt, DEC) \ 399 __octeon_cop2_des_ed_8(cbc_encrypt, ENC_CBC) \ 400 __octeon_cop2_des_ed_8(cbc_decrypt, DEC_CBC) 401 402 __octeon_cop2_des_8 403 404 #define __octeon_cop2_des_ed_block_au_vaddr64(ed, ED, au, AU) \ 405 static inline void \ 406 octeon_cop2_des_##ed##_block_##au##_vaddr64(uint64_t d, uint64_t s, int n) \ 407 { \ 408 uint64_t tmp0; \ 409 uint64_t x = d + 8 * n; \ 410 \ 411 asm volatile ( \ 412 CNASM_START() \ 413 "1: \n" \ 414 CNASM_##AU##LD(tmp0, 0, s) \ 415 CNASM_MT2(tmp0, CVM_MT_3DES_##ED, 0) \ 416 CNASM_MF2(tmp0, CVM_MF_3DES_RESULT, 0) \ 417 CNASM_##AU##SD(tmp0, 0, d) \ 418 " daddu %[d], %[d], 8 \n" \ 419 " bne %[d], %[x], 1b \n" \ 420 " daddu %[s], %[s], 8 \n" \ 421 CNASM_END() \ 422 : [d] "=d" (d), \ 423 [s] "=d" (s), \ 424 [tmp0] "=&r" (tmp0) \ 425 : "0" (d), \ 426 "1" (s), \ 427 [x] "d" (x)); \ 428 } 429 430 #define __octeon_cop2_des_ed_block_au_ptr(ed, ED, au, AU, ptr) \ 431 static inline void \ 432 octeon_cop2_des_##ed##_block_##au(ptr d, ptr s, int n) \ 433 { \ 434 octeon_cop2_des_##ed##_block_##au##_vaddr64((intptr_t)d, (intptr_t)s, n); \ 435 } 436 437 #define __octeon_cop2_des_ed_block_au(ed, ED, au, AU) \ 438 __octeon_cop2_des_ed_block_au_vaddr64(ed, ED, au, AU) \ 439 __octeon_cop2_des_ed_block_au_ptr(ed, ED, au, AU, __##au##_t *) 440 441 #define __octeon_cop2_des_ed_block(ed, ED) \ 442 __octeon_cop2_des_ed_block_au(ed, ED, aligned, A) \ 443 __octeon_cop2_des_ed_block_au(ed, ED, unaligned, U) 444 445 #define __octeon_cop2_des_block \ 446 /* __octeon_cop2_des_ed_block(encrypt, ENC) */ \ 447 /* __octeon_cop2_des_ed_block(decrypt, DEC) */ \ 448 __octeon_cop2_des_ed_block(cbc_encrypt, ENC_CBC) \ 449 __octeon_cop2_des_ed_block(cbc_decrypt, DEC_CBC) 450 451 __octeon_cop2_des_block 452 453 #define __octeon_cop2_des_ed_64_au_vaddr64(ed, ED, au, AU) \ 454 static inline void \ 455 octeon_cop2_des_##ed##_64_##au##_vaddr64(uint64_t d, uint64_t s) \ 456 { \ 457 uint64_t tmp0, tmp1, tmp2, tmp3; \ 458 \ 459 asm volatile ( \ 460 CNASM_START() \ 461 CNASM_##AU##LD(tmp0, 0, s) \ 462 CNASM_##AU##LD(tmp1, 8, s) \ 463 CNASM_MT2(tmp0, CVM_MT_3DES_##ED, 0) \ 464 CNASM_##AU##LD(tmp2, 16, s) \ 465 CNASM_MF2(tmp0, CVM_MF_3DES_RESULT, 0) \ 466 CNASM_MT2(tmp1, CVM_MT_3DES_##ED, 0) \ 467 CNASM_##AU##LD(tmp3, 24, s) \ 468 CNASM_MF2(tmp1, CVM_MF_3DES_RESULT, 0) \ 469 CNASM_MT2(tmp2, CVM_MT_3DES_##ED, 0) \ 470 CNASM_##AU##SD(tmp0, 0, d) \ 471 CNASM_MF2(tmp2, CVM_MF_3DES_RESULT, 0) \ 472 CNASM_MT2(tmp3, CVM_MT_3DES_##ED, 0) \ 473 CNASM_##AU##SD(tmp1, 8, d) \ 474 CNASM_MF2(tmp3, CVM_MF_3DES_RESULT, 0) \ 475 CNASM_##AU##SD(tmp2, 16, d) \ 476 CNASM_##AU##SD(tmp3, 24, d) \ 477 CNASM_##AU##LD(tmp0, 32, s) \ 478 CNASM_##AU##LD(tmp1, 40, s) \ 479 CNASM_MT2(tmp0, CVM_MT_3DES_##ED, 0) \ 480 CNASM_##AU##LD(tmp2, 48, s) \ 481 CNASM_MF2(tmp0, CVM_MF_3DES_RESULT, 0) \ 482 CNASM_MT2(tmp1, CVM_MT_3DES_##ED, 0) \ 483 CNASM_##AU##LD(tmp3, 56, s) \ 484 CNASM_MF2(tmp1, CVM_MF_3DES_RESULT, 0) \ 485 CNASM_MT2(tmp2, CVM_MT_3DES_##ED, 0) \ 486 CNASM_##AU##SD(tmp0, 32, d) \ 487 CNASM_MF2(tmp2, CVM_MF_3DES_RESULT, 0) \ 488 CNASM_MT2(tmp3, CVM_MT_3DES_##ED, 0) \ 489 CNASM_##AU##SD(tmp1, 40, d) \ 490 CNASM_MF2(tmp3, CVM_MF_3DES_RESULT, 0) \ 491 CNASM_##AU##SD(tmp2, 48, d) \ 492 CNASM_##AU##SD(tmp3, 56, d) \ 493 CNASM_END() \ 494 : [tmp0] "=&r" (tmp0), \ 495 [tmp1] "=&r" (tmp1), \ 496 [tmp2] "=&r" (tmp2), \ 497 [tmp3] "=&r" (tmp3) \ 498 : [d] "d" (d), \ 499 [s] "d" (s)); \ 500 } 501 502 #define __octeon_cop2_des_ed_64_au_ptr(ed, ED, au, AU, ptr) \ 503 static inline void \ 504 octeon_cop2_des_##ed##_64_##au(ptr d, ptr s) \ 505 { \ 506 octeon_cop2_des_##ed##_64_##au##_vaddr64((intptr_t)d, (intptr_t)s); \ 507 } 508 509 #define __octeon_cop2_des_ed_64_au(ed, ED, au, AU) \ 510 __octeon_cop2_des_ed_64_au_vaddr64(ed, ED, au, AU) \ 511 __octeon_cop2_des_ed_64_au_ptr(ed, ED, au, AU, __##au##_t *) 512 513 #define __octeon_cop2_des_ed_64(ed, ED) \ 514 __octeon_cop2_des_ed_64_au(ed, ED, aligned, A) \ 515 __octeon_cop2_des_ed_64_au(ed, ED, unaligned, U) 516 517 #define __octeon_cop2_des_64 \ 518 /* __octeon_cop2_des_ed_64(encrypt, ENC) */ \ 519 /* __octeon_cop2_des_ed_64(decrypt, DEC) */ \ 520 __octeon_cop2_des_ed_64(cbc_encrypt, ENC_CBC) \ 521 __octeon_cop2_des_ed_64(cbc_decrypt, DEC_CBC) 522 523 __octeon_cop2_des_64 524 525 /* -------------------------------------------------------------------------- */ 526 527 /* MD5 */ 528 529 static inline void 530 octeon_cop2_md5_set_iv_unaligned_vaddr64(uint64_t iv) 531 { 532 uint64_t tmp0, tmp1; 533 534 asm volatile ( 535 CNASM_START() 536 /* Load IV from context */ 537 CNASM_ULD(tmp0, 0, iv) 538 CNASM_ULD(tmp1, 8, iv) 539 CNASM_MT2(tmp0, CVM_MT_HSH_IV, 0) 540 CNASM_MT2(tmp1, CVM_MT_HSH_IV, 1) 541 CNASM_MT2ZERO( CVM_MT_HSH_IV, 2) 542 CNASM_MT2ZERO( CVM_MT_HSH_IV, 3) 543 CNASM_END() 544 : [tmp0] "=&r" (tmp0), 545 [tmp1] "=&r" (tmp1) 546 : [iv] "d" (iv)); 547 } 548 549 static inline void 550 octeon_cop2_md5_set_iv_unaligned(uint64_t *iv) 551 { 552 octeon_cop2_md5_set_iv_unaligned_vaddr64((intptr_t)iv); 553 } 554 555 static inline void 556 octeon_cop2_md5_get_iv_unaligned_vaddr64(uint64_t iv) 557 { 558 uint64_t tmp0, tmp1; 559 560 asm volatile ( 561 CNASM_START() 562 /* Store IV to context */ 563 CNASM_MF2(tmp0, CVM_MF_HSH_IV, 0) 564 CNASM_MF2(tmp1, CVM_MF_HSH_IV, 1) 565 CNASM_USD(tmp0, 0, iv) 566 CNASM_USD(tmp1, 8, iv) 567 CNASM_END() 568 : [tmp0] "=&r" (tmp0), 569 [tmp1] "=&r" (tmp1) 570 : [iv] "d" (iv)); 571 } 572 573 static inline void 574 octeon_cop2_md5_get_iv_unaligned(uint64_t *iv) 575 { 576 octeon_cop2_md5_get_iv_unaligned_vaddr64((intptr_t)iv); 577 } 578 579 static inline void 580 octeon_cop2_md5_update_unaligned_vaddr64(uint64_t src) 581 { 582 uint64_t tmp0, tmp1, tmp2, tmp3; 583 584 asm volatile ( 585 CNASM_START() 586 /* Update HASH */ 587 CNASM_ULD(tmp0, 0, src) 588 CNASM_ULD(tmp1, 8, src) 589 CNASM_ULD(tmp2, 16, src) 590 CNASM_ULD(tmp3, 24, src) 591 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 0) 592 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 1) 593 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 2) 594 CNASM_MT2(tmp3, CVM_MT_HSH_DAT, 3) 595 CNASM_ULD(tmp0, 32, src) 596 CNASM_ULD(tmp1, 40, src) 597 CNASM_ULD(tmp2, 48, src) 598 CNASM_ULD(tmp3, 56, src) 599 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 4) 600 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 5) 601 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 6) 602 CNASM_MT2(tmp3, CVM_MT_HSH_STANDARD5, 0) 603 CNASM_END() 604 : [tmp0] "=&r" (tmp0), 605 [tmp1] "=&r" (tmp1), 606 [tmp2] "=&r" (tmp2), 607 [tmp3] "=&r" (tmp3) 608 : [src] "d" (src)); 609 } 610 611 static inline void 612 octeon_cop2_md5_update_unaligned(uint64_t *src) 613 { 614 octeon_cop2_md5_update_unaligned_vaddr64((intptr_t)src); 615 } 616 617 /* -------------------------------------------------------------------------- */ 618 619 /* SHA1 */ 620 621 static inline void 622 octeon_cop2_sha1_set_iv_unaligned_vaddr64(uint64_t iv) 623 { 624 uint64_t tmp0, tmp1, tmp2; 625 626 asm volatile ( 627 CNASM_START() 628 /* Load IV from context */ 629 CNASM_ULD(tmp0, 0, iv) 630 CNASM_ULD(tmp1, 8, iv) 631 CNASM_ULW(tmp2, 16, iv) 632 "dsll %[tmp2], %[tmp2], 32 \n\t" 633 CNASM_MT2(tmp0, CVM_MT_HSH_IV, 0) 634 CNASM_MT2(tmp1, CVM_MT_HSH_IV, 1) 635 CNASM_MT2(tmp2, CVM_MT_HSH_IV, 2) 636 CNASM_MT2ZERO( CVM_MT_HSH_IV, 3) 637 CNASM_END() 638 : [tmp0] "=&r" (tmp0), 639 [tmp1] "=&r" (tmp1), 640 [tmp2] "=&r" (tmp2) 641 : [iv] "d" (iv)); 642 } 643 644 static inline void 645 octeon_cop2_sha1_set_iv_unaligned(uint8_t *iv) 646 { 647 octeon_cop2_sha1_set_iv_unaligned_vaddr64((intptr_t)iv); 648 } 649 650 static inline void 651 octeon_cop2_sha1_get_iv_unaligned_vaddr64(uint64_t iv) 652 { 653 uint64_t tmp0, tmp1, tmp2; 654 655 asm volatile ( 656 CNASM_START() 657 /* Store IV to context */ 658 CNASM_MF2(tmp0, CVM_MF_HSH_IV, 0) 659 CNASM_MF2(tmp1, CVM_MF_HSH_IV, 1) 660 CNASM_MF2(tmp2, CVM_MF_HSH_IV, 2) 661 CNASM_USD(tmp0, 0, iv) 662 CNASM_USD(tmp1, 8, iv) 663 "dsrl %[tmp2], %[tmp2], 32 \n\t" 664 CNASM_USW(tmp2, 16, iv) 665 CNASM_END() 666 : [tmp0] "=&r" (tmp0), 667 [tmp1] "=&r" (tmp1), 668 [tmp2] "=&r" (tmp2) 669 : [iv] "d" (iv)); 670 } 671 672 static inline void 673 octeon_cop2_sha1_get_iv_unaligned(uint8_t *iv) 674 { 675 octeon_cop2_sha1_get_iv_unaligned_vaddr64((intptr_t)iv); 676 } 677 678 static inline void 679 octeon_cop2_sha1_update_unaligned_vaddr64(uint64_t src) 680 { 681 uint64_t tmp0, tmp1, tmp2, tmp3; 682 683 asm volatile ( 684 CNASM_START() 685 /* Update HASH */ 686 CNASM_ULD(tmp0, 0, src) 687 CNASM_ULD(tmp1, 8, src) 688 CNASM_ULD(tmp2, 16, src) 689 CNASM_ULD(tmp3, 24, src) 690 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 0) 691 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 1) 692 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 2) 693 CNASM_MT2(tmp3, CVM_MT_HSH_DAT, 3) 694 CNASM_ULD(tmp0, 32, src) 695 CNASM_ULD(tmp1, 40, src) 696 CNASM_ULD(tmp2, 48, src) 697 CNASM_ULD(tmp3, 56, src) 698 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 4) 699 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 5) 700 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 6) 701 CNASM_MT2(tmp3, CVM_MT_HSH_STARTSHA, 0) 702 CNASM_END() 703 : [tmp0] "=&r" (tmp0), 704 [tmp1] "=&r" (tmp1), 705 [tmp2] "=&r" (tmp2), 706 [tmp3] "=&r" (tmp3) 707 : [src] "d" (src)); 708 } 709 710 static inline void 711 octeon_cop2_sha1_update_unaligned(uint8_t *src) 712 { 713 octeon_cop2_sha1_update_unaligned_vaddr64((intptr_t)src); 714 } 715 716 /* -------------------------------------------------------------------------- */ 717 718 /* SHA256 */ 719 720 static inline void 721 octeon_cop2_sha256_set_iv_unaligned_vaddr64(uint64_t iv) 722 { 723 uint64_t tmp0, tmp1, tmp2, tmp3; 724 725 asm volatile ( 726 CNASM_START() 727 /* Load IV from context */ 728 CNASM_ULD(tmp0, 0, iv) 729 CNASM_ULD(tmp1, 8, iv) 730 CNASM_ULD(tmp2, 16, iv) 731 CNASM_ULD(tmp3, 24, iv) 732 CNASM_MT2(tmp0, CVM_MT_HSH_IV, 0) 733 CNASM_MT2(tmp1, CVM_MT_HSH_IV, 1) 734 CNASM_MT2(tmp2, CVM_MT_HSH_IV, 2) 735 CNASM_MT2(tmp3, CVM_MT_HSH_IV, 3) 736 CNASM_END() 737 : [tmp0] "=&r" (tmp0), 738 [tmp1] "=&r" (tmp1), 739 [tmp2] "=&r" (tmp2), 740 [tmp3] "=&r" (tmp3) 741 : [iv] "d" (iv)); 742 } 743 744 static inline void 745 octeon_cop2_sha256_set_iv_unaligned(uint8_t *iv) 746 { 747 octeon_cop2_sha256_set_iv_unaligned_vaddr64((intptr_t)iv); 748 } 749 750 static inline void 751 octeon_cop2_sha256_get_iv_unaligned_vaddr64(uint64_t iv) 752 { 753 uint64_t tmp0, tmp1, tmp2, tmp3; 754 755 asm volatile ( 756 CNASM_START() 757 /* Store IV to context */ 758 CNASM_MF2(tmp0, CVM_MF_HSH_IV, 0) 759 CNASM_MF2(tmp1, CVM_MF_HSH_IV, 1) 760 CNASM_MF2(tmp2, CVM_MF_HSH_IV, 2) 761 CNASM_MF2(tmp3, CVM_MF_HSH_IV, 3) 762 CNASM_USD(tmp0, 0, iv) 763 CNASM_USD(tmp1, 8, iv) 764 CNASM_USD(tmp2, 16, iv) 765 CNASM_USD(tmp3, 24, iv) 766 CNASM_END() 767 : [tmp0] "=&r" (tmp0), 768 [tmp1] "=&r" (tmp1), 769 [tmp2] "=&r" (tmp2), 770 [tmp3] "=&r" (tmp3) 771 : [iv] "d" (iv)); 772 } 773 774 static inline void 775 octeon_cop2_sha256_get_iv_unaligned(uint8_t *iv) 776 { 777 octeon_cop2_sha256_get_iv_unaligned_vaddr64((intptr_t)iv); 778 } 779 780 static inline void 781 octeon_cop2_sha256_update_unaligned_vaddr64(uint64_t src) 782 { 783 uint64_t tmp0, tmp1, tmp2, tmp3; 784 785 asm volatile ( 786 CNASM_START() 787 /* Update HASH */ 788 CNASM_ULD(tmp0, 0, src) 789 CNASM_ULD(tmp1, 8, src) 790 CNASM_ULD(tmp2, 16, src) 791 CNASM_ULD(tmp3, 24, src) 792 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 0) 793 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 1) 794 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 2) 795 CNASM_MT2(tmp3, CVM_MT_HSH_DAT, 3) 796 CNASM_ULD(tmp0, 32, src) 797 CNASM_ULD(tmp1, 40, src) 798 CNASM_ULD(tmp2, 48, src) 799 CNASM_ULD(tmp3, 56, src) 800 CNASM_MT2(tmp0, CVM_MT_HSH_DAT, 4) 801 CNASM_MT2(tmp1, CVM_MT_HSH_DAT, 5) 802 CNASM_MT2(tmp2, CVM_MT_HSH_DAT, 6) 803 CNASM_MT2(tmp3, CVM_MT_HSH_STARTSHA256, 0) 804 CNASM_END() 805 : [tmp0] "=&r" (tmp0), 806 [tmp1] "=&r" (tmp1), 807 [tmp2] "=&r" (tmp2), 808 [tmp3] "=&r" (tmp3) 809 : [src] "d" (src)); 810 } 811 812 static inline void 813 octeon_cop2_sha256_update_unaligned(uint8_t *src) 814 { 815 octeon_cop2_sha256_update_unaligned_vaddr64((intptr_t)src); 816 } 817 818 /* -------------------------------------------------------------------------- */ 819 820 /* SHA512 */ 821 822 static inline void 823 octeon_cop2_sha512_set_iv_unaligned_vaddr64(uint64_t iv) 824 { 825 uint64_t tmp0, tmp1, tmp2, tmp3; 826 827 asm volatile ( 828 CNASM_START() 829 /* Load IV from context */ 830 CNASM_ULD(tmp0, 0, iv) 831 CNASM_ULD(tmp1, 8, iv) 832 CNASM_ULD(tmp2, 16, iv) 833 CNASM_ULD(tmp3, 24, iv) 834 CNASM_MT2(tmp0, CVM_MT_HSH_IVW, 0) 835 CNASM_MT2(tmp1, CVM_MT_HSH_IVW, 1) 836 CNASM_MT2(tmp2, CVM_MT_HSH_IVW, 2) 837 CNASM_MT2(tmp3, CVM_MT_HSH_IVW, 3) 838 CNASM_ULD(tmp0, 32, iv) 839 CNASM_ULD(tmp1, 40, iv) 840 CNASM_ULD(tmp2, 48, iv) 841 CNASM_ULD(tmp3, 56, iv) 842 CNASM_MT2(tmp0, CVM_MT_HSH_IVW, 4) 843 CNASM_MT2(tmp1, CVM_MT_HSH_IVW, 5) 844 CNASM_MT2(tmp2, CVM_MT_HSH_IVW, 6) 845 CNASM_MT2(tmp3, CVM_MT_HSH_IVW, 7) 846 CNASM_END() 847 : [tmp0] "=&r" (tmp0), 848 [tmp1] "=&r" (tmp1), 849 [tmp2] "=&r" (tmp2), 850 [tmp3] "=&r" (tmp3) 851 : [iv] "d" (iv)); 852 } 853 854 static inline void 855 octeon_cop2_sha512_set_iv_unaligned(uint8_t *iv) 856 { 857 octeon_cop2_sha512_set_iv_unaligned_vaddr64((intptr_t)iv); 858 } 859 860 static inline void 861 octeon_cop2_sha512_get_iv_unaligned_vaddr64(uint64_t iv) 862 { 863 uint64_t tmp0, tmp1, tmp2, tmp3; 864 865 asm volatile ( 866 CNASM_START() 867 /* Store IV to context */ 868 CNASM_MF2(tmp0, CVM_MF_HSH_IVW, 0) 869 CNASM_MF2(tmp1, CVM_MF_HSH_IVW, 1) 870 CNASM_MF2(tmp2, CVM_MF_HSH_IVW, 2) 871 CNASM_MF2(tmp3, CVM_MF_HSH_IVW, 3) 872 CNASM_USD(tmp0, 0, iv) 873 CNASM_USD(tmp1, 8, iv) 874 CNASM_USD(tmp2, 16, iv) 875 CNASM_USD(tmp3, 24, iv) 876 CNASM_MF2(tmp0, CVM_MF_HSH_IVW, 4) 877 CNASM_MF2(tmp1, CVM_MF_HSH_IVW, 5) 878 CNASM_MF2(tmp2, CVM_MF_HSH_IVW, 6) 879 CNASM_MF2(tmp3, CVM_MF_HSH_IVW, 7) 880 CNASM_USD(tmp0, 32, iv) 881 CNASM_USD(tmp1, 40, iv) 882 CNASM_USD(tmp2, 48, iv) 883 CNASM_USD(tmp3, 56, iv) 884 CNASM_END() 885 : [tmp0] "=&r" (tmp0), 886 [tmp1] "=&r" (tmp1), 887 [tmp2] "=&r" (tmp2), 888 [tmp3] "=&r" (tmp3) 889 : [iv] "d" (iv)); 890 } 891 892 static inline void 893 octeon_cop2_sha512_get_iv_unaligned(uint8_t *iv) 894 { 895 octeon_cop2_sha512_get_iv_unaligned_vaddr64((intptr_t)iv); 896 } 897 898 static inline void 899 octeon_cop2_sha512_update_unaligned_vaddr64(uint64_t src) 900 { 901 uint64_t tmp0, tmp1, tmp2, tmp3; 902 903 asm volatile ( 904 CNASM_START() 905 /* Update HASH */ 906 CNASM_ULD(tmp0, 0, src) 907 CNASM_ULD(tmp1, 8, src) 908 CNASM_ULD(tmp2, 16, src) 909 CNASM_ULD(tmp3, 24, src) 910 CNASM_MT2(tmp0, CVM_MT_HSH_DATW, 0) 911 CNASM_MT2(tmp1, CVM_MT_HSH_DATW, 1) 912 CNASM_MT2(tmp2, CVM_MT_HSH_DATW, 2) 913 CNASM_MT2(tmp3, CVM_MT_HSH_DATW, 3) 914 CNASM_ULD(tmp0, 32, src) 915 CNASM_ULD(tmp1, 40, src) 916 CNASM_ULD(tmp2, 48, src) 917 CNASM_ULD(tmp3, 56, src) 918 CNASM_MT2(tmp0, CVM_MT_HSH_DATW, 4) 919 CNASM_MT2(tmp1, CVM_MT_HSH_DATW, 5) 920 CNASM_MT2(tmp2, CVM_MT_HSH_DATW, 6) 921 CNASM_MT2(tmp3, CVM_MT_HSH_DATW, 7) 922 CNASM_ULD(tmp0, 64, src) 923 CNASM_ULD(tmp1, 72, src) 924 CNASM_ULD(tmp2, 80, src) 925 CNASM_ULD(tmp3, 88, src) 926 CNASM_MT2(tmp0, CVM_MT_HSH_DATW, 8) 927 CNASM_MT2(tmp1, CVM_MT_HSH_DATW, 9) 928 CNASM_MT2(tmp2, CVM_MT_HSH_DATW, 10) 929 CNASM_MT2(tmp3, CVM_MT_HSH_DATW, 11) 930 CNASM_ULD(tmp0, 96, src) 931 CNASM_ULD(tmp1, 104, src) 932 CNASM_ULD(tmp2, 112, src) 933 CNASM_ULD(tmp3, 120, src) 934 CNASM_MT2(tmp0, CVM_MT_HSH_DATW, 12) 935 CNASM_MT2(tmp1, CVM_MT_HSH_DATW, 13) 936 CNASM_MT2(tmp2, CVM_MT_HSH_DATW, 14) 937 CNASM_MT2(tmp3, CVM_MT_HSH_STARTSHA512, 0) 938 CNASM_END() 939 : [tmp0] "=&r" (tmp0), 940 [tmp1] "=&r" (tmp1), 941 [tmp2] "=&r" (tmp2), 942 [tmp3] "=&r" (tmp3) 943 : [src] "d" (src)); 944 } 945 946 static inline void 947 octeon_cop2_sha512_update_unaligned(uint8_t *src) 948 { 949 octeon_cop2_sha512_update_unaligned_vaddr64((intptr_t)src); 950 } 951 952 /* -------------------------------------------------------------------------- */ 953 954 /* CRC */ 955 956 /* XXX */ 957 958 #ifdef notyet 959 static inline void 960 octeon_cop2_crc_polynomial(val) 961 { 962 __asm __volatile ( 963 CNASM_START() 964 " dmtc2 %[val], 0x4200" 965 CNASM_END() 966 : 967 : [val] "d" (val)) 968 } 969 970 #define CVMX_MT_CRC_IV(val) \ 971 __asm __volatile (__PUSH "dmtc2 %0,0x0201" __POP :: "d"(val)) 972 #define CVMX_MT_CRC_BYTE_REFLECT(val) \ 973 __asm __volatile (__PUSH "dmtc2 %0,0x0214" __POP :: "d"(val)) 974 #define CVMX_MT_CRC_HALF_REFLECT(val) \ 975 __asm __volatile (__PUSH "dmtc2 %0,0x0215" __POP :: "d"(val)) 976 #define CVMX_MT_CRC_DWORD_REFLECT(val) \ 977 __asm __volatile (__PUSH "dmtc2 %0,0x1217" __POP :: "d"(val)) 978 #define CVMX_MF_CRC_IV_REFLECT(val) \ 979 __asm __volatile (__PUSH "dmfc2 %0,0x0203" __POP : "=d"(val)) 980 981 static inline void 982 octeon_cop2_crc_reflect(XXX) 983 { 984 __asm __volatile ( 985 CNASM_START() 986 " and %[val], %[len], 15 \n" 987 " beq %[val], %[len], 2f \n" 988 " subu %[tmp], %[len], %[val] \n" 989 " move %[len], %[val] \n" 990 " addu %[tmp], %[buf] \n" 991 992 " .align 3 \n" 993 "1: \n" 994 CNASM_ULD(val, 0, buf) 995 " addu %[buf], 16 \n" 996 CNASM_MT2(val, CVM_MT_CRC_DWORD_REFLECT, 0) 997 CNASM_ULD(val, -8, buf) 998 " bne %[buf], %[tmp], 1b \n" 999 CNASM_MT2(val, CVM_MT_CRC_DWORD_REFLECT, 0) 1000 1001 " .align 3 \n" 1002 "2: and %[val], %[len], 1 \n" 1003 " beq %[val], %[len], 4f \n" 1004 " subu %[tmp], %[len], %[val] \n" 1005 " move %[len], %[val] \n" 1006 " addu %[tmp], %[buf] \n" 1007 1008 " .align 3 \n" 1009 "3: addu %[buf], 2 \n" 1010 " lhu %[val], -2(%[buf]) \n" 1011 " bne %[buf], %[tmp], 3b \n" 1012 CNASM_MT2(val, CVM_MT_CRC_HALF_REFLECT, 0) 1013 1014 " .align 3 \n" 1015 "4: beqz %[len], 5f \n" 1016 " nop \n" 1017 " lbu %[val], 0(%[buf]) \n" 1018 CNASM_MT2(val, CVM_MT_CRC_BYTE_REFLECT, 0) 1019 1020 " .align 3 \n" 1021 "5: \n" 1022 CNASM_END() 1023 : [len] "=d" (len), 1024 [buf] "=d" (buf), 1025 [val] "=d" (val), 1026 [tmp] "=d" (tmp) 1027 : "0" (len), 1028 "1" (buf) 1029 ); 1030 #endif 1031 1032 /* -------------------------------------------------------------------------- */ 1033 1034 /* GFM */ 1035 1036 /* XXX */ 1037 1038 #endif /* _OCTEON_COP2VAR_H_ */ 1039
Indexes created Sat Sep 20 22:09:52 GMT 2025