1 #include "arm_asm.h" 2 #include "arm_arch.h" 3 4 .text 5 6 7 .hidden OPENSSL_armcap_P 8 9 .align 5 10 .Lsigma: 11 .quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral 12 .Lone: 13 .long 1,0,0,0 14 .LOPENSSL_armcap_P: 15 #ifdef __ILP32__ 16 .long OPENSSL_armcap_P-. 17 #else 18 .quad OPENSSL_armcap_P-. 19 #endif 20 .byte 67,104,97,67,104,97,50,48,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 21 .align 2 22 23 .globl ChaCha20_ctr32 24 .type ChaCha20_ctr32,%function 25 .align 5 26 ChaCha20_ctr32: 27 cbz x2,.Labort 28 adr x5,.LOPENSSL_armcap_P 29 cmp x2,#192 30 b.lo .Lshort 31 #ifdef __ILP32__ 32 ldrsw x6,[x5] 33 #else 34 ldr x6,[x5] 35 #endif 36 ldr w17,[x6,x5] 37 tst w17,#ARMV7_NEON 38 b.ne ChaCha20_neon 39 40 .Lshort: 41 .inst 0xd503233f // paciasp 42 stp x29,x30,[sp,#-96]! 43 add x29,sp,#0 44 45 adr x5,.Lsigma 46 stp x19,x20,[sp,#16] 47 stp x21,x22,[sp,#32] 48 stp x23,x24,[sp,#48] 49 stp x25,x26,[sp,#64] 50 stp x27,x28,[sp,#80] 51 sub sp,sp,#64 52 53 ldp x22,x23,[x5] // load sigma 54 ldp x24,x25,[x3] // load key 55 ldp x26,x27,[x3,#16] 56 ldp x28,x30,[x4] // load counter 57 #ifdef __ARMEB__ 58 ror x24,x24,#32 59 ror x25,x25,#32 60 ror x26,x26,#32 61 ror x27,x27,#32 62 ror x28,x28,#32 63 ror x30,x30,#32 64 #endif 65 66 .Loop_outer: 67 mov w5,w22 // unpack key block 68 lsr x6,x22,#32 69 mov w7,w23 70 lsr x8,x23,#32 71 mov w9,w24 72 lsr x10,x24,#32 73 mov w11,w25 74 lsr x12,x25,#32 75 mov w13,w26 76 lsr x14,x26,#32 77 mov w15,w27 78 lsr x16,x27,#32 79 mov w17,w28 80 lsr x19,x28,#32 81 mov w20,w30 82 lsr x21,x30,#32 83 84 mov x4,#10 85 subs x2,x2,#64 86 .Loop: 87 sub x4,x4,#1 88 add w5,w5,w9 89 add w6,w6,w10 90 add w7,w7,w11 91 add w8,w8,w12 92 eor w17,w17,w5 93 eor w19,w19,w6 94 eor w20,w20,w7 95 eor w21,w21,w8 96 ror w17,w17,#16 97 ror w19,w19,#16 98 ror w20,w20,#16 99 ror w21,w21,#16 100 add w13,w13,w17 101 add w14,w14,w19 102 add w15,w15,w20 103 add w16,w16,w21 104 eor w9,w9,w13 105 eor w10,w10,w14 106 eor w11,w11,w15 107 eor w12,w12,w16 108 ror w9,w9,#20 109 ror w10,w10,#20 110 ror w11,w11,#20 111 ror w12,w12,#20 112 add w5,w5,w9 113 add w6,w6,w10 114 add w7,w7,w11 115 add w8,w8,w12 116 eor w17,w17,w5 117 eor w19,w19,w6 118 eor w20,w20,w7 119 eor w21,w21,w8 120 ror w17,w17,#24 121 ror w19,w19,#24 122 ror w20,w20,#24 123 ror w21,w21,#24 124 add w13,w13,w17 125 add w14,w14,w19 126 add w15,w15,w20 127 add w16,w16,w21 128 eor w9,w9,w13 129 eor w10,w10,w14 130 eor w11,w11,w15 131 eor w12,w12,w16 132 ror w9,w9,#25 133 ror w10,w10,#25 134 ror w11,w11,#25 135 ror w12,w12,#25 136 add w5,w5,w10 137 add w6,w6,w11 138 add w7,w7,w12 139 add w8,w8,w9 140 eor w21,w21,w5 141 eor w17,w17,w6 142 eor w19,w19,w7 143 eor w20,w20,w8 144 ror w21,w21,#16 145 ror w17,w17,#16 146 ror w19,w19,#16 147 ror w20,w20,#16 148 add w15,w15,w21 149 add w16,w16,w17 150 add w13,w13,w19 151 add w14,w14,w20 152 eor w10,w10,w15 153 eor w11,w11,w16 154 eor w12,w12,w13 155 eor w9,w9,w14 156 ror w10,w10,#20 157 ror w11,w11,#20 158 ror w12,w12,#20 159 ror w9,w9,#20 160 add w5,w5,w10 161 add w6,w6,w11 162 add w7,w7,w12 163 add w8,w8,w9 164 eor w21,w21,w5 165 eor w17,w17,w6 166 eor w19,w19,w7 167 eor w20,w20,w8 168 ror w21,w21,#24 169 ror w17,w17,#24 170 ror w19,w19,#24 171 ror w20,w20,#24 172 add w15,w15,w21 173 add w16,w16,w17 174 add w13,w13,w19 175 add w14,w14,w20 176 eor w10,w10,w15 177 eor w11,w11,w16 178 eor w12,w12,w13 179 eor w9,w9,w14 180 ror w10,w10,#25 181 ror w11,w11,#25 182 ror w12,w12,#25 183 ror w9,w9,#25 184 cbnz x4,.Loop 185 186 add w5,w5,w22 // accumulate key block 187 add x6,x6,x22,lsr#32 188 add w7,w7,w23 189 add x8,x8,x23,lsr#32 190 add w9,w9,w24 191 add x10,x10,x24,lsr#32 192 add w11,w11,w25 193 add x12,x12,x25,lsr#32 194 add w13,w13,w26 195 add x14,x14,x26,lsr#32 196 add w15,w15,w27 197 add x16,x16,x27,lsr#32 198 add w17,w17,w28 199 add x19,x19,x28,lsr#32 200 add w20,w20,w30 201 add x21,x21,x30,lsr#32 202 203 b.lo .Ltail 204 205 add x5,x5,x6,lsl#32 // pack 206 add x7,x7,x8,lsl#32 207 ldp x6,x8,[x1,#0] // load input 208 add x9,x9,x10,lsl#32 209 add x11,x11,x12,lsl#32 210 ldp x10,x12,[x1,#16] 211 add x13,x13,x14,lsl#32 212 add x15,x15,x16,lsl#32 213 ldp x14,x16,[x1,#32] 214 add x17,x17,x19,lsl#32 215 add x20,x20,x21,lsl#32 216 ldp x19,x21,[x1,#48] 217 add x1,x1,#64 218 #ifdef __ARMEB__ 219 rev x5,x5 220 rev x7,x7 221 rev x9,x9 222 rev x11,x11 223 rev x13,x13 224 rev x15,x15 225 rev x17,x17 226 rev x20,x20 227 #endif 228 eor x5,x5,x6 229 eor x7,x7,x8 230 eor x9,x9,x10 231 eor x11,x11,x12 232 eor x13,x13,x14 233 eor x15,x15,x16 234 eor x17,x17,x19 235 eor x20,x20,x21 236 237 stp x5,x7,[x0,#0] // store output 238 add x28,x28,#1 // increment counter 239 stp x9,x11,[x0,#16] 240 stp x13,x15,[x0,#32] 241 stp x17,x20,[x0,#48] 242 add x0,x0,#64 243 244 b.hi .Loop_outer 245 246 ldp x19,x20,[x29,#16] 247 add sp,sp,#64 248 ldp x21,x22,[x29,#32] 249 ldp x23,x24,[x29,#48] 250 ldp x25,x26,[x29,#64] 251 ldp x27,x28,[x29,#80] 252 ldp x29,x30,[sp],#96 253 .inst 0xd50323bf // autiasp 254 .Labort: 255 ret 256 257 .align 4 258 .Ltail: 259 add x2,x2,#64 260 .Less_than_64: 261 sub x0,x0,#1 262 add x1,x1,x2 263 add x0,x0,x2 264 add x4,sp,x2 265 neg x2,x2 266 267 add x5,x5,x6,lsl#32 // pack 268 add x7,x7,x8,lsl#32 269 add x9,x9,x10,lsl#32 270 add x11,x11,x12,lsl#32 271 add x13,x13,x14,lsl#32 272 add x15,x15,x16,lsl#32 273 add x17,x17,x19,lsl#32 274 add x20,x20,x21,lsl#32 275 #ifdef __ARMEB__ 276 rev x5,x5 277 rev x7,x7 278 rev x9,x9 279 rev x11,x11 280 rev x13,x13 281 rev x15,x15 282 rev x17,x17 283 rev x20,x20 284 #endif 285 stp x5,x7,[sp,#0] 286 stp x9,x11,[sp,#16] 287 stp x13,x15,[sp,#32] 288 stp x17,x20,[sp,#48] 289 290 .Loop_tail: 291 ldrb w10,[x1,x2] 292 ldrb w11,[x4,x2] 293 add x2,x2,#1 294 eor w10,w10,w11 295 strb w10,[x0,x2] 296 cbnz x2,.Loop_tail 297 298 stp xzr,xzr,[sp,#0] 299 stp xzr,xzr,[sp,#16] 300 stp xzr,xzr,[sp,#32] 301 stp xzr,xzr,[sp,#48] 302 303 ldp x19,x20,[x29,#16] 304 add sp,sp,#64 305 ldp x21,x22,[x29,#32] 306 ldp x23,x24,[x29,#48] 307 ldp x25,x26,[x29,#64] 308 ldp x27,x28,[x29,#80] 309 ldp x29,x30,[sp],#96 310 .inst 0xd50323bf // autiasp 311 ret 312 .size ChaCha20_ctr32,.-ChaCha20_ctr32 313 314 .type ChaCha20_neon,%function 315 .align 5 316 ChaCha20_neon: 317 .inst 0xd503233f // paciasp 318 stp x29,x30,[sp,#-96]! 319 add x29,sp,#0 320 321 adr x5,.Lsigma 322 stp x19,x20,[sp,#16] 323 stp x21,x22,[sp,#32] 324 stp x23,x24,[sp,#48] 325 stp x25,x26,[sp,#64] 326 stp x27,x28,[sp,#80] 327 cmp x2,#512 328 b.hs .L512_or_more_neon 329 330 sub sp,sp,#64 331 332 ldp x22,x23,[x5] // load sigma 333 ld1 {v24.4s},[x5],#16 334 ldp x24,x25,[x3] // load key 335 ldp x26,x27,[x3,#16] 336 ld1 {v25.4s,v26.4s},[x3] 337 ldp x28,x30,[x4] // load counter 338 ld1 {v27.4s},[x4] 339 ld1 {v31.4s},[x5] 340 #ifdef __ARMEB__ 341 rev64 v24.4s,v24.4s 342 ror x24,x24,#32 343 ror x25,x25,#32 344 ror x26,x26,#32 345 ror x27,x27,#32 346 ror x28,x28,#32 347 ror x30,x30,#32 348 #endif 349 add v27.4s,v27.4s,v31.4s // += 1 350 add v28.4s,v27.4s,v31.4s 351 add v29.4s,v28.4s,v31.4s 352 shl v31.4s,v31.4s,#2 // 1 -> 4 353 354 .Loop_outer_neon: 355 mov w5,w22 // unpack key block 356 lsr x6,x22,#32 357 mov v0.16b,v24.16b 358 mov w7,w23 359 lsr x8,x23,#32 360 mov v4.16b,v24.16b 361 mov w9,w24 362 lsr x10,x24,#32 363 mov v16.16b,v24.16b 364 mov w11,w25 365 mov v1.16b,v25.16b 366 lsr x12,x25,#32 367 mov v5.16b,v25.16b 368 mov w13,w26 369 mov v17.16b,v25.16b 370 lsr x14,x26,#32 371 mov v3.16b,v27.16b 372 mov w15,w27 373 mov v7.16b,v28.16b 374 lsr x16,x27,#32 375 mov v19.16b,v29.16b 376 mov w17,w28 377 mov v2.16b,v26.16b 378 lsr x19,x28,#32 379 mov v6.16b,v26.16b 380 mov w20,w30 381 mov v18.16b,v26.16b 382 lsr x21,x30,#32 383 384 mov x4,#10 385 subs x2,x2,#256 386 .Loop_neon: 387 sub x4,x4,#1 388 add v0.4s,v0.4s,v1.4s 389 add w5,w5,w9 390 add v4.4s,v4.4s,v5.4s 391 add w6,w6,w10 392 add v16.4s,v16.4s,v17.4s 393 add w7,w7,w11 394 eor v3.16b,v3.16b,v0.16b 395 add w8,w8,w12 396 eor v7.16b,v7.16b,v4.16b 397 eor w17,w17,w5 398 eor v19.16b,v19.16b,v16.16b 399 eor w19,w19,w6 400 rev32 v3.8h,v3.8h 401 eor w20,w20,w7 402 rev32 v7.8h,v7.8h 403 eor w21,w21,w8 404 rev32 v19.8h,v19.8h 405 ror w17,w17,#16 406 add v2.4s,v2.4s,v3.4s 407 ror w19,w19,#16 408 add v6.4s,v6.4s,v7.4s 409 ror w20,w20,#16 410 add v18.4s,v18.4s,v19.4s 411 ror w21,w21,#16 412 eor v20.16b,v1.16b,v2.16b 413 add w13,w13,w17 414 eor v21.16b,v5.16b,v6.16b 415 add w14,w14,w19 416 eor v22.16b,v17.16b,v18.16b 417 add w15,w15,w20 418 ushr v1.4s,v20.4s,#20 419 add w16,w16,w21 420 ushr v5.4s,v21.4s,#20 421 eor w9,w9,w13 422 ushr v17.4s,v22.4s,#20 423 eor w10,w10,w14 424 sli v1.4s,v20.4s,#12 425 eor w11,w11,w15 426 sli v5.4s,v21.4s,#12 427 eor w12,w12,w16 428 sli v17.4s,v22.4s,#12 429 ror w9,w9,#20 430 add v0.4s,v0.4s,v1.4s 431 ror w10,w10,#20 432 add v4.4s,v4.4s,v5.4s 433 ror w11,w11,#20 434 add v16.4s,v16.4s,v17.4s 435 ror w12,w12,#20 436 eor v20.16b,v3.16b,v0.16b 437 add w5,w5,w9 438 eor v21.16b,v7.16b,v4.16b 439 add w6,w6,w10 440 eor v22.16b,v19.16b,v16.16b 441 add w7,w7,w11 442 ushr v3.4s,v20.4s,#24 443 add w8,w8,w12 444 ushr v7.4s,v21.4s,#24 445 eor w17,w17,w5 446 ushr v19.4s,v22.4s,#24 447 eor w19,w19,w6 448 sli v3.4s,v20.4s,#8 449 eor w20,w20,w7 450 sli v7.4s,v21.4s,#8 451 eor w21,w21,w8 452 sli v19.4s,v22.4s,#8 453 ror w17,w17,#24 454 add v2.4s,v2.4s,v3.4s 455 ror w19,w19,#24 456 add v6.4s,v6.4s,v7.4s 457 ror w20,w20,#24 458 add v18.4s,v18.4s,v19.4s 459 ror w21,w21,#24 460 eor v20.16b,v1.16b,v2.16b 461 add w13,w13,w17 462 eor v21.16b,v5.16b,v6.16b 463 add w14,w14,w19 464 eor v22.16b,v17.16b,v18.16b 465 add w15,w15,w20 466 ushr v1.4s,v20.4s,#25 467 add w16,w16,w21 468 ushr v5.4s,v21.4s,#25 469 eor w9,w9,w13 470 ushr v17.4s,v22.4s,#25 471 eor w10,w10,w14 472 sli v1.4s,v20.4s,#7 473 eor w11,w11,w15 474 sli v5.4s,v21.4s,#7 475 eor w12,w12,w16 476 sli v17.4s,v22.4s,#7 477 ror w9,w9,#25 478 ext v2.16b,v2.16b,v2.16b,#8 479 ror w10,w10,#25 480 ext v6.16b,v6.16b,v6.16b,#8 481 ror w11,w11,#25 482 ext v18.16b,v18.16b,v18.16b,#8 483 ror w12,w12,#25 484 ext v3.16b,v3.16b,v3.16b,#12 485 ext v7.16b,v7.16b,v7.16b,#12 486 ext v19.16b,v19.16b,v19.16b,#12 487 ext v1.16b,v1.16b,v1.16b,#4 488 ext v5.16b,v5.16b,v5.16b,#4 489 ext v17.16b,v17.16b,v17.16b,#4 490 add v0.4s,v0.4s,v1.4s 491 add w5,w5,w10 492 add v4.4s,v4.4s,v5.4s 493 add w6,w6,w11 494 add v16.4s,v16.4s,v17.4s 495 add w7,w7,w12 496 eor v3.16b,v3.16b,v0.16b 497 add w8,w8,w9 498 eor v7.16b,v7.16b,v4.16b 499 eor w21,w21,w5 500 eor v19.16b,v19.16b,v16.16b 501 eor w17,w17,w6 502 rev32 v3.8h,v3.8h 503 eor w19,w19,w7 504 rev32 v7.8h,v7.8h 505 eor w20,w20,w8 506 rev32 v19.8h,v19.8h 507 ror w21,w21,#16 508 add v2.4s,v2.4s,v3.4s 509 ror w17,w17,#16 510 add v6.4s,v6.4s,v7.4s 511 ror w19,w19,#16 512 add v18.4s,v18.4s,v19.4s 513 ror w20,w20,#16 514 eor v20.16b,v1.16b,v2.16b 515 add w15,w15,w21 516 eor v21.16b,v5.16b,v6.16b 517 add w16,w16,w17 518 eor v22.16b,v17.16b,v18.16b 519 add w13,w13,w19 520 ushr v1.4s,v20.4s,#20 521 add w14,w14,w20 522 ushr v5.4s,v21.4s,#20 523 eor w10,w10,w15 524 ushr v17.4s,v22.4s,#20 525 eor w11,w11,w16 526 sli v1.4s,v20.4s,#12 527 eor w12,w12,w13 528 sli v5.4s,v21.4s,#12 529 eor w9,w9,w14 530 sli v17.4s,v22.4s,#12 531 ror w10,w10,#20 532 add v0.4s,v0.4s,v1.4s 533 ror w11,w11,#20 534 add v4.4s,v4.4s,v5.4s 535 ror w12,w12,#20 536 add v16.4s,v16.4s,v17.4s 537 ror w9,w9,#20 538 eor v20.16b,v3.16b,v0.16b 539 add w5,w5,w10 540 eor v21.16b,v7.16b,v4.16b 541 add w6,w6,w11 542 eor v22.16b,v19.16b,v16.16b 543 add w7,w7,w12 544 ushr v3.4s,v20.4s,#24 545 add w8,w8,w9 546 ushr v7.4s,v21.4s,#24 547 eor w21,w21,w5 548 ushr v19.4s,v22.4s,#24 549 eor w17,w17,w6 550 sli v3.4s,v20.4s,#8 551 eor w19,w19,w7 552 sli v7.4s,v21.4s,#8 553 eor w20,w20,w8 554 sli v19.4s,v22.4s,#8 555 ror w21,w21,#24 556 add v2.4s,v2.4s,v3.4s 557 ror w17,w17,#24 558 add v6.4s,v6.4s,v7.4s 559 ror w19,w19,#24 560 add v18.4s,v18.4s,v19.4s 561 ror w20,w20,#24 562 eor v20.16b,v1.16b,v2.16b 563 add w15,w15,w21 564 eor v21.16b,v5.16b,v6.16b 565 add w16,w16,w17 566 eor v22.16b,v17.16b,v18.16b 567 add w13,w13,w19 568 ushr v1.4s,v20.4s,#25 569 add w14,w14,w20 570 ushr v5.4s,v21.4s,#25 571 eor w10,w10,w15 572 ushr v17.4s,v22.4s,#25 573 eor w11,w11,w16 574 sli v1.4s,v20.4s,#7 575 eor w12,w12,w13 576 sli v5.4s,v21.4s,#7 577 eor w9,w9,w14 578 sli v17.4s,v22.4s,#7 579 ror w10,w10,#25 580 ext v2.16b,v2.16b,v2.16b,#8 581 ror w11,w11,#25 582 ext v6.16b,v6.16b,v6.16b,#8 583 ror w12,w12,#25 584 ext v18.16b,v18.16b,v18.16b,#8 585 ror w9,w9,#25 586 ext v3.16b,v3.16b,v3.16b,#4 587 ext v7.16b,v7.16b,v7.16b,#4 588 ext v19.16b,v19.16b,v19.16b,#4 589 ext v1.16b,v1.16b,v1.16b,#12 590 ext v5.16b,v5.16b,v5.16b,#12 591 ext v17.16b,v17.16b,v17.16b,#12 592 cbnz x4,.Loop_neon 593 594 add w5,w5,w22 // accumulate key block 595 add v0.4s,v0.4s,v24.4s 596 add x6,x6,x22,lsr#32 597 add v4.4s,v4.4s,v24.4s 598 add w7,w7,w23 599 add v16.4s,v16.4s,v24.4s 600 add x8,x8,x23,lsr#32 601 add v2.4s,v2.4s,v26.4s 602 add w9,w9,w24 603 add v6.4s,v6.4s,v26.4s 604 add x10,x10,x24,lsr#32 605 add v18.4s,v18.4s,v26.4s 606 add w11,w11,w25 607 add v3.4s,v3.4s,v27.4s 608 add x12,x12,x25,lsr#32 609 add w13,w13,w26 610 add v7.4s,v7.4s,v28.4s 611 add x14,x14,x26,lsr#32 612 add w15,w15,w27 613 add v19.4s,v19.4s,v29.4s 614 add x16,x16,x27,lsr#32 615 add w17,w17,w28 616 add v1.4s,v1.4s,v25.4s 617 add x19,x19,x28,lsr#32 618 add w20,w20,w30 619 add v5.4s,v5.4s,v25.4s 620 add x21,x21,x30,lsr#32 621 add v17.4s,v17.4s,v25.4s 622 623 b.lo .Ltail_neon 624 625 add x5,x5,x6,lsl#32 // pack 626 add x7,x7,x8,lsl#32 627 ldp x6,x8,[x1,#0] // load input 628 add x9,x9,x10,lsl#32 629 add x11,x11,x12,lsl#32 630 ldp x10,x12,[x1,#16] 631 add x13,x13,x14,lsl#32 632 add x15,x15,x16,lsl#32 633 ldp x14,x16,[x1,#32] 634 add x17,x17,x19,lsl#32 635 add x20,x20,x21,lsl#32 636 ldp x19,x21,[x1,#48] 637 add x1,x1,#64 638 #ifdef __ARMEB__ 639 rev x5,x5 640 rev x7,x7 641 rev x9,x9 642 rev x11,x11 643 rev x13,x13 644 rev x15,x15 645 rev x17,x17 646 rev x20,x20 647 #endif 648 ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 649 eor x5,x5,x6 650 eor x7,x7,x8 651 eor x9,x9,x10 652 eor x11,x11,x12 653 eor x13,x13,x14 654 eor v0.16b,v0.16b,v20.16b 655 eor x15,x15,x16 656 eor v1.16b,v1.16b,v21.16b 657 eor x17,x17,x19 658 eor v2.16b,v2.16b,v22.16b 659 eor x20,x20,x21 660 eor v3.16b,v3.16b,v23.16b 661 ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 662 663 stp x5,x7,[x0,#0] // store output 664 add x28,x28,#4 // increment counter 665 stp x9,x11,[x0,#16] 666 add v27.4s,v27.4s,v31.4s // += 4 667 stp x13,x15,[x0,#32] 668 add v28.4s,v28.4s,v31.4s 669 stp x17,x20,[x0,#48] 670 add v29.4s,v29.4s,v31.4s 671 add x0,x0,#64 672 673 st1 {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64 674 ld1 {v0.16b,v1.16b,v2.16b,v3.16b},[x1],#64 675 676 eor v4.16b,v4.16b,v20.16b 677 eor v5.16b,v5.16b,v21.16b 678 eor v6.16b,v6.16b,v22.16b 679 eor v7.16b,v7.16b,v23.16b 680 st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 681 682 eor v16.16b,v16.16b,v0.16b 683 eor v17.16b,v17.16b,v1.16b 684 eor v18.16b,v18.16b,v2.16b 685 eor v19.16b,v19.16b,v3.16b 686 st1 {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64 687 688 b.hi .Loop_outer_neon 689 690 ldp x19,x20,[x29,#16] 691 add sp,sp,#64 692 ldp x21,x22,[x29,#32] 693 ldp x23,x24,[x29,#48] 694 ldp x25,x26,[x29,#64] 695 ldp x27,x28,[x29,#80] 696 ldp x29,x30,[sp],#96 697 .inst 0xd50323bf // autiasp 698 ret 699 700 .Ltail_neon: 701 add x2,x2,#256 702 cmp x2,#64 703 b.lo .Less_than_64 704 705 add x5,x5,x6,lsl#32 // pack 706 add x7,x7,x8,lsl#32 707 ldp x6,x8,[x1,#0] // load input 708 add x9,x9,x10,lsl#32 709 add x11,x11,x12,lsl#32 710 ldp x10,x12,[x1,#16] 711 add x13,x13,x14,lsl#32 712 add x15,x15,x16,lsl#32 713 ldp x14,x16,[x1,#32] 714 add x17,x17,x19,lsl#32 715 add x20,x20,x21,lsl#32 716 ldp x19,x21,[x1,#48] 717 add x1,x1,#64 718 #ifdef __ARMEB__ 719 rev x5,x5 720 rev x7,x7 721 rev x9,x9 722 rev x11,x11 723 rev x13,x13 724 rev x15,x15 725 rev x17,x17 726 rev x20,x20 727 #endif 728 eor x5,x5,x6 729 eor x7,x7,x8 730 eor x9,x9,x10 731 eor x11,x11,x12 732 eor x13,x13,x14 733 eor x15,x15,x16 734 eor x17,x17,x19 735 eor x20,x20,x21 736 737 stp x5,x7,[x0,#0] // store output 738 add x28,x28,#4 // increment counter 739 stp x9,x11,[x0,#16] 740 stp x13,x15,[x0,#32] 741 stp x17,x20,[x0,#48] 742 add x0,x0,#64 743 b.eq .Ldone_neon 744 sub x2,x2,#64 745 cmp x2,#64 746 b.lo .Less_than_128 747 748 ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 749 eor v0.16b,v0.16b,v20.16b 750 eor v1.16b,v1.16b,v21.16b 751 eor v2.16b,v2.16b,v22.16b 752 eor v3.16b,v3.16b,v23.16b 753 st1 {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64 754 b.eq .Ldone_neon 755 sub x2,x2,#64 756 cmp x2,#64 757 b.lo .Less_than_192 758 759 ld1 {v20.16b,v21.16b,v22.16b,v23.16b},[x1],#64 760 eor v4.16b,v4.16b,v20.16b 761 eor v5.16b,v5.16b,v21.16b 762 eor v6.16b,v6.16b,v22.16b 763 eor v7.16b,v7.16b,v23.16b 764 st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 765 b.eq .Ldone_neon 766 sub x2,x2,#64 767 768 st1 {v16.16b,v17.16b,v18.16b,v19.16b},[sp] 769 b .Last_neon 770 771 .Less_than_128: 772 st1 {v0.16b,v1.16b,v2.16b,v3.16b},[sp] 773 b .Last_neon 774 .Less_than_192: 775 st1 {v4.16b,v5.16b,v6.16b,v7.16b},[sp] 776 b .Last_neon 777 778 .align 4 779 .Last_neon: 780 sub x0,x0,#1 781 add x1,x1,x2 782 add x0,x0,x2 783 add x4,sp,x2 784 neg x2,x2 785 786 .Loop_tail_neon: 787 ldrb w10,[x1,x2] 788 ldrb w11,[x4,x2] 789 add x2,x2,#1 790 eor w10,w10,w11 791 strb w10,[x0,x2] 792 cbnz x2,.Loop_tail_neon 793 794 stp xzr,xzr,[sp,#0] 795 stp xzr,xzr,[sp,#16] 796 stp xzr,xzr,[sp,#32] 797 stp xzr,xzr,[sp,#48] 798 799 .Ldone_neon: 800 ldp x19,x20,[x29,#16] 801 add sp,sp,#64 802 ldp x21,x22,[x29,#32] 803 ldp x23,x24,[x29,#48] 804 ldp x25,x26,[x29,#64] 805 ldp x27,x28,[x29,#80] 806 ldp x29,x30,[sp],#96 807 .inst 0xd50323bf // autiasp 808 ret 809 .size ChaCha20_neon,.-ChaCha20_neon 810 .type ChaCha20_512_neon,%function 811 .align 5 812 ChaCha20_512_neon: 813 .inst 0xd503233f // paciasp 814 stp x29,x30,[sp,#-96]! 815 add x29,sp,#0 816 817 adr x5,.Lsigma 818 stp x19,x20,[sp,#16] 819 stp x21,x22,[sp,#32] 820 stp x23,x24,[sp,#48] 821 stp x25,x26,[sp,#64] 822 stp x27,x28,[sp,#80] 823 824 .L512_or_more_neon: 825 sub sp,sp,#128+64 826 827 ldp x22,x23,[x5] // load sigma 828 ld1 {v24.4s},[x5],#16 829 ldp x24,x25,[x3] // load key 830 ldp x26,x27,[x3,#16] 831 ld1 {v25.4s,v26.4s},[x3] 832 ldp x28,x30,[x4] // load counter 833 ld1 {v27.4s},[x4] 834 ld1 {v31.4s},[x5] 835 #ifdef __ARMEB__ 836 rev64 v24.4s,v24.4s 837 ror x24,x24,#32 838 ror x25,x25,#32 839 ror x26,x26,#32 840 ror x27,x27,#32 841 ror x28,x28,#32 842 ror x30,x30,#32 843 #endif 844 add v27.4s,v27.4s,v31.4s // += 1 845 stp q24,q25,[sp,#0] // off-load key block, invariant part 846 add v27.4s,v27.4s,v31.4s // not typo 847 str q26,[sp,#32] 848 add v28.4s,v27.4s,v31.4s 849 add v29.4s,v28.4s,v31.4s 850 add v30.4s,v29.4s,v31.4s 851 shl v31.4s,v31.4s,#2 // 1 -> 4 852 853 stp d8,d9,[sp,#128+0] // meet ABI requirements 854 stp d10,d11,[sp,#128+16] 855 stp d12,d13,[sp,#128+32] 856 stp d14,d15,[sp,#128+48] 857 858 sub x2,x2,#512 // not typo 859 860 .Loop_outer_512_neon: 861 mov v0.16b,v24.16b 862 mov v4.16b,v24.16b 863 mov v8.16b,v24.16b 864 mov v12.16b,v24.16b 865 mov v16.16b,v24.16b 866 mov v20.16b,v24.16b 867 mov v1.16b,v25.16b 868 mov w5,w22 // unpack key block 869 mov v5.16b,v25.16b 870 lsr x6,x22,#32 871 mov v9.16b,v25.16b 872 mov w7,w23 873 mov v13.16b,v25.16b 874 lsr x8,x23,#32 875 mov v17.16b,v25.16b 876 mov w9,w24 877 mov v21.16b,v25.16b 878 lsr x10,x24,#32 879 mov v3.16b,v27.16b 880 mov w11,w25 881 mov v7.16b,v28.16b 882 lsr x12,x25,#32 883 mov v11.16b,v29.16b 884 mov w13,w26 885 mov v15.16b,v30.16b 886 lsr x14,x26,#32 887 mov v2.16b,v26.16b 888 mov w15,w27 889 mov v6.16b,v26.16b 890 lsr x16,x27,#32 891 add v19.4s,v3.4s,v31.4s // +4 892 mov w17,w28 893 add v23.4s,v7.4s,v31.4s // +4 894 lsr x19,x28,#32 895 mov v10.16b,v26.16b 896 mov w20,w30 897 mov v14.16b,v26.16b 898 lsr x21,x30,#32 899 mov v18.16b,v26.16b 900 stp q27,q28,[sp,#48] // off-load key block, variable part 901 mov v22.16b,v26.16b 902 str q29,[sp,#80] 903 904 mov x4,#5 905 subs x2,x2,#512 906 .Loop_upper_neon: 907 sub x4,x4,#1 908 add v0.4s,v0.4s,v1.4s 909 add w5,w5,w9 910 add v4.4s,v4.4s,v5.4s 911 add w6,w6,w10 912 add v8.4s,v8.4s,v9.4s 913 add w7,w7,w11 914 add v12.4s,v12.4s,v13.4s 915 add w8,w8,w12 916 add v16.4s,v16.4s,v17.4s 917 eor w17,w17,w5 918 add v20.4s,v20.4s,v21.4s 919 eor w19,w19,w6 920 eor v3.16b,v3.16b,v0.16b 921 eor w20,w20,w7 922 eor v7.16b,v7.16b,v4.16b 923 eor w21,w21,w8 924 eor v11.16b,v11.16b,v8.16b 925 ror w17,w17,#16 926 eor v15.16b,v15.16b,v12.16b 927 ror w19,w19,#16 928 eor v19.16b,v19.16b,v16.16b 929 ror w20,w20,#16 930 eor v23.16b,v23.16b,v20.16b 931 ror w21,w21,#16 932 rev32 v3.8h,v3.8h 933 add w13,w13,w17 934 rev32 v7.8h,v7.8h 935 add w14,w14,w19 936 rev32 v11.8h,v11.8h 937 add w15,w15,w20 938 rev32 v15.8h,v15.8h 939 add w16,w16,w21 940 rev32 v19.8h,v19.8h 941 eor w9,w9,w13 942 rev32 v23.8h,v23.8h 943 eor w10,w10,w14 944 add v2.4s,v2.4s,v3.4s 945 eor w11,w11,w15 946 add v6.4s,v6.4s,v7.4s 947 eor w12,w12,w16 948 add v10.4s,v10.4s,v11.4s 949 ror w9,w9,#20 950 add v14.4s,v14.4s,v15.4s 951 ror w10,w10,#20 952 add v18.4s,v18.4s,v19.4s 953 ror w11,w11,#20 954 add v22.4s,v22.4s,v23.4s 955 ror w12,w12,#20 956 eor v24.16b,v1.16b,v2.16b 957 add w5,w5,w9 958 eor v25.16b,v5.16b,v6.16b 959 add w6,w6,w10 960 eor v26.16b,v9.16b,v10.16b 961 add w7,w7,w11 962 eor v27.16b,v13.16b,v14.16b 963 add w8,w8,w12 964 eor v28.16b,v17.16b,v18.16b 965 eor w17,w17,w5 966 eor v29.16b,v21.16b,v22.16b 967 eor w19,w19,w6 968 ushr v1.4s,v24.4s,#20 969 eor w20,w20,w7 970 ushr v5.4s,v25.4s,#20 971 eor w21,w21,w8 972 ushr v9.4s,v26.4s,#20 973 ror w17,w17,#24 974 ushr v13.4s,v27.4s,#20 975 ror w19,w19,#24 976 ushr v17.4s,v28.4s,#20 977 ror w20,w20,#24 978 ushr v21.4s,v29.4s,#20 979 ror w21,w21,#24 980 sli v1.4s,v24.4s,#12 981 add w13,w13,w17 982 sli v5.4s,v25.4s,#12 983 add w14,w14,w19 984 sli v9.4s,v26.4s,#12 985 add w15,w15,w20 986 sli v13.4s,v27.4s,#12 987 add w16,w16,w21 988 sli v17.4s,v28.4s,#12 989 eor w9,w9,w13 990 sli v21.4s,v29.4s,#12 991 eor w10,w10,w14 992 add v0.4s,v0.4s,v1.4s 993 eor w11,w11,w15 994 add v4.4s,v4.4s,v5.4s 995 eor w12,w12,w16 996 add v8.4s,v8.4s,v9.4s 997 ror w9,w9,#25 998 add v12.4s,v12.4s,v13.4s 999 ror w10,w10,#25 1000 add v16.4s,v16.4s,v17.4s 1001 ror w11,w11,#25 1002 add v20.4s,v20.4s,v21.4s 1003 ror w12,w12,#25 1004 eor v24.16b,v3.16b,v0.16b 1005 add w5,w5,w10 1006 eor v25.16b,v7.16b,v4.16b 1007 add w6,w6,w11 1008 eor v26.16b,v11.16b,v8.16b 1009 add w7,w7,w12 1010 eor v27.16b,v15.16b,v12.16b 1011 add w8,w8,w9 1012 eor v28.16b,v19.16b,v16.16b 1013 eor w21,w21,w5 1014 eor v29.16b,v23.16b,v20.16b 1015 eor w17,w17,w6 1016 ushr v3.4s,v24.4s,#24 1017 eor w19,w19,w7 1018 ushr v7.4s,v25.4s,#24 1019 eor w20,w20,w8 1020 ushr v11.4s,v26.4s,#24 1021 ror w21,w21,#16 1022 ushr v15.4s,v27.4s,#24 1023 ror w17,w17,#16 1024 ushr v19.4s,v28.4s,#24 1025 ror w19,w19,#16 1026 ushr v23.4s,v29.4s,#24 1027 ror w20,w20,#16 1028 sli v3.4s,v24.4s,#8 1029 add w15,w15,w21 1030 sli v7.4s,v25.4s,#8 1031 add w16,w16,w17 1032 sli v11.4s,v26.4s,#8 1033 add w13,w13,w19 1034 sli v15.4s,v27.4s,#8 1035 add w14,w14,w20 1036 sli v19.4s,v28.4s,#8 1037 eor w10,w10,w15 1038 sli v23.4s,v29.4s,#8 1039 eor w11,w11,w16 1040 add v2.4s,v2.4s,v3.4s 1041 eor w12,w12,w13 1042 add v6.4s,v6.4s,v7.4s 1043 eor w9,w9,w14 1044 add v10.4s,v10.4s,v11.4s 1045 ror w10,w10,#20 1046 add v14.4s,v14.4s,v15.4s 1047 ror w11,w11,#20 1048 add v18.4s,v18.4s,v19.4s 1049 ror w12,w12,#20 1050 add v22.4s,v22.4s,v23.4s 1051 ror w9,w9,#20 1052 eor v24.16b,v1.16b,v2.16b 1053 add w5,w5,w10 1054 eor v25.16b,v5.16b,v6.16b 1055 add w6,w6,w11 1056 eor v26.16b,v9.16b,v10.16b 1057 add w7,w7,w12 1058 eor v27.16b,v13.16b,v14.16b 1059 add w8,w8,w9 1060 eor v28.16b,v17.16b,v18.16b 1061 eor w21,w21,w5 1062 eor v29.16b,v21.16b,v22.16b 1063 eor w17,w17,w6 1064 ushr v1.4s,v24.4s,#25 1065 eor w19,w19,w7 1066 ushr v5.4s,v25.4s,#25 1067 eor w20,w20,w8 1068 ushr v9.4s,v26.4s,#25 1069 ror w21,w21,#24 1070 ushr v13.4s,v27.4s,#25 1071 ror w17,w17,#24 1072 ushr v17.4s,v28.4s,#25 1073 ror w19,w19,#24 1074 ushr v21.4s,v29.4s,#25 1075 ror w20,w20,#24 1076 sli v1.4s,v24.4s,#7 1077 add w15,w15,w21 1078 sli v5.4s,v25.4s,#7 1079 add w16,w16,w17 1080 sli v9.4s,v26.4s,#7 1081 add w13,w13,w19 1082 sli v13.4s,v27.4s,#7 1083 add w14,w14,w20 1084 sli v17.4s,v28.4s,#7 1085 eor w10,w10,w15 1086 sli v21.4s,v29.4s,#7 1087 eor w11,w11,w16 1088 ext v2.16b,v2.16b,v2.16b,#8 1089 eor w12,w12,w13 1090 ext v6.16b,v6.16b,v6.16b,#8 1091 eor w9,w9,w14 1092 ext v10.16b,v10.16b,v10.16b,#8 1093 ror w10,w10,#25 1094 ext v14.16b,v14.16b,v14.16b,#8 1095 ror w11,w11,#25 1096 ext v18.16b,v18.16b,v18.16b,#8 1097 ror w12,w12,#25 1098 ext v22.16b,v22.16b,v22.16b,#8 1099 ror w9,w9,#25 1100 ext v3.16b,v3.16b,v3.16b,#12 1101 ext v7.16b,v7.16b,v7.16b,#12 1102 ext v11.16b,v11.16b,v11.16b,#12 1103 ext v15.16b,v15.16b,v15.16b,#12 1104 ext v19.16b,v19.16b,v19.16b,#12 1105 ext v23.16b,v23.16b,v23.16b,#12 1106 ext v1.16b,v1.16b,v1.16b,#4 1107 ext v5.16b,v5.16b,v5.16b,#4 1108 ext v9.16b,v9.16b,v9.16b,#4 1109 ext v13.16b,v13.16b,v13.16b,#4 1110 ext v17.16b,v17.16b,v17.16b,#4 1111 ext v21.16b,v21.16b,v21.16b,#4 1112 add v0.4s,v0.4s,v1.4s 1113 add w5,w5,w9 1114 add v4.4s,v4.4s,v5.4s 1115 add w6,w6,w10 1116 add v8.4s,v8.4s,v9.4s 1117 add w7,w7,w11 1118 add v12.4s,v12.4s,v13.4s 1119 add w8,w8,w12 1120 add v16.4s,v16.4s,v17.4s 1121 eor w17,w17,w5 1122 add v20.4s,v20.4s,v21.4s 1123 eor w19,w19,w6 1124 eor v3.16b,v3.16b,v0.16b 1125 eor w20,w20,w7 1126 eor v7.16b,v7.16b,v4.16b 1127 eor w21,w21,w8 1128 eor v11.16b,v11.16b,v8.16b 1129 ror w17,w17,#16 1130 eor v15.16b,v15.16b,v12.16b 1131 ror w19,w19,#16 1132 eor v19.16b,v19.16b,v16.16b 1133 ror w20,w20,#16 1134 eor v23.16b,v23.16b,v20.16b 1135 ror w21,w21,#16 1136 rev32 v3.8h,v3.8h 1137 add w13,w13,w17 1138 rev32 v7.8h,v7.8h 1139 add w14,w14,w19 1140 rev32 v11.8h,v11.8h 1141 add w15,w15,w20 1142 rev32 v15.8h,v15.8h 1143 add w16,w16,w21 1144 rev32 v19.8h,v19.8h 1145 eor w9,w9,w13 1146 rev32 v23.8h,v23.8h 1147 eor w10,w10,w14 1148 add v2.4s,v2.4s,v3.4s 1149 eor w11,w11,w15 1150 add v6.4s,v6.4s,v7.4s 1151 eor w12,w12,w16 1152 add v10.4s,v10.4s,v11.4s 1153 ror w9,w9,#20 1154 add v14.4s,v14.4s,v15.4s 1155 ror w10,w10,#20 1156 add v18.4s,v18.4s,v19.4s 1157 ror w11,w11,#20 1158 add v22.4s,v22.4s,v23.4s 1159 ror w12,w12,#20 1160 eor v24.16b,v1.16b,v2.16b 1161 add w5,w5,w9 1162 eor v25.16b,v5.16b,v6.16b 1163 add w6,w6,w10 1164 eor v26.16b,v9.16b,v10.16b 1165 add w7,w7,w11 1166 eor v27.16b,v13.16b,v14.16b 1167 add w8,w8,w12 1168 eor v28.16b,v17.16b,v18.16b 1169 eor w17,w17,w5 1170 eor v29.16b,v21.16b,v22.16b 1171 eor w19,w19,w6 1172 ushr v1.4s,v24.4s,#20 1173 eor w20,w20,w7 1174 ushr v5.4s,v25.4s,#20 1175 eor w21,w21,w8 1176 ushr v9.4s,v26.4s,#20 1177 ror w17,w17,#24 1178 ushr v13.4s,v27.4s,#20 1179 ror w19,w19,#24 1180 ushr v17.4s,v28.4s,#20 1181 ror w20,w20,#24 1182 ushr v21.4s,v29.4s,#20 1183 ror w21,w21,#24 1184 sli v1.4s,v24.4s,#12 1185 add w13,w13,w17 1186 sli v5.4s,v25.4s,#12 1187 add w14,w14,w19 1188 sli v9.4s,v26.4s,#12 1189 add w15,w15,w20 1190 sli v13.4s,v27.4s,#12 1191 add w16,w16,w21 1192 sli v17.4s,v28.4s,#12 1193 eor w9,w9,w13 1194 sli v21.4s,v29.4s,#12 1195 eor w10,w10,w14 1196 add v0.4s,v0.4s,v1.4s 1197 eor w11,w11,w15 1198 add v4.4s,v4.4s,v5.4s 1199 eor w12,w12,w16 1200 add v8.4s,v8.4s,v9.4s 1201 ror w9,w9,#25 1202 add v12.4s,v12.4s,v13.4s 1203 ror w10,w10,#25 1204 add v16.4s,v16.4s,v17.4s 1205 ror w11,w11,#25 1206 add v20.4s,v20.4s,v21.4s 1207 ror w12,w12,#25 1208 eor v24.16b,v3.16b,v0.16b 1209 add w5,w5,w10 1210 eor v25.16b,v7.16b,v4.16b 1211 add w6,w6,w11 1212 eor v26.16b,v11.16b,v8.16b 1213 add w7,w7,w12 1214 eor v27.16b,v15.16b,v12.16b 1215 add w8,w8,w9 1216 eor v28.16b,v19.16b,v16.16b 1217 eor w21,w21,w5 1218 eor v29.16b,v23.16b,v20.16b 1219 eor w17,w17,w6 1220 ushr v3.4s,v24.4s,#24 1221 eor w19,w19,w7 1222 ushr v7.4s,v25.4s,#24 1223 eor w20,w20,w8 1224 ushr v11.4s,v26.4s,#24 1225 ror w21,w21,#16 1226 ushr v15.4s,v27.4s,#24 1227 ror w17,w17,#16 1228 ushr v19.4s,v28.4s,#24 1229 ror w19,w19,#16 1230 ushr v23.4s,v29.4s,#24 1231 ror w20,w20,#16 1232 sli v3.4s,v24.4s,#8 1233 add w15,w15,w21 1234 sli v7.4s,v25.4s,#8 1235 add w16,w16,w17 1236 sli v11.4s,v26.4s,#8 1237 add w13,w13,w19 1238 sli v15.4s,v27.4s,#8 1239 add w14,w14,w20 1240 sli v19.4s,v28.4s,#8 1241 eor w10,w10,w15 1242 sli v23.4s,v29.4s,#8 1243 eor w11,w11,w16 1244 add v2.4s,v2.4s,v3.4s 1245 eor w12,w12,w13 1246 add v6.4s,v6.4s,v7.4s 1247 eor w9,w9,w14 1248 add v10.4s,v10.4s,v11.4s 1249 ror w10,w10,#20 1250 add v14.4s,v14.4s,v15.4s 1251 ror w11,w11,#20 1252 add v18.4s,v18.4s,v19.4s 1253 ror w12,w12,#20 1254 add v22.4s,v22.4s,v23.4s 1255 ror w9,w9,#20 1256 eor v24.16b,v1.16b,v2.16b 1257 add w5,w5,w10 1258 eor v25.16b,v5.16b,v6.16b 1259 add w6,w6,w11 1260 eor v26.16b,v9.16b,v10.16b 1261 add w7,w7,w12 1262 eor v27.16b,v13.16b,v14.16b 1263 add w8,w8,w9 1264 eor v28.16b,v17.16b,v18.16b 1265 eor w21,w21,w5 1266 eor v29.16b,v21.16b,v22.16b 1267 eor w17,w17,w6 1268 ushr v1.4s,v24.4s,#25 1269 eor w19,w19,w7 1270 ushr v5.4s,v25.4s,#25 1271 eor w20,w20,w8 1272 ushr v9.4s,v26.4s,#25 1273 ror w21,w21,#24 1274 ushr v13.4s,v27.4s,#25 1275 ror w17,w17,#24 1276 ushr v17.4s,v28.4s,#25 1277 ror w19,w19,#24 1278 ushr v21.4s,v29.4s,#25 1279 ror w20,w20,#24 1280 sli v1.4s,v24.4s,#7 1281 add w15,w15,w21 1282 sli v5.4s,v25.4s,#7 1283 add w16,w16,w17 1284 sli v9.4s,v26.4s,#7 1285 add w13,w13,w19 1286 sli v13.4s,v27.4s,#7 1287 add w14,w14,w20 1288 sli v17.4s,v28.4s,#7 1289 eor w10,w10,w15 1290 sli v21.4s,v29.4s,#7 1291 eor w11,w11,w16 1292 ext v2.16b,v2.16b,v2.16b,#8 1293 eor w12,w12,w13 1294 ext v6.16b,v6.16b,v6.16b,#8 1295 eor w9,w9,w14 1296 ext v10.16b,v10.16b,v10.16b,#8 1297 ror w10,w10,#25 1298 ext v14.16b,v14.16b,v14.16b,#8 1299 ror w11,w11,#25 1300 ext v18.16b,v18.16b,v18.16b,#8 1301 ror w12,w12,#25 1302 ext v22.16b,v22.16b,v22.16b,#8 1303 ror w9,w9,#25 1304 ext v3.16b,v3.16b,v3.16b,#4 1305 ext v7.16b,v7.16b,v7.16b,#4 1306 ext v11.16b,v11.16b,v11.16b,#4 1307 ext v15.16b,v15.16b,v15.16b,#4 1308 ext v19.16b,v19.16b,v19.16b,#4 1309 ext v23.16b,v23.16b,v23.16b,#4 1310 ext v1.16b,v1.16b,v1.16b,#12 1311 ext v5.16b,v5.16b,v5.16b,#12 1312 ext v9.16b,v9.16b,v9.16b,#12 1313 ext v13.16b,v13.16b,v13.16b,#12 1314 ext v17.16b,v17.16b,v17.16b,#12 1315 ext v21.16b,v21.16b,v21.16b,#12 1316 cbnz x4,.Loop_upper_neon 1317 1318 add w5,w5,w22 // accumulate key block 1319 add x6,x6,x22,lsr#32 1320 add w7,w7,w23 1321 add x8,x8,x23,lsr#32 1322 add w9,w9,w24 1323 add x10,x10,x24,lsr#32 1324 add w11,w11,w25 1325 add x12,x12,x25,lsr#32 1326 add w13,w13,w26 1327 add x14,x14,x26,lsr#32 1328 add w15,w15,w27 1329 add x16,x16,x27,lsr#32 1330 add w17,w17,w28 1331 add x19,x19,x28,lsr#32 1332 add w20,w20,w30 1333 add x21,x21,x30,lsr#32 1334 1335 add x5,x5,x6,lsl#32 // pack 1336 add x7,x7,x8,lsl#32 1337 ldp x6,x8,[x1,#0] // load input 1338 add x9,x9,x10,lsl#32 1339 add x11,x11,x12,lsl#32 1340 ldp x10,x12,[x1,#16] 1341 add x13,x13,x14,lsl#32 1342 add x15,x15,x16,lsl#32 1343 ldp x14,x16,[x1,#32] 1344 add x17,x17,x19,lsl#32 1345 add x20,x20,x21,lsl#32 1346 ldp x19,x21,[x1,#48] 1347 add x1,x1,#64 1348 #ifdef __ARMEB__ 1349 rev x5,x5 1350 rev x7,x7 1351 rev x9,x9 1352 rev x11,x11 1353 rev x13,x13 1354 rev x15,x15 1355 rev x17,x17 1356 rev x20,x20 1357 #endif 1358 eor x5,x5,x6 1359 eor x7,x7,x8 1360 eor x9,x9,x10 1361 eor x11,x11,x12 1362 eor x13,x13,x14 1363 eor x15,x15,x16 1364 eor x17,x17,x19 1365 eor x20,x20,x21 1366 1367 stp x5,x7,[x0,#0] // store output 1368 add x28,x28,#1 // increment counter 1369 mov w5,w22 // unpack key block 1370 lsr x6,x22,#32 1371 stp x9,x11,[x0,#16] 1372 mov w7,w23 1373 lsr x8,x23,#32 1374 stp x13,x15,[x0,#32] 1375 mov w9,w24 1376 lsr x10,x24,#32 1377 stp x17,x20,[x0,#48] 1378 add x0,x0,#64 1379 mov w11,w25 1380 lsr x12,x25,#32 1381 mov w13,w26 1382 lsr x14,x26,#32 1383 mov w15,w27 1384 lsr x16,x27,#32 1385 mov w17,w28 1386 lsr x19,x28,#32 1387 mov w20,w30 1388 lsr x21,x30,#32 1389 1390 mov x4,#5 1391 .Loop_lower_neon: 1392 sub x4,x4,#1 1393 add v0.4s,v0.4s,v1.4s 1394 add w5,w5,w9 1395 add v4.4s,v4.4s,v5.4s 1396 add w6,w6,w10 1397 add v8.4s,v8.4s,v9.4s 1398 add w7,w7,w11 1399 add v12.4s,v12.4s,v13.4s 1400 add w8,w8,w12 1401 add v16.4s,v16.4s,v17.4s 1402 eor w17,w17,w5 1403 add v20.4s,v20.4s,v21.4s 1404 eor w19,w19,w6 1405 eor v3.16b,v3.16b,v0.16b 1406 eor w20,w20,w7 1407 eor v7.16b,v7.16b,v4.16b 1408 eor w21,w21,w8 1409 eor v11.16b,v11.16b,v8.16b 1410 ror w17,w17,#16 1411 eor v15.16b,v15.16b,v12.16b 1412 ror w19,w19,#16 1413 eor v19.16b,v19.16b,v16.16b 1414 ror w20,w20,#16 1415 eor v23.16b,v23.16b,v20.16b 1416 ror w21,w21,#16 1417 rev32 v3.8h,v3.8h 1418 add w13,w13,w17 1419 rev32 v7.8h,v7.8h 1420 add w14,w14,w19 1421 rev32 v11.8h,v11.8h 1422 add w15,w15,w20 1423 rev32 v15.8h,v15.8h 1424 add w16,w16,w21 1425 rev32 v19.8h,v19.8h 1426 eor w9,w9,w13 1427 rev32 v23.8h,v23.8h 1428 eor w10,w10,w14 1429 add v2.4s,v2.4s,v3.4s 1430 eor w11,w11,w15 1431 add v6.4s,v6.4s,v7.4s 1432 eor w12,w12,w16 1433 add v10.4s,v10.4s,v11.4s 1434 ror w9,w9,#20 1435 add v14.4s,v14.4s,v15.4s 1436 ror w10,w10,#20 1437 add v18.4s,v18.4s,v19.4s 1438 ror w11,w11,#20 1439 add v22.4s,v22.4s,v23.4s 1440 ror w12,w12,#20 1441 eor v24.16b,v1.16b,v2.16b 1442 add w5,w5,w9 1443 eor v25.16b,v5.16b,v6.16b 1444 add w6,w6,w10 1445 eor v26.16b,v9.16b,v10.16b 1446 add w7,w7,w11 1447 eor v27.16b,v13.16b,v14.16b 1448 add w8,w8,w12 1449 eor v28.16b,v17.16b,v18.16b 1450 eor w17,w17,w5 1451 eor v29.16b,v21.16b,v22.16b 1452 eor w19,w19,w6 1453 ushr v1.4s,v24.4s,#20 1454 eor w20,w20,w7 1455 ushr v5.4s,v25.4s,#20 1456 eor w21,w21,w8 1457 ushr v9.4s,v26.4s,#20 1458 ror w17,w17,#24 1459 ushr v13.4s,v27.4s,#20 1460 ror w19,w19,#24 1461 ushr v17.4s,v28.4s,#20 1462 ror w20,w20,#24 1463 ushr v21.4s,v29.4s,#20 1464 ror w21,w21,#24 1465 sli v1.4s,v24.4s,#12 1466 add w13,w13,w17 1467 sli v5.4s,v25.4s,#12 1468 add w14,w14,w19 1469 sli v9.4s,v26.4s,#12 1470 add w15,w15,w20 1471 sli v13.4s,v27.4s,#12 1472 add w16,w16,w21 1473 sli v17.4s,v28.4s,#12 1474 eor w9,w9,w13 1475 sli v21.4s,v29.4s,#12 1476 eor w10,w10,w14 1477 add v0.4s,v0.4s,v1.4s 1478 eor w11,w11,w15 1479 add v4.4s,v4.4s,v5.4s 1480 eor w12,w12,w16 1481 add v8.4s,v8.4s,v9.4s 1482 ror w9,w9,#25 1483 add v12.4s,v12.4s,v13.4s 1484 ror w10,w10,#25 1485 add v16.4s,v16.4s,v17.4s 1486 ror w11,w11,#25 1487 add v20.4s,v20.4s,v21.4s 1488 ror w12,w12,#25 1489 eor v24.16b,v3.16b,v0.16b 1490 add w5,w5,w10 1491 eor v25.16b,v7.16b,v4.16b 1492 add w6,w6,w11 1493 eor v26.16b,v11.16b,v8.16b 1494 add w7,w7,w12 1495 eor v27.16b,v15.16b,v12.16b 1496 add w8,w8,w9 1497 eor v28.16b,v19.16b,v16.16b 1498 eor w21,w21,w5 1499 eor v29.16b,v23.16b,v20.16b 1500 eor w17,w17,w6 1501 ushr v3.4s,v24.4s,#24 1502 eor w19,w19,w7 1503 ushr v7.4s,v25.4s,#24 1504 eor w20,w20,w8 1505 ushr v11.4s,v26.4s,#24 1506 ror w21,w21,#16 1507 ushr v15.4s,v27.4s,#24 1508 ror w17,w17,#16 1509 ushr v19.4s,v28.4s,#24 1510 ror w19,w19,#16 1511 ushr v23.4s,v29.4s,#24 1512 ror w20,w20,#16 1513 sli v3.4s,v24.4s,#8 1514 add w15,w15,w21 1515 sli v7.4s,v25.4s,#8 1516 add w16,w16,w17 1517 sli v11.4s,v26.4s,#8 1518 add w13,w13,w19 1519 sli v15.4s,v27.4s,#8 1520 add w14,w14,w20 1521 sli v19.4s,v28.4s,#8 1522 eor w10,w10,w15 1523 sli v23.4s,v29.4s,#8 1524 eor w11,w11,w16 1525 add v2.4s,v2.4s,v3.4s 1526 eor w12,w12,w13 1527 add v6.4s,v6.4s,v7.4s 1528 eor w9,w9,w14 1529 add v10.4s,v10.4s,v11.4s 1530 ror w10,w10,#20 1531 add v14.4s,v14.4s,v15.4s 1532 ror w11,w11,#20 1533 add v18.4s,v18.4s,v19.4s 1534 ror w12,w12,#20 1535 add v22.4s,v22.4s,v23.4s 1536 ror w9,w9,#20 1537 eor v24.16b,v1.16b,v2.16b 1538 add w5,w5,w10 1539 eor v25.16b,v5.16b,v6.16b 1540 add w6,w6,w11 1541 eor v26.16b,v9.16b,v10.16b 1542 add w7,w7,w12 1543 eor v27.16b,v13.16b,v14.16b 1544 add w8,w8,w9 1545 eor v28.16b,v17.16b,v18.16b 1546 eor w21,w21,w5 1547 eor v29.16b,v21.16b,v22.16b 1548 eor w17,w17,w6 1549 ushr v1.4s,v24.4s,#25 1550 eor w19,w19,w7 1551 ushr v5.4s,v25.4s,#25 1552 eor w20,w20,w8 1553 ushr v9.4s,v26.4s,#25 1554 ror w21,w21,#24 1555 ushr v13.4s,v27.4s,#25 1556 ror w17,w17,#24 1557 ushr v17.4s,v28.4s,#25 1558 ror w19,w19,#24 1559 ushr v21.4s,v29.4s,#25 1560 ror w20,w20,#24 1561 sli v1.4s,v24.4s,#7 1562 add w15,w15,w21 1563 sli v5.4s,v25.4s,#7 1564 add w16,w16,w17 1565 sli v9.4s,v26.4s,#7 1566 add w13,w13,w19 1567 sli v13.4s,v27.4s,#7 1568 add w14,w14,w20 1569 sli v17.4s,v28.4s,#7 1570 eor w10,w10,w15 1571 sli v21.4s,v29.4s,#7 1572 eor w11,w11,w16 1573 ext v2.16b,v2.16b,v2.16b,#8 1574 eor w12,w12,w13 1575 ext v6.16b,v6.16b,v6.16b,#8 1576 eor w9,w9,w14 1577 ext v10.16b,v10.16b,v10.16b,#8 1578 ror w10,w10,#25 1579 ext v14.16b,v14.16b,v14.16b,#8 1580 ror w11,w11,#25 1581 ext v18.16b,v18.16b,v18.16b,#8 1582 ror w12,w12,#25 1583 ext v22.16b,v22.16b,v22.16b,#8 1584 ror w9,w9,#25 1585 ext v3.16b,v3.16b,v3.16b,#12 1586 ext v7.16b,v7.16b,v7.16b,#12 1587 ext v11.16b,v11.16b,v11.16b,#12 1588 ext v15.16b,v15.16b,v15.16b,#12 1589 ext v19.16b,v19.16b,v19.16b,#12 1590 ext v23.16b,v23.16b,v23.16b,#12 1591 ext v1.16b,v1.16b,v1.16b,#4 1592 ext v5.16b,v5.16b,v5.16b,#4 1593 ext v9.16b,v9.16b,v9.16b,#4 1594 ext v13.16b,v13.16b,v13.16b,#4 1595 ext v17.16b,v17.16b,v17.16b,#4 1596 ext v21.16b,v21.16b,v21.16b,#4 1597 add v0.4s,v0.4s,v1.4s 1598 add w5,w5,w9 1599 add v4.4s,v4.4s,v5.4s 1600 add w6,w6,w10 1601 add v8.4s,v8.4s,v9.4s 1602 add w7,w7,w11 1603 add v12.4s,v12.4s,v13.4s 1604 add w8,w8,w12 1605 add v16.4s,v16.4s,v17.4s 1606 eor w17,w17,w5 1607 add v20.4s,v20.4s,v21.4s 1608 eor w19,w19,w6 1609 eor v3.16b,v3.16b,v0.16b 1610 eor w20,w20,w7 1611 eor v7.16b,v7.16b,v4.16b 1612 eor w21,w21,w8 1613 eor v11.16b,v11.16b,v8.16b 1614 ror w17,w17,#16 1615 eor v15.16b,v15.16b,v12.16b 1616 ror w19,w19,#16 1617 eor v19.16b,v19.16b,v16.16b 1618 ror w20,w20,#16 1619 eor v23.16b,v23.16b,v20.16b 1620 ror w21,w21,#16 1621 rev32 v3.8h,v3.8h 1622 add w13,w13,w17 1623 rev32 v7.8h,v7.8h 1624 add w14,w14,w19 1625 rev32 v11.8h,v11.8h 1626 add w15,w15,w20 1627 rev32 v15.8h,v15.8h 1628 add w16,w16,w21 1629 rev32 v19.8h,v19.8h 1630 eor w9,w9,w13 1631 rev32 v23.8h,v23.8h 1632 eor w10,w10,w14 1633 add v2.4s,v2.4s,v3.4s 1634 eor w11,w11,w15 1635 add v6.4s,v6.4s,v7.4s 1636 eor w12,w12,w16 1637 add v10.4s,v10.4s,v11.4s 1638 ror w9,w9,#20 1639 add v14.4s,v14.4s,v15.4s 1640 ror w10,w10,#20 1641 add v18.4s,v18.4s,v19.4s 1642 ror w11,w11,#20 1643 add v22.4s,v22.4s,v23.4s 1644 ror w12,w12,#20 1645 eor v24.16b,v1.16b,v2.16b 1646 add w5,w5,w9 1647 eor v25.16b,v5.16b,v6.16b 1648 add w6,w6,w10 1649 eor v26.16b,v9.16b,v10.16b 1650 add w7,w7,w11 1651 eor v27.16b,v13.16b,v14.16b 1652 add w8,w8,w12 1653 eor v28.16b,v17.16b,v18.16b 1654 eor w17,w17,w5 1655 eor v29.16b,v21.16b,v22.16b 1656 eor w19,w19,w6 1657 ushr v1.4s,v24.4s,#20 1658 eor w20,w20,w7 1659 ushr v5.4s,v25.4s,#20 1660 eor w21,w21,w8 1661 ushr v9.4s,v26.4s,#20 1662 ror w17,w17,#24 1663 ushr v13.4s,v27.4s,#20 1664 ror w19,w19,#24 1665 ushr v17.4s,v28.4s,#20 1666 ror w20,w20,#24 1667 ushr v21.4s,v29.4s,#20 1668 ror w21,w21,#24 1669 sli v1.4s,v24.4s,#12 1670 add w13,w13,w17 1671 sli v5.4s,v25.4s,#12 1672 add w14,w14,w19 1673 sli v9.4s,v26.4s,#12 1674 add w15,w15,w20 1675 sli v13.4s,v27.4s,#12 1676 add w16,w16,w21 1677 sli v17.4s,v28.4s,#12 1678 eor w9,w9,w13 1679 sli v21.4s,v29.4s,#12 1680 eor w10,w10,w14 1681 add v0.4s,v0.4s,v1.4s 1682 eor w11,w11,w15 1683 add v4.4s,v4.4s,v5.4s 1684 eor w12,w12,w16 1685 add v8.4s,v8.4s,v9.4s 1686 ror w9,w9,#25 1687 add v12.4s,v12.4s,v13.4s 1688 ror w10,w10,#25 1689 add v16.4s,v16.4s,v17.4s 1690 ror w11,w11,#25 1691 add v20.4s,v20.4s,v21.4s 1692 ror w12,w12,#25 1693 eor v24.16b,v3.16b,v0.16b 1694 add w5,w5,w10 1695 eor v25.16b,v7.16b,v4.16b 1696 add w6,w6,w11 1697 eor v26.16b,v11.16b,v8.16b 1698 add w7,w7,w12 1699 eor v27.16b,v15.16b,v12.16b 1700 add w8,w8,w9 1701 eor v28.16b,v19.16b,v16.16b 1702 eor w21,w21,w5 1703 eor v29.16b,v23.16b,v20.16b 1704 eor w17,w17,w6 1705 ushr v3.4s,v24.4s,#24 1706 eor w19,w19,w7 1707 ushr v7.4s,v25.4s,#24 1708 eor w20,w20,w8 1709 ushr v11.4s,v26.4s,#24 1710 ror w21,w21,#16 1711 ushr v15.4s,v27.4s,#24 1712 ror w17,w17,#16 1713 ushr v19.4s,v28.4s,#24 1714 ror w19,w19,#16 1715 ushr v23.4s,v29.4s,#24 1716 ror w20,w20,#16 1717 sli v3.4s,v24.4s,#8 1718 add w15,w15,w21 1719 sli v7.4s,v25.4s,#8 1720 add w16,w16,w17 1721 sli v11.4s,v26.4s,#8 1722 add w13,w13,w19 1723 sli v15.4s,v27.4s,#8 1724 add w14,w14,w20 1725 sli v19.4s,v28.4s,#8 1726 eor w10,w10,w15 1727 sli v23.4s,v29.4s,#8 1728 eor w11,w11,w16 1729 add v2.4s,v2.4s,v3.4s 1730 eor w12,w12,w13 1731 add v6.4s,v6.4s,v7.4s 1732 eor w9,w9,w14 1733 add v10.4s,v10.4s,v11.4s 1734 ror w10,w10,#20 1735 add v14.4s,v14.4s,v15.4s 1736 ror w11,w11,#20 1737 add v18.4s,v18.4s,v19.4s 1738 ror w12,w12,#20 1739 add v22.4s,v22.4s,v23.4s 1740 ror w9,w9,#20 1741 eor v24.16b,v1.16b,v2.16b 1742 add w5,w5,w10 1743 eor v25.16b,v5.16b,v6.16b 1744 add w6,w6,w11 1745 eor v26.16b,v9.16b,v10.16b 1746 add w7,w7,w12 1747 eor v27.16b,v13.16b,v14.16b 1748 add w8,w8,w9 1749 eor v28.16b,v17.16b,v18.16b 1750 eor w21,w21,w5 1751 eor v29.16b,v21.16b,v22.16b 1752 eor w17,w17,w6 1753 ushr v1.4s,v24.4s,#25 1754 eor w19,w19,w7 1755 ushr v5.4s,v25.4s,#25 1756 eor w20,w20,w8 1757 ushr v9.4s,v26.4s,#25 1758 ror w21,w21,#24 1759 ushr v13.4s,v27.4s,#25 1760 ror w17,w17,#24 1761 ushr v17.4s,v28.4s,#25 1762 ror w19,w19,#24 1763 ushr v21.4s,v29.4s,#25 1764 ror w20,w20,#24 1765 sli v1.4s,v24.4s,#7 1766 add w15,w15,w21 1767 sli v5.4s,v25.4s,#7 1768 add w16,w16,w17 1769 sli v9.4s,v26.4s,#7 1770 add w13,w13,w19 1771 sli v13.4s,v27.4s,#7 1772 add w14,w14,w20 1773 sli v17.4s,v28.4s,#7 1774 eor w10,w10,w15 1775 sli v21.4s,v29.4s,#7 1776 eor w11,w11,w16 1777 ext v2.16b,v2.16b,v2.16b,#8 1778 eor w12,w12,w13 1779 ext v6.16b,v6.16b,v6.16b,#8 1780 eor w9,w9,w14 1781 ext v10.16b,v10.16b,v10.16b,#8 1782 ror w10,w10,#25 1783 ext v14.16b,v14.16b,v14.16b,#8 1784 ror w11,w11,#25 1785 ext v18.16b,v18.16b,v18.16b,#8 1786 ror w12,w12,#25 1787 ext v22.16b,v22.16b,v22.16b,#8 1788 ror w9,w9,#25 1789 ext v3.16b,v3.16b,v3.16b,#4 1790 ext v7.16b,v7.16b,v7.16b,#4 1791 ext v11.16b,v11.16b,v11.16b,#4 1792 ext v15.16b,v15.16b,v15.16b,#4 1793 ext v19.16b,v19.16b,v19.16b,#4 1794 ext v23.16b,v23.16b,v23.16b,#4 1795 ext v1.16b,v1.16b,v1.16b,#12 1796 ext v5.16b,v5.16b,v5.16b,#12 1797 ext v9.16b,v9.16b,v9.16b,#12 1798 ext v13.16b,v13.16b,v13.16b,#12 1799 ext v17.16b,v17.16b,v17.16b,#12 1800 ext v21.16b,v21.16b,v21.16b,#12 1801 cbnz x4,.Loop_lower_neon 1802 1803 add w5,w5,w22 // accumulate key block 1804 ldp q24,q25,[sp,#0] 1805 add x6,x6,x22,lsr#32 1806 ldp q26,q27,[sp,#32] 1807 add w7,w7,w23 1808 ldp q28,q29,[sp,#64] 1809 add x8,x8,x23,lsr#32 1810 add v0.4s,v0.4s,v24.4s 1811 add w9,w9,w24 1812 add v4.4s,v4.4s,v24.4s 1813 add x10,x10,x24,lsr#32 1814 add v8.4s,v8.4s,v24.4s 1815 add w11,w11,w25 1816 add v12.4s,v12.4s,v24.4s 1817 add x12,x12,x25,lsr#32 1818 add v16.4s,v16.4s,v24.4s 1819 add w13,w13,w26 1820 add v20.4s,v20.4s,v24.4s 1821 add x14,x14,x26,lsr#32 1822 add v2.4s,v2.4s,v26.4s 1823 add w15,w15,w27 1824 add v6.4s,v6.4s,v26.4s 1825 add x16,x16,x27,lsr#32 1826 add v10.4s,v10.4s,v26.4s 1827 add w17,w17,w28 1828 add v14.4s,v14.4s,v26.4s 1829 add x19,x19,x28,lsr#32 1830 add v18.4s,v18.4s,v26.4s 1831 add w20,w20,w30 1832 add v22.4s,v22.4s,v26.4s 1833 add x21,x21,x30,lsr#32 1834 add v19.4s,v19.4s,v31.4s // +4 1835 add x5,x5,x6,lsl#32 // pack 1836 add v23.4s,v23.4s,v31.4s // +4 1837 add x7,x7,x8,lsl#32 1838 add v3.4s,v3.4s,v27.4s 1839 ldp x6,x8,[x1,#0] // load input 1840 add v7.4s,v7.4s,v28.4s 1841 add x9,x9,x10,lsl#32 1842 add v11.4s,v11.4s,v29.4s 1843 add x11,x11,x12,lsl#32 1844 add v15.4s,v15.4s,v30.4s 1845 ldp x10,x12,[x1,#16] 1846 add v19.4s,v19.4s,v27.4s 1847 add x13,x13,x14,lsl#32 1848 add v23.4s,v23.4s,v28.4s 1849 add x15,x15,x16,lsl#32 1850 add v1.4s,v1.4s,v25.4s 1851 ldp x14,x16,[x1,#32] 1852 add v5.4s,v5.4s,v25.4s 1853 add x17,x17,x19,lsl#32 1854 add v9.4s,v9.4s,v25.4s 1855 add x20,x20,x21,lsl#32 1856 add v13.4s,v13.4s,v25.4s 1857 ldp x19,x21,[x1,#48] 1858 add v17.4s,v17.4s,v25.4s 1859 add x1,x1,#64 1860 add v21.4s,v21.4s,v25.4s 1861 1862 #ifdef __ARMEB__ 1863 rev x5,x5 1864 rev x7,x7 1865 rev x9,x9 1866 rev x11,x11 1867 rev x13,x13 1868 rev x15,x15 1869 rev x17,x17 1870 rev x20,x20 1871 #endif 1872 ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x1],#64 1873 eor x5,x5,x6 1874 eor x7,x7,x8 1875 eor x9,x9,x10 1876 eor x11,x11,x12 1877 eor x13,x13,x14 1878 eor v0.16b,v0.16b,v24.16b 1879 eor x15,x15,x16 1880 eor v1.16b,v1.16b,v25.16b 1881 eor x17,x17,x19 1882 eor v2.16b,v2.16b,v26.16b 1883 eor x20,x20,x21 1884 eor v3.16b,v3.16b,v27.16b 1885 ld1 {v24.16b,v25.16b,v26.16b,v27.16b},[x1],#64 1886 1887 stp x5,x7,[x0,#0] // store output 1888 add x28,x28,#7 // increment counter 1889 stp x9,x11,[x0,#16] 1890 stp x13,x15,[x0,#32] 1891 stp x17,x20,[x0,#48] 1892 add x0,x0,#64 1893 st1 {v0.16b,v1.16b,v2.16b,v3.16b},[x0],#64 1894 1895 ld1 {v0.16b,v1.16b,v2.16b,v3.16b},[x1],#64 1896 eor v4.16b,v4.16b,v24.16b 1897 eor v5.16b,v5.16b,v25.16b 1898 eor v6.16b,v6.16b,v26.16b 1899 eor v7.16b,v7.16b,v27.16b 1900 st1 {v4.16b,v5.16b,v6.16b,v7.16b},[x0],#64 1901 1902 ld1 {v4.16b,v5.16b,v6.16b,v7.16b},[x1],#64 1903 eor v8.16b,v8.16b,v0.16b 1904 ldp q24,q25,[sp,#0] 1905 eor v9.16b,v9.16b,v1.16b 1906 ldp q26,q27,[sp,#32] 1907 eor v10.16b,v10.16b,v2.16b 1908 eor v11.16b,v11.16b,v3.16b 1909 st1 {v8.16b,v9.16b,v10.16b,v11.16b},[x0],#64 1910 1911 ld1 {v8.16b,v9.16b,v10.16b,v11.16b},[x1],#64 1912 eor v12.16b,v12.16b,v4.16b 1913 eor v13.16b,v13.16b,v5.16b 1914 eor v14.16b,v14.16b,v6.16b 1915 eor v15.16b,v15.16b,v7.16b 1916 st1 {v12.16b,v13.16b,v14.16b,v15.16b},[x0],#64 1917 1918 ld1 {v12.16b,v13.16b,v14.16b,v15.16b},[x1],#64 1919 eor v16.16b,v16.16b,v8.16b 1920 eor v17.16b,v17.16b,v9.16b 1921 eor v18.16b,v18.16b,v10.16b 1922 eor v19.16b,v19.16b,v11.16b 1923 st1 {v16.16b,v17.16b,v18.16b,v19.16b},[x0],#64 1924 1925 shl v0.4s,v31.4s,#1 // 4 -> 8 1926 eor v20.16b,v20.16b,v12.16b 1927 eor v21.16b,v21.16b,v13.16b 1928 eor v22.16b,v22.16b,v14.16b 1929 eor v23.16b,v23.16b,v15.16b 1930 st1 {v20.16b,v21.16b,v22.16b,v23.16b},[x0],#64 1931 1932 add v27.4s,v27.4s,v0.4s // += 8 1933 add v28.4s,v28.4s,v0.4s 1934 add v29.4s,v29.4s,v0.4s 1935 add v30.4s,v30.4s,v0.4s 1936 1937 b.hs .Loop_outer_512_neon 1938 1939 adds x2,x2,#512 1940 ushr v0.4s,v31.4s,#2 // 4 -> 1 1941 1942 ldp d8,d9,[sp,#128+0] // meet ABI requirements 1943 ldp d10,d11,[sp,#128+16] 1944 ldp d12,d13,[sp,#128+32] 1945 ldp d14,d15,[sp,#128+48] 1946 1947 stp q24,q31,[sp,#0] // wipe off-load area 1948 stp q24,q31,[sp,#32] 1949 stp q24,q31,[sp,#64] 1950 1951 b.eq .Ldone_512_neon 1952 1953 cmp x2,#192 1954 sub v27.4s,v27.4s,v0.4s // -= 1 1955 sub v28.4s,v28.4s,v0.4s 1956 sub v29.4s,v29.4s,v0.4s 1957 add sp,sp,#128 1958 b.hs .Loop_outer_neon 1959 1960 eor v25.16b,v25.16b,v25.16b 1961 eor v26.16b,v26.16b,v26.16b 1962 eor v27.16b,v27.16b,v27.16b 1963 eor v28.16b,v28.16b,v28.16b 1964 eor v29.16b,v29.16b,v29.16b 1965 eor v30.16b,v30.16b,v30.16b 1966 b .Loop_outer 1967 1968 .Ldone_512_neon: 1969 ldp x19,x20,[x29,#16] 1970 add sp,sp,#128+64 1971 ldp x21,x22,[x29,#32] 1972 ldp x23,x24,[x29,#48] 1973 ldp x25,x26,[x29,#64] 1974 ldp x27,x28,[x29,#80] 1975 ldp x29,x30,[sp],#96 1976 .inst 0xd50323bf // autiasp 1977 ret 1978 .size ChaCha20_512_neon,.-ChaCha20_512_neon 1979
Indexes created Mon Mar 02 05:31:46 UTC 2026