1 #include <machine/asm.h> 2 .text 3 4 .globl ossl_rsaz_amm52x40_x1_avxifma256 5 .type ossl_rsaz_amm52x40_x1_avxifma256,@function 6 .align 32 7 ossl_rsaz_amm52x40_x1_avxifma256: 8 .cfi_startproc 9 .byte 243,15,30,250 10 pushq %rbx 11 .cfi_adjust_cfa_offset 8 12 .cfi_offset %rbx,-16 13 pushq %rbp 14 .cfi_adjust_cfa_offset 8 15 .cfi_offset %rbp,-24 16 pushq %r12 17 .cfi_adjust_cfa_offset 8 18 .cfi_offset %r12,-32 19 pushq %r13 20 .cfi_adjust_cfa_offset 8 21 .cfi_offset %r13,-40 22 pushq %r14 23 .cfi_adjust_cfa_offset 8 24 .cfi_offset %r14,-48 25 pushq %r15 26 .cfi_adjust_cfa_offset 8 27 .cfi_offset %r15,-56 28 29 vpxor %ymm0,%ymm0,%ymm0 30 vmovapd %ymm0,%ymm3 31 vmovapd %ymm0,%ymm4 32 vmovapd %ymm0,%ymm5 33 vmovapd %ymm0,%ymm6 34 vmovapd %ymm0,%ymm7 35 vmovapd %ymm0,%ymm8 36 vmovapd %ymm0,%ymm9 37 vmovapd %ymm0,%ymm10 38 vmovapd %ymm0,%ymm11 39 vmovapd %ymm0,%ymm12 40 41 xorl %r9d,%r9d 42 43 movq %rdx,%r11 44 movq $0xfffffffffffff,%rax 45 46 47 movl $10,%ebx 48 49 .align 32 50 .Lloop10: 51 movq 0(%r11),%r13 52 53 vpbroadcastq 0(%r11),%ymm1 54 movq 0(%rsi),%rdx 55 mulxq %r13,%r13,%r12 56 addq %r13,%r9 57 movq %r12,%r10 58 adcq $0,%r10 59 60 movq %r8,%r13 61 imulq %r9,%r13 62 andq %rax,%r13 63 64 vmovq %r13,%xmm2 65 vpbroadcastq %xmm2,%ymm2 66 movq 0(%rcx),%rdx 67 mulxq %r13,%r13,%r12 68 addq %r13,%r9 69 adcq %r12,%r10 70 71 shrq $52,%r9 72 salq $12,%r10 73 orq %r10,%r9 74 75 leaq -328(%rsp),%rsp 76 77 {vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3 78 {vex} vpmadd52luq 32(%rsi),%ymm1,%ymm4 79 {vex} vpmadd52luq 64(%rsi),%ymm1,%ymm5 80 {vex} vpmadd52luq 96(%rsi),%ymm1,%ymm6 81 {vex} vpmadd52luq 128(%rsi),%ymm1,%ymm7 82 {vex} vpmadd52luq 160(%rsi),%ymm1,%ymm8 83 {vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9 84 {vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10 85 {vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11 86 {vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12 87 88 {vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3 89 {vex} vpmadd52luq 32(%rcx),%ymm2,%ymm4 90 {vex} vpmadd52luq 64(%rcx),%ymm2,%ymm5 91 {vex} vpmadd52luq 96(%rcx),%ymm2,%ymm6 92 {vex} vpmadd52luq 128(%rcx),%ymm2,%ymm7 93 {vex} vpmadd52luq 160(%rcx),%ymm2,%ymm8 94 {vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9 95 {vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10 96 {vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11 97 {vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12 98 vmovdqu %ymm3,0(%rsp) 99 vmovdqu %ymm4,32(%rsp) 100 vmovdqu %ymm5,64(%rsp) 101 vmovdqu %ymm6,96(%rsp) 102 vmovdqu %ymm7,128(%rsp) 103 vmovdqu %ymm8,160(%rsp) 104 vmovdqu %ymm9,192(%rsp) 105 vmovdqu %ymm10,224(%rsp) 106 vmovdqu %ymm11,256(%rsp) 107 vmovdqu %ymm12,288(%rsp) 108 movq $0,320(%rsp) 109 110 vmovdqu 8(%rsp),%ymm3 111 vmovdqu 40(%rsp),%ymm4 112 vmovdqu 72(%rsp),%ymm5 113 vmovdqu 104(%rsp),%ymm6 114 vmovdqu 136(%rsp),%ymm7 115 vmovdqu 168(%rsp),%ymm8 116 vmovdqu 200(%rsp),%ymm9 117 vmovdqu 232(%rsp),%ymm10 118 vmovdqu 264(%rsp),%ymm11 119 vmovdqu 296(%rsp),%ymm12 120 121 addq 8(%rsp),%r9 122 123 {vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3 124 {vex} vpmadd52huq 32(%rsi),%ymm1,%ymm4 125 {vex} vpmadd52huq 64(%rsi),%ymm1,%ymm5 126 {vex} vpmadd52huq 96(%rsi),%ymm1,%ymm6 127 {vex} vpmadd52huq 128(%rsi),%ymm1,%ymm7 128 {vex} vpmadd52huq 160(%rsi),%ymm1,%ymm8 129 {vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9 130 {vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10 131 {vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11 132 {vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12 133 134 {vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3 135 {vex} vpmadd52huq 32(%rcx),%ymm2,%ymm4 136 {vex} vpmadd52huq 64(%rcx),%ymm2,%ymm5 137 {vex} vpmadd52huq 96(%rcx),%ymm2,%ymm6 138 {vex} vpmadd52huq 128(%rcx),%ymm2,%ymm7 139 {vex} vpmadd52huq 160(%rcx),%ymm2,%ymm8 140 {vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9 141 {vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10 142 {vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11 143 {vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12 144 leaq 328(%rsp),%rsp 145 movq 8(%r11),%r13 146 147 vpbroadcastq 8(%r11),%ymm1 148 movq 0(%rsi),%rdx 149 mulxq %r13,%r13,%r12 150 addq %r13,%r9 151 movq %r12,%r10 152 adcq $0,%r10 153 154 movq %r8,%r13 155 imulq %r9,%r13 156 andq %rax,%r13 157 158 vmovq %r13,%xmm2 159 vpbroadcastq %xmm2,%ymm2 160 movq 0(%rcx),%rdx 161 mulxq %r13,%r13,%r12 162 addq %r13,%r9 163 adcq %r12,%r10 164 165 shrq $52,%r9 166 salq $12,%r10 167 orq %r10,%r9 168 169 leaq -328(%rsp),%rsp 170 171 {vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3 172 {vex} vpmadd52luq 32(%rsi),%ymm1,%ymm4 173 {vex} vpmadd52luq 64(%rsi),%ymm1,%ymm5 174 {vex} vpmadd52luq 96(%rsi),%ymm1,%ymm6 175 {vex} vpmadd52luq 128(%rsi),%ymm1,%ymm7 176 {vex} vpmadd52luq 160(%rsi),%ymm1,%ymm8 177 {vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9 178 {vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10 179 {vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11 180 {vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12 181 182 {vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3 183 {vex} vpmadd52luq 32(%rcx),%ymm2,%ymm4 184 {vex} vpmadd52luq 64(%rcx),%ymm2,%ymm5 185 {vex} vpmadd52luq 96(%rcx),%ymm2,%ymm6 186 {vex} vpmadd52luq 128(%rcx),%ymm2,%ymm7 187 {vex} vpmadd52luq 160(%rcx),%ymm2,%ymm8 188 {vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9 189 {vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10 190 {vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11 191 {vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12 192 vmovdqu %ymm3,0(%rsp) 193 vmovdqu %ymm4,32(%rsp) 194 vmovdqu %ymm5,64(%rsp) 195 vmovdqu %ymm6,96(%rsp) 196 vmovdqu %ymm7,128(%rsp) 197 vmovdqu %ymm8,160(%rsp) 198 vmovdqu %ymm9,192(%rsp) 199 vmovdqu %ymm10,224(%rsp) 200 vmovdqu %ymm11,256(%rsp) 201 vmovdqu %ymm12,288(%rsp) 202 movq $0,320(%rsp) 203 204 vmovdqu 8(%rsp),%ymm3 205 vmovdqu 40(%rsp),%ymm4 206 vmovdqu 72(%rsp),%ymm5 207 vmovdqu 104(%rsp),%ymm6 208 vmovdqu 136(%rsp),%ymm7 209 vmovdqu 168(%rsp),%ymm8 210 vmovdqu 200(%rsp),%ymm9 211 vmovdqu 232(%rsp),%ymm10 212 vmovdqu 264(%rsp),%ymm11 213 vmovdqu 296(%rsp),%ymm12 214 215 addq 8(%rsp),%r9 216 217 {vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3 218 {vex} vpmadd52huq 32(%rsi),%ymm1,%ymm4 219 {vex} vpmadd52huq 64(%rsi),%ymm1,%ymm5 220 {vex} vpmadd52huq 96(%rsi),%ymm1,%ymm6 221 {vex} vpmadd52huq 128(%rsi),%ymm1,%ymm7 222 {vex} vpmadd52huq 160(%rsi),%ymm1,%ymm8 223 {vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9 224 {vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10 225 {vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11 226 {vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12 227 228 {vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3 229 {vex} vpmadd52huq 32(%rcx),%ymm2,%ymm4 230 {vex} vpmadd52huq 64(%rcx),%ymm2,%ymm5 231 {vex} vpmadd52huq 96(%rcx),%ymm2,%ymm6 232 {vex} vpmadd52huq 128(%rcx),%ymm2,%ymm7 233 {vex} vpmadd52huq 160(%rcx),%ymm2,%ymm8 234 {vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9 235 {vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10 236 {vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11 237 {vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12 238 leaq 328(%rsp),%rsp 239 movq 16(%r11),%r13 240 241 vpbroadcastq 16(%r11),%ymm1 242 movq 0(%rsi),%rdx 243 mulxq %r13,%r13,%r12 244 addq %r13,%r9 245 movq %r12,%r10 246 adcq $0,%r10 247 248 movq %r8,%r13 249 imulq %r9,%r13 250 andq %rax,%r13 251 252 vmovq %r13,%xmm2 253 vpbroadcastq %xmm2,%ymm2 254 movq 0(%rcx),%rdx 255 mulxq %r13,%r13,%r12 256 addq %r13,%r9 257 adcq %r12,%r10 258 259 shrq $52,%r9 260 salq $12,%r10 261 orq %r10,%r9 262 263 leaq -328(%rsp),%rsp 264 265 {vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3 266 {vex} vpmadd52luq 32(%rsi),%ymm1,%ymm4 267 {vex} vpmadd52luq 64(%rsi),%ymm1,%ymm5 268 {vex} vpmadd52luq 96(%rsi),%ymm1,%ymm6 269 {vex} vpmadd52luq 128(%rsi),%ymm1,%ymm7 270 {vex} vpmadd52luq 160(%rsi),%ymm1,%ymm8 271 {vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9 272 {vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10 273 {vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11 274 {vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12 275 276 {vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3 277 {vex} vpmadd52luq 32(%rcx),%ymm2,%ymm4 278 {vex} vpmadd52luq 64(%rcx),%ymm2,%ymm5 279 {vex} vpmadd52luq 96(%rcx),%ymm2,%ymm6 280 {vex} vpmadd52luq 128(%rcx),%ymm2,%ymm7 281 {vex} vpmadd52luq 160(%rcx),%ymm2,%ymm8 282 {vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9 283 {vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10 284 {vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11 285 {vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12 286 vmovdqu %ymm3,0(%rsp) 287 vmovdqu %ymm4,32(%rsp) 288 vmovdqu %ymm5,64(%rsp) 289 vmovdqu %ymm6,96(%rsp) 290 vmovdqu %ymm7,128(%rsp) 291 vmovdqu %ymm8,160(%rsp) 292 vmovdqu %ymm9,192(%rsp) 293 vmovdqu %ymm10,224(%rsp) 294 vmovdqu %ymm11,256(%rsp) 295 vmovdqu %ymm12,288(%rsp) 296 movq $0,320(%rsp) 297 298 vmovdqu 8(%rsp),%ymm3 299 vmovdqu 40(%rsp),%ymm4 300 vmovdqu 72(%rsp),%ymm5 301 vmovdqu 104(%rsp),%ymm6 302 vmovdqu 136(%rsp),%ymm7 303 vmovdqu 168(%rsp),%ymm8 304 vmovdqu 200(%rsp),%ymm9 305 vmovdqu 232(%rsp),%ymm10 306 vmovdqu 264(%rsp),%ymm11 307 vmovdqu 296(%rsp),%ymm12 308 309 addq 8(%rsp),%r9 310 311 {vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3 312 {vex} vpmadd52huq 32(%rsi),%ymm1,%ymm4 313 {vex} vpmadd52huq 64(%rsi),%ymm1,%ymm5 314 {vex} vpmadd52huq 96(%rsi),%ymm1,%ymm6 315 {vex} vpmadd52huq 128(%rsi),%ymm1,%ymm7 316 {vex} vpmadd52huq 160(%rsi),%ymm1,%ymm8 317 {vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9 318 {vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10 319 {vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11 320 {vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12 321 322 {vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3 323 {vex} vpmadd52huq 32(%rcx),%ymm2,%ymm4 324 {vex} vpmadd52huq 64(%rcx),%ymm2,%ymm5 325 {vex} vpmadd52huq 96(%rcx),%ymm2,%ymm6 326 {vex} vpmadd52huq 128(%rcx),%ymm2,%ymm7 327 {vex} vpmadd52huq 160(%rcx),%ymm2,%ymm8 328 {vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9 329 {vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10 330 {vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11 331 {vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12 332 leaq 328(%rsp),%rsp 333 movq 24(%r11),%r13 334 335 vpbroadcastq 24(%r11),%ymm1 336 movq 0(%rsi),%rdx 337 mulxq %r13,%r13,%r12 338 addq %r13,%r9 339 movq %r12,%r10 340 adcq $0,%r10 341 342 movq %r8,%r13 343 imulq %r9,%r13 344 andq %rax,%r13 345 346 vmovq %r13,%xmm2 347 vpbroadcastq %xmm2,%ymm2 348 movq 0(%rcx),%rdx 349 mulxq %r13,%r13,%r12 350 addq %r13,%r9 351 adcq %r12,%r10 352 353 shrq $52,%r9 354 salq $12,%r10 355 orq %r10,%r9 356 357 leaq -328(%rsp),%rsp 358 359 {vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3 360 {vex} vpmadd52luq 32(%rsi),%ymm1,%ymm4 361 {vex} vpmadd52luq 64(%rsi),%ymm1,%ymm5 362 {vex} vpmadd52luq 96(%rsi),%ymm1,%ymm6 363 {vex} vpmadd52luq 128(%rsi),%ymm1,%ymm7 364 {vex} vpmadd52luq 160(%rsi),%ymm1,%ymm8 365 {vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9 366 {vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10 367 {vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11 368 {vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12 369 370 {vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3 371 {vex} vpmadd52luq 32(%rcx),%ymm2,%ymm4 372 {vex} vpmadd52luq 64(%rcx),%ymm2,%ymm5 373 {vex} vpmadd52luq 96(%rcx),%ymm2,%ymm6 374 {vex} vpmadd52luq 128(%rcx),%ymm2,%ymm7 375 {vex} vpmadd52luq 160(%rcx),%ymm2,%ymm8 376 {vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9 377 {vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10 378 {vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11 379 {vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12 380 vmovdqu %ymm3,0(%rsp) 381 vmovdqu %ymm4,32(%rsp) 382 vmovdqu %ymm5,64(%rsp) 383 vmovdqu %ymm6,96(%rsp) 384 vmovdqu %ymm7,128(%rsp) 385 vmovdqu %ymm8,160(%rsp) 386 vmovdqu %ymm9,192(%rsp) 387 vmovdqu %ymm10,224(%rsp) 388 vmovdqu %ymm11,256(%rsp) 389 vmovdqu %ymm12,288(%rsp) 390 movq $0,320(%rsp) 391 392 vmovdqu 8(%rsp),%ymm3 393 vmovdqu 40(%rsp),%ymm4 394 vmovdqu 72(%rsp),%ymm5 395 vmovdqu 104(%rsp),%ymm6 396 vmovdqu 136(%rsp),%ymm7 397 vmovdqu 168(%rsp),%ymm8 398 vmovdqu 200(%rsp),%ymm9 399 vmovdqu 232(%rsp),%ymm10 400 vmovdqu 264(%rsp),%ymm11 401 vmovdqu 296(%rsp),%ymm12 402 403 addq 8(%rsp),%r9 404 405 {vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3 406 {vex} vpmadd52huq 32(%rsi),%ymm1,%ymm4 407 {vex} vpmadd52huq 64(%rsi),%ymm1,%ymm5 408 {vex} vpmadd52huq 96(%rsi),%ymm1,%ymm6 409 {vex} vpmadd52huq 128(%rsi),%ymm1,%ymm7 410 {vex} vpmadd52huq 160(%rsi),%ymm1,%ymm8 411 {vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9 412 {vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10 413 {vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11 414 {vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12 415 416 {vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3 417 {vex} vpmadd52huq 32(%rcx),%ymm2,%ymm4 418 {vex} vpmadd52huq 64(%rcx),%ymm2,%ymm5 419 {vex} vpmadd52huq 96(%rcx),%ymm2,%ymm6 420 {vex} vpmadd52huq 128(%rcx),%ymm2,%ymm7 421 {vex} vpmadd52huq 160(%rcx),%ymm2,%ymm8 422 {vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9 423 {vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10 424 {vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11 425 {vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12 426 leaq 328(%rsp),%rsp 427 leaq 32(%r11),%r11 428 decl %ebx 429 jne .Lloop10 430 431 vmovq %r9,%xmm0 432 vpbroadcastq %xmm0,%ymm0 433 vpblendd $3,%ymm0,%ymm3,%ymm3 434 435 leaq -640(%rsp),%rsp 436 vmovupd %ymm3,0(%rsp) 437 vmovupd %ymm4,32(%rsp) 438 vmovupd %ymm5,64(%rsp) 439 vmovupd %ymm6,96(%rsp) 440 vmovupd %ymm7,128(%rsp) 441 vmovupd %ymm8,160(%rsp) 442 vmovupd %ymm9,192(%rsp) 443 vmovupd %ymm10,224(%rsp) 444 vmovupd %ymm11,256(%rsp) 445 vmovupd %ymm12,288(%rsp) 446 447 448 449 vpsrlq $52,%ymm3,%ymm3 450 vpsrlq $52,%ymm4,%ymm4 451 vpsrlq $52,%ymm5,%ymm5 452 vpsrlq $52,%ymm6,%ymm6 453 vpsrlq $52,%ymm7,%ymm7 454 vpsrlq $52,%ymm8,%ymm8 455 vpsrlq $52,%ymm9,%ymm9 456 vpsrlq $52,%ymm10,%ymm10 457 vpsrlq $52,%ymm11,%ymm11 458 vpsrlq $52,%ymm12,%ymm12 459 460 461 vpermq $144,%ymm12,%ymm12 462 vpermq $3,%ymm11,%ymm13 463 vblendpd $1,%ymm13,%ymm12,%ymm12 464 465 vpermq $144,%ymm11,%ymm11 466 vpermq $3,%ymm10,%ymm13 467 vblendpd $1,%ymm13,%ymm11,%ymm11 468 469 vpermq $144,%ymm10,%ymm10 470 vpermq $3,%ymm9,%ymm13 471 vblendpd $1,%ymm13,%ymm10,%ymm10 472 473 vpermq $144,%ymm9,%ymm9 474 vpermq $3,%ymm8,%ymm13 475 vblendpd $1,%ymm13,%ymm9,%ymm9 476 477 vpermq $144,%ymm8,%ymm8 478 vpermq $3,%ymm7,%ymm13 479 vblendpd $1,%ymm13,%ymm8,%ymm8 480 481 vpermq $144,%ymm7,%ymm7 482 vpermq $3,%ymm6,%ymm13 483 vblendpd $1,%ymm13,%ymm7,%ymm7 484 485 vpermq $144,%ymm6,%ymm6 486 vpermq $3,%ymm5,%ymm13 487 vblendpd $1,%ymm13,%ymm6,%ymm6 488 489 vpermq $144,%ymm5,%ymm5 490 vpermq $3,%ymm4,%ymm13 491 vblendpd $1,%ymm13,%ymm5,%ymm5 492 493 vpermq $144,%ymm4,%ymm4 494 vpermq $3,%ymm3,%ymm13 495 vblendpd $1,%ymm13,%ymm4,%ymm4 496 497 vpermq $144,%ymm3,%ymm3 498 vpand .Lhigh64x3(%rip),%ymm3,%ymm3 499 500 vmovupd %ymm3,320(%rsp) 501 vmovupd %ymm4,352(%rsp) 502 vmovupd %ymm5,384(%rsp) 503 vmovupd %ymm6,416(%rsp) 504 vmovupd %ymm7,448(%rsp) 505 vmovupd %ymm8,480(%rsp) 506 vmovupd %ymm9,512(%rsp) 507 vmovupd %ymm10,544(%rsp) 508 vmovupd %ymm11,576(%rsp) 509 vmovupd %ymm12,608(%rsp) 510 511 vmovupd 0(%rsp),%ymm3 512 vmovupd 32(%rsp),%ymm4 513 vmovupd 64(%rsp),%ymm5 514 vmovupd 96(%rsp),%ymm6 515 vmovupd 128(%rsp),%ymm7 516 vmovupd 160(%rsp),%ymm8 517 vmovupd 192(%rsp),%ymm9 518 vmovupd 224(%rsp),%ymm10 519 vmovupd 256(%rsp),%ymm11 520 vmovupd 288(%rsp),%ymm12 521 522 523 vpand .Lmask52x4(%rip),%ymm3,%ymm3 524 vpand .Lmask52x4(%rip),%ymm4,%ymm4 525 vpand .Lmask52x4(%rip),%ymm5,%ymm5 526 vpand .Lmask52x4(%rip),%ymm6,%ymm6 527 vpand .Lmask52x4(%rip),%ymm7,%ymm7 528 vpand .Lmask52x4(%rip),%ymm8,%ymm8 529 vpand .Lmask52x4(%rip),%ymm9,%ymm9 530 vpand .Lmask52x4(%rip),%ymm10,%ymm10 531 vpand .Lmask52x4(%rip),%ymm11,%ymm11 532 vpand .Lmask52x4(%rip),%ymm12,%ymm12 533 534 535 vpaddq 320(%rsp),%ymm3,%ymm3 536 vpaddq 352(%rsp),%ymm4,%ymm4 537 vpaddq 384(%rsp),%ymm5,%ymm5 538 vpaddq 416(%rsp),%ymm6,%ymm6 539 vpaddq 448(%rsp),%ymm7,%ymm7 540 vpaddq 480(%rsp),%ymm8,%ymm8 541 vpaddq 512(%rsp),%ymm9,%ymm9 542 vpaddq 544(%rsp),%ymm10,%ymm10 543 vpaddq 576(%rsp),%ymm11,%ymm11 544 vpaddq 608(%rsp),%ymm12,%ymm12 545 546 leaq 640(%rsp),%rsp 547 548 549 550 vpcmpgtq .Lmask52x4(%rip),%ymm3,%ymm13 551 vmovmskpd %ymm13,%r14d 552 vpcmpgtq .Lmask52x4(%rip),%ymm4,%ymm13 553 vmovmskpd %ymm13,%r13d 554 shlb $4,%r13b 555 orb %r13b,%r14b 556 557 vpcmpgtq .Lmask52x4(%rip),%ymm5,%ymm13 558 vmovmskpd %ymm13,%r13d 559 vpcmpgtq .Lmask52x4(%rip),%ymm6,%ymm13 560 vmovmskpd %ymm13,%r12d 561 shlb $4,%r12b 562 orb %r12b,%r13b 563 564 vpcmpgtq .Lmask52x4(%rip),%ymm7,%ymm13 565 vmovmskpd %ymm13,%r12d 566 vpcmpgtq .Lmask52x4(%rip),%ymm8,%ymm13 567 vmovmskpd %ymm13,%r11d 568 shlb $4,%r11b 569 orb %r11b,%r12b 570 571 vpcmpgtq .Lmask52x4(%rip),%ymm9,%ymm13 572 vmovmskpd %ymm13,%r11d 573 vpcmpgtq .Lmask52x4(%rip),%ymm10,%ymm13 574 vmovmskpd %ymm13,%r10d 575 shlb $4,%r10b 576 orb %r10b,%r11b 577 578 vpcmpgtq .Lmask52x4(%rip),%ymm11,%ymm13 579 vmovmskpd %ymm13,%r10d 580 vpcmpgtq .Lmask52x4(%rip),%ymm12,%ymm13 581 vmovmskpd %ymm13,%r9d 582 shlb $4,%r9b 583 orb %r9b,%r10b 584 585 addb %r14b,%r14b 586 adcb %r13b,%r13b 587 adcb %r12b,%r12b 588 adcb %r11b,%r11b 589 adcb %r10b,%r10b 590 591 592 vpcmpeqq .Lmask52x4(%rip),%ymm3,%ymm13 593 vmovmskpd %ymm13,%r9d 594 vpcmpeqq .Lmask52x4(%rip),%ymm4,%ymm13 595 vmovmskpd %ymm13,%r8d 596 shlb $4,%r8b 597 orb %r8b,%r9b 598 599 vpcmpeqq .Lmask52x4(%rip),%ymm5,%ymm13 600 vmovmskpd %ymm13,%r8d 601 vpcmpeqq .Lmask52x4(%rip),%ymm6,%ymm13 602 vmovmskpd %ymm13,%edx 603 shlb $4,%dl 604 orb %dl,%r8b 605 606 vpcmpeqq .Lmask52x4(%rip),%ymm7,%ymm13 607 vmovmskpd %ymm13,%edx 608 vpcmpeqq .Lmask52x4(%rip),%ymm8,%ymm13 609 vmovmskpd %ymm13,%ecx 610 shlb $4,%cl 611 orb %cl,%dl 612 613 vpcmpeqq .Lmask52x4(%rip),%ymm9,%ymm13 614 vmovmskpd %ymm13,%ecx 615 vpcmpeqq .Lmask52x4(%rip),%ymm10,%ymm13 616 vmovmskpd %ymm13,%ebx 617 shlb $4,%bl 618 orb %bl,%cl 619 620 vpcmpeqq .Lmask52x4(%rip),%ymm11,%ymm13 621 vmovmskpd %ymm13,%ebx 622 vpcmpeqq .Lmask52x4(%rip),%ymm12,%ymm13 623 vmovmskpd %ymm13,%eax 624 shlb $4,%al 625 orb %al,%bl 626 627 addb %r9b,%r14b 628 adcb %r8b,%r13b 629 adcb %dl,%r12b 630 adcb %cl,%r11b 631 adcb %bl,%r10b 632 633 xorb %r9b,%r14b 634 xorb %r8b,%r13b 635 xorb %dl,%r12b 636 xorb %cl,%r11b 637 xorb %bl,%r10b 638 639 pushq %r9 640 pushq %r8 641 642 leaq .Lkmasklut(%rip),%r8 643 644 movb %r14b,%r9b 645 andq $0xf,%r14 646 vpsubq .Lmask52x4(%rip),%ymm3,%ymm13 647 shlq $5,%r14 648 vmovapd (%r8,%r14,1),%ymm14 649 vblendvpd %ymm14,%ymm13,%ymm3,%ymm3 650 651 shrb $4,%r9b 652 andq $0xf,%r9 653 vpsubq .Lmask52x4(%rip),%ymm4,%ymm13 654 shlq $5,%r9 655 vmovapd (%r8,%r9,1),%ymm14 656 vblendvpd %ymm14,%ymm13,%ymm4,%ymm4 657 658 movb %r13b,%r9b 659 andq $0xf,%r13 660 vpsubq .Lmask52x4(%rip),%ymm5,%ymm13 661 shlq $5,%r13 662 vmovapd (%r8,%r13,1),%ymm14 663 vblendvpd %ymm14,%ymm13,%ymm5,%ymm5 664 665 shrb $4,%r9b 666 andq $0xf,%r9 667 vpsubq .Lmask52x4(%rip),%ymm6,%ymm13 668 shlq $5,%r9 669 vmovapd (%r8,%r9,1),%ymm14 670 vblendvpd %ymm14,%ymm13,%ymm6,%ymm6 671 672 movb %r12b,%r9b 673 andq $0xf,%r12 674 vpsubq .Lmask52x4(%rip),%ymm7,%ymm13 675 shlq $5,%r12 676 vmovapd (%r8,%r12,1),%ymm14 677 vblendvpd %ymm14,%ymm13,%ymm7,%ymm7 678 679 shrb $4,%r9b 680 andq $0xf,%r9 681 vpsubq .Lmask52x4(%rip),%ymm8,%ymm13 682 shlq $5,%r9 683 vmovapd (%r8,%r9,1),%ymm14 684 vblendvpd %ymm14,%ymm13,%ymm8,%ymm8 685 686 movb %r11b,%r9b 687 andq $0xf,%r11 688 vpsubq .Lmask52x4(%rip),%ymm9,%ymm13 689 shlq $5,%r11 690 vmovapd (%r8,%r11,1),%ymm14 691 vblendvpd %ymm14,%ymm13,%ymm9,%ymm9 692 693 shrb $4,%r9b 694 andq $0xf,%r9 695 vpsubq .Lmask52x4(%rip),%ymm10,%ymm13 696 shlq $5,%r9 697 vmovapd (%r8,%r9,1),%ymm14 698 vblendvpd %ymm14,%ymm13,%ymm10,%ymm10 699 700 movb %r10b,%r9b 701 andq $0xf,%r10 702 vpsubq .Lmask52x4(%rip),%ymm11,%ymm13 703 shlq $5,%r10 704 vmovapd (%r8,%r10,1),%ymm14 705 vblendvpd %ymm14,%ymm13,%ymm11,%ymm11 706 707 shrb $4,%r9b 708 andq $0xf,%r9 709 vpsubq .Lmask52x4(%rip),%ymm12,%ymm13 710 shlq $5,%r9 711 vmovapd (%r8,%r9,1),%ymm14 712 vblendvpd %ymm14,%ymm13,%ymm12,%ymm12 713 714 popq %r8 715 popq %r9 716 717 vpand .Lmask52x4(%rip),%ymm3,%ymm3 718 vpand .Lmask52x4(%rip),%ymm4,%ymm4 719 vpand .Lmask52x4(%rip),%ymm5,%ymm5 720 vpand .Lmask52x4(%rip),%ymm6,%ymm6 721 vpand .Lmask52x4(%rip),%ymm7,%ymm7 722 vpand .Lmask52x4(%rip),%ymm8,%ymm8 723 vpand .Lmask52x4(%rip),%ymm9,%ymm9 724 725 vpand .Lmask52x4(%rip),%ymm10,%ymm10 726 vpand .Lmask52x4(%rip),%ymm11,%ymm11 727 vpand .Lmask52x4(%rip),%ymm12,%ymm12 728 729 vmovdqu %ymm3,0(%rdi) 730 vmovdqu %ymm4,32(%rdi) 731 vmovdqu %ymm5,64(%rdi) 732 vmovdqu %ymm6,96(%rdi) 733 vmovdqu %ymm7,128(%rdi) 734 vmovdqu %ymm8,160(%rdi) 735 vmovdqu %ymm9,192(%rdi) 736 vmovdqu %ymm10,224(%rdi) 737 vmovdqu %ymm11,256(%rdi) 738 vmovdqu %ymm12,288(%rdi) 739 740 vzeroupper 741 leaq (%rsp),%rax 742 .cfi_def_cfa_register %rax 743 movq 0(%rax),%r15 744 .cfi_restore %r15 745 movq 8(%rax),%r14 746 .cfi_restore %r14 747 movq 16(%rax),%r13 748 .cfi_restore %r13 749 movq 24(%rax),%r12 750 .cfi_restore %r12 751 movq 32(%rax),%rbp 752 .cfi_restore %rbp 753 movq 40(%rax),%rbx 754 .cfi_restore %rbx 755 leaq 48(%rax),%rsp 756 .cfi_def_cfa %rsp,8 757 .Lossl_rsaz_amm52x40_x1_avxifma256_epilogue: 758 759 .byte 0xf3,0xc3 760 .cfi_endproc 761 .size ossl_rsaz_amm52x40_x1_avxifma256, .-ossl_rsaz_amm52x40_x1_avxifma256 762 .section .rodata 763 .align 32 764 .Lmask52x4: 765 .quad 0xfffffffffffff 766 .quad 0xfffffffffffff 767 .quad 0xfffffffffffff 768 .quad 0xfffffffffffff 769 .Lhigh64x3: 770 .quad 0x0 771 .quad 0xffffffffffffffff 772 .quad 0xffffffffffffffff 773 .quad 0xffffffffffffffff 774 .Lkmasklut: 775 776 .quad 0x0 777 .quad 0x0 778 .quad 0x0 779 .quad 0x0 780 781 .quad 0xffffffffffffffff 782 .quad 0x0 783 .quad 0x0 784 .quad 0x0 785 786 .quad 0x0 787 .quad 0xffffffffffffffff 788 .quad 0x0 789 .quad 0x0 790 791 .quad 0xffffffffffffffff 792 .quad 0xffffffffffffffff 793 .quad 0x0 794 .quad 0x0 795 796 .quad 0x0 797 .quad 0x0 798 .quad 0xffffffffffffffff 799 .quad 0x0 800 801 .quad 0xffffffffffffffff 802 .quad 0x0 803 .quad 0xffffffffffffffff 804 .quad 0x0 805 806 .quad 0x0 807 .quad 0xffffffffffffffff 808 .quad 0xffffffffffffffff 809 .quad 0x0 810 811 .quad 0xffffffffffffffff 812 .quad 0xffffffffffffffff 813 .quad 0xffffffffffffffff 814 .quad 0x0 815 816 .quad 0x0 817 .quad 0x0 818 .quad 0x0 819 .quad 0xffffffffffffffff 820 821 .quad 0xffffffffffffffff 822 .quad 0x0 823 .quad 0x0 824 .quad 0xffffffffffffffff 825 826 .quad 0x0 827 .quad 0xffffffffffffffff 828 .quad 0x0 829 .quad 0xffffffffffffffff 830 831 .quad 0xffffffffffffffff 832 .quad 0xffffffffffffffff 833 .quad 0x0 834 .quad 0xffffffffffffffff 835 836 .quad 0x0 837 .quad 0x0 838 .quad 0xffffffffffffffff 839 .quad 0xffffffffffffffff 840 841 .quad 0xffffffffffffffff 842 .quad 0x0 843 .quad 0xffffffffffffffff 844 .quad 0xffffffffffffffff 845 846 .quad 0x0 847 .quad 0xffffffffffffffff 848 .quad 0xffffffffffffffff 849 .quad 0xffffffffffffffff 850 851 .quad 0xffffffffffffffff 852 .quad 0xffffffffffffffff 853 .quad 0xffffffffffffffff 854 .quad 0xffffffffffffffff 855 .text 856 857 .globl ossl_rsaz_amm52x40_x2_avxifma256 858 .type ossl_rsaz_amm52x40_x2_avxifma256,@function 859 .align 32 860 ossl_rsaz_amm52x40_x2_avxifma256: 861 .cfi_startproc 862 .byte 243,15,30,250 863 pushq %rbx 864 .cfi_adjust_cfa_offset 8 865 .cfi_offset %rbx,-16 866 pushq %rbp 867 .cfi_adjust_cfa_offset 8 868 .cfi_offset %rbp,-24 869 pushq %r12 870 .cfi_adjust_cfa_offset 8 871 .cfi_offset %r12,-32 872 pushq %r13 873 .cfi_adjust_cfa_offset 8 874 .cfi_offset %r13,-40 875 pushq %r14 876 .cfi_adjust_cfa_offset 8 877 .cfi_offset %r14,-48 878 pushq %r15 879 .cfi_adjust_cfa_offset 8 880 .cfi_offset %r15,-56 881 882 vpxor %ymm0,%ymm0,%ymm0 883 vmovapd %ymm0,%ymm3 884 vmovapd %ymm0,%ymm4 885 vmovapd %ymm0,%ymm5 886 vmovapd %ymm0,%ymm6 887 vmovapd %ymm0,%ymm7 888 vmovapd %ymm0,%ymm8 889 vmovapd %ymm0,%ymm9 890 vmovapd %ymm0,%ymm10 891 vmovapd %ymm0,%ymm11 892 vmovapd %ymm0,%ymm12 893 894 xorl %r9d,%r9d 895 896 movq %rdx,%r11 897 movq $0xfffffffffffff,%rax 898 899 movl $40,%ebx 900 901 .align 32 902 .Lloop40: 903 movq 0(%r11),%r13 904 905 vpbroadcastq 0(%r11),%ymm1 906 movq 0(%rsi),%rdx 907 mulxq %r13,%r13,%r12 908 addq %r13,%r9 909 movq %r12,%r10 910 adcq $0,%r10 911 912 movq (%r8),%r13 913 imulq %r9,%r13 914 andq %rax,%r13 915 916 vmovq %r13,%xmm2 917 vpbroadcastq %xmm2,%ymm2 918 movq 0(%rcx),%rdx 919 mulxq %r13,%r13,%r12 920 addq %r13,%r9 921 adcq %r12,%r10 922 923 shrq $52,%r9 924 salq $12,%r10 925 orq %r10,%r9 926 927 leaq -328(%rsp),%rsp 928 929 {vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3 930 {vex} vpmadd52luq 32(%rsi),%ymm1,%ymm4 931 {vex} vpmadd52luq 64(%rsi),%ymm1,%ymm5 932 {vex} vpmadd52luq 96(%rsi),%ymm1,%ymm6 933 {vex} vpmadd52luq 128(%rsi),%ymm1,%ymm7 934 {vex} vpmadd52luq 160(%rsi),%ymm1,%ymm8 935 {vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9 936 {vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10 937 {vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11 938 {vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12 939 940 {vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3 941 {vex} vpmadd52luq 32(%rcx),%ymm2,%ymm4 942 {vex} vpmadd52luq 64(%rcx),%ymm2,%ymm5 943 {vex} vpmadd52luq 96(%rcx),%ymm2,%ymm6 944 {vex} vpmadd52luq 128(%rcx),%ymm2,%ymm7 945 {vex} vpmadd52luq 160(%rcx),%ymm2,%ymm8 946 {vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9 947 {vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10 948 {vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11 949 {vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12 950 vmovdqu %ymm3,0(%rsp) 951 vmovdqu %ymm4,32(%rsp) 952 vmovdqu %ymm5,64(%rsp) 953 vmovdqu %ymm6,96(%rsp) 954 vmovdqu %ymm7,128(%rsp) 955 vmovdqu %ymm8,160(%rsp) 956 vmovdqu %ymm9,192(%rsp) 957 vmovdqu %ymm10,224(%rsp) 958 vmovdqu %ymm11,256(%rsp) 959 vmovdqu %ymm12,288(%rsp) 960 movq $0,320(%rsp) 961 962 vmovdqu 8(%rsp),%ymm3 963 vmovdqu 40(%rsp),%ymm4 964 vmovdqu 72(%rsp),%ymm5 965 vmovdqu 104(%rsp),%ymm6 966 vmovdqu 136(%rsp),%ymm7 967 vmovdqu 168(%rsp),%ymm8 968 vmovdqu 200(%rsp),%ymm9 969 vmovdqu 232(%rsp),%ymm10 970 vmovdqu 264(%rsp),%ymm11 971 vmovdqu 296(%rsp),%ymm12 972 973 addq 8(%rsp),%r9 974 975 {vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3 976 {vex} vpmadd52huq 32(%rsi),%ymm1,%ymm4 977 {vex} vpmadd52huq 64(%rsi),%ymm1,%ymm5 978 {vex} vpmadd52huq 96(%rsi),%ymm1,%ymm6 979 {vex} vpmadd52huq 128(%rsi),%ymm1,%ymm7 980 {vex} vpmadd52huq 160(%rsi),%ymm1,%ymm8 981 {vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9 982 {vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10 983 {vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11 984 {vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12 985 986 {vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3 987 {vex} vpmadd52huq 32(%rcx),%ymm2,%ymm4 988 {vex} vpmadd52huq 64(%rcx),%ymm2,%ymm5 989 {vex} vpmadd52huq 96(%rcx),%ymm2,%ymm6 990 {vex} vpmadd52huq 128(%rcx),%ymm2,%ymm7 991 {vex} vpmadd52huq 160(%rcx),%ymm2,%ymm8 992 {vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9 993 {vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10 994 {vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11 995 {vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12 996 leaq 328(%rsp),%rsp 997 leaq 8(%r11),%r11 998 decl %ebx 999 jne .Lloop40 1000 1001 pushq %r11 1002 pushq %rsi 1003 pushq %rcx 1004 pushq %r8 1005 1006 vmovq %r9,%xmm0 1007 vpbroadcastq %xmm0,%ymm0 1008 vpblendd $3,%ymm0,%ymm3,%ymm3 1009 1010 leaq -640(%rsp),%rsp 1011 vmovupd %ymm3,0(%rsp) 1012 vmovupd %ymm4,32(%rsp) 1013 vmovupd %ymm5,64(%rsp) 1014 vmovupd %ymm6,96(%rsp) 1015 vmovupd %ymm7,128(%rsp) 1016 vmovupd %ymm8,160(%rsp) 1017 vmovupd %ymm9,192(%rsp) 1018 vmovupd %ymm10,224(%rsp) 1019 vmovupd %ymm11,256(%rsp) 1020 vmovupd %ymm12,288(%rsp) 1021 1022 1023 1024 vpsrlq $52,%ymm3,%ymm3 1025 vpsrlq $52,%ymm4,%ymm4 1026 vpsrlq $52,%ymm5,%ymm5 1027 vpsrlq $52,%ymm6,%ymm6 1028 vpsrlq $52,%ymm7,%ymm7 1029 vpsrlq $52,%ymm8,%ymm8 1030 vpsrlq $52,%ymm9,%ymm9 1031 vpsrlq $52,%ymm10,%ymm10 1032 vpsrlq $52,%ymm11,%ymm11 1033 vpsrlq $52,%ymm12,%ymm12 1034 1035 1036 vpermq $144,%ymm12,%ymm12 1037 vpermq $3,%ymm11,%ymm13 1038 vblendpd $1,%ymm13,%ymm12,%ymm12 1039 1040 vpermq $144,%ymm11,%ymm11 1041 vpermq $3,%ymm10,%ymm13 1042 vblendpd $1,%ymm13,%ymm11,%ymm11 1043 1044 vpermq $144,%ymm10,%ymm10 1045 vpermq $3,%ymm9,%ymm13 1046 vblendpd $1,%ymm13,%ymm10,%ymm10 1047 1048 vpermq $144,%ymm9,%ymm9 1049 vpermq $3,%ymm8,%ymm13 1050 vblendpd $1,%ymm13,%ymm9,%ymm9 1051 1052 vpermq $144,%ymm8,%ymm8 1053 vpermq $3,%ymm7,%ymm13 1054 vblendpd $1,%ymm13,%ymm8,%ymm8 1055 1056 vpermq $144,%ymm7,%ymm7 1057 vpermq $3,%ymm6,%ymm13 1058 vblendpd $1,%ymm13,%ymm7,%ymm7 1059 1060 vpermq $144,%ymm6,%ymm6 1061 vpermq $3,%ymm5,%ymm13 1062 vblendpd $1,%ymm13,%ymm6,%ymm6 1063 1064 vpermq $144,%ymm5,%ymm5 1065 vpermq $3,%ymm4,%ymm13 1066 vblendpd $1,%ymm13,%ymm5,%ymm5 1067 1068 vpermq $144,%ymm4,%ymm4 1069 vpermq $3,%ymm3,%ymm13 1070 vblendpd $1,%ymm13,%ymm4,%ymm4 1071 1072 vpermq $144,%ymm3,%ymm3 1073 vpand .Lhigh64x3(%rip),%ymm3,%ymm3 1074 1075 vmovupd %ymm3,320(%rsp) 1076 vmovupd %ymm4,352(%rsp) 1077 vmovupd %ymm5,384(%rsp) 1078 vmovupd %ymm6,416(%rsp) 1079 vmovupd %ymm7,448(%rsp) 1080 vmovupd %ymm8,480(%rsp) 1081 vmovupd %ymm9,512(%rsp) 1082 vmovupd %ymm10,544(%rsp) 1083 vmovupd %ymm11,576(%rsp) 1084 vmovupd %ymm12,608(%rsp) 1085 1086 vmovupd 0(%rsp),%ymm3 1087 vmovupd 32(%rsp),%ymm4 1088 vmovupd 64(%rsp),%ymm5 1089 vmovupd 96(%rsp),%ymm6 1090 vmovupd 128(%rsp),%ymm7 1091 vmovupd 160(%rsp),%ymm8 1092 vmovupd 192(%rsp),%ymm9 1093 vmovupd 224(%rsp),%ymm10 1094 vmovupd 256(%rsp),%ymm11 1095 vmovupd 288(%rsp),%ymm12 1096 1097 1098 vpand .Lmask52x4(%rip),%ymm3,%ymm3 1099 vpand .Lmask52x4(%rip),%ymm4,%ymm4 1100 vpand .Lmask52x4(%rip),%ymm5,%ymm5 1101 vpand .Lmask52x4(%rip),%ymm6,%ymm6 1102 vpand .Lmask52x4(%rip),%ymm7,%ymm7 1103 vpand .Lmask52x4(%rip),%ymm8,%ymm8 1104 vpand .Lmask52x4(%rip),%ymm9,%ymm9 1105 vpand .Lmask52x4(%rip),%ymm10,%ymm10 1106 vpand .Lmask52x4(%rip),%ymm11,%ymm11 1107 vpand .Lmask52x4(%rip),%ymm12,%ymm12 1108 1109 1110 vpaddq 320(%rsp),%ymm3,%ymm3 1111 vpaddq 352(%rsp),%ymm4,%ymm4 1112 vpaddq 384(%rsp),%ymm5,%ymm5 1113 vpaddq 416(%rsp),%ymm6,%ymm6 1114 vpaddq 448(%rsp),%ymm7,%ymm7 1115 vpaddq 480(%rsp),%ymm8,%ymm8 1116 vpaddq 512(%rsp),%ymm9,%ymm9 1117 vpaddq 544(%rsp),%ymm10,%ymm10 1118 vpaddq 576(%rsp),%ymm11,%ymm11 1119 vpaddq 608(%rsp),%ymm12,%ymm12 1120 1121 leaq 640(%rsp),%rsp 1122 1123 1124 1125 vpcmpgtq .Lmask52x4(%rip),%ymm3,%ymm13 1126 vmovmskpd %ymm13,%r14d 1127 vpcmpgtq .Lmask52x4(%rip),%ymm4,%ymm13 1128 vmovmskpd %ymm13,%r13d 1129 shlb $4,%r13b 1130 orb %r13b,%r14b 1131 1132 vpcmpgtq .Lmask52x4(%rip),%ymm5,%ymm13 1133 vmovmskpd %ymm13,%r13d 1134 vpcmpgtq .Lmask52x4(%rip),%ymm6,%ymm13 1135 vmovmskpd %ymm13,%r12d 1136 shlb $4,%r12b 1137 orb %r12b,%r13b 1138 1139 vpcmpgtq .Lmask52x4(%rip),%ymm7,%ymm13 1140 vmovmskpd %ymm13,%r12d 1141 vpcmpgtq .Lmask52x4(%rip),%ymm8,%ymm13 1142 vmovmskpd %ymm13,%r11d 1143 shlb $4,%r11b 1144 orb %r11b,%r12b 1145 1146 vpcmpgtq .Lmask52x4(%rip),%ymm9,%ymm13 1147 vmovmskpd %ymm13,%r11d 1148 vpcmpgtq .Lmask52x4(%rip),%ymm10,%ymm13 1149 vmovmskpd %ymm13,%r10d 1150 shlb $4,%r10b 1151 orb %r10b,%r11b 1152 1153 vpcmpgtq .Lmask52x4(%rip),%ymm11,%ymm13 1154 vmovmskpd %ymm13,%r10d 1155 vpcmpgtq .Lmask52x4(%rip),%ymm12,%ymm13 1156 vmovmskpd %ymm13,%r9d 1157 shlb $4,%r9b 1158 orb %r9b,%r10b 1159 1160 addb %r14b,%r14b 1161 adcb %r13b,%r13b 1162 adcb %r12b,%r12b 1163 adcb %r11b,%r11b 1164 adcb %r10b,%r10b 1165 1166 1167 vpcmpeqq .Lmask52x4(%rip),%ymm3,%ymm13 1168 vmovmskpd %ymm13,%r9d 1169 vpcmpeqq .Lmask52x4(%rip),%ymm4,%ymm13 1170 vmovmskpd %ymm13,%r8d 1171 shlb $4,%r8b 1172 orb %r8b,%r9b 1173 1174 vpcmpeqq .Lmask52x4(%rip),%ymm5,%ymm13 1175 vmovmskpd %ymm13,%r8d 1176 vpcmpeqq .Lmask52x4(%rip),%ymm6,%ymm13 1177 vmovmskpd %ymm13,%edx 1178 shlb $4,%dl 1179 orb %dl,%r8b 1180 1181 vpcmpeqq .Lmask52x4(%rip),%ymm7,%ymm13 1182 vmovmskpd %ymm13,%edx 1183 vpcmpeqq .Lmask52x4(%rip),%ymm8,%ymm13 1184 vmovmskpd %ymm13,%ecx 1185 shlb $4,%cl 1186 orb %cl,%dl 1187 1188 vpcmpeqq .Lmask52x4(%rip),%ymm9,%ymm13 1189 vmovmskpd %ymm13,%ecx 1190 vpcmpeqq .Lmask52x4(%rip),%ymm10,%ymm13 1191 vmovmskpd %ymm13,%ebx 1192 shlb $4,%bl 1193 orb %bl,%cl 1194 1195 vpcmpeqq .Lmask52x4(%rip),%ymm11,%ymm13 1196 vmovmskpd %ymm13,%ebx 1197 vpcmpeqq .Lmask52x4(%rip),%ymm12,%ymm13 1198 vmovmskpd %ymm13,%eax 1199 shlb $4,%al 1200 orb %al,%bl 1201 1202 addb %r9b,%r14b 1203 adcb %r8b,%r13b 1204 adcb %dl,%r12b 1205 adcb %cl,%r11b 1206 adcb %bl,%r10b 1207 1208 xorb %r9b,%r14b 1209 xorb %r8b,%r13b 1210 xorb %dl,%r12b 1211 xorb %cl,%r11b 1212 xorb %bl,%r10b 1213 1214 pushq %r9 1215 pushq %r8 1216 1217 leaq .Lkmasklut(%rip),%r8 1218 1219 movb %r14b,%r9b 1220 andq $0xf,%r14 1221 vpsubq .Lmask52x4(%rip),%ymm3,%ymm13 1222 shlq $5,%r14 1223 vmovapd (%r8,%r14,1),%ymm14 1224 vblendvpd %ymm14,%ymm13,%ymm3,%ymm3 1225 1226 shrb $4,%r9b 1227 andq $0xf,%r9 1228 vpsubq .Lmask52x4(%rip),%ymm4,%ymm13 1229 shlq $5,%r9 1230 vmovapd (%r8,%r9,1),%ymm14 1231 vblendvpd %ymm14,%ymm13,%ymm4,%ymm4 1232 1233 movb %r13b,%r9b 1234 andq $0xf,%r13 1235 vpsubq .Lmask52x4(%rip),%ymm5,%ymm13 1236 shlq $5,%r13 1237 vmovapd (%r8,%r13,1),%ymm14 1238 vblendvpd %ymm14,%ymm13,%ymm5,%ymm5 1239 1240 shrb $4,%r9b 1241 andq $0xf,%r9 1242 vpsubq .Lmask52x4(%rip),%ymm6,%ymm13 1243 shlq $5,%r9 1244 vmovapd (%r8,%r9,1),%ymm14 1245 vblendvpd %ymm14,%ymm13,%ymm6,%ymm6 1246 1247 movb %r12b,%r9b 1248 andq $0xf,%r12 1249 vpsubq .Lmask52x4(%rip),%ymm7,%ymm13 1250 shlq $5,%r12 1251 vmovapd (%r8,%r12,1),%ymm14 1252 vblendvpd %ymm14,%ymm13,%ymm7,%ymm7 1253 1254 shrb $4,%r9b 1255 andq $0xf,%r9 1256 vpsubq .Lmask52x4(%rip),%ymm8,%ymm13 1257 shlq $5,%r9 1258 vmovapd (%r8,%r9,1),%ymm14 1259 vblendvpd %ymm14,%ymm13,%ymm8,%ymm8 1260 1261 movb %r11b,%r9b 1262 andq $0xf,%r11 1263 vpsubq .Lmask52x4(%rip),%ymm9,%ymm13 1264 shlq $5,%r11 1265 vmovapd (%r8,%r11,1),%ymm14 1266 vblendvpd %ymm14,%ymm13,%ymm9,%ymm9 1267 1268 shrb $4,%r9b 1269 andq $0xf,%r9 1270 vpsubq .Lmask52x4(%rip),%ymm10,%ymm13 1271 shlq $5,%r9 1272 vmovapd (%r8,%r9,1),%ymm14 1273 vblendvpd %ymm14,%ymm13,%ymm10,%ymm10 1274 1275 movb %r10b,%r9b 1276 andq $0xf,%r10 1277 vpsubq .Lmask52x4(%rip),%ymm11,%ymm13 1278 shlq $5,%r10 1279 vmovapd (%r8,%r10,1),%ymm14 1280 vblendvpd %ymm14,%ymm13,%ymm11,%ymm11 1281 1282 shrb $4,%r9b 1283 andq $0xf,%r9 1284 vpsubq .Lmask52x4(%rip),%ymm12,%ymm13 1285 shlq $5,%r9 1286 vmovapd (%r8,%r9,1),%ymm14 1287 vblendvpd %ymm14,%ymm13,%ymm12,%ymm12 1288 1289 popq %r8 1290 popq %r9 1291 1292 vpand .Lmask52x4(%rip),%ymm3,%ymm3 1293 vpand .Lmask52x4(%rip),%ymm4,%ymm4 1294 vpand .Lmask52x4(%rip),%ymm5,%ymm5 1295 vpand .Lmask52x4(%rip),%ymm6,%ymm6 1296 vpand .Lmask52x4(%rip),%ymm7,%ymm7 1297 vpand .Lmask52x4(%rip),%ymm8,%ymm8 1298 vpand .Lmask52x4(%rip),%ymm9,%ymm9 1299 1300 vpand .Lmask52x4(%rip),%ymm10,%ymm10 1301 vpand .Lmask52x4(%rip),%ymm11,%ymm11 1302 vpand .Lmask52x4(%rip),%ymm12,%ymm12 1303 1304 popq %r8 1305 popq %rcx 1306 popq %rsi 1307 popq %r11 1308 1309 vmovdqu %ymm3,0(%rdi) 1310 vmovdqu %ymm4,32(%rdi) 1311 vmovdqu %ymm5,64(%rdi) 1312 vmovdqu %ymm6,96(%rdi) 1313 vmovdqu %ymm7,128(%rdi) 1314 vmovdqu %ymm8,160(%rdi) 1315 vmovdqu %ymm9,192(%rdi) 1316 vmovdqu %ymm10,224(%rdi) 1317 vmovdqu %ymm11,256(%rdi) 1318 vmovdqu %ymm12,288(%rdi) 1319 1320 xorl %r15d,%r15d 1321 1322 movq $0xfffffffffffff,%rax 1323 1324 movl $40,%ebx 1325 1326 vpxor %ymm0,%ymm0,%ymm0 1327 vmovapd %ymm0,%ymm3 1328 vmovapd %ymm0,%ymm4 1329 vmovapd %ymm0,%ymm5 1330 vmovapd %ymm0,%ymm6 1331 vmovapd %ymm0,%ymm7 1332 vmovapd %ymm0,%ymm8 1333 vmovapd %ymm0,%ymm9 1334 vmovapd %ymm0,%ymm10 1335 vmovapd %ymm0,%ymm11 1336 vmovapd %ymm0,%ymm12 1337 .align 32 1338 .Lloop40_1: 1339 movq 0(%r11),%r13 1340 1341 vpbroadcastq 0(%r11),%ymm1 1342 movq 320(%rsi),%rdx 1343 mulxq %r13,%r13,%r12 1344 addq %r13,%r9 1345 movq %r12,%r10 1346 adcq $0,%r10 1347 1348 movq 8(%r8),%r13 1349 imulq %r9,%r13 1350 andq %rax,%r13 1351 1352 vmovq %r13,%xmm2 1353 vpbroadcastq %xmm2,%ymm2 1354 movq 320(%rcx),%rdx 1355 mulxq %r13,%r13,%r12 1356 addq %r13,%r9 1357 adcq %r12,%r10 1358 1359 shrq $52,%r9 1360 salq $12,%r10 1361 orq %r10,%r9 1362 1363 leaq -328(%rsp),%rsp 1364 1365 {vex} vpmadd52luq 320(%rsi),%ymm1,%ymm3 1366 {vex} vpmadd52luq 352(%rsi),%ymm1,%ymm4 1367 {vex} vpmadd52luq 384(%rsi),%ymm1,%ymm5 1368 {vex} vpmadd52luq 416(%rsi),%ymm1,%ymm6 1369 {vex} vpmadd52luq 448(%rsi),%ymm1,%ymm7 1370 {vex} vpmadd52luq 480(%rsi),%ymm1,%ymm8 1371 {vex} vpmadd52luq 512(%rsi),%ymm1,%ymm9 1372 {vex} vpmadd52luq 544(%rsi),%ymm1,%ymm10 1373 {vex} vpmadd52luq 576(%rsi),%ymm1,%ymm11 1374 {vex} vpmadd52luq 608(%rsi),%ymm1,%ymm12 1375 1376 {vex} vpmadd52luq 320(%rcx),%ymm2,%ymm3 1377 {vex} vpmadd52luq 352(%rcx),%ymm2,%ymm4 1378 {vex} vpmadd52luq 384(%rcx),%ymm2,%ymm5 1379 {vex} vpmadd52luq 416(%rcx),%ymm2,%ymm6 1380 {vex} vpmadd52luq 448(%rcx),%ymm2,%ymm7 1381 {vex} vpmadd52luq 480(%rcx),%ymm2,%ymm8 1382 {vex} vpmadd52luq 512(%rcx),%ymm2,%ymm9 1383 {vex} vpmadd52luq 544(%rcx),%ymm2,%ymm10 1384 {vex} vpmadd52luq 576(%rcx),%ymm2,%ymm11 1385 {vex} vpmadd52luq 608(%rcx),%ymm2,%ymm12 1386 vmovdqu %ymm3,0(%rsp) 1387 vmovdqu %ymm4,32(%rsp) 1388 vmovdqu %ymm5,64(%rsp) 1389 vmovdqu %ymm6,96(%rsp) 1390 vmovdqu %ymm7,128(%rsp) 1391 vmovdqu %ymm8,160(%rsp) 1392 vmovdqu %ymm9,192(%rsp) 1393 vmovdqu %ymm10,224(%rsp) 1394 vmovdqu %ymm11,256(%rsp) 1395 vmovdqu %ymm12,288(%rsp) 1396 movq $0,320(%rsp) 1397 1398 vmovdqu 8(%rsp),%ymm3 1399 vmovdqu 40(%rsp),%ymm4 1400 vmovdqu 72(%rsp),%ymm5 1401 vmovdqu 104(%rsp),%ymm6 1402 vmovdqu 136(%rsp),%ymm7 1403 vmovdqu 168(%rsp),%ymm8 1404 vmovdqu 200(%rsp),%ymm9 1405 vmovdqu 232(%rsp),%ymm10 1406 vmovdqu 264(%rsp),%ymm11 1407 vmovdqu 296(%rsp),%ymm12 1408 1409 addq 8(%rsp),%r9 1410 1411 {vex} vpmadd52huq 320(%rsi),%ymm1,%ymm3 1412 {vex} vpmadd52huq 352(%rsi),%ymm1,%ymm4 1413 {vex} vpmadd52huq 384(%rsi),%ymm1,%ymm5 1414 {vex} vpmadd52huq 416(%rsi),%ymm1,%ymm6 1415 {vex} vpmadd52huq 448(%rsi),%ymm1,%ymm7 1416 {vex} vpmadd52huq 480(%rsi),%ymm1,%ymm8 1417 {vex} vpmadd52huq 512(%rsi),%ymm1,%ymm9 1418 {vex} vpmadd52huq 544(%rsi),%ymm1,%ymm10 1419 {vex} vpmadd52huq 576(%rsi),%ymm1,%ymm11 1420 {vex} vpmadd52huq 608(%rsi),%ymm1,%ymm12 1421 1422 {vex} vpmadd52huq 320(%rcx),%ymm2,%ymm3 1423 {vex} vpmadd52huq 352(%rcx),%ymm2,%ymm4 1424 {vex} vpmadd52huq 384(%rcx),%ymm2,%ymm5 1425 {vex} vpmadd52huq 416(%rcx),%ymm2,%ymm6 1426 {vex} vpmadd52huq 448(%rcx),%ymm2,%ymm7 1427 {vex} vpmadd52huq 480(%rcx),%ymm2,%ymm8 1428 {vex} vpmadd52huq 512(%rcx),%ymm2,%ymm9 1429 {vex} vpmadd52huq 544(%rcx),%ymm2,%ymm10 1430 {vex} vpmadd52huq 576(%rcx),%ymm2,%ymm11 1431 {vex} vpmadd52huq 608(%rcx),%ymm2,%ymm12 1432 leaq 328(%rsp),%rsp 1433 leaq 8(%r11),%r11 1434 decl %ebx 1435 jne .Lloop40_1 1436 1437 vmovq %r9,%xmm0 1438 vpbroadcastq %xmm0,%ymm0 1439 vpblendd $3,%ymm0,%ymm3,%ymm3 1440 1441 leaq -640(%rsp),%rsp 1442 vmovupd %ymm3,0(%rsp) 1443 vmovupd %ymm4,32(%rsp) 1444 vmovupd %ymm5,64(%rsp) 1445 vmovupd %ymm6,96(%rsp) 1446 vmovupd %ymm7,128(%rsp) 1447 vmovupd %ymm8,160(%rsp) 1448 vmovupd %ymm9,192(%rsp) 1449 vmovupd %ymm10,224(%rsp) 1450 vmovupd %ymm11,256(%rsp) 1451 vmovupd %ymm12,288(%rsp) 1452 1453 1454 1455 vpsrlq $52,%ymm3,%ymm3 1456 vpsrlq $52,%ymm4,%ymm4 1457 vpsrlq $52,%ymm5,%ymm5 1458 vpsrlq $52,%ymm6,%ymm6 1459 vpsrlq $52,%ymm7,%ymm7 1460 vpsrlq $52,%ymm8,%ymm8 1461 vpsrlq $52,%ymm9,%ymm9 1462 vpsrlq $52,%ymm10,%ymm10 1463 vpsrlq $52,%ymm11,%ymm11 1464 vpsrlq $52,%ymm12,%ymm12 1465 1466 1467 vpermq $144,%ymm12,%ymm12 1468 vpermq $3,%ymm11,%ymm13 1469 vblendpd $1,%ymm13,%ymm12,%ymm12 1470 1471 vpermq $144,%ymm11,%ymm11 1472 vpermq $3,%ymm10,%ymm13 1473 vblendpd $1,%ymm13,%ymm11,%ymm11 1474 1475 vpermq $144,%ymm10,%ymm10 1476 vpermq $3,%ymm9,%ymm13 1477 vblendpd $1,%ymm13,%ymm10,%ymm10 1478 1479 vpermq $144,%ymm9,%ymm9 1480 vpermq $3,%ymm8,%ymm13 1481 vblendpd $1,%ymm13,%ymm9,%ymm9 1482 1483 vpermq $144,%ymm8,%ymm8 1484 vpermq $3,%ymm7,%ymm13 1485 vblendpd $1,%ymm13,%ymm8,%ymm8 1486 1487 vpermq $144,%ymm7,%ymm7 1488 vpermq $3,%ymm6,%ymm13 1489 vblendpd $1,%ymm13,%ymm7,%ymm7 1490 1491 vpermq $144,%ymm6,%ymm6 1492 vpermq $3,%ymm5,%ymm13 1493 vblendpd $1,%ymm13,%ymm6,%ymm6 1494 1495 vpermq $144,%ymm5,%ymm5 1496 vpermq $3,%ymm4,%ymm13 1497 vblendpd $1,%ymm13,%ymm5,%ymm5 1498 1499 vpermq $144,%ymm4,%ymm4 1500 vpermq $3,%ymm3,%ymm13 1501 vblendpd $1,%ymm13,%ymm4,%ymm4 1502 1503 vpermq $144,%ymm3,%ymm3 1504 vpand .Lhigh64x3(%rip),%ymm3,%ymm3 1505 1506 vmovupd %ymm3,320(%rsp) 1507 vmovupd %ymm4,352(%rsp) 1508 vmovupd %ymm5,384(%rsp) 1509 vmovupd %ymm6,416(%rsp) 1510 vmovupd %ymm7,448(%rsp) 1511 vmovupd %ymm8,480(%rsp) 1512 vmovupd %ymm9,512(%rsp) 1513 vmovupd %ymm10,544(%rsp) 1514 vmovupd %ymm11,576(%rsp) 1515 vmovupd %ymm12,608(%rsp) 1516 1517 vmovupd 0(%rsp),%ymm3 1518 vmovupd 32(%rsp),%ymm4 1519 vmovupd 64(%rsp),%ymm5 1520 vmovupd 96(%rsp),%ymm6 1521 vmovupd 128(%rsp),%ymm7 1522 vmovupd 160(%rsp),%ymm8 1523 vmovupd 192(%rsp),%ymm9 1524 vmovupd 224(%rsp),%ymm10 1525 vmovupd 256(%rsp),%ymm11 1526 vmovupd 288(%rsp),%ymm12 1527 1528 1529 vpand .Lmask52x4(%rip),%ymm3,%ymm3 1530 vpand .Lmask52x4(%rip),%ymm4,%ymm4 1531 vpand .Lmask52x4(%rip),%ymm5,%ymm5 1532 vpand .Lmask52x4(%rip),%ymm6,%ymm6 1533 vpand .Lmask52x4(%rip),%ymm7,%ymm7 1534 vpand .Lmask52x4(%rip),%ymm8,%ymm8 1535 vpand .Lmask52x4(%rip),%ymm9,%ymm9 1536 vpand .Lmask52x4(%rip),%ymm10,%ymm10 1537 vpand .Lmask52x4(%rip),%ymm11,%ymm11 1538 vpand .Lmask52x4(%rip),%ymm12,%ymm12 1539 1540 1541 vpaddq 320(%rsp),%ymm3,%ymm3 1542 vpaddq 352(%rsp),%ymm4,%ymm4 1543 vpaddq 384(%rsp),%ymm5,%ymm5 1544 vpaddq 416(%rsp),%ymm6,%ymm6 1545 vpaddq 448(%rsp),%ymm7,%ymm7 1546 vpaddq 480(%rsp),%ymm8,%ymm8 1547 vpaddq 512(%rsp),%ymm9,%ymm9 1548 vpaddq 544(%rsp),%ymm10,%ymm10 1549 vpaddq 576(%rsp),%ymm11,%ymm11 1550 vpaddq 608(%rsp),%ymm12,%ymm12 1551 1552 leaq 640(%rsp),%rsp 1553 1554 1555 1556 vpcmpgtq .Lmask52x4(%rip),%ymm3,%ymm13 1557 vmovmskpd %ymm13,%r14d 1558 vpcmpgtq .Lmask52x4(%rip),%ymm4,%ymm13 1559 vmovmskpd %ymm13,%r13d 1560 shlb $4,%r13b 1561 orb %r13b,%r14b 1562 1563 vpcmpgtq .Lmask52x4(%rip),%ymm5,%ymm13 1564 vmovmskpd %ymm13,%r13d 1565 vpcmpgtq .Lmask52x4(%rip),%ymm6,%ymm13 1566 vmovmskpd %ymm13,%r12d 1567 shlb $4,%r12b 1568 orb %r12b,%r13b 1569 1570 vpcmpgtq .Lmask52x4(%rip),%ymm7,%ymm13 1571 vmovmskpd %ymm13,%r12d 1572 vpcmpgtq .Lmask52x4(%rip),%ymm8,%ymm13 1573 vmovmskpd %ymm13,%r11d 1574 shlb $4,%r11b 1575 orb %r11b,%r12b 1576 1577 vpcmpgtq .Lmask52x4(%rip),%ymm9,%ymm13 1578 vmovmskpd %ymm13,%r11d 1579 vpcmpgtq .Lmask52x4(%rip),%ymm10,%ymm13 1580 vmovmskpd %ymm13,%r10d 1581 shlb $4,%r10b 1582 orb %r10b,%r11b 1583 1584 vpcmpgtq .Lmask52x4(%rip),%ymm11,%ymm13 1585 vmovmskpd %ymm13,%r10d 1586 vpcmpgtq .Lmask52x4(%rip),%ymm12,%ymm13 1587 vmovmskpd %ymm13,%r9d 1588 shlb $4,%r9b 1589 orb %r9b,%r10b 1590 1591 addb %r14b,%r14b 1592 adcb %r13b,%r13b 1593 adcb %r12b,%r12b 1594 adcb %r11b,%r11b 1595 adcb %r10b,%r10b 1596 1597 1598 vpcmpeqq .Lmask52x4(%rip),%ymm3,%ymm13 1599 vmovmskpd %ymm13,%r9d 1600 vpcmpeqq .Lmask52x4(%rip),%ymm4,%ymm13 1601 vmovmskpd %ymm13,%r8d 1602 shlb $4,%r8b 1603 orb %r8b,%r9b 1604 1605 vpcmpeqq .Lmask52x4(%rip),%ymm5,%ymm13 1606 vmovmskpd %ymm13,%r8d 1607 vpcmpeqq .Lmask52x4(%rip),%ymm6,%ymm13 1608 vmovmskpd %ymm13,%edx 1609 shlb $4,%dl 1610 orb %dl,%r8b 1611 1612 vpcmpeqq .Lmask52x4(%rip),%ymm7,%ymm13 1613 vmovmskpd %ymm13,%edx 1614 vpcmpeqq .Lmask52x4(%rip),%ymm8,%ymm13 1615 vmovmskpd %ymm13,%ecx 1616 shlb $4,%cl 1617 orb %cl,%dl 1618 1619 vpcmpeqq .Lmask52x4(%rip),%ymm9,%ymm13 1620 vmovmskpd %ymm13,%ecx 1621 vpcmpeqq .Lmask52x4(%rip),%ymm10,%ymm13 1622 vmovmskpd %ymm13,%ebx 1623 shlb $4,%bl 1624 orb %bl,%cl 1625 1626 vpcmpeqq .Lmask52x4(%rip),%ymm11,%ymm13 1627 vmovmskpd %ymm13,%ebx 1628 vpcmpeqq .Lmask52x4(%rip),%ymm12,%ymm13 1629 vmovmskpd %ymm13,%eax 1630 shlb $4,%al 1631 orb %al,%bl 1632 1633 addb %r9b,%r14b 1634 adcb %r8b,%r13b 1635 adcb %dl,%r12b 1636 adcb %cl,%r11b 1637 adcb %bl,%r10b 1638 1639 xorb %r9b,%r14b 1640 xorb %r8b,%r13b 1641 xorb %dl,%r12b 1642 xorb %cl,%r11b 1643 xorb %bl,%r10b 1644 1645 pushq %r9 1646 pushq %r8 1647 1648 leaq .Lkmasklut(%rip),%r8 1649 1650 movb %r14b,%r9b 1651 andq $0xf,%r14 1652 vpsubq .Lmask52x4(%rip),%ymm3,%ymm13 1653 shlq $5,%r14 1654 vmovapd (%r8,%r14,1),%ymm14 1655 vblendvpd %ymm14,%ymm13,%ymm3,%ymm3 1656 1657 shrb $4,%r9b 1658 andq $0xf,%r9 1659 vpsubq .Lmask52x4(%rip),%ymm4,%ymm13 1660 shlq $5,%r9 1661 vmovapd (%r8,%r9,1),%ymm14 1662 vblendvpd %ymm14,%ymm13,%ymm4,%ymm4 1663 1664 movb %r13b,%r9b 1665 andq $0xf,%r13 1666 vpsubq .Lmask52x4(%rip),%ymm5,%ymm13 1667 shlq $5,%r13 1668 vmovapd (%r8,%r13,1),%ymm14 1669 vblendvpd %ymm14,%ymm13,%ymm5,%ymm5 1670 1671 shrb $4,%r9b 1672 andq $0xf,%r9 1673 vpsubq .Lmask52x4(%rip),%ymm6,%ymm13 1674 shlq $5,%r9 1675 vmovapd (%r8,%r9,1),%ymm14 1676 vblendvpd %ymm14,%ymm13,%ymm6,%ymm6 1677 1678 movb %r12b,%r9b 1679 andq $0xf,%r12 1680 vpsubq .Lmask52x4(%rip),%ymm7,%ymm13 1681 shlq $5,%r12 1682 vmovapd (%r8,%r12,1),%ymm14 1683 vblendvpd %ymm14,%ymm13,%ymm7,%ymm7 1684 1685 shrb $4,%r9b 1686 andq $0xf,%r9 1687 vpsubq .Lmask52x4(%rip),%ymm8,%ymm13 1688 shlq $5,%r9 1689 vmovapd (%r8,%r9,1),%ymm14 1690 vblendvpd %ymm14,%ymm13,%ymm8,%ymm8 1691 1692 movb %r11b,%r9b 1693 andq $0xf,%r11 1694 vpsubq .Lmask52x4(%rip),%ymm9,%ymm13 1695 shlq $5,%r11 1696 vmovapd (%r8,%r11,1),%ymm14 1697 vblendvpd %ymm14,%ymm13,%ymm9,%ymm9 1698 1699 shrb $4,%r9b 1700 andq $0xf,%r9 1701 vpsubq .Lmask52x4(%rip),%ymm10,%ymm13 1702 shlq $5,%r9 1703 vmovapd (%r8,%r9,1),%ymm14 1704 vblendvpd %ymm14,%ymm13,%ymm10,%ymm10 1705 1706 movb %r10b,%r9b 1707 andq $0xf,%r10 1708 vpsubq .Lmask52x4(%rip),%ymm11,%ymm13 1709 shlq $5,%r10 1710 vmovapd (%r8,%r10,1),%ymm14 1711 vblendvpd %ymm14,%ymm13,%ymm11,%ymm11 1712 1713 shrb $4,%r9b 1714 andq $0xf,%r9 1715 vpsubq .Lmask52x4(%rip),%ymm12,%ymm13 1716 shlq $5,%r9 1717 vmovapd (%r8,%r9,1),%ymm14 1718 vblendvpd %ymm14,%ymm13,%ymm12,%ymm12 1719 1720 popq %r8 1721 popq %r9 1722 1723 vpand .Lmask52x4(%rip),%ymm3,%ymm3 1724 vpand .Lmask52x4(%rip),%ymm4,%ymm4 1725 vpand .Lmask52x4(%rip),%ymm5,%ymm5 1726 vpand .Lmask52x4(%rip),%ymm6,%ymm6 1727 vpand .Lmask52x4(%rip),%ymm7,%ymm7 1728 vpand .Lmask52x4(%rip),%ymm8,%ymm8 1729 vpand .Lmask52x4(%rip),%ymm9,%ymm9 1730 1731 vpand .Lmask52x4(%rip),%ymm10,%ymm10 1732 vpand .Lmask52x4(%rip),%ymm11,%ymm11 1733 vpand .Lmask52x4(%rip),%ymm12,%ymm12 1734 1735 vmovdqu %ymm3,320(%rdi) 1736 vmovdqu %ymm4,352(%rdi) 1737 vmovdqu %ymm5,384(%rdi) 1738 vmovdqu %ymm6,416(%rdi) 1739 vmovdqu %ymm7,448(%rdi) 1740 vmovdqu %ymm8,480(%rdi) 1741 vmovdqu %ymm9,512(%rdi) 1742 vmovdqu %ymm10,544(%rdi) 1743 vmovdqu %ymm11,576(%rdi) 1744 vmovdqu %ymm12,608(%rdi) 1745 1746 vzeroupper 1747 leaq (%rsp),%rax 1748 .cfi_def_cfa_register %rax 1749 movq 0(%rax),%r15 1750 .cfi_restore %r15 1751 movq 8(%rax),%r14 1752 .cfi_restore %r14 1753 movq 16(%rax),%r13 1754 .cfi_restore %r13 1755 movq 24(%rax),%r12 1756 .cfi_restore %r12 1757 movq 32(%rax),%rbp 1758 .cfi_restore %rbp 1759 movq 40(%rax),%rbx 1760 .cfi_restore %rbx 1761 leaq 48(%rax),%rsp 1762 .cfi_def_cfa %rsp,8 1763 .Lossl_rsaz_amm52x40_x2_avxifma256_epilogue: 1764 .byte 0xf3,0xc3 1765 .cfi_endproc 1766 .size ossl_rsaz_amm52x40_x2_avxifma256, .-ossl_rsaz_amm52x40_x2_avxifma256 1767 .text 1768 1769 .align 32 1770 .globl ossl_extract_multiplier_2x40_win5_avx 1771 .type ossl_extract_multiplier_2x40_win5_avx,@function 1772 ossl_extract_multiplier_2x40_win5_avx: 1773 .cfi_startproc 1774 .byte 243,15,30,250 1775 vmovapd .Lones(%rip),%ymm14 1776 vmovq %rdx,%xmm10 1777 vpbroadcastq %xmm10,%ymm12 1778 vmovq %rcx,%xmm10 1779 vpbroadcastq %xmm10,%ymm13 1780 leaq 20480(%rsi),%rax 1781 1782 1783 movq %rsi,%r10 1784 1785 1786 vpxor %xmm0,%xmm0,%xmm0 1787 vmovapd %ymm0,%ymm1 1788 vmovapd %ymm0,%ymm2 1789 vmovapd %ymm0,%ymm3 1790 vmovapd %ymm0,%ymm4 1791 vmovapd %ymm0,%ymm5 1792 vmovapd %ymm0,%ymm6 1793 vmovapd %ymm0,%ymm7 1794 vmovapd %ymm0,%ymm8 1795 vmovapd %ymm0,%ymm9 1796 vpxor %ymm11,%ymm11,%ymm11 1797 .align 32 1798 .Lloop_0: 1799 vpcmpeqq %ymm11,%ymm12,%ymm15 1800 vmovdqu 0(%rsi),%ymm10 1801 1802 vblendvpd %ymm15,%ymm10,%ymm0,%ymm0 1803 vmovdqu 32(%rsi),%ymm10 1804 1805 vblendvpd %ymm15,%ymm10,%ymm1,%ymm1 1806 vmovdqu 64(%rsi),%ymm10 1807 1808 vblendvpd %ymm15,%ymm10,%ymm2,%ymm2 1809 vmovdqu 96(%rsi),%ymm10 1810 1811 vblendvpd %ymm15,%ymm10,%ymm3,%ymm3 1812 vmovdqu 128(%rsi),%ymm10 1813 1814 vblendvpd %ymm15,%ymm10,%ymm4,%ymm4 1815 vmovdqu 160(%rsi),%ymm10 1816 1817 vblendvpd %ymm15,%ymm10,%ymm5,%ymm5 1818 vmovdqu 192(%rsi),%ymm10 1819 1820 vblendvpd %ymm15,%ymm10,%ymm6,%ymm6 1821 vmovdqu 224(%rsi),%ymm10 1822 1823 vblendvpd %ymm15,%ymm10,%ymm7,%ymm7 1824 vmovdqu 256(%rsi),%ymm10 1825 1826 vblendvpd %ymm15,%ymm10,%ymm8,%ymm8 1827 vmovdqu 288(%rsi),%ymm10 1828 1829 vblendvpd %ymm15,%ymm10,%ymm9,%ymm9 1830 vpaddq %ymm14,%ymm11,%ymm11 1831 addq $640,%rsi 1832 cmpq %rsi,%rax 1833 jne .Lloop_0 1834 vmovdqu %ymm0,0(%rdi) 1835 vmovdqu %ymm1,32(%rdi) 1836 vmovdqu %ymm2,64(%rdi) 1837 vmovdqu %ymm3,96(%rdi) 1838 vmovdqu %ymm4,128(%rdi) 1839 vmovdqu %ymm5,160(%rdi) 1840 vmovdqu %ymm6,192(%rdi) 1841 vmovdqu %ymm7,224(%rdi) 1842 vmovdqu %ymm8,256(%rdi) 1843 vmovdqu %ymm9,288(%rdi) 1844 movq %r10,%rsi 1845 vpxor %ymm11,%ymm11,%ymm11 1846 .align 32 1847 .Lloop_320: 1848 vpcmpeqq %ymm11,%ymm13,%ymm15 1849 vmovdqu 320(%rsi),%ymm10 1850 1851 vblendvpd %ymm15,%ymm10,%ymm0,%ymm0 1852 vmovdqu 352(%rsi),%ymm10 1853 1854 vblendvpd %ymm15,%ymm10,%ymm1,%ymm1 1855 vmovdqu 384(%rsi),%ymm10 1856 1857 vblendvpd %ymm15,%ymm10,%ymm2,%ymm2 1858 vmovdqu 416(%rsi),%ymm10 1859 1860 vblendvpd %ymm15,%ymm10,%ymm3,%ymm3 1861 vmovdqu 448(%rsi),%ymm10 1862 1863 vblendvpd %ymm15,%ymm10,%ymm4,%ymm4 1864 vmovdqu 480(%rsi),%ymm10 1865 1866 vblendvpd %ymm15,%ymm10,%ymm5,%ymm5 1867 vmovdqu 512(%rsi),%ymm10 1868 1869 vblendvpd %ymm15,%ymm10,%ymm6,%ymm6 1870 vmovdqu 544(%rsi),%ymm10 1871 1872 vblendvpd %ymm15,%ymm10,%ymm7,%ymm7 1873 vmovdqu 576(%rsi),%ymm10 1874 1875 vblendvpd %ymm15,%ymm10,%ymm8,%ymm8 1876 vmovdqu 608(%rsi),%ymm10 1877 1878 vblendvpd %ymm15,%ymm10,%ymm9,%ymm9 1879 vpaddq %ymm14,%ymm11,%ymm11 1880 addq $640,%rsi 1881 cmpq %rsi,%rax 1882 jne .Lloop_320 1883 vmovdqu %ymm0,320(%rdi) 1884 vmovdqu %ymm1,352(%rdi) 1885 vmovdqu %ymm2,384(%rdi) 1886 vmovdqu %ymm3,416(%rdi) 1887 vmovdqu %ymm4,448(%rdi) 1888 vmovdqu %ymm5,480(%rdi) 1889 vmovdqu %ymm6,512(%rdi) 1890 vmovdqu %ymm7,544(%rdi) 1891 vmovdqu %ymm8,576(%rdi) 1892 vmovdqu %ymm9,608(%rdi) 1893 1894 .byte 0xf3,0xc3 1895 .cfi_endproc 1896 .size ossl_extract_multiplier_2x40_win5_avx, .-ossl_extract_multiplier_2x40_win5_avx 1897 .section .rodata 1898 .align 32 1899 .Lones: 1900 .quad 1,1,1,1 1901 .Lzeros: 1902 .quad 0,0,0,0 1903 .section ".note.gnu.property", "a" 1904 .p2align 3 1905 .long 1f - 0f 1906 .long 4f - 1f 1907 .long 5 1908 0: 1909 # "GNU" encoded with .byte, since .asciz isn't supported 1910 # on Solaris. 1911 .byte 0x47 1912 .byte 0x4e 1913 .byte 0x55 1914 .byte 0 1915 1: 1916 .p2align 3 1917 .long 0xc0000002 1918 .long 3f - 2f 1919 2: 1920 .long 3 1921 3: 1922 .p2align 3 1923 4: 1924
Indexes created Sun Mar 01 05:31:48 UTC 2026