xref: /src/crypto/external/apache2/openssl/lib/libcrypto/arch/aarch64/armv8-mont.S

#include "arm_arch.h"
#ifndef	__KERNEL__

.hidden	OPENSSL_armv8_rsa_neonized
#endif
.text

.globl	bn_mul_mont
.type	bn_mul_mont,%function
.align	5
bn_mul_mont:
	AARCH64_SIGN_LINK_REGISTER
.Lbn_mul_mont:
	tst	x5,#3
	b.ne	.Lmul_mont
	cmp	x5,#32
	b.le	.Lscalar_impl
#ifndef	__KERNEL__
#ifndef	__AARCH64EB__
	adrp	x17,OPENSSL_armv8_rsa_neonized
	ldr	w17,[x17,#:lo12:OPENSSL_armv8_rsa_neonized]
	cbnz	w17, bn_mul8x_mont_neon
#endif
#endif

.Lscalar_impl:
	tst	x5,#7
	b.eq	__bn_sqr8x_mont
	tst	x5,#3
	b.eq	__bn_mul4x_mont

.Lmul_mont:
	stp	x29,x30,[sp,#-64]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]

	ldr	x9,[x2],#8		// bp[0]
	sub	x22,sp,x5,lsl#3
	ldp	x7,x8,[x1],#16	// ap[0..1]
	lsl	x5,x5,#3
	ldr	x4,[x4]		// *n0
	and	x22,x22,#-16		// ABI says so
	ldp	x13,x14,[x3],#16	// np[0..1]

	mul	x6,x7,x9		// ap[0]*bp[0]
	sub	x21,x5,#16		// j=num-2
	umulh	x7,x7,x9
	mul	x10,x8,x9		// ap[1]*bp[0]
	umulh	x11,x8,x9

	mul	x15,x6,x4		// "tp[0]"*n0
	mov	sp,x22			// alloca

	// (*)	mul	x12,x13,x15	// np[0]*m1
	umulh	x13,x13,x15
	mul	x16,x14,x15		// np[1]*m1
	// (*)	adds	x12,x12,x6	// discarded
	// (*)	As for removal of first multiplication and addition
	//	instructions. The outcome of first addition is
	//	guaranteed to be zero, which leaves two computationally
	//	significant outcomes: it either carries or not. Then
	//	question is when does it carry? Is there alternative
	//	way to deduce it? If you follow operations, you can
	//	observe that condition for carry is quite simple:
	//	x6 being non-zero. So that carry can be calculated
	//	by adding -1 to x6. That's what next instruction does.
	subs	xzr,x6,#1		// (*)
	umulh	x17,x14,x15
	adc	x13,x13,xzr
	cbz	x21,.L1st_skip

.L1st:
	ldr	x8,[x1],#8
	adds	x6,x10,x7
	sub	x21,x21,#8		// j--
	adc	x7,x11,xzr

	ldr	x14,[x3],#8
	adds	x12,x16,x13
	mul	x10,x8,x9		// ap[j]*bp[0]
	adc	x13,x17,xzr
	umulh	x11,x8,x9

	adds	x12,x12,x6
	mul	x16,x14,x15		// np[j]*m1
	adc	x13,x13,xzr
	umulh	x17,x14,x15
	str	x12,[x22],#8		// tp[j-1]
	cbnz	x21,.L1st

.L1st_skip:
	adds	x6,x10,x7
	sub	x1,x1,x5		// rewind x1
	adc	x7,x11,xzr

	adds	x12,x16,x13
	sub	x3,x3,x5		// rewind x3
	adc	x13,x17,xzr

	adds	x12,x12,x6
	sub	x20,x5,#8		// i=num-1
	adcs	x13,x13,x7

	adc	x19,xzr,xzr		// upmost overflow bit
	stp	x12,x13,[x22]

.Louter:
	ldr	x9,[x2],#8		// bp[i]
	ldp	x7,x8,[x1],#16
	ldr	x23,[sp]		// tp[0]
	add	x22,sp,#8

	mul	x6,x7,x9		// ap[0]*bp[i]
	sub	x21,x5,#16		// j=num-2
	umulh	x7,x7,x9
	ldp	x13,x14,[x3],#16
	mul	x10,x8,x9		// ap[1]*bp[i]
	adds	x6,x6,x23
	umulh	x11,x8,x9
	adc	x7,x7,xzr

	mul	x15,x6,x4
	sub	x20,x20,#8		// i--

	// (*)	mul	x12,x13,x15	// np[0]*m1
	umulh	x13,x13,x15
	mul	x16,x14,x15		// np[1]*m1
	// (*)	adds	x12,x12,x6
	subs	xzr,x6,#1		// (*)
	umulh	x17,x14,x15
	cbz	x21,.Linner_skip

.Linner:
	ldr	x8,[x1],#8
	adc	x13,x13,xzr
	ldr	x23,[x22],#8		// tp[j]
	adds	x6,x10,x7
	sub	x21,x21,#8		// j--
	adc	x7,x11,xzr

	adds	x12,x16,x13
	ldr	x14,[x3],#8
	adc	x13,x17,xzr

	mul	x10,x8,x9		// ap[j]*bp[i]
	adds	x6,x6,x23
	umulh	x11,x8,x9
	adc	x7,x7,xzr

	mul	x16,x14,x15		// np[j]*m1
	adds	x12,x12,x6
	umulh	x17,x14,x15
	stur	x12,[x22,#-16]		// tp[j-1]
	cbnz	x21,.Linner

.Linner_skip:
	ldr	x23,[x22],#8		// tp[j]
	adc	x13,x13,xzr
	adds	x6,x10,x7
	sub	x1,x1,x5		// rewind x1
	adc	x7,x11,xzr

	adds	x12,x16,x13
	sub	x3,x3,x5		// rewind x3
	adcs	x13,x17,x19
	adc	x19,xzr,xzr

	adds	x6,x6,x23
	adc	x7,x7,xzr

	adds	x12,x12,x6
	adcs	x13,x13,x7
	adc	x19,x19,xzr		// upmost overflow bit
	stp	x12,x13,[x22,#-16]

	cbnz	x20,.Louter

	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	ldr	x23,[sp]		// tp[0]
	add	x22,sp,#8
	ldr	x14,[x3],#8		// np[0]
	subs	x21,x5,#8		// j=num-1 and clear borrow
	mov	x1,x0
.Lsub:
	sbcs	x8,x23,x14		// tp[j]-np[j]
	ldr	x23,[x22],#8
	sub	x21,x21,#8		// j--
	ldr	x14,[x3],#8
	str	x8,[x1],#8		// rp[j]=tp[j]-np[j]
	cbnz	x21,.Lsub

	sbcs	x8,x23,x14
	sbcs	x19,x19,xzr		// did it borrow?
	str	x8,[x1],#8		// rp[num-1]

	ldr	x23,[sp]		// tp[0]
	add	x22,sp,#8
	ldr	x8,[x0],#8		// rp[0]
	sub	x5,x5,#8		// num--
	nop
.Lcond_copy:
	sub	x5,x5,#8		// num--
	csel	x14,x23,x8,lo		// did it borrow?
	ldr	x23,[x22],#8
	ldr	x8,[x0],#8
	stur	xzr,[x22,#-16]		// wipe tp
	stur	x14,[x0,#-16]
	cbnz	x5,.Lcond_copy

	csel	x14,x23,x8,lo
	stur	xzr,[x22,#-8]		// wipe tp
	stur	x14,[x0,#-8]

	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldr	x29,[sp],#64
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	bn_mul_mont,.-bn_mul_mont
.type	bn_mul8x_mont_neon,%function
.align	5
bn_mul8x_mont_neon:
	// Not adding AARCH64_SIGN_LINK_REGISTER here because bn_mul8x_mont_neon is jumped to
	// only from bn_mul_mont which has already signed the return address.
	stp	x29,x30,[sp,#-80]!
	mov	x16,sp
	stp	d8,d9,[sp,#16]
	stp	d10,d11,[sp,#32]
	stp	d12,d13,[sp,#48]
	stp	d14,d15,[sp,#64]
	lsl	x5,x5,#1
	eor	v14.16b,v14.16b,v14.16b

.align	4
.LNEON_8n:
	eor	v6.16b,v6.16b,v6.16b
	sub	x7,sp,#128
	eor	v7.16b,v7.16b,v7.16b
	sub	x7,x7,x5,lsl#4
	eor	v8.16b,v8.16b,v8.16b
	and	x7,x7,#-64
	eor	v9.16b,v9.16b,v9.16b
	mov	sp,x7		// alloca
	eor	v10.16b,v10.16b,v10.16b
	add	x7,x7,#256
	eor	v11.16b,v11.16b,v11.16b
	sub	x8,x5,#8
	eor	v12.16b,v12.16b,v12.16b
	eor	v13.16b,v13.16b,v13.16b

.LNEON_8n_init:
	st1	{v6.2d,v7.2d},[x7],#32
	subs	x8,x8,#8
	st1	{v8.2d,v9.2d},[x7],#32
	st1	{v10.2d,v11.2d},[x7],#32
	st1	{v12.2d,v13.2d},[x7],#32
	bne	.LNEON_8n_init

	add	x6,sp,#256
	ld1	{v0.4s,v1.4s},[x1],#32
	add	x10,sp,#8
	ldr	s30,[x4],#4
	mov	x9,x5
	b	.LNEON_8n_outer

.align	4
.LNEON_8n_outer:
	ldr	s28,[x2],#4   // *b++
	uxtl	v28.4s,v28.4h
	add	x7,sp,#128
	ld1	{v2.4s,v3.4s},[x3],#32

	umlal	v6.2d,v28.2s,v0.s[0]
	umlal	v7.2d,v28.2s,v0.s[1]
	umlal	v8.2d,v28.2s,v0.s[2]
	shl	v29.2d,v6.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v9.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v6.2d
	umlal	v10.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v11.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[sp]		// put aside smashed b[8*i+0]
	umlal	v12.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v13.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v6.2d,v29.2s,v2.s[0]
	umlal	v7.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v8.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v6.2d,#16
	umlal	v9.2d,v29.2s,v2.s[3]
	umlal	v10.2d,v29.2s,v3.s[0]
	ext	v6.16b,v6.16b,v6.16b,#8
	add	v6.2d,v6.2d,v15.2d
	umlal	v11.2d,v29.2s,v3.s[1]
	ushr	v6.2d,v6.2d,#16
	umlal	v12.2d,v29.2s,v3.s[2]
	umlal	v13.2d,v29.2s,v3.s[3]
	add	v16.2d,v7.2d,v6.2d
	ins	v7.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+0]
	umlal	v7.2d,v28.2s,v0.s[0]
	ld1	{v6.2d},[x6],#16
	umlal	v8.2d,v28.2s,v0.s[1]
	umlal	v9.2d,v28.2s,v0.s[2]
	shl	v29.2d,v7.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v10.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v7.2d
	umlal	v11.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v12.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+1]
	umlal	v13.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v6.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v7.2d,v29.2s,v2.s[0]
	umlal	v8.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v9.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v7.2d,#16
	umlal	v10.2d,v29.2s,v2.s[3]
	umlal	v11.2d,v29.2s,v3.s[0]
	ext	v7.16b,v7.16b,v7.16b,#8
	add	v7.2d,v7.2d,v15.2d
	umlal	v12.2d,v29.2s,v3.s[1]
	ushr	v7.2d,v7.2d,#16
	umlal	v13.2d,v29.2s,v3.s[2]
	umlal	v6.2d,v29.2s,v3.s[3]
	add	v16.2d,v8.2d,v7.2d
	ins	v8.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+1]
	umlal	v8.2d,v28.2s,v0.s[0]
	ld1	{v7.2d},[x6],#16
	umlal	v9.2d,v28.2s,v0.s[1]
	umlal	v10.2d,v28.2s,v0.s[2]
	shl	v29.2d,v8.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v11.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v8.2d
	umlal	v12.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v13.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+2]
	umlal	v6.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v7.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v8.2d,v29.2s,v2.s[0]
	umlal	v9.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v10.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v8.2d,#16
	umlal	v11.2d,v29.2s,v2.s[3]
	umlal	v12.2d,v29.2s,v3.s[0]
	ext	v8.16b,v8.16b,v8.16b,#8
	add	v8.2d,v8.2d,v15.2d
	umlal	v13.2d,v29.2s,v3.s[1]
	ushr	v8.2d,v8.2d,#16
	umlal	v6.2d,v29.2s,v3.s[2]
	umlal	v7.2d,v29.2s,v3.s[3]
	add	v16.2d,v9.2d,v8.2d
	ins	v9.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+2]
	umlal	v9.2d,v28.2s,v0.s[0]
	ld1	{v8.2d},[x6],#16
	umlal	v10.2d,v28.2s,v0.s[1]
	umlal	v11.2d,v28.2s,v0.s[2]
	shl	v29.2d,v9.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v12.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v9.2d
	umlal	v13.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v6.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+3]
	umlal	v7.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v8.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v9.2d,v29.2s,v2.s[0]
	umlal	v10.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v11.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v9.2d,#16
	umlal	v12.2d,v29.2s,v2.s[3]
	umlal	v13.2d,v29.2s,v3.s[0]
	ext	v9.16b,v9.16b,v9.16b,#8
	add	v9.2d,v9.2d,v15.2d
	umlal	v6.2d,v29.2s,v3.s[1]
	ushr	v9.2d,v9.2d,#16
	umlal	v7.2d,v29.2s,v3.s[2]
	umlal	v8.2d,v29.2s,v3.s[3]
	add	v16.2d,v10.2d,v9.2d
	ins	v10.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+3]
	umlal	v10.2d,v28.2s,v0.s[0]
	ld1	{v9.2d},[x6],#16
	umlal	v11.2d,v28.2s,v0.s[1]
	umlal	v12.2d,v28.2s,v0.s[2]
	shl	v29.2d,v10.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v13.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v10.2d
	umlal	v6.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v7.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+4]
	umlal	v8.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v9.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v10.2d,v29.2s,v2.s[0]
	umlal	v11.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v12.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v10.2d,#16
	umlal	v13.2d,v29.2s,v2.s[3]
	umlal	v6.2d,v29.2s,v3.s[0]
	ext	v10.16b,v10.16b,v10.16b,#8
	add	v10.2d,v10.2d,v15.2d
	umlal	v7.2d,v29.2s,v3.s[1]
	ushr	v10.2d,v10.2d,#16
	umlal	v8.2d,v29.2s,v3.s[2]
	umlal	v9.2d,v29.2s,v3.s[3]
	add	v16.2d,v11.2d,v10.2d
	ins	v11.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+4]
	umlal	v11.2d,v28.2s,v0.s[0]
	ld1	{v10.2d},[x6],#16
	umlal	v12.2d,v28.2s,v0.s[1]
	umlal	v13.2d,v28.2s,v0.s[2]
	shl	v29.2d,v11.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v6.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v11.2d
	umlal	v7.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v8.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+5]
	umlal	v9.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v10.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v11.2d,v29.2s,v2.s[0]
	umlal	v12.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v13.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v11.2d,#16
	umlal	v6.2d,v29.2s,v2.s[3]
	umlal	v7.2d,v29.2s,v3.s[0]
	ext	v11.16b,v11.16b,v11.16b,#8
	add	v11.2d,v11.2d,v15.2d
	umlal	v8.2d,v29.2s,v3.s[1]
	ushr	v11.2d,v11.2d,#16
	umlal	v9.2d,v29.2s,v3.s[2]
	umlal	v10.2d,v29.2s,v3.s[3]
	add	v16.2d,v12.2d,v11.2d
	ins	v12.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+5]
	umlal	v12.2d,v28.2s,v0.s[0]
	ld1	{v11.2d},[x6],#16
	umlal	v13.2d,v28.2s,v0.s[1]
	umlal	v6.2d,v28.2s,v0.s[2]
	shl	v29.2d,v12.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v7.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v12.2d
	umlal	v8.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v9.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+6]
	umlal	v10.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v11.2d,v28.2s,v1.s[3]
	ldr	s28,[x2],#4   // *b++
	umlal	v12.2d,v29.2s,v2.s[0]
	umlal	v13.2d,v29.2s,v2.s[1]
	uxtl	v28.4s,v28.4h
	umlal	v6.2d,v29.2s,v2.s[2]
	ushr	v15.2d,v12.2d,#16
	umlal	v7.2d,v29.2s,v2.s[3]
	umlal	v8.2d,v29.2s,v3.s[0]
	ext	v12.16b,v12.16b,v12.16b,#8
	add	v12.2d,v12.2d,v15.2d
	umlal	v9.2d,v29.2s,v3.s[1]
	ushr	v12.2d,v12.2d,#16
	umlal	v10.2d,v29.2s,v3.s[2]
	umlal	v11.2d,v29.2s,v3.s[3]
	add	v16.2d,v13.2d,v12.2d
	ins	v13.d[0],v16.d[0]
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+6]
	umlal	v13.2d,v28.2s,v0.s[0]
	ld1	{v12.2d},[x6],#16
	umlal	v6.2d,v28.2s,v0.s[1]
	umlal	v7.2d,v28.2s,v0.s[2]
	shl	v29.2d,v13.2d,#16
	ext	v29.16b,v29.16b,v29.16b,#8
	umlal	v8.2d,v28.2s,v0.s[3]
	add	v29.2d,v29.2d,v13.2d
	umlal	v9.2d,v28.2s,v1.s[0]
	mul	v29.2s,v29.2s,v30.2s
	umlal	v10.2d,v28.2s,v1.s[1]
	st1	{v28.2s},[x10],#8	// put aside smashed b[8*i+7]
	umlal	v11.2d,v28.2s,v1.s[2]
	uxtl	v29.4s,v29.4h
	umlal	v12.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[sp]		// pull smashed b[8*i+0]
	umlal	v13.2d,v29.2s,v2.s[0]
	ld1	{v0.4s,v1.4s},[x1],#32
	umlal	v6.2d,v29.2s,v2.s[1]
	umlal	v7.2d,v29.2s,v2.s[2]
	mov	v5.16b,v13.16b
	ushr	v5.2d,v5.2d,#16
	ext	v13.16b,v13.16b,v13.16b,#8
	umlal	v8.2d,v29.2s,v2.s[3]
	umlal	v9.2d,v29.2s,v3.s[0]
	add	v13.2d,v13.2d,v5.2d
	umlal	v10.2d,v29.2s,v3.s[1]
	ushr	v13.2d,v13.2d,#16
	eor	v15.16b,v15.16b,v15.16b
	ins	v13.d[1],v15.d[0]
	umlal	v11.2d,v29.2s,v3.s[2]
	umlal	v12.2d,v29.2s,v3.s[3]
	add	v6.2d,v6.2d,v13.2d
	st1	{v29.2s},[x10],#8	// put aside smashed m[8*i+7]
	add	x10,sp,#8		// rewind
	sub	x8,x5,#8
	b	.LNEON_8n_inner

.align	4
.LNEON_8n_inner:
	subs	x8,x8,#8
	umlal	v6.2d,v28.2s,v0.s[0]
	ld1	{v13.2d},[x6]
	umlal	v7.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+0]
	umlal	v8.2d,v28.2s,v0.s[2]
	ld1	{v2.4s,v3.4s},[x3],#32
	umlal	v9.2d,v28.2s,v0.s[3]
	b.eq	.LInner_jump
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump:
	umlal	v10.2d,v28.2s,v1.s[0]
	umlal	v11.2d,v28.2s,v1.s[1]
	umlal	v12.2d,v28.2s,v1.s[2]
	umlal	v13.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+1]
	umlal	v6.2d,v29.2s,v2.s[0]
	umlal	v7.2d,v29.2s,v2.s[1]
	umlal	v8.2d,v29.2s,v2.s[2]
	umlal	v9.2d,v29.2s,v2.s[3]
	umlal	v10.2d,v29.2s,v3.s[0]
	umlal	v11.2d,v29.2s,v3.s[1]
	umlal	v12.2d,v29.2s,v3.s[2]
	umlal	v13.2d,v29.2s,v3.s[3]
	st1	{v6.2d},[x7],#16
	umlal	v7.2d,v28.2s,v0.s[0]
	ld1	{v6.2d},[x6]
	umlal	v8.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+1]
	umlal	v9.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump1
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump1:
	umlal	v10.2d,v28.2s,v0.s[3]
	umlal	v11.2d,v28.2s,v1.s[0]
	umlal	v12.2d,v28.2s,v1.s[1]
	umlal	v13.2d,v28.2s,v1.s[2]
	umlal	v6.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+2]
	umlal	v7.2d,v29.2s,v2.s[0]
	umlal	v8.2d,v29.2s,v2.s[1]
	umlal	v9.2d,v29.2s,v2.s[2]
	umlal	v10.2d,v29.2s,v2.s[3]
	umlal	v11.2d,v29.2s,v3.s[0]
	umlal	v12.2d,v29.2s,v3.s[1]
	umlal	v13.2d,v29.2s,v3.s[2]
	umlal	v6.2d,v29.2s,v3.s[3]
	st1	{v7.2d},[x7],#16
	umlal	v8.2d,v28.2s,v0.s[0]
	ld1	{v7.2d},[x6]
	umlal	v9.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+2]
	umlal	v10.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump2
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump2:
	umlal	v11.2d,v28.2s,v0.s[3]
	umlal	v12.2d,v28.2s,v1.s[0]
	umlal	v13.2d,v28.2s,v1.s[1]
	umlal	v6.2d,v28.2s,v1.s[2]
	umlal	v7.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+3]
	umlal	v8.2d,v29.2s,v2.s[0]
	umlal	v9.2d,v29.2s,v2.s[1]
	umlal	v10.2d,v29.2s,v2.s[2]
	umlal	v11.2d,v29.2s,v2.s[3]
	umlal	v12.2d,v29.2s,v3.s[0]
	umlal	v13.2d,v29.2s,v3.s[1]
	umlal	v6.2d,v29.2s,v3.s[2]
	umlal	v7.2d,v29.2s,v3.s[3]
	st1	{v8.2d},[x7],#16
	umlal	v9.2d,v28.2s,v0.s[0]
	ld1	{v8.2d},[x6]
	umlal	v10.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+3]
	umlal	v11.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump3
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump3:
	umlal	v12.2d,v28.2s,v0.s[3]
	umlal	v13.2d,v28.2s,v1.s[0]
	umlal	v6.2d,v28.2s,v1.s[1]
	umlal	v7.2d,v28.2s,v1.s[2]
	umlal	v8.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+4]
	umlal	v9.2d,v29.2s,v2.s[0]
	umlal	v10.2d,v29.2s,v2.s[1]
	umlal	v11.2d,v29.2s,v2.s[2]
	umlal	v12.2d,v29.2s,v2.s[3]
	umlal	v13.2d,v29.2s,v3.s[0]
	umlal	v6.2d,v29.2s,v3.s[1]
	umlal	v7.2d,v29.2s,v3.s[2]
	umlal	v8.2d,v29.2s,v3.s[3]
	st1	{v9.2d},[x7],#16
	umlal	v10.2d,v28.2s,v0.s[0]
	ld1	{v9.2d},[x6]
	umlal	v11.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+4]
	umlal	v12.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump4
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump4:
	umlal	v13.2d,v28.2s,v0.s[3]
	umlal	v6.2d,v28.2s,v1.s[0]
	umlal	v7.2d,v28.2s,v1.s[1]
	umlal	v8.2d,v28.2s,v1.s[2]
	umlal	v9.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+5]
	umlal	v10.2d,v29.2s,v2.s[0]
	umlal	v11.2d,v29.2s,v2.s[1]
	umlal	v12.2d,v29.2s,v2.s[2]
	umlal	v13.2d,v29.2s,v2.s[3]
	umlal	v6.2d,v29.2s,v3.s[0]
	umlal	v7.2d,v29.2s,v3.s[1]
	umlal	v8.2d,v29.2s,v3.s[2]
	umlal	v9.2d,v29.2s,v3.s[3]
	st1	{v10.2d},[x7],#16
	umlal	v11.2d,v28.2s,v0.s[0]
	ld1	{v10.2d},[x6]
	umlal	v12.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+5]
	umlal	v13.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump5
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump5:
	umlal	v6.2d,v28.2s,v0.s[3]
	umlal	v7.2d,v28.2s,v1.s[0]
	umlal	v8.2d,v28.2s,v1.s[1]
	umlal	v9.2d,v28.2s,v1.s[2]
	umlal	v10.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+6]
	umlal	v11.2d,v29.2s,v2.s[0]
	umlal	v12.2d,v29.2s,v2.s[1]
	umlal	v13.2d,v29.2s,v2.s[2]
	umlal	v6.2d,v29.2s,v2.s[3]
	umlal	v7.2d,v29.2s,v3.s[0]
	umlal	v8.2d,v29.2s,v3.s[1]
	umlal	v9.2d,v29.2s,v3.s[2]
	umlal	v10.2d,v29.2s,v3.s[3]
	st1	{v11.2d},[x7],#16
	umlal	v12.2d,v28.2s,v0.s[0]
	ld1	{v11.2d},[x6]
	umlal	v13.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+6]
	umlal	v6.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump6
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump6:
	umlal	v7.2d,v28.2s,v0.s[3]
	umlal	v8.2d,v28.2s,v1.s[0]
	umlal	v9.2d,v28.2s,v1.s[1]
	umlal	v10.2d,v28.2s,v1.s[2]
	umlal	v11.2d,v28.2s,v1.s[3]
	ld1	{v28.2s},[x10],#8	// pull smashed b[8*i+7]
	umlal	v12.2d,v29.2s,v2.s[0]
	umlal	v13.2d,v29.2s,v2.s[1]
	umlal	v6.2d,v29.2s,v2.s[2]
	umlal	v7.2d,v29.2s,v2.s[3]
	umlal	v8.2d,v29.2s,v3.s[0]
	umlal	v9.2d,v29.2s,v3.s[1]
	umlal	v10.2d,v29.2s,v3.s[2]
	umlal	v11.2d,v29.2s,v3.s[3]
	st1	{v12.2d},[x7],#16
	umlal	v13.2d,v28.2s,v0.s[0]
	ld1	{v12.2d},[x6]
	umlal	v6.2d,v28.2s,v0.s[1]
	ld1	{v29.2s},[x10],#8	// pull smashed m[8*i+7]
	umlal	v7.2d,v28.2s,v0.s[2]
	b.eq	.LInner_jump7
	add	x6,x6,#16	// don't advance in last iteration
.LInner_jump7:
	umlal	v8.2d,v28.2s,v0.s[3]
	umlal	v9.2d,v28.2s,v1.s[0]
	umlal	v10.2d,v28.2s,v1.s[1]
	umlal	v11.2d,v28.2s,v1.s[2]
	umlal	v12.2d,v28.2s,v1.s[3]
	b.ne	.LInner_after_rewind8
	sub	x1,x1,x5,lsl#2	// rewind
.LInner_after_rewind8:
	umlal	v13.2d,v29.2s,v2.s[0]
	ld1	{v28.2s},[sp]		// pull smashed b[8*i+0]
	umlal	v6.2d,v29.2s,v2.s[1]
	ld1	{v0.4s,v1.4s},[x1],#32
	umlal	v7.2d,v29.2s,v2.s[2]
	add	x10,sp,#8		// rewind
	umlal	v8.2d,v29.2s,v2.s[3]
	umlal	v9.2d,v29.2s,v3.s[0]
	umlal	v10.2d,v29.2s,v3.s[1]
	umlal	v11.2d,v29.2s,v3.s[2]
	st1	{v13.2d},[x7],#16
	umlal	v12.2d,v29.2s,v3.s[3]

	bne	.LNEON_8n_inner
	add	x6,sp,#128
	st1	{v6.2d,v7.2d},[x7],#32
	eor	v2.16b,v2.16b,v2.16b	// v2
	st1	{v8.2d,v9.2d},[x7],#32
	eor	v3.16b,v3.16b,v3.16b	// v3
	st1	{v10.2d,v11.2d},[x7],#32
	st1	{v12.2d},[x7]

	subs	x9,x9,#8
	ld1	{v6.2d,v7.2d},[x6],#32
	ld1	{v8.2d,v9.2d},[x6],#32
	ld1	{v10.2d,v11.2d},[x6],#32
	ld1	{v12.2d,v13.2d},[x6],#32

	b.eq	.LInner_8n_jump_2steps
	sub	x3,x3,x5,lsl#2	// rewind
	b	.LNEON_8n_outer

.LInner_8n_jump_2steps:
	add	x7,sp,#128
	st1	{v2.2d,v3.2d}, [sp],#32	// start wiping stack frame
	mov	v5.16b,v6.16b
	ushr	v15.2d,v6.2d,#16
	ext	v6.16b,v6.16b,v6.16b,#8
	st1	{v2.2d,v3.2d}, [sp],#32
	add	v6.2d,v6.2d,v15.2d
	st1	{v2.2d,v3.2d}, [sp],#32
	ushr	v15.2d,v6.2d,#16
	st1	{v2.2d,v3.2d}, [sp],#32
	zip1	v6.4h,v5.4h,v6.4h
	ins	v15.d[1],v14.d[0]

	mov	x8,x5
	b	.LNEON_tail_entry

.align	4
.LNEON_tail:
	add	v6.2d,v6.2d,v15.2d
	mov	v5.16b,v6.16b
	ushr	v15.2d,v6.2d,#16
	ext	v6.16b,v6.16b,v6.16b,#8
	ld1	{v8.2d,v9.2d}, [x6],#32
	add	v6.2d,v6.2d,v15.2d
	ld1	{v10.2d,v11.2d}, [x6],#32
	ushr	v15.2d,v6.2d,#16
	ld1	{v12.2d,v13.2d}, [x6],#32
	zip1	v6.4h,v5.4h,v6.4h
	ins	v15.d[1],v14.d[0]

.LNEON_tail_entry:
	add	v7.2d,v7.2d,v15.2d
	st1	{v6.s}[0], [x7],#4
	ushr	v15.2d,v7.2d,#16
	mov	v5.16b,v7.16b
	ext	v7.16b,v7.16b,v7.16b,#8
	add	v7.2d,v7.2d,v15.2d
	ushr	v15.2d,v7.2d,#16
	zip1	v7.4h,v5.4h,v7.4h
	ins	v15.d[1],v14.d[0]
	add	v8.2d,v8.2d,v15.2d
	st1	{v7.s}[0], [x7],#4
	ushr	v15.2d,v8.2d,#16
	mov	v5.16b,v8.16b
	ext	v8.16b,v8.16b,v8.16b,#8
	add	v8.2d,v8.2d,v15.2d
	ushr	v15.2d,v8.2d,#16
	zip1	v8.4h,v5.4h,v8.4h
	ins	v15.d[1],v14.d[0]
	add	v9.2d,v9.2d,v15.2d
	st1	{v8.s}[0], [x7],#4
	ushr	v15.2d,v9.2d,#16
	mov	v5.16b,v9.16b
	ext	v9.16b,v9.16b,v9.16b,#8
	add	v9.2d,v9.2d,v15.2d
	ushr	v15.2d,v9.2d,#16
	zip1	v9.4h,v5.4h,v9.4h
	ins	v15.d[1],v14.d[0]
	add	v10.2d,v10.2d,v15.2d
	st1	{v9.s}[0], [x7],#4
	ushr	v15.2d,v10.2d,#16
	mov	v5.16b,v10.16b
	ext	v10.16b,v10.16b,v10.16b,#8
	add	v10.2d,v10.2d,v15.2d
	ushr	v15.2d,v10.2d,#16
	zip1	v10.4h,v5.4h,v10.4h
	ins	v15.d[1],v14.d[0]
	add	v11.2d,v11.2d,v15.2d
	st1	{v10.s}[0], [x7],#4
	ushr	v15.2d,v11.2d,#16
	mov	v5.16b,v11.16b
	ext	v11.16b,v11.16b,v11.16b,#8
	add	v11.2d,v11.2d,v15.2d
	ushr	v15.2d,v11.2d,#16
	zip1	v11.4h,v5.4h,v11.4h
	ins	v15.d[1],v14.d[0]
	add	v12.2d,v12.2d,v15.2d
	st1	{v11.s}[0], [x7],#4
	ushr	v15.2d,v12.2d,#16
	mov	v5.16b,v12.16b
	ext	v12.16b,v12.16b,v12.16b,#8
	add	v12.2d,v12.2d,v15.2d
	ushr	v15.2d,v12.2d,#16
	zip1	v12.4h,v5.4h,v12.4h
	ins	v15.d[1],v14.d[0]
	add	v13.2d,v13.2d,v15.2d
	st1	{v12.s}[0], [x7],#4
	ushr	v15.2d,v13.2d,#16
	mov	v5.16b,v13.16b
	ext	v13.16b,v13.16b,v13.16b,#8
	add	v13.2d,v13.2d,v15.2d
	ushr	v15.2d,v13.2d,#16
	zip1	v13.4h,v5.4h,v13.4h
	ins	v15.d[1],v14.d[0]
	ld1	{v6.2d,v7.2d}, [x6],#32
	subs	x8,x8,#8
	st1	{v13.s}[0], [x7],#4
	bne	.LNEON_tail

	st1	{v15.s}[0], [x7],#4	// top-most bit
	sub	x3,x3,x5,lsl#2		// rewind x3
	subs	x1,sp,#0			// clear carry flag
	add	x2,sp,x5,lsl#2

.LNEON_sub:
	ldp	w4,w5,[x1],#8
	ldp	w6,w7,[x1],#8
	ldp	w8,w9,[x3],#8
	ldp	w10,w11,[x3],#8
	sbcs	w8,w4,w8
	sbcs	w9,w5,w9
	sbcs	w10,w6,w10
	sbcs	w11,w7,w11
	sub	x17,x2,x1
	stp	w8,w9,[x0],#8
	stp	w10,w11,[x0],#8
	cbnz	x17,.LNEON_sub

	ldr	w10, [x1]		// load top-most bit
	mov	x11,sp
	eor	v0.16b,v0.16b,v0.16b
	sub	x11,x2,x11		// this is num*4
	eor	v1.16b,v1.16b,v1.16b
	mov	x1,sp
	sub	x0,x0,x11		// rewind x0
	mov	x3,x2		// second 3/4th of frame
	sbcs	w10,w10,wzr		// result is carry flag

.LNEON_copy_n_zap:
	ldp	w4,w5,[x1],#8
	ldp	w6,w7,[x1],#8
	ldp	w8,w9,[x0],#8
	ldp	w10,w11,[x0]
	sub	x0,x0,#8
	b.cs	.LCopy_1
	mov	w8,w4
	mov	w9,w5
	mov	w10,w6
	mov	w11,w7
.LCopy_1:
	st1	{v0.2d,v1.2d}, [x3],#32		// wipe
	st1	{v0.2d,v1.2d}, [x3],#32		// wipe
	ldp	w4,w5,[x1],#8
	ldp	w6,w7,[x1],#8
	stp	w8,w9,[x0],#8
	stp	w10,w11,[x0],#8
	sub	x1,x1,#32
	ldp	w8,w9,[x0],#8
	ldp	w10,w11,[x0]
	sub	x0,x0,#8
	b.cs	.LCopy_2
	mov	w8, w4
	mov	w9, w5
	mov	w10, w6
	mov	w11, w7
.LCopy_2:
	st1	{v0.2d,v1.2d}, [x1],#32		// wipe
	st1	{v0.2d,v1.2d}, [x3],#32		// wipe
	sub	x17,x2,x1		// preserves carry
	stp	w8,w9,[x0],#8
	stp	w10,w11,[x0],#8
	cbnz	x17,.LNEON_copy_n_zap

	mov	sp,x16
	ldp	d14,d15,[sp,#64]
	ldp	d12,d13,[sp,#48]
	ldp	d10,d11,[sp,#32]
	ldp	d8,d9,[sp,#16]
	ldr	x29,[sp],#80
	AARCH64_VALIDATE_LINK_REGISTER
	ret	// bx lr

.size	bn_mul8x_mont_neon,.-bn_mul8x_mont_neon
.type	__bn_sqr8x_mont,%function
.align	5
__bn_sqr8x_mont:
	cmp	x1,x2
	b.ne	__bn_mul4x_mont
.Lsqr8x_mont:
	// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to
	// only from bn_mul_mont which has already signed the return address.
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]
	stp	x0,x3,[sp,#96]	// offload rp and np

	ldp	x6,x7,[x1,#8*0]
	ldp	x8,x9,[x1,#8*2]
	ldp	x10,x11,[x1,#8*4]
	ldp	x12,x13,[x1,#8*6]

	sub	x2,sp,x5,lsl#4
	lsl	x5,x5,#3
	ldr	x4,[x4]		// *n0
	mov	sp,x2			// alloca
	sub	x27,x5,#8*8
	b	.Lsqr8x_zero_start

.Lsqr8x_zero:
	sub	x27,x27,#8*8
	stp	xzr,xzr,[x2,#8*0]
	stp	xzr,xzr,[x2,#8*2]
	stp	xzr,xzr,[x2,#8*4]
	stp	xzr,xzr,[x2,#8*6]
.Lsqr8x_zero_start:
	stp	xzr,xzr,[x2,#8*8]
	stp	xzr,xzr,[x2,#8*10]
	stp	xzr,xzr,[x2,#8*12]
	stp	xzr,xzr,[x2,#8*14]
	add	x2,x2,#8*16
	cbnz	x27,.Lsqr8x_zero

	add	x3,x1,x5
	add	x1,x1,#8*8
	mov	x19,xzr
	mov	x20,xzr
	mov	x21,xzr
	mov	x22,xzr
	mov	x23,xzr
	mov	x24,xzr
	mov	x25,xzr
	mov	x26,xzr
	mov	x2,sp
	str	x4,[x29,#112]		// offload n0

	// Multiply everything but a[i]*a[i]
.align	4
.Lsqr8x_outer_loop:
        //                                                 a[1]a[0]	(i)
        //                                             a[2]a[0]
        //                                         a[3]a[0]
        //                                     a[4]a[0]
        //                                 a[5]a[0]
        //                             a[6]a[0]
        //                         a[7]a[0]
        //                                         a[2]a[1]		(ii)
        //                                     a[3]a[1]
        //                                 a[4]a[1]
        //                             a[5]a[1]
        //                         a[6]a[1]
        //                     a[7]a[1]
        //                                 a[3]a[2]			(iii)
        //                             a[4]a[2]
        //                         a[5]a[2]
        //                     a[6]a[2]
        //                 a[7]a[2]
        //                         a[4]a[3]				(iv)
        //                     a[5]a[3]
        //                 a[6]a[3]
        //             a[7]a[3]
        //                 a[5]a[4]					(v)
        //             a[6]a[4]
        //         a[7]a[4]
        //         a[6]a[5]						(vi)
        //     a[7]a[5]
        // a[7]a[6]							(vii)

	mul	x14,x7,x6		// lo(a[1..7]*a[0])		(i)
	mul	x15,x8,x6
	mul	x16,x9,x6
	mul	x17,x10,x6
	adds	x20,x20,x14		// t[1]+lo(a[1]*a[0])
	mul	x14,x11,x6
	adcs	x21,x21,x15
	mul	x15,x12,x6
	adcs	x22,x22,x16
	mul	x16,x13,x6
	adcs	x23,x23,x17
	umulh	x17,x7,x6		// hi(a[1..7]*a[0])
	adcs	x24,x24,x14
	umulh	x14,x8,x6
	adcs	x25,x25,x15
	umulh	x15,x9,x6
	adcs	x26,x26,x16
	umulh	x16,x10,x6
	stp	x19,x20,[x2],#8*2	// t[0..1]
	adc	x19,xzr,xzr		// t[8]
	adds	x21,x21,x17		// t[2]+lo(a[1]*a[0])
	umulh	x17,x11,x6
	adcs	x22,x22,x14
	umulh	x14,x12,x6
	adcs	x23,x23,x15
	umulh	x15,x13,x6
	adcs	x24,x24,x16
	mul	x16,x8,x7		// lo(a[2..7]*a[1])		(ii)
	adcs	x25,x25,x17
	mul	x17,x9,x7
	adcs	x26,x26,x14
	mul	x14,x10,x7
	adc	x19,x19,x15

	mul	x15,x11,x7
	adds	x22,x22,x16
	mul	x16,x12,x7
	adcs	x23,x23,x17
	mul	x17,x13,x7
	adcs	x24,x24,x14
	umulh	x14,x8,x7		// hi(a[2..7]*a[1])
	adcs	x25,x25,x15
	umulh	x15,x9,x7
	adcs	x26,x26,x16
	umulh	x16,x10,x7
	adcs	x19,x19,x17
	umulh	x17,x11,x7
	stp	x21,x22,[x2],#8*2	// t[2..3]
	adc	x20,xzr,xzr		// t[9]
	adds	x23,x23,x14
	umulh	x14,x12,x7
	adcs	x24,x24,x15
	umulh	x15,x13,x7
	adcs	x25,x25,x16
	mul	x16,x9,x8		// lo(a[3..7]*a[2])		(iii)
	adcs	x26,x26,x17
	mul	x17,x10,x8
	adcs	x19,x19,x14
	mul	x14,x11,x8
	adc	x20,x20,x15

	mul	x15,x12,x8
	adds	x24,x24,x16
	mul	x16,x13,x8
	adcs	x25,x25,x17
	umulh	x17,x9,x8		// hi(a[3..7]*a[2])
	adcs	x26,x26,x14
	umulh	x14,x10,x8
	adcs	x19,x19,x15
	umulh	x15,x11,x8
	adcs	x20,x20,x16
	umulh	x16,x12,x8
	stp	x23,x24,[x2],#8*2	// t[4..5]
	adc	x21,xzr,xzr		// t[10]
	adds	x25,x25,x17
	umulh	x17,x13,x8
	adcs	x26,x26,x14
	mul	x14,x10,x9		// lo(a[4..7]*a[3])		(iv)
	adcs	x19,x19,x15
	mul	x15,x11,x9
	adcs	x20,x20,x16
	mul	x16,x12,x9
	adc	x21,x21,x17

	mul	x17,x13,x9
	adds	x26,x26,x14
	umulh	x14,x10,x9		// hi(a[4..7]*a[3])
	adcs	x19,x19,x15
	umulh	x15,x11,x9
	adcs	x20,x20,x16
	umulh	x16,x12,x9
	adcs	x21,x21,x17
	umulh	x17,x13,x9
	stp	x25,x26,[x2],#8*2	// t[6..7]
	adc	x22,xzr,xzr		// t[11]
	adds	x19,x19,x14
	mul	x14,x11,x10		// lo(a[5..7]*a[4])		(v)
	adcs	x20,x20,x15
	mul	x15,x12,x10
	adcs	x21,x21,x16
	mul	x16,x13,x10
	adc	x22,x22,x17

	umulh	x17,x11,x10		// hi(a[5..7]*a[4])
	adds	x20,x20,x14
	umulh	x14,x12,x10
	adcs	x21,x21,x15
	umulh	x15,x13,x10
	adcs	x22,x22,x16
	mul	x16,x12,x11		// lo(a[6..7]*a[5])		(vi)
	adc	x23,xzr,xzr		// t[12]
	adds	x21,x21,x17
	mul	x17,x13,x11
	adcs	x22,x22,x14
	umulh	x14,x12,x11		// hi(a[6..7]*a[5])
	adc	x23,x23,x15

	umulh	x15,x13,x11
	adds	x22,x22,x16
	mul	x16,x13,x12		// lo(a[7]*a[6])		(vii)
	adcs	x23,x23,x17
	umulh	x17,x13,x12		// hi(a[7]*a[6])
	adc	x24,xzr,xzr		// t[13]
	adds	x23,x23,x14
	sub	x27,x3,x1	// done yet?
	adc	x24,x24,x15

	adds	x24,x24,x16
	sub	x14,x3,x5	// rewinded ap
	adc	x25,xzr,xzr		// t[14]
	add	x25,x25,x17

	cbz	x27,.Lsqr8x_outer_break

	mov	x4,x6
	ldp	x6,x7,[x2,#8*0]
	ldp	x8,x9,[x2,#8*2]
	ldp	x10,x11,[x2,#8*4]
	ldp	x12,x13,[x2,#8*6]
	adds	x19,x19,x6
	adcs	x20,x20,x7
	ldp	x6,x7,[x1,#8*0]
	adcs	x21,x21,x8
	adcs	x22,x22,x9
	ldp	x8,x9,[x1,#8*2]
	adcs	x23,x23,x10
	adcs	x24,x24,x11
	ldp	x10,x11,[x1,#8*4]
	adcs	x25,x25,x12
	mov	x0,x1
	adcs	x26,xzr,x13
	ldp	x12,x13,[x1,#8*6]
	add	x1,x1,#8*8
	//adc	x28,xzr,xzr		// moved below
	mov	x27,#-8*8

	//                                                         a[8]a[0]
	//                                                     a[9]a[0]
	//                                                 a[a]a[0]
	//                                             a[b]a[0]
	//                                         a[c]a[0]
	//                                     a[d]a[0]
	//                                 a[e]a[0]
	//                             a[f]a[0]
	//                                                     a[8]a[1]
	//                         a[f]a[1]........................
	//                                                 a[8]a[2]
	//                     a[f]a[2]........................
	//                                             a[8]a[3]
	//                 a[f]a[3]........................
	//                                         a[8]a[4]
	//             a[f]a[4]........................
	//                                     a[8]a[5]
	//         a[f]a[5]........................
	//                                 a[8]a[6]
	//     a[f]a[6]........................
	//                             a[8]a[7]
	// a[f]a[7]........................
.Lsqr8x_mul:
	mul	x14,x6,x4
	adc	x28,xzr,xzr		// carry bit, modulo-scheduled
	mul	x15,x7,x4
	add	x27,x27,#8
	mul	x16,x8,x4
	mul	x17,x9,x4
	adds	x19,x19,x14
	mul	x14,x10,x4
	adcs	x20,x20,x15
	mul	x15,x11,x4
	adcs	x21,x21,x16
	mul	x16,x12,x4
	adcs	x22,x22,x17
	mul	x17,x13,x4
	adcs	x23,x23,x14
	umulh	x14,x6,x4
	adcs	x24,x24,x15
	umulh	x15,x7,x4
	adcs	x25,x25,x16
	umulh	x16,x8,x4
	adcs	x26,x26,x17
	umulh	x17,x9,x4
	adc	x28,x28,xzr
	str	x19,[x2],#8
	adds	x19,x20,x14
	umulh	x14,x10,x4
	adcs	x20,x21,x15
	umulh	x15,x11,x4
	adcs	x21,x22,x16
	umulh	x16,x12,x4
	adcs	x22,x23,x17
	umulh	x17,x13,x4
	ldr	x4,[x0,x27]
	adcs	x23,x24,x14
	adcs	x24,x25,x15
	adcs	x25,x26,x16
	adcs	x26,x28,x17
	//adc	x28,xzr,xzr		// moved above
	cbnz	x27,.Lsqr8x_mul
					// note that carry flag is guaranteed
					// to be zero at this point
	cmp	x1,x3		// done yet?
	b.eq	.Lsqr8x_break

	ldp	x6,x7,[x2,#8*0]
	ldp	x8,x9,[x2,#8*2]
	ldp	x10,x11,[x2,#8*4]
	ldp	x12,x13,[x2,#8*6]
	adds	x19,x19,x6
	ldur	x4,[x0,#-8*8]
	adcs	x20,x20,x7
	ldp	x6,x7,[x1,#8*0]
	adcs	x21,x21,x8
	adcs	x22,x22,x9
	ldp	x8,x9,[x1,#8*2]
	adcs	x23,x23,x10
	adcs	x24,x24,x11
	ldp	x10,x11,[x1,#8*4]
	adcs	x25,x25,x12
	mov	x27,#-8*8
	adcs	x26,x26,x13
	ldp	x12,x13,[x1,#8*6]
	add	x1,x1,#8*8
	//adc	x28,xzr,xzr		// moved above
	b	.Lsqr8x_mul

.align	4
.Lsqr8x_break:
	ldp	x6,x7,[x0,#8*0]
	add	x1,x0,#8*8
	ldp	x8,x9,[x0,#8*2]
	sub	x14,x3,x1		// is it last iteration?
	ldp	x10,x11,[x0,#8*4]
	sub	x15,x2,x14
	ldp	x12,x13,[x0,#8*6]
	cbz	x14,.Lsqr8x_outer_loop

	stp	x19,x20,[x2,#8*0]
	ldp	x19,x20,[x15,#8*0]
	stp	x21,x22,[x2,#8*2]
	ldp	x21,x22,[x15,#8*2]
	stp	x23,x24,[x2,#8*4]
	ldp	x23,x24,[x15,#8*4]
	stp	x25,x26,[x2,#8*6]
	mov	x2,x15
	ldp	x25,x26,[x15,#8*6]
	b	.Lsqr8x_outer_loop

.align	4
.Lsqr8x_outer_break:
	// Now multiply above result by 2 and add a[n-1]*a[n-1]|...|a[0]*a[0]
	ldp	x7,x9,[x14,#8*0]	// recall that x14 is &a[0]
	ldp	x15,x16,[sp,#8*1]
	ldp	x11,x13,[x14,#8*2]
	add	x1,x14,#8*4
	ldp	x17,x14,[sp,#8*3]

	stp	x19,x20,[x2,#8*0]
	mul	x19,x7,x7
	stp	x21,x22,[x2,#8*2]
	umulh	x7,x7,x7
	stp	x23,x24,[x2,#8*4]
	mul	x8,x9,x9
	stp	x25,x26,[x2,#8*6]
	mov	x2,sp
	umulh	x9,x9,x9
	adds	x20,x7,x15,lsl#1
	extr	x15,x16,x15,#63
	sub	x27,x5,#8*4

.Lsqr4x_shift_n_add:
	adcs	x21,x8,x15
	extr	x16,x17,x16,#63
	sub	x27,x27,#8*4
	adcs	x22,x9,x16
	ldp	x15,x16,[x2,#8*5]
	mul	x10,x11,x11
	ldp	x7,x9,[x1],#8*2
	umulh	x11,x11,x11
	mul	x12,x13,x13
	umulh	x13,x13,x13
	extr	x17,x14,x17,#63
	stp	x19,x20,[x2,#8*0]
	adcs	x23,x10,x17
	extr	x14,x15,x14,#63
	stp	x21,x22,[x2,#8*2]
	adcs	x24,x11,x14
	ldp	x17,x14,[x2,#8*7]
	extr	x15,x16,x15,#63
	adcs	x25,x12,x15
	extr	x16,x17,x16,#63
	adcs	x26,x13,x16
	ldp	x15,x16,[x2,#8*9]
	mul	x6,x7,x7
	ldp	x11,x13,[x1],#8*2
	umulh	x7,x7,x7
	mul	x8,x9,x9
	umulh	x9,x9,x9
	stp	x23,x24,[x2,#8*4]
	extr	x17,x14,x17,#63
	stp	x25,x26,[x2,#8*6]
	add	x2,x2,#8*8
	adcs	x19,x6,x17
	extr	x14,x15,x14,#63
	adcs	x20,x7,x14
	ldp	x17,x14,[x2,#8*3]
	extr	x15,x16,x15,#63
	cbnz	x27,.Lsqr4x_shift_n_add
	ldp	x1,x4,[x29,#104]	// pull np and n0

	adcs	x21,x8,x15
	extr	x16,x17,x16,#63
	adcs	x22,x9,x16
	ldp	x15,x16,[x2,#8*5]
	mul	x10,x11,x11
	umulh	x11,x11,x11
	stp	x19,x20,[x2,#8*0]
	mul	x12,x13,x13
	umulh	x13,x13,x13
	stp	x21,x22,[x2,#8*2]
	extr	x17,x14,x17,#63
	adcs	x23,x10,x17
	extr	x14,x15,x14,#63
	ldp	x19,x20,[sp,#8*0]
	adcs	x24,x11,x14
	extr	x15,x16,x15,#63
	ldp	x6,x7,[x1,#8*0]
	adcs	x25,x12,x15
	extr	x16,xzr,x16,#63
	ldp	x8,x9,[x1,#8*2]
	adc	x26,x13,x16
	ldp	x10,x11,[x1,#8*4]

	// Reduce by 512 bits per iteration
	mul	x28,x4,x19		// t[0]*n0
	ldp	x12,x13,[x1,#8*6]
	add	x3,x1,x5
	ldp	x21,x22,[sp,#8*2]
	stp	x23,x24,[x2,#8*4]
	ldp	x23,x24,[sp,#8*4]
	stp	x25,x26,[x2,#8*6]
	ldp	x25,x26,[sp,#8*6]
	add	x1,x1,#8*8
	mov	x30,xzr		// initial top-most carry
	mov	x2,sp
	mov	x27,#8

.Lsqr8x_reduction:
	// (*)	mul	x14,x6,x28	// lo(n[0-7])*lo(t[0]*n0)
	mul	x15,x7,x28
	sub	x27,x27,#1
	mul	x16,x8,x28
	str	x28,[x2],#8		// put aside t[0]*n0 for tail processing
	mul	x17,x9,x28
	// (*)	adds	xzr,x19,x14
	subs	xzr,x19,#1		// (*)
	mul	x14,x10,x28
	adcs	x19,x20,x15
	mul	x15,x11,x28
	adcs	x20,x21,x16
	mul	x16,x12,x28
	adcs	x21,x22,x17
	mul	x17,x13,x28
	adcs	x22,x23,x14
	umulh	x14,x6,x28		// hi(n[0-7])*lo(t[0]*n0)
	adcs	x23,x24,x15
	umulh	x15,x7,x28
	adcs	x24,x25,x16
	umulh	x16,x8,x28
	adcs	x25,x26,x17
	umulh	x17,x9,x28
	adc	x26,xzr,xzr
	adds	x19,x19,x14
	umulh	x14,x10,x28
	adcs	x20,x20,x15
	umulh	x15,x11,x28
	adcs	x21,x21,x16
	umulh	x16,x12,x28
	adcs	x22,x22,x17
	umulh	x17,x13,x28
	mul	x28,x4,x19		// next t[0]*n0
	adcs	x23,x23,x14
	adcs	x24,x24,x15
	adcs	x25,x25,x16
	adc	x26,x26,x17
	cbnz	x27,.Lsqr8x_reduction

	ldp	x14,x15,[x2,#8*0]
	ldp	x16,x17,[x2,#8*2]
	mov	x0,x2
	sub	x27,x3,x1	// done yet?
	adds	x19,x19,x14
	adcs	x20,x20,x15
	ldp	x14,x15,[x2,#8*4]
	adcs	x21,x21,x16
	adcs	x22,x22,x17
	ldp	x16,x17,[x2,#8*6]
	adcs	x23,x23,x14
	adcs	x24,x24,x15
	adcs	x25,x25,x16
	adcs	x26,x26,x17
	//adc	x28,xzr,xzr		// moved below
	cbz	x27,.Lsqr8x8_post_condition

	ldur	x4,[x2,#-8*8]
	ldp	x6,x7,[x1,#8*0]
	ldp	x8,x9,[x1,#8*2]
	ldp	x10,x11,[x1,#8*4]
	mov	x27,#-8*8
	ldp	x12,x13,[x1,#8*6]
	add	x1,x1,#8*8

.Lsqr8x_tail:
	mul	x14,x6,x4
	adc	x28,xzr,xzr		// carry bit, modulo-scheduled
	mul	x15,x7,x4
	add	x27,x27,#8
	mul	x16,x8,x4
	mul	x17,x9,x4
	adds	x19,x19,x14
	mul	x14,x10,x4
	adcs	x20,x20,x15
	mul	x15,x11,x4
	adcs	x21,x21,x16
	mul	x16,x12,x4
	adcs	x22,x22,x17
	mul	x17,x13,x4
	adcs	x23,x23,x14
	umulh	x14,x6,x4
	adcs	x24,x24,x15
	umulh	x15,x7,x4
	adcs	x25,x25,x16
	umulh	x16,x8,x4
	adcs	x26,x26,x17
	umulh	x17,x9,x4
	adc	x28,x28,xzr
	str	x19,[x2],#8
	adds	x19,x20,x14
	umulh	x14,x10,x4
	adcs	x20,x21,x15
	umulh	x15,x11,x4
	adcs	x21,x22,x16
	umulh	x16,x12,x4
	adcs	x22,x23,x17
	umulh	x17,x13,x4
	ldr	x4,[x0,x27]
	adcs	x23,x24,x14
	adcs	x24,x25,x15
	adcs	x25,x26,x16
	adcs	x26,x28,x17
	//adc	x28,xzr,xzr		// moved above
	cbnz	x27,.Lsqr8x_tail
					// note that carry flag is guaranteed
					// to be zero at this point
	ldp	x6,x7,[x2,#8*0]
	sub	x27,x3,x1	// done yet?
	sub	x16,x3,x5	// rewinded np
	ldp	x8,x9,[x2,#8*2]
	ldp	x10,x11,[x2,#8*4]
	ldp	x12,x13,[x2,#8*6]
	cbz	x27,.Lsqr8x_tail_break

	ldur	x4,[x0,#-8*8]
	adds	x19,x19,x6
	adcs	x20,x20,x7
	ldp	x6,x7,[x1,#8*0]
	adcs	x21,x21,x8
	adcs	x22,x22,x9
	ldp	x8,x9,[x1,#8*2]
	adcs	x23,x23,x10
	adcs	x24,x24,x11
	ldp	x10,x11,[x1,#8*4]
	adcs	x25,x25,x12
	mov	x27,#-8*8
	adcs	x26,x26,x13
	ldp	x12,x13,[x1,#8*6]
	add	x1,x1,#8*8
	//adc	x28,xzr,xzr		// moved above
	b	.Lsqr8x_tail

.align	4
.Lsqr8x_tail_break:
	ldr	x4,[x29,#112]		// pull n0
	add	x27,x2,#8*8		// end of current t[num] window

	subs	xzr,x30,#1		// "move" top-most carry to carry bit
	adcs	x14,x19,x6
	adcs	x15,x20,x7
	ldp	x19,x20,[x0,#8*0]
	adcs	x21,x21,x8
	ldp	x6,x7,[x16,#8*0]	// recall that x16 is &n[0]
	adcs	x22,x22,x9
	ldp	x8,x9,[x16,#8*2]
	adcs	x23,x23,x10
	adcs	x24,x24,x11
	ldp	x10,x11,[x16,#8*4]
	adcs	x25,x25,x12
	adcs	x26,x26,x13
	ldp	x12,x13,[x16,#8*6]
	add	x1,x16,#8*8
	adc	x30,xzr,xzr	// top-most carry
	mul	x28,x4,x19
	stp	x14,x15,[x2,#8*0]
	stp	x21,x22,[x2,#8*2]
	ldp	x21,x22,[x0,#8*2]
	stp	x23,x24,[x2,#8*4]
	ldp	x23,x24,[x0,#8*4]
	cmp	x27,x29		// did we hit the bottom?
	stp	x25,x26,[x2,#8*6]
	mov	x2,x0			// slide the window
	ldp	x25,x26,[x0,#8*6]
	mov	x27,#8
	b.ne	.Lsqr8x_reduction

	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	ldr	x0,[x29,#96]		// pull rp
	add	x2,x2,#8*8
	subs	x14,x19,x6
	sbcs	x15,x20,x7
	sub	x27,x5,#8*8
	mov	x3,x0		// x0 copy

.Lsqr8x_sub:
	sbcs	x16,x21,x8
	ldp	x6,x7,[x1,#8*0]
	sbcs	x17,x22,x9
	stp	x14,x15,[x0,#8*0]
	sbcs	x14,x23,x10
	ldp	x8,x9,[x1,#8*2]
	sbcs	x15,x24,x11
	stp	x16,x17,[x0,#8*2]
	sbcs	x16,x25,x12
	ldp	x10,x11,[x1,#8*4]
	sbcs	x17,x26,x13
	ldp	x12,x13,[x1,#8*6]
	add	x1,x1,#8*8
	ldp	x19,x20,[x2,#8*0]
	sub	x27,x27,#8*8
	ldp	x21,x22,[x2,#8*2]
	ldp	x23,x24,[x2,#8*4]
	ldp	x25,x26,[x2,#8*6]
	add	x2,x2,#8*8
	stp	x14,x15,[x0,#8*4]
	sbcs	x14,x19,x6
	stp	x16,x17,[x0,#8*6]
	add	x0,x0,#8*8
	sbcs	x15,x20,x7
	cbnz	x27,.Lsqr8x_sub

	sbcs	x16,x21,x8
	mov	x2,sp
	add	x1,sp,x5
	ldp	x6,x7,[x3,#8*0]
	sbcs	x17,x22,x9
	stp	x14,x15,[x0,#8*0]
	sbcs	x14,x23,x10
	ldp	x8,x9,[x3,#8*2]
	sbcs	x15,x24,x11
	stp	x16,x17,[x0,#8*2]
	sbcs	x16,x25,x12
	ldp	x19,x20,[x1,#8*0]
	sbcs	x17,x26,x13
	ldp	x21,x22,[x1,#8*2]
	sbcs	xzr,x30,xzr	// did it borrow?
	ldr	x30,[x29,#8]		// pull return address
	stp	x14,x15,[x0,#8*4]
	stp	x16,x17,[x0,#8*6]

	sub	x27,x5,#8*4
.Lsqr4x_cond_copy:
	sub	x27,x27,#8*4
	csel	x14,x19,x6,lo
	stp	xzr,xzr,[x2,#8*0]
	csel	x15,x20,x7,lo
	ldp	x6,x7,[x3,#8*4]
	ldp	x19,x20,[x1,#8*4]
	csel	x16,x21,x8,lo
	stp	xzr,xzr,[x2,#8*2]
	add	x2,x2,#8*4
	csel	x17,x22,x9,lo
	ldp	x8,x9,[x3,#8*6]
	ldp	x21,x22,[x1,#8*6]
	add	x1,x1,#8*4
	stp	x14,x15,[x3,#8*0]
	stp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4
	stp	xzr,xzr,[x1,#8*0]
	stp	xzr,xzr,[x1,#8*2]
	cbnz	x27,.Lsqr4x_cond_copy

	csel	x14,x19,x6,lo
	stp	xzr,xzr,[x2,#8*0]
	csel	x15,x20,x7,lo
	stp	xzr,xzr,[x2,#8*2]
	csel	x16,x21,x8,lo
	csel	x17,x22,x9,lo
	stp	x14,x15,[x3,#8*0]
	stp	x16,x17,[x3,#8*2]

	b	.Lsqr8x_done

.align	4
.Lsqr8x8_post_condition:
	adc	x28,xzr,xzr
	ldr	x30,[x29,#8]		// pull return address
	// x19-7,x28 hold result, x6-7 hold modulus
	subs	x6,x19,x6
	ldr	x1,[x29,#96]		// pull rp
	sbcs	x7,x20,x7
	stp	xzr,xzr,[sp,#8*0]
	sbcs	x8,x21,x8
	stp	xzr,xzr,[sp,#8*2]
	sbcs	x9,x22,x9
	stp	xzr,xzr,[sp,#8*4]
	sbcs	x10,x23,x10
	stp	xzr,xzr,[sp,#8*6]
	sbcs	x11,x24,x11
	stp	xzr,xzr,[sp,#8*8]
	sbcs	x12,x25,x12
	stp	xzr,xzr,[sp,#8*10]
	sbcs	x13,x26,x13
	stp	xzr,xzr,[sp,#8*12]
	sbcs	x28,x28,xzr	// did it borrow?
	stp	xzr,xzr,[sp,#8*14]

	// x6-7 hold result-modulus
	csel	x6,x19,x6,lo
	csel	x7,x20,x7,lo
	csel	x8,x21,x8,lo
	csel	x9,x22,x9,lo
	stp	x6,x7,[x1,#8*0]
	csel	x10,x23,x10,lo
	csel	x11,x24,x11,lo
	stp	x8,x9,[x1,#8*2]
	csel	x12,x25,x12,lo
	csel	x13,x26,x13,lo
	stp	x10,x11,[x1,#8*4]
	stp	x12,x13,[x1,#8*6]

.Lsqr8x_done:
	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldr	x29,[sp],#128
	// x30 is loaded earlier
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	__bn_sqr8x_mont,.-__bn_sqr8x_mont
.type	__bn_mul4x_mont,%function
.align	5
__bn_mul4x_mont:
	// Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to
	// only from bn_mul_mont (or __bn_sqr8x_mont from bn_mul_mont) which has already signed the return address.
	stp	x29,x30,[sp,#-128]!
	add	x29,sp,#0
	stp	x19,x20,[sp,#16]
	stp	x21,x22,[sp,#32]
	stp	x23,x24,[sp,#48]
	stp	x25,x26,[sp,#64]
	stp	x27,x28,[sp,#80]

	sub	x26,sp,x5,lsl#3
	lsl	x5,x5,#3
	ldr	x4,[x4]		// *n0
	sub	sp,x26,#8*4		// alloca

	add	x10,x2,x5
	add	x27,x1,x5
	stp	x0,x10,[x29,#96]	// offload rp and &b[num]

	ldr	x24,[x2,#8*0]		// b[0]
	ldp	x6,x7,[x1,#8*0]	// a[0..3]
	ldp	x8,x9,[x1,#8*2]
	add	x1,x1,#8*4
	mov	x19,xzr
	mov	x20,xzr
	mov	x21,xzr
	mov	x22,xzr
	ldp	x14,x15,[x3,#8*0]	// n[0..3]
	ldp	x16,x17,[x3,#8*2]
	adds	x3,x3,#8*4		// clear carry bit
	mov	x0,xzr
	mov	x28,#0
	mov	x26,sp

.Loop_mul4x_1st_reduction:
	mul	x10,x6,x24		// lo(a[0..3]*b[0])
	adc	x0,x0,xzr	// modulo-scheduled
	mul	x11,x7,x24
	add	x28,x28,#8
	mul	x12,x8,x24
	and	x28,x28,#31
	mul	x13,x9,x24
	adds	x19,x19,x10
	umulh	x10,x6,x24		// hi(a[0..3]*b[0])
	adcs	x20,x20,x11
	mul	x25,x19,x4		// t[0]*n0
	adcs	x21,x21,x12
	umulh	x11,x7,x24
	adcs	x22,x22,x13
	umulh	x12,x8,x24
	adc	x23,xzr,xzr
	umulh	x13,x9,x24
	ldr	x24,[x2,x28]		// next b[i] (or b[0])
	adds	x20,x20,x10
	// (*)	mul	x10,x14,x25	// lo(n[0..3]*t[0]*n0)
	str	x25,[x26],#8		// put aside t[0]*n0 for tail processing
	adcs	x21,x21,x11
	mul	x11,x15,x25
	adcs	x22,x22,x12
	mul	x12,x16,x25
	adc	x23,x23,x13		// can't overflow
	mul	x13,x17,x25
	// (*)	adds	xzr,x19,x10
	subs	xzr,x19,#1		// (*)
	umulh	x10,x14,x25		// hi(n[0..3]*t[0]*n0)
	adcs	x19,x20,x11
	umulh	x11,x15,x25
	adcs	x20,x21,x12
	umulh	x12,x16,x25
	adcs	x21,x22,x13
	umulh	x13,x17,x25
	adcs	x22,x23,x0
	adc	x0,xzr,xzr
	adds	x19,x19,x10
	sub	x10,x27,x1
	adcs	x20,x20,x11
	adcs	x21,x21,x12
	adcs	x22,x22,x13
	//adc	x0,x0,xzr
	cbnz	x28,.Loop_mul4x_1st_reduction

	cbz	x10,.Lmul4x4_post_condition

	ldp	x6,x7,[x1,#8*0]	// a[4..7]
	ldp	x8,x9,[x1,#8*2]
	add	x1,x1,#8*4
	ldr	x25,[sp]		// a[0]*n0
	ldp	x14,x15,[x3,#8*0]	// n[4..7]
	ldp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4

.Loop_mul4x_1st_tail:
	mul	x10,x6,x24		// lo(a[4..7]*b[i])
	adc	x0,x0,xzr	// modulo-scheduled
	mul	x11,x7,x24
	add	x28,x28,#8
	mul	x12,x8,x24
	and	x28,x28,#31
	mul	x13,x9,x24
	adds	x19,x19,x10
	umulh	x10,x6,x24		// hi(a[4..7]*b[i])
	adcs	x20,x20,x11
	umulh	x11,x7,x24
	adcs	x21,x21,x12
	umulh	x12,x8,x24
	adcs	x22,x22,x13
	umulh	x13,x9,x24
	adc	x23,xzr,xzr
	ldr	x24,[x2,x28]		// next b[i] (or b[0])
	adds	x20,x20,x10
	mul	x10,x14,x25		// lo(n[4..7]*a[0]*n0)
	adcs	x21,x21,x11
	mul	x11,x15,x25
	adcs	x22,x22,x12
	mul	x12,x16,x25
	adc	x23,x23,x13		// can't overflow
	mul	x13,x17,x25
	adds	x19,x19,x10
	umulh	x10,x14,x25		// hi(n[4..7]*a[0]*n0)
	adcs	x20,x20,x11
	umulh	x11,x15,x25
	adcs	x21,x21,x12
	umulh	x12,x16,x25
	adcs	x22,x22,x13
	adcs	x23,x23,x0
	umulh	x13,x17,x25
	adc	x0,xzr,xzr
	ldr	x25,[sp,x28]		// next t[0]*n0
	str	x19,[x26],#8		// result!!!
	adds	x19,x20,x10
	sub	x10,x27,x1		// done yet?
	adcs	x20,x21,x11
	adcs	x21,x22,x12
	adcs	x22,x23,x13
	//adc	x0,x0,xzr
	cbnz	x28,.Loop_mul4x_1st_tail

	sub	x11,x27,x5	// rewinded x1
	cbz	x10,.Lmul4x_proceed

	ldp	x6,x7,[x1,#8*0]
	ldp	x8,x9,[x1,#8*2]
	add	x1,x1,#8*4
	ldp	x14,x15,[x3,#8*0]
	ldp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4
	b	.Loop_mul4x_1st_tail

.align	5
.Lmul4x_proceed:
	ldr	x24,[x2,#8*4]!		// *++b
	adc	x30,x0,xzr
	ldp	x6,x7,[x11,#8*0]	// a[0..3]
	sub	x3,x3,x5		// rewind np
	ldp	x8,x9,[x11,#8*2]
	add	x1,x11,#8*4

	stp	x19,x20,[x26,#8*0]	// result!!!
	ldp	x19,x20,[sp,#8*4]	// t[0..3]
	stp	x21,x22,[x26,#8*2]	// result!!!
	ldp	x21,x22,[sp,#8*6]

	ldp	x14,x15,[x3,#8*0]	// n[0..3]
	mov	x26,sp
	ldp	x16,x17,[x3,#8*2]
	adds	x3,x3,#8*4		// clear carry bit
	mov	x0,xzr

.align	4
.Loop_mul4x_reduction:
	mul	x10,x6,x24		// lo(a[0..3]*b[4])
	adc	x0,x0,xzr	// modulo-scheduled
	mul	x11,x7,x24
	add	x28,x28,#8
	mul	x12,x8,x24
	and	x28,x28,#31
	mul	x13,x9,x24
	adds	x19,x19,x10
	umulh	x10,x6,x24		// hi(a[0..3]*b[4])
	adcs	x20,x20,x11
	mul	x25,x19,x4		// t[0]*n0
	adcs	x21,x21,x12
	umulh	x11,x7,x24
	adcs	x22,x22,x13
	umulh	x12,x8,x24
	adc	x23,xzr,xzr
	umulh	x13,x9,x24
	ldr	x24,[x2,x28]		// next b[i]
	adds	x20,x20,x10
	// (*)	mul	x10,x14,x25
	str	x25,[x26],#8		// put aside t[0]*n0 for tail processing
	adcs	x21,x21,x11
	mul	x11,x15,x25		// lo(n[0..3]*t[0]*n0
	adcs	x22,x22,x12
	mul	x12,x16,x25
	adc	x23,x23,x13		// can't overflow
	mul	x13,x17,x25
	// (*)	adds	xzr,x19,x10
	subs	xzr,x19,#1		// (*)
	umulh	x10,x14,x25		// hi(n[0..3]*t[0]*n0
	adcs	x19,x20,x11
	umulh	x11,x15,x25
	adcs	x20,x21,x12
	umulh	x12,x16,x25
	adcs	x21,x22,x13
	umulh	x13,x17,x25
	adcs	x22,x23,x0
	adc	x0,xzr,xzr
	adds	x19,x19,x10
	adcs	x20,x20,x11
	adcs	x21,x21,x12
	adcs	x22,x22,x13
	//adc	x0,x0,xzr
	cbnz	x28,.Loop_mul4x_reduction

	adc	x0,x0,xzr
	ldp	x10,x11,[x26,#8*4]	// t[4..7]
	ldp	x12,x13,[x26,#8*6]
	ldp	x6,x7,[x1,#8*0]	// a[4..7]
	ldp	x8,x9,[x1,#8*2]
	add	x1,x1,#8*4
	adds	x19,x19,x10
	adcs	x20,x20,x11
	adcs	x21,x21,x12
	adcs	x22,x22,x13
	//adc	x0,x0,xzr

	ldr	x25,[sp]		// t[0]*n0
	ldp	x14,x15,[x3,#8*0]	// n[4..7]
	ldp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4

.align	4
.Loop_mul4x_tail:
	mul	x10,x6,x24		// lo(a[4..7]*b[4])
	adc	x0,x0,xzr	// modulo-scheduled
	mul	x11,x7,x24
	add	x28,x28,#8
	mul	x12,x8,x24
	and	x28,x28,#31
	mul	x13,x9,x24
	adds	x19,x19,x10
	umulh	x10,x6,x24		// hi(a[4..7]*b[4])
	adcs	x20,x20,x11
	umulh	x11,x7,x24
	adcs	x21,x21,x12
	umulh	x12,x8,x24
	adcs	x22,x22,x13
	umulh	x13,x9,x24
	adc	x23,xzr,xzr
	ldr	x24,[x2,x28]		// next b[i]
	adds	x20,x20,x10
	mul	x10,x14,x25		// lo(n[4..7]*t[0]*n0)
	adcs	x21,x21,x11
	mul	x11,x15,x25
	adcs	x22,x22,x12
	mul	x12,x16,x25
	adc	x23,x23,x13		// can't overflow
	mul	x13,x17,x25
	adds	x19,x19,x10
	umulh	x10,x14,x25		// hi(n[4..7]*t[0]*n0)
	adcs	x20,x20,x11
	umulh	x11,x15,x25
	adcs	x21,x21,x12
	umulh	x12,x16,x25
	adcs	x22,x22,x13
	umulh	x13,x17,x25
	adcs	x23,x23,x0
	ldr	x25,[sp,x28]		// next a[0]*n0
	adc	x0,xzr,xzr
	str	x19,[x26],#8		// result!!!
	adds	x19,x20,x10
	sub	x10,x27,x1		// done yet?
	adcs	x20,x21,x11
	adcs	x21,x22,x12
	adcs	x22,x23,x13
	//adc	x0,x0,xzr
	cbnz	x28,.Loop_mul4x_tail

	sub	x11,x3,x5		// rewinded np?
	adc	x0,x0,xzr
	cbz	x10,.Loop_mul4x_break

	ldp	x10,x11,[x26,#8*4]
	ldp	x12,x13,[x26,#8*6]
	ldp	x6,x7,[x1,#8*0]
	ldp	x8,x9,[x1,#8*2]
	add	x1,x1,#8*4
	adds	x19,x19,x10
	adcs	x20,x20,x11
	adcs	x21,x21,x12
	adcs	x22,x22,x13
	//adc	x0,x0,xzr
	ldp	x14,x15,[x3,#8*0]
	ldp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4
	b	.Loop_mul4x_tail

.align	4
.Loop_mul4x_break:
	ldp	x12,x13,[x29,#96]	// pull rp and &b[num]
	adds	x19,x19,x30
	add	x2,x2,#8*4		// bp++
	adcs	x20,x20,xzr
	sub	x1,x1,x5		// rewind ap
	adcs	x21,x21,xzr
	stp	x19,x20,[x26,#8*0]	// result!!!
	adcs	x22,x22,xzr
	ldp	x19,x20,[sp,#8*4]	// t[0..3]
	adc	x30,x0,xzr
	stp	x21,x22,[x26,#8*2]	// result!!!
	cmp	x2,x13			// done yet?
	ldp	x21,x22,[sp,#8*6]
	ldp	x14,x15,[x11,#8*0]	// n[0..3]
	ldp	x16,x17,[x11,#8*2]
	add	x3,x11,#8*4
	b.eq	.Lmul4x_post

	ldr	x24,[x2]
	ldp	x6,x7,[x1,#8*0]	// a[0..3]
	ldp	x8,x9,[x1,#8*2]
	adds	x1,x1,#8*4		// clear carry bit
	mov	x0,xzr
	mov	x26,sp
	b	.Loop_mul4x_reduction

.align	4
.Lmul4x_post:
	// Final step. We see if result is larger than modulus, and
	// if it is, subtract the modulus. But comparison implies
	// subtraction. So we subtract modulus, see if it borrowed,
	// and conditionally copy original value.
	mov	x0,x12
	mov	x27,x12		// x0 copy
	subs	x10,x19,x14
	add	x26,sp,#8*8
	sbcs	x11,x20,x15
	sub	x28,x5,#8*4

.Lmul4x_sub:
	sbcs	x12,x21,x16
	ldp	x14,x15,[x3,#8*0]
	sub	x28,x28,#8*4
	ldp	x19,x20,[x26,#8*0]
	sbcs	x13,x22,x17
	ldp	x16,x17,[x3,#8*2]
	add	x3,x3,#8*4
	ldp	x21,x22,[x26,#8*2]
	add	x26,x26,#8*4
	stp	x10,x11,[x0,#8*0]
	sbcs	x10,x19,x14
	stp	x12,x13,[x0,#8*2]
	add	x0,x0,#8*4
	sbcs	x11,x20,x15
	cbnz	x28,.Lmul4x_sub

	sbcs	x12,x21,x16
	mov	x26,sp
	add	x1,sp,#8*4
	ldp	x6,x7,[x27,#8*0]
	sbcs	x13,x22,x17
	stp	x10,x11,[x0,#8*0]
	ldp	x8,x9,[x27,#8*2]
	stp	x12,x13,[x0,#8*2]
	ldp	x19,x20,[x1,#8*0]
	ldp	x21,x22,[x1,#8*2]
	sbcs	xzr,x30,xzr	// did it borrow?
	ldr	x30,[x29,#8]		// pull return address

	sub	x28,x5,#8*4
.Lmul4x_cond_copy:
	sub	x28,x28,#8*4
	csel	x10,x19,x6,lo
	stp	xzr,xzr,[x26,#8*0]
	csel	x11,x20,x7,lo
	ldp	x6,x7,[x27,#8*4]
	ldp	x19,x20,[x1,#8*4]
	csel	x12,x21,x8,lo
	stp	xzr,xzr,[x26,#8*2]
	add	x26,x26,#8*4
	csel	x13,x22,x9,lo
	ldp	x8,x9,[x27,#8*6]
	ldp	x21,x22,[x1,#8*6]
	add	x1,x1,#8*4
	stp	x10,x11,[x27,#8*0]
	stp	x12,x13,[x27,#8*2]
	add	x27,x27,#8*4
	cbnz	x28,.Lmul4x_cond_copy

	csel	x10,x19,x6,lo
	stp	xzr,xzr,[x26,#8*0]
	csel	x11,x20,x7,lo
	stp	xzr,xzr,[x26,#8*2]
	csel	x12,x21,x8,lo
	stp	xzr,xzr,[x26,#8*3]
	csel	x13,x22,x9,lo
	stp	xzr,xzr,[x26,#8*4]
	stp	x10,x11,[x27,#8*0]
	stp	x12,x13,[x27,#8*2]

	b	.Lmul4x_done

.align	4
.Lmul4x4_post_condition:
	adc	x0,x0,xzr
	ldr	x1,[x29,#96]		// pull rp
	// x19-3,x0 hold result, x14-7 hold modulus
	subs	x6,x19,x14
	ldr	x30,[x29,#8]		// pull return address
	sbcs	x7,x20,x15
	stp	xzr,xzr,[sp,#8*0]
	sbcs	x8,x21,x16
	stp	xzr,xzr,[sp,#8*2]
	sbcs	x9,x22,x17
	stp	xzr,xzr,[sp,#8*4]
	sbcs	xzr,x0,xzr		// did it borrow?
	stp	xzr,xzr,[sp,#8*6]

	// x6-3 hold result-modulus
	csel	x6,x19,x6,lo
	csel	x7,x20,x7,lo
	csel	x8,x21,x8,lo
	csel	x9,x22,x9,lo
	stp	x6,x7,[x1,#8*0]
	stp	x8,x9,[x1,#8*2]

.Lmul4x_done:
	ldp	x19,x20,[x29,#16]
	mov	sp,x29
	ldp	x21,x22,[x29,#32]
	mov	x0,#1
	ldp	x23,x24,[x29,#48]
	ldp	x25,x26,[x29,#64]
	ldp	x27,x28,[x29,#80]
	ldr	x29,[sp],#128
	// x30 loaded earlier
	AARCH64_VALIDATE_LINK_REGISTER
	ret
.size	__bn_mul4x_mont,.-__bn_mul4x_mont
.section	.rodata
.byte	77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
.align	2
.align	4