xref: /src/crypto/external/apache2/openssl/lib/libcrypto/arch/aarch64/aes-gcm-armv8_64.S

#include "arm_arch.h"

#if __ARM_MAX_ARCH__>=8
.arch	armv8-a+crypto
.text
.globl	aes_gcm_enc_128_kernel
.type	aes_gcm_enc_128_kernel,%function
.align	4
aes_gcm_enc_128_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L128_enc_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	ldp	x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #160]                     //load rk10
#ifdef __AARCH64EB__
	ror	x13, x13, #32
	ror	x14, x14, #32
#endif
	ld1	{v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	lsr	x5, x1, #3              //byte_len
	mov	x15, x5

	ld1	{v18.4s}, [x8], #16								  //load rk0
	add	x4, x0, x1, lsr #3   //end_input_ptr
	sub	x5, x5, #1      //byte_len - 1

	lsr	x12, x11, #32
	ldr	q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif
	fmov	d1, x10                               //CTR block 1
	rev	w12, w12                                //rev_ctr32

	add	w12, w12, #1                            //increment rev_ctr32
	orr	w11, w11, w11
	ld1	{v19.4s}, [x8], #16								  //load rk1

	rev	w9, w12                                 //CTR block 1
	add	w12, w12, #1                            //CTR block 1
	fmov	d3, x10                               //CTR block 3

	orr	x9, x11, x9, lsl #32            //CTR block 1
	ld1	{ v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

	fmov	v1.d[1], x9                               //CTR block 1
	rev	w9, w12                                 //CTR block 2

	fmov	d2, x10                               //CTR block 2
	orr	x9, x11, x9, lsl #32            //CTR block 2
	add	w12, w12, #1                            //CTR block 2

	fmov	v2.d[1], x9                               //CTR block 2
	rev	w9, w12                                 //CTR block 3

	orr	x9, x11, x9, lsl #32            //CTR block 3
	ld1	{v20.4s}, [x8], #16								  //load rk2

	add	w12, w12, #1                            //CTR block 3
	fmov	v3.d[1], x9                               //CTR block 3

	ldr	q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif
	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 0
	ld1	{v21.4s}, [x8], #16								  //load rk3

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 0
	ldr	q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 0
	ld1	{v22.4s}, [x8], #16								  //load rk4

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 0
	ld1	{v23.4s}, [x8], #16								  //load rk5

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 1
	trn2	v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 1
	ld1	{v24.4s}, [x8], #16								  //load rk6

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 1
	ld1	{v25.4s}, [x8], #16								  //load rk7

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 1
	trn1	v9.2d, v14.2d,    v15.2d                      //h4h | h3h

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 2
	ld1	{v26.4s}, [x8], #16								  //load rk8

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 2
	ldr	q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 2
	eor	v17.16b, v17.16b, v9.16b                  //h4k | h3k

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 3

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 3

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 3
	ld1	{v27.4s}, [x8], #16								  //load rk9

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 3

	and	x5, x5, #0xffffffffffffffc0    //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
	trn2	v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 4
	add	x5, x5, x0

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 4
	cmp	x0, x5                   //check if we have <= 4 blocks

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 4

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 5

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 5

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 6

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 4

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 6
	trn1	v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 6

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 5

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 7

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 7

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 6

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 8

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 7

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 8

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 8

	aese	v2.16b, v27.16b                                      //AES block 2 - round 9

	aese	v0.16b, v27.16b                                      //AES block 0 - round 9

	eor	v16.16b, v16.16b, v8.16b                     //h2k | h1k

	aese	v1.16b, v27.16b                                      //AES block 1 - round 9

	aese	v3.16b, v27.16b                                      //AES block 3 - round 9
	b.ge	.L128_enc_tail                                    //handle tail

	ldp	x6, x7, [x0, #0]            //AES block 0 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	ldp	x21, x22, [x0, #32]           //AES block 2 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	ldp	x19, x20, [x0, #16]           //AES block 1 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	ldp	x23, x24, [x0, #48]           //AES block 3 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	eor	x6, x6, x13                     //AES block 0 - round 10 low
	eor	x7, x7, x14                     //AES block 0 - round 10 high

	eor	x21, x21, x13                     //AES block 2 - round 10 low
	fmov	d4, x6                               //AES block 0 - mov low

	eor	x19, x19, x13                     //AES block 1 - round 10 low
	eor	x22, x22, x14                     //AES block 2 - round 10 high
	fmov	v4.d[1], x7                           //AES block 0 - mov high

	fmov	d5, x19                               //AES block 1 - mov low
	eor	x20, x20, x14                     //AES block 1 - round 10 high

	eor	x23, x23, x13                     //AES block 3 - round 10 low
	fmov	v5.d[1], x20                           //AES block 1 - mov high

	fmov	d6, x21                               //AES block 2 - mov low
	eor	x24, x24, x14                     //AES block 3 - round 10 high
	rev	w9, w12                                 //CTR block 4

	fmov	v6.d[1], x22                           //AES block 2 - mov high
	orr	x9, x11, x9, lsl #32            //CTR block 4

	eor	v4.16b, v4.16b, v0.16b                          //AES block 0 - result
	fmov	d0, x10                               //CTR block 4
	add	w12, w12, #1                            //CTR block 4

	fmov	v0.d[1], x9                               //CTR block 4
	rev	w9, w12                                 //CTR block 5

	eor	v5.16b, v5.16b, v1.16b                          //AES block 1 - result
	fmov	d1, x10                               //CTR block 5
	orr	x9, x11, x9, lsl #32            //CTR block 5

	add	w12, w12, #1                            //CTR block 5
	add	x0, x0, #64                       //AES input_ptr update
	fmov	v1.d[1], x9                               //CTR block 5

	fmov	d7, x23                               //AES block 3 - mov low
	rev	w9, w12                                 //CTR block 6
	st1	{ v4.16b}, [x2], #16                     //AES block 0 - store result

	fmov	v7.d[1], x24                           //AES block 3 - mov high
	orr	x9, x11, x9, lsl #32            //CTR block 6

	add	w12, w12, #1                            //CTR block 6
	eor	v6.16b, v6.16b, v2.16b                          //AES block 2 - result
	st1	{ v5.16b}, [x2], #16                     //AES block 1 - store result

	fmov	d2, x10                               //CTR block 6
	cmp	x0, x5                   //check if we have <= 8 blocks

	fmov	v2.d[1], x9                               //CTR block 6
	rev	w9, w12                                 //CTR block 7
	st1	{ v6.16b}, [x2], #16                     //AES block 2 - store result

	orr	x9, x11, x9, lsl #32            //CTR block 7

	eor	v7.16b, v7.16b, v3.16b                          //AES block 3 - result
	st1	{ v7.16b}, [x2], #16                     //AES block 3 - store result
	b.ge	.L128_enc_prepretail                              //do prepretail

.L128_enc_main_loop:	//main	loop start
	ldp	x23, x24, [x0, #48]           //AES block 4k+3 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	rev64	v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	fmov	d3, x10                               //CTR block 4k+3

	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	rev64	v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	add	w12, w12, #1                            //CTR block 4k+3
	fmov	v3.d[1], x9                               //CTR block 4k+3

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	mov	d30, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	eor	x24, x24, x14                     //AES block 4k+3 - round 10 high

	pmull2	v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid
	ldp	x6, x7, [x0, #0]            //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	rev	w9, w12                                 //CTR block 4k+8

	eor	v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid
	orr	x9, x11, x9, lsl #32            //CTR block 4k+8

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	add	w12, w12, #1                            //CTR block 4k+8
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	eor	v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

	pmull	v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

	pmull	v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid

	pmull	v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	pmull2	v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	eor	x7, x7, x14                     //AES block 4k+4 - round 10 high

	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	eor	x6, x6, x13                     //AES block 4k+4 - round 10 low

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	pmull2	v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3
	eor	v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid

	pmull	v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
	movi	v8.8b, #0xc2

	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	shl	d8, d8, #56               //mod_constant

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5
	ldp	x19, x20, [x0, #16]           //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	ldp	x21, x22, [x0, #32]           //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	eor	x19, x19, x13                     //AES block 4k+5 - round 10 low

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6
	eor	x23, x23, x13                     //AES block 4k+3 - round 10 low

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5
	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	fmov	d4, x6                               //AES block 4k+4 - mov low
	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	fmov	v4.d[1], x7                           //AES block 4k+4 - mov high

	add	x0, x0, #64                       //AES input_ptr update
	fmov	d7, x23                               //AES block 4k+3 - mov low
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	fmov	d5, x19                               //AES block 4k+5 - mov low

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7
	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	eor	x20, x20, x14                     //AES block 4k+5 - round 10 high

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7
	fmov	v5.d[1], x20                           //AES block 4k+5 - mov high

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	fmov	v7.d[1], x24                           //AES block 4k+3 - mov high

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	cmp	x0, x5                   //.LOOP CONTROL

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	aese	v0.16b, v27.16b                                      //AES block 4k+4 - round 9
	eor	x21, x21, x13                     //AES block 4k+6 - round 10 low
	eor	x22, x22, x14                     //AES block 4k+6 - round 10 high

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	fmov	d6, x21                               //AES block 4k+6 - mov low

	aese	v1.16b, v27.16b                                      //AES block 4k+5 - round 9
	fmov	v6.d[1], x22                           //AES block 4k+6 - mov high

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	eor	v4.16b, v4.16b, v0.16b                          //AES block 4k+4 - result

	fmov	d0, x10                               //CTR block 4k+8
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8

	fmov	v0.d[1], x9                               //CTR block 4k+8
	rev	w9, w12                                 //CTR block 4k+9
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	eor	v5.16b, v5.16b, v1.16b                          //AES block 4k+5 - result

	add	w12, w12, #1                            //CTR block 4k+9
	orr	x9, x11, x9, lsl #32            //CTR block 4k+9
	fmov	d1, x10                               //CTR block 4k+9

	pmull	v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low
	fmov	v1.d[1], x9                               //CTR block 4k+9
	rev	w9, w12                                 //CTR block 4k+10

	aese	v2.16b, v27.16b                                      //AES block 4k+6 - round 9
	st1	{ v4.16b}, [x2], #16                     //AES block 4k+4 - store result
	eor	v6.16b, v6.16b, v2.16b                          //AES block 4k+6 - result
	orr	x9, x11, x9, lsl #32            //CTR block 4k+10

	aese	v3.16b, v27.16b                                      //AES block 4k+7 - round 9
	add	w12, w12, #1                            //CTR block 4k+10
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment
	fmov	d2, x10                               //CTR block 4k+10

	eor	v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
	st1	{ v5.16b}, [x2], #16                     //AES block 4k+5 - store result

	fmov	v2.d[1], x9                               //CTR block 4k+10
	st1	{ v6.16b}, [x2], #16                     //AES block 4k+6 - store result
	rev	w9, w12                                 //CTR block 4k+11

	orr	x9, x11, x9, lsl #32            //CTR block 4k+11
	eor	v7.16b, v7.16b, v3.16b                          //AES block 4k+3 - result

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	st1	{ v7.16b}, [x2], #16                     //AES block 4k+3 - store result
	b.lt	.L128_enc_main_loop

.L128_enc_prepretail:	//PREPRETAIL
	rev64	v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)
	fmov	d3, x10                               //CTR block 4k+3
	rev64	v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	add	w12, w12, #1                            //CTR block 4k+3
	fmov	v3.d[1], x9                               //CTR block 4k+3

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

	pmull	v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1

	pmull2	v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	mov	d30, v5.d[1]                                  //GHASH block 4k+1 - mid

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid

	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	eor	v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1

	pmull	v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0

	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	eor	v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid

	pmull2	v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	pmull2	v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	pmull	v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	eor	v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2

	pmull	v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
	movi	v8.8b, #0xc2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	eor	v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2

	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3

	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid
	shl	d8, d8, #56               //mod_constant

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4

	pmull	v28.1q, v9.1d, v8.1d
	eor	v10.16b, v10.16b, v9.16b                         //karatsuba tidy up

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	ext	v9.16b, v9.16b, v9.16b, #8

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	eor	v10.16b, v10.16b, v11.16b

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5
	eor	v10.16b, v10.16b, v28.16b

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	eor	v10.16b, v10.16b, v9.16b

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7

	pmull	v28.1q, v10.1d, v8.1d

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7
	ext	v10.16b, v10.16b, v10.16b, #8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	eor	v11.16b, v11.16b, v28.16b

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8

	aese	v3.16b, v27.16b                                      //AES block 4k+7 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8

	aese	v0.16b, v27.16b                                      //AES block 4k+4 - round 9

	aese	v1.16b, v27.16b                                      //AES block 4k+5 - round 9
	eor	v11.16b, v11.16b, v10.16b

	aese	v2.16b, v27.16b                                      //AES block 4k+6 - round 9
.L128_enc_tail:	//TAIL

	sub	x5, x4, x0   //main_end_input_ptr is number of bytes left to process
	ldp	x6, x7, [x0], #16           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	cmp	x5, #48

	ext	v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
	eor	x6, x6, x13                     //AES block 4k+4 - round 10 low
	eor	x7, x7, x14                     //AES block 4k+4 - round 10 high

	fmov	d4, x6                               //AES block 4k+4 - mov low

	fmov	v4.d[1], x7                           //AES block 4k+4 - mov high

	eor	v5.16b, v4.16b, v0.16b                          //AES block 4k+4 - result

	b.gt	.L128_enc_blocks_more_than_3

	sub	w12, w12, #1
	movi	v11.8b, #0
	mov	v3.16b, v2.16b

	cmp	x5, #32
	mov	v2.16b, v1.16b
	movi	v9.8b, #0

	movi	v10.8b, #0
	b.gt	.L128_enc_blocks_more_than_2

	mov	v3.16b, v1.16b
	cmp	x5, #16

	sub	w12, w12, #1
	b.gt	.L128_enc_blocks_more_than_1

	sub	w12, w12, #1
	b	.L128_enc_blocks_less_than_1
.L128_enc_blocks_more_than_3:	//blocks	left >  3
	st1	{ v5.16b}, [x2], #16                     //AES final-3 block  - store result

	ldp	x6, x7, [x0], #16           //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	rev64	v4.16b, v5.16b                                    //GHASH final-3 block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag
	eor	x7, x7, x14                     //AES final-2 block - round 10 high
	eor	x6, x6, x13                     //AES final-2 block - round 10 low

	fmov	d5, x6                                 //AES final-2 block - mov low

	movi	v8.8b, #0                                        //suppress further partial tag feed in
	fmov	v5.d[1], x7                             //AES final-2 block - mov high

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
	mov	d22, v4.d[1]                                 //GHASH final-3 block - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

	mov	d10, v17.d[1]                               //GHASH final-3 block - mid

	eor	v5.16b, v5.16b, v1.16b                            //AES final-2 block - result
	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

	pmull	v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
.L128_enc_blocks_more_than_2:	//blocks	left >  2

	st1	{ v5.16b}, [x2], #16                     //AES final-2 block - store result

	rev64	v4.16b, v5.16b                                    //GHASH final-2 block
	ldp	x6, x7, [x0], #16           //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	eor	x6, x6, x13                     //AES final-1 block - round 10 low

	fmov	d5, x6                                 //AES final-1 block - mov low
	eor	x7, x7, x14                     //AES final-1 block - round 10 high

	pmull2	v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high
	fmov	v5.d[1], x7                             //AES final-1 block - mov high

	mov	d22, v4.d[1]                                 //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid

	eor	v5.16b, v5.16b, v2.16b                            //AES final-1 block - result

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low

	pmull	v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

	movi	v8.8b, #0                                        //suppress further partial tag feed in

	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
.L128_enc_blocks_more_than_1:	//blocks	left >  1

	st1	{ v5.16b}, [x2], #16                     //AES final-1 block - store result

	rev64	v4.16b, v5.16b                                    //GHASH final-1 block
	ldp	x6, x7, [x0], #16           //AES final block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	eor	x7, x7, x14                     //AES final block - round 10 high
	eor	x6, x6, x13                     //AES final block - round 10 low

	fmov	d5, x6                                 //AES final block - mov low

	pmull2	v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high
	fmov	v5.d[1], x7                             //AES final block - mov high

	mov	d22, v4.d[1]                                 //GHASH final-1 block - mid

	pmull	v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

	eor	v5.16b, v5.16b, v3.16b                            //AES final block - result

	ins	v22.d[1], v22.d[0]                            //GHASH final-1 block - mid

	pmull2	v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high

	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
	movi	v8.8b, #0                                        //suppress further partial tag feed in
.L128_enc_blocks_less_than_1:	//blocks	left <= 1

	and	x1, x1, #127                    //bit_length %= 128
	mvn	x13, xzr                                      //rk10_l = 0xffffffffffffffff

	mvn	x14, xzr                                      //rk10_h = 0xffffffffffffffff
	sub	x1, x1, #128                    //bit_length -= 128

	neg	x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127                    //bit_length %= 128

	lsr	x14, x14, x1                     //rk10_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x6, x13, x14, lt
	csel	x7, x14, xzr, lt

	fmov	d0, x6                                 //ctr0b is mask for last block

	fmov	v0.d[1], x7

	and	v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

	rev64	v4.16b, v5.16b                                    //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	mov	d8, v4.d[1]                                  //GHASH final block - mid

	pmull	v21.1q, v4.1d, v12.1d                          //GHASH final block - low
	ld1	{ v18.16b}, [x2]                            //load existing bytes where the possibly partial last block is to be stored

	eor	v8.8b, v8.8b, v4.8b                          //GHASH final block - mid
#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif
	pmull2	v20.1q, v4.2d, v12.2d                          //GHASH final block - high

	pmull	v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final block - low

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final block - high

	eor	v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	shl	d8, d8, #56               //mod_constant

	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	pmull	v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low

	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	bif	v5.16b, v18.16b, v0.16b                              //insert existing bytes in top end of result before storing

	eor	v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
	st1	{ v5.16b}, [x2]                          //store all 16B

	str	w9, [x16, #12]                          //store the updated counter

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]
	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L128_enc_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_enc_128_kernel,.-aes_gcm_enc_128_kernel
.globl	aes_gcm_dec_128_kernel
.type	aes_gcm_dec_128_kernel,%function
.align	4
aes_gcm_dec_128_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L128_dec_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	lsr	x5, x1, #3              //byte_len
	mov	x15, x5
	ldp	x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #160]                     //load rk10
#ifdef __AARCH64EB__
	ror	x14, x14, 32
	ror	x13, x13, 32
#endif
	sub	x5, x5, #1      //byte_len - 1
	ld1	{v18.4s}, [x8], #16                                //load rk0

	and	x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
	ld1	{ v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

	ldr	q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif
	lsr	x12, x11, #32
	fmov	d2, x10                               //CTR block 2

	ld1	{v19.4s}, [x8], #16                                //load rk1
	orr	w11, w11, w11
	rev	w12, w12                                //rev_ctr32

	fmov	d1, x10                               //CTR block 1
	add	w12, w12, #1                            //increment rev_ctr32

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 0
	rev	w9, w12                                 //CTR block 1

	orr	x9, x11, x9, lsl #32            //CTR block 1
	ld1	{v20.4s}, [x8], #16                                //load rk2
	add	w12, w12, #1                            //CTR block 1

	fmov	v1.d[1], x9                               //CTR block 1
	rev	w9, w12                                 //CTR block 2
	add	w12, w12, #1                            //CTR block 2

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 1
	orr	x9, x11, x9, lsl #32            //CTR block 2

	fmov	v2.d[1], x9                               //CTR block 2
	rev	w9, w12                                 //CTR block 3

	fmov	d3, x10                               //CTR block 3
	orr	x9, x11, x9, lsl #32            //CTR block 3
	add	w12, w12, #1                            //CTR block 3

	fmov	v3.d[1], x9                               //CTR block 3
	add	x4, x0, x1, lsr #3   //end_input_ptr

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 0
	ld1	{v21.4s}, [x8], #16                                //load rk3

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 2
	ld1	{v22.4s}, [x8], #16                                //load rk4

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 0
	ld1	{v23.4s}, [x8], #16                                //load rk5

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 1
	ld1	{v24.4s}, [x8], #16                                //load rk6

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 0

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 1

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 2

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 1
	ld1	{ v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 3
	ld1	{v25.4s}, [x8], #16                                //load rk7

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 3

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 2
	ld1	{v26.4s}, [x8], #16                                //load rk8

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 4

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 3

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 3
	ldr	q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif
	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 4
	ld1	{v27.4s}, [x8], #16                                //load rk9

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 5

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 4

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 4

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 5
	ldr	q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif
	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 5

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 6

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 6

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 6

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 6
	trn1	v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

	ldr	q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif
	trn2	v16.2d,  v12.2d,    v13.2d                      //h2l | h1l
	add	x5, x5, x0

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 7

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 7

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 7
	eor	v16.16b, v16.16b, v8.16b                     //h2k | h1k

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 7

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 8
	trn2	v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 8

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 8
	trn1	v9.2d, v14.2d,    v15.2d                      //h4h | h3h

	aese	v2.16b, v27.16b                                      //AES block 2 - round 9

	aese	v3.16b, v27.16b                                      //AES block 3 - round 9

	aese	v0.16b, v27.16b                                      //AES block 0 - round 9
	cmp	x0, x5                   //check if we have <= 4 blocks

	aese	v1.16b, v27.16b                                      //AES block 1 - round 9
	eor	v17.16b, v17.16b, v9.16b                  //h4k | h3k
	b.ge	.L128_dec_tail                                    //handle tail

	ld1	{v4.16b, v5.16b}, [x0], #32               //AES block 0 - load ciphertext; AES block 1 - load ciphertext

	eor	v1.16b, v5.16b, v1.16b                            //AES block 1 - result
	ld1	{v6.16b}, [x0], #16                       //AES block 2 - load ciphertext

	eor	v0.16b, v4.16b, v0.16b                            //AES block 0 - result
	rev64	v4.16b, v4.16b                                    //GHASH block 0
	rev	w9, w12                                 //CTR block 4

	orr	x9, x11, x9, lsl #32            //CTR block 4
	add	w12, w12, #1                            //CTR block 4
	ld1	{v7.16b}, [x0], #16                       //AES block 3 - load ciphertext

	rev64	v5.16b, v5.16b                                    //GHASH block 1
	mov	x19, v1.d[0]                            //AES block 1 - mov low

	mov	x20, v1.d[1]                            //AES block 1 - mov high

	mov	x6, v0.d[0]                            //AES block 0 - mov low
	cmp	x0, x5                   //check if we have <= 8 blocks

	mov	x7, v0.d[1]                            //AES block 0 - mov high

	fmov	d0, x10                               //CTR block 4

	fmov	v0.d[1], x9                               //CTR block 4
	rev	w9, w12                                 //CTR block 5
	eor	x19, x19, x13                   //AES block 1 - round 10 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	fmov	d1, x10                               //CTR block 5
	add	w12, w12, #1                            //CTR block 5
	orr	x9, x11, x9, lsl #32            //CTR block 5

	fmov	v1.d[1], x9                               //CTR block 5
	rev	w9, w12                                 //CTR block 6
	add	w12, w12, #1                            //CTR block 6

	orr	x9, x11, x9, lsl #32            //CTR block 6

	eor	x20, x20, x14                   //AES block 1 - round 10 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	eor	x6, x6, x13                   //AES block 0 - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v2.16b, v6.16b, v2.16b                            //AES block 2 - result

	eor	x7, x7, x14                   //AES block 0 - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	stp	x6, x7, [x2], #16        //AES block 0 - store result

	stp	x19, x20, [x2], #16        //AES block 1 - store result
	b.ge	.L128_dec_prepretail                              //do prepretail

.L128_dec_main_loop:	//main	loop start
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low

	pmull2	v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	fmov	d2, x10                               //CTR block 4k+6

	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2
	fmov	v2.d[1], x9                               //CTR block 4k+6
	rev	w9, w12                                 //CTR block 4k+7

	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1
	mov	d30, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3

	pmull	v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high
	orr	x9, x11, x9, lsl #32            //CTR block 4k+7

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	fmov	d3, x10                               //CTR block 4k+7
	eor	v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	fmov	v3.d[1], x9                               //CTR block 4k+7

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

	pmull	v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	eor	v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0

	pmull	v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	eor	x23, x23, x13                   //AES block 4k+3 - round 10 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	pmull	v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
	eor	x22, x22, x14                   //AES block 4k+2 - round 10 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	eor	v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid

	pmull2	v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	pmull2	v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4
	eor	v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
	eor	x24, x24, x14                   //AES block 4k+3 - round 10 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif
	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5
	eor	x21, x21, x13                   //AES block 4k+2 - round 10 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif
	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	movi	v8.8b, #0xc2

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high
	ld1	{v4.16b}, [x0], #16                       //AES block 4k+3 - load ciphertext

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7
	add	w12, w12, #1                            //CTR block 4k+7

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7
	shl	d8, d8, #56               //mod_constant

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	rev	w9, w12                                 //CTR block 4k+8

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	ld1	{v5.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v0.16b, v27.16b                                      //AES block 4k+4 - round 9
	orr	x9, x11, x9, lsl #32            //CTR block 4k+8

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	aese	v1.16b, v27.16b                                      //AES block 4k+5 - round 9

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	eor	v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	ld1	{v6.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext

	add	w12, w12, #1                            //CTR block 4k+8
	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid
	eor	v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	ld1	{v7.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6

	rev64	v5.16b, v5.16b                                    //GHASH block 4k+5
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid
	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	fmov	d0, x10                               //CTR block 4k+8

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	fmov	v0.d[1], x9                               //CTR block 4k+8
	rev	w9, w12                                 //CTR block 4k+9

	aese	v2.16b, v27.16b                                      //AES block 4k+6 - round 9
	orr	x9, x11, x9, lsl #32            //CTR block 4k+9
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8
	eor	x7, x7, x14                   //AES block 4k+4 - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low
	mov	x20, v1.d[1]                            //AES block 4k+5 - mov high
	eor	x6, x6, x13                   //AES block 4k+4 - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
	mov	x19, v1.d[0]                            //AES block 4k+5 - mov low
	add	w12, w12, #1                            //CTR block 4k+9

	aese	v3.16b, v27.16b                                      //AES block 4k+7 - round 9
	fmov	d1, x10                               //CTR block 4k+9
	cmp	x0, x5                   //.LOOP CONTROL

	rev64	v4.16b, v4.16b                                    //GHASH block 4k+4
	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	fmov	v1.d[1], x9                               //CTR block 4k+9

	rev	w9, w12                                 //CTR block 4k+10
	add	w12, w12, #1                            //CTR block 4k+10

	eor	x20, x20, x14                   //AES block 4k+5 - round 10 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	stp	x6, x7, [x2], #16        //AES block 4k+4 - store result

	eor	x19, x19, x13                   //AES block 4k+5 - round 10 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	stp	x19, x20, [x2], #16        //AES block 4k+5 - store result

	orr	x9, x11, x9, lsl #32            //CTR block 4k+10
	b.lt	.L128_dec_main_loop

.L128_dec_prepretail:	//PREPRETAIL
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low
	mov	d30, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high

	eor	v4.16b, v4.16b, v11.16b                           //PRE 1
	fmov	d2, x10                               //CTR block 4k+6
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	fmov	v2.d[1], x9                               //CTR block 4k+6

	rev	w9, w12                                 //CTR block 4k+7
	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low
	eor	v30.8b, v30.8b, v5.8b                          //GHASH block 4k+1 - mid

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d10, v17.d[1]                               //GHASH block 4k - mid
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2
	orr	x9, x11, x9, lsl #32            //CTR block 4k+7

	pmull	v29.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid
	fmov	d3, x10                               //CTR block 4k+7

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	fmov	v3.d[1], x9                               //CTR block 4k+7

	pmull	v30.1q, v30.1d, v17.1d                          //GHASH block 4k+1 - mid
	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	pmull2	v28.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+1 - low

	pmull	v29.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
	eor	v9.16b, v9.16b, v28.16b                         //GHASH block 4k+1 - high

	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+1 - mid

	pmull2	v4.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	pmull2	v8.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	pmull	v28.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	eor	v9.16b, v9.16b, v8.16b                         //GHASH block 4k+2 - high
	movi	v8.8b, #0xc2

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	eor	v11.16b, v11.16b, v28.16b                         //GHASH block 4k+2 - low

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+3 - high

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	eor	x23, x23, x13                   //AES block 4k+3 - round 10 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	x21, x21, x13                   //AES block 4k+2 - round 10 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif
	eor	v11.16b, v11.16b, v29.16b                         //GHASH block 4k+3 - low

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	shl	d8, d8, #56               //mod_constant

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	aese	v1.16b, v27.16b                                      //AES block 4k+5 - round 9

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	eor	x24, x24, x14                   //AES block 4k+3 - round 10 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif
	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8
	eor	x22, x22, x14                   //AES block 4k+2 - round 10 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	aese	v0.16b, v27.16b                                      //AES block 4k+4 - round 9
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	aese	v2.16b, v27.16b                                      //AES block 4k+6 - round 9
	add	w12, w12, #1                            //CTR block 4k+7
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result

	aese	v3.16b, v27.16b                                      //AES block 4k+7 - round 9
	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L128_dec_tail:	//TAIL

	sub	x5, x4, x0   //main_end_input_ptr is number of bytes left to process
	ld1	{ v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

	eor	v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high

	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low

	cmp	x5, #48

	eor	x7, x7, x14                   //AES block 4k+4 - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	ext	v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
	eor	x6, x6, x13                   //AES block 4k+4 - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	b.gt	.L128_dec_blocks_more_than_3

	mov	v3.16b, v2.16b
	sub	w12, w12, #1
	movi	v11.8b, #0

	movi	v9.8b, #0
	mov	v2.16b, v1.16b

	movi	v10.8b, #0
	cmp	x5, #32
	b.gt	.L128_dec_blocks_more_than_2

	cmp	x5, #16

	mov	v3.16b, v1.16b
	sub	w12, w12, #1
	b.gt	.L128_dec_blocks_more_than_1

	sub	w12, w12, #1
	b	.L128_dec_blocks_less_than_1
.L128_dec_blocks_more_than_3:	//blocks	left >  3
	rev64	v4.16b, v5.16b                                    //GHASH final-3 block
	ld1	{ v5.16b}, [x0], #16                      //AES final-2 block - load ciphertext

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	mov	d10, v17.d[1]                               //GHASH final-3 block - mid
	stp	x6, x7, [x2], #16        //AES final-3 block  - store result
	eor	v0.16b, v5.16b, v1.16b                            //AES final-2 block - result

	mov	d22, v4.d[1]                                 //GHASH final-3 block - mid
	mov	x7, v0.d[1]                            //AES final-2 block - mov high

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
	mov	x6, v0.d[0]                            //AES final-2 block - mov low

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

	movi	v8.8b, #0                                        //suppress further partial tag feed in
	eor	x7, x7, x14                   //AES final-2 block - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	pmull	v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
	eor	x6, x6, x13                   //AES final-2 block - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
.L128_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v4.16b, v5.16b                                    //GHASH final-2 block
	ld1	{ v5.16b}, [x0], #16                      //AES final-1 block - load ciphertext

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	eor	v0.16b, v5.16b, v2.16b                            //AES final-1 block - result
	stp	x6, x7, [x2], #16        //AES final-2 block  - store result

	mov	d22, v4.d[1]                                 //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

	pmull2	v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high
	mov	x6, v0.d[0]                            //AES final-1 block - mov low

	mov	x7, v0.d[1]                            //AES final-1 block - mov high
	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid

	movi	v8.8b, #0                                        //suppress further partial tag feed in

	pmull	v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

	eor	x6, x6, x13                   //AES final-1 block - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high

	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
	eor	x7, x7, x14                   //AES final-1 block - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
.L128_dec_blocks_more_than_1:	//blocks	left >  1

	rev64	v4.16b, v5.16b                                    //GHASH final-1 block

	ld1	{ v5.16b}, [x0], #16                      //AES final block - load ciphertext
	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	mov	d22, v4.d[1]                                 //GHASH final-1 block - mid

	eor	v0.16b, v5.16b, v3.16b                            //AES final block - result

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

	stp	x6, x7, [x2], #16        //AES final-1 block  - store result
	mov	x6, v0.d[0]                            //AES final block - mov low

	mov	x7, v0.d[1]                            //AES final block - mov high
	ins	v22.d[1], v22.d[0]                            //GHASH final-1 block - mid

	pmull	v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low

	pmull2	v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high

	pmull2	v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid
	movi	v8.8b, #0                                        //suppress further partial tag feed in

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high
	eor	x7, x7, x14                   //AES final block - round 10 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	x6, x6, x13                   //AES final block - round 10 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
.L128_dec_blocks_less_than_1:	//blocks	left <= 1

	mvn	x14, xzr                                      //rk10_h = 0xffffffffffffffff
	and	x1, x1, #127                    //bit_length %= 128

	mvn	x13, xzr                                      //rk10_l = 0xffffffffffffffff
	sub	x1, x1, #128                    //bit_length -= 128

	neg	x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127                    //bit_length %= 128

	lsr	x14, x14, x1                     //rk10_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x10, x14, xzr, lt
	csel	x9, x13, x14, lt

	fmov	d0, x9                                   //ctr0b is mask for last block

	mov	v0.d[1], x10

	and	v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

	rev64	v4.16b, v5.16b                                    //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	ldp	x4, x5, [x2] //load existing bytes we need to not overwrite

	and	x7, x7, x10

	pmull2	v20.1q, v4.2d, v12.2d                          //GHASH final block - high
	mov	d8, v4.d[1]                                  //GHASH final block - mid

	eor	v8.8b, v8.8b, v4.8b                          //GHASH final block - mid
	eor	v9.16b, v9.16b, v20.16b                            //GHASH final block - high

	pmull	v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

	pmull	v21.1q, v4.1d, v12.1d                          //GHASH final block - low
	bic	x4, x4, x9           //mask out low existing bytes
	and	x6, x6, x9

#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif

	eor	v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final block - low

	bic	x5, x5, x10   //mask out high existing bytes
	shl	d8, d8, #56               //mod_constant

	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	orr	x6, x6, x4
	str	w9, [x16, #12]                          //store the updated counter

	orr	x7, x7, x5
	stp	x6, x7, [x2]
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]

	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L128_dec_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_dec_128_kernel,.-aes_gcm_dec_128_kernel
.globl	aes_gcm_enc_192_kernel
.type	aes_gcm_enc_192_kernel,%function
.align	4
aes_gcm_enc_192_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L192_enc_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	ldp	x10, x11, [x16]             //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #192]                     //load rk12
#ifdef __AARCH64EB__
	ror	x13, x13, #32
	ror	x14, x14, #32
#endif
	ld1	{v18.4s}, [x8], #16	                             //load rk0

	ld1	{v19.4s}, [x8], #16	                             //load rk1

	ld1	{v20.4s}, [x8], #16	                             //load rk2

	lsr	x12, x11, #32
	ld1	{v21.4s}, [x8], #16	                             //load rk3
	orr	w11, w11, w11

	ld1	{v22.4s}, [x8], #16	                             //load rk4
	rev	w12, w12                               //rev_ctr32

	add	w12, w12, #1                           //increment rev_ctr32
	fmov	d3, x10                              //CTR block 3

	rev	w9, w12                                //CTR block 1
	add	w12, w12, #1                           //CTR block 1
	fmov	d1, x10                              //CTR block 1

	orr	x9, x11, x9, lsl #32           //CTR block 1
	ld1	{ v0.16b}, [x16]                            //special case vector load initial counter so we can start first AES block as quickly as possible

	fmov	v1.d[1], x9                              //CTR block 1
	rev	w9, w12                                //CTR block 2
	add	w12, w12, #1                           //CTR block 2

	fmov	d2, x10                              //CTR block 2
	orr	x9, x11, x9, lsl #32           //CTR block 2

	fmov	v2.d[1], x9                              //CTR block 2
	rev	w9, w12                                //CTR block 3

	orr	x9, x11, x9, lsl #32           //CTR block 3
	ld1	{v23.4s}, [x8], #16	                             //load rk5

	fmov	v3.d[1], x9                              //CTR block 3

	ld1	{v24.4s}, [x8], #16	                             //load rk6

	ld1	{v25.4s}, [x8], #16	                             //load rk7

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 0
	ld1	{ v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 0
	ld1	{v26.4s}, [x8], #16	                             //load rk8

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 0
	ldr	q15, [x3, #112]                       //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif
	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 0
	ld1	{v27.4s}, [x8], #16	                             //load rk9

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 1
	ld1	{v28.4s}, [x8], #16	                         //load rk10

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 1
	ldr	q12, [x3, #32]                        //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif
	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 1
	ld1	{v29.4s}, [x8], #16	                         //load rk11

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 1
	ldr	q14, [x3, #80]                        //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif
	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 2

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 2

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 3
	trn1	v9.2d, v14.2d,    v15.2d                     //h4h | h3h

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 3

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 2
	trn2	v17.2d,  v14.2d,    v15.2d                     //h4l | h3l

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 4

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 3

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 3

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 5

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 4

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 4

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 6

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 4

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 5

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 5

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 5

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 6
	ldr	q13, [x3, #64]                        //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif
	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 6

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 6

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 7

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 7
	trn2	v16.2d,  v12.2d,    v13.2d                     //h2l | h1l

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 8

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 7
	trn1	v8.2d,    v12.2d,    v13.2d                     //h2h | h1h

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 8

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 8

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 9

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 9

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 9

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 9

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b         //AES block 0 - round 10

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b         //AES block 2 - round 10

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b         //AES block 1 - round 10
	lsr	x5, x1, #3             //byte_len
	mov	x15, x5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b         //AES block 3 - round 10
	sub	x5, x5, #1     //byte_len - 1

	eor	v16.16b, v16.16b, v8.16b                    //h2k | h1k
	and	x5, x5, #0xffffffffffffffc0   //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	eor	v17.16b, v17.16b, v9.16b                 //h4k | h3k

	aese	v2.16b, v29.16b                                    //AES block 2 - round 11
	add	x4, x0, x1, lsr #3  //end_input_ptr
	add	x5, x5, x0

	aese	v1.16b, v29.16b                                    //AES block 1 - round 11
	cmp	x0, x5                  //check if we have <= 4 blocks

	aese	v0.16b, v29.16b                                    //AES block 0 - round 11
	add	w12, w12, #1                           //CTR block 3

	aese	v3.16b, v29.16b                                    //AES block 3 - round 11
	b.ge	.L192_enc_tail                                   //handle tail

	rev	w9, w12                                //CTR block 4
	ldp	x6, x7, [x0, #0]           //AES block 0 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	orr	x9, x11, x9, lsl #32           //CTR block 4
	ldp	x21, x22, [x0, #32]          //AES block 2 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	ldp	x23, x24, [x0, #48]          //AES block 3 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	ldp	x19, x20, [x0, #16]          //AES block 1 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	add	x0, x0, #64                      //AES input_ptr update
	cmp	x0, x5                  //check if we have <= 8 blocks

	eor	x6, x6, x13                    //AES block 0 - round 12 low

	eor	x7, x7, x14                    //AES block 0 - round 12 high
	eor	x22, x22, x14                    //AES block 2 - round 12 high
	fmov	d4, x6                              //AES block 0 - mov low

	eor	x24, x24, x14                    //AES block 3 - round 12 high
	fmov	v4.d[1], x7                          //AES block 0 - mov high

	eor	x21, x21, x13                    //AES block 2 - round 12 low
	eor	x19, x19, x13                    //AES block 1 - round 12 low

	fmov	d5, x19                              //AES block 1 - mov low
	eor	x20, x20, x14                    //AES block 1 - round 12 high

	fmov	v5.d[1], x20                          //AES block 1 - mov high

	eor	x23, x23, x13                    //AES block 3 - round 12 low
	fmov	d6, x21                              //AES block 2 - mov low

	add	w12, w12, #1                           //CTR block 4
	eor	v4.16b, v4.16b, v0.16b                         //AES block 0 - result
	fmov	d0, x10                              //CTR block 4

	fmov	v0.d[1], x9                              //CTR block 4
	rev	w9, w12                                //CTR block 5

	orr	x9, x11, x9, lsl #32           //CTR block 5
	add	w12, w12, #1                           //CTR block 5

	fmov	d7, x23                              //AES block 3 - mov low
	st1	{ v4.16b}, [x2], #16                    //AES block 0 - store result

	fmov	v6.d[1], x22                          //AES block 2 - mov high

	eor	v5.16b, v5.16b, v1.16b                         //AES block 1 - result
	fmov	d1, x10                              //CTR block 5
	st1	{ v5.16b}, [x2], #16                    //AES block 1 - store result

	fmov	v7.d[1], x24                          //AES block 3 - mov high

	fmov	v1.d[1], x9                              //CTR block 5
	rev	w9, w12                                //CTR block 6

	orr	x9, x11, x9, lsl #32           //CTR block 6

	add	w12, w12, #1                           //CTR block 6
	eor	v6.16b, v6.16b, v2.16b                         //AES block 2 - result
	fmov	d2, x10                              //CTR block 6

	fmov	v2.d[1], x9                              //CTR block 6
	rev	w9, w12                                //CTR block 7

	orr	x9, x11, x9, lsl #32           //CTR block 7
	st1	{ v6.16b}, [x2], #16                    //AES block 2 - store result

	eor	v7.16b, v7.16b, v3.16b                         //AES block 3 - result
	st1	{ v7.16b}, [x2], #16                    //AES block 3 - store result
	b.ge	.L192_enc_prepretail                             //do prepretail

.L192_enc_main_loop:	//main	loop start
	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 0
	rev64	v5.16b, v5.16b                                   //GHASH block 4k+1 (t0 and t1 free)

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 0
	ldp	x19, x20, [x0, #16]          //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	ext	v11.16b, v11.16b, v11.16b, #8                    //PRE 0
	fmov	d3, x10                              //CTR block 4k+3
	rev64	v4.16b, v4.16b                                   //GHASH block 4k (only t0 is free)

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 1
	fmov	v3.d[1], x9                              //CTR block 4k+3

	pmull2	v30.1q, v5.2d, v14.2d                         //GHASH block 4k+1 - high
	rev64	v7.16b, v7.16b                                   //GHASH block 4k+3 (t0, t1, t2 and t3 free)
	ldp	x21, x22, [x0, #32]          //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 0
	ldp	x23, x24, [x0, #48]          //AES block 4k+3 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	pmull	v31.1q, v5.1d, v14.1d                         //GHASH block 4k+1 - low
	eor	v4.16b, v4.16b, v11.16b                          //PRE 1

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 1

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 1
	rev64	v6.16b, v6.16b                                   //GHASH block 4k+2 (t0, t1, and t2 free)

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 0
	eor	x24, x24, x14                    //AES block 4k+3 - round 12 high

	pmull	v11.1q, v4.1d, v15.1d                      //GHASH block 4k - low
	mov	d8, v4.d[1]                                 //GHASH block 4k - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 2

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 1
	eor	x21, x21, x13                    //AES block 4k+6 - round 12 low

	eor	v8.8b, v8.8b, v4.8b                         //GHASH block 4k - mid
	eor	v11.16b, v11.16b, v31.16b                        //GHASH block 4k+1 - low

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 3
	eor	x19, x19, x13                    //AES block 4k+5 - round 12 low

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 2
	mov	d31, v6.d[1]                                 //GHASH block 4k+2 - mid

	pmull2	v9.1q, v4.2d, v15.2d                      //GHASH block 4k - high
	mov	d4, v5.d[1]                                 //GHASH block 4k+1 - mid

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 2

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 3

	mov	d10, v17.d[1]                              //GHASH block 4k - mid
	eor	v9.16b, v9.16b, v30.16b                        //GHASH block 4k+1 - high

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 2
	eor	v31.8b, v31.8b, v6.8b                         //GHASH block 4k+2 - mid

	pmull2	v30.1q, v6.2d, v13.2d                         //GHASH block 4k+2 - high

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 4
	eor	v4.8b, v4.8b, v5.8b                         //GHASH block 4k+1 - mid

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 3

	pmull2	v5.1q, v7.2d, v12.2d                         //GHASH block 4k+3 - high
	eor	x20, x20, x14                    //AES block 4k+5 - round 12 high
	ins	v31.d[1], v31.d[0]                               //GHASH block 4k+2 - mid

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 5
	add	w12, w12, #1                           //CTR block 4k+3

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 4
	eor	v9.16b, v9.16b, v30.16b                        //GHASH block 4k+2 - high

	pmull	v4.1q, v4.1d, v17.1d                         //GHASH block 4k+1 - mid
	eor	x22, x22, x14                    //AES block 4k+6 - round 12 high

	pmull2	v31.1q, v31.2d, v16.2d                         //GHASH block 4k+2 - mid
	eor	x23, x23, x13                    //AES block 4k+3 - round 12 low
	mov	d30, v7.d[1]                                 //GHASH block 4k+3 - mid

	pmull	v10.1q, v8.1d, v10.1d                     //GHASH block 4k - mid
	rev	w9, w12                                //CTR block 4k+8

	pmull	v8.1q, v6.1d, v13.1d                         //GHASH block 4k+2 - low
	orr	x9, x11, x9, lsl #32           //CTR block 4k+8

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 3
	eor	v30.8b, v30.8b, v7.8b                         //GHASH block 4k+3 - mid

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 4
	ldp	x6, x7, [x0, #0]           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 6
	eor	v11.16b, v11.16b, v8.16b                        //GHASH block 4k+2 - low

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 4
	add	x0, x0, #64                      //AES input_ptr update

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 5
	movi	v8.8b, #0xc2

	pmull	v6.1q, v7.1d, v12.1d                         //GHASH block 4k+3 - low
	eor	x7, x7, x14                    //AES block 4k+4 - round 12 high
	eor	v10.16b, v10.16b, v4.16b                        //GHASH block 4k+1 - mid

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 5
	eor	x6, x6, x13                    //AES block 4k+4 - round 12 low

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 6
	shl	d8, d8, #56              //mod_constant

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 5
	eor	v9.16b, v9.16b, v5.16b                        //GHASH block 4k+3 - high

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 7
	fmov	d5, x19                              //AES block 4k+5 - mov low

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 7
	eor	v10.16b, v10.16b, v31.16b                        //GHASH block 4k+2 - mid

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 6
	fmov	v5.d[1], x20                          //AES block 4k+5 - mov high

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 8
	eor	v11.16b, v11.16b, v6.16b                        //GHASH block 4k+3 - low

	pmull	v30.1q, v30.1d, v16.1d                         //GHASH block 4k+3 - mid
	cmp	x0, x5                  //.LOOP CONTROL
	fmov	d4, x6                              //AES block 4k+4 - mov low

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 6
	fmov	v4.d[1], x7                          //AES block 4k+4 - mov high

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 8
	fmov	d7, x23                              //AES block 4k+3 - mov low

	eor	v10.16b, v10.16b, v30.16b                        //GHASH block 4k+3 - mid
	eor	v30.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up
	add	w12, w12, #1                           //CTR block 4k+8

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 7
	fmov	v7.d[1], x24                          //AES block 4k+3 - mov high

	pmull	v31.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid
	ext	v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment
	fmov	d6, x21                              //AES block 4k+6 - mov low

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 7

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 9
	eor	v10.16b, v10.16b, v30.16b                        //MODULO - karatsuba tidy up

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 8

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 9

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 10
	eor	v10.16b, v10.16b, v31.16b                     //MODULO - fold into mid

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 9

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 9

	aese	v0.16b, v29.16b                                    //AES block 4k+4 - round 11

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 10
	eor	v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 10

	eor	v4.16b, v4.16b, v0.16b                         //AES block 4k+4 - result
	fmov	d0, x10                              //CTR block 4k+8

	aese	v1.16b, v29.16b                                    //AES block 4k+5 - round 11
	fmov	v0.d[1], x9                              //CTR block 4k+8
	rev	w9, w12                                //CTR block 4k+9

	pmull	v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low
	fmov	v6.d[1], x22                          //AES block 4k+6 - mov high
	st1	{ v4.16b}, [x2], #16                    //AES block 4k+4 - store result

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 10
	orr	x9, x11, x9, lsl #32           //CTR block 4k+9

	eor	v5.16b, v5.16b, v1.16b                         //AES block 4k+5 - result
	add	w12, w12, #1                           //CTR block 4k+9
	fmov	d1, x10                              //CTR block 4k+9

	aese	v2.16b, v29.16b                                    //AES block 4k+6 - round 11
	fmov	v1.d[1], x9                              //CTR block 4k+9
	rev	w9, w12                                //CTR block 4k+10

	add	w12, w12, #1                           //CTR block 4k+10
	ext	v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment
	orr	x9, x11, x9, lsl #32           //CTR block 4k+10

	st1	{ v5.16b}, [x2], #16                    //AES block 4k+5 - store result
	eor	v11.16b, v11.16b, v9.16b                        //MODULO - fold into low

	aese	v3.16b, v29.16b                                    //AES block 4k+7 - round 11
	eor	v6.16b, v6.16b, v2.16b                         //AES block 4k+6 - result
	fmov	d2, x10                              //CTR block 4k+10

	st1	{ v6.16b}, [x2], #16                    //AES block 4k+6 - store result
	fmov	v2.d[1], x9                              //CTR block 4k+10
	rev	w9, w12                                //CTR block 4k+11

	eor	v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
	orr	x9, x11, x9, lsl #32           //CTR block 4k+11

	eor	v7.16b, v7.16b, v3.16b                         //AES block 4k+3 - result
	st1	{ v7.16b}, [x2], #16                    //AES block 4k+3 - store result
	b.lt	.L192_enc_main_loop

.L192_enc_prepretail:	//PREPRETAIL
	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 0
	rev64	v4.16b, v4.16b                                   //GHASH block 4k (only t0 is free)

	fmov	d3, x10                              //CTR block 4k+3
	ext	v11.16b, v11.16b, v11.16b, #8                    //PRE 0
	add	w12, w12, #1                           //CTR block 4k+3

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 0
	rev64	v5.16b, v5.16b                                   //GHASH block 4k+1 (t0 and t1 free)

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 0

	fmov	v3.d[1], x9                              //CTR block 4k+3
	eor	v4.16b, v4.16b, v11.16b                          //PRE 1
	mov	d10, v17.d[1]                              //GHASH block 4k - mid

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 1
	rev64	v6.16b, v6.16b                                   //GHASH block 4k+2 (t0, t1, and t2 free)

	pmull2	v30.1q, v5.2d, v14.2d                         //GHASH block 4k+1 - high

	pmull	v11.1q, v4.1d, v15.1d                      //GHASH block 4k - low
	mov	d8, v4.d[1]                                 //GHASH block 4k - mid

	pmull	v31.1q, v5.1d, v14.1d                         //GHASH block 4k+1 - low
	rev64	v7.16b, v7.16b                                   //GHASH block 4k+3 (t0, t1, t2 and t3 free)

	pmull2	v9.1q, v4.2d, v15.2d                      //GHASH block 4k - high

	eor	v8.8b, v8.8b, v4.8b                         //GHASH block 4k - mid
	mov	d4, v5.d[1]                                 //GHASH block 4k+1 - mid

	eor	v11.16b, v11.16b, v31.16b                        //GHASH block 4k+1 - low
	mov	d31, v6.d[1]                                 //GHASH block 4k+2 - mid

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 0
	eor	v9.16b, v9.16b, v30.16b                        //GHASH block 4k+1 - high

	pmull2	v30.1q, v6.2d, v13.2d                         //GHASH block 4k+2 - high

	eor	v4.8b, v4.8b, v5.8b                         //GHASH block 4k+1 - mid
	eor	v31.8b, v31.8b, v6.8b                         //GHASH block 4k+2 - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 1

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 1
	eor	v9.16b, v9.16b, v30.16b                        //GHASH block 4k+2 - high

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 1

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 2
	mov	d30, v7.d[1]                                 //GHASH block 4k+3 - mid

	pmull2	v5.1q, v7.2d, v12.2d                         //GHASH block 4k+3 - high
	ins	v31.d[1], v31.d[0]                               //GHASH block 4k+2 - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 2

	pmull	v10.1q, v8.1d, v10.1d                     //GHASH block 4k - mid
	eor	v30.8b, v30.8b, v7.8b                         //GHASH block 4k+3 - mid

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 3

	pmull2	v31.1q, v31.2d, v16.2d                         //GHASH block 4k+2 - mid

	pmull	v4.1q, v4.1d, v17.1d                         //GHASH block 4k+1 - mid

	pmull	v30.1q, v30.1d, v16.1d                         //GHASH block 4k+3 - mid
	eor	v9.16b, v9.16b, v5.16b                        //GHASH block 4k+3 - high

	pmull	v8.1q, v6.1d, v13.1d                         //GHASH block 4k+2 - low

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 3
	eor	v10.16b, v10.16b, v4.16b                        //GHASH block 4k+1 - mid

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 2
	eor	v11.16b, v11.16b, v8.16b                        //GHASH block 4k+2 - low

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 4

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 3
	eor	v10.16b, v10.16b, v31.16b                        //GHASH block 4k+2 - mid

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 3

	pmull	v6.1q, v7.1d, v12.1d                         //GHASH block 4k+3 - low
	movi	v8.8b, #0xc2

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 4

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 4

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 4
	eor	v10.16b, v10.16b, v30.16b                        //GHASH block 4k+3 - mid

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 5

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 5
	eor	v11.16b, v11.16b, v6.16b                        //GHASH block 4k+3 - low

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 5

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 6
	eor	v10.16b, v10.16b, v9.16b                        //karatsuba tidy up

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 6
	shl	d8, d8, #56              //mod_constant

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 7

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 7
	eor	v10.16b, v10.16b, v11.16b

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 7

	pmull	v30.1q, v9.1d, v8.1d

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 6
	ext	v9.16b, v9.16b, v9.16b, #8

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 8

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 8
	eor	v10.16b, v10.16b, v30.16b

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 8

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 8
	eor	v10.16b, v10.16b, v9.16b

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 9

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 9

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 9

	pmull	v30.1q, v10.1d, v8.1d

	ext	v10.16b, v10.16b, v10.16b, #8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b         //AES block 4k+7 - round 10

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b         //AES block 4k+4 - round 10

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b         //AES block 4k+6 - round 10

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b         //AES block 4k+5 - round 10
	eor	v11.16b, v11.16b, v30.16b

	aese	v0.16b, v29.16b                                    //AES block 4k+4 - round 11

	aese	v3.16b, v29.16b                                    //AES block 4k+7 - round 11

	aese	v2.16b, v29.16b                                    //AES block 4k+6 - round 11

	aese	v1.16b, v29.16b                                    //AES block 4k+5 - round 11
	eor	v11.16b, v11.16b, v10.16b
.L192_enc_tail:	//TAIL

	sub	x5, x4, x0  //main_end_input_ptr is number of bytes left to process
	ldp	x6, x7, [x0], #16          //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	x6, x6, x13                    //AES block 4k+4 - round 12 low
	eor	x7, x7, x14                    //AES block 4k+4 - round 12 high

	fmov	d4, x6                              //AES block 4k+4 - mov low

	fmov	v4.d[1], x7                          //AES block 4k+4 - mov high
	cmp	x5, #48

	eor	v5.16b, v4.16b, v0.16b                         //AES block 4k+4 - result

	ext	v8.16b, v11.16b, v11.16b, #8                    //prepare final partial tag
	b.gt	.L192_enc_blocks_more_than_3

	sub	w12, w12, #1
	movi	v10.8b, #0

	mov	v3.16b, v2.16b
	movi	v9.8b, #0
	cmp	x5, #32

	mov	v2.16b, v1.16b
	movi	v11.8b, #0
	b.gt	.L192_enc_blocks_more_than_2

	sub	w12, w12, #1

	mov	v3.16b, v1.16b
	cmp	x5, #16
	b.gt	.L192_enc_blocks_more_than_1

	sub	w12, w12, #1
	b	.L192_enc_blocks_less_than_1
.L192_enc_blocks_more_than_3:	//blocks	left >  3
	st1	{ v5.16b}, [x2], #16                    //AES final-3 block  - store result

	ldp	x6, x7, [x0], #16          //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	rev64	v4.16b, v5.16b                                   //GHASH final-3 block

	eor	x6, x6, x13                    //AES final-2 block - round 12 low
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	eor	x7, x7, x14                    //AES final-2 block - round 12 high
	fmov	d5, x6                                //AES final-2 block - mov low

	fmov	v5.d[1], x7                            //AES final-2 block - mov high

	mov	d22, v4.d[1]                                //GHASH final-3 block - mid

	pmull	v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low

	mov	d10, v17.d[1]                              //GHASH final-3 block - mid

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid

	movi	v8.8b, #0                                       //suppress further partial tag feed in

	pmull2	v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

	pmull	v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
	eor	v5.16b, v5.16b, v1.16b                           //AES final-2 block - result
.L192_enc_blocks_more_than_2:	//blocks	left >  2

	st1	{ v5.16b}, [x2], #16                    //AES final-2 block - store result

	rev64	v4.16b, v5.16b                                   //GHASH final-2 block
	ldp	x6, x7, [x0], #16          //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	eor	x7, x7, x14                    //AES final-1 block - round 12 high

	pmull2	v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high
	mov	d22, v4.d[1]                                //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low
	eor	x6, x6, x13                    //AES final-1 block - round 12 low

	fmov	d5, x6                                //AES final-1 block - mov low

	fmov	v5.d[1], x7                            //AES final-1 block - mov high
	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high
	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low

	pmull	v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

	movi	v8.8b, #0                                       //suppress further partial tag feed in

	eor	v5.16b, v5.16b, v2.16b                           //AES final-1 block - result

	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
.L192_enc_blocks_more_than_1:	//blocks	left >  1

	st1	{ v5.16b}, [x2], #16                    //AES final-1 block - store result

	ldp	x6, x7, [x0], #16          //AES final block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	rev64	v4.16b, v5.16b                                   //GHASH final-1 block

	eor	x6, x6, x13                    //AES final block - round 12 low
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag
	movi	v8.8b, #0                                       //suppress further partial tag feed in

	mov	d22, v4.d[1]                                //GHASH final-1 block - mid

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid
	eor	x7, x7, x14                    //AES final block - round 12 high
	fmov	d5, x6                                //AES final block - mov low

	pmull2	v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high
	fmov	v5.d[1], x7                            //AES final block - mov high

	ins	v22.d[1], v22.d[0]                           //GHASH final-1 block - mid

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

	pmull	v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low

	pmull2	v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid

	eor	v5.16b, v5.16b, v3.16b                           //AES final block - result

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low

	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid
.L192_enc_blocks_less_than_1:	//blocks	left <= 1

	ld1	{ v18.16b}, [x2]                           //load existing bytes where the possibly partial last block is to be stored
#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif
	and	x1, x1, #127                   //bit_length %= 128

	sub	x1, x1, #128                   //bit_length -= 128
	mvn	x14, xzr                                     //rk12_h = 0xffffffffffffffff

	neg	x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])
	mvn	x13, xzr                                     //rk12_l = 0xffffffffffffffff

	and	x1, x1, #127                   //bit_length %= 128

	lsr	x14, x14, x1                    //rk12_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x6, x13, x14, lt
	csel	x7, x14, xzr, lt

	fmov	d0, x6                                //ctr0b is mask for last block

	fmov	v0.d[1], x7

	and	v5.16b, v5.16b, v0.16b                           //possibly partial last block has zeroes in highest bits

	rev64	v4.16b, v5.16b                                   //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	mov	d8, v4.d[1]                                 //GHASH final block - mid

	pmull	v21.1q, v4.1d, v12.1d                         //GHASH final block - low

	pmull2	v20.1q, v4.2d, v12.2d                         //GHASH final block - high

	eor	v8.8b, v8.8b, v4.8b                         //GHASH final block - mid

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final block - low

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final block - high

	pmull	v8.1q, v8.1d, v16.1d                         //GHASH final block - mid

	eor	v10.16b, v10.16b, v8.16b                        //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v30.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up

	shl	d8, d8, #56              //mod_constant

	bif	v5.16b, v18.16b, v0.16b                             //insert existing bytes in top end of result before storing

	eor	v10.16b, v10.16b, v30.16b                        //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid

	ext	v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment

	eor	v10.16b, v10.16b, v31.16b                     //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

	pmull	v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low

	ext	v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment

	eor	v11.16b, v11.16b, v9.16b                        //MODULO - fold into low
	str	w9, [x16, #12]                         //store the updated counter

	st1	{ v5.16b}, [x2]                         //store all 16B

	eor	v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]

	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L192_enc_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_enc_192_kernel,.-aes_gcm_enc_192_kernel
.globl	aes_gcm_dec_192_kernel
.type	aes_gcm_dec_192_kernel,%function
.align	4
aes_gcm_dec_192_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L192_dec_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	add	x4, x0, x1, lsr #3   //end_input_ptr
	ldp	x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #192]                     //load rk12
#ifdef __AARCH64EB__
	ror	x13, x13, #32
	ror	x14, x14, #32
#endif
	ld1	{ v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

	ld1	{v18.4s}, [x8], #16                                  //load rk0

	lsr	x5, x1, #3              //byte_len
	mov	x15, x5
	ld1	{v19.4s}, [x8], #16                               //load rk1

	lsr	x12, x11, #32
	orr	w11, w11, w11
	fmov	d3, x10                               //CTR block 3

	rev	w12, w12                                //rev_ctr32
	fmov	d1, x10                               //CTR block 1

	add	w12, w12, #1                            //increment rev_ctr32
	ld1	{v20.4s}, [x8], #16                               //load rk2

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 0
	rev	w9, w12                                 //CTR block 1

	add	w12, w12, #1                            //CTR block 1
	orr	x9, x11, x9, lsl #32            //CTR block 1
	ld1	{v21.4s}, [x8], #16                               //load rk3

	fmov	v1.d[1], x9                               //CTR block 1
	rev	w9, w12                                 //CTR block 2
	add	w12, w12, #1                            //CTR block 2

	fmov	d2, x10                               //CTR block 2
	orr	x9, x11, x9, lsl #32            //CTR block 2

	fmov	v2.d[1], x9                               //CTR block 2
	rev	w9, w12                                 //CTR block 3

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 1
	orr	x9, x11, x9, lsl #32            //CTR block 3

	fmov	v3.d[1], x9                               //CTR block 3

	ld1	{v22.4s}, [x8], #16                               //load rk4

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 2

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 0
	ld1	{v23.4s}, [x8], #16                               //load rk5

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 0
	ldr	q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif
	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 0
	ldr	q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif
	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 1
	ldr	q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif
	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 1

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 1
	ldr	q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif
	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 2
	ld1	{v24.4s}, [x8], #16                               //load rk6

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 3
	ld1	{v25.4s}, [x8], #16                               //load rk7

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 2
	ld1	{v26.4s}, [x8], #16                               //load rk8

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 2
	ld1	{v27.4s}, [x8], #16                               //load rk9

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 3
	ld1	{ v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 3
	add	w12, w12, #1                            //CTR block 3

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 3
	trn1	v9.2d, v14.2d,    v15.2d                      //h4h | h3h

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 4
	ld1	{v28.4s}, [x8], #16                              //load rk10

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 4
	trn2	v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 4

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 4
	trn2	v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 5
	ld1	{v29.4s}, [x8], #16                              //load rk11

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 5

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 5

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 6

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 6

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 6

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 7

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 7

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 8

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 7

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 9

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 9

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 8
	sub	x5, x5, #1      //byte_len - 1

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 8
	and	x5, x5, #0xffffffffffffffc0    //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 10
	add	x5, x5, x0

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 9
	cmp	x0, x5                   //check if we have <= 4 blocks

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 9
	trn1	v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

	aese	v3.16b, v29.16b                                     //AES block 3 - round 11

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 10

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 10

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 10
	eor	v16.16b, v16.16b, v8.16b                     //h2k | h1k

	aese	v2.16b, v29.16b                                     //AES block 2 - round 11

	aese	v1.16b, v29.16b                                     //AES block 1 - round 11
	eor	v17.16b, v17.16b, v9.16b                  //h4k | h3k

	aese	v0.16b, v29.16b                                     //AES block 0 - round 11
	b.ge	.L192_dec_tail                                    //handle tail

	ld1	{v4.16b, v5.16b}, [x0], #32               //AES block 0,1 - load ciphertext

	eor	v1.16b, v5.16b, v1.16b                            //AES block 1 - result

	eor	v0.16b, v4.16b, v0.16b                            //AES block 0 - result
	rev	w9, w12                                 //CTR block 4
	ld1	{v6.16b, v7.16b}, [x0], #32               //AES block 2,3 - load ciphertext

	mov	x19, v1.d[0]                            //AES block 1 - mov low

	mov	x20, v1.d[1]                            //AES block 1 - mov high

	mov	x6, v0.d[0]                            //AES block 0 - mov low
	orr	x9, x11, x9, lsl #32            //CTR block 4
	add	w12, w12, #1                            //CTR block 4

	mov	x7, v0.d[1]                            //AES block 0 - mov high
	rev64	v4.16b, v4.16b                                    //GHASH block 0

	fmov	d0, x10                               //CTR block 4
	rev64	v5.16b, v5.16b                                    //GHASH block 1
	cmp	x0, x5                   //check if we have <= 8 blocks

	eor	x19, x19, x13                   //AES block 1 - round 12 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	fmov	v0.d[1], x9                               //CTR block 4
	rev	w9, w12                                 //CTR block 5

	orr	x9, x11, x9, lsl #32            //CTR block 5
	fmov	d1, x10                               //CTR block 5
	eor	x20, x20, x14                   //AES block 1 - round 12 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	add	w12, w12, #1                            //CTR block 5
	fmov	v1.d[1], x9                               //CTR block 5
	eor	x6, x6, x13                   //AES block 0 - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	rev	w9, w12                                 //CTR block 6
	eor	x7, x7, x14                   //AES block 0 - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	stp	x6, x7, [x2], #16        //AES block 0 - store result
	orr	x9, x11, x9, lsl #32            //CTR block 6

	stp	x19, x20, [x2], #16        //AES block 1 - store result

	add	w12, w12, #1                            //CTR block 6
	eor	v2.16b, v6.16b, v2.16b                            //AES block 2 - result
	b.ge	.L192_dec_prepretail                              //do prepretail

.L192_dec_main_loop:	//main	loop start
	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0

	pmull	v31.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low

	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	fmov	d2, x10                               //CTR block 4k+6

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1

	pmull2	v30.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
	fmov	v2.d[1], x9                               //CTR block 4k+6

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	fmov	d3, x10                               //CTR block 4k+7
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d10, v17.d[1]                               //GHASH block 4k - mid
	rev	w9, w12                                 //CTR block 4k+7

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	orr	x9, x11, x9, lsl #32            //CTR block 4k+7

	fmov	v3.d[1], x9                               //CTR block 4k+7
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2
	eor	x22, x22, x14                   //AES block 4k+2 - round 12 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
	eor	v11.16b, v11.16b, v31.16b                         //GHASH block 4k+1 - low
	eor	x21, x21, x13                   //AES block 4k+2 - round 12 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif
	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3

	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid
	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	eor	v9.16b, v9.16b, v30.16b                         //GHASH block 4k+1 - high

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4

	pmull2	v30.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

	pmull	v8.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5

	eor	v9.16b, v9.16b, v30.16b                         //GHASH block 4k+2 - high
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3

	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+2 - low

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	movi	v8.8b, #0xc2

	pmull	v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9
	eor	v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9
	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	shl	d8, d8, #56               //mod_constant

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	ld1	{v4.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	ld1	{v5.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext
	eor	x23, x23, x13                   //AES block 4k+3 - round 12 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v0.16b, v29.16b                                     //AES block 4k+4 - round 11
	add	w12, w12, #1                            //CTR block 4k+7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	ld1	{v6.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

	aese	v1.16b, v29.16b                                     //AES block 4k+5 - round 11
	ld1	{v7.16b}, [x0], #16                       //AES block 4k+7 - load ciphertext
	rev	w9, w12                                 //CTR block 4k+8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	cmp	x0, x5                   //.LOOP CONTROL

	eor	v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result
	eor	x24, x24, x14                   //AES block 4k+3 - round 12 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif
	eor	v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10
	orr	x9, x11, x9, lsl #32            //CTR block 4k+8

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	mov	x19, v1.d[0]                            //AES block 4k+5 - mov low

	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result
	rev64	v5.16b, v5.16b                                    //GHASH block 4k+5

	aese	v2.16b, v29.16b                                     //AES block 4k+6 - round 11
	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10
	mov	x20, v1.d[1]                            //AES block 4k+5 - mov high

	fmov	d0, x10                               //CTR block 4k+8
	add	w12, w12, #1                            //CTR block 4k+8
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	eor	v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
	fmov	v0.d[1], x9                               //CTR block 4k+8
	rev	w9, w12                                 //CTR block 4k+9

	eor	x6, x6, x13                   //AES block 4k+4 - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	orr	x9, x11, x9, lsl #32            //CTR block 4k+9
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	fmov	d1, x10                               //CTR block 4k+9
	add	w12, w12, #1                            //CTR block 4k+9
	eor	x19, x19, x13                   //AES block 4k+5 - round 12 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	fmov	v1.d[1], x9                               //CTR block 4k+9
	rev	w9, w12                                 //CTR block 4k+10
	eor	x20, x20, x14                   //AES block 4k+5 - round 12 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	eor	x7, x7, x14                   //AES block 4k+4 - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	stp	x6, x7, [x2], #16        //AES block 4k+4 - store result
	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low

	add	w12, w12, #1                            //CTR block 4k+10
	rev64	v4.16b, v4.16b                                    //GHASH block 4k+4
	orr	x9, x11, x9, lsl #32            //CTR block 4k+10

	aese	v3.16b, v29.16b                                     //AES block 4k+7 - round 11
	stp	x19, x20, [x2], #16        //AES block 4k+5 - store result
	b.lt	.L192_dec_main_loop

.L192_dec_prepretail:	//PREPRETAIL
	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	eor	v4.16b, v4.16b, v11.16b                           //PRE 1
	fmov	d2, x10                               //CTR block 4k+6

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid
	fmov	d3, x10                               //CTR block 4k+7

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	fmov	v2.d[1], x9                               //CTR block 4k+6
	rev	w9, w12                                 //CTR block 4k+7

	orr	x9, x11, x9, lsl #32            //CTR block 4k+7
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	pmull	v31.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	eor	x24, x24, x14                   //AES block 4k+3 - round 12 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif
	fmov	v3.d[1], x9                               //CTR block 4k+7

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2
	eor	x21, x21, x13                   //AES block 4k+2 - round 12 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif
	pmull2	v30.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high
	eor	x22, x22, x14                   //AES block 4k+2 - round 12 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	eor	x23, x23, x13                   //AES block 4k+3 - round 12 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	eor	v9.16b, v9.16b, v30.16b                         //GHASH block 4k+1 - high

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
	add	w12, w12, #1                            //CTR block 4k+7

	pmull2	v30.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	eor	v11.16b, v11.16b, v31.16b                         //GHASH block 4k+1 - low

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0

	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid
	mov	d31, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	eor	v9.16b, v9.16b, v30.16b                         //GHASH block 4k+2 - high

	eor	v31.8b, v31.8b, v6.8b                          //GHASH block 4k+2 - mid

	pmull	v8.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	mov	d30, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	ins	v31.d[1], v31.d[0]                                //GHASH block 4k+2 - mid

	pmull	v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	eor	v30.8b, v30.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3

	pmull2	v31.1q, v31.2d, v16.2d                          //GHASH block 4k+2 - mid
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+2 - low

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
	movi	v8.8b, #0xc2

	pmull	v30.1q, v30.1d, v16.1d                          //GHASH block 4k+3 - mid

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	shl	d8, d8, #56               //mod_constant
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	eor	v10.16b, v10.16b, v31.16b                         //GHASH block 4k+2 - mid

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	eor	v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v10.16b, v10.16b, v30.16b                         //GHASH block 4k+3 - mid

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7
	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10

	aese	v0.16b, v29.16b
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	aese	v2.16b, v29.16b

	aese	v1.16b, v29.16b

	aese	v3.16b, v29.16b

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L192_dec_tail:	//TAIL

	sub	x5, x4, x0   //main_end_input_ptr is number of bytes left to process
	ld1	{ v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

	eor	v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high

	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low

	ext	v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag

	cmp	x5, #48

	eor	x7, x7, x14                   //AES block 4k+4 - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	x6, x6, x13                   //AES block 4k+4 - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	b.gt	.L192_dec_blocks_more_than_3

	movi	v11.8b, #0
	movi	v9.8b, #0

	mov	v3.16b, v2.16b
	mov	v2.16b, v1.16b
	sub	w12, w12, #1

	movi	v10.8b, #0
	cmp	x5, #32
	b.gt	.L192_dec_blocks_more_than_2

	mov	v3.16b, v1.16b
	cmp	x5, #16
	sub	w12, w12, #1

	b.gt	.L192_dec_blocks_more_than_1

	sub	w12, w12, #1
	b	.L192_dec_blocks_less_than_1
.L192_dec_blocks_more_than_3:	//blocks	left >  3
	rev64	v4.16b, v5.16b                                    //GHASH final-3 block
	ld1	{ v5.16b}, [x0], #16                      //AES final-2 block - load ciphertext

	stp	x6, x7, [x2], #16        //AES final-3 block  - store result

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	eor	v0.16b, v5.16b, v1.16b                            //AES final-2 block - result

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH final-3 block - low
	mov	x6, v0.d[0]                            //AES final-2 block - mov low
	mov	d22, v4.d[1]                                 //GHASH final-3 block - mid

	mov	x7, v0.d[1]                            //AES final-2 block - mov high

	mov	d10, v17.d[1]                               //GHASH final-3 block - mid
	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-3 block - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH final-3 block - high

	eor	x6, x6, x13                   //AES final-2 block - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	movi	v8.8b, #0                                        //suppress further partial tag feed in

	pmull	v10.1q, v22.1d, v10.1d                    //GHASH final-3 block - mid
	eor	x7, x7, x14                   //AES final-2 block - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
.L192_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v4.16b, v5.16b                                    //GHASH final-2 block
	ld1	{ v5.16b}, [x0], #16                      //AES final-1 block - load ciphertext

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	movi	v8.8b, #0                                        //suppress further partial tag feed in

	eor	v0.16b, v5.16b, v2.16b                            //AES final-1 block - result

	mov	d22, v4.d[1]                                 //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                          //GHASH final-2 block - low

	stp	x6, x7, [x2], #16        //AES final-2 block  - store result

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-2 block - mid
	mov	x7, v0.d[1]                            //AES final-1 block - mov high

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-2 block - low
	mov	x6, v0.d[0]                            //AES final-1 block - mov low

	pmull2	v20.1q, v4.2d, v14.2d                          //GHASH final-2 block - high

	pmull	v22.1q, v22.1d, v17.1d                      //GHASH final-2 block - mid

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-2 block - high
	eor	x7, x7, x14                   //AES final-1 block - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	x6, x6, x13                   //AES final-1 block - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-2 block - mid
.L192_dec_blocks_more_than_1:	//blocks	left >  1

	rev64	v4.16b, v5.16b                                    //GHASH final-1 block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag
	ld1	{ v5.16b}, [x0], #16                      //AES final block - load ciphertext

	mov	d22, v4.d[1]                                 //GHASH final-1 block - mid

	pmull2	v20.1q, v4.2d, v13.2d                          //GHASH final-1 block - high

	eor	v0.16b, v5.16b, v3.16b                            //AES final block - result
	stp	x6, x7, [x2], #16        //AES final-1 block  - store result

	eor	v22.8b, v22.8b, v4.8b                      //GHASH final-1 block - mid

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final-1 block - high

	pmull	v21.1q, v4.1d, v13.1d                          //GHASH final-1 block - low
	mov	x7, v0.d[1]                            //AES final block - mov high

	ins	v22.d[1], v22.d[0]                            //GHASH final-1 block - mid
	mov	x6, v0.d[0]                            //AES final block - mov low

	pmull2	v22.1q, v22.2d, v16.2d                      //GHASH final-1 block - mid

	movi	v8.8b, #0                                        //suppress further partial tag feed in
	eor	v11.16b, v11.16b, v21.16b                            //GHASH final-1 block - low
	eor	x7, x7, x14                   //AES final block - round 12 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	x6, x6, x13                   //AES final block - round 12 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v10.16b, v10.16b, v22.16b                       //GHASH final-1 block - mid
.L192_dec_blocks_less_than_1:	//blocks	left <= 1

	mvn	x13, xzr                                      //rk12_l = 0xffffffffffffffff
	ldp	x4, x5, [x2]  //load existing bytes we need to not overwrite
	and	x1, x1, #127                    //bit_length %= 128

	sub	x1, x1, #128                    //bit_length -= 128

	neg	x1, x1                          //bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127                    //bit_length %= 128
	mvn	x14, xzr                                      //rk12_h = 0xffffffffffffffff

	lsr	x14, x14, x1                     //rk12_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x9, x13, x14, lt
	csel	x10, x14, xzr, lt

	fmov	d0, x9                                   //ctr0b is mask for last block
	and	x6, x6, x9
	bic	x4, x4, x9           //mask out low existing bytes

	orr	x6, x6, x4
	mov	v0.d[1], x10
#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif

	and	v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits
	str	w9, [x16, #12]                          //store the updated counter

	rev64	v4.16b, v5.16b                                    //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag
	bic	x5, x5, x10 //mask out high existing bytes

	and	x7, x7, x10

	pmull2	v20.1q, v4.2d, v12.2d                          //GHASH final block - high
	mov	d8, v4.d[1]                                  //GHASH final block - mid

	pmull	v21.1q, v4.1d, v12.1d                          //GHASH final block - low

	eor	v8.8b, v8.8b, v4.8b                          //GHASH final block - mid

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final block - high

	pmull	v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final block - low

	eor	v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v30.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	shl	d8, d8, #56               //mod_constant

	eor	v10.16b, v10.16b, v30.16b                         //MODULO - karatsuba tidy up

	pmull	v31.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	orr	x7, x7, x5
	stp	x6, x7, [x2]

	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	eor	v10.16b, v10.16b, v31.16b                      //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]

	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L192_dec_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_dec_192_kernel,.-aes_gcm_dec_192_kernel
.globl	aes_gcm_enc_256_kernel
.type	aes_gcm_enc_256_kernel,%function
.align	4
aes_gcm_enc_256_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L256_enc_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	add	x4, x0, x1, lsr #3   //end_input_ptr
	lsr	x5, x1, #3              //byte_len
	mov	x15, x5
	ldp	x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #224]                     //load rk14
#ifdef __AARCH64EB__
	ror	x13, x13, #32
	ror	x14, x14, #32
#endif
	ld1	{ v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible
	sub	x5, x5, #1      //byte_len - 1

	ld1	{v18.4s}, [x8], #16                               //load rk0
	and	x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	ld1	{v19.4s}, [x8], #16                               //load rk1
	add	x5, x5, x0

	lsr	x12, x11, #32
	fmov	d2, x10                               //CTR block 2
	orr	w11, w11, w11

	rev	w12, w12                                //rev_ctr32
	cmp	x0, x5                   //check if we have <= 4 blocks
	fmov	d1, x10                               //CTR block 1

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 0
	add	w12, w12, #1                            //increment rev_ctr32

	rev	w9, w12                                 //CTR block 1
	fmov	d3, x10                               //CTR block 3

	orr	x9, x11, x9, lsl #32            //CTR block 1
	add	w12, w12, #1                            //CTR block 1
	ld1	{v20.4s}, [x8], #16                               //load rk2

	fmov	v1.d[1], x9                               //CTR block 1
	rev	w9, w12                                 //CTR block 2
	add	w12, w12, #1                            //CTR block 2

	orr	x9, x11, x9, lsl #32            //CTR block 2
	ld1	{v21.4s}, [x8], #16                               //load rk3

	fmov	v2.d[1], x9                               //CTR block 2
	rev	w9, w12                                 //CTR block 3

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 1
	orr	x9, x11, x9, lsl #32            //CTR block 3

	fmov	v3.d[1], x9                               //CTR block 3

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 0
	ld1	{v22.4s}, [x8], #16                               //load rk4

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 2
	ld1	{v23.4s}, [x8], #16                               //load rk5

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 0
	ld1	{v24.4s}, [x8], #16                               //load rk6

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 1
	ldr	q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif
	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 0
	ld1	{v25.4s}, [x8], #16                               //load rk7

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 1
	ld1	{v26.4s}, [x8], #16                               //load rk8

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 2
	ldr	q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif
	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 1
	ld1	{v27.4s}, [x8], #16                               //load rk9

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 2
	ldr	q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif
	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 3
	ld1	{v28.4s}, [x8], #16                              //load rk10

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 2
	ld1	{v29.4s}, [x8], #16                              //load rk11

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 3
	add	w12, w12, #1                            //CTR block 3

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 3

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 3
	ld1	{ v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 4

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 4

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 4

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 4

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 5

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 5

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 5

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 6
	trn2	v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 6
	ld1	{v30.4s}, [x8], #16                              //load rk12

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 6
	ldr	q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif
	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 6
	ld1	{v31.4s}, [x8], #16                              //load rk13

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 7
	trn1	v9.2d, v14.2d,    v15.2d                      //h4h | h3h

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 7

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 7
	trn2	v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 8

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 8

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 9

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 9

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 8

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 10

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 9

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 9

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 10

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 10

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 11

	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 11

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 10

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 12

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 12

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 11
	eor	v17.16b, v17.16b, v9.16b                  //h4k | h3k

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 11

	aese	v2.16b, v31.16b                                     //AES block 2 - round 13
	trn1	v8.2d,    v12.2d,    v13.2d                      //h2h | h1h

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 12

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 12

	aese	v1.16b, v31.16b                                     //AES block 1 - round 13

	aese	v0.16b, v31.16b                                     //AES block 0 - round 13

	aese	v3.16b, v31.16b                                     //AES block 3 - round 13
	eor	v16.16b, v16.16b, v8.16b                     //h2k | h1k
	b.ge	.L256_enc_tail                                    //handle tail

	ldp	x19, x20, [x0, #16]           //AES block 1 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	rev	w9, w12                                 //CTR block 4
	ldp	x6, x7, [x0, #0]            //AES block 0 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	ldp	x23, x24, [x0, #48]           //AES block 3 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	ldp	x21, x22, [x0, #32]           //AES block 2 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	add	x0, x0, #64                       //AES input_ptr update

	eor	x19, x19, x13                     //AES block 1 - round 14 low
	eor	x20, x20, x14                     //AES block 1 - round 14 high

	fmov	d5, x19                               //AES block 1 - mov low
	eor	x6, x6, x13                     //AES block 0 - round 14 low

	eor	x7, x7, x14                     //AES block 0 - round 14 high
	eor	x24, x24, x14                     //AES block 3 - round 14 high
	fmov	d4, x6                               //AES block 0 - mov low

	cmp	x0, x5                   //check if we have <= 8 blocks
	fmov	v4.d[1], x7                           //AES block 0 - mov high
	eor	x23, x23, x13                     //AES block 3 - round 14 low

	eor	x21, x21, x13                     //AES block 2 - round 14 low
	fmov	v5.d[1], x20                           //AES block 1 - mov high

	fmov	d6, x21                               //AES block 2 - mov low
	add	w12, w12, #1                            //CTR block 4

	orr	x9, x11, x9, lsl #32            //CTR block 4
	fmov	d7, x23                               //AES block 3 - mov low
	eor	x22, x22, x14                     //AES block 2 - round 14 high

	fmov	v6.d[1], x22                           //AES block 2 - mov high

	eor	v4.16b, v4.16b, v0.16b                          //AES block 0 - result
	fmov	d0, x10                               //CTR block 4

	fmov	v0.d[1], x9                               //CTR block 4
	rev	w9, w12                                 //CTR block 5
	add	w12, w12, #1                            //CTR block 5

	eor	v5.16b, v5.16b, v1.16b                          //AES block 1 - result
	fmov	d1, x10                               //CTR block 5
	orr	x9, x11, x9, lsl #32            //CTR block 5

	fmov	v1.d[1], x9                               //CTR block 5
	rev	w9, w12                                 //CTR block 6
	st1	{ v4.16b}, [x2], #16                     //AES block 0 - store result

	fmov	v7.d[1], x24                           //AES block 3 - mov high
	orr	x9, x11, x9, lsl #32            //CTR block 6
	eor	v6.16b, v6.16b, v2.16b                          //AES block 2 - result

	st1	{ v5.16b}, [x2], #16                     //AES block 1 - store result

	add	w12, w12, #1                            //CTR block 6
	fmov	d2, x10                               //CTR block 6

	fmov	v2.d[1], x9                               //CTR block 6
	st1	{ v6.16b}, [x2], #16                     //AES block 2 - store result
	rev	w9, w12                                 //CTR block 7

	orr	x9, x11, x9, lsl #32            //CTR block 7

	eor	v7.16b, v7.16b, v3.16b                          //AES block 3 - result
	st1	{ v7.16b}, [x2], #16                     //AES block 3 - store result
	b.ge	.L256_enc_prepretail                               //do prepretail

.L256_enc_main_loop:	//main	loop start
	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	rev64	v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	fmov	d3, x10                               //CTR block 4k+3

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	fmov	v3.d[1], x9                               //CTR block 4k+3

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	ldp	x23, x24, [x0, #48]           //AES block 4k+7 - load plaintext
#ifdef __AARCH64EB__
	rev	x23, x23
	rev	x24, x24
#endif
	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	ldp	x21, x22, [x0, #32]           //AES block 4k+6 - load plaintext
#ifdef __AARCH64EB__
	rev	x21, x21
	rev	x22, x22
#endif
	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	eor	x23, x23, x13                     //AES block 4k+7 - round 14 low

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	eor	x22, x22, x14                     //AES block 4k+6 - round 14 high
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	rev64	v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

	pmull2	v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

	pmull	v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4
	mov	d8, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	eor	v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	ins	v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

	pmull2	v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

	pmull	v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7

	pmull	v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	ldp	x19, x20, [x0, #16]           //AES block 4k+5 - load plaintext
#ifdef __AARCH64EB__
	rev	x19, x19
	rev	x20, x20
#endif
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	mov	d4, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	eor	v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

	pmull2	v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
	eor	v4.8b, v4.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	eor	x19, x19, x13                     //AES block 4k+5 - round 14 low

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9
	eor	v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	eor	x21, x21, x13                     //AES block 4k+6 - round 14 low

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9
	movi	v8.8b, #0xc2

	pmull	v4.1q, v4.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high
	fmov	d5, x19                               //AES block 4k+5 - mov low

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	ldp	x6, x7, [x0, #0]            //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10
	shl	d8, d8, #56               //mod_constant

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8
	eor	v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+3 - mid

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9
	add	w12, w12, #1                            //CTR block 4k+3

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 11
	eor	v4.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 11
	add	x0, x0, #64                       //AES input_ptr update

	pmull	v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	rev	w9, w12                                 //CTR block 4k+8
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10
	eor	x6, x6, x13                     //AES block 4k+4 - round 14 low

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 12
	eor	v10.16b, v10.16b, v4.16b                         //MODULO - karatsuba tidy up

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10
	eor	x7, x7, x14                     //AES block 4k+4 - round 14 high

	fmov	d4, x6                               //AES block 4k+4 - mov low
	orr	x9, x11, x9, lsl #32            //CTR block 4k+8
	eor	v7.16b, v9.16b, v7.16b                   //MODULO - fold into mid

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 12
	eor	x20, x20, x14                     //AES block 4k+5 - round 14 high

	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 11
	eor	x24, x24, x14                     //AES block 4k+7 - round 14 high

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 11
	add	w12, w12, #1                            //CTR block 4k+8

	aese	v0.16b, v31.16b                                     //AES block 4k+4 - round 13
	fmov	v4.d[1], x7                           //AES block 4k+4 - mov high
	eor	v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 12
	fmov	d7, x23                               //AES block 4k+7 - mov low

	aese	v1.16b, v31.16b                                     //AES block 4k+5 - round 13
	fmov	v5.d[1], x20                           //AES block 4k+5 - mov high

	fmov	d6, x21                               //AES block 4k+6 - mov low
	cmp	x0, x5                   //.LOOP CONTROL

	fmov	v6.d[1], x22                           //AES block 4k+6 - mov high

	pmull	v9.1q, v10.1d, v8.1d            //MODULO - mid 64b align with low
	eor	v4.16b, v4.16b, v0.16b                          //AES block 4k+4 - result
	fmov	d0, x10                               //CTR block 4k+8

	fmov	v0.d[1], x9                               //CTR block 4k+8
	rev	w9, w12                                 //CTR block 4k+9
	add	w12, w12, #1                            //CTR block 4k+9

	eor	v5.16b, v5.16b, v1.16b                          //AES block 4k+5 - result
	fmov	d1, x10                               //CTR block 4k+9
	orr	x9, x11, x9, lsl #32            //CTR block 4k+9

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 12
	fmov	v1.d[1], x9                               //CTR block 4k+9

	aese	v2.16b, v31.16b                                     //AES block 4k+6 - round 13
	rev	w9, w12                                 //CTR block 4k+10
	st1	{ v4.16b}, [x2], #16                     //AES block 4k+4 - store result

	orr	x9, x11, x9, lsl #32            //CTR block 4k+10
	eor	v11.16b, v11.16b, v9.16b                         //MODULO - fold into low
	fmov	v7.d[1], x24                           //AES block 4k+7 - mov high

	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment
	st1	{ v5.16b}, [x2], #16                     //AES block 4k+5 - store result
	add	w12, w12, #1                            //CTR block 4k+10

	aese	v3.16b, v31.16b                                     //AES block 4k+7 - round 13
	eor	v6.16b, v6.16b, v2.16b                          //AES block 4k+6 - result
	fmov	d2, x10                               //CTR block 4k+10

	st1	{ v6.16b}, [x2], #16                     //AES block 4k+6 - store result
	fmov	v2.d[1], x9                               //CTR block 4k+10
	rev	w9, w12                                 //CTR block 4k+11

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	orr	x9, x11, x9, lsl #32            //CTR block 4k+11

	eor	v7.16b, v7.16b, v3.16b                          //AES block 4k+7 - result
	st1	{ v7.16b}, [x2], #16                     //AES block 4k+7 - store result
	b.lt	.L256_enc_main_loop

.L256_enc_prepretail:	//PREPRETAIL
	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2 (t0, t1, and t2 free)

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	fmov	d3, x10                               //CTR block 4k+3

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	rev64	v4.16b, v4.16b                                    //GHASH block 4k (only t0 is free)

	fmov	v3.d[1], x9                               //CTR block 4k+3
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1

	eor	v4.16b, v4.16b, v11.16b                           //PRE 1
	rev64	v5.16b, v5.16b                                    //GHASH block 4k+1 (t0 and t1 free)

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid

	pmull2	v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	pmull	v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2

	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3

	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid
	mov	d8, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3 (t0, t1, t2 and t3 free)

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
	eor	v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid
	add	w12, w12, #1                            //CTR block 4k+3

	pmull	v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

	pmull2	v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

	eor	v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low
	ins	v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5

	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high
	mov	d4, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	pmull2	v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

	eor	v4.8b, v4.8b, v7.8b                          //GHASH block 4k+3 - mid

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	pmull	v4.1q, v4.1d, v16.1d                          //GHASH block 4k+3 - mid
	eor	v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	movi	v8.8b, #0xc2

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	shl	d8, d8, #56               //mod_constant

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+3 - mid

	pmull	v6.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	eor	v11.16b, v11.16b, v6.16b                         //GHASH block 4k+3 - low

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9

	eor	v10.16b, v10.16b, v9.16b                         //karatsuba tidy up

	pmull	v4.1q, v9.1d, v8.1d
	ext	v9.16b, v9.16b, v9.16b, #8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	eor	v10.16b, v10.16b, v11.16b

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 11
	eor	v10.16b, v10.16b, v4.16b

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 12

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 11
	eor	v10.16b, v10.16b, v9.16b

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 11

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 12

	pmull	v4.1q, v10.1d, v8.1d

	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 11
	ext	v10.16b, v10.16b, v10.16b, #8

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 12

	aese	v1.16b, v31.16b                                     //AES block 4k+5 - round 13
	eor	v11.16b, v11.16b, v4.16b

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 12

	aese	v3.16b, v31.16b                                     //AES block 4k+7 - round 13

	aese	v0.16b, v31.16b                                     //AES block 4k+4 - round 13

	aese	v2.16b, v31.16b                                     //AES block 4k+6 - round 13
	eor	v11.16b, v11.16b, v10.16b
.L256_enc_tail:	//TAIL

	ext	v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag
	sub	x5, x4, x0   //main_end_input_ptr is number of bytes left to process
	ldp	x6, x7, [x0], #16           //AES block 4k+4 - load plaintext
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	x6, x6, x13                     //AES block 4k+4 - round 14 low
	eor	x7, x7, x14                     //AES block 4k+4 - round 14 high

	cmp	x5, #48
	fmov	d4, x6                               //AES block 4k+4 - mov low

	fmov	v4.d[1], x7                           //AES block 4k+4 - mov high

	eor	v5.16b, v4.16b, v0.16b                          //AES block 4k+4 - result
	b.gt	.L256_enc_blocks_more_than_3

	cmp	x5, #32
	mov	v3.16b, v2.16b
	movi	v11.8b, #0

	movi	v9.8b, #0
	sub	w12, w12, #1

	mov	v2.16b, v1.16b
	movi	v10.8b, #0
	b.gt	.L256_enc_blocks_more_than_2

	mov	v3.16b, v1.16b
	sub	w12, w12, #1
	cmp	x5, #16

	b.gt	.L256_enc_blocks_more_than_1

	sub	w12, w12, #1
	b	.L256_enc_blocks_less_than_1
.L256_enc_blocks_more_than_3:	//blocks	left >  3
	st1	{ v5.16b}, [x2], #16                    //AES final-3 block  - store result

	ldp	x6, x7, [x0], #16          //AES final-2 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	rev64	v4.16b, v5.16b                                   //GHASH final-3 block

	eor	x6, x6, x13                    //AES final-2 block - round 14 low
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	eor	x7, x7, x14                    //AES final-2 block - round 14 high

	mov	d22, v4.d[1]                                //GHASH final-3 block - mid
	fmov	d5, x6                                //AES final-2 block - mov low

	fmov	v5.d[1], x7                            //AES final-2 block - mov high

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid
	movi	v8.8b, #0                                       //suppress further partial tag feed in

	mov	d10, v17.d[1]                              //GHASH final-3 block - mid

	pmull	v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low

	pmull2	v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

	pmull	v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
	eor	v5.16b, v5.16b, v1.16b                           //AES final-2 block - result
.L256_enc_blocks_more_than_2:	//blocks	left >  2

	st1	{ v5.16b}, [x2], #16                    //AES final-2 block - store result

	ldp	x6, x7, [x0], #16          //AES final-1 block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	rev64	v4.16b, v5.16b                                   //GHASH final-2 block

	eor	x6, x6, x13                    //AES final-1 block - round 14 low
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	fmov	d5, x6                                //AES final-1 block - mov low
	eor	x7, x7, x14                    //AES final-1 block - round 14 high

	fmov	v5.d[1], x7                            //AES final-1 block - mov high

	movi	v8.8b, #0                                       //suppress further partial tag feed in

	pmull2	v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high
	mov	d22, v4.d[1]                                //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid

	eor	v5.16b, v5.16b, v2.16b                           //AES final-1 block - result

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high

	pmull	v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low

	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
.L256_enc_blocks_more_than_1:	//blocks	left >  1

	st1	{ v5.16b}, [x2], #16                    //AES final-1 block - store result

	rev64	v4.16b, v5.16b                                   //GHASH final-1 block

	ldp	x6, x7, [x0], #16          //AES final block - load input low & high
#ifdef __AARCH64EB__
	rev	x6, x6
	rev	x7, x7
#endif
	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	movi	v8.8b, #0                                       //suppress further partial tag feed in

	eor	x6, x6, x13                    //AES final block - round 14 low
	mov	d22, v4.d[1]                                //GHASH final-1 block - mid

	pmull2	v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high
	eor	x7, x7, x14                    //AES final block - round 14 high

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

	ins	v22.d[1], v22.d[0]                           //GHASH final-1 block - mid
	fmov	d5, x6                                //AES final block - mov low

	fmov	v5.d[1], x7                            //AES final block - mov high

	pmull2	v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid

	pmull	v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low

	eor	v5.16b, v5.16b, v3.16b                           //AES final block - result
	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low
.L256_enc_blocks_less_than_1:	//blocks	left <= 1

	and	x1, x1, #127                   //bit_length %= 128

	mvn	x13, xzr                                     //rk14_l = 0xffffffffffffffff
	sub	x1, x1, #128                   //bit_length -= 128

	neg	x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])
	ld1	{ v18.16b}, [x2]                           //load existing bytes where the possibly partial last block is to be stored

	mvn	x14, xzr                                     //rk14_h = 0xffffffffffffffff
	and	x1, x1, #127                   //bit_length %= 128

	lsr	x14, x14, x1                    //rk14_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x6, x13, x14, lt
	csel	x7, x14, xzr, lt

	fmov	d0, x6                                //ctr0b is mask for last block

	fmov	v0.d[1], x7

	and	v5.16b, v5.16b, v0.16b                           //possibly partial last block has zeroes in highest bits

	rev64	v4.16b, v5.16b                                   //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	bif	v5.16b, v18.16b, v0.16b                             //insert existing bytes in top end of result before storing

	pmull2	v20.1q, v4.2d, v12.2d                         //GHASH final block - high
	mov	d8, v4.d[1]                                 //GHASH final block - mid
#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif

	pmull	v21.1q, v4.1d, v12.1d                         //GHASH final block - low

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final block - high
	eor	v8.8b, v8.8b, v4.8b                         //GHASH final block - mid

	pmull	v8.1q, v8.1d, v16.1d                         //GHASH final block - mid

	eor	v11.16b, v11.16b, v21.16b                           //GHASH final block - low

	eor	v10.16b, v10.16b, v8.16b                        //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v4.16b, v11.16b, v9.16b                        //MODULO - karatsuba tidy up

	shl	d8, d8, #56              //mod_constant

	eor	v10.16b, v10.16b, v4.16b                        //MODULO - karatsuba tidy up

	pmull	v7.1q, v9.1d, v8.1d           //MODULO - top 64b align with mid

	ext	v9.16b, v9.16b, v9.16b, #8                    //MODULO - other top alignment

	eor	v10.16b, v10.16b, v7.16b                     //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                        //MODULO - fold into mid

	pmull	v9.1q, v10.1d, v8.1d           //MODULO - mid 64b align with low

	ext	v10.16b, v10.16b, v10.16b, #8                    //MODULO - other mid alignment

	str	w9, [x16, #12]                         //store the updated counter

	st1	{ v5.16b}, [x2]                         //store all 16B
	eor	v11.16b, v11.16b, v9.16b                        //MODULO - fold into low

	eor	v11.16b, v11.16b, v10.16b                        //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]

	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L256_enc_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_enc_256_kernel,.-aes_gcm_enc_256_kernel
.globl	aes_gcm_dec_256_kernel
.type	aes_gcm_dec_256_kernel,%function
.align	4
aes_gcm_dec_256_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L256_dec_ret
	stp	x19, x20, [sp, #-112]!
	mov	x16, x4
	mov	x8, x5
	stp	x21, x22, [sp, #16]
	stp	x23, x24, [sp, #32]
	stp	d8, d9, [sp, #48]
	stp	d10, d11, [sp, #64]
	stp	d12, d13, [sp, #80]
	stp	d14, d15, [sp, #96]

	lsr	x5, x1, #3              //byte_len
	mov	x15, x5
	ldp	x10, x11, [x16]              //ctr96_b64, ctr96_t32
#ifdef __AARCH64EB__
	rev	x10, x10
	rev	x11, x11
#endif
	ldp	x13, x14, [x8, #224]                     //load rk14
#ifdef __AARCH64EB__
	ror	x14, x14, #32
	ror	x13, x13, #32
#endif
	ld1	{v18.4s}, [x8], #16                               //load rk0
	sub	x5, x5, #1      //byte_len - 1

	ld1	{v19.4s}, [x8], #16                               //load rk1
	and	x5, x5, #0xffffffffffffffc0 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	add	x4, x0, x1, lsr #3   //end_input_ptr
	ld1	{v20.4s}, [x8], #16                               //load rk2

	lsr	x12, x11, #32
	ld1	{v21.4s}, [x8], #16                               //load rk3
	orr	w11, w11, w11

	ld1	{v22.4s}, [x8], #16                               //load rk4
	add	x5, x5, x0
	rev	w12, w12                                //rev_ctr32

	add	w12, w12, #1                            //increment rev_ctr32
	fmov	d3, x10                               //CTR block 3

	rev	w9, w12                                 //CTR block 1
	add	w12, w12, #1                            //CTR block 1
	fmov	d1, x10                               //CTR block 1

	orr	x9, x11, x9, lsl #32            //CTR block 1
	ld1	{ v0.16b}, [x16]                             //special case vector load initial counter so we can start first AES block as quickly as possible

	fmov	v1.d[1], x9                               //CTR block 1
	rev	w9, w12                                 //CTR block 2
	add	w12, w12, #1                            //CTR block 2

	fmov	d2, x10                               //CTR block 2
	orr	x9, x11, x9, lsl #32            //CTR block 2

	fmov	v2.d[1], x9                               //CTR block 2
	rev	w9, w12                                 //CTR block 3

	orr	x9, x11, x9, lsl #32            //CTR block 3
	ld1	{v23.4s}, [x8], #16                               //load rk5

	fmov	v3.d[1], x9                               //CTR block 3
	add	w12, w12, #1                            //CTR block 3

	ld1	{v24.4s}, [x8], #16                               //load rk6

	ld1	{v25.4s}, [x8], #16                               //load rk7

	ld1	{v26.4s}, [x8], #16                               //load rk8

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 0
	ldr	q14, [x3, #80]                         //load h3l | h3h
#ifndef __AARCH64EB__
	ext	v14.16b, v14.16b, v14.16b, #8
#endif

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 0
	ldr	q15, [x3, #112]                        //load h4l | h4h
#ifndef __AARCH64EB__
	ext	v15.16b, v15.16b, v15.16b, #8
#endif

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 0
	ldr	q13, [x3, #64]                         //load h2l | h2h
#ifndef __AARCH64EB__
	ext	v13.16b, v13.16b, v13.16b, #8
#endif

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 0
	ld1	{v27.4s}, [x8], #16                                 //load rk9

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 1

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 1
	ld1	{ v11.16b}, [x3]
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 1
	ld1	{v28.4s}, [x8], #16                              //load rk10

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 1
	ld1	{v29.4s}, [x8], #16                              //load rk11

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 2
	ldr	q12, [x3, #32]                         //load h1l | h1h
#ifndef __AARCH64EB__
	ext	v12.16b, v12.16b, v12.16b, #8
#endif
	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 2
	ld1	{v30.4s}, [x8], #16                              //load rk12

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 2

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 3

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 2

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 3

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 4
	cmp	x0, x5                   //check if we have <= 4 blocks

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 3

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 3

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 4

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 4

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 4

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 5

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 5

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 5

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 5

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 6

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 6

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 6

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 6

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 7

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 7

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 8

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 8

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 8

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 8
	ld1	{v31.4s}, [x8], #16                             //load rk13

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 9

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 10

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 9

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 10

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 9

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 10

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 11

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 10

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 11

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 11

	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 11

	trn1	v9.2d, v14.2d,    v15.2d                      //h4h | h3h

	trn2	v17.2d,  v14.2d,    v15.2d                      //h4l | h3l

	trn1	v8.2d,    v12.2d,    v13.2d                      //h2h | h1h
	trn2	v16.2d,  v12.2d,    v13.2d                      //h2l | h1l

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 1 - round 12

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 0 - round 12

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 2 - round 12

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 3 - round 12
	eor	v17.16b, v17.16b, v9.16b                  //h4k | h3k

	aese	v1.16b, v31.16b                                     //AES block 1 - round 13

	aese	v2.16b, v31.16b                                     //AES block 2 - round 13
	eor	v16.16b, v16.16b, v8.16b                     //h2k | h1k

	aese	v3.16b, v31.16b                                     //AES block 3 - round 13

	aese	v0.16b, v31.16b                                     //AES block 0 - round 13
	b.ge	.L256_dec_tail                                    //handle tail

	ld1	{v4.16b, v5.16b}, [x0], #32               //AES block 0,1 - load ciphertext

	rev	w9, w12                                 //CTR block 4

	eor	v0.16b, v4.16b, v0.16b                            //AES block 0 - result

	eor	v1.16b, v5.16b, v1.16b                            //AES block 1 - result
	rev64	v5.16b, v5.16b                                    //GHASH block 1
	ld1	{v6.16b}, [x0], #16                       //AES block 2 - load ciphertext

	mov	x7, v0.d[1]                            //AES block 0 - mov high

	mov	x6, v0.d[0]                            //AES block 0 - mov low
	rev64	v4.16b, v4.16b                                    //GHASH block 0
	add	w12, w12, #1                            //CTR block 4

	fmov	d0, x10                               //CTR block 4
	orr	x9, x11, x9, lsl #32            //CTR block 4

	fmov	v0.d[1], x9                               //CTR block 4
	rev	w9, w12                                 //CTR block 5
	add	w12, w12, #1                            //CTR block 5

	mov	x19, v1.d[0]                            //AES block 1 - mov low

	orr	x9, x11, x9, lsl #32            //CTR block 5
	mov	x20, v1.d[1]                            //AES block 1 - mov high
	eor	x7, x7, x14                   //AES block 0 - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	eor	x6, x6, x13                   //AES block 0 - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	stp	x6, x7, [x2], #16        //AES block 0 - store result
	fmov	d1, x10                               //CTR block 5

	ld1	{v7.16b}, [x0], #16                       //AES block 3 - load ciphertext

	fmov	v1.d[1], x9                               //CTR block 5
	rev	w9, w12                                 //CTR block 6
	add	w12, w12, #1                            //CTR block 6

	eor	x19, x19, x13                   //AES block 1 - round 14 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	orr	x9, x11, x9, lsl #32            //CTR block 6

	eor	x20, x20, x14                   //AES block 1 - round 14 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	stp	x19, x20, [x2], #16        //AES block 1 - store result

	eor	v2.16b, v6.16b, v2.16b                            //AES block 2 - result
	cmp	x0, x5                   //check if we have <= 8 blocks
	b.ge	.L256_dec_prepretail                              //do prepretail

.L256_dec_main_loop:	//main	loop start
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	fmov	d2, x10                               //CTR block 4k+6

	fmov	v2.d[1], x9                               //CTR block 4k+6
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1
	rev	w9, w12                                 //CTR block 4k+7

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid
	fmov	d3, x10                               //CTR block 4k+7

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2
	orr	x9, x11, x9, lsl #32            //CTR block 4k+7

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	fmov	v3.d[1], x9                               //CTR block 4k+7

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	eor	x22, x22, x14                   //AES block 4k+2 - round 14 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0
	eor	x21, x21, x13                   //AES block 4k+2 - round 14 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif
	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low

	pmull2	v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	eor	x23, x23, x13                   //AES block 4k+3 - round 14 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	pmull	v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low
	eor	x24, x24, x14                   //AES block 4k+3 - round 14 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5
	add	w12, w12, #1                            //CTR block 4k+7

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	mov	d8, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4
	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

	pmull	v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	eor	v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid
	rev	w9, w12                                 //CTR block 4k+8

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6
	ins	v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	add	w12, w12, #1                            //CTR block 4k+8

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7

	pmull2	v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high
	mov	d6, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5

	pmull2	v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6

	pmull	v4.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low
	orr	x9, x11, x9, lsl #32            //CTR block 4k+8
	eor	v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9
	eor	v6.8b, v6.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10

	pmull	v6.1q, v6.1d, v16.1d                          //GHASH block 4k+3 - mid
	movi	v8.8b, #0xc2

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	eor	v11.16b, v11.16b, v4.16b                         //GHASH block 4k+3 - low

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 11

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	shl	d8, d8, #56               //mod_constant

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8
	eor	v10.16b, v10.16b, v6.16b                         //GHASH block 4k+3 - mid

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 12

	pmull	v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid
	eor	v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9
	ld1	{v4.16b}, [x0], #16                       //AES block 4k+4 - load ciphertext

	aese	v0.16b, v31.16b                                     //AES block 4k+4 - round 13
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10
	eor	v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9
	ld1	{v5.16b}, [x0], #16                       //AES block 4k+5 - load ciphertext

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8
	eor	v0.16b, v4.16b, v0.16b                            //AES block 4k+4 - result

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 11
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10
	eor	v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9
	ld1	{v6.16b}, [x0], #16                       //AES block 4k+6 - load ciphertext

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 12
	ld1	{v7.16b}, [x0], #16                       //AES block 4k+7 - load ciphertext

	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 11
	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	aese	v1.16b, v31.16b                                     //AES block 4k+5 - round 13
	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 12
	fmov	d0, x10                               //CTR block 4k+8

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 11
	fmov	v0.d[1], x9                               //CTR block 4k+8

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	eor	v1.16b, v5.16b, v1.16b                            //AES block 4k+5 - result
	rev	w9, w12                                 //CTR block 4k+9

	aese	v2.16b, v31.16b                                     //AES block 4k+6 - round 13
	orr	x9, x11, x9, lsl #32            //CTR block 4k+9
	cmp	x0, x5                   //.LOOP CONTROL

	add	w12, w12, #1                            //CTR block 4k+9

	eor	x6, x6, x13                   //AES block 4k+4 - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	x7, x7, x14                   //AES block 4k+4 - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	mov	x20, v1.d[1]                            //AES block 4k+5 - mov high
	eor	v2.16b, v6.16b, v2.16b                            //AES block 4k+6 - result
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 12
	mov	x19, v1.d[0]                            //AES block 4k+5 - mov low

	fmov	d1, x10                               //CTR block 4k+9
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	fmov	v1.d[1], x9                               //CTR block 4k+9
	rev	w9, w12                                 //CTR block 4k+10
	add	w12, w12, #1                            //CTR block 4k+10

	aese	v3.16b, v31.16b                                     //AES block 4k+7 - round 13
	orr	x9, x11, x9, lsl #32            //CTR block 4k+10

	rev64	v5.16b, v5.16b                                    //GHASH block 4k+5
	eor	x20, x20, x14                   //AES block 4k+5 - round 14 high
#ifdef __AARCH64EB__
	rev	x20, x20
#endif
	stp	x6, x7, [x2], #16        //AES block 4k+4 - store result

	eor	x19, x19, x13                   //AES block 4k+5 - round 14 low
#ifdef __AARCH64EB__
	rev	x19, x19
#endif
	stp	x19, x20, [x2], #16        //AES block 4k+5 - store result

	rev64	v4.16b, v4.16b                                    //GHASH block 4k+4
	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	b.lt	.L256_dec_main_loop


.L256_dec_prepretail:	//PREPRETAIL
	ext	v11.16b, v11.16b, v11.16b, #8                     //PRE 0
	mov	x21, v2.d[0]                            //AES block 4k+2 - mov low
	eor	v3.16b, v7.16b, v3.16b                            //AES block 4k+3 - result

	aese	v0.16b, v18.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 0
	mov	x22, v2.d[1]                            //AES block 4k+2 - mov high

	aese	v1.16b, v18.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 0
	fmov	d2, x10                               //CTR block 4k+6

	fmov	v2.d[1], x9                               //CTR block 4k+6
	rev	w9, w12                                 //CTR block 4k+7
	eor	v4.16b, v4.16b, v11.16b                           //PRE 1

	rev64	v6.16b, v6.16b                                    //GHASH block 4k+2
	orr	x9, x11, x9, lsl #32            //CTR block 4k+7
	mov	x23, v3.d[0]                            //AES block 4k+3 - mov low

	aese	v1.16b, v19.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 1
	mov	x24, v3.d[1]                            //AES block 4k+3 - mov high

	pmull	v11.1q, v4.1d, v15.1d                       //GHASH block 4k - low
	mov	d8, v4.d[1]                                  //GHASH block 4k - mid
	fmov	d3, x10                               //CTR block 4k+7

	pmull2	v9.1q, v4.2d, v15.2d                       //GHASH block 4k - high
	fmov	v3.d[1], x9                               //CTR block 4k+7

	aese	v2.16b, v18.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 0
	mov	d10, v17.d[1]                               //GHASH block 4k - mid

	aese	v0.16b, v19.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 1
	eor	v8.8b, v8.8b, v4.8b                          //GHASH block 4k - mid

	pmull2	v4.1q, v5.2d, v14.2d                          //GHASH block 4k+1 - high

	aese	v2.16b, v19.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 1
	rev64	v7.16b, v7.16b                                    //GHASH block 4k+3

	aese	v3.16b, v18.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 0

	pmull	v10.1q, v8.1d, v10.1d                      //GHASH block 4k - mid
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+1 - high

	pmull	v8.1q, v5.1d, v14.1d                          //GHASH block 4k+1 - low

	aese	v3.16b, v19.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 1
	mov	d4, v5.d[1]                                  //GHASH block 4k+1 - mid

	aese	v0.16b, v20.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 2

	aese	v1.16b, v20.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 2
	eor	v11.16b, v11.16b, v8.16b                         //GHASH block 4k+1 - low

	aese	v2.16b, v20.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 2

	aese	v0.16b, v21.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 3
	mov	d8, v6.d[1]                                  //GHASH block 4k+2 - mid

	aese	v3.16b, v20.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 2
	eor	v4.8b, v4.8b, v5.8b                          //GHASH block 4k+1 - mid

	pmull	v5.1q, v6.1d, v13.1d                          //GHASH block 4k+2 - low

	aese	v0.16b, v22.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 4

	aese	v3.16b, v21.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 3
	eor	v8.8b, v8.8b, v6.8b                          //GHASH block 4k+2 - mid

	pmull	v4.1q, v4.1d, v17.1d                          //GHASH block 4k+1 - mid

	aese	v0.16b, v23.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 5
	eor	v11.16b, v11.16b, v5.16b                         //GHASH block 4k+2 - low

	aese	v3.16b, v22.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 4

	pmull2	v5.1q, v7.2d, v12.2d                          //GHASH block 4k+3 - high
	eor	v10.16b, v10.16b, v4.16b                         //GHASH block 4k+1 - mid

	pmull2	v4.1q, v6.2d, v13.2d                          //GHASH block 4k+2 - high

	aese	v3.16b, v23.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 5
	ins	v8.d[1], v8.d[0]                                //GHASH block 4k+2 - mid

	aese	v2.16b, v21.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 3

	aese	v1.16b, v21.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 3
	eor	v9.16b, v9.16b, v4.16b                         //GHASH block 4k+2 - high

	pmull	v4.1q, v7.1d, v12.1d                          //GHASH block 4k+3 - low

	aese	v2.16b, v22.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 4
	mov	d6, v7.d[1]                                  //GHASH block 4k+3 - mid

	aese	v1.16b, v22.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 4

	pmull2	v8.1q, v8.2d, v16.2d                          //GHASH block 4k+2 - mid

	aese	v2.16b, v23.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 5
	eor	v6.8b, v6.8b, v7.8b                          //GHASH block 4k+3 - mid

	aese	v1.16b, v23.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 5

	aese	v3.16b, v24.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 6
	eor	v10.16b, v10.16b, v8.16b                         //GHASH block 4k+2 - mid

	aese	v2.16b, v24.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 6

	aese	v0.16b, v24.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 6
	movi	v8.8b, #0xc2

	aese	v1.16b, v24.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 6
	eor	v11.16b, v11.16b, v4.16b                         //GHASH block 4k+3 - low

	pmull	v6.1q, v6.1d, v16.1d                          //GHASH block 4k+3 - mid

	aese	v3.16b, v25.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 7
	eor	v9.16b, v9.16b, v5.16b                         //GHASH block 4k+3 - high

	aese	v1.16b, v25.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 7

	aese	v0.16b, v25.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 7
	eor	v10.16b, v10.16b, v6.16b                         //GHASH block 4k+3 - mid

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 8

	aese	v2.16b, v25.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 7
	eor	v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 8

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 8
	shl	d8, d8, #56               //mod_constant

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 8

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 9
	eor	v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

	pmull	v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 9
	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 9

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 9
	eor	v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 10

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 10

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 10
	eor	x22, x22, x14                   //AES block 4k+2 - round 14 high
#ifdef __AARCH64EB__
	rev	x22, x22
#endif
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 10
	eor	x23, x23, x13                   //AES block 4k+3 - round 14 low
#ifdef __AARCH64EB__
	rev	x23, x23
#endif
	aese	v2.16b, v29.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 11
	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	aese	v0.16b, v29.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 11
	add	w12, w12, #1                            //CTR block 4k+7

	aese	v1.16b, v29.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 11
	eor	x21, x21, x13                   //AES block 4k+2 - round 14 low
#ifdef __AARCH64EB__
	rev	x21, x21
#endif

	aese	v2.16b, v30.16b
	aesmc	v2.16b, v2.16b          //AES block 4k+6 - round 12

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low
	eor	x24, x24, x14                   //AES block 4k+3 - round 14 high
#ifdef __AARCH64EB__
	rev	x24, x24
#endif

	aese	v3.16b, v29.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 11
	stp	x21, x22, [x2], #16        //AES block 4k+2 - store result

	aese	v1.16b, v30.16b
	aesmc	v1.16b, v1.16b          //AES block 4k+5 - round 12
	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	aese	v0.16b, v30.16b
	aesmc	v0.16b, v0.16b          //AES block 4k+4 - round 12
	stp	x23, x24, [x2], #16        //AES block 4k+3 - store result

	aese	v3.16b, v30.16b
	aesmc	v3.16b, v3.16b          //AES block 4k+7 - round 12
	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	aese	v1.16b, v31.16b                                     //AES block 4k+5 - round 13

	aese	v0.16b, v31.16b                                     //AES block 4k+4 - round 13

	aese	v3.16b, v31.16b                                     //AES block 4k+7 - round 13

	aese	v2.16b, v31.16b                                     //AES block 4k+6 - round 13
	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
.L256_dec_tail:	//TAIL

	sub	x5, x4, x0   //main_end_input_ptr is number of bytes left to process
	ld1	{ v5.16b}, [x0], #16                      //AES block 4k+4 - load ciphertext

	eor	v0.16b, v5.16b, v0.16b                            //AES block 4k+4 - result

	mov	x6, v0.d[0]                            //AES block 4k+4 - mov low

	mov	x7, v0.d[1]                            //AES block 4k+4 - mov high
	ext	v8.16b, v11.16b, v11.16b, #8                     //prepare final partial tag

	cmp	x5, #48

	eor	x6, x6, x13                   //AES block 4k+4 - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif

	eor	x7, x7, x14                   //AES block 4k+4 - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
	b.gt	.L256_dec_blocks_more_than_3

	sub	w12, w12, #1
	mov	v3.16b, v2.16b
	movi	v10.8b, #0

	movi	v11.8b, #0
	cmp	x5, #32

	movi	v9.8b, #0
	mov	v2.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_2

	sub	w12, w12, #1

	mov	v3.16b, v1.16b
	cmp	x5, #16
	b.gt	.L256_dec_blocks_more_than_1

	sub	w12, w12, #1
	b	.L256_dec_blocks_less_than_1
.L256_dec_blocks_more_than_3:	//blocks	left >  3
	rev64	v4.16b, v5.16b                                   //GHASH final-3 block
	ld1	{ v5.16b}, [x0], #16                     //AES final-2 block - load ciphertext

	stp	x6, x7, [x2], #16       //AES final-3 block  - store result

	mov	d10, v17.d[1]                              //GHASH final-3 block - mid

	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag

	eor	v0.16b, v5.16b, v1.16b                           //AES final-2 block - result

	mov	d22, v4.d[1]                                //GHASH final-3 block - mid

	mov	x6, v0.d[0]                           //AES final-2 block - mov low

	mov	x7, v0.d[1]                           //AES final-2 block - mov high

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-3 block - mid

	movi	v8.8b, #0                                       //suppress further partial tag feed in

	pmull2	v9.1q, v4.2d, v15.2d                      //GHASH final-3 block - high

	pmull	v10.1q, v22.1d, v10.1d                   //GHASH final-3 block - mid
	eor	x6, x6, x13                  //AES final-2 block - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif

	pmull	v11.1q, v4.1d, v15.1d                      //GHASH final-3 block - low
	eor	x7, x7, x14                  //AES final-2 block - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
.L256_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v4.16b, v5.16b                                   //GHASH final-2 block
	ld1	{ v5.16b}, [x0], #16                     //AES final-1 block - load ciphertext

	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag
	stp	x6, x7, [x2], #16       //AES final-2 block  - store result

	eor	v0.16b, v5.16b, v2.16b                           //AES final-1 block - result

	mov	d22, v4.d[1]                                //GHASH final-2 block - mid

	pmull	v21.1q, v4.1d, v14.1d                         //GHASH final-2 block - low

	pmull2	v20.1q, v4.2d, v14.2d                         //GHASH final-2 block - high

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-2 block - mid
	mov	x6, v0.d[0]                           //AES final-1 block - mov low

	mov	x7, v0.d[1]                           //AES final-1 block - mov high
	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-2 block - low
	movi	v8.8b, #0                                       //suppress further partial tag feed in

	pmull	v22.1q, v22.1d, v17.1d                     //GHASH final-2 block - mid

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-2 block - high
	eor	x6, x6, x13                  //AES final-1 block - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif

	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-2 block - mid
	eor	x7, x7, x14                  //AES final-1 block - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
.L256_dec_blocks_more_than_1:	//blocks	left >  1

	stp	x6, x7, [x2], #16       //AES final-1 block  - store result
	rev64	v4.16b, v5.16b                                   //GHASH final-1 block

	ld1	{ v5.16b}, [x0], #16                     //AES final block - load ciphertext

	eor	v4.16b, v4.16b, v8.16b                          //feed in partial tag
	movi	v8.8b, #0                                       //suppress further partial tag feed in

	mov	d22, v4.d[1]                                //GHASH final-1 block - mid

	eor	v0.16b, v5.16b, v3.16b                           //AES final block - result

	pmull2	v20.1q, v4.2d, v13.2d                         //GHASH final-1 block - high

	eor	v22.8b, v22.8b, v4.8b                     //GHASH final-1 block - mid

	pmull	v21.1q, v4.1d, v13.1d                         //GHASH final-1 block - low
	mov	x6, v0.d[0]                           //AES final block - mov low

	ins	v22.d[1], v22.d[0]                           //GHASH final-1 block - mid

	mov	x7, v0.d[1]                           //AES final block - mov high

	pmull2	v22.1q, v22.2d, v16.2d                     //GHASH final-1 block - mid
	eor	x6, x6, x13                  //AES final block - round 14 low
#ifdef __AARCH64EB__
	rev	x6, x6
#endif
	eor	v11.16b, v11.16b, v21.16b                           //GHASH final-1 block - low

	eor	v9.16b, v9.16b, v20.16b                           //GHASH final-1 block - high

	eor	v10.16b, v10.16b, v22.16b                      //GHASH final-1 block - mid
	eor	x7, x7, x14                  //AES final block - round 14 high
#ifdef __AARCH64EB__
	rev	x7, x7
#endif
.L256_dec_blocks_less_than_1:	//blocks	left <= 1

	and	x1, x1, #127                   //bit_length %= 128
	mvn	x14, xzr                                     //rk14_h = 0xffffffffffffffff

	sub	x1, x1, #128                   //bit_length -= 128
	mvn	x13, xzr                                     //rk14_l = 0xffffffffffffffff

	ldp	x4, x5, [x2] //load existing bytes we need to not overwrite
	neg	x1, x1                         //bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127                   //bit_length %= 128

	lsr	x14, x14, x1                    //rk14_h is mask for top 64b of last block
	cmp	x1, #64

	csel	x9, x13, x14, lt
	csel	x10, x14, xzr, lt

	fmov	d0, x9                                  //ctr0b is mask for last block
	and	x6, x6, x9

	mov	v0.d[1], x10
	bic	x4, x4, x9          //mask out low existing bytes

#ifndef __AARCH64EB__
	rev	w9, w12
#else
	mov	w9, w12
#endif

	bic	x5, x5, x10      //mask out high existing bytes

	orr	x6, x6, x4

	and	x7, x7, x10

	orr	x7, x7, x5

	and	v5.16b, v5.16b, v0.16b                            //possibly partial last block has zeroes in highest bits

	rev64	v4.16b, v5.16b                                    //GHASH final block

	eor	v4.16b, v4.16b, v8.16b                           //feed in partial tag

	pmull	v21.1q, v4.1d, v12.1d                          //GHASH final block - low

	mov	d8, v4.d[1]                                  //GHASH final block - mid

	eor	v8.8b, v8.8b, v4.8b                          //GHASH final block - mid

	pmull2	v20.1q, v4.2d, v12.2d                          //GHASH final block - high

	pmull	v8.1q, v8.1d, v16.1d                          //GHASH final block - mid

	eor	v9.16b, v9.16b, v20.16b                            //GHASH final block - high

	eor	v11.16b, v11.16b, v21.16b                            //GHASH final block - low

	eor	v10.16b, v10.16b, v8.16b                         //GHASH final block - mid
	movi	v8.8b, #0xc2

	eor	v6.16b, v11.16b, v9.16b                         //MODULO - karatsuba tidy up

	shl	d8, d8, #56               //mod_constant

	eor	v10.16b, v10.16b, v6.16b                         //MODULO - karatsuba tidy up

	pmull	v7.1q, v9.1d, v8.1d            //MODULO - top 64b align with mid

	ext	v9.16b, v9.16b, v9.16b, #8                     //MODULO - other top alignment

	eor	v10.16b, v10.16b, v7.16b                      //MODULO - fold into mid

	eor	v10.16b, v10.16b, v9.16b                         //MODULO - fold into mid

	pmull	v8.1q, v10.1d, v8.1d     //MODULO - mid 64b align with low

	ext	v10.16b, v10.16b, v10.16b, #8                     //MODULO - other mid alignment

	eor	v11.16b, v11.16b, v8.16b               //MODULO - fold into low

	stp	x6, x7, [x2]

	str	w9, [x16, #12]                          //store the updated counter

	eor	v11.16b, v11.16b, v10.16b                         //MODULO - fold into low
	ext	v11.16b, v11.16b, v11.16b, #8
	rev64	v11.16b, v11.16b
	mov	x0, x15
	st1	{ v11.16b }, [x3]

	ldp	x21, x22, [sp, #16]
	ldp	x23, x24, [sp, #32]
	ldp	d8, d9, [sp, #48]
	ldp	d10, d11, [sp, #64]
	ldp	d12, d13, [sp, #80]
	ldp	d14, d15, [sp, #96]
	ldp	x19, x20, [sp], #112
	ret

.L256_dec_ret:
	mov	w0, #0x0
	ret
.size	aes_gcm_dec_256_kernel,.-aes_gcm_dec_256_kernel
.section	.rodata
.byte	71,72,65,83,72,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
.align	2
.align	2
#endif