Home | History | Annotate | Line # | Download | only in asm 
      1 #! /usr/bin/env perl
      2 # Copyright 2014-2023 The OpenSSL Project Authors. All Rights Reserved.
      3 #
      4 # Licensed under the Apache License 2.0 (the "License").  You may not use
      5 # this file except in compliance with the License.  You can obtain a copy
      6 # in the file LICENSE in the source distribution or at
      7 # https://www.openssl.org/source/license.html
      8 
      9 #
     10 # ====================================================================
     11 # Written by Andy Polyakov <appro (at] openssl.org> for the OpenSSL
     12 # project. The module is, however, dual licensed under OpenSSL and
     13 # CRYPTOGAMS licenses depending on where you obtain it. For further
     14 # details see http://www.openssl.org/~appro/cryptogams/.
     15 # ====================================================================
     16 #
     17 # GHASH for ARMv8 Crypto Extension, 64-bit polynomial multiplication.
     18 #
     19 # June 2014
     20 #
     21 # Initial version was developed in tight cooperation with Ard
     22 # Biesheuvel of Linaro from bits-n-pieces from other assembly modules.
     23 # Just like aesv8-armx.pl this module supports both AArch32 and
     24 # AArch64 execution modes.
     25 #
     26 # July 2014
     27 #
     28 # Implement 2x aggregated reduction [see ghash-x86.pl for background
     29 # information].
     30 #
     31 # November 2017
     32 #
     33 # AArch64 register bank to "accommodate" 4x aggregated reduction and
     34 # improve performance by 20-70% depending on processor.
     35 #
     36 # Current performance in cycles per processed byte:
     37 #
     38 #		64-bit PMULL	32-bit PMULL	32-bit NEON(*)
     39 # Apple A7	0.58		0.92		5.62
     40 # Cortex-A53	0.85		1.01		8.39
     41 # Cortex-A57	0.73		1.17		7.61
     42 # Denver	0.51		0.65		6.02
     43 # Mongoose	0.65		1.10		8.06
     44 # Kryo		0.76		1.16		8.00
     45 # ThunderX2	1.05
     46 #
     47 # (*)	presented for reference/comparison purposes;
     48 
     49 # $output is the last argument if it looks like a file (it has an extension)
     50 # $flavour is the first argument if it doesn't look like a file
     51 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
     52 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
     53 
     54 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
     55 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
     56 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
     57 die "can't locate arm-xlate.pl";
     58 
     59 open OUT,"| \"$^X\" $xlate $flavour \"$output\""
     60     or die "can't call $xlate: $!";
     61 *STDOUT=*OUT;
     62 
     63 $Xi="x0";	# argument block
     64 $Htbl="x1";
     65 $inp="x2";
     66 $len="x3";
     67 
     68 $inc="x12";
     69 
     70 {
     71 my ($Xl,$Xm,$Xh,$IN)=map("q$_",(0..3));
     72 my ($t0,$t1,$t2,$xC2,$H,$Hhl,$H2)=map("q$_",(8..14));
     73 my $_byte = ($flavour =~ /win/ ? "DCB" : ".byte");
     74 
     75 $code=<<___;
     76 #include "arm_arch.h"
     77 
     78 #if __ARM_MAX_ARCH__>=7
     79 ___
     80 $code.=".arch	armv8-a+crypto\n.text\n"	if ($flavour =~ /64/);
     81 $code.=<<___					if ($flavour !~ /64/);
     82 .fpu	neon
     83 #ifdef __thumb2__
     84 .syntax        unified
     85 .thumb
     86 # define INST(a,b,c,d) $_byte  c,0xef,a,b
     87 #else
     88 .code  32
     89 # define INST(a,b,c,d) $_byte  a,b,c,0xf2
     90 #endif
     91 
     92 .text
     93 ___
     94 
     95 ################################################################################
     96 # void gcm_init_v8(u128 Htable[16],const u64 H[2]);
     97 #
     98 # input:	128-bit H - secret parameter E(K,0^128)
     99 # output:	precomputed table filled with degrees of twisted H;
    100 #		H is twisted to handle reverse bitness of GHASH;
    101 #		only few of 16 slots of Htable[16] are used;
    102 #		data is opaque to outside world (which allows to
    103 #		optimize the code independently);
    104 #
    105 $code.=<<___;
    106 .global	gcm_init_v8
    107 .type	gcm_init_v8,%function
    108 .align	4
    109 gcm_init_v8:
    110 	vld1.64		{$t1},[x1]		@ load input H
    111 	vmov.i8		$xC2,#0xe1
    112 	vshl.i64	$xC2,$xC2,#57		@ 0xc2.0
    113 	vext.8		$IN,$t1,$t1,#8
    114 	vshr.u64	$t2,$xC2,#63
    115 	vdup.32		$t1,${t1}[1]
    116 	vext.8		$t0,$t2,$xC2,#8		@ t0=0xc2....01
    117 	vshr.u64	$t2,$IN,#63
    118 	vshr.s32	$t1,$t1,#31		@ broadcast carry bit
    119 	vand		$t2,$t2,$t0
    120 	vshl.i64	$IN,$IN,#1
    121 	vext.8		$t2,$t2,$t2,#8
    122 	vand		$t0,$t0,$t1
    123 	vorr		$IN,$IN,$t2		@ H<<<=1
    124 	veor		$H,$IN,$t0		@ twisted H
    125 	vst1.64		{$H},[x0],#16		@ store Htable[0]
    126 
    127 	@ calculate H^2
    128 	vext.8		$t0,$H,$H,#8		@ Karatsuba pre-processing
    129 	vpmull.p64	$Xl,$H,$H
    130 	veor		$t0,$t0,$H
    131 	vpmull2.p64	$Xh,$H,$H
    132 	vpmull.p64	$Xm,$t0,$t0
    133 
    134 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    135 	veor		$t2,$Xl,$Xh
    136 	veor		$Xm,$Xm,$t1
    137 	veor		$Xm,$Xm,$t2
    138 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase
    139 
    140 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    141 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    142 	veor		$Xl,$Xm,$t2
    143 
    144 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
    145 	vpmull.p64	$Xl,$Xl,$xC2
    146 	veor		$t2,$t2,$Xh
    147 	veor		$H2,$Xl,$t2
    148 
    149 	vext.8		$t1,$H2,$H2,#8		@ Karatsuba pre-processing
    150 	veor		$t1,$t1,$H2
    151 	vext.8		$Hhl,$t0,$t1,#8		@ pack Karatsuba pre-processed
    152 	vst1.64		{$Hhl-$H2},[x0],#32	@ store Htable[1..2]
    153 ___
    154 if ($flavour =~ /64/) {
    155 my ($t3,$Yl,$Ym,$Yh) = map("q$_",(4..7));
    156 
    157 $code.=<<___;
    158 	@ calculate H^3 and H^4
    159 	vpmull.p64	$Xl,$H, $H2
    160 	 vpmull.p64	$Yl,$H2,$H2
    161 	vpmull2.p64	$Xh,$H, $H2
    162 	 vpmull2.p64	$Yh,$H2,$H2
    163 	vpmull.p64	$Xm,$t0,$t1
    164 	 vpmull.p64	$Ym,$t1,$t1
    165 
    166 	vext.8		$t0,$Xl,$Xh,#8		@ Karatsuba post-processing
    167 	 vext.8		$t1,$Yl,$Yh,#8
    168 	veor		$t2,$Xl,$Xh
    169 	veor		$Xm,$Xm,$t0
    170 	 veor		$t3,$Yl,$Yh
    171 	 veor		$Ym,$Ym,$t1
    172 	veor		$Xm,$Xm,$t2
    173 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase
    174 	 veor		$Ym,$Ym,$t3
    175 	 vpmull.p64	$t3,$Yl,$xC2
    176 
    177 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    178 	 vmov		$Yh#lo,$Ym#hi
    179 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    180 	 vmov		$Ym#hi,$Yl#lo
    181 	veor		$Xl,$Xm,$t2
    182 	 veor		$Yl,$Ym,$t3
    183 
    184 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase
    185 	 vext.8		$t3,$Yl,$Yl,#8
    186 	vpmull.p64	$Xl,$Xl,$xC2
    187 	 vpmull.p64	$Yl,$Yl,$xC2
    188 	veor		$t2,$t2,$Xh
    189 	 veor		$t3,$t3,$Yh
    190 	veor		$H, $Xl,$t2		@ H^3
    191 	 veor		$H2,$Yl,$t3		@ H^4
    192 
    193 	vext.8		$t0,$H, $H,#8		@ Karatsuba pre-processing
    194 	 vext.8		$t1,$H2,$H2,#8
    195 	veor		$t0,$t0,$H
    196 	 veor		$t1,$t1,$H2
    197 	vext.8		$Hhl,$t0,$t1,#8		@ pack Karatsuba pre-processed
    198 	vst1.64		{$H-$H2},[x0]		@ store Htable[3..5]
    199 ___
    200 }
    201 $code.=<<___;
    202 	ret
    203 .size	gcm_init_v8,.-gcm_init_v8
    204 ___
    205 ################################################################################
    206 # void gcm_gmult_v8(u64 Xi[2],const u128 Htable[16]);
    207 #
    208 # input:	Xi - current hash value;
    209 #		Htable - table precomputed in gcm_init_v8;
    210 # output:	Xi - next hash value Xi;
    211 #
    212 $code.=<<___;
    213 .global	gcm_gmult_v8
    214 .type	gcm_gmult_v8,%function
    215 .align	4
    216 gcm_gmult_v8:
    217 	vld1.64		{$t1},[$Xi]		@ load Xi
    218 	vmov.i8		$xC2,#0xe1
    219 	vld1.64		{$H-$Hhl},[$Htbl]	@ load twisted H, ...
    220 	vshl.u64	$xC2,$xC2,#57
    221 #ifndef __ARMEB__
    222 	vrev64.8	$t1,$t1
    223 #endif
    224 	vext.8		$IN,$t1,$t1,#8
    225 
    226 	vpmull.p64	$Xl,$H,$IN		@ H.loXi.lo
    227 	veor		$t1,$t1,$IN		@ Karatsuba pre-processing
    228 	vpmull2.p64	$Xh,$H,$IN		@ H.hiXi.hi
    229 	vpmull.p64	$Xm,$Hhl,$t1		@ (H.lo+H.hi)(Xi.lo+Xi.hi)
    230 
    231 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    232 	veor		$t2,$Xl,$Xh
    233 	veor		$Xm,$Xm,$t1
    234 	veor		$Xm,$Xm,$t2
    235 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    236 
    237 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    238 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    239 	veor		$Xl,$Xm,$t2
    240 
    241 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    242 	vpmull.p64	$Xl,$Xl,$xC2
    243 	veor		$t2,$t2,$Xh
    244 	veor		$Xl,$Xl,$t2
    245 
    246 #ifndef __ARMEB__
    247 	vrev64.8	$Xl,$Xl
    248 #endif
    249 	vext.8		$Xl,$Xl,$Xl,#8
    250 	vst1.64		{$Xl},[$Xi]		@ write out Xi
    251 
    252 	ret
    253 .size	gcm_gmult_v8,.-gcm_gmult_v8
    254 ___
    255 ################################################################################
    256 # void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
    257 #
    258 # input:	table precomputed in gcm_init_v8;
    259 #		current hash value Xi;
    260 #		pointer to input data;
    261 #		length of input data in bytes, but divisible by block size;
    262 # output:	next hash value Xi;
    263 #
    264 $code.=<<___;
    265 .global	gcm_ghash_v8
    266 .type	gcm_ghash_v8,%function
    267 .align	4
    268 gcm_ghash_v8:
    269 ___
    270 $code.=<<___	if ($flavour =~ /64/);
    271 	cmp		$len,#64
    272 	b.hs		.Lgcm_ghash_v8_4x
    273 ___
    274 $code.=<<___		if ($flavour !~ /64/);
    275 	vstmdb		sp!,{d8-d15}		@ 32-bit ABI says so
    276 ___
    277 $code.=<<___;
    278 	vld1.64		{$Xl},[$Xi]		@ load [rotated] Xi
    279 						@ "[rotated]" means that
    280 						@ loaded value would have
    281 						@ to be rotated in order to
    282 						@ make it appear as in
    283 						@ algorithm specification
    284 	subs		$len,$len,#32		@ see if $len is 32 or larger
    285 	mov		$inc,#16		@ $inc is used as post-
    286 						@ increment for input pointer;
    287 						@ as loop is modulo-scheduled
    288 						@ $inc is zeroed just in time
    289 						@ to preclude overstepping
    290 						@ inp[len], which means that
    291 						@ last block[s] are actually
    292 						@ loaded twice, but last
    293 						@ copy is not processed
    294 	vld1.64		{$H-$Hhl},[$Htbl],#32	@ load twisted H, ..., H^2
    295 	vmov.i8		$xC2,#0xe1
    296 	vld1.64		{$H2},[$Htbl]
    297 	cclr		$inc,eq			@ is it time to zero $inc?
    298 	vext.8		$Xl,$Xl,$Xl,#8		@ rotate Xi
    299 	vld1.64		{$t0},[$inp],#16	@ load [rotated] I[0]
    300 	vshl.u64	$xC2,$xC2,#57		@ compose 0xc2.0 constant
    301 #ifndef __ARMEB__
    302 	vrev64.8	$t0,$t0
    303 	vrev64.8	$Xl,$Xl
    304 #endif
    305 	vext.8		$IN,$t0,$t0,#8		@ rotate I[0]
    306 	b.lo		.Lodd_tail_v8		@ $len was less than 32
    307 ___
    308 { my ($Xln,$Xmn,$Xhn,$In) = map("q$_",(4..7));
    309 	#######
    310 	# Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
    311 	#	[(H*Ii+1) + (H*Xi+1)] mod P =
    312 	#	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
    313 	#
    314 $code.=<<___;
    315 	vld1.64		{$t1},[$inp],$inc	@ load [rotated] I[1]
    316 #ifndef __ARMEB__
    317 	vrev64.8	$t1,$t1
    318 #endif
    319 	vext.8		$In,$t1,$t1,#8
    320 	veor		$IN,$IN,$Xl		@ I[i]^=Xi
    321 	vpmull.p64	$Xln,$H,$In		@ HIi+1
    322 	veor		$t1,$t1,$In		@ Karatsuba pre-processing
    323 	vpmull2.p64	$Xhn,$H,$In
    324 	b		.Loop_mod2x_v8
    325 
    326 .align	4
    327 .Loop_mod2x_v8:
    328 	vext.8		$t2,$IN,$IN,#8
    329 	subs		$len,$len,#32		@ is there more data?
    330 	vpmull.p64	$Xl,$H2,$IN		@ H^2.loXi.lo
    331 	cclr		$inc,lo			@ is it time to zero $inc?
    332 
    333 	 vpmull.p64	$Xmn,$Hhl,$t1
    334 	veor		$t2,$t2,$IN		@ Karatsuba pre-processing
    335 	vpmull2.p64	$Xh,$H2,$IN		@ H^2.hiXi.hi
    336 	veor		$Xl,$Xl,$Xln		@ accumulate
    337 	vpmull2.p64	$Xm,$Hhl,$t2		@ (H^2.lo+H^2.hi)(Xi.lo+Xi.hi)
    338 	 vld1.64	{$t0},[$inp],$inc	@ load [rotated] I[i+2]
    339 
    340 	veor		$Xh,$Xh,$Xhn
    341 	 cclr		$inc,eq			@ is it time to zero $inc?
    342 	veor		$Xm,$Xm,$Xmn
    343 
    344 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    345 	veor		$t2,$Xl,$Xh
    346 	veor		$Xm,$Xm,$t1
    347 	 vld1.64	{$t1},[$inp],$inc	@ load [rotated] I[i+3]
    348 #ifndef __ARMEB__
    349 	 vrev64.8	$t0,$t0
    350 #endif
    351 	veor		$Xm,$Xm,$t2
    352 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    353 
    354 #ifndef __ARMEB__
    355 	 vrev64.8	$t1,$t1
    356 #endif
    357 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    358 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    359 	 vext.8		$In,$t1,$t1,#8
    360 	 vext.8		$IN,$t0,$t0,#8
    361 	veor		$Xl,$Xm,$t2
    362 	 vpmull.p64	$Xln,$H,$In		@ HIi+1
    363 	veor		$IN,$IN,$Xh		@ accumulate $IN early
    364 
    365 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    366 	vpmull.p64	$Xl,$Xl,$xC2
    367 	veor		$IN,$IN,$t2
    368 	 veor		$t1,$t1,$In		@ Karatsuba pre-processing
    369 	veor		$IN,$IN,$Xl
    370 	 vpmull2.p64	$Xhn,$H,$In
    371 	b.hs		.Loop_mod2x_v8		@ there was at least 32 more bytes
    372 
    373 	veor		$Xh,$Xh,$t2
    374 	vext.8		$IN,$t0,$t0,#8		@ re-construct $IN
    375 	adds		$len,$len,#32		@ re-construct $len
    376 	veor		$Xl,$Xl,$Xh		@ re-construct $Xl
    377 	b.eq		.Ldone_v8		@ is $len zero?
    378 ___
    379 }
    380 $code.=<<___;
    381 .Lodd_tail_v8:
    382 	vext.8		$t2,$Xl,$Xl,#8
    383 	veor		$IN,$IN,$Xl		@ inp^=Xi
    384 	veor		$t1,$t0,$t2		@ $t1 is rotated inp^Xi
    385 
    386 	vpmull.p64	$Xl,$H,$IN		@ H.loXi.lo
    387 	veor		$t1,$t1,$IN		@ Karatsuba pre-processing
    388 	vpmull2.p64	$Xh,$H,$IN		@ H.hiXi.hi
    389 	vpmull.p64	$Xm,$Hhl,$t1		@ (H.lo+H.hi)(Xi.lo+Xi.hi)
    390 
    391 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    392 	veor		$t2,$Xl,$Xh
    393 	veor		$Xm,$Xm,$t1
    394 	veor		$Xm,$Xm,$t2
    395 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    396 
    397 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    398 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    399 	veor		$Xl,$Xm,$t2
    400 
    401 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    402 	vpmull.p64	$Xl,$Xl,$xC2
    403 	veor		$t2,$t2,$Xh
    404 	veor		$Xl,$Xl,$t2
    405 
    406 .Ldone_v8:
    407 #ifndef __ARMEB__
    408 	vrev64.8	$Xl,$Xl
    409 #endif
    410 	vext.8		$Xl,$Xl,$Xl,#8
    411 	vst1.64		{$Xl},[$Xi]		@ write out Xi
    412 
    413 ___
    414 $code.=<<___		if ($flavour !~ /64/);
    415 	vldmia		sp!,{d8-d15}		@ 32-bit ABI says so
    416 ___
    417 $code.=<<___;
    418 	ret
    419 .size	gcm_ghash_v8,.-gcm_ghash_v8
    420 ___
    421 
    422 if ($flavour =~ /64/) {				# 4x subroutine
    423 my ($I0,$j1,$j2,$j3,
    424     $I1,$I2,$I3,$H3,$H34,$H4,$Yl,$Ym,$Yh) = map("q$_",(4..7,15..23));
    425 
    426 $code.=<<___;
    427 .type	gcm_ghash_v8_4x,%function
    428 .align	4
    429 gcm_ghash_v8_4x:
    430 .Lgcm_ghash_v8_4x:
    431 	vld1.64		{$Xl},[$Xi]		@ load [rotated] Xi
    432 	vld1.64		{$H-$H2},[$Htbl],#48	@ load twisted H, ..., H^2
    433 	vmov.i8		$xC2,#0xe1
    434 	vld1.64		{$H3-$H4},[$Htbl]	@ load twisted H^3, ..., H^4
    435 	vshl.u64	$xC2,$xC2,#57		@ compose 0xc2.0 constant
    436 
    437 	vld1.64		{$I0-$j3},[$inp],#64
    438 #ifndef __ARMEB__
    439 	vrev64.8	$Xl,$Xl
    440 	vrev64.8	$j1,$j1
    441 	vrev64.8	$j2,$j2
    442 	vrev64.8	$j3,$j3
    443 	vrev64.8	$I0,$I0
    444 #endif
    445 	vext.8		$I3,$j3,$j3,#8
    446 	vext.8		$I2,$j2,$j2,#8
    447 	vext.8		$I1,$j1,$j1,#8
    448 
    449 	vpmull.p64	$Yl,$H,$I3		@ HIi+3
    450 	veor		$j3,$j3,$I3
    451 	vpmull2.p64	$Yh,$H,$I3
    452 	vpmull.p64	$Ym,$Hhl,$j3
    453 
    454 	vpmull.p64	$t0,$H2,$I2		@ H^2Ii+2
    455 	veor		$j2,$j2,$I2
    456 	vpmull2.p64	$I2,$H2,$I2
    457 	vpmull2.p64	$j2,$Hhl,$j2
    458 
    459 	veor		$Yl,$Yl,$t0
    460 	veor		$Yh,$Yh,$I2
    461 	veor		$Ym,$Ym,$j2
    462 
    463 	vpmull.p64	$j3,$H3,$I1		@ H^3Ii+1
    464 	veor		$j1,$j1,$I1
    465 	vpmull2.p64	$I1,$H3,$I1
    466 	vpmull.p64	$j1,$H34,$j1
    467 
    468 	veor		$Yl,$Yl,$j3
    469 	veor		$Yh,$Yh,$I1
    470 	veor		$Ym,$Ym,$j1
    471 
    472 	subs		$len,$len,#128
    473 	b.lo		.Ltail4x
    474 
    475 	b		.Loop4x
    476 
    477 .align	4
    478 .Loop4x:
    479 	veor		$t0,$I0,$Xl
    480 	 vld1.64	{$I0-$j3},[$inp],#64
    481 	vext.8		$IN,$t0,$t0,#8
    482 #ifndef __ARMEB__
    483 	 vrev64.8	$j1,$j1
    484 	 vrev64.8	$j2,$j2
    485 	 vrev64.8	$j3,$j3
    486 	 vrev64.8	$I0,$I0
    487 #endif
    488 
    489 	vpmull.p64	$Xl,$H4,$IN		@ H^4(Xi+Ii)
    490 	veor		$t0,$t0,$IN
    491 	vpmull2.p64	$Xh,$H4,$IN
    492 	 vext.8		$I3,$j3,$j3,#8
    493 	vpmull2.p64	$Xm,$H34,$t0
    494 
    495 	veor		$Xl,$Xl,$Yl
    496 	veor		$Xh,$Xh,$Yh
    497 	 vext.8		$I2,$j2,$j2,#8
    498 	veor		$Xm,$Xm,$Ym
    499 	 vext.8		$I1,$j1,$j1,#8
    500 
    501 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    502 	veor		$t2,$Xl,$Xh
    503 	 vpmull.p64	$Yl,$H,$I3		@ HIi+3
    504 	 veor		$j3,$j3,$I3
    505 	veor		$Xm,$Xm,$t1
    506 	 vpmull2.p64	$Yh,$H,$I3
    507 	veor		$Xm,$Xm,$t2
    508 	 vpmull.p64	$Ym,$Hhl,$j3
    509 
    510 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    511 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    512 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    513 	 vpmull.p64	$t0,$H2,$I2		@ H^2Ii+2
    514 	 veor		$j2,$j2,$I2
    515 	 vpmull2.p64	$I2,$H2,$I2
    516 	veor		$Xl,$Xm,$t2
    517 	 vpmull2.p64	$j2,$Hhl,$j2
    518 
    519 	 veor		$Yl,$Yl,$t0
    520 	 veor		$Yh,$Yh,$I2
    521 	 veor		$Ym,$Ym,$j2
    522 
    523 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    524 	vpmull.p64	$Xl,$Xl,$xC2
    525 	 vpmull.p64	$j3,$H3,$I1		@ H^3Ii+1
    526 	 veor		$j1,$j1,$I1
    527 	veor		$t2,$t2,$Xh
    528 	 vpmull2.p64	$I1,$H3,$I1
    529 	 vpmull.p64	$j1,$H34,$j1
    530 
    531 	veor		$Xl,$Xl,$t2
    532 	 veor		$Yl,$Yl,$j3
    533 	 veor		$Yh,$Yh,$I1
    534 	vext.8		$Xl,$Xl,$Xl,#8
    535 	 veor		$Ym,$Ym,$j1
    536 
    537 	subs		$len,$len,#64
    538 	b.hs		.Loop4x
    539 
    540 .Ltail4x:
    541 	veor		$t0,$I0,$Xl
    542 	vext.8		$IN,$t0,$t0,#8
    543 
    544 	vpmull.p64	$Xl,$H4,$IN		@ H^4(Xi+Ii)
    545 	veor		$t0,$t0,$IN
    546 	vpmull2.p64	$Xh,$H4,$IN
    547 	vpmull2.p64	$Xm,$H34,$t0
    548 
    549 	veor		$Xl,$Xl,$Yl
    550 	veor		$Xh,$Xh,$Yh
    551 	veor		$Xm,$Xm,$Ym
    552 
    553 	adds		$len,$len,#64
    554 	b.eq		.Ldone4x
    555 
    556 	cmp		$len,#32
    557 	b.lo		.Lone
    558 	b.eq		.Ltwo
    559 .Lthree:
    560 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    561 	veor		$t2,$Xl,$Xh
    562 	veor		$Xm,$Xm,$t1
    563 	 vld1.64	{$I0-$j2},[$inp]
    564 	veor		$Xm,$Xm,$t2
    565 #ifndef	__ARMEB__
    566 	 vrev64.8	$j1,$j1
    567 	 vrev64.8	$j2,$j2
    568 	 vrev64.8	$I0,$I0
    569 #endif
    570 
    571 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    572 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    573 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    574 	 vext.8		$I2,$j2,$j2,#8
    575 	 vext.8		$I1,$j1,$j1,#8
    576 	veor		$Xl,$Xm,$t2
    577 
    578 	 vpmull.p64	$Yl,$H,$I2		@ HIi+2
    579 	 veor		$j2,$j2,$I2
    580 
    581 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    582 	vpmull.p64	$Xl,$Xl,$xC2
    583 	veor		$t2,$t2,$Xh
    584 	 vpmull2.p64	$Yh,$H,$I2
    585 	 vpmull.p64	$Ym,$Hhl,$j2
    586 	veor		$Xl,$Xl,$t2
    587 	 vpmull.p64	$j3,$H2,$I1		@ H^2Ii+1
    588 	 veor		$j1,$j1,$I1
    589 	vext.8		$Xl,$Xl,$Xl,#8
    590 
    591 	 vpmull2.p64	$I1,$H2,$I1
    592 	veor		$t0,$I0,$Xl
    593 	 vpmull2.p64	$j1,$Hhl,$j1
    594 	vext.8		$IN,$t0,$t0,#8
    595 
    596 	 veor		$Yl,$Yl,$j3
    597 	 veor		$Yh,$Yh,$I1
    598 	 veor		$Ym,$Ym,$j1
    599 
    600 	vpmull.p64	$Xl,$H3,$IN		@ H^3(Xi+Ii)
    601 	veor		$t0,$t0,$IN
    602 	vpmull2.p64	$Xh,$H3,$IN
    603 	vpmull.p64	$Xm,$H34,$t0
    604 
    605 	veor		$Xl,$Xl,$Yl
    606 	veor		$Xh,$Xh,$Yh
    607 	veor		$Xm,$Xm,$Ym
    608 	b		.Ldone4x
    609 
    610 .align	4
    611 .Ltwo:
    612 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    613 	veor		$t2,$Xl,$Xh
    614 	veor		$Xm,$Xm,$t1
    615 	 vld1.64	{$I0-$j1},[$inp]
    616 	veor		$Xm,$Xm,$t2
    617 #ifndef	__ARMEB__
    618 	 vrev64.8	$j1,$j1
    619 	 vrev64.8	$I0,$I0
    620 #endif
    621 
    622 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    623 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    624 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    625 	 vext.8		$I1,$j1,$j1,#8
    626 	veor		$Xl,$Xm,$t2
    627 
    628 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    629 	vpmull.p64	$Xl,$Xl,$xC2
    630 	veor		$t2,$t2,$Xh
    631 	veor		$Xl,$Xl,$t2
    632 	vext.8		$Xl,$Xl,$Xl,#8
    633 
    634 	 vpmull.p64	$Yl,$H,$I1		@ HIi+1
    635 	 veor		$j1,$j1,$I1
    636 
    637 	veor		$t0,$I0,$Xl
    638 	vext.8		$IN,$t0,$t0,#8
    639 
    640 	 vpmull2.p64	$Yh,$H,$I1
    641 	 vpmull.p64	$Ym,$Hhl,$j1
    642 
    643 	vpmull.p64	$Xl,$H2,$IN		@ H^2(Xi+Ii)
    644 	veor		$t0,$t0,$IN
    645 	vpmull2.p64	$Xh,$H2,$IN
    646 	vpmull2.p64	$Xm,$Hhl,$t0
    647 
    648 	veor		$Xl,$Xl,$Yl
    649 	veor		$Xh,$Xh,$Yh
    650 	veor		$Xm,$Xm,$Ym
    651 	b		.Ldone4x
    652 
    653 .align	4
    654 .Lone:
    655 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    656 	veor		$t2,$Xl,$Xh
    657 	veor		$Xm,$Xm,$t1
    658 	 vld1.64	{$I0},[$inp]
    659 	veor		$Xm,$Xm,$t2
    660 #ifndef	__ARMEB__
    661 	 vrev64.8	$I0,$I0
    662 #endif
    663 
    664 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    665 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    666 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    667 	veor		$Xl,$Xm,$t2
    668 
    669 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    670 	vpmull.p64	$Xl,$Xl,$xC2
    671 	veor		$t2,$t2,$Xh
    672 	veor		$Xl,$Xl,$t2
    673 	vext.8		$Xl,$Xl,$Xl,#8
    674 
    675 	veor		$t0,$I0,$Xl
    676 	vext.8		$IN,$t0,$t0,#8
    677 
    678 	vpmull.p64	$Xl,$H,$IN
    679 	veor		$t0,$t0,$IN
    680 	vpmull2.p64	$Xh,$H,$IN
    681 	vpmull.p64	$Xm,$Hhl,$t0
    682 
    683 .Ldone4x:
    684 	vext.8		$t1,$Xl,$Xh,#8		@ Karatsuba post-processing
    685 	veor		$t2,$Xl,$Xh
    686 	veor		$Xm,$Xm,$t1
    687 	veor		$Xm,$Xm,$t2
    688 
    689 	vpmull.p64	$t2,$Xl,$xC2		@ 1st phase of reduction
    690 	vmov		$Xh#lo,$Xm#hi		@ Xh|Xm - 256-bit result
    691 	vmov		$Xm#hi,$Xl#lo		@ Xm is rotated Xl
    692 	veor		$Xl,$Xm,$t2
    693 
    694 	vext.8		$t2,$Xl,$Xl,#8		@ 2nd phase of reduction
    695 	vpmull.p64	$Xl,$Xl,$xC2
    696 	veor		$t2,$t2,$Xh
    697 	veor		$Xl,$Xl,$t2
    698 	vext.8		$Xl,$Xl,$Xl,#8
    699 
    700 #ifndef __ARMEB__
    701 	vrev64.8	$Xl,$Xl
    702 #endif
    703 	vst1.64		{$Xl},[$Xi]		@ write out Xi
    704 
    705 	ret
    706 .size	gcm_ghash_v8_4x,.-gcm_ghash_v8_4x
    707 ___
    708 
    709 }
    710 }
    711 
    712 $code.=<<___;
    713 .asciz  "GHASH for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
    714 .align  2
    715 #endif
    716 ___
    717 
    718 if ($flavour =~ /64/) {			######## 64-bit code
    719     sub unvmov {
    720 	my $arg=shift;
    721 
    722 	$arg =~ m/q([0-9]+)#(lo|hi),\s*q([0-9]+)#(lo|hi)/o &&
    723 	sprintf	"ins	v%d.d[%d],v%d.d[%d]",$1<8?$1:$1+8,($2 eq "lo")?0:1,
    724 					     $3<8?$3:$3+8,($4 eq "lo")?0:1;
    725     }
    726     foreach(split("\n",$code)) {
    727 	s/cclr\s+([wx])([^,]+),\s*([a-z]+)/csel	$1$2,$1zr,$1$2,$3/o	or
    728 	s/vmov\.i8/movi/o		or	# fix up legacy mnemonics
    729 	s/vmov\s+(.*)/unvmov($1)/geo	or
    730 	s/vext\.8/ext/o			or
    731 	s/vshr\.s/sshr\.s/o		or
    732 	s/vshr/ushr/o			or
    733 	s/^(\s+)v/$1/o			or	# strip off v prefix
    734 	s/\bbx\s+lr\b/ret/o;
    735 
    736 	s/\bq([0-9]+)\b/"v".($1<8?$1:$1+8).".16b"/geo;	# old->new registers
    737 	s/@\s/\/\//o;				# old->new style commentary
    738 
    739 	# fix up remaining legacy suffixes
    740 	s/\.[ui]?8(\s)/$1/o;
    741 	s/\.[uis]?32//o and s/\.16b/\.4s/go;
    742 	m/\.p64/o and s/\.16b/\.1q/o;		# 1st pmull argument
    743 	m/l\.p64/o and s/\.16b/\.1d/go;		# 2nd and 3rd pmull arguments
    744 	s/\.[uisp]?64//o and s/\.16b/\.2d/go;
    745 	s/\.[42]([sd])\[([0-3])\]/\.$1\[$2\]/o;
    746 
    747 	# Switch preprocessor checks to aarch64 versions.
    748 	s/__ARME([BL])__/__AARCH64E$1__/go;
    749 
    750 	print $_,"\n";
    751     }
    752 } else {				######## 32-bit code
    753     sub unvdup32 {
    754 	my $arg=shift;
    755 
    756 	$arg =~ m/q([0-9]+),\s*q([0-9]+)\[([0-3])\]/o &&
    757 	sprintf	"vdup.32	q%d,d%d[%d]",$1,2*$2+($3>>1),$3&1;
    758     }
    759     sub unvpmullp64 {
    760 	my ($mnemonic,$arg)=@_;
    761 
    762 	if ($arg =~ m/q([0-9]+),\s*q([0-9]+),\s*q([0-9]+)/o) {
    763 	    my $word = 0xf2a00e00|(($1&7)<<13)|(($1&8)<<19)
    764 				 |(($2&7)<<17)|(($2&8)<<4)
    765 				 |(($3&7)<<1) |(($3&8)<<2);
    766 	    $word |= 0x00010001	 if ($mnemonic =~ "2");
    767 	    # since ARMv7 instructions are always encoded little-endian.
    768 	    # correct solution is to use .inst directive, but older
    769 	    # assemblers don't implement it:-(
    770 	    sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s",
    771 			$word&0xff,($word>>8)&0xff,
    772 			($word>>16)&0xff,($word>>24)&0xff,
    773 			$mnemonic,$arg;
    774 	}
    775     }
    776 
    777     foreach(split("\n",$code)) {
    778 	s/\b[wx]([0-9]+)\b/r$1/go;		# new->old registers
    779 	s/\bv([0-9])\.[12468]+[bsd]\b/q$1/go;	# new->old registers
    780 	s/\/\/\s?/@ /o;				# new->old style commentary
    781 
    782 	# fix up remaining new-style suffixes
    783 	s/\],#[0-9]+/]!/o;
    784 
    785 	s/cclr\s+([^,]+),\s*([a-z]+)/mov.$2	$1,#0/o			or
    786 	s/vdup\.32\s+(.*)/unvdup32($1)/geo				or
    787 	s/v?(pmull2?)\.p64\s+(.*)/unvpmullp64($1,$2)/geo		or
    788 	s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo	or
    789 	s/^(\s+)b\./$1b/o						or
    790 	s/^(\s+)ret/$1bx\tlr/o;
    791 
    792 	if (s/^(\s+)mov\.([a-z]+)/$1mov$2/) {
    793 	    print "     it      $2\n";
    794 	}
    795 
    796 	print $_,"\n";
    797     }
    798 }
    799 
    800 close STDOUT or die "error closing STDOUT: $!"; # enforce flush
    801