xref: /src/external/lgpl3/gmp/dist/mpn/x86/k7/mmx/lshift.asm

dnl  AMD K7 mpn_lshift -- mpn left shift.

dnl  Copyright 1999-2002 Free Software Foundation, Inc.

dnl  This file is part of the GNU MP Library.
dnl
dnl  The GNU MP Library is free software; you can redistribute it and/or modify
dnl  it under the terms of either:
dnl
dnl    * the GNU Lesser General Public License as published by the Free
dnl      Software Foundation; either version 3 of the License, or (at your
dnl      option) any later version.
dnl
dnl  or
dnl
dnl    * the GNU General Public License as published by the Free Software
dnl      Foundation; either version 2 of the License, or (at your option) any
dnl      later version.
dnl
dnl  or both in parallel, as here.
dnl
dnl  The GNU MP Library is distributed in the hope that it will be useful, but
dnl  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
dnl  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
dnl  for more details.
dnl
dnl  You should have received copies of the GNU General Public License and the
dnl  GNU Lesser General Public License along with the GNU MP Library.  If not,
dnl  see https://www.gnu.org/licenses/.

include(`../config.m4')


C K7: 1.21 cycles/limb (at 16 limbs/loop).



dnl  K7: UNROLL_COUNT cycles/limb
dnl           4           1.51
dnl           8           1.26
dnl          16           1.21
dnl          32           1.2
dnl  Maximum possible with the current code is 64.

deflit(UNROLL_COUNT, 16)


C mp_limb_t mpn_lshift (mp_ptr dst, mp_srcptr src, mp_size_t size,
C                       unsigned shift);
C
C Shift src,size left by shift many bits and store the result in dst,size.
C Zeros are shifted in at the right.  The bits shifted out at the left are
C the return value.
C
C The comments in mpn_rshift apply here too.

ifdef(`PIC',`
deflit(UNROLL_THRESHOLD, 10)
',`
deflit(UNROLL_THRESHOLD, 10)
')

defframe(PARAM_SHIFT,16)
defframe(PARAM_SIZE, 12)
defframe(PARAM_SRC,  8)
defframe(PARAM_DST,  4)

defframe(SAVE_EDI, -4)
defframe(SAVE_ESI, -8)
defframe(SAVE_EBX, -12)
deflit(SAVE_SIZE, 12)

	TEXT
	ALIGN(32)

PROLOGUE(mpn_lshift)
deflit(`FRAME',0)

	movl	PARAM_SIZE, %eax
	movl	PARAM_SRC, %edx
	subl	$SAVE_SIZE, %esp
deflit(`FRAME',SAVE_SIZE)

	movl	PARAM_SHIFT, %ecx
	movl	%edi, SAVE_EDI

	movl	PARAM_DST, %edi
	decl	%eax
	jnz	L(more_than_one_limb)

	movl	(%edx), %edx

	shldl(	%cl, %edx, %eax)	C eax was decremented to zero

	shll	%cl, %edx

	movl	%edx, (%edi)
	movl	SAVE_EDI, %edi
	addl	$SAVE_SIZE, %esp

	ret


C -----------------------------------------------------------------------------
L(more_than_one_limb):
	C eax	size-1
	C ebx
	C ecx	shift
	C edx	src
	C esi
	C edi	dst
	C ebp

	movd	PARAM_SHIFT, %mm6
	movd	(%edx,%eax,4), %mm5	C src high limb
	cmp	$UNROLL_THRESHOLD-1, %eax

	jae	L(unroll)
	negl	%ecx
	movd	(%edx), %mm4		C src low limb

	addl	$32, %ecx

	movd	%ecx, %mm7

L(simple_top):
	C eax	loop counter, limbs
	C ebx
	C ecx
	C edx	src
	C esi
	C edi	dst
	C ebp
	C
	C mm0	scratch
	C mm4	src low limb
	C mm5	src high limb
	C mm6	shift
	C mm7	32-shift

	movq	-4(%edx,%eax,4), %mm0
	decl	%eax

	psrlq	%mm7, %mm0

	movd	%mm0, 4(%edi,%eax,4)
	jnz	L(simple_top)


	psllq	%mm6, %mm5
	psllq	%mm6, %mm4

	psrlq	$32, %mm5
	movd	%mm4, (%edi)		C dst low limb

	movd	%mm5, %eax		C return value

	movl	SAVE_EDI, %edi
	addl	$SAVE_SIZE, %esp
	emms

	ret


C -----------------------------------------------------------------------------
	ALIGN(16)
L(unroll):
	C eax	size-1
	C ebx	(saved)
	C ecx	shift
	C edx	src
	C esi
	C edi	dst
	C ebp
	C
	C mm5	src high limb, for return value
	C mm6	lshift

	movl	%esi, SAVE_ESI
	movl	%ebx, SAVE_EBX
	leal	-4(%edx,%eax,4), %edx   C &src[size-2]

	testb	$4, %dl
	movq	(%edx), %mm1		C src high qword

	jz	L(start_src_aligned)


	C src isn't aligned, process high limb (marked xxx) separately to
	C make it so
	C
	C  source    -4(edx,%eax,4)
	C                  |
	C  +-------+-------+-------+--
	C  |  xxx          |
	C  +-------+-------+-------+--
	C        0mod8   4mod8   0mod8
	C
	C  dest      -4(edi,%eax,4)
	C                  |
	C  +-------+-------+--
	C  |  xxx  |       |
	C  +-------+-------+--

	psllq	%mm6, %mm1
	subl	$4, %edx
	movl	%eax, PARAM_SIZE	C size-1

	psrlq	$32, %mm1
	decl	%eax			C size-2 is new size-1

	movd	%mm1, 4(%edi,%eax,4)
	movq	(%edx), %mm1		C new src high qword
L(start_src_aligned):


	leal	-4(%edi,%eax,4), %edi   C &dst[size-2]
	psllq	%mm6, %mm5

	testl	$4, %edi
	psrlq	$32, %mm5		C return value

	jz	L(start_dst_aligned)


	C dst isn't aligned, subtract 4 bytes to make it so, and pretend the
	C shift is 32 bits extra.  High limb of dst (marked xxx) handled
	C here separately.
	C
	C  source       %edx
	C  +-------+-------+--
	C  |      mm1      |
	C  +-------+-------+--
	C                0mod8   4mod8
	C
	C  dest         %edi
	C  +-------+-------+-------+--
	C  |  xxx  |
	C  +-------+-------+-------+--
	C        0mod8   4mod8   0mod8

	movq	%mm1, %mm0
	psllq	%mm6, %mm1
	addl	$32, %ecx		C shift+32

	psrlq	$32, %mm1

	movd	%mm1, 4(%edi)
	movq	%mm0, %mm1
	subl	$4, %edi

	movd	%ecx, %mm6		C new lshift
L(start_dst_aligned):

	decl	%eax			C size-2, two last limbs handled at end
	movq	%mm1, %mm2		C copy of src high qword
	negl	%ecx

	andl	$-2, %eax		C round size down to even
	addl	$64, %ecx

	movl	%eax, %ebx
	negl	%eax

	andl	$UNROLL_MASK, %eax
	decl	%ebx

	shll	%eax

	movd	%ecx, %mm7		C rshift = 64-lshift

ifdef(`PIC',`
	call	L(pic_calc)
L(here):
',`
	leal	L(entry) (%eax,%eax,4), %esi
')
	shrl	$UNROLL_LOG2, %ebx	C loop counter

	leal	ifelse(UNROLL_BYTES,256,128) -8(%edx,%eax,2), %edx
	leal	ifelse(UNROLL_BYTES,256,128) (%edi,%eax,2), %edi
	movl	PARAM_SIZE, %eax	C for use at end
	jmp	*%esi


ifdef(`PIC',`
L(pic_calc):
	C See mpn/x86/README about old gas bugs
	leal	(%eax,%eax,4), %esi
	addl	$L(entry)-L(here), %esi
	addl	(%esp), %esi

	ret_internal
')


C -----------------------------------------------------------------------------
	ALIGN(32)
L(top):
	C eax	size (for use at end)
	C ebx	loop counter
	C ecx	rshift
	C edx	src
	C esi	computed jump
	C edi	dst
	C ebp
	C
	C mm0	scratch
	C mm1	\ carry (alternating, mm2 first)
	C mm2	/
	C mm6	lshift
	C mm7	rshift
	C
	C 10 code bytes/limb
	C
	C The two chunks differ in whether mm1 or mm2 hold the carry.
	C The computed jump puts the initial carry in both mm1 and mm2.

L(entry):
deflit(CHUNK_COUNT, 4)
forloop(i, 0, UNROLL_COUNT/CHUNK_COUNT-1, `
	deflit(`disp0', eval(-i*CHUNK_COUNT*4 ifelse(UNROLL_BYTES,256,-128)))
	deflit(`disp1', eval(disp0 - 8))

Zdisp(	movq,	disp0,(%edx), %mm0)
	psllq	%mm6, %mm2

	movq	%mm0, %mm1
	psrlq	%mm7, %mm0

	por	%mm2, %mm0
Zdisp(	movq,	%mm0, disp0,(%edi))


Zdisp(	movq,	disp1,(%edx), %mm0)
	psllq	%mm6, %mm1

	movq	%mm0, %mm2
	psrlq	%mm7, %mm0

	por	%mm1, %mm0
Zdisp(	movq,	%mm0, disp1,(%edi))
')

	subl	$UNROLL_BYTES, %edx
	subl	$UNROLL_BYTES, %edi
	decl	%ebx

	jns	L(top)



define(`disp', `m4_empty_if_zero(eval($1 ifelse(UNROLL_BYTES,256,-128)))')

L(end):
	testb	$1, %al
	movl	SAVE_EBX, %ebx
	psllq	%mm6, %mm2	C wanted left shifted in all cases below

	movd	%mm5, %eax

	movl	SAVE_ESI, %esi
	jz	L(end_even)


L(end_odd):

	C Size odd, destination was aligned.
	C
	C                 source        edx+8   edx+4
	C                 --+---------------+-------+
	C                   |      mm2      |       |
	C                 --+---------------+-------+
	C
	C dest                            edi
	C --+---------------+---------------+-------+
	C   |   written     |               |       |
	C --+---------------+---------------+-------+
	C
	C mm6 = shift
	C mm7 = ecx = 64-shift


	C Size odd, destination was unaligned.
	C
	C                 source        edx+8   edx+4
	C                 --+---------------+-------+
	C                   |      mm2      |       |
	C                 --+---------------+-------+
	C
	C         dest                            edi
	C         --+---------------+---------------+
	C           |   written     |               |
	C         --+---------------+---------------+
	C
	C mm6 = shift+32
	C mm7 = ecx = 64-(shift+32)


	C In both cases there's one extra limb of src to fetch and combine
	C with mm2 to make a qword at (%edi), and in the aligned case
	C there's an extra limb of dst to be formed from that extra src limb
	C left shifted.

	movd	disp(4) (%edx), %mm0
	testb	$32, %cl

	movq	%mm0, %mm1
	psllq	$32, %mm0

	psrlq	%mm7, %mm0
	psllq	%mm6, %mm1

	por	%mm2, %mm0

	movq	%mm0, disp(0) (%edi)
	jz	L(end_odd_unaligned)
	movd	%mm1, disp(-4) (%edi)
L(end_odd_unaligned):

	movl	SAVE_EDI, %edi
	addl	$SAVE_SIZE, %esp
	emms

	ret


L(end_even):

	C Size even, destination was aligned.
	C
	C                 source        edx+8
	C                 --+---------------+
	C                   |      mm2      |
	C                 --+---------------+
	C
	C dest                            edi
	C --+---------------+---------------+
	C   |   written     |               |
	C --+---------------+---------------+
	C
	C mm6 = shift
	C mm7 = ecx = 64-shift


	C Size even, destination was unaligned.
	C
	C               source          edx+8
	C                 --+---------------+
	C                   |      mm2      |
	C                 --+---------------+
	C
	C         dest                  edi+4
	C         --+---------------+-------+
	C           |    written    |       |
	C         --+---------------+-------+
	C
	C mm6 = shift+32
	C mm7 = ecx = 64-(shift+32)


	C The movq for the aligned case overwrites the movd for the
	C unaligned case.

	movq	%mm2, %mm0
	psrlq	$32, %mm2

	testb	$32, %cl
	movd	%mm2, disp(4) (%edi)

	jz	L(end_even_unaligned)
	movq	%mm0, disp(0) (%edi)
L(end_even_unaligned):

	movl	SAVE_EDI, %edi
	addl	$SAVE_SIZE, %esp
	emms

	ret

EPILOGUE()