xref: /src/sys/lib/libkern/arch/hppa/bcopy.S

/*	$NetBSD: bcopy.S,v 1.14.14.1 2017/12/03 11:38:46 jdolecek Exp $	*/

/*
 * Copyright (c) 2002 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Matthew Fredette.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copy routines for NetBSD/hppa.
 */

#undef _LOCORE
#define _LOCORE	/* XXX fredette - unfortunate */

#if defined(SPCOPY) && !defined(_STANDALONE)

#include "opt_diagnostic.h"
#include "opt_multiprocessor.h"

#include <machine/cpu.h>

#endif

#include <machine/asm.h>
#include <machine/frame.h>
#include <machine/reg.h>

#if defined(LIBC_SCCS) && !defined(lint)
RCSID("$NetBSD: bcopy.S,v 1.14.14.1 2017/12/03 11:38:46 jdolecek Exp $")
#endif /* LIBC_SCCS and not lint */

/*
 * The stbys instruction is a little asymmetric.  When (%r2 & 3)
 * is zero, stbys,b,m %r1, 4(%r2) works like stws,ma.  You
 * might then wish that when (%r2 & 3) == 0, stbys,e,m %r1, -4(%r2)
 * worked like stws,mb.  But it doesn't.
 *
 * This macro works around this problem.  It requires that %t2
 * hold the number of bytes that will be written by this store
 * (meaning that it ranges from one to four).
 *
 * Watch the delay-slot trickery here.  The comib is used to set
 * up which instruction, either the stws or the stbys, is run
 * in the delay slot of the b instruction.
 */
#define _STBYS_E_M(r, dst_spc, dst_off)				  \
	comib,<>	4, %t2, 4				! \
	b		4					! \
	stws,mb		r, -4(dst_spc, dst_off)			! \
	stbys,e,m	r, 0(dst_spc, dst_off)

/*
 * This macro does a bulk copy with no shifting.  cmplt and m are
 * the completer and displacement multiplier, respectively, for
 * the load and store instructions.
 */
#define _COPY(src_spc, src_off, dst_spc, dst_off, count, cmplt, m) \
								! \
	/*							! \
	 * Loop storing 16 bytes at a time.  Since count 	! \
	 * may be > INT_MAX, we have to be careful and		! \
	 * avoid comparisons that treat it as a signed 		! \
	 * quantity, until after this loop, when count		! \
	 * is guaranteed to be less than 16.			! \
	 */							! \
	comib,>>=,n	15, count, _LABEL(_skip16)		! \
.label _LABEL(_loop16)						! \
	addi		-16, count, count			! \
	ldws,cmplt	m*4(src_spc, src_off), %t1		! \
	ldws,cmplt	m*4(src_spc, src_off), %t2		! \
	ldws,cmplt	m*4(src_spc, src_off), %t3		! \
	ldws,cmplt	m*4(src_spc, src_off), %t4		! \
	stws,cmplt	%t1, m*4(dst_spc, dst_off)		! \
	stws,cmplt	%t2, m*4(dst_spc, dst_off)		! \
	stws,cmplt	%t3, m*4(dst_spc, dst_off)		! \
	comib,<<	15, count, _LABEL(_loop16)		! \
	stws,cmplt	%t4, m*4(dst_spc, dst_off)		! \
.label _LABEL(_skip16)						! \
								! \
	/* Loop storing 4 bytes at a time. */			! \
	addib,<,n	-4, count, _LABEL(_skip4)		! \
.label _LABEL(_loop4)						! \
	ldws,cmplt	m*4(src_spc, src_off), %t1		! \
	addib,>=	-4, count, _LABEL(_loop4)		! \
	stws,cmplt	%t1, m*4(dst_spc, dst_off)		! \
.label _LABEL(_skip4)						! \
	/* Restore the correct count. */			! \
	addi		4, count, count				! \
								! \
.label _LABEL(_do1)						! \
								! \
	/* Loop storing 1 byte at a time. */			! \
	addib,<,n	-1, count, _LABEL(_skip1)		! \
.label _LABEL(_loop1)						! \
	ldbs,cmplt	m*1(src_spc, src_off), %t1		! \
	addib,>=	-1, count, _LABEL(_loop1)		! \
	stbs,cmplt	%t1, m*1(dst_spc, dst_off)		! \
.label _LABEL(_skip1)						! \
	/* Restore the correct count. */			! \
	b		_LABEL(_done)				! \
	addi		1, count, count

/*
 * This macro is definitely strange.  It exists purely to
 * allow the _COPYS macro to be reused, but because it
 * requires this long attempt to explain it, I'm starting
 * to doubt the value of that.
 *
 * Part of the expansion of the _COPYS macro below are loops
 * that copy four words or one word at a time, performing shifts
 * to get data to line up correctly in the destination buffer.
 *
 * The _COPYS macro is used when copying backwards, as well
 * as forwards.  The 4-word loop always loads into %t1, %t2, %t3,
 * and %t4 in that order.  This means that when copying forward,
 * %t1 will have the word from the lowest address, and %t4 will
 * have the word from the highest address.  When copying
 * backwards, the opposite is true.
 *
 * The shift instructions need pairs of registers with adjacent
 * words, with the register containing the word from the lowest
 * address *always* coming first.  It is this assymetry that
 * gives rise to this macro - depending on which direction
 * we're copying in, these ordered pairs are different.
 *
 * Fortunately, we can compute those register numbers at compile
 * time, and assemble them manually into a shift instruction.
 * That's what this macro does.
 *
 * This macro takes two arguments.  n ranges from 0 to 3 and
 * is the "shift number", i.e., n = 0 means we're doing the
 * shift for what will be the first store.
 *
 * m is the displacement multiplier from the _COPYS macro call.
 * This is 1 for a forward copy and -1 for a backwards copy.
 * So, the ((m + 1) / 2) term yields 0 for a backwards copy and
 * 1 for a forward copy, and the ((m - 1) / 2) term yields
 * 0 for a forward copy, and -1 for a backwards copy.
 * These terms are used to discriminate the register computations
 * below.
 *
 * When copying forward, then, the first register used with
 * the first vshd will be 19 + (3 - ((0 - 1) & 3)), or %t4,
 * which matches _COPYS' requirement that the word last loaded
 * be in %t4.  The first register used for the second vshd
 * will then "wrap" around to 19 + (3 - ((1 - 1) & 3)), or %t1.
 * And so on to %t2 and %t3.
 *
 * When copying forward, the second register used with the first
 * vshd will be (19 + (3 - ((n + 0) & 3)), or %t1.  It will
 * continue to be %t2, then %t3, and finally %t4.
 *
 * When copying backwards, the values for the first and second
 * register for each vshd are reversed from the forwards case.
 * (Symmetry reclaimed!)  Proving this is "left as an exercise
 * for the reader" (remember the different discriminating values!)
 */
#define _VSHD(n, m, t)						  \
	.word (0xd0000000					| \
	((19 + (3 - ((n - 1 * ((m + 1) / 2)) & 3))) << 16)	| \
	((19 + (3 - ((n + 1 * ((m - 1) / 2)) & 3))) << 21)	| \
	(t))

/*
 * This macro does a bulk copy with shifting.  cmplt and m are
 * the completer and displacement multiplier, respectively, for
 * the load and store instructions.  It is assumed that the
 * word last loaded is already in %t4.
 */
#define _COPYS(src_spc, src_off, dst_spc, dst_off, count, cmplt, m) \
								! \
	/*							! \
	 * Loop storing 16 bytes at a time.  Since count 	! \
	 * may be > INT_MAX, we have to be careful and		! \
	 * avoid comparisons that treat it as a signed 		! \
	 * quantity, until after this loop, when count		! \
	 * is guaranteed to be less than 16.			! \
	 */							! \
	comib,>>=,n	15, count, _LABEL(S_skip16)		! \
.label _LABEL(S_loop16)						! \
	addi		-16, count, count			! \
	ldws,cmplt	m*4(src_spc, src_off), %t1		! \
	ldws,cmplt	m*4(src_spc, src_off), %t2		! \
	ldws,cmplt	m*4(src_spc, src_off), %t3		! \
	_VSHD(0, m, 1)	/* vshd %t4, %t1, %r1 */		! \
	ldws,cmplt	m*4(src_spc, src_off), %t4		! \
	_VSHD(1, m, 22)	/* vshd %t1, %t2, %t1 */		! \
	_VSHD(2, m, 21)	/* vshd %t2, %t3, %t2 */		! \
	_VSHD(3, m, 20)	/* vshd %t3, %t4, %t3 */		! \
	stws,cmplt	%r1, m*4(dst_spc, dst_off)		! \
	stws,cmplt	%t1, m*4(dst_spc, dst_off)		! \
	stws,cmplt	%t2, m*4(dst_spc, dst_off)		! \
	comib,<<	15, count, _LABEL(S_loop16)		! \
	stws,cmplt	%t3, m*4(dst_spc, dst_off)		! \
.label _LABEL(S_skip16)						! \
								! \
	/* Loop storing 4 bytes at a time. */			! \
	addib,<,n	-4, count, _LABEL(S_skip4)		! \
.label _LABEL(S_loop4)						! \
	ldws,cmplt	m*4(src_spc, src_off), %t1		! \
	_VSHD(0, m, 1)	/* into %r1 (1) */			! \
	copy		%t1, %t4				! \
	addib,>=	-4, count, _LABEL(S_loop4)		! \
	stws,cmplt	%r1, m*4(dst_spc, dst_off)		! \
.label _LABEL(S_skip4)						! \
								! \
	/*							! \
 	 * We now need to "back up" src_off by the		! \
	 * number of bytes remaining in the FIFO		! \
	 * (i.e., the number of bytes remaining in %t4),	! \
	 * because (the correct) count still includes		! \
	 * these bytes, and we intent to keep it that		! \
	 * way, and finish with the single-byte copier.		! \
	 *							! \
	 * The number of bytes remaining in the FIFO is		! \
	 * related to the shift count, so recover it,		! \
	 * restoring the correct count at the same time.	! \
	 */							! \
	mfctl	%cr11, %t1					! \
	addi	4, count, count					! \
	shd	%r0, %t1, 3, %t1				! \
								! \
	/*							! \
	 * If we're copying forward, the shift count		! \
	 * is the number of bytes remaining in the		! \
	 * FIFO, and we want to subtract it from src_off.	! \
	 * If we're copying backwards, (4 - shift count)	! \
	 * is the number of bytes remaining in the FIFO,	! \
	 * and we want to add it to src_off.			! \
	 *							! \
	 * We observe that x + (4 - y) = x - (y - 4),		! \
	 * and introduce this instruction to add -4 when	! \
	 * m is -1, although this does mean one extra		! \
	 * instruction in the forward case.			! \
	 */							! \
	addi	4*((m - 1) / 2), %t1, %t1			! \
								! \
	/* Now branch to the byte-at-a-time loop. */		! \
	b	_LABEL(_do1)					! \
	sub	src_off, %t1, src_off

/*
 * This macro copies a region in the forward direction.
 */
#define _COPY_FORWARD(src_spc, src_off, dst_spc, dst_off, count)  \
								! \
	/*							! \
	 * Since in the shifting-left case we will		! \
	 * load 8 bytes before checking count, to		! \
	 * keep things simple, branch to the byte 		! \
	 * copier unless we're copying at least 8.		! \
	 */							! \
	comib,>>,n	8, count, _LABEL(_do1)			! \
								! \
	/*							! \
	 * Once we 4-byte align the source offset, 		! \
	 * figure out how many bytes from the region		! \
	 * will be in the first 4-byte word we read.		! \
	 * Ditto for writing the destination offset.		! \
	 */							! \
	extru		src_off, 31, 2, %t1			! \
	extru		dst_off, 31, 2, %t2			! \
	subi		4, %t1, %t1				! \
	subi		4, %t2, %t2				! \
								! \
	/*							! \
	 * Calculate the byte shift required.  A 		! \
	 * positive value means a source 4-byte word 		! \
	 * has to be shifted to the right to line up 		! \
	 * as a destination 4-byte word.			! \
	 */							! \
	sub		%t1, %t2, %t1				! \
								! \
	/* 4-byte align src_off. */				! \
	depi		0, 31, 2, src_off			! \
								! \
	/*							! \
	 * It's somewhat important to note that this		! \
	 * code thinks of count as "the number of bytes		! \
	 * that haven't been stored yet", as opposed to		! \
	 * "the number of bytes that haven't been copied	! \
	 * yet".  The distinction is subtle, but becomes	! \
	 * apparent at the end of the shifting code, where	! \
	 * we "back up" src_off to correspond to count,		! \
	 * as opposed to flushing the FIFO.			! \
	 *							! \
	 * We calculated above how many bytes our first		! \
	 * store will store, so update count now.		! \
	 *							! \
	 * If the shift is zero, strictly as an optimization	! \
	 * we use a copy loop that does no shifting.		! \
	 */							! \
	comb,<>		%r0, %t1, _LABEL(_shifting)		! \
	sub		count, %t2, count			! \
								! \
	/* Load and store the first word. */			! \
	ldws,ma		4(src_spc, src_off), %t4		! \
	stbys,b,m	%t4, 4(dst_spc, dst_off)		! \
								! \
	/* Do the rest of the copy. */				! \
	_COPY(src_spc,src_off,dst_spc,dst_off,count,ma,1)	! \
								! \
.label _LABEL(_shifting)					! \
								! \
	/*							! \
	 * If shift < 0, we need to shift words to the		! \
	 * left.  Since we can't do this directly, we		! \
	 * adjust the shift so it's a shift to the right	! \
	 * and load the first word into the high word of	! \
	 * the FIFO.  Otherwise, we load a zero into the	! \
	 * high word of the FIFO.				! \
	 */							! \
	comb,<=		%r0, %t1, _LABEL(_shiftingrt)		! \
	copy		%r0, %t3				! \
	addi		4, %t1, %t1				! \
	ldws,ma		4(src_spc, src_off), %t3		! \
.label _LABEL(_shiftingrt)					! \
								! \
	/*							! \
	 * Turn the shift byte count into a bit count,		! \
	 * load the next word, set the Shift Amount 		! \
	 * Register, and form and store the first word.		! \
	 */							! \
	sh3add		%t1, %r0, %t1				! \
	ldws,ma		4(src_spc, src_off), %t4		! \
	mtctl		%t1, %cr11				! \
	vshd		%t3, %t4, %r1				! \
	stbys,b,m	%r1, 4(dst_spc, dst_off)		! \
								! \
	/* Do the rest of the copy. */				! \
	_COPYS(src_spc,src_off,dst_spc,dst_off,count,ma,1)

/* This macro copies a region in the reverse direction. */
#define _COPY_REVERSE(src_spc, src_off, dst_spc, dst_off, count)  \
								! \
	/* Immediately add count to both offsets. */		! \
	add	src_off, count, src_off				! \
	add	dst_off, count, dst_off				! \
								! \
	/*							! \
	 * Since in the shifting-right case we 			! \
	 * will load 8 bytes before checking 			! \
	 * count, to keep things simple, branch 		! \
	 * to the byte copier unless we're 			! \
	 * copying at least 8 bytes.				! \
	 */							! \
	comib,>>,n	8, count, _LABEL(_do1)			! \
								! \
	/*							! \
	 * Once we 4-byte align the source offset, 		! \
	 * figure out how many bytes from the region		! \
	 * will be in the first 4-byte word we read.		! \
	 * Ditto for writing the destination offset.		! \
	 */							! \
	extru,<>	src_off, 31, 2, %t1			! \
	ldi		4, %t1					! \
	extru,<>	dst_off, 31, 2, %t2			! \
	ldi		4, %t2					! \
								! \
	/*							! \
	 * Calculate the byte shift required.  A 		! \
	 * positive value means a source 4-byte 		! \
	 * word has to be shifted to the right to 		! \
	 * line up as a destination 4-byte word.		! \
	 */							! \
	sub		%t2, %t1, %t1				! \
								! \
	/*							! \
	 * 4-byte align src_off, leaving it pointing 		! \
	 * to the 4-byte word *after* the next word 		! \
	 * we intend to load.					! \
	 *							! \
	 * It's somewhat important to note that this		! \
	 * code thinks of count as "the number of bytes		! \
	 * that haven't been stored yet", as opposed to		! \
	 * "the number of bytes that haven't been copied	! \
	 * yet".  The distinction is subtle, but becomes	! \
	 * apparent at the end of the shifting code, where	! \
	 * we "back up" src_off to correspond to count,		! \
	 * as opposed to flushing the FIFO.			! \
	 *							! \
	 * We calculated above how many bytes our first		! \
	 * store will store, so update count now.		! \
	 *							! \
	 * If the shift is zero, we use a copy loop that	! \
	 * does no shifting.  NB: unlike the forward case,	! \
	 * this is NOT strictly an optimization.  If the	! \
	 * SAR is zero the vshds do NOT do the right thing.	! \
	 * This is another assymetry more or less the "fault"	! \
	 * of vshd.						! \
	 */							! \
	addi		3, src_off, src_off			! \
	sub		count, %t2, count			! \
	comb,<>		%r0, %t1, _LABEL(_shifting)		! \
	depi		0, 31, 2, src_off			! \
								! \
	/* Load and store the first word. */			! \
	ldws,mb		-4(src_spc, src_off), %t4		! \
	_STBYS_E_M(%t4, dst_spc, dst_off)			! \
								! \
	/* Do the rest of the copy. */				! \
	_COPY(src_spc,src_off,dst_spc,dst_off,count,mb,-1)	! \
								! \
.label _LABEL(_shifting)					! \
								! \
	/*							! \
	 * If shift < 0, we need to shift words to the		! \
	 * left.  Since we can't do this directly, we		! \
	 * adjust the shift so it's a shift to the right	! \
	 * and load a zero in to the low word of the FIFO.	! \
	 * Otherwise, we load the first word into the		! \
	 * low word of the FIFO.				! \
	 *							! \
	 * Note the nullification trickery here.  We 		! \
	 * assume that we're shifting to the left, and		! \
	 * load zero into the low word of the FIFO.  Then	! \
	 * we nullify the addi if we're shifting to the		! \
	 * right.  If the addi is not nullified, we are		! \
 	 * shifting to the left, so we nullify the load.	! \
	 * we branch if we're shifting to the 			! \
	 */							! \
	copy		%r0, %t3				! \
	comb,<=,n	%r0, %t1, 0				! \
	addi,tr		4, %t1, %t1				! \
	ldws,mb		-4(src_spc, src_off), %t3		! \
								! \
	/*							! \
	 * Turn the shift byte count into a bit count,		! \
	 * load the next word, set the Shift Amount 		! \
	 * Register, and form and store the first word.		! \
	 */							! \
	sh3add		%t1, %r0, %t1				! \
	ldws,mb		-4(src_spc, src_off), %t4		! \
	mtctl		%t1, %cr11				! \
	vshd		%t4, %t3, %r1				! \
	_STBYS_E_M(%r1, dst_spc, dst_off)			! \
								! \
	/* Do the rest of the copy. */				! \
	_COPYS(src_spc,src_off,dst_spc,dst_off,count,mb,-1)

/*
 * For paranoia, when things aren't going well, enable this
 * code to assemble byte-at-a-time-only copying.
 */
#if 1
#undef _COPY_FORWARD
#define _COPY_FORWARD(src_spc, src_off, dst_spc, dst_off, count)  \
	comb,=,n	%r0, count, _LABEL(_done)		! \
	ldbs,ma		1(src_spc, src_off), %r1		! \
	addib,<>	-1, count, -12				! \
	stbs,ma		%r1, 1(dst_spc, dst_off)		! \
	b,n		_LABEL(_done)
#undef _COPY_REVERSE
#define _COPY_REVERSE(src_spc, src_off, dst_spc, dst_off, count)  \
	comb,=		%r0, count, _LABEL(_done)		! \
	add		src_off, count, src_off			! \
	add		dst_off, count, dst_off			! \
	ldbs,mb		-1(src_spc, src_off), %r1		! \
	addib,<>	-1, count, -12				! \
	stbs,mb		%r1, -1(dst_spc, dst_off)		! \
	b,n		_LABEL(_done)
#endif

/*
 * If none of the following are defined, define BCOPY.
 */
#if !(defined(SPCOPY) || defined(MEMCPY) || defined(MEMMOVE))
#define BCOPY
#endif

#if defined(SPCOPY) && !defined(_STANDALONE)

#include <sys/errno.h>
#include "assym.h"

/*
 * int spcopy(pa_space_t ssp, const void *src, pa_space_t dsp, void *dst,
 * 	size_t len)
 *
 * We assume that the regions do not overlap.
 */
LEAF_ENTRY(spcopy)

        /*
	 * Setup the fault handler, which will fill in %ret0 if triggered.
	 */
	GET_CURLWP(%r31)
#ifdef	DIAGNOSTIC
	comb,<>,n %r0, %r31, Lspcopy_curlwp_ok
	ldil	L%panic, %r1
	ldil	L%Lspcopy_curlwp_bad, %arg0
	ldo	R%panic(%r1), %r1
	ldo	R%Lspcopy_curlwp_bad(%arg0), %arg0
	.call
	bv,n    %r0(%r1)
	nop
Lspcopy_curlwp_bad:
	.asciz	"spcopy: curlwp == NULL\n"
	.align	8
Lspcopy_curlwp_ok:
#endif /* DIAGNOSTIC */
	ldil    L%spcopy_fault, %r1
	ldw     L_PCB(%r31), %r31
	ldo     R%spcopy_fault(%r1), %r1
	stw     %r1, PCB_ONFAULT(%r31)

	/* Setup the space registers. */
	mfsp	%sr2, %ret1
	mtsp	%arg0, %sr1
	mtsp	%arg2, %sr2

	/* Get the len argument and do the copy. */
	ldw	HPPA_FRAME_ARG(4)(%sp), %arg0
#define	_LABEL(l) __CONCAT(spcopy,l)
	_COPY_FORWARD(%sr1,%arg1,%sr2,%arg3,%arg0)
_LABEL(_done):

	/* Return. */
	copy	%r0, %ret0
ALTENTRY(spcopy_fault)
	stw     %r0, PCB_ONFAULT(%r31)
	bv	%r0(%rp)
	mtsp	%ret1, %sr2
EXIT(spcopy)
#endif /* SPCOPY && !_STANDALONE */

#ifdef MEMCPY
/*
 * void *memcpy(void *restrict dst, const void *restrict src, size_t len);
 *
 * memcpy is specifically restricted to working on
 * non-overlapping regions, so we can just copy forward.
 */
LEAF_ENTRY(memcpy)
	copy	%arg0, %ret0
#define	_LABEL(l) __CONCAT(memcpy,l)
	_COPY_FORWARD(%sr0,%arg1,%sr0,%arg0,%arg2)
_LABEL(_done):
	bv,n	%r0(%rp)
	nop
EXIT(memcpy)
#endif /* MEMCPY */

#ifdef BCOPY
/*
 * void bcopy(const void *src, void *dst, size_t len);
 */
LEAF_ENTRY(bcopy)
	copy	%arg0, %r1
	copy	%arg1, %arg0
	copy	%r1, %arg1
	/* FALLTHROUGH */
#define _LABEL_F(l) __CONCAT(bcopy_F,l)
#define _LABEL_R(l) __CONCAT(bcopy_R,l)
#endif

#ifdef MEMMOVE
/*
 * void *memmove(void *dst, const void *src, size_t len);
 */
LEAF_ENTRY(memmove)
#define _LABEL_F(l) __CONCAT(memmove_F,l)
#define _LABEL_R(l) __CONCAT(memmove_R,l)
	copy	%arg0, %ret0
#endif /* MEMMOVE */

#if defined(BCOPY) || defined(MEMMOVE)

	/*
	 * If src >= dst or src + len <= dst, we copy
	 * forward, else we copy in reverse.
	 */
	add		%arg1, %arg2, %r1
	comb,>>=,n	%arg1, %arg0, 0
	comb,>>,n	%r1, %arg0, _LABEL_R(_go)

#define _LABEL _LABEL_F
	_COPY_FORWARD(%sr0,%arg1,%sr0,%arg0,%arg2)
#undef _LABEL

_LABEL_R(_go):
#define _LABEL _LABEL_R
	_COPY_REVERSE(%sr0,%arg1,%sr0,%arg0,%arg2)
#undef _LABEL

_LABEL_F(_done):
_LABEL_R(_done):
	bv,n	%r0(%rp)
	nop
#ifdef BCOPY
EXIT(bcopy)
#else
EXIT(memmove)
#endif
#endif /* BCOPY || MEMMOVE */