/*	$NetBSD: cpu_in_cksum_asm_neon.S,v 1.6 2025/10/02 13:25:18 riastradh Exp $	*/

/*-
 * Copyright (c) 2012 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Matt Thomas of 3am Software Foundry.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <machine/asm.h>

RCSID("$NetBSD: cpu_in_cksum_asm_neon.S,v 1.6 2025/10/02 13:25:18 riastradh Exp $")

/*
 * uint32_t
 * cpu_in_cksum_neon(const void *dptr, size_t dlen)
 *
 *	r0 = dptr
 *	r1 = dlen
 */
ENTRY(cpu_in_cksum_neon)
	mov		ip, r0		/* leave r0 as temp */
	add		r3, r1, ip	/* get end pointer */
	and		r1, ip, #7	/* get start offset (leading bytes) */
	and		r2, r3, #7	/* get end offset (trailing bytes) */
	bic		ip, ip, #7	/* start on a dword boundary */
	add		r3, r3, #7	/* round up to a dword boundary */
	bic		r3, r3, #7	/* end on a dword boundary */
	veor		q2, q2, q2	/* clear accumulator */
	vmvn.u64	q1, q2		/* create leading/trailing masks */
	/*
	 * Normally the lower addressed is in d6 but in this case we want to
	 * reverse it since we might only have a single dword and the final
	 * fold will want the dword to trim in d7 so put the first dword in
	 * d7 until we know we are going to read more than one.
	 */
	veor		d6, d6, d6	/* clear second dword */
	vld1.64		{d7}, [ip:64]!	/* load first dword */
	orrs		r0, r1, r2	/* do we have any offsets */
	beq		.Lpre_main_loop	/*   no, proceed to main loop. */
	mov		r1, r1, lsl #3	/* leading bytes -> bits */
	movs		r2, r2, lsl #3	/* trailing bytes -> bits */
#ifdef __ARMEL__
	subne		r2, r2, #64	/* trim trailing MSBs */
#else
	rsb		r1, r1, #0	/* trim leading MSBs */
	rsbne		r2, r2, #64	/* trim trailing LSBs */
#endif
	vmov		d0, r1, r2	/* move shifts */
	vmovl.u32	q0, d0		/* 2 U32 -> 2 U64 */
	vshl.u64	q1, q1, q0	/* apply shifts to masks */
	vand.u32	d7, d7, d2	/* apply leading mask to 1st dword */
	tst		r1, #8		/* was the starting address odd? */
	beq		.Lpre_main_loop	/*   no, go to pre_main_loop */
	veor		d2, d2, d2	/* clear d2 (indicate odd addr) */

.Lpre_main_loop:
	cmp		ip, r3		/* do we just have a single dword? */
	beq		.Lfinish_up	/*   yes, let finish up! */
	vmov		d6, d7		/* move 1st dword to loaddr reg */
	vld1.64		{d7}, [ip:64]!	/* read rest of initial qword */

.Lmain_loop:
	subs		r1, r3, ip	/* how much left to do? */
	beq		.Lfinish_up	/*   = 0? we are done. */

	bics		r0, r1, #31	/* we deal with octawords only */
	beq		.Lloop_end	/*   no octawords? exit loop */
	rsbs		r0, r0, #128	/* subtract from 128 */
	ble		.Lloop128	/*   <= 0?, do 128 at a time. */
	add		r0, r0, r0, lsr #2 /* multiple by 1.25 */
	add		pc, pc, r0	/* and jump! */
	nop

.Lloop128:
	vld1.64		{d8-d9}, [ip:64]!	/* 128 left */
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6-d7}, [ip:64]!
	vmovl.u16	q0, d8		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d9		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */

	vld1.64		{d8-d9}, [ip:64]!	/* 96 left */
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6-d7}, [ip:64]!
	vmovl.u16	q0, d8		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d9		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */

	vld1.64		{d8-d9}, [ip:64]!	/* 64 left */
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6-d7}, [ip:64]!
	vmovl.u16	q0, d8		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d9		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */

	vld1.64		{d8-d9}, [ip:64]!	/* 32 left */
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6-d7}, [ip:64]!
	vmovl.u16	q0, d8		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vmovl.u16	q0, d9		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */

	b		.Lmain_loop

.Lloop_end:
	/*
	 * We have one to 3 more dwords to process
	 */
	rsb		r0, r1, #24
	add		r0, r0, r0, lsr #1
	add		pc, pc, r0
	nop
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6}, [ip:64]!
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d6}, [ip:64]!
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vld1.64		{d7}, [ip:64]!

.Lfinish_up:
	/*
	 * Apply remaining data in d6 and d7
	 */
	vmovl.u16	q0, d6		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */
	vand		d7, d7, d3	/* apply trailing mask */
	vmovl.u16	q0, d7		/* 4 U16 -> 4 U32 */
	vadd.u32	q2, q2, q0	/* add 4 U32 to accumulator */

	/*
	 * We now have 4 32-bit sums in q2 (each is 20-bits or less).
	 * Now to get to 1 I32 bit sum.
	 */
	vadd.u32	d4, d4, d5	/* 4 I32 -> 2 I32 */
	vmov		r2, s4		/* get flag for odd start */
	teq		r2, #0		/* was start addr even? */
	vmov		r0, r1, d4	/* extract two I32 */
	rev16eq		r0, r0		/* byte swap if start was odd */
	rev16eq		r1, r1		/* byte swap if start was odd */
	adds		ip, r0, r1	/* add them producing carry */
#include "arm/arm/cpu_in_cksum_fold.S"
END(cpu_in_cksum_neon)
