OpenGrok

/*	$NetBSD: cache_r5k.c,v 1.21 2020/06/14 15:12:56 tsutsui Exp $	*/

/*
 * Copyright 2001 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Jason R. Thorpe for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed for the NetBSD Project by
 *	Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: cache_r5k.c,v 1.21 2020/06/14 15:12:56 tsutsui Exp $");

#include <sys/param.h>

#include <mips/cache.h>
#include <mips/cache_r4k.h>
#include <mips/cache_r5k.h>
#include <mips/locore.h>

/*
 * Cache operations for R5000-style caches:
 *
 *	- 2-way set-associative
 *	- Write-back
 *	- Virtually indexed, physically tagged
 *
 * Since the R4600 is so similar (2-way set-associative, 32b/l),
 * we handle that here, too.  Note for R4600, we have to work
 * around some chip bugs.  From the v1.7 errata:
 *
 *  18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D,
 *      Hit_Invalidate_D and Create_Dirty_Excl_D should only be
 *      executed if there is no other dcache activity. If the dcache is
 *      accessed for another instruction immeidately preceding when these
 *      cache instructions are executing, it is possible that the dcache
 *      tag match outputs used by these cache instructions will be
 *      incorrect. These cache instructions should be preceded by at least
 *      four instructions that are not any kind of load or store
 *      instruction.
 *
 * ...and from the v2.0 errata:
 *
 *   The CACHE instructions Hit_Writeback_Inv_D, Hit_Writeback_D,
 *   Hit_Invalidate_D and Create_Dirty_Exclusive_D will only operate
 *   correctly if the internal data cache refill buffer is empty.  These
 *   CACHE instructions should be separated from any potential data cache
 *   miss by a load instruction to an uncached address to empty the response
 *   buffer.
 *
 * XXX Does not handle split secondary caches.
 */

#define	round_line16(x)		round_line(x, 16)
#define	trunc_line16(x)		trunc_line(x, 16)
#define	round_line32(x)		round_line(x, 32)
#define	trunc_line32(x)		trunc_line(x, 32)
#define	round_line(x,n)		(((x) + (register_t)(n) - 1) & -(register_t)(n))
#define	trunc_line(x,n)		((x) & -(register_t)(n))

__asm(".set mips3");

void
r5k_picache_sync_all(void)
{
        struct mips_cache_info * const mci = &mips_cache_info;

        /*
         * Since we're hitting the whole thing, we don't have to
         * worry about the N different "ways".
         */
        mips_intern_dcache_sync_all();
	__asm volatile("sync");
        mips_intern_icache_sync_range_index(MIPS_KSEG0_START,
            mci->mci_picache_size);
}

void
r5k_picache_sync_range(register_t va, vsize_t size)
{

	mips_intern_dcache_sync_range(va, size);
	mips_intern_icache_sync_range(va, size);
}

void
r5k_picache_sync_range_index(vaddr_t va, vsize_t size)
{
	struct mips_cache_info * const mci = &mips_cache_info;
	const size_t ways = mci->mci_picache_ways;
	const size_t line_size = mci->mci_picache_line_size;
	const size_t way_size = mci->mci_picache_way_size;
	const size_t way_mask = way_size - 1;
	vaddr_t eva;

	/*
	 * Since we're doing Index ops, we expect to not be able
	 * to access the address we've been given.  So, get the
	 * bits that determine the cache index, and make a KSEG0
	 * address out of them.
	 */
	va = MIPS_PHYS_TO_KSEG0(va & way_mask);

	eva = round_line(va + size, line_size);
	va = trunc_line(va, line_size);
	size = eva - va;

	/*
	 * If we are going to flush more than is in a way (or the stride
	 * needed for that way), we are flushing everything.
	 */
	if (size >= way_size) {
		r5k_picache_sync_all();
		return;
	}

	for (size_t way = 0; way < ways; way++) {
		mips_intern_dcache_sync_range_index(va, size);
		mips_intern_icache_sync_range_index(va, size);
		va += way_size;
		eva += way_size;
	}
}

void
r5k_pdcache_wbinv_all(void)
{
	struct mips_cache_info * const mci = &mips_cache_info;

	/*
	 * Since we're hitting the whole thing, we don't have to
	 * worry about the N different "ways".
	 */
	mips_intern_pdcache_wbinv_range_index(MIPS_KSEG0_START,
	    mci->mci_pdcache_size);
}

void
r5k_pdcache_wbinv_range_index(vaddr_t va, vsize_t size)
{
	struct mips_cache_info * const mci = &mips_cache_info;
	const size_t ways = mci->mci_pdcache_ways;
	const size_t line_size = mci->mci_pdcache_line_size;
	const vaddr_t way_size = mci->mci_pdcache_way_size;
	const vaddr_t way_mask = way_size - 1;
	vaddr_t eva;

	/*
	 * Since we're doing Index ops, we expect to not be able
	 * to access the address we've been given.  So, get the
	 * bits that determine the cache index, and make a KSEG0
	 * address out of them.
	 */
	va = MIPS_PHYS_TO_KSEG0(va & way_mask);
	eva = round_line(va + size, line_size);
	va = trunc_line(va, line_size);
	size = eva - va;

	/*
	 * If we are going to flush more than is in a way, we are flushing
	 * everything.
	 */
	if (size >= way_size) {
		mips_intern_pdcache_wbinv_range_index(MIPS_KSEG0_START,
		    mci->mci_pdcache_size);
		return;
	}

	/*
	 * Invalidate each way.  If the address range wraps past the end of
	 * the way, we will be invalidating in two ways but eventually things
	 * work out since the last way will wrap into the first way.
	 */
	for (size_t way = 0; way < ways; way++) {
		mips_intern_pdcache_wbinv_range_index(va, size);
		va += way_size;
		eva += way_size;
	}
}

void
r4600v1_pdcache_wbinv_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	/*
	 * This is pathetically slow, but the chip bug is pretty
	 * nasty, and we hope that not too many v1.x R4600s are
	 * around.
	 */

	va = trunc_line32(va);

	/*
	 * To make this a little less painful, just hit the entire
	 * cache if we have a range >= the cache size.
	 */
	if (eva - va >= mips_cache_info.mci_pdcache_size) {
		r5k_pdcache_wbinv_all();
		return;
	}

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	while (va < eva) {
		__asm volatile("nop; nop; nop; nop");
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
		va += 32;
	}

	mips_cp0_status_write(ostatus);
}

void
r4600v2_pdcache_wbinv_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	va = trunc_line32(va);

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	for (; (eva - va) >= (32 * 32); va += (32 * 32)) {
		(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
		cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
	}

	(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
	for (; va < eva; va += 32) {
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);

	}

	mips_cp0_status_write(ostatus);
}

void
vr4131v1_pdcache_wbinv_range_16(register_t va, vsize_t size)
{
	register_t eva = round_line16(va + size);

	va = trunc_line16(va);

	for (; (eva - va) >= (32 * 16); va += (32 * 16)) {
		cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
		cache_r4k_op_32lines_16(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);

	}

	for (; va < eva; va += 16) {
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
	}
}

void
r4600v1_pdcache_inv_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	/*
	 * This is pathetically slow, but the chip bug is pretty
	 * nasty, and we hope that not too many v1.x R4600s are
	 * around.
	 */

	va = trunc_line32(va);

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	for (; va < eva; va += 32) {
		__asm volatile("nop; nop; nop; nop;");
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);

	}

	mips_cp0_status_write(ostatus);
}

void
r4600v2_pdcache_inv_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	va = trunc_line32(va);

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	/*
	 * Between blasts of big cache chunks, give interrupts
	 * a chance to get though.
	 */
	for (; (eva - va) >= (32 * 32); va += (32 * 32)) {
		(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
		cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);

	}

	(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
	for (; va < eva; va += 32) {
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);

	}

	mips_cp0_status_write(ostatus);
}

void
r4600v1_pdcache_wb_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	/*
	 * This is pathetically slow, but the chip bug is pretty
	 * nasty, and we hope that not too many v1.x R4600s are
	 * around.
	 */

	va = trunc_line32(va);

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	for (; va < eva; va += 32) {
		__asm volatile("nop; nop; nop; nop;");
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);

	}

	mips_cp0_status_write(ostatus);
}

void
r4600v2_pdcache_wb_range_32(register_t va, vsize_t size)
{
	const register_t eva = round_line32(va + size);

	va = trunc_line32(va);

	const uint32_t ostatus = mips_cp0_status_read();

	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	/*
	 * Between blasts of big cache chunks, give interrupts
	 * a chance to get though.
	 */
	for (; (eva - va) >= (32 * 32); va += (32 * 32)) {
		(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
		cache_r4k_op_32lines_32(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);

	}

	(void) *(volatile int *)MIPS_PHYS_TO_KSEG1(0);
	for (; va < eva; va += 32) {
		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
	}

	mips_cp0_status_write(ostatus);
}

/*
 * Cache operations for R5000-style secondary caches:
 *
 *	- Direct-mapped
 *	- Write-through
 *	- Physically indexed, physically tagged
 *
 */


__asm(".set mips3");

void
r5k_sdcache_wbinv_all(void)
{

	r5k_sdcache_wbinv_range(MIPS_PHYS_TO_KSEG0(0), mips_cache_info.mci_sdcache_size);
}

void
r5k_sdcache_wbinv_range_index(vaddr_t va, vsize_t size)
{

	/*
	 * Since we're doing Index ops, we expect to not be able
	 * to access the address we've been given.  So, get the
	 * bits that determine the cache index, and make a KSEG0
	 * address out of them.
	 */
	va = MIPS_PHYS_TO_KSEG0(va & (mips_cache_info.mci_sdcache_size - 1));
	r5k_sdcache_wbinv_range((intptr_t)va, size);
}

void
r5k_sdcache_wbinv_range(register_t va, vsize_t size)
{
	uint32_t ostatus, taglo;
	register_t eva = mips_r5k_round_page(va + size);

	va = mips_r5k_trunc_page(va);

	ostatus = mips_cp0_status_read();
	mips_cp0_status_write(ostatus & ~MIPS_SR_INT_IE);

	__asm volatile("mfc0 %0, $28" : "=r"(taglo));
	__asm volatile("mtc0 $0, $28");

	for (; va < eva; va += R5K_SC_PAGESIZE) {
		cache_op_r4k_line(va, CACHEOP_R5K_Page_Invalidate_S);
	}

	mips_cp0_status_write(ostatus);
	__asm volatile("mtc0 %0, $28; nop" :: "r"(taglo));
}