Home | History | Annotate | Line # | Download | only in booke 
      1 /*	$NetBSD: booke_cache.c,v 1.5 2020/07/06 09:34:16 rin Exp $	*/
      2 /*-
      3  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
      4  * All rights reserved.
      5  *
      6  * This code is derived from software contributed to The NetBSD Foundation
      7  * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
      8  * Agency and which was developed by Matt Thomas of 3am Software Foundry.
      9  *
     10  * This material is based upon work supported by the Defense Advanced Research
     11  * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
     12  * Contract No. N66001-09-C-2073.
     13  * Approved for Public Release, Distribution Unlimited
     14  *
     15  * Redistribution and use in source and binary forms, with or without
     16  * modification, are permitted provided that the following conditions
     17  * are met:
     18  * 1. Redistributions of source code must retain the above copyright
     19  *    notice, this list of conditions and the following disclaimer.
     20  * 2. Redistributions in binary form must reproduce the above copyright
     21  *    notice, this list of conditions and the following disclaimer in the
     22  *    documentation and/or other materials provided with the distribution.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     26  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     27  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     34  * POSSIBILITY OF SUCH DAMAGE.
     35  */
     36 
     37 #include <sys/cdefs.h>
     38 __KERNEL_RCSID(0, "$NetBSD: booke_cache.c,v 1.5 2020/07/06 09:34:16 rin Exp $");
     39 
     40 #include <sys/param.h>
     41 #include <sys/cpu.h>
     42 #include <sys/atomic.h>
     43 
     44 enum cache_op { OP_DCBF, OP_DCBST, OP_DCBI, OP_DCBZ, OP_DCBA, OP_ICBI };
     45 
     46 static void inline
     47 dcbf(vaddr_t va, vsize_t off)
     48 {
     49 	__asm volatile("dcbf\t%0,%1" : : "b" (va), "r" (off));
     50 }
     51 
     52 static void inline
     53 dcbst(vaddr_t va, vsize_t off)
     54 {
     55 	__asm volatile("dcbst\t%0,%1" : : "b" (va), "r" (off));
     56 }
     57 
     58 static void inline
     59 dcbi(vaddr_t va, vsize_t off)
     60 {
     61 	__asm volatile("dcbi\t%0,%1" : : "b" (va), "r" (off));
     62 }
     63 
     64 static void inline
     65 dcbz(vaddr_t va, vsize_t off)
     66 {
     67 	__asm volatile("dcbz\t%0,%1" : : "b" (va), "r" (off));
     68 }
     69 
     70 static void inline
     71 dcba(vaddr_t va, vsize_t off)
     72 {
     73 	__asm volatile("dcba\t%0,%1" : : "b" (va), "r" (off));
     74 }
     75 
     76 static void inline
     77 icbi(vaddr_t va, vsize_t off)
     78 {
     79 	__asm volatile("icbi\t%0,%1" : : "b" (va), "r" (off));
     80 }
     81 
     82 static inline void
     83 cache_op(vaddr_t va, vsize_t len, vsize_t line_size, enum cache_op op)
     84 {
     85 	KASSERT(line_size > 0);
     86 
     87 	if (len == 0)
     88 		return;
     89 
     90 	/* Make sure we flush all cache lines */
     91 	len += va & (line_size - 1);
     92 	va &= -line_size;
     93 
     94 	for (vsize_t i = 0; i < len; i += line_size) {
     95 		switch (op) {
     96 		case OP_DCBF: dcbf(va, i); break;
     97 		case OP_DCBST: dcbst(va, i); break;
     98 		case OP_DCBI: dcbi(va, i); break;
     99 		case OP_DCBZ: dcbz(va, i); break;
    100 		case OP_DCBA: dcba(va, i); break;
    101 		case OP_ICBI: icbi(va, i); break;
    102 		}
    103 	}
    104 	if (op != OP_ICBI)
    105 		membar_producer();
    106 }
    107 
    108 void
    109 dcache_wb_page(vaddr_t va)
    110 {
    111 	cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBST);
    112 }
    113 
    114 void
    115 dcache_wbinv_page(vaddr_t va)
    116 {
    117 	cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBF);
    118 }
    119 
    120 void
    121 dcache_inv_page(vaddr_t va)
    122 {
    123 	cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBI);
    124 }
    125 
    126 void
    127 dcache_zero_page(vaddr_t va)
    128 {
    129 	cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBZ);
    130 }
    131 
    132 void
    133 icache_inv_page(vaddr_t va)
    134 {
    135 	membar_sync();
    136 	cache_op(va, PAGE_SIZE, curcpu()->ci_ci.icache_line_size, OP_ICBI);
    137 	membar_sync();
    138 	/* synchronizing instruction will be the rfi to user mode */
    139 }
    140 
    141 void
    142 dcache_wb(vaddr_t va, vsize_t len)
    143 {
    144 	cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBST);
    145 }
    146 
    147 void
    148 dcache_wbinv(vaddr_t va, vsize_t len)
    149 {
    150 	cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBF);
    151 }
    152 
    153 void
    154 dcache_inv(vaddr_t va, vsize_t len)
    155 {
    156 	cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBI);
    157 }
    158 
    159 void
    160 icache_inv(vaddr_t va, vsize_t len)
    161 {
    162 	membar_sync();
    163 	cache_op(va, len, curcpu()->ci_ci.icache_line_size, OP_ICBI);
    164 	membar_sync();
    165 }
    166