Home | History | Annotate | Line # | Download | only in aarch64
      1 /* $NetBSD: bus_space.c,v 1.18 2024/02/07 04:20:26 msaitoh Exp $ */
      2 
      3 /*
      4  * Copyright (c) 2017 Ryo Shimizu
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     19  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
     20  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
     24  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
     25  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(1, "$NetBSD: bus_space.c,v 1.18 2024/02/07 04:20:26 msaitoh Exp $");
     31 
     32 #include <sys/param.h>
     33 #include <sys/systm.h>
     34 #include <sys/bus.h>
     35 
     36 #include <uvm/uvm_extern.h>
     37 
     38 #include <aarch64/bus_funcs.h>
     39 #include <aarch64/machdep.h>
     40 
     41 #include <arm/cpufunc.h>
     42 
     43 /* Prototypes for all the bus_space structure functions */
     44 bs_protos(generic)
     45 bs_protos(generic_dsb)
     46 
     47 #if __AARCH64EB__
     48 #define NSWAP(n)	n ## _swap
     49 #else
     50 #define NSWAP(n)	n
     51 #endif
     52 
     53 struct bus_space arm_generic_bs_tag = {
     54 	.bs_cookie = &arm_generic_bs_tag,
     55 
     56 	.bs_stride = 0,
     57 	.bs_flags = 0,
     58 
     59 	.bs_map = generic_bs_map,
     60 	.bs_unmap = generic_bs_unmap,
     61 	.bs_subregion = generic_bs_subregion,
     62 	.bs_alloc = generic_bs_alloc,
     63 	.bs_free = generic_bs_free,
     64 	.bs_vaddr = generic_bs_vaddr,
     65 	.bs_mmap = generic_bs_mmap,
     66 	.bs_barrier = generic_bs_barrier,
     67 
     68 	/* read */
     69 	.bs_r_1 = generic_bs_r_1,
     70 	.bs_r_2 = NSWAP(generic_bs_r_2),
     71 	.bs_r_4 = NSWAP(generic_bs_r_4),
     72 	.bs_r_8 = NSWAP(generic_bs_r_8),
     73 
     74 	/* write */
     75 	.bs_w_1 = generic_bs_w_1,
     76 	.bs_w_2 = NSWAP(generic_bs_w_2),
     77 	.bs_w_4 = NSWAP(generic_bs_w_4),
     78 	.bs_w_8 = NSWAP(generic_bs_w_8),
     79 
     80 	/* read region */
     81 	.bs_rr_1 = generic_bs_rr_1,
     82 	.bs_rr_2 = NSWAP(generic_bs_rr_2),
     83 	.bs_rr_4 = NSWAP(generic_bs_rr_4),
     84 	.bs_rr_8 = NSWAP(generic_bs_rr_8),
     85 
     86 	/* write region */
     87 	.bs_wr_1 = generic_bs_wr_1,
     88 	.bs_wr_2 = NSWAP(generic_bs_wr_2),
     89 	.bs_wr_4 = NSWAP(generic_bs_wr_4),
     90 	.bs_wr_8 = NSWAP(generic_bs_wr_8),
     91 
     92 	/* copy region */
     93 	.bs_c_1 = generic_bs_c_1,
     94 	.bs_c_2 = generic_bs_c_2,
     95 	.bs_c_4 = generic_bs_c_4,
     96 	.bs_c_8 = generic_bs_c_8,
     97 
     98 	/* set region */
     99 	.bs_sr_1 = generic_bs_sr_1,
    100 	.bs_sr_2 = NSWAP(generic_bs_sr_2),
    101 	.bs_sr_4 = NSWAP(generic_bs_sr_4),
    102 	.bs_sr_8 = NSWAP(generic_bs_sr_8),
    103 
    104 	/* read multi */
    105 	.bs_rm_1 = generic_bs_rm_1,
    106 	.bs_rm_2 = NSWAP(generic_bs_rm_2),
    107 	.bs_rm_4 = NSWAP(generic_bs_rm_4),
    108 	.bs_rm_8 = NSWAP(generic_bs_rm_8),
    109 
    110 	/* write multi */
    111 	.bs_wm_1 = generic_bs_wm_1,
    112 	.bs_wm_2 = NSWAP(generic_bs_wm_2),
    113 	.bs_wm_4 = NSWAP(generic_bs_wm_4),
    114 	.bs_wm_8 = NSWAP(generic_bs_wm_8),
    115 
    116 	/* set multi */
    117 	.bs_sm_1 = generic_bs_sm_1,
    118 	.bs_sm_2 = NSWAP(generic_bs_sm_2),
    119 	.bs_sm_4 = NSWAP(generic_bs_sm_4),
    120 	.bs_sm_8 = NSWAP(generic_bs_sm_8),
    121 
    122 #ifdef __BUS_SPACE_HAS_STREAM_METHODS
    123 	/* read stream */
    124 	.bs_r_1_s = generic_bs_r_1,
    125 	.bs_r_2_s = generic_bs_r_2,
    126 	.bs_r_4_s = generic_bs_r_4,
    127 	.bs_r_8_s = generic_bs_r_8,
    128 
    129 	/* write stream */
    130 	.bs_w_1_s = generic_bs_w_1,
    131 	.bs_w_2_s = generic_bs_w_2,
    132 	.bs_w_4_s = generic_bs_w_4,
    133 	.bs_w_8_s = generic_bs_w_8,
    134 
    135 	/* read region stream */
    136 	.bs_rr_1_s = generic_bs_rr_1,
    137 	.bs_rr_2_s = generic_bs_rr_2,
    138 	.bs_rr_4_s = generic_bs_rr_4,
    139 	.bs_rr_8_s = generic_bs_rr_8,
    140 
    141 	/* write region stream */
    142 	.bs_wr_1_s = generic_bs_wr_1,
    143 	.bs_wr_2_s = generic_bs_wr_2,
    144 	.bs_wr_4_s = generic_bs_wr_4,
    145 	.bs_wr_8_s = generic_bs_wr_8,
    146 
    147 	/* read multi stream */
    148 	.bs_rm_1_s = generic_bs_rm_1,
    149 	.bs_rm_2_s = generic_bs_rm_2,
    150 	.bs_rm_4_s = generic_bs_rm_4,
    151 	.bs_rm_8_s = generic_bs_rm_8,
    152 
    153 	/* write multi stream */
    154 	.bs_wm_1_s = generic_bs_wm_1,
    155 	.bs_wm_2_s = generic_bs_wm_2,
    156 	.bs_wm_4_s = generic_bs_wm_4,
    157 	.bs_wm_8_s = generic_bs_wm_8,
    158 #endif
    159 
    160 #ifdef __BUS_SPACE_HAS_PROBING_METHODS
    161 	/* peek */
    162 	.bs_pe_1 = generic_bs_pe_1,
    163 	.bs_pe_2 = generic_bs_pe_2,
    164 	.bs_pe_4 = generic_bs_pe_4,
    165 	.bs_pe_8 = generic_bs_pe_8,
    166 
    167 	/* poke */
    168 	.bs_po_1 = generic_bs_po_1,
    169 	.bs_po_2 = generic_bs_po_2,
    170 	.bs_po_4 = generic_bs_po_4,
    171 	.bs_po_8 = generic_bs_po_8,
    172 #endif
    173 };
    174 
    175 struct bus_space aarch64_generic_dsb_bs_tag = {
    176 	.bs_cookie = &aarch64_generic_dsb_bs_tag,
    177 
    178 	.bs_stride = 0,
    179 	.bs_flags = 0,
    180 
    181 	.bs_map = generic_bs_map,
    182 	.bs_unmap = generic_bs_unmap,
    183 	.bs_subregion = generic_bs_subregion,
    184 	.bs_alloc = generic_bs_alloc,
    185 	.bs_free = generic_bs_free,
    186 	.bs_vaddr = generic_bs_vaddr,
    187 	.bs_mmap = generic_bs_mmap,
    188 	.bs_barrier = generic_bs_barrier,
    189 
    190 	/* read */
    191 	.bs_r_1 = generic_dsb_bs_r_1,
    192 	.bs_r_2 = NSWAP(generic_dsb_bs_r_2),
    193 	.bs_r_4 = NSWAP(generic_dsb_bs_r_4),
    194 	.bs_r_8 = NSWAP(generic_dsb_bs_r_8),
    195 
    196 	/* write */
    197 	.bs_w_1 = generic_dsb_bs_w_1,
    198 	.bs_w_2 = NSWAP(generic_dsb_bs_w_2),
    199 	.bs_w_4 = NSWAP(generic_dsb_bs_w_4),
    200 	.bs_w_8 = NSWAP(generic_dsb_bs_w_8),
    201 
    202 	/* read region */
    203 	.bs_rr_1 = generic_dsb_bs_rr_1,
    204 	.bs_rr_2 = NSWAP(generic_dsb_bs_rr_2),
    205 	.bs_rr_4 = NSWAP(generic_dsb_bs_rr_4),
    206 	.bs_rr_8 = NSWAP(generic_dsb_bs_rr_8),
    207 
    208 	/* write region */
    209 	.bs_wr_1 = generic_dsb_bs_wr_1,
    210 	.bs_wr_2 = NSWAP(generic_dsb_bs_wr_2),
    211 	.bs_wr_4 = NSWAP(generic_dsb_bs_wr_4),
    212 	.bs_wr_8 = NSWAP(generic_dsb_bs_wr_8),
    213 
    214 	/* copy region */
    215 	.bs_c_1 = generic_dsb_bs_c_1,
    216 	.bs_c_2 = generic_dsb_bs_c_2,
    217 	.bs_c_4 = generic_dsb_bs_c_4,
    218 	.bs_c_8 = generic_dsb_bs_c_8,
    219 
    220 	/* set region */
    221 	.bs_sr_1 = generic_dsb_bs_sr_1,
    222 	.bs_sr_2 = NSWAP(generic_dsb_bs_sr_2),
    223 	.bs_sr_4 = NSWAP(generic_dsb_bs_sr_4),
    224 	.bs_sr_8 = NSWAP(generic_dsb_bs_sr_8),
    225 
    226 	/* read multi */
    227 	.bs_rm_1 = generic_dsb_bs_rm_1,
    228 	.bs_rm_2 = NSWAP(generic_dsb_bs_rm_2),
    229 	.bs_rm_4 = NSWAP(generic_dsb_bs_rm_4),
    230 	.bs_rm_8 = NSWAP(generic_dsb_bs_rm_8),
    231 
    232 	/* write multi */
    233 	.bs_wm_1 = generic_dsb_bs_wm_1,
    234 	.bs_wm_2 = NSWAP(generic_dsb_bs_wm_2),
    235 	.bs_wm_4 = NSWAP(generic_dsb_bs_wm_4),
    236 	.bs_wm_8 = NSWAP(generic_dsb_bs_wm_8),
    237 
    238 	/* set multi */
    239 	.bs_sm_1 = generic_dsb_bs_sm_1,
    240 	.bs_sm_2 = NSWAP(generic_dsb_bs_sm_2),
    241 	.bs_sm_4 = NSWAP(generic_dsb_bs_sm_4),
    242 	.bs_sm_8 = NSWAP(generic_dsb_bs_sm_8),
    243 
    244 #ifdef __BUS_SPACE_HAS_STREAM_METHODS
    245 	/* read stream */
    246 	.bs_r_1_s = generic_dsb_bs_r_1,
    247 	.bs_r_2_s = generic_dsb_bs_r_2,
    248 	.bs_r_4_s = generic_dsb_bs_r_4,
    249 	.bs_r_8_s = generic_dsb_bs_r_8,
    250 
    251 	/* write stream */
    252 	.bs_w_1_s = generic_dsb_bs_w_1,
    253 	.bs_w_2_s = generic_dsb_bs_w_2,
    254 	.bs_w_4_s = generic_dsb_bs_w_4,
    255 	.bs_w_8_s = generic_dsb_bs_w_8,
    256 
    257 	/* read region stream */
    258 	.bs_rr_1_s = generic_dsb_bs_rr_1,
    259 	.bs_rr_2_s = generic_dsb_bs_rr_2,
    260 	.bs_rr_4_s = generic_dsb_bs_rr_4,
    261 	.bs_rr_8_s = generic_dsb_bs_rr_8,
    262 
    263 	/* write region stream */
    264 	.bs_wr_1_s = generic_dsb_bs_wr_1,
    265 	.bs_wr_2_s = generic_dsb_bs_wr_2,
    266 	.bs_wr_4_s = generic_dsb_bs_wr_4,
    267 	.bs_wr_8_s = generic_dsb_bs_wr_8,
    268 
    269 	/* read multi stream */
    270 	.bs_rm_1_s = generic_dsb_bs_rm_1,
    271 	.bs_rm_2_s = generic_dsb_bs_rm_2,
    272 	.bs_rm_4_s = generic_dsb_bs_rm_4,
    273 	.bs_rm_8_s = generic_dsb_bs_rm_8,
    274 
    275 	/* write multi stream */
    276 	.bs_wm_1_s = generic_dsb_bs_wm_1,
    277 	.bs_wm_2_s = generic_dsb_bs_wm_2,
    278 	.bs_wm_4_s = generic_dsb_bs_wm_4,
    279 	.bs_wm_8_s = generic_dsb_bs_wm_8,
    280 #endif
    281 
    282 #ifdef __BUS_SPACE_HAS_PROBING_METHODS
    283 	/* peek */
    284 	.bs_pe_1 = generic_bs_pe_1,
    285 	.bs_pe_2 = generic_bs_pe_2,
    286 	.bs_pe_4 = generic_bs_pe_4,
    287 	.bs_pe_8 = generic_bs_pe_8,
    288 
    289 	/* poke */
    290 	.bs_po_1 = generic_bs_po_1,
    291 	.bs_po_2 = generic_bs_po_2,
    292 	.bs_po_4 = generic_bs_po_4,
    293 	.bs_po_8 = generic_bs_po_8,
    294 #endif
    295 };
    296 
    297 struct bus_space arm_generic_a4x_bs_tag = {
    298 	.bs_cookie = &arm_generic_a4x_bs_tag,
    299 
    300 	.bs_stride = 2,
    301 	.bs_flags = 0,
    302 
    303 	.bs_map = generic_bs_map,
    304 	.bs_unmap = generic_bs_unmap,
    305 	.bs_subregion = generic_bs_subregion,
    306 	.bs_alloc = generic_bs_alloc,
    307 	.bs_free = generic_bs_free,
    308 	.bs_vaddr = generic_bs_vaddr,
    309 	.bs_mmap = generic_bs_mmap,
    310 	.bs_barrier = generic_bs_barrier,
    311 
    312 	/* read */
    313 	.bs_r_1 = generic_bs_r_1,
    314 	.bs_r_2 = NSWAP(generic_bs_r_2),
    315 	.bs_r_4 = NSWAP(generic_bs_r_4),
    316 	.bs_r_8 = NSWAP(generic_bs_r_8),
    317 
    318 	/* write */
    319 	.bs_w_1 = generic_bs_w_1,
    320 	.bs_w_2 = NSWAP(generic_bs_w_2),
    321 	.bs_w_4 = NSWAP(generic_bs_w_4),
    322 	.bs_w_8 = NSWAP(generic_bs_w_8),
    323 
    324 	/* read region */
    325 	.bs_rr_1 = generic_bs_rr_1,
    326 	.bs_rr_2 = NSWAP(generic_bs_rr_2),
    327 	.bs_rr_4 = NSWAP(generic_bs_rr_4),
    328 	.bs_rr_8 = NSWAP(generic_bs_rr_8),
    329 
    330 	/* write region */
    331 	.bs_wr_1 = generic_bs_wr_1,
    332 	.bs_wr_2 = NSWAP(generic_bs_wr_2),
    333 	.bs_wr_4 = NSWAP(generic_bs_wr_4),
    334 	.bs_wr_8 = NSWAP(generic_bs_wr_8),
    335 
    336 	/* copy region */
    337 	.bs_c_1 = generic_bs_c_1,
    338 	.bs_c_2 = generic_bs_c_2,
    339 	.bs_c_4 = generic_bs_c_4,
    340 	.bs_c_8 = generic_bs_c_8,
    341 
    342 	/* set region */
    343 	.bs_sr_1 = generic_bs_sr_1,
    344 	.bs_sr_2 = NSWAP(generic_bs_sr_2),
    345 	.bs_sr_4 = NSWAP(generic_bs_sr_4),
    346 	.bs_sr_8 = NSWAP(generic_bs_sr_8),
    347 
    348 	/* read multi */
    349 	.bs_rm_1 = generic_bs_rm_1,
    350 	.bs_rm_2 = NSWAP(generic_bs_rm_2),
    351 	.bs_rm_4 = NSWAP(generic_bs_rm_4),
    352 	.bs_rm_8 = NSWAP(generic_bs_rm_8),
    353 
    354 	/* write multi */
    355 	.bs_wm_1 = generic_bs_wm_1,
    356 	.bs_wm_2 = NSWAP(generic_bs_wm_2),
    357 	.bs_wm_4 = NSWAP(generic_bs_wm_4),
    358 	.bs_wm_8 = NSWAP(generic_bs_wm_8),
    359 
    360 	/* set multi */
    361 	.bs_sm_1 = generic_bs_sm_1,
    362 	.bs_sm_2 = NSWAP(generic_bs_sm_2),
    363 	.bs_sm_4 = NSWAP(generic_bs_sm_4),
    364 	.bs_sm_8 = NSWAP(generic_bs_sm_8),
    365 
    366 #ifdef __BUS_SPACE_HAS_STREAM_METHODS
    367 	/* read stream */
    368 	.bs_r_1_s = generic_bs_r_1,
    369 	.bs_r_2_s = generic_bs_r_2,
    370 	.bs_r_4_s = generic_bs_r_4,
    371 	.bs_r_8_s = generic_bs_r_8,
    372 
    373 	/* write stream */
    374 	.bs_w_1_s = generic_bs_w_1,
    375 	.bs_w_2_s = generic_bs_w_2,
    376 	.bs_w_4_s = generic_bs_w_4,
    377 	.bs_w_8_s = generic_bs_w_8,
    378 
    379 	/* read region stream */
    380 	.bs_rr_1_s = generic_bs_rr_1,
    381 	.bs_rr_2_s = generic_bs_rr_2,
    382 	.bs_rr_4_s = generic_bs_rr_4,
    383 	.bs_rr_8_s = generic_bs_rr_8,
    384 
    385 	/* write region stream */
    386 	.bs_wr_1_s = generic_bs_wr_1,
    387 	.bs_wr_2_s = generic_bs_wr_2,
    388 	.bs_wr_4_s = generic_bs_wr_4,
    389 	.bs_wr_8_s = generic_bs_wr_8,
    390 
    391 	/* read multi stream */
    392 	.bs_rm_1_s = generic_bs_rm_1,
    393 	.bs_rm_2_s = generic_bs_rm_2,
    394 	.bs_rm_4_s = generic_bs_rm_4,
    395 	.bs_rm_8_s = generic_bs_rm_8,
    396 
    397 	/* write multi stream */
    398 	.bs_wm_1_s = generic_bs_wm_1,
    399 	.bs_wm_2_s = generic_bs_wm_2,
    400 	.bs_wm_4_s = generic_bs_wm_4,
    401 	.bs_wm_8_s = generic_bs_wm_8,
    402 #endif
    403 
    404 #ifdef __BUS_SPACE_HAS_PROBING_METHODS
    405 	/* peek */
    406 	.bs_pe_1 = generic_bs_pe_1,
    407 	.bs_pe_2 = generic_bs_pe_2,
    408 	.bs_pe_4 = generic_bs_pe_4,
    409 	.bs_pe_8 = generic_bs_pe_8,
    410 
    411 	/* poke */
    412 	.bs_po_1 = generic_bs_po_1,
    413 	.bs_po_2 = generic_bs_po_2,
    414 	.bs_po_4 = generic_bs_po_4,
    415 	.bs_po_8 = generic_bs_po_8,
    416 #endif
    417 };
    418 
    419 struct bus_space aarch64_generic_a4x_dsb_bs_tag = {
    420 	.bs_cookie = &aarch64_generic_a4x_dsb_bs_tag,
    421 
    422 	.bs_stride = 2,
    423 	.bs_flags = 0,
    424 
    425 	.bs_map = generic_bs_map,
    426 	.bs_unmap = generic_bs_unmap,
    427 	.bs_subregion = generic_bs_subregion,
    428 	.bs_alloc = generic_bs_alloc,
    429 	.bs_free = generic_bs_free,
    430 	.bs_vaddr = generic_bs_vaddr,
    431 	.bs_mmap = generic_bs_mmap,
    432 	.bs_barrier = generic_bs_barrier,
    433 
    434 	/* read */
    435 	.bs_r_1 = generic_dsb_bs_r_1,
    436 	.bs_r_2 = NSWAP(generic_dsb_bs_r_2),
    437 	.bs_r_4 = NSWAP(generic_dsb_bs_r_4),
    438 	.bs_r_8 = NSWAP(generic_dsb_bs_r_8),
    439 
    440 	/* write */
    441 	.bs_w_1 = generic_dsb_bs_w_1,
    442 	.bs_w_2 = NSWAP(generic_dsb_bs_w_2),
    443 	.bs_w_4 = NSWAP(generic_dsb_bs_w_4),
    444 	.bs_w_8 = NSWAP(generic_dsb_bs_w_8),
    445 
    446 	/* read region */
    447 	.bs_rr_1 = generic_dsb_bs_rr_1,
    448 	.bs_rr_2 = NSWAP(generic_dsb_bs_rr_2),
    449 	.bs_rr_4 = NSWAP(generic_dsb_bs_rr_4),
    450 	.bs_rr_8 = NSWAP(generic_dsb_bs_rr_8),
    451 
    452 	/* write region */
    453 	.bs_wr_1 = generic_dsb_bs_wr_1,
    454 	.bs_wr_2 = NSWAP(generic_dsb_bs_wr_2),
    455 	.bs_wr_4 = NSWAP(generic_dsb_bs_wr_4),
    456 	.bs_wr_8 = NSWAP(generic_dsb_bs_wr_8),
    457 
    458 	/* copy region */
    459 	.bs_c_1 = generic_dsb_bs_c_1,
    460 	.bs_c_2 = generic_dsb_bs_c_2,
    461 	.bs_c_4 = generic_dsb_bs_c_4,
    462 	.bs_c_8 = generic_dsb_bs_c_8,
    463 
    464 	/* set region */
    465 	.bs_sr_1 = generic_dsb_bs_sr_1,
    466 	.bs_sr_2 = NSWAP(generic_dsb_bs_sr_2),
    467 	.bs_sr_4 = NSWAP(generic_dsb_bs_sr_4),
    468 	.bs_sr_8 = NSWAP(generic_dsb_bs_sr_8),
    469 
    470 	/* read multi */
    471 	.bs_rm_1 = generic_dsb_bs_rm_1,
    472 	.bs_rm_2 = NSWAP(generic_dsb_bs_rm_2),
    473 	.bs_rm_4 = NSWAP(generic_dsb_bs_rm_4),
    474 	.bs_rm_8 = NSWAP(generic_dsb_bs_rm_8),
    475 
    476 	/* write multi */
    477 	.bs_wm_1 = generic_dsb_bs_wm_1,
    478 	.bs_wm_2 = NSWAP(generic_dsb_bs_wm_2),
    479 	.bs_wm_4 = NSWAP(generic_dsb_bs_wm_4),
    480 	.bs_wm_8 = NSWAP(generic_dsb_bs_wm_8),
    481 
    482 	/* set multi */
    483 	.bs_sm_1 = generic_dsb_bs_sm_1,
    484 	.bs_sm_2 = NSWAP(generic_dsb_bs_sm_2),
    485 	.bs_sm_4 = NSWAP(generic_dsb_bs_sm_4),
    486 	.bs_sm_8 = NSWAP(generic_dsb_bs_sm_8),
    487 
    488 #ifdef __BUS_SPACE_HAS_STREAM_METHODS
    489 	/* read stream */
    490 	.bs_r_1_s = generic_dsb_bs_r_1,
    491 	.bs_r_2_s = generic_dsb_bs_r_2,
    492 	.bs_r_4_s = generic_dsb_bs_r_4,
    493 	.bs_r_8_s = generic_dsb_bs_r_8,
    494 
    495 	/* write stream */
    496 	.bs_w_1_s = generic_dsb_bs_w_1,
    497 	.bs_w_2_s = generic_dsb_bs_w_2,
    498 	.bs_w_4_s = generic_dsb_bs_w_4,
    499 	.bs_w_8_s = generic_dsb_bs_w_8,
    500 
    501 	/* read region stream */
    502 	.bs_rr_1_s = generic_dsb_bs_rr_1,
    503 	.bs_rr_2_s = generic_dsb_bs_rr_2,
    504 	.bs_rr_4_s = generic_dsb_bs_rr_4,
    505 	.bs_rr_8_s = generic_dsb_bs_rr_8,
    506 
    507 	/* write region stream */
    508 	.bs_wr_1_s = generic_dsb_bs_wr_1,
    509 	.bs_wr_2_s = generic_dsb_bs_wr_2,
    510 	.bs_wr_4_s = generic_dsb_bs_wr_4,
    511 	.bs_wr_8_s = generic_dsb_bs_wr_8,
    512 
    513 	/* read multi stream */
    514 	.bs_rm_1_s = generic_dsb_bs_rm_1,
    515 	.bs_rm_2_s = generic_dsb_bs_rm_2,
    516 	.bs_rm_4_s = generic_dsb_bs_rm_4,
    517 	.bs_rm_8_s = generic_dsb_bs_rm_8,
    518 
    519 	/* write multi stream */
    520 	.bs_wm_1_s = generic_dsb_bs_wm_1,
    521 	.bs_wm_2_s = generic_dsb_bs_wm_2,
    522 	.bs_wm_4_s = generic_dsb_bs_wm_4,
    523 	.bs_wm_8_s = generic_dsb_bs_wm_8,
    524 #endif
    525 
    526 #ifdef __BUS_SPACE_HAS_PROBING_METHODS
    527 	/* peek */
    528 	.bs_pe_1 = generic_bs_pe_1,
    529 	.bs_pe_2 = generic_bs_pe_2,
    530 	.bs_pe_4 = generic_bs_pe_4,
    531 	.bs_pe_8 = generic_bs_pe_8,
    532 
    533 	/* poke */
    534 	.bs_po_1 = generic_bs_po_1,
    535 	.bs_po_2 = generic_bs_po_2,
    536 	.bs_po_4 = generic_bs_po_4,
    537 	.bs_po_8 = generic_bs_po_8,
    538 #endif
    539 };
    540 
    541 int
    542 generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flag,
    543     bus_space_handle_t *bshp)
    544 {
    545 	const struct pmap_devmap *pd;
    546 	paddr_t startpa, endpa, pa;
    547 	vaddr_t va;
    548 	int pmapflags;
    549 
    550 	if ((pd = pmap_devmap_find_pa(bpa, size)) != NULL) {
    551 		*bshp = pd->pd_va + (bpa - pd->pd_pa);
    552 		return 0;
    553 	}
    554 
    555 	startpa = trunc_page(bpa);
    556 	endpa = round_page(bpa + size);
    557 
    558 	/* XXX use extent manager to check duplicate mapping */
    559 
    560 	va = uvm_km_alloc(kernel_map, endpa - startpa, 0,
    561 	    UVM_KMF_VAONLY | UVM_KMF_NOWAIT);
    562 	if (va == 0)
    563 		return ENOMEM;
    564 
    565 	*bshp = (bus_space_handle_t)(va + (bpa - startpa));
    566 
    567 	if ((flag & BUS_SPACE_MAP_PREFETCHABLE) != 0)
    568 		pmapflags = PMAP_WRITE_COMBINE;
    569 	else if ((flag & BUS_SPACE_MAP_CACHEABLE) != 0)
    570 		pmapflags = PMAP_WRITE_BACK;
    571 	else if ((flag & BUS_SPACE_MAP_NONPOSTED) != 0)
    572 		pmapflags = PMAP_DEV_NP;
    573 	else
    574 		pmapflags = PMAP_DEV;
    575 
    576 	for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE) {
    577 		pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE, pmapflags);
    578 	}
    579 	pmap_update(pmap_kernel());
    580 
    581 	return 0;
    582 }
    583 
    584 void
    585 generic_bs_unmap(void *t, bus_space_handle_t bsh, bus_size_t size)
    586 {
    587 	vaddr_t va;
    588 	vsize_t sz;
    589 
    590 	if (pmap_devmap_find_va(bsh, size) != NULL)
    591 		return;
    592 
    593 	va = trunc_page(bsh);
    594 	sz = round_page(bsh + size) - va;
    595 
    596 	pmap_kremove(va, sz);
    597 	pmap_update(pmap_kernel());
    598 	uvm_km_free(kernel_map, va, sz, UVM_KMF_VAONLY);
    599 }
    600 
    601 
    602 int
    603 generic_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
    604     bus_size_t size, bus_space_handle_t *nbshp)
    605 {
    606 	*nbshp = bsh + (offset << ((struct bus_space *)t)->bs_stride);
    607 	return 0;
    608 }
    609 
    610 void
    611 generic_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
    612     bus_size_t len, int flags)
    613 {
    614 	flags &= BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE;
    615 
    616 	/*
    617 	 * For default mappings, which are mapped with nGnRE memory
    618 	 * regions, all loads and stores are issued in program order
    619 	 * (non-reordered).
    620 	 *
    621 	 * For strongly ordered mappings, which are mapped with nGnRnE
    622 	 * regions, all loads and stores are issued in program order
    623 	 * (non-reordered) and will complete at the endpoint, thus
    624 	 * not requiring any barrier.
    625 	 *
    626 	 * For BUS_SPACE_MAP_PREFETCHABLE mappings, which are mapped
    627 	 * as normal memory with the non-cacheable cacheability attr-
    628 	 * ibute, loads and stores may be issued out of order, and
    629 	 * writes may be buffered, potentially requiring any of the
    630 	 * read, write, and read/write barriers.
    631 	 *
    632 	 * For BUS_SPACE_MAP_CACHEABLE mappings, which are mapped as
    633 	 * normal memory with the write-back cacheability attribute
    634 	 * (just like normal memory), the same potential for any of
    635 	 * the barriers exists.
    636 	 *
    637 	 * We can't easily tell here how the region was mapped (without
    638 	 * consulting the page tables), so just issue the barrier
    639 	 * unconditionally.  Chances are either it's necessary or the
    640 	 * cost is small in comparison to device register I/O.
    641 	 *
    642 	 * The bus_space(9) man page is not clear whether barriers
    643 	 * should enforce ordering or completion. To be safe, use dsb
    644 	 * (ensure completion) here instead of dmb (ordering).
    645 	 */
    646 	switch (flags) {
    647 	case BUS_SPACE_BARRIER_READ:
    648 		dsb(ld);
    649 		break;
    650 	case BUS_SPACE_BARRIER_WRITE:
    651 		dsb(st);
    652 		break;
    653 	case BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE:
    654 		dsb(sy);
    655 		break;
    656 	}
    657 }
    658 
    659 void *
    660 generic_bs_vaddr(void *t, bus_space_handle_t bsh)
    661 {
    662 	return (void *)bsh;
    663 }
    664 
    665 paddr_t
    666 generic_bs_mmap(void *t, bus_addr_t bpa, off_t offset, int prot, int flags)
    667 {
    668 	paddr_t bus_flags = 0;
    669 
    670 	if ((flags & BUS_SPACE_MAP_CACHEABLE) != 0)
    671 		bus_flags |= ARM_MMAP_WRITEBACK;
    672 	else if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0)
    673 		bus_flags |= ARM_MMAP_WRITECOMBINE;
    674 	else
    675 		bus_flags |= ARM_MMAP_DEVICE;
    676 
    677 	return (atop(bpa + (offset << ((struct bus_space *)t)->bs_stride)) |
    678 	    bus_flags);
    679 }
    680 
    681 int
    682 generic_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
    683     bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
    684     bus_addr_t *bpap, bus_space_handle_t *bshp)
    685 {
    686 	panic("%s(): not implemented\n", __func__);
    687 }
    688 
    689 void
    690 generic_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
    691 {
    692 	panic("%s(): not implemented\n", __func__);
    693 }
    694 
    695 #ifdef __BUS_SPACE_HAS_PROBING_METHODS
    696 int
    697 generic_bs_pe_1(void *t, bus_space_handle_t bsh, bus_size_t offset,
    698     uint8_t *datap)
    699 {
    700 	struct faultbuf fb;
    701 	int error;
    702 
    703 	if ((error = cpu_set_onfault(&fb)) == 0) {
    704 		*datap = generic_dsb_bs_r_1(t, bsh, offset);
    705 		dsb(ld);
    706 		cpu_unset_onfault();
    707 	}
    708 	return error;
    709 }
    710 
    711 int
    712 generic_bs_pe_2(void *t, bus_space_handle_t bsh, bus_size_t offset,
    713     uint16_t *datap)
    714 {
    715 	struct faultbuf fb;
    716 	int error;
    717 
    718 	if ((error = cpu_set_onfault(&fb)) == 0) {
    719 		*datap = NSWAP(generic_dsb_bs_r_2)(t, bsh, offset);
    720 		dsb(ld);
    721 		cpu_unset_onfault();
    722 	}
    723 	return error;
    724 }
    725 
    726 int
    727 generic_bs_pe_4(void *t, bus_space_handle_t bsh, bus_size_t offset,
    728     uint32_t *datap)
    729 {
    730 	struct faultbuf fb;
    731 	int error;
    732 
    733 	if ((error = cpu_set_onfault(&fb)) == 0) {
    734 		*datap = NSWAP(generic_dsb_bs_r_4)(t, bsh, offset);
    735 		dsb(ld);
    736 		cpu_unset_onfault();
    737 	}
    738 	return error;
    739 }
    740 
    741 int
    742 generic_bs_pe_8(void *t, bus_space_handle_t bsh, bus_size_t offset,
    743     uint64_t *datap)
    744 {
    745 	struct faultbuf fb;
    746 	int error;
    747 
    748 	if ((error = cpu_set_onfault(&fb)) == 0) {
    749 		*datap = NSWAP(generic_dsb_bs_r_8)(t, bsh, offset);
    750 		dsb(ld);
    751 		cpu_unset_onfault();
    752 	}
    753 	return error;
    754 }
    755 
    756 int
    757 generic_bs_po_1(void *t, bus_space_handle_t bsh, bus_size_t offset,
    758     uint8_t data)
    759 {
    760 	struct faultbuf fb;
    761 	int error;
    762 
    763 	if ((error = cpu_set_onfault(&fb)) == 0) {
    764 		generic_dsb_bs_w_1(t, bsh, offset, data);
    765 		cpu_unset_onfault();
    766 	}
    767 	return error;
    768 }
    769 
    770 int
    771 generic_bs_po_2(void *t, bus_space_handle_t bsh, bus_size_t offset,
    772     uint16_t data)
    773 {
    774 	struct faultbuf fb;
    775 	int error;
    776 
    777 	if ((error = cpu_set_onfault(&fb)) == 0) {
    778 		NSWAP(generic_dsb_bs_w_2)(t, bsh, offset, data);
    779 		cpu_unset_onfault();
    780 	}
    781 	return error;
    782 }
    783 
    784 int
    785 generic_bs_po_4(void *t, bus_space_handle_t bsh, bus_size_t offset,
    786     uint32_t data)
    787 {
    788 	struct faultbuf fb;
    789 	int error;
    790 
    791 	if ((error = cpu_set_onfault(&fb)) == 0) {
    792 		NSWAP(generic_dsb_bs_w_4)(t, bsh, offset, data);
    793 		cpu_unset_onfault();
    794 	}
    795 	return error;
    796 }
    797 
    798 int
    799 generic_bs_po_8(void *t, bus_space_handle_t bsh, bus_size_t offset,
    800     uint64_t data)
    801 {
    802 	struct faultbuf fb;
    803 	int error;
    804 
    805 	if ((error = cpu_set_onfault(&fb)) == 0) {
    806 		NSWAP(generic_dsb_bs_w_8)(t, bsh, offset, data);
    807 		cpu_unset_onfault();
    808 	}
    809 	return error;
    810 }
    811 #endif /* __BUS_SPACE_HAS_PROBING_METHODS */
    812