Home | History | Annotate | Line # | Download | only in tc
tc_bus_mem.c revision 1.29
      1 /* $NetBSD: tc_bus_mem.c,v 1.29 2009/03/14 15:36:00 dsl Exp $ */
      2 
      3 /*
      4  * Copyright (c) 1996 Carnegie-Mellon University.
      5  * All rights reserved.
      6  *
      7  * Author: Chris G. Demetriou
      8  *
      9  * Permission to use, copy, modify and distribute this software and
     10  * its documentation is hereby granted, provided that both the copyright
     11  * notice and this permission notice appear in all copies of the
     12  * software, derivative works or modified versions, and any portions
     13  * thereof, and that both notices appear in supporting documentation.
     14  *
     15  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
     16  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
     17  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
     18  *
     19  * Carnegie Mellon requests users of this software to return to
     20  *
     21  *  Software Distribution Coordinator  or  Software.Distribution (at) CS.CMU.EDU
     22  *  School of Computer Science
     23  *  Carnegie Mellon University
     24  *  Pittsburgh PA 15213-3890
     25  *
     26  * any improvements or extensions that they make and grant Carnegie the
     27  * rights to redistribute these changes.
     28  */
     29 
     30 /*
     31  * Common TurboChannel Chipset "bus memory" functions.
     32  */
     33 
     34 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
     35 
     36 __KERNEL_RCSID(0, "$NetBSD: tc_bus_mem.c,v 1.29 2009/03/14 15:36:00 dsl Exp $");
     37 
     38 #include <sys/param.h>
     39 #include <sys/systm.h>
     40 #include <sys/malloc.h>
     41 #include <sys/syslog.h>
     42 #include <sys/device.h>
     43 
     44 #include <uvm/uvm_extern.h>
     45 
     46 #include <machine/bus.h>
     47 #include <dev/tc/tcvar.h>
     48 
     49 #define	__C(A,B)	__CONCAT(A,B)
     50 
     51 /* mapping/unmapping */
     52 int		tc_mem_map(void *, bus_addr_t, bus_size_t, int,
     53 		    bus_space_handle_t *, int);
     54 void		tc_mem_unmap(void *, bus_space_handle_t, bus_size_t, int);
     55 int		tc_mem_subregion(void *, bus_space_handle_t, bus_size_t,
     56 		    bus_size_t, bus_space_handle_t *);
     57 
     58 int		tc_mem_translate(void *, bus_addr_t, bus_size_t,
     59 		    int, struct alpha_bus_space_translation *);
     60 int		tc_mem_get_window(void *, int,
     61 		    struct alpha_bus_space_translation *);
     62 
     63 /* allocation/deallocation */
     64 int		tc_mem_alloc(void *, bus_addr_t, bus_addr_t, bus_size_t,
     65 		    bus_size_t, bus_addr_t, int, bus_addr_t *,
     66 		    bus_space_handle_t *);
     67 void		tc_mem_free(void *, bus_space_handle_t, bus_size_t);
     68 
     69 /* get kernel virtual address */
     70 void *		tc_mem_vaddr(void *, bus_space_handle_t);
     71 
     72 /* mmap for user */
     73 paddr_t		tc_mem_mmap(void *, bus_addr_t, off_t, int, int);
     74 
     75 /* barrier */
     76 inline void	tc_mem_barrier(void *, bus_space_handle_t,
     77 		    bus_size_t, bus_size_t, int);
     78 
     79 /* read (single) */
     80 inline u_int8_t	tc_mem_read_1(void *, bus_space_handle_t, bus_size_t);
     81 inline u_int16_t tc_mem_read_2(void *, bus_space_handle_t, bus_size_t);
     82 inline u_int32_t tc_mem_read_4(void *, bus_space_handle_t, bus_size_t);
     83 inline u_int64_t tc_mem_read_8(void *, bus_space_handle_t, bus_size_t);
     84 
     85 /* read multiple */
     86 void		tc_mem_read_multi_1(void *, bus_space_handle_t,
     87 		    bus_size_t, u_int8_t *, bus_size_t);
     88 void		tc_mem_read_multi_2(void *, bus_space_handle_t,
     89 		    bus_size_t, u_int16_t *, bus_size_t);
     90 void		tc_mem_read_multi_4(void *, bus_space_handle_t,
     91 		    bus_size_t, u_int32_t *, bus_size_t);
     92 void		tc_mem_read_multi_8(void *, bus_space_handle_t,
     93 		    bus_size_t, u_int64_t *, bus_size_t);
     94 
     95 /* read region */
     96 void		tc_mem_read_region_1(void *, bus_space_handle_t,
     97 		    bus_size_t, u_int8_t *, bus_size_t);
     98 void		tc_mem_read_region_2(void *, bus_space_handle_t,
     99 		    bus_size_t, u_int16_t *, bus_size_t);
    100 void		tc_mem_read_region_4(void *, bus_space_handle_t,
    101 		    bus_size_t, u_int32_t *, bus_size_t);
    102 void		tc_mem_read_region_8(void *, bus_space_handle_t,
    103 		    bus_size_t, u_int64_t *, bus_size_t);
    104 
    105 /* write (single) */
    106 inline void	tc_mem_write_1(void *, bus_space_handle_t, bus_size_t,
    107 		    u_int8_t);
    108 inline void	tc_mem_write_2(void *, bus_space_handle_t, bus_size_t,
    109 		    u_int16_t);
    110 inline void	tc_mem_write_4(void *, bus_space_handle_t, bus_size_t,
    111 		    u_int32_t);
    112 inline void	tc_mem_write_8(void *, bus_space_handle_t, bus_size_t,
    113 		    u_int64_t);
    114 
    115 /* write multiple */
    116 void		tc_mem_write_multi_1(void *, bus_space_handle_t,
    117 		    bus_size_t, const u_int8_t *, bus_size_t);
    118 void		tc_mem_write_multi_2(void *, bus_space_handle_t,
    119 		    bus_size_t, const u_int16_t *, bus_size_t);
    120 void		tc_mem_write_multi_4(void *, bus_space_handle_t,
    121 		    bus_size_t, const u_int32_t *, bus_size_t);
    122 void		tc_mem_write_multi_8(void *, bus_space_handle_t,
    123 		    bus_size_t, const u_int64_t *, bus_size_t);
    124 
    125 /* write region */
    126 void		tc_mem_write_region_1(void *, bus_space_handle_t,
    127 		    bus_size_t, const u_int8_t *, bus_size_t);
    128 void		tc_mem_write_region_2(void *, bus_space_handle_t,
    129 		    bus_size_t, const u_int16_t *, bus_size_t);
    130 void		tc_mem_write_region_4(void *, bus_space_handle_t,
    131 		    bus_size_t, const u_int32_t *, bus_size_t);
    132 void		tc_mem_write_region_8(void *, bus_space_handle_t,
    133 		    bus_size_t, const u_int64_t *, bus_size_t);
    134 
    135 /* set multiple */
    136 void		tc_mem_set_multi_1(void *, bus_space_handle_t,
    137 		    bus_size_t, u_int8_t, bus_size_t);
    138 void		tc_mem_set_multi_2(void *, bus_space_handle_t,
    139 		    bus_size_t, u_int16_t, bus_size_t);
    140 void		tc_mem_set_multi_4(void *, bus_space_handle_t,
    141 		    bus_size_t, u_int32_t, bus_size_t);
    142 void		tc_mem_set_multi_8(void *, bus_space_handle_t,
    143 		    bus_size_t, u_int64_t, bus_size_t);
    144 
    145 /* set region */
    146 void		tc_mem_set_region_1(void *, bus_space_handle_t,
    147 		    bus_size_t, u_int8_t, bus_size_t);
    148 void		tc_mem_set_region_2(void *, bus_space_handle_t,
    149 		    bus_size_t, u_int16_t, bus_size_t);
    150 void		tc_mem_set_region_4(void *, bus_space_handle_t,
    151 		    bus_size_t, u_int32_t, bus_size_t);
    152 void		tc_mem_set_region_8(void *, bus_space_handle_t,
    153 		    bus_size_t, u_int64_t, bus_size_t);
    154 
    155 /* copy */
    156 void		tc_mem_copy_region_1(void *, bus_space_handle_t,
    157 		    bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t);
    158 void		tc_mem_copy_region_2(void *, bus_space_handle_t,
    159 		    bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t);
    160 void		tc_mem_copy_region_4(void *, bus_space_handle_t,
    161 		    bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t);
    162 void		tc_mem_copy_region_8(void *, bus_space_handle_t,
    163 		    bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t);
    164 
    165 static struct alpha_bus_space tc_mem_space = {
    166 	/* cookie */
    167 	NULL,
    168 
    169 	/* mapping/unmapping */
    170 	tc_mem_map,
    171 	tc_mem_unmap,
    172 	tc_mem_subregion,
    173 
    174 	tc_mem_translate,
    175 	tc_mem_get_window,
    176 
    177 	/* allocation/deallocation */
    178 	tc_mem_alloc,
    179 	tc_mem_free,
    180 
    181 	/* get kernel virtual address */
    182 	tc_mem_vaddr,
    183 
    184 	/* mmap for user */
    185 	tc_mem_mmap,
    186 
    187 	/* barrier */
    188 	tc_mem_barrier,
    189 
    190 	/* read (single) */
    191 	tc_mem_read_1,
    192 	tc_mem_read_2,
    193 	tc_mem_read_4,
    194 	tc_mem_read_8,
    195 
    196 	/* read multiple */
    197 	tc_mem_read_multi_1,
    198 	tc_mem_read_multi_2,
    199 	tc_mem_read_multi_4,
    200 	tc_mem_read_multi_8,
    201 
    202 	/* read region */
    203 	tc_mem_read_region_1,
    204 	tc_mem_read_region_2,
    205 	tc_mem_read_region_4,
    206 	tc_mem_read_region_8,
    207 
    208 	/* write (single) */
    209 	tc_mem_write_1,
    210 	tc_mem_write_2,
    211 	tc_mem_write_4,
    212 	tc_mem_write_8,
    213 
    214 	/* write multiple */
    215 	tc_mem_write_multi_1,
    216 	tc_mem_write_multi_2,
    217 	tc_mem_write_multi_4,
    218 	tc_mem_write_multi_8,
    219 
    220 	/* write region */
    221 	tc_mem_write_region_1,
    222 	tc_mem_write_region_2,
    223 	tc_mem_write_region_4,
    224 	tc_mem_write_region_8,
    225 
    226 	/* set multiple */
    227 	tc_mem_set_multi_1,
    228 	tc_mem_set_multi_2,
    229 	tc_mem_set_multi_4,
    230 	tc_mem_set_multi_8,
    231 
    232 	/* set region */
    233 	tc_mem_set_region_1,
    234 	tc_mem_set_region_2,
    235 	tc_mem_set_region_4,
    236 	tc_mem_set_region_8,
    237 
    238 	/* copy */
    239 	tc_mem_copy_region_1,
    240 	tc_mem_copy_region_2,
    241 	tc_mem_copy_region_4,
    242 	tc_mem_copy_region_8,
    243 };
    244 
    245 bus_space_tag_t
    246 tc_bus_mem_init(void *memv)
    247 {
    248 	bus_space_tag_t h = &tc_mem_space;
    249 
    250 	h->abs_cookie = memv;
    251 	return (h);
    252 }
    253 
    254 /* ARGSUSED */
    255 int
    256 tc_mem_translate(void *v, bus_addr_t memaddr, bus_size_t memlen, int flags, struct alpha_bus_space_translation *abst)
    257 {
    258 
    259 	return (EOPNOTSUPP);
    260 }
    261 
    262 /* ARGSUSED */
    263 int
    264 tc_mem_get_window(void *v, int window, struct alpha_bus_space_translation *abst)
    265 {
    266 
    267 	return (EOPNOTSUPP);
    268 }
    269 
    270 /* ARGSUSED */
    271 int
    272 tc_mem_map(void *v, bus_addr_t memaddr, bus_size_t memsize, int flags, bus_space_handle_t *memhp, int acct)
    273 {
    274 	int cacheable = flags & BUS_SPACE_MAP_CACHEABLE;
    275 	int linear = flags & BUS_SPACE_MAP_LINEAR;
    276 
    277 	/* Requests for linear uncacheable space can't be satisfied. */
    278 	if (linear && !cacheable)
    279 		return (EOPNOTSUPP);
    280 
    281 	if (memaddr & 0x7)
    282 		panic("tc_mem_map needs 8 byte alignment");
    283 	if (cacheable)
    284 		*memhp = ALPHA_PHYS_TO_K0SEG(memaddr);
    285 	else
    286 		*memhp = ALPHA_PHYS_TO_K0SEG(TC_DENSE_TO_SPARSE(memaddr));
    287 	return (0);
    288 }
    289 
    290 /* ARGSUSED */
    291 void
    292 tc_mem_unmap(void *v, bus_space_handle_t memh, bus_size_t memsize, int acct)
    293 {
    294 
    295 	/* XXX XX XXX nothing to do. */
    296 }
    297 
    298 int
    299 tc_mem_subregion(v, memh, offset, size, nmemh)
    300 	void *v;
    301 	bus_space_handle_t memh, *nmemh;
    302 	bus_size_t offset, size;
    303 {
    304 
    305 	/* Disallow subregioning that would make the handle unaligned. */
    306 	if ((offset & 0x7) != 0)
    307 		return (1);
    308 
    309 	if ((memh & TC_SPACE_SPARSE) != 0)
    310 		*nmemh = memh + (offset << 1);
    311 	else
    312 		*nmemh = memh + offset;
    313 
    314 	return (0);
    315 }
    316 
    317 int
    318 tc_mem_alloc(v, rstart, rend, size, align, boundary, flags, addrp, bshp)
    319 	void *v;
    320 	bus_addr_t rstart, rend, *addrp;
    321 	bus_size_t size, align, boundary;
    322 	int flags;
    323 	bus_space_handle_t *bshp;
    324 {
    325 
    326 	/* XXX XXX XXX XXX XXX XXX */
    327 	panic("tc_mem_alloc unimplemented");
    328 }
    329 
    330 void
    331 tc_mem_free(void *v, bus_space_handle_t bsh, bus_size_t size)
    332 {
    333 
    334 	/* XXX XXX XXX XXX XXX XXX */
    335 	panic("tc_mem_free unimplemented");
    336 }
    337 
    338 void *
    339 tc_mem_vaddr(void *v, bus_space_handle_t bsh)
    340 {
    341 #ifdef DIAGNOSTIC
    342 	if ((bsh & TC_SPACE_SPARSE) != 0) {
    343 		/*
    344 		 * tc_mem_map() catches linear && !cacheable,
    345 		 * so we shouldn't come here
    346 		 */
    347 		panic("tc_mem_vaddr");
    348 	}
    349 #endif
    350 	return ((void *)bsh);
    351 }
    352 
    353 paddr_t
    354 tc_mem_mmap(void *v, bus_addr_t addr, off_t off, int prot, int flags)
    355 {
    356 	int linear = flags & BUS_SPACE_MAP_LINEAR;
    357 	bus_addr_t rv;
    358 
    359 	if (linear)
    360 		rv = addr + off;
    361 	else
    362 		rv = TC_DENSE_TO_SPARSE(addr + off);
    363 
    364 	return (alpha_btop(rv));
    365 }
    366 
    367 inline void
    368 tc_mem_barrier(v, h, o, l, f)
    369 	void *v;
    370 	bus_space_handle_t h;
    371 	bus_size_t o, l;
    372 	int f;
    373 {
    374 
    375 	if ((f & BUS_SPACE_BARRIER_READ) != 0)
    376 		alpha_mb();
    377 	else if ((f & BUS_SPACE_BARRIER_WRITE) != 0)
    378 		alpha_wmb();
    379 }
    380 
    381 inline u_int8_t
    382 tc_mem_read_1(void *v, bus_space_handle_t memh, bus_size_t off)
    383 {
    384 	volatile u_int8_t *p;
    385 
    386 	alpha_mb();		/* XXX XXX XXX */
    387 
    388 	if ((memh & TC_SPACE_SPARSE) != 0)
    389 		panic("tc_mem_read_1 not implemented for sparse space");
    390 
    391 	p = (u_int8_t *)(memh + off);
    392 	return (*p);
    393 }
    394 
    395 inline u_int16_t
    396 tc_mem_read_2(void *v, bus_space_handle_t memh, bus_size_t off)
    397 {
    398 	volatile u_int16_t *p;
    399 
    400 	alpha_mb();		/* XXX XXX XXX */
    401 
    402 	if ((memh & TC_SPACE_SPARSE) != 0)
    403 		panic("tc_mem_read_2 not implemented for sparse space");
    404 
    405 	p = (u_int16_t *)(memh + off);
    406 	return (*p);
    407 }
    408 
    409 inline u_int32_t
    410 tc_mem_read_4(void *v, bus_space_handle_t memh, bus_size_t off)
    411 {
    412 	volatile u_int32_t *p;
    413 
    414 	alpha_mb();		/* XXX XXX XXX */
    415 
    416 	if ((memh & TC_SPACE_SPARSE) != 0)
    417 		/* Nothing special to do for 4-byte sparse space accesses */
    418 		p = (u_int32_t *)(memh + (off << 1));
    419 	else
    420 		p = (u_int32_t *)(memh + off);
    421 	return (*p);
    422 }
    423 
    424 inline u_int64_t
    425 tc_mem_read_8(void *v, bus_space_handle_t memh, bus_size_t off)
    426 {
    427 	volatile u_int64_t *p;
    428 
    429 	alpha_mb();		/* XXX XXX XXX */
    430 
    431 	if ((memh & TC_SPACE_SPARSE) != 0)
    432 		panic("tc_mem_read_8 not implemented for sparse space");
    433 
    434 	p = (u_int64_t *)(memh + off);
    435 	return (*p);
    436 }
    437 
    438 #define	tc_mem_read_multi_N(BYTES,TYPE)					\
    439 void									\
    440 __C(tc_mem_read_multi_,BYTES)(v, h, o, a, c)				\
    441 	void *v;							\
    442 	bus_space_handle_t h;						\
    443 	bus_size_t o, c;						\
    444 	TYPE *a;							\
    445 {									\
    446 									\
    447 	while (c-- > 0) {						\
    448 		tc_mem_barrier(v, h, o, sizeof *a, BUS_SPACE_BARRIER_READ); \
    449 		*a++ = __C(tc_mem_read_,BYTES)(v, h, o);		\
    450 	}								\
    451 }
    452 tc_mem_read_multi_N(1,u_int8_t)
    453 tc_mem_read_multi_N(2,u_int16_t)
    454 tc_mem_read_multi_N(4,u_int32_t)
    455 tc_mem_read_multi_N(8,u_int64_t)
    456 
    457 #define	tc_mem_read_region_N(BYTES,TYPE)				\
    458 void									\
    459 __C(tc_mem_read_region_,BYTES)(v, h, o, a, c)				\
    460 	void *v;							\
    461 	bus_space_handle_t h;						\
    462 	bus_size_t o, c;						\
    463 	TYPE *a;							\
    464 {									\
    465 									\
    466 	while (c-- > 0) {						\
    467 		*a++ = __C(tc_mem_read_,BYTES)(v, h, o);		\
    468 		o += sizeof *a;						\
    469 	}								\
    470 }
    471 tc_mem_read_region_N(1,u_int8_t)
    472 tc_mem_read_region_N(2,u_int16_t)
    473 tc_mem_read_region_N(4,u_int32_t)
    474 tc_mem_read_region_N(8,u_int64_t)
    475 
    476 inline void
    477 tc_mem_write_1(void *v, bus_space_handle_t memh, bus_size_t off, u_int8_t val)
    478 {
    479 
    480 	if ((memh & TC_SPACE_SPARSE) != 0) {
    481 		volatile u_int64_t *p, vl;
    482 		u_int64_t shift, msk;
    483 
    484 		shift = off & 0x3;
    485 		off &= 0x3;
    486 
    487 		p = (u_int64_t *)(memh + (off << 1));
    488 
    489 		msk = ~(0x1 << shift) & 0xf;
    490 		vl = (msk << 32) | (((u_int64_t)val) << (shift * 8));
    491 
    492 		*p = val;
    493 	} else {
    494 		volatile u_int8_t *p;
    495 
    496 		p = (u_int8_t *)(memh + off);
    497 		*p = val;
    498 	}
    499         alpha_mb();		/* XXX XXX XXX */
    500 }
    501 
    502 inline void
    503 tc_mem_write_2(void *v, bus_space_handle_t memh, bus_size_t off, u_int16_t val)
    504 {
    505 
    506 	if ((memh & TC_SPACE_SPARSE) != 0) {
    507 		volatile u_int64_t *p, vl;
    508 		u_int64_t shift, msk;
    509 
    510 		shift = off & 0x2;
    511 		off &= 0x3;
    512 
    513 		p = (u_int64_t *)(memh + (off << 1));
    514 
    515 		msk = ~(0x3 << shift) & 0xf;
    516 		vl = (msk << 32) | (((u_int64_t)val) << (shift * 8));
    517 
    518 		*p = val;
    519 	} else {
    520 		volatile u_int16_t *p;
    521 
    522 		p = (u_int16_t *)(memh + off);
    523 		*p = val;
    524 	}
    525         alpha_mb();		/* XXX XXX XXX */
    526 }
    527 
    528 inline void
    529 tc_mem_write_4(void *v, bus_space_handle_t memh, bus_size_t off, u_int32_t val)
    530 {
    531 	volatile u_int32_t *p;
    532 
    533 	if ((memh & TC_SPACE_SPARSE) != 0)
    534 		/* Nothing special to do for 4-byte sparse space accesses */
    535 		p = (u_int32_t *)(memh + (off << 1));
    536 	else
    537 		p = (u_int32_t *)(memh + off);
    538 	*p = val;
    539         alpha_mb();		/* XXX XXX XXX */
    540 }
    541 
    542 inline void
    543 tc_mem_write_8(void *v, bus_space_handle_t memh, bus_size_t off, u_int64_t val)
    544 {
    545 	volatile u_int64_t *p;
    546 
    547 	if ((memh & TC_SPACE_SPARSE) != 0)
    548 		panic("tc_mem_read_8 not implemented for sparse space");
    549 
    550 	p = (u_int64_t *)(memh + off);
    551 	*p = val;
    552         alpha_mb();		/* XXX XXX XXX */
    553 }
    554 
    555 #define	tc_mem_write_multi_N(BYTES,TYPE)				\
    556 void									\
    557 __C(tc_mem_write_multi_,BYTES)(v, h, o, a, c)				\
    558 	void *v;							\
    559 	bus_space_handle_t h;						\
    560 	bus_size_t o, c;						\
    561 	const TYPE *a;							\
    562 {									\
    563 									\
    564 	while (c-- > 0) {						\
    565 		__C(tc_mem_write_,BYTES)(v, h, o, *a++);		\
    566 		tc_mem_barrier(v, h, o, sizeof *a, BUS_SPACE_BARRIER_WRITE); \
    567 	}								\
    568 }
    569 tc_mem_write_multi_N(1,u_int8_t)
    570 tc_mem_write_multi_N(2,u_int16_t)
    571 tc_mem_write_multi_N(4,u_int32_t)
    572 tc_mem_write_multi_N(8,u_int64_t)
    573 
    574 #define	tc_mem_write_region_N(BYTES,TYPE)				\
    575 void									\
    576 __C(tc_mem_write_region_,BYTES)(v, h, o, a, c)				\
    577 	void *v;							\
    578 	bus_space_handle_t h;						\
    579 	bus_size_t o, c;						\
    580 	const TYPE *a;							\
    581 {									\
    582 									\
    583 	while (c-- > 0) {						\
    584 		__C(tc_mem_write_,BYTES)(v, h, o, *a++);		\
    585 		o += sizeof *a;						\
    586 	}								\
    587 }
    588 tc_mem_write_region_N(1,u_int8_t)
    589 tc_mem_write_region_N(2,u_int16_t)
    590 tc_mem_write_region_N(4,u_int32_t)
    591 tc_mem_write_region_N(8,u_int64_t)
    592 
    593 #define	tc_mem_set_multi_N(BYTES,TYPE)					\
    594 void									\
    595 __C(tc_mem_set_multi_,BYTES)(v, h, o, val, c)				\
    596 	void *v;							\
    597 	bus_space_handle_t h;						\
    598 	bus_size_t o, c;						\
    599 	TYPE val;							\
    600 {									\
    601 									\
    602 	while (c-- > 0) {						\
    603 		__C(tc_mem_write_,BYTES)(v, h, o, val);			\
    604 		tc_mem_barrier(v, h, o, sizeof val, BUS_SPACE_BARRIER_WRITE); \
    605 	}								\
    606 }
    607 tc_mem_set_multi_N(1,u_int8_t)
    608 tc_mem_set_multi_N(2,u_int16_t)
    609 tc_mem_set_multi_N(4,u_int32_t)
    610 tc_mem_set_multi_N(8,u_int64_t)
    611 
    612 #define	tc_mem_set_region_N(BYTES,TYPE)					\
    613 void									\
    614 __C(tc_mem_set_region_,BYTES)(v, h, o, val, c)				\
    615 	void *v;							\
    616 	bus_space_handle_t h;						\
    617 	bus_size_t o, c;						\
    618 	TYPE val;							\
    619 {									\
    620 									\
    621 	while (c-- > 0) {						\
    622 		__C(tc_mem_write_,BYTES)(v, h, o, val);			\
    623 		o += sizeof val;					\
    624 	}								\
    625 }
    626 tc_mem_set_region_N(1,u_int8_t)
    627 tc_mem_set_region_N(2,u_int16_t)
    628 tc_mem_set_region_N(4,u_int32_t)
    629 tc_mem_set_region_N(8,u_int64_t)
    630 
    631 #define	tc_mem_copy_region_N(BYTES)					\
    632 void									\
    633 __C(tc_mem_copy_region_,BYTES)(v, h1, o1, h2, o2, c)			\
    634 	void *v;							\
    635 	bus_space_handle_t h1, h2;					\
    636 	bus_size_t o1, o2, c;						\
    637 {									\
    638 	bus_size_t o;							\
    639 									\
    640 	if ((h1 & TC_SPACE_SPARSE) != 0 &&				\
    641 	    (h2 & TC_SPACE_SPARSE) != 0) {				\
    642 		memmove((void *)(h2 + o2), (void *)(h1 + o1), c * BYTES); \
    643 		return;							\
    644 	}								\
    645 									\
    646 	if (h1 + o1 >= h2 + o2)						\
    647 		/* src after dest: copy forward */			\
    648 		for (o = 0; c > 0; c--, o += BYTES)			\
    649 			__C(tc_mem_write_,BYTES)(v, h2, o2 + o,		\
    650 			    __C(tc_mem_read_,BYTES)(v, h1, o1 + o));	\
    651 	else								\
    652 		/* dest after src: copy backwards */			\
    653 		for (o = (c - 1) * BYTES; c > 0; c--, o -= BYTES)	\
    654 			__C(tc_mem_write_,BYTES)(v, h2, o2 + o,		\
    655 			    __C(tc_mem_read_,BYTES)(v, h1, o1 + o));	\
    656 }
    657 tc_mem_copy_region_N(1)
    658 tc_mem_copy_region_N(2)
    659 tc_mem_copy_region_N(4)
    660 tc_mem_copy_region_N(8)
    661