1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of copyright holder nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef ENA_PLAT_H_ 35 #define ENA_PLAT_H_ 36 37 #include <sys/cdefs.h> 38 #if 0 39 __FBSDID("$FreeBSD: head/sys/contrib/ena-com/ena_plat.h 333453 2018-05-10 09:25:51Z mw $"); 40 #endif 41 __KERNEL_RCSID(0, "$NetBSD: ena_plat.h,v 1.10 2023/11/05 18:15:02 jdolecek Exp $"); 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 46 #include <sys/bus.h> 47 #include <sys/condvar.h> 48 #include <sys/endian.h> 49 #include <sys/kernel.h> 50 #include <sys/kmem.h> 51 #include <sys/kthread.h> 52 #include <sys/mbuf.h> 53 #include <sys/module.h> 54 #include <sys/proc.h> 55 #include <sys/socket.h> 56 #include <sys/sockio.h> 57 #include <sys/sysctl.h> 58 #include <sys/types.h> 59 #include <sys/bus.h> 60 #include <sys/atomic.h> 61 62 #include <net/if.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/if_ether.h> 66 67 #include <net/bpf.h> 68 69 #include <net/rss_config.h> 70 71 #include <netinet/in.h> /* XXX for struct ip */ 72 #include <netinet/in_systm.h> /* XXX for struct ip */ 73 #include <netinet/ip.h> /* XXX for struct ip */ 74 #include <netinet/ip6.h> /* XXX for struct ip6_hdr */ 75 #include <netinet/tcp.h> /* XXX for struct tcphdr */ 76 77 #include <dev/pci/pcivar.h> 78 #include <dev/pci/pcireg.h> 79 80 extern struct ena_bus_space ebs; 81 82 /* Levels */ 83 #define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */ 84 #define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */ 85 #define ENA_INFO (1 << 2) /* Provides additional driver info. */ 86 #define ENA_DBG (1 << 3) /* Driver output for debugging. */ 87 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */ 88 #define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */ 89 #define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */ 90 #define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */ 91 #define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */ 92 #define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */ 93 94 extern int ena_log_level; 95 96 #define ena_trace_raw(level, fmt, args...) \ 97 do { \ 98 if (((level) & ena_log_level) != (level)) \ 99 break; \ 100 printf(fmt, ##args); \ 101 } while (0) 102 103 #define ena_trace(level, fmt, args...) \ 104 ena_trace_raw(level, "%s() [LID:%d]: " \ 105 fmt " \n", __func__, curlwp->l_lid, ##args) 106 107 108 #define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg) 109 #define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg) 110 #define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg) 111 #define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg) 112 113 #define unlikely(x) __predict_false(x) 114 #define likely(x) __predict_true(x) 115 116 #define __iomem 117 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 118 119 #define MAX_ERRNO 4095 120 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 121 122 #define ENA_ASSERT(cond, format, arg...) \ 123 do { \ 124 if (unlikely(!(cond))) { \ 125 ena_trc_err( \ 126 "Assert failed on %s:%s:%d:" format, \ 127 __FILE__, __func__, __LINE__, ##arg); \ 128 } \ 129 } while (0) 130 131 #define ENA_WARN(cond, format, arg...) \ 132 do { \ 133 if (unlikely((cond))) { \ 134 ena_trc_warn(format, ##arg); \ 135 } \ 136 } while (0) 137 138 static inline long IS_ERR(const void *ptr) 139 { 140 return IS_ERR_VALUE((unsigned long)ptr); 141 } 142 143 static inline void *ERR_PTR(long error) 144 { 145 return (void *)error; 146 } 147 148 static inline long PTR_ERR(const void *ptr) 149 { 150 return (long) ptr; 151 } 152 153 #define GENMASK(h, l) (((1U << ((h) - (l) + 1)) - 1) << (l)) 154 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 155 #define BIT(x) (1UL << (x)) 156 157 #define ENA_ABORT() BUG() 158 #define BUG() panic("ENA BUG") 159 160 #define SZ_256 (256) 161 #define SZ_4K (4096) 162 163 #define ENA_COM_OK 0 164 #define ENA_COM_FAULT EFAULT 165 #define ENA_COM_INVAL EINVAL 166 #define ENA_COM_NO_MEM ENOMEM 167 #define ENA_COM_NO_SPACE ENOSPC 168 #define ENA_COM_TRY_AGAIN -1 169 #define ENA_COM_UNSUPPORTED EOPNOTSUPP 170 #define ENA_COM_NO_DEVICE ENODEV 171 #define ENA_COM_PERMISSION EPERM 172 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT 173 174 static inline int 175 ENA_MSLEEP(int x) 176 { 177 if (cold) { 178 while (x >= 1000000) { 179 delay(1000000); 180 x -= 1000000; 181 } 182 if (x > 0) 183 delay(x); 184 return EWOULDBLOCK; 185 } else { 186 return kpause("enaw", false, mstohz(x), NULL); 187 } 188 } 189 190 #define ENA_UDELAY(x) DELAY(x) 191 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 192 mstohz(timeout_us * (1000 / 100)) /* XXX assumes 100 ms sleep */ 193 #define ENA_TIME_EXPIRE(timeout) ((timeout)-- <= 0) 194 #define ENA_MIGHT_SLEEP() 195 196 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 197 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 198 199 #define ENA_MIN32(x,y) MIN(x, y) 200 #define ENA_MIN16(x,y) MIN(x, y) 201 #define ENA_MIN8(x,y) MIN(x, y) 202 203 #define ENA_MAX32(x,y) MAX(x, y) 204 #define ENA_MAX16(x,y) MAX(x, y) 205 #define ENA_MAX8(x,y) MAX(x, y) 206 207 /* Spinlock related methods */ 208 #define ena_spinlock_t kmutex_t 209 #define ENA_SPINLOCK_INIT(spinlock) \ 210 mutex_init(&(spinlock), MUTEX_DEFAULT, IPL_NET) 211 #define ENA_SPINLOCK_DESTROY(spinlock) \ 212 do { \ 213 mutex_destroy(&(spinlock)); \ 214 } while (0) 215 #define ENA_SPINLOCK_LOCK(spinlock, flags) \ 216 do { \ 217 (void)(flags); \ 218 mutex_enter(&(spinlock)); \ 219 } while (0) 220 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 221 do { \ 222 (void)(flags); \ 223 mutex_exit(&(spinlock)); \ 224 } while (0) 225 226 227 /* Wait queue related methods */ 228 #define ena_wait_event_t struct { kcondvar_t wq; kmutex_t mtx; } 229 #define ENA_WAIT_EVENT_INIT(waitqueue) \ 230 do { \ 231 cv_init(&((waitqueue).wq), "enacv"); \ 232 mutex_init(&((waitqueue).mtx), MUTEX_DEFAULT, IPL_NET); \ 233 } while (0) 234 #define ENA_WAIT_EVENT_DESTROY(waitqueue) \ 235 do { \ 236 cv_destroy(&((waitqueue).wq)); \ 237 mutex_destroy(&((waitqueue).mtx)); \ 238 } while (0) 239 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 240 cv_init(&((waitqueue).wq), "enacv") 241 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 242 do { \ 243 mutex_enter(&((waitqueue).mtx)); \ 244 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 245 timeout_us * hz / 1000 / 1000 ); \ 246 mutex_exit(&((waitqueue).mtx)); \ 247 } while (0) 248 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 249 do { \ 250 mutex_enter(&((waitqueue).mtx)); \ 251 cv_broadcast(&((waitqueue).wq)); \ 252 mutex_exit(&((waitqueue).mtx)); \ 253 } while (0) 254 255 #define dma_addr_t bus_addr_t 256 #define u8 uint8_t 257 #define u16 uint16_t 258 #define u32 uint32_t 259 #define u64 uint64_t 260 261 typedef struct { 262 paddr_t paddr; 263 void *vaddr; 264 bus_dma_tag_t tag; 265 bus_dmamap_t map; 266 bus_dma_segment_t seg; 267 int nseg; 268 } ena_mem_handle_t; 269 270 struct ena_bus { 271 bus_space_handle_t reg_bar_h; 272 bus_space_tag_t reg_bar_t; 273 bus_space_handle_t mem_bar_h; 274 bus_space_tag_t mem_bar_t; 275 }; 276 277 typedef uint32_t ena_atomic32_t; 278 279 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 280 int error); 281 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 282 int mapflags); 283 284 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 285 do { \ 286 int count, i; \ 287 volatile uint64_t *to = (volatile uint64_t *)(dst); \ 288 const uint64_t *from = (const uint64_t *)(src); \ 289 count = (size) / 8; \ 290 \ 291 for (i = 0; i < count; i++, from++, to++) \ 292 *to = *from; \ 293 } while (0) 294 295 #define ENA_MEM_ALLOC(dmadev, size) kmem_zalloc(size, KM_NOSLEEP) 296 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL) 297 #define ENA_MEM_FREE(dmadev, ptr, size) kmem_free(ptr, size) 298 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \ 299 dev_node) \ 300 do { \ 301 ((virt) = NULL); \ 302 (void)(dev_node); \ 303 } while (0) 304 305 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 306 do { \ 307 ena_dma_alloc((dmadev), (size), &(dma), 0); \ 308 (virt) = (void *)(dma).vaddr; \ 309 (phys) = (dma).paddr; \ 310 } while (0) 311 312 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 313 do { \ 314 (void)size; \ 315 size_t mapsize = (dma).map->dm_mapsize; \ 316 bus_dmamap_unload((dma).tag, (dma).map); \ 317 bus_dmamem_unmap((dma).tag, (dma).vaddr, mapsize); \ 318 bus_dmamem_free((dma).tag, &(dma).seg, (dma).nseg); \ 319 bus_dmamap_destroy((dma).tag, (dma).map); \ 320 (dma).tag = NULL; \ 321 (virt) = NULL; \ 322 } while (0) 323 324 /* Register R/W methods */ 325 #define ENA_REG_WRITE32(bus, value, offset) \ 326 bus_space_write_4( \ 327 ((struct ena_bus*)bus)->reg_bar_t, \ 328 ((struct ena_bus*)bus)->reg_bar_h, \ 329 (bus_size_t)(offset), (value)) 330 331 #define ENA_REG_READ32(bus, offset) \ 332 bus_space_read_4( \ 333 ((struct ena_bus*)bus)->reg_bar_t, \ 334 ((struct ena_bus*)bus)->reg_bar_h, \ 335 (bus_size_t)(offset)) 336 337 #define ENA_DB_SYNC(mem_handle) bus_dmamap_sync((mem_handle)->tag, \ 338 (mem_handle)->map, 0, (mem_handle)->map->dm_mapsize, \ 339 BUS_DMASYNC_PREREAD) 340 341 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 342 343 #define VLAN_HLEN sizeof(struct ether_vlan_header) 344 #define CSUM_OFFLOAD (M_CSUM_IPv4|M_CSUM_TCPv4|M_CSUM_UDPv4) 345 346 #if defined(__i386__) || defined(__amd64__) 347 static __inline 348 void prefetch(void *x) 349 { 350 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); 351 } 352 #else 353 #define prefetch(x) 354 #endif 355 356 /* DMA buffers access */ 357 #define dma_unmap_addr(p, name) ((p)->dma->name) 358 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 359 #define dma_unmap_len(p, name) ((p)->name) 360 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 361 362 #define memcpy_toio memcpy 363 364 #define ATOMIC32_INC(I32_PTR) atomic_inc_32(I32_PTR) 365 #define ATOMIC32_DEC(I32_PTR) atomic_dec_32(I32_PTR) 366 #define ATOMIC32_READ(I32_PTR) atomic_cas_32(I32_PTR, 0, 0) 367 #define ATOMIC32_SET(I32_PTR, VAL) atomic_swap_32(I32_PTR, VAL) 368 369 #define barrier() __asm__ __volatile__("": : :"memory") 370 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 371 #define READ_ONCE(x) ({ \ 372 __typeof(x) __var; \ 373 barrier(); \ 374 __var = ACCESS_ONCE(x); \ 375 barrier(); \ 376 __var; \ 377 }) 378 379 #include "ena_defs/ena_includes.h" 380 381 /* 382 * XXX This is not really right. Need to adjust the driver to use 383 * bus_space_barrier or bus_dmamap_sync. 384 */ 385 #if defined(__i386__) || defined(__x86_64__) 386 #include <x86/cpufunc.h> 387 #define rmb() x86_lfence() 388 #define wmb() x86_sfence() 389 #define mb() x86_mfence() 390 #elif defined(__aarch64__) 391 #define rmb() __asm __volatile("dsb ld" ::: "memory") 392 #define wmb() __asm __volatile("dsb st" ::: "memory") 393 #define mb() __asm __volatile("dsb sy" ::: "memory") 394 #else 395 #define rmb() membar_acquire() 396 #define wmb() membar_release() 397 #define mb() membar_sync() 398 #endif 399 400 #endif /* ENA_PLAT_H_ */ 401
Indexes created Sat Sep 20 22:09:52 GMT 2025