cxgb_sge.c revision 1.1.2.2
1 1.1.2.2 uebayasi /************************************************************************** 2 1.1.2.2 uebayasi 3 1.1.2.2 uebayasi Copyright (c) 2007, Chelsio Inc. 4 1.1.2.2 uebayasi All rights reserved. 5 1.1.2.2 uebayasi 6 1.1.2.2 uebayasi Redistribution and use in source and binary forms, with or without 7 1.1.2.2 uebayasi modification, are permitted provided that the following conditions are met: 8 1.1.2.2 uebayasi 9 1.1.2.2 uebayasi 1. Redistributions of source code must retain the above copyright notice, 10 1.1.2.2 uebayasi this list of conditions and the following disclaimer. 11 1.1.2.2 uebayasi 12 1.1.2.2 uebayasi 2. Neither the name of the Chelsio Corporation nor the names of its 13 1.1.2.2 uebayasi contributors may be used to endorse or promote products derived from 14 1.1.2.2 uebayasi this software without specific prior written permission. 15 1.1.2.2 uebayasi 16 1.1.2.2 uebayasi THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 1.1.2.2 uebayasi AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 1.1.2.2 uebayasi IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 1.1.2.2 uebayasi ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 1.1.2.2 uebayasi LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 1.1.2.2 uebayasi CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 1.1.2.2 uebayasi SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 1.1.2.2 uebayasi INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 1.1.2.2 uebayasi CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 1.1.2.2 uebayasi ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 1.1.2.2 uebayasi POSSIBILITY OF SUCH DAMAGE. 27 1.1.2.2 uebayasi 28 1.1.2.2 uebayasi ***************************************************************************/ 29 1.1.2.2 uebayasi 30 1.1.2.2 uebayasi #include <sys/cdefs.h> 31 1.1.2.2 uebayasi __KERNEL_RCSID(0, "$NetBSD: cxgb_sge.c,v 1.1.2.2 2010/04/30 14:43:45 uebayasi Exp $"); 32 1.1.2.2 uebayasi 33 1.1.2.2 uebayasi #include <sys/param.h> 34 1.1.2.2 uebayasi #include <sys/systm.h> 35 1.1.2.2 uebayasi #include <sys/kernel.h> 36 1.1.2.2 uebayasi #include <sys/conf.h> 37 1.1.2.2 uebayasi #include <machine/bus.h> 38 1.1.2.2 uebayasi #include <sys/queue.h> 39 1.1.2.2 uebayasi #include <sys/sysctl.h> 40 1.1.2.2 uebayasi 41 1.1.2.2 uebayasi #include <sys/proc.h> 42 1.1.2.2 uebayasi #include <sys/sched.h> 43 1.1.2.2 uebayasi #include <sys/systm.h> 44 1.1.2.2 uebayasi 45 1.1.2.2 uebayasi #include <netinet/in_systm.h> 46 1.1.2.2 uebayasi #include <netinet/in.h> 47 1.1.2.2 uebayasi #include <netinet/ip.h> 48 1.1.2.2 uebayasi #include <netinet/tcp.h> 49 1.1.2.2 uebayasi 50 1.1.2.2 uebayasi #include <dev/pci/pcireg.h> 51 1.1.2.2 uebayasi #include <dev/pci/pcivar.h> 52 1.1.2.2 uebayasi 53 1.1.2.2 uebayasi #ifdef CONFIG_DEFINED 54 1.1.2.2 uebayasi #include <cxgb_include.h> 55 1.1.2.2 uebayasi #else 56 1.1.2.2 uebayasi #include <dev/pci/cxgb/cxgb_include.h> 57 1.1.2.2 uebayasi #endif 58 1.1.2.2 uebayasi 59 1.1.2.2 uebayasi uint32_t collapse_free = 0; 60 1.1.2.2 uebayasi uint32_t mb_free_vec_free = 0; 61 1.1.2.2 uebayasi int txq_fills = 0; 62 1.1.2.2 uebayasi int collapse_mbufs = 0; 63 1.1.2.2 uebayasi static int bogus_imm = 0; 64 1.1.2.2 uebayasi #ifndef DISABLE_MBUF_IOVEC 65 1.1.2.2 uebayasi static int recycle_enable = 1; 66 1.1.2.2 uebayasi #endif 67 1.1.2.2 uebayasi 68 1.1.2.2 uebayasi #define USE_GTS 0 69 1.1.2.2 uebayasi 70 1.1.2.2 uebayasi #define SGE_RX_SM_BUF_SIZE 1536 71 1.1.2.2 uebayasi #define SGE_RX_DROP_THRES 16 72 1.1.2.2 uebayasi #define SGE_RX_COPY_THRES 128 73 1.1.2.2 uebayasi 74 1.1.2.2 uebayasi /* 75 1.1.2.2 uebayasi * Period of the Tx buffer reclaim timer. This timer does not need to run 76 1.1.2.2 uebayasi * frequently as Tx buffers are usually reclaimed by new Tx packets. 77 1.1.2.2 uebayasi */ 78 1.1.2.2 uebayasi #define TX_RECLAIM_PERIOD (hz >> 1) 79 1.1.2.2 uebayasi 80 1.1.2.2 uebayasi /* 81 1.1.2.2 uebayasi * work request size in bytes 82 1.1.2.2 uebayasi */ 83 1.1.2.2 uebayasi #define WR_LEN (WR_FLITS * 8) 84 1.1.2.2 uebayasi 85 1.1.2.2 uebayasi /* 86 1.1.2.2 uebayasi * Values for sge_txq.flags 87 1.1.2.2 uebayasi */ 88 1.1.2.2 uebayasi enum { 89 1.1.2.2 uebayasi TXQ_RUNNING = 1 << 0, /* fetch engine is running */ 90 1.1.2.2 uebayasi TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */ 91 1.1.2.2 uebayasi }; 92 1.1.2.2 uebayasi 93 1.1.2.2 uebayasi struct tx_desc { 94 1.1.2.2 uebayasi uint64_t flit[TX_DESC_FLITS]; 95 1.1.2.2 uebayasi } __packed; 96 1.1.2.2 uebayasi 97 1.1.2.2 uebayasi struct rx_desc { 98 1.1.2.2 uebayasi uint32_t addr_lo; 99 1.1.2.2 uebayasi uint32_t len_gen; 100 1.1.2.2 uebayasi uint32_t gen2; 101 1.1.2.2 uebayasi uint32_t addr_hi; 102 1.1.2.2 uebayasi } __packed; 103 1.1.2.2 uebayasi 104 1.1.2.2 uebayasi struct rsp_desc { /* response queue descriptor */ 105 1.1.2.2 uebayasi struct rss_header rss_hdr; 106 1.1.2.2 uebayasi uint32_t flags; 107 1.1.2.2 uebayasi uint32_t len_cq; 108 1.1.2.2 uebayasi uint8_t imm_data[47]; 109 1.1.2.2 uebayasi uint8_t intr_gen; 110 1.1.2.2 uebayasi } __packed; 111 1.1.2.2 uebayasi 112 1.1.2.2 uebayasi #define RX_SW_DESC_MAP_CREATED (1 << 0) 113 1.1.2.2 uebayasi #define TX_SW_DESC_MAP_CREATED (1 << 1) 114 1.1.2.2 uebayasi #define RX_SW_DESC_INUSE (1 << 3) 115 1.1.2.2 uebayasi #define TX_SW_DESC_MAPPED (1 << 4) 116 1.1.2.2 uebayasi 117 1.1.2.2 uebayasi #define RSPQ_NSOP_NEOP G_RSPD_SOP_EOP(0) 118 1.1.2.2 uebayasi #define RSPQ_EOP G_RSPD_SOP_EOP(F_RSPD_EOP) 119 1.1.2.2 uebayasi #define RSPQ_SOP G_RSPD_SOP_EOP(F_RSPD_SOP) 120 1.1.2.2 uebayasi #define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) 121 1.1.2.2 uebayasi 122 1.1.2.2 uebayasi struct tx_sw_desc { /* SW state per Tx descriptor */ 123 1.1.2.2 uebayasi struct mbuf *m; 124 1.1.2.2 uebayasi bus_dma_segment_t segs[1]; 125 1.1.2.2 uebayasi bus_dmamap_t map; 126 1.1.2.2 uebayasi int flags; 127 1.1.2.2 uebayasi }; 128 1.1.2.2 uebayasi 129 1.1.2.2 uebayasi struct rx_sw_desc { /* SW state per Rx descriptor */ 130 1.1.2.2 uebayasi void *cl; 131 1.1.2.2 uebayasi bus_dmamap_t map; 132 1.1.2.2 uebayasi int flags; 133 1.1.2.2 uebayasi }; 134 1.1.2.2 uebayasi 135 1.1.2.2 uebayasi struct txq_state { 136 1.1.2.2 uebayasi unsigned int compl; 137 1.1.2.2 uebayasi unsigned int gen; 138 1.1.2.2 uebayasi unsigned int pidx; 139 1.1.2.2 uebayasi }; 140 1.1.2.2 uebayasi 141 1.1.2.2 uebayasi /* 142 1.1.2.2 uebayasi * Maps a number of flits to the number of Tx descriptors that can hold them. 143 1.1.2.2 uebayasi * The formula is 144 1.1.2.2 uebayasi * 145 1.1.2.2 uebayasi * desc = 1 + (flits - 2) / (WR_FLITS - 1). 146 1.1.2.2 uebayasi * 147 1.1.2.2 uebayasi * HW allows up to 4 descriptors to be combined into a WR. 148 1.1.2.2 uebayasi */ 149 1.1.2.2 uebayasi static uint8_t flit_desc_map[] = { 150 1.1.2.2 uebayasi 0, 151 1.1.2.2 uebayasi #if SGE_NUM_GENBITS == 1 152 1.1.2.2 uebayasi 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 153 1.1.2.2 uebayasi 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 154 1.1.2.2 uebayasi 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 155 1.1.2.2 uebayasi 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 156 1.1.2.2 uebayasi #elif SGE_NUM_GENBITS == 2 157 1.1.2.2 uebayasi 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 158 1.1.2.2 uebayasi 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 159 1.1.2.2 uebayasi 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 160 1.1.2.2 uebayasi 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 161 1.1.2.2 uebayasi #else 162 1.1.2.2 uebayasi # error "SGE_NUM_GENBITS must be 1 or 2" 163 1.1.2.2 uebayasi #endif 164 1.1.2.2 uebayasi }; 165 1.1.2.2 uebayasi 166 1.1.2.2 uebayasi 167 1.1.2.2 uebayasi static int lro_default = 0; 168 1.1.2.2 uebayasi int cxgb_debug = 0; 169 1.1.2.2 uebayasi 170 1.1.2.2 uebayasi static void t3_free_qset(adapter_t *sc, struct sge_qset *q); 171 1.1.2.2 uebayasi static void sge_timer_cb(void *arg); 172 1.1.2.2 uebayasi static void sge_timer_reclaim(struct work *wk, void *arg); 173 1.1.2.2 uebayasi static void sge_txq_reclaim_handler(struct work *wk, void *arg); 174 1.1.2.2 uebayasi static int free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec); 175 1.1.2.2 uebayasi 176 1.1.2.2 uebayasi /** 177 1.1.2.2 uebayasi * reclaim_completed_tx - reclaims completed Tx descriptors 178 1.1.2.2 uebayasi * @adapter: the adapter 179 1.1.2.2 uebayasi * @q: the Tx queue to reclaim completed descriptors from 180 1.1.2.2 uebayasi * 181 1.1.2.2 uebayasi * Reclaims Tx descriptors that the SGE has indicated it has processed, 182 1.1.2.2 uebayasi * and frees the associated buffers if possible. Called with the Tx 183 1.1.2.2 uebayasi * queue's lock held. 184 1.1.2.2 uebayasi */ 185 1.1.2.2 uebayasi static __inline int 186 1.1.2.2 uebayasi reclaim_completed_tx(struct sge_txq *q, int nbufs, struct mbuf **mvec) 187 1.1.2.2 uebayasi { 188 1.1.2.2 uebayasi int reclaimed, reclaim = desc_reclaimable(q); 189 1.1.2.2 uebayasi int n = 0; 190 1.1.2.2 uebayasi 191 1.1.2.2 uebayasi mtx_assert(&q->lock, MA_OWNED); 192 1.1.2.2 uebayasi if (reclaim > 0) { 193 1.1.2.2 uebayasi n = free_tx_desc(q, min(reclaim, nbufs), mvec); 194 1.1.2.2 uebayasi reclaimed = min(reclaim, nbufs); 195 1.1.2.2 uebayasi q->cleaned += reclaimed; 196 1.1.2.2 uebayasi q->in_use -= reclaimed; 197 1.1.2.2 uebayasi } 198 1.1.2.2 uebayasi return (n); 199 1.1.2.2 uebayasi } 200 1.1.2.2 uebayasi 201 1.1.2.2 uebayasi /** 202 1.1.2.2 uebayasi * should_restart_tx - are there enough resources to restart a Tx queue? 203 1.1.2.2 uebayasi * @q: the Tx queue 204 1.1.2.2 uebayasi * 205 1.1.2.2 uebayasi * Checks if there are enough descriptors to restart a suspended Tx queue. 206 1.1.2.2 uebayasi */ 207 1.1.2.2 uebayasi static __inline int 208 1.1.2.2 uebayasi should_restart_tx(const struct sge_txq *q) 209 1.1.2.2 uebayasi { 210 1.1.2.2 uebayasi unsigned int r = q->processed - q->cleaned; 211 1.1.2.2 uebayasi 212 1.1.2.2 uebayasi return q->in_use - r < (q->size >> 1); 213 1.1.2.2 uebayasi } 214 1.1.2.2 uebayasi 215 1.1.2.2 uebayasi /** 216 1.1.2.2 uebayasi * t3_sge_init - initialize SGE 217 1.1.2.2 uebayasi * @adap: the adapter 218 1.1.2.2 uebayasi * @p: the SGE parameters 219 1.1.2.2 uebayasi * 220 1.1.2.2 uebayasi * Performs SGE initialization needed every time after a chip reset. 221 1.1.2.2 uebayasi * We do not initialize any of the queue sets here, instead the driver 222 1.1.2.2 uebayasi * top-level must request those individually. We also do not enable DMA 223 1.1.2.2 uebayasi * here, that should be done after the queues have been set up. 224 1.1.2.2 uebayasi */ 225 1.1.2.2 uebayasi void 226 1.1.2.2 uebayasi t3_sge_init(adapter_t *adap, struct sge_params *p) 227 1.1.2.2 uebayasi { 228 1.1.2.2 uebayasi u_int ctrl, ups; 229 1.1.2.2 uebayasi 230 1.1.2.2 uebayasi ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ 231 1.1.2.2 uebayasi 232 1.1.2.2 uebayasi ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | 233 1.1.2.2 uebayasi F_CQCRDTCTRL | 234 1.1.2.2 uebayasi V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | 235 1.1.2.2 uebayasi V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; 236 1.1.2.2 uebayasi #if SGE_NUM_GENBITS == 1 237 1.1.2.2 uebayasi ctrl |= F_EGRGENCTRL; 238 1.1.2.2 uebayasi #endif 239 1.1.2.2 uebayasi if (adap->params.rev > 0) { 240 1.1.2.2 uebayasi if (!(adap->flags & (USING_MSIX | USING_MSI))) 241 1.1.2.2 uebayasi ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ; 242 1.1.2.2 uebayasi ctrl |= F_CQCRDTCTRL | F_AVOIDCQOVFL; 243 1.1.2.2 uebayasi } 244 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_CONTROL, ctrl); 245 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) | 246 1.1.2.2 uebayasi V_LORCQDRBTHRSH(512)); 247 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10); 248 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) | 249 1.1.2.2 uebayasi V_TIMEOUT(200 * core_ticks_per_usec(adap))); 250 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 1000); 251 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256); 252 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000); 253 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256); 254 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff)); 255 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024); 256 1.1.2.2 uebayasi } 257 1.1.2.2 uebayasi 258 1.1.2.2 uebayasi 259 1.1.2.2 uebayasi /** 260 1.1.2.2 uebayasi * sgl_len - calculates the size of an SGL of the given capacity 261 1.1.2.2 uebayasi * @n: the number of SGL entries 262 1.1.2.2 uebayasi * 263 1.1.2.2 uebayasi * Calculates the number of flits needed for a scatter/gather list that 264 1.1.2.2 uebayasi * can hold the given number of entries. 265 1.1.2.2 uebayasi */ 266 1.1.2.2 uebayasi static __inline unsigned int 267 1.1.2.2 uebayasi sgl_len(unsigned int n) 268 1.1.2.2 uebayasi { 269 1.1.2.2 uebayasi return ((3 * n) / 2 + (n & 1)); 270 1.1.2.2 uebayasi } 271 1.1.2.2 uebayasi 272 1.1.2.2 uebayasi /** 273 1.1.2.2 uebayasi * get_imm_packet - return the next ingress packet buffer from a response 274 1.1.2.2 uebayasi * @resp: the response descriptor containing the packet data 275 1.1.2.2 uebayasi * 276 1.1.2.2 uebayasi * Return a packet containing the immediate data of the given response. 277 1.1.2.2 uebayasi */ 278 1.1.2.2 uebayasi #ifdef DISABLE_MBUF_IOVEC 279 1.1.2.2 uebayasi static __inline int 280 1.1.2.2 uebayasi get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh) 281 1.1.2.2 uebayasi { 282 1.1.2.2 uebayasi struct mbuf *m; 283 1.1.2.2 uebayasi int len; 284 1.1.2.2 uebayasi uint32_t flags = ntohl(resp->flags); 285 1.1.2.2 uebayasi uint8_t sopeop = G_RSPD_SOP_EOP(flags); 286 1.1.2.2 uebayasi 287 1.1.2.2 uebayasi /* 288 1.1.2.2 uebayasi * would be a firmware bug 289 1.1.2.2 uebayasi */ 290 1.1.2.2 uebayasi if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) 291 1.1.2.2 uebayasi return (0); 292 1.1.2.2 uebayasi 293 1.1.2.2 uebayasi m = m_gethdr(M_NOWAIT, MT_DATA); 294 1.1.2.2 uebayasi len = G_RSPD_LEN(ntohl(resp->len_cq)); 295 1.1.2.2 uebayasi 296 1.1.2.2 uebayasi if (m) { 297 1.1.2.2 uebayasi MH_ALIGN(m, IMMED_PKT_SIZE); 298 1.1.2.2 uebayasi memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE); 299 1.1.2.2 uebayasi m->m_len = len; 300 1.1.2.2 uebayasi 301 1.1.2.2 uebayasi switch (sopeop) { 302 1.1.2.2 uebayasi case RSPQ_SOP_EOP: 303 1.1.2.2 uebayasi mh->mh_head = mh->mh_tail = m; 304 1.1.2.2 uebayasi m->m_pkthdr.len = len; 305 1.1.2.2 uebayasi m->m_flags |= M_PKTHDR; 306 1.1.2.2 uebayasi break; 307 1.1.2.2 uebayasi case RSPQ_EOP: 308 1.1.2.2 uebayasi m->m_flags &= ~M_PKTHDR; 309 1.1.2.2 uebayasi mh->mh_head->m_pkthdr.len += len; 310 1.1.2.2 uebayasi mh->mh_tail->m_next = m; 311 1.1.2.2 uebayasi mh->mh_tail = m; 312 1.1.2.2 uebayasi break; 313 1.1.2.2 uebayasi } 314 1.1.2.2 uebayasi } 315 1.1.2.2 uebayasi return (m != NULL); 316 1.1.2.2 uebayasi } 317 1.1.2.2 uebayasi 318 1.1.2.2 uebayasi #else 319 1.1.2.2 uebayasi static int 320 1.1.2.2 uebayasi get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m, void *cl, uint32_t flags) 321 1.1.2.2 uebayasi { 322 1.1.2.2 uebayasi int len, error; 323 1.1.2.2 uebayasi uint8_t sopeop = G_RSPD_SOP_EOP(flags); 324 1.1.2.2 uebayasi 325 1.1.2.2 uebayasi /* 326 1.1.2.2 uebayasi * would be a firmware bug 327 1.1.2.2 uebayasi */ 328 1.1.2.2 uebayasi len = G_RSPD_LEN(ntohl(resp->len_cq)); 329 1.1.2.2 uebayasi if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) { 330 1.1.2.2 uebayasi if (cxgb_debug) 331 1.1.2.2 uebayasi device_printf(sc->dev, "unexpected value sopeop=%d flags=0x%x len=%din get_imm_packet\n", sopeop, flags, len); 332 1.1.2.2 uebayasi bogus_imm++; 333 1.1.2.2 uebayasi return (EINVAL); 334 1.1.2.2 uebayasi } 335 1.1.2.2 uebayasi error = 0; 336 1.1.2.2 uebayasi switch (sopeop) { 337 1.1.2.2 uebayasi case RSPQ_SOP_EOP: 338 1.1.2.2 uebayasi m->m_len = m->m_pkthdr.len = len; 339 1.1.2.2 uebayasi memcpy(mtod(m, uint8_t *), resp->imm_data, len); 340 1.1.2.2 uebayasi break; 341 1.1.2.2 uebayasi case RSPQ_EOP: 342 1.1.2.2 uebayasi memcpy(cl, resp->imm_data, len); 343 1.1.2.2 uebayasi m_iovappend(m, cl, MSIZE, len, 0); 344 1.1.2.2 uebayasi break; 345 1.1.2.2 uebayasi default: 346 1.1.2.2 uebayasi bogus_imm++; 347 1.1.2.2 uebayasi error = EINVAL; 348 1.1.2.2 uebayasi } 349 1.1.2.2 uebayasi 350 1.1.2.2 uebayasi return (error); 351 1.1.2.2 uebayasi } 352 1.1.2.2 uebayasi #endif 353 1.1.2.2 uebayasi 354 1.1.2.2 uebayasi static __inline u_int 355 1.1.2.2 uebayasi flits_to_desc(u_int n) 356 1.1.2.2 uebayasi { 357 1.1.2.2 uebayasi return (flit_desc_map[n]); 358 1.1.2.2 uebayasi } 359 1.1.2.2 uebayasi 360 1.1.2.2 uebayasi void 361 1.1.2.2 uebayasi t3_sge_err_intr_handler(adapter_t *adapter) 362 1.1.2.2 uebayasi { 363 1.1.2.2 uebayasi unsigned int v, status; 364 1.1.2.2 uebayasi 365 1.1.2.2 uebayasi 366 1.1.2.2 uebayasi status = t3_read_reg(adapter, A_SG_INT_CAUSE); 367 1.1.2.2 uebayasi 368 1.1.2.2 uebayasi if (status & F_RSPQCREDITOVERFOW) 369 1.1.2.2 uebayasi CH_ALERT(adapter, "SGE response queue credit overflow\n"); 370 1.1.2.2 uebayasi 371 1.1.2.2 uebayasi if (status & F_RSPQDISABLED) { 372 1.1.2.2 uebayasi v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS); 373 1.1.2.2 uebayasi 374 1.1.2.2 uebayasi CH_ALERT(adapter, 375 1.1.2.2 uebayasi "packet delivered to disabled response queue (0x%x)\n", 376 1.1.2.2 uebayasi (v >> S_RSPQ0DISABLED) & 0xff); 377 1.1.2.2 uebayasi } 378 1.1.2.2 uebayasi 379 1.1.2.2 uebayasi t3_write_reg(adapter, A_SG_INT_CAUSE, status); 380 1.1.2.2 uebayasi if (status & (F_RSPQCREDITOVERFOW | F_RSPQDISABLED)) 381 1.1.2.2 uebayasi t3_fatal_err(adapter); 382 1.1.2.2 uebayasi } 383 1.1.2.2 uebayasi 384 1.1.2.2 uebayasi void 385 1.1.2.2 uebayasi t3_sge_prep(adapter_t *adap, struct sge_params *p) 386 1.1.2.2 uebayasi { 387 1.1.2.2 uebayasi int i; 388 1.1.2.2 uebayasi 389 1.1.2.2 uebayasi /* XXX Does ETHER_ALIGN need to be accounted for here? */ 390 1.1.2.2 uebayasi p->max_pkt_size = MJUM16BYTES - sizeof(struct cpl_rx_data); 391 1.1.2.2 uebayasi 392 1.1.2.2 uebayasi for (i = 0; i < SGE_QSETS; ++i) { 393 1.1.2.2 uebayasi struct qset_params *q = p->qset + i; 394 1.1.2.2 uebayasi 395 1.1.2.2 uebayasi q->polling = adap->params.rev > 0; 396 1.1.2.2 uebayasi 397 1.1.2.2 uebayasi if (adap->params.nports > 2) 398 1.1.2.2 uebayasi q->coalesce_nsecs = 50000; 399 1.1.2.2 uebayasi else 400 1.1.2.2 uebayasi q->coalesce_nsecs = 5000; 401 1.1.2.2 uebayasi 402 1.1.2.2 uebayasi q->rspq_size = RSPQ_Q_SIZE; 403 1.1.2.2 uebayasi q->fl_size = FL_Q_SIZE; 404 1.1.2.2 uebayasi q->jumbo_size = JUMBO_Q_SIZE; 405 1.1.2.2 uebayasi q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; 406 1.1.2.2 uebayasi q->txq_size[TXQ_OFLD] = 1024; 407 1.1.2.2 uebayasi q->txq_size[TXQ_CTRL] = 256; 408 1.1.2.2 uebayasi q->cong_thres = 0; 409 1.1.2.2 uebayasi } 410 1.1.2.2 uebayasi } 411 1.1.2.2 uebayasi 412 1.1.2.2 uebayasi int 413 1.1.2.2 uebayasi t3_sge_alloc(adapter_t *sc) 414 1.1.2.2 uebayasi { 415 1.1.2.2 uebayasi /* The parent tag. */ 416 1.1.2.2 uebayasi sc->parent_dmat = sc->pa.pa_dmat; 417 1.1.2.2 uebayasi 418 1.1.2.2 uebayasi /* 419 1.1.2.2 uebayasi * DMA tag for normal sized RX frames 420 1.1.2.2 uebayasi */ 421 1.1.2.2 uebayasi sc->rx_dmat = sc->pa.pa_dmat; 422 1.1.2.2 uebayasi 423 1.1.2.2 uebayasi /* 424 1.1.2.2 uebayasi * DMA tag for jumbo sized RX frames. 425 1.1.2.2 uebayasi */ 426 1.1.2.2 uebayasi sc->rx_jumbo_dmat = sc->pa.pa_dmat; 427 1.1.2.2 uebayasi 428 1.1.2.2 uebayasi /* 429 1.1.2.2 uebayasi * DMA tag for TX frames. 430 1.1.2.2 uebayasi */ 431 1.1.2.2 uebayasi sc->tx_dmat = sc->pa.pa_dmat; 432 1.1.2.2 uebayasi 433 1.1.2.2 uebayasi return (0); 434 1.1.2.2 uebayasi } 435 1.1.2.2 uebayasi 436 1.1.2.2 uebayasi int 437 1.1.2.2 uebayasi t3_sge_free(struct adapter * sc) 438 1.1.2.2 uebayasi { 439 1.1.2.2 uebayasi return (0); 440 1.1.2.2 uebayasi } 441 1.1.2.2 uebayasi 442 1.1.2.2 uebayasi void 443 1.1.2.2 uebayasi t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p) 444 1.1.2.2 uebayasi { 445 1.1.2.2 uebayasi 446 1.1.2.2 uebayasi qs->rspq.holdoff_tmr = max(p->coalesce_nsecs/100, 1U); 447 1.1.2.2 uebayasi qs->rspq.polling = 0 /* p->polling */; 448 1.1.2.2 uebayasi } 449 1.1.2.2 uebayasi 450 1.1.2.2 uebayasi /** 451 1.1.2.2 uebayasi * refill_fl - refill an SGE free-buffer list 452 1.1.2.2 uebayasi * @sc: the controller softc 453 1.1.2.2 uebayasi * @q: the free-list to refill 454 1.1.2.2 uebayasi * @n: the number of new buffers to allocate 455 1.1.2.2 uebayasi * 456 1.1.2.2 uebayasi * (Re)populate an SGE free-buffer list with up to @n new packet buffers. 457 1.1.2.2 uebayasi * The caller must assure that @n does not exceed the queue's capacity. 458 1.1.2.2 uebayasi */ 459 1.1.2.2 uebayasi static void 460 1.1.2.2 uebayasi refill_fl(adapter_t *sc, struct sge_fl *q, int n) 461 1.1.2.2 uebayasi { 462 1.1.2.2 uebayasi struct rx_sw_desc *sd = &q->sdesc[q->pidx]; 463 1.1.2.2 uebayasi struct rx_desc *d = &q->desc[q->pidx]; 464 1.1.2.2 uebayasi void *cl; 465 1.1.2.2 uebayasi int err; 466 1.1.2.2 uebayasi 467 1.1.2.2 uebayasi while (n--) { 468 1.1.2.2 uebayasi /* 469 1.1.2.2 uebayasi * We only allocate a cluster, mbuf allocation happens after rx 470 1.1.2.2 uebayasi */ 471 1.1.2.2 uebayasi if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) 472 1.1.2.2 uebayasi { 473 1.1.2.2 uebayasi err = bus_dmamap_create(sc->pa.pa_dmat, 474 1.1.2.2 uebayasi q->buf_size, 1, q->buf_size, 0, 475 1.1.2.2 uebayasi BUS_DMA_ALLOCNOW, &sd->map); 476 1.1.2.2 uebayasi if (err != 0) 477 1.1.2.2 uebayasi { 478 1.1.2.2 uebayasi log(LOG_WARNING, "failure in refill_fl\n"); 479 1.1.2.2 uebayasi return; 480 1.1.2.2 uebayasi } 481 1.1.2.2 uebayasi sd->flags |= RX_SW_DESC_MAP_CREATED; 482 1.1.2.2 uebayasi } 483 1.1.2.2 uebayasi cl = malloc(q->buf_size, M_DEVBUF, M_NOWAIT); 484 1.1.2.2 uebayasi if (cl == NULL) 485 1.1.2.2 uebayasi { 486 1.1.2.2 uebayasi log(LOG_WARNING, "Failed to allocate cluster\n"); 487 1.1.2.2 uebayasi break; 488 1.1.2.2 uebayasi } 489 1.1.2.2 uebayasi err = bus_dmamap_load(sc->pa.pa_dmat, sd->map, cl, q->buf_size, NULL, BUS_DMA_NOWAIT); 490 1.1.2.2 uebayasi if (err) 491 1.1.2.2 uebayasi { 492 1.1.2.2 uebayasi log(LOG_WARNING, "failure in refill_fl\n"); 493 1.1.2.2 uebayasi free(cl, M_DEVBUF); 494 1.1.2.2 uebayasi return; 495 1.1.2.2 uebayasi } 496 1.1.2.2 uebayasi 497 1.1.2.2 uebayasi sd->flags |= RX_SW_DESC_INUSE; 498 1.1.2.2 uebayasi sd->cl = cl; 499 1.1.2.2 uebayasi d->addr_lo = htobe32(sd->map->dm_segs[0].ds_addr & 0xffffffff); 500 1.1.2.2 uebayasi d->addr_hi = htobe32(((uint64_t)sd->map->dm_segs[0].ds_addr>>32) & 0xffffffff); 501 1.1.2.2 uebayasi d->len_gen = htobe32(V_FLD_GEN1(q->gen)); 502 1.1.2.2 uebayasi d->gen2 = htobe32(V_FLD_GEN2(q->gen)); 503 1.1.2.2 uebayasi 504 1.1.2.2 uebayasi d++; 505 1.1.2.2 uebayasi sd++; 506 1.1.2.2 uebayasi 507 1.1.2.2 uebayasi if (++q->pidx == q->size) { 508 1.1.2.2 uebayasi q->pidx = 0; 509 1.1.2.2 uebayasi q->gen ^= 1; 510 1.1.2.2 uebayasi sd = q->sdesc; 511 1.1.2.2 uebayasi d = q->desc; 512 1.1.2.2 uebayasi } 513 1.1.2.2 uebayasi q->credits++; 514 1.1.2.2 uebayasi } 515 1.1.2.2 uebayasi 516 1.1.2.2 uebayasi t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 517 1.1.2.2 uebayasi } 518 1.1.2.2 uebayasi 519 1.1.2.2 uebayasi 520 1.1.2.2 uebayasi /** 521 1.1.2.2 uebayasi * free_rx_bufs - free the Rx buffers on an SGE free list 522 1.1.2.2 uebayasi * @sc: the controle softc 523 1.1.2.2 uebayasi * @q: the SGE free list to clean up 524 1.1.2.2 uebayasi * 525 1.1.2.2 uebayasi * Release the buffers on an SGE free-buffer Rx queue. HW fetching from 526 1.1.2.2 uebayasi * this queue should be stopped before calling this function. 527 1.1.2.2 uebayasi */ 528 1.1.2.2 uebayasi static void 529 1.1.2.2 uebayasi free_rx_bufs(adapter_t *sc, struct sge_fl *q) 530 1.1.2.2 uebayasi { 531 1.1.2.2 uebayasi u_int cidx = q->cidx; 532 1.1.2.2 uebayasi 533 1.1.2.2 uebayasi while (q->credits--) { 534 1.1.2.2 uebayasi struct rx_sw_desc *d = &q->sdesc[cidx]; 535 1.1.2.2 uebayasi 536 1.1.2.2 uebayasi if (d->flags & RX_SW_DESC_INUSE) { 537 1.1.2.2 uebayasi bus_dmamap_unload(q->entry_tag, d->map); 538 1.1.2.2 uebayasi bus_dmamap_destroy(q->entry_tag, d->map); 539 1.1.2.2 uebayasi d->map = NULL; 540 1.1.2.2 uebayasi free(d->cl, M_DEVBUF); 541 1.1.2.2 uebayasi d->cl = NULL; 542 1.1.2.2 uebayasi } 543 1.1.2.2 uebayasi d->cl = NULL; 544 1.1.2.2 uebayasi if (++cidx == q->size) 545 1.1.2.2 uebayasi cidx = 0; 546 1.1.2.2 uebayasi } 547 1.1.2.2 uebayasi } 548 1.1.2.2 uebayasi 549 1.1.2.2 uebayasi static __inline void 550 1.1.2.2 uebayasi __refill_fl(adapter_t *adap, struct sge_fl *fl) 551 1.1.2.2 uebayasi { 552 1.1.2.2 uebayasi refill_fl(adap, fl, min(16U, fl->size - fl->credits)); 553 1.1.2.2 uebayasi } 554 1.1.2.2 uebayasi 555 1.1.2.2 uebayasi #ifndef DISABLE_MBUF_IOVEC 556 1.1.2.2 uebayasi /** 557 1.1.2.2 uebayasi * recycle_rx_buf - recycle a receive buffer 558 1.1.2.2 uebayasi * @adapter: the adapter 559 1.1.2.2 uebayasi * @q: the SGE free list 560 1.1.2.2 uebayasi * @idx: index of buffer to recycle 561 1.1.2.2 uebayasi * 562 1.1.2.2 uebayasi * Recycles the specified buffer on the given free list by adding it at 563 1.1.2.2 uebayasi * the next available slot on the list. 564 1.1.2.2 uebayasi */ 565 1.1.2.2 uebayasi static void 566 1.1.2.2 uebayasi recycle_rx_buf(adapter_t *adap, struct sge_fl *q, unsigned int idx) 567 1.1.2.2 uebayasi { 568 1.1.2.2 uebayasi struct rx_desc *from = &q->desc[idx]; 569 1.1.2.2 uebayasi struct rx_desc *to = &q->desc[q->pidx]; 570 1.1.2.2 uebayasi 571 1.1.2.2 uebayasi q->sdesc[q->pidx] = q->sdesc[idx]; 572 1.1.2.2 uebayasi to->addr_lo = from->addr_lo; // already big endian 573 1.1.2.2 uebayasi to->addr_hi = from->addr_hi; // likewise 574 1.1.2.2 uebayasi wmb(); 575 1.1.2.2 uebayasi to->len_gen = htobe32(V_FLD_GEN1(q->gen)); 576 1.1.2.2 uebayasi to->gen2 = htobe32(V_FLD_GEN2(q->gen)); 577 1.1.2.2 uebayasi q->credits++; 578 1.1.2.2 uebayasi 579 1.1.2.2 uebayasi if (++q->pidx == q->size) { 580 1.1.2.2 uebayasi q->pidx = 0; 581 1.1.2.2 uebayasi q->gen ^= 1; 582 1.1.2.2 uebayasi } 583 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); 584 1.1.2.2 uebayasi } 585 1.1.2.2 uebayasi #endif 586 1.1.2.2 uebayasi 587 1.1.2.2 uebayasi static int 588 1.1.2.2 uebayasi alloc_ring(adapter_t *sc, size_t nelem, size_t elem_size, size_t sw_size, 589 1.1.2.2 uebayasi bus_addr_t *phys, 590 1.1.2.2 uebayasi void *desc, void *sdesc, bus_dma_tag_t *tag, 591 1.1.2.2 uebayasi bus_dmamap_t *map, bus_dma_tag_t parent_entry_tag, bus_dma_tag_t *entry_tag) 592 1.1.2.2 uebayasi { 593 1.1.2.2 uebayasi size_t len = nelem * elem_size; 594 1.1.2.2 uebayasi void *s = NULL; 595 1.1.2.2 uebayasi void *p = NULL; 596 1.1.2.2 uebayasi int err; 597 1.1.2.2 uebayasi bus_dma_segment_t phys_seg; 598 1.1.2.2 uebayasi 599 1.1.2.2 uebayasi int nsegs; 600 1.1.2.2 uebayasi 601 1.1.2.2 uebayasi *tag = sc->pa.pa_dmat; 602 1.1.2.2 uebayasi 603 1.1.2.2 uebayasi /* allocate wired physical memory for DMA descriptor array */ 604 1.1.2.2 uebayasi err = bus_dmamem_alloc(*tag, len, PAGE_SIZE, 0, &phys_seg, 1, 605 1.1.2.2 uebayasi &nsegs, BUS_DMA_NOWAIT); 606 1.1.2.2 uebayasi if (err != 0) 607 1.1.2.2 uebayasi { 608 1.1.2.2 uebayasi device_printf(sc->dev, "Cannot allocate descriptor memory\n"); 609 1.1.2.2 uebayasi return (ENOMEM); 610 1.1.2.2 uebayasi } 611 1.1.2.2 uebayasi *phys = phys_seg.ds_addr; 612 1.1.2.2 uebayasi 613 1.1.2.2 uebayasi /* map physical address to kernel virtual address */ 614 1.1.2.2 uebayasi err = bus_dmamem_map(*tag, &phys_seg, 1, len, &p, 615 1.1.2.2 uebayasi BUS_DMA_NOWAIT|BUS_DMA_COHERENT); 616 1.1.2.2 uebayasi if (err != 0) 617 1.1.2.2 uebayasi { 618 1.1.2.2 uebayasi device_printf(sc->dev, "Cannot map descriptor memory\n"); 619 1.1.2.2 uebayasi return (ENOMEM); 620 1.1.2.2 uebayasi } 621 1.1.2.2 uebayasi 622 1.1.2.2 uebayasi memset(p, 0, len); 623 1.1.2.2 uebayasi *(void **)desc = p; 624 1.1.2.2 uebayasi 625 1.1.2.2 uebayasi if (sw_size) 626 1.1.2.2 uebayasi { 627 1.1.2.2 uebayasi len = nelem * sw_size; 628 1.1.2.2 uebayasi s = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO); 629 1.1.2.2 uebayasi *(void **)sdesc = s; 630 1.1.2.2 uebayasi } 631 1.1.2.2 uebayasi 632 1.1.2.2 uebayasi if (parent_entry_tag == NULL) 633 1.1.2.2 uebayasi return (0); 634 1.1.2.2 uebayasi *entry_tag = sc->pa.pa_dmat; 635 1.1.2.2 uebayasi 636 1.1.2.2 uebayasi return (0); 637 1.1.2.2 uebayasi } 638 1.1.2.2 uebayasi 639 1.1.2.2 uebayasi static void 640 1.1.2.2 uebayasi sge_slow_intr_handler(struct work *wk, void *arg) 641 1.1.2.2 uebayasi { 642 1.1.2.2 uebayasi adapter_t *sc = arg; 643 1.1.2.2 uebayasi 644 1.1.2.2 uebayasi t3_slow_intr_handler(sc); 645 1.1.2.2 uebayasi } 646 1.1.2.2 uebayasi 647 1.1.2.2 uebayasi /** 648 1.1.2.2 uebayasi * sge_timer_cb - perform periodic maintenance of an SGE qset 649 1.1.2.2 uebayasi * @data: the SGE queue set to maintain 650 1.1.2.2 uebayasi * 651 1.1.2.2 uebayasi * Runs periodically from a timer to perform maintenance of an SGE queue 652 1.1.2.2 uebayasi * set. It performs two tasks: 653 1.1.2.2 uebayasi * 654 1.1.2.2 uebayasi * a) Cleans up any completed Tx descriptors that may still be pending. 655 1.1.2.2 uebayasi * Normal descriptor cleanup happens when new packets are added to a Tx 656 1.1.2.2 uebayasi * queue so this timer is relatively infrequent and does any cleanup only 657 1.1.2.2 uebayasi * if the Tx queue has not seen any new packets in a while. We make a 658 1.1.2.2 uebayasi * best effort attempt to reclaim descriptors, in that we don't wait 659 1.1.2.2 uebayasi * around if we cannot get a queue's lock (which most likely is because 660 1.1.2.2 uebayasi * someone else is queueing new packets and so will also handle the clean 661 1.1.2.2 uebayasi * up). Since control queues use immediate data exclusively we don't 662 1.1.2.2 uebayasi * bother cleaning them up here. 663 1.1.2.2 uebayasi * 664 1.1.2.2 uebayasi * b) Replenishes Rx queues that have run out due to memory shortage. 665 1.1.2.2 uebayasi * Normally new Rx buffers are added when existing ones are consumed but 666 1.1.2.2 uebayasi * when out of memory a queue can become empty. We try to add only a few 667 1.1.2.2 uebayasi * buffers here, the queue will be replenished fully as these new buffers 668 1.1.2.2 uebayasi * are used up if memory shortage has subsided. 669 1.1.2.2 uebayasi * 670 1.1.2.2 uebayasi * c) Return coalesced response queue credits in case a response queue is 671 1.1.2.2 uebayasi * starved. 672 1.1.2.2 uebayasi * 673 1.1.2.2 uebayasi * d) Ring doorbells for T304 tunnel queues since we have seen doorbell 674 1.1.2.2 uebayasi * fifo overflows and the FW doesn't implement any recovery scheme yet. 675 1.1.2.2 uebayasi */ 676 1.1.2.2 uebayasi 677 1.1.2.2 uebayasi static void 678 1.1.2.2 uebayasi sge_timer_cb(void *arg) 679 1.1.2.2 uebayasi { 680 1.1.2.2 uebayasi adapter_t *sc = arg; 681 1.1.2.2 uebayasi struct port_info *p; 682 1.1.2.2 uebayasi struct sge_qset *qs; 683 1.1.2.2 uebayasi struct sge_txq *txq; 684 1.1.2.2 uebayasi int i, j; 685 1.1.2.2 uebayasi int reclaim_eth, reclaim_ofl, refill_rx; 686 1.1.2.2 uebayasi 687 1.1.2.2 uebayasi for (i = 0; i < sc->params.nports; i++) 688 1.1.2.2 uebayasi for (j = 0; j < sc->port[i].nqsets; j++) { 689 1.1.2.2 uebayasi qs = &sc->sge.qs[i + j]; 690 1.1.2.2 uebayasi txq = &qs->txq[0]; 691 1.1.2.2 uebayasi reclaim_eth = txq[TXQ_ETH].processed - txq[TXQ_ETH].cleaned; 692 1.1.2.2 uebayasi reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; 693 1.1.2.2 uebayasi refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || 694 1.1.2.2 uebayasi (qs->fl[1].credits < qs->fl[1].size)); 695 1.1.2.2 uebayasi if (reclaim_eth || reclaim_ofl || refill_rx) { 696 1.1.2.2 uebayasi p = &sc->port[i]; 697 1.1.2.2 uebayasi workqueue_enqueue(p->timer_reclaim_task.wq, &p->timer_reclaim_task.w, NULL); 698 1.1.2.2 uebayasi break; 699 1.1.2.2 uebayasi } 700 1.1.2.2 uebayasi } 701 1.1.2.2 uebayasi if (sc->params.nports > 2) { 702 1.1.2.2 uebayasi int k; 703 1.1.2.2 uebayasi 704 1.1.2.2 uebayasi for_each_port(sc, k) { 705 1.1.2.2 uebayasi struct port_info *pi = &sc->port[k]; 706 1.1.2.2 uebayasi 707 1.1.2.2 uebayasi t3_write_reg(sc, A_SG_KDOORBELL, 708 1.1.2.2 uebayasi F_SELEGRCNTX | 709 1.1.2.2 uebayasi (FW_TUNNEL_SGEEC_START + pi->first_qset)); 710 1.1.2.2 uebayasi } 711 1.1.2.2 uebayasi } 712 1.1.2.2 uebayasi if (sc->open_device_map != 0) 713 1.1.2.2 uebayasi callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 714 1.1.2.2 uebayasi } 715 1.1.2.2 uebayasi 716 1.1.2.2 uebayasi /* 717 1.1.2.2 uebayasi * This is meant to be a catch-all function to keep sge state private 718 1.1.2.2 uebayasi * to sge.c 719 1.1.2.2 uebayasi * 720 1.1.2.2 uebayasi */ 721 1.1.2.2 uebayasi int 722 1.1.2.2 uebayasi t3_sge_init_adapter(adapter_t *sc) 723 1.1.2.2 uebayasi { 724 1.1.2.2 uebayasi callout_init(&sc->sge_timer_ch, 0); 725 1.1.2.2 uebayasi callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); 726 1.1.2.2 uebayasi sc->slow_intr_task.name = "sge_slow_intr"; 727 1.1.2.2 uebayasi sc->slow_intr_task.func = sge_slow_intr_handler; 728 1.1.2.2 uebayasi sc->slow_intr_task.context = sc; 729 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &sc->slow_intr_task, NULL, "cxgb_make_task"); 730 1.1.2.2 uebayasi return (0); 731 1.1.2.2 uebayasi } 732 1.1.2.2 uebayasi 733 1.1.2.2 uebayasi int 734 1.1.2.2 uebayasi t3_sge_init_port(struct port_info *p) 735 1.1.2.2 uebayasi { 736 1.1.2.2 uebayasi p->timer_reclaim_task.name = "sge_timer_reclaim"; 737 1.1.2.2 uebayasi p->timer_reclaim_task.func = sge_timer_reclaim; 738 1.1.2.2 uebayasi p->timer_reclaim_task.context = p; 739 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &p->timer_reclaim_task, NULL, "cxgb_make_task"); 740 1.1.2.2 uebayasi 741 1.1.2.2 uebayasi return (0); 742 1.1.2.2 uebayasi } 743 1.1.2.2 uebayasi 744 1.1.2.2 uebayasi void 745 1.1.2.2 uebayasi t3_sge_deinit_sw(adapter_t *sc) 746 1.1.2.2 uebayasi { 747 1.1.2.2 uebayasi callout_drain(&sc->sge_timer_ch); 748 1.1.2.2 uebayasi } 749 1.1.2.2 uebayasi 750 1.1.2.2 uebayasi /** 751 1.1.2.2 uebayasi * refill_rspq - replenish an SGE response queue 752 1.1.2.2 uebayasi * @adapter: the adapter 753 1.1.2.2 uebayasi * @q: the response queue to replenish 754 1.1.2.2 uebayasi * @credits: how many new responses to make available 755 1.1.2.2 uebayasi * 756 1.1.2.2 uebayasi * Replenishes a response queue by making the supplied number of responses 757 1.1.2.2 uebayasi * available to HW. 758 1.1.2.2 uebayasi */ 759 1.1.2.2 uebayasi static __inline void 760 1.1.2.2 uebayasi refill_rspq(adapter_t *sc, const struct sge_rspq *q, u_int credits) 761 1.1.2.2 uebayasi { 762 1.1.2.2 uebayasi 763 1.1.2.2 uebayasi /* mbufs are allocated on demand when a rspq entry is processed. */ 764 1.1.2.2 uebayasi t3_write_reg(sc, A_SG_RSPQ_CREDIT_RETURN, 765 1.1.2.2 uebayasi V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); 766 1.1.2.2 uebayasi } 767 1.1.2.2 uebayasi 768 1.1.2.2 uebayasi static __inline void 769 1.1.2.2 uebayasi sge_txq_reclaim_(struct sge_txq *txq) 770 1.1.2.2 uebayasi { 771 1.1.2.2 uebayasi int reclaimable, i, n; 772 1.1.2.2 uebayasi struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; 773 1.1.2.2 uebayasi struct port_info *p; 774 1.1.2.2 uebayasi 775 1.1.2.2 uebayasi p = txq->port; 776 1.1.2.2 uebayasi reclaim_more: 777 1.1.2.2 uebayasi n = 0; 778 1.1.2.2 uebayasi reclaimable = desc_reclaimable(txq); 779 1.1.2.2 uebayasi if (reclaimable > 0 && mtx_trylock(&txq->lock)) { 780 1.1.2.2 uebayasi n = reclaim_completed_tx(txq, TX_CLEAN_MAX_DESC, m_vec); 781 1.1.2.2 uebayasi mtx_unlock(&txq->lock); 782 1.1.2.2 uebayasi } 783 1.1.2.2 uebayasi if (n == 0) 784 1.1.2.2 uebayasi return; 785 1.1.2.2 uebayasi 786 1.1.2.2 uebayasi for (i = 0; i < n; i++) { 787 1.1.2.2 uebayasi m_freem_vec(m_vec[i]); 788 1.1.2.2 uebayasi } 789 1.1.2.2 uebayasi if (p && p->ifp->if_drv_flags & IFF_DRV_OACTIVE && 790 1.1.2.2 uebayasi txq->size - txq->in_use >= TX_START_MAX_DESC) { 791 1.1.2.2 uebayasi txq_fills++; 792 1.1.2.2 uebayasi p->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 793 1.1.2.2 uebayasi workqueue_enqueue(p->start_task.wq, &p->start_task.w, NULL); 794 1.1.2.2 uebayasi } 795 1.1.2.2 uebayasi 796 1.1.2.2 uebayasi if (n) 797 1.1.2.2 uebayasi goto reclaim_more; 798 1.1.2.2 uebayasi } 799 1.1.2.2 uebayasi 800 1.1.2.2 uebayasi static void 801 1.1.2.2 uebayasi sge_txq_reclaim_handler(struct work *wk, void *arg) 802 1.1.2.2 uebayasi { 803 1.1.2.2 uebayasi struct sge_txq *q = arg; 804 1.1.2.2 uebayasi 805 1.1.2.2 uebayasi sge_txq_reclaim_(q); 806 1.1.2.2 uebayasi } 807 1.1.2.2 uebayasi 808 1.1.2.2 uebayasi static void 809 1.1.2.2 uebayasi sge_timer_reclaim(struct work *wk, void *arg) 810 1.1.2.2 uebayasi { 811 1.1.2.2 uebayasi struct port_info *p = arg; 812 1.1.2.2 uebayasi int i, nqsets = p->nqsets; 813 1.1.2.2 uebayasi adapter_t *sc = p->adapter; 814 1.1.2.2 uebayasi struct sge_qset *qs; 815 1.1.2.2 uebayasi struct sge_txq *txq; 816 1.1.2.2 uebayasi struct mtx *lock; 817 1.1.2.2 uebayasi 818 1.1.2.2 uebayasi for (i = 0; i < nqsets; i++) { 819 1.1.2.2 uebayasi qs = &sc->sge.qs[i]; 820 1.1.2.2 uebayasi txq = &qs->txq[TXQ_ETH]; 821 1.1.2.2 uebayasi sge_txq_reclaim_(txq); 822 1.1.2.2 uebayasi 823 1.1.2.2 uebayasi txq = &qs->txq[TXQ_OFLD]; 824 1.1.2.2 uebayasi sge_txq_reclaim_(txq); 825 1.1.2.2 uebayasi 826 1.1.2.2 uebayasi lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : 827 1.1.2.2 uebayasi &sc->sge.qs[0].rspq.lock; 828 1.1.2.2 uebayasi 829 1.1.2.2 uebayasi if (mtx_trylock(lock)) { 830 1.1.2.2 uebayasi /* XXX currently assume that we are *NOT* polling */ 831 1.1.2.2 uebayasi uint32_t status = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS); 832 1.1.2.2 uebayasi 833 1.1.2.2 uebayasi if (qs->fl[0].credits < qs->fl[0].size - 16) 834 1.1.2.2 uebayasi __refill_fl(sc, &qs->fl[0]); 835 1.1.2.2 uebayasi if (qs->fl[1].credits < qs->fl[1].size - 16) 836 1.1.2.2 uebayasi __refill_fl(sc, &qs->fl[1]); 837 1.1.2.2 uebayasi 838 1.1.2.2 uebayasi if (status & (1 << qs->rspq.cntxt_id)) { 839 1.1.2.2 uebayasi if (qs->rspq.credits) { 840 1.1.2.2 uebayasi refill_rspq(sc, &qs->rspq, 1); 841 1.1.2.2 uebayasi qs->rspq.credits--; 842 1.1.2.2 uebayasi t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, 843 1.1.2.2 uebayasi 1 << qs->rspq.cntxt_id); 844 1.1.2.2 uebayasi } 845 1.1.2.2 uebayasi } 846 1.1.2.2 uebayasi mtx_unlock(lock); 847 1.1.2.2 uebayasi } 848 1.1.2.2 uebayasi } 849 1.1.2.2 uebayasi } 850 1.1.2.2 uebayasi 851 1.1.2.2 uebayasi /** 852 1.1.2.2 uebayasi * init_qset_cntxt - initialize an SGE queue set context info 853 1.1.2.2 uebayasi * @qs: the queue set 854 1.1.2.2 uebayasi * @id: the queue set id 855 1.1.2.2 uebayasi * 856 1.1.2.2 uebayasi * Initializes the TIDs and context ids for the queues of a queue set. 857 1.1.2.2 uebayasi */ 858 1.1.2.2 uebayasi static void 859 1.1.2.2 uebayasi init_qset_cntxt(struct sge_qset *qs, u_int id) 860 1.1.2.2 uebayasi { 861 1.1.2.2 uebayasi 862 1.1.2.2 uebayasi qs->rspq.cntxt_id = id; 863 1.1.2.2 uebayasi qs->fl[0].cntxt_id = 2 * id; 864 1.1.2.2 uebayasi qs->fl[1].cntxt_id = 2 * id + 1; 865 1.1.2.2 uebayasi qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; 866 1.1.2.2 uebayasi qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; 867 1.1.2.2 uebayasi qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; 868 1.1.2.2 uebayasi qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; 869 1.1.2.2 uebayasi qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; 870 1.1.2.2 uebayasi } 871 1.1.2.2 uebayasi 872 1.1.2.2 uebayasi 873 1.1.2.2 uebayasi static void 874 1.1.2.2 uebayasi txq_prod(struct sge_txq *txq, unsigned int ndesc, struct txq_state *txqs) 875 1.1.2.2 uebayasi { 876 1.1.2.2 uebayasi txq->in_use += ndesc; 877 1.1.2.2 uebayasi /* 878 1.1.2.2 uebayasi * XXX we don't handle stopping of queue 879 1.1.2.2 uebayasi * presumably start handles this when we bump against the end 880 1.1.2.2 uebayasi */ 881 1.1.2.2 uebayasi txqs->gen = txq->gen; 882 1.1.2.2 uebayasi txq->unacked += ndesc; 883 1.1.2.2 uebayasi txqs->compl = (txq->unacked & 8) << (S_WR_COMPL - 3); 884 1.1.2.2 uebayasi txq->unacked &= 7; 885 1.1.2.2 uebayasi txqs->pidx = txq->pidx; 886 1.1.2.2 uebayasi txq->pidx += ndesc; 887 1.1.2.2 uebayasi 888 1.1.2.2 uebayasi if (txq->pidx >= txq->size) { 889 1.1.2.2 uebayasi txq->pidx -= txq->size; 890 1.1.2.2 uebayasi txq->gen ^= 1; 891 1.1.2.2 uebayasi } 892 1.1.2.2 uebayasi 893 1.1.2.2 uebayasi } 894 1.1.2.2 uebayasi 895 1.1.2.2 uebayasi /** 896 1.1.2.2 uebayasi * calc_tx_descs - calculate the number of Tx descriptors for a packet 897 1.1.2.2 uebayasi * @m: the packet mbufs 898 1.1.2.2 uebayasi * @nsegs: the number of segments 899 1.1.2.2 uebayasi * 900 1.1.2.2 uebayasi * Returns the number of Tx descriptors needed for the given Ethernet 901 1.1.2.2 uebayasi * packet. Ethernet packets require addition of WR and CPL headers. 902 1.1.2.2 uebayasi */ 903 1.1.2.2 uebayasi static __inline unsigned int 904 1.1.2.2 uebayasi calc_tx_descs(const struct mbuf *m, int nsegs) 905 1.1.2.2 uebayasi { 906 1.1.2.2 uebayasi unsigned int flits; 907 1.1.2.2 uebayasi 908 1.1.2.2 uebayasi if (m->m_pkthdr.len <= WR_LEN - sizeof(struct cpl_tx_pkt)) 909 1.1.2.2 uebayasi return 1; 910 1.1.2.2 uebayasi 911 1.1.2.2 uebayasi flits = sgl_len(nsegs) + 2; 912 1.1.2.2 uebayasi #ifdef TSO_SUPPORTED 913 1.1.2.2 uebayasi if (m->m_pkthdr.csum_flags & (CSUM_TSO)) 914 1.1.2.2 uebayasi flits++; 915 1.1.2.2 uebayasi #endif 916 1.1.2.2 uebayasi return flits_to_desc(flits); 917 1.1.2.2 uebayasi } 918 1.1.2.2 uebayasi 919 1.1.2.2 uebayasi static unsigned int 920 1.1.2.2 uebayasi busdma_map_mbufs(struct mbuf **m, struct sge_txq *txq, 921 1.1.2.2 uebayasi struct tx_sw_desc *stx, bus_dma_segment_t *segs, int *nsegs) 922 1.1.2.2 uebayasi { 923 1.1.2.2 uebayasi struct mbuf *m0; 924 1.1.2.2 uebayasi int err, pktlen; 925 1.1.2.2 uebayasi int i, total_len; 926 1.1.2.2 uebayasi 927 1.1.2.2 uebayasi m0 = *m; 928 1.1.2.2 uebayasi pktlen = m0->m_pkthdr.len; 929 1.1.2.2 uebayasi 930 1.1.2.2 uebayasi m0 = *m; 931 1.1.2.2 uebayasi i = 0; 932 1.1.2.2 uebayasi total_len = 0; 933 1.1.2.2 uebayasi while (m0) 934 1.1.2.2 uebayasi { 935 1.1.2.2 uebayasi i++; 936 1.1.2.2 uebayasi total_len += m0->m_len; 937 1.1.2.2 uebayasi m0 = m0->m_next; 938 1.1.2.2 uebayasi } 939 1.1.2.2 uebayasi err = bus_dmamap_create(txq->entry_tag, total_len, TX_MAX_SEGS, total_len, 0, BUS_DMA_NOWAIT, &stx->map); 940 1.1.2.2 uebayasi if (err) 941 1.1.2.2 uebayasi return (err); 942 1.1.2.2 uebayasi err = bus_dmamap_load_mbuf(txq->entry_tag, stx->map, *m, 0); 943 1.1.2.2 uebayasi if (err) 944 1.1.2.2 uebayasi return (err); 945 1.1.2.2 uebayasi // feed out the physical mappings 946 1.1.2.2 uebayasi *nsegs = stx->map->dm_nsegs; 947 1.1.2.2 uebayasi for (i=0; i<*nsegs; i++) 948 1.1.2.2 uebayasi { 949 1.1.2.2 uebayasi segs[i] = stx->map->dm_segs[i]; 950 1.1.2.2 uebayasi } 951 1.1.2.2 uebayasi #ifdef DEBUG 952 1.1.2.2 uebayasi if (err) { 953 1.1.2.2 uebayasi int n = 0; 954 1.1.2.2 uebayasi struct mbuf *mtmp = m0; 955 1.1.2.2 uebayasi while(mtmp) { 956 1.1.2.2 uebayasi n++; 957 1.1.2.2 uebayasi mtmp = mtmp->m_next; 958 1.1.2.2 uebayasi } 959 1.1.2.2 uebayasi printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n", 960 1.1.2.2 uebayasi err, m0->m_pkthdr.len, n); 961 1.1.2.2 uebayasi } 962 1.1.2.2 uebayasi #endif 963 1.1.2.2 uebayasi if (err == EFBIG) { 964 1.1.2.2 uebayasi /* Too many segments, try to defrag */ 965 1.1.2.2 uebayasi m0 = m_defrag(m0, M_DONTWAIT); 966 1.1.2.2 uebayasi if (m0 == NULL) { 967 1.1.2.2 uebayasi m_freem(*m); 968 1.1.2.2 uebayasi *m = NULL; 969 1.1.2.2 uebayasi return (ENOBUFS); 970 1.1.2.2 uebayasi } 971 1.1.2.2 uebayasi *m = m0; 972 1.1.2.2 uebayasi INT3; // XXXXXXXXXXXXXXXXXX like above! 973 1.1.2.2 uebayasi } 974 1.1.2.2 uebayasi 975 1.1.2.2 uebayasi if (err == ENOMEM) { 976 1.1.2.2 uebayasi return (err); 977 1.1.2.2 uebayasi } 978 1.1.2.2 uebayasi 979 1.1.2.2 uebayasi if (err) { 980 1.1.2.2 uebayasi if (cxgb_debug) 981 1.1.2.2 uebayasi printf("map failure err=%d pktlen=%d\n", err, pktlen); 982 1.1.2.2 uebayasi m_freem_vec(m0); 983 1.1.2.2 uebayasi *m = NULL; 984 1.1.2.2 uebayasi return (err); 985 1.1.2.2 uebayasi } 986 1.1.2.2 uebayasi 987 1.1.2.2 uebayasi bus_dmamap_sync(txq->entry_tag, stx->map, 0, pktlen, BUS_DMASYNC_PREWRITE); 988 1.1.2.2 uebayasi stx->flags |= TX_SW_DESC_MAPPED; 989 1.1.2.2 uebayasi 990 1.1.2.2 uebayasi return (0); 991 1.1.2.2 uebayasi } 992 1.1.2.2 uebayasi 993 1.1.2.2 uebayasi /** 994 1.1.2.2 uebayasi * make_sgl - populate a scatter/gather list for a packet 995 1.1.2.2 uebayasi * @sgp: the SGL to populate 996 1.1.2.2 uebayasi * @segs: the packet dma segments 997 1.1.2.2 uebayasi * @nsegs: the number of segments 998 1.1.2.2 uebayasi * 999 1.1.2.2 uebayasi * Generates a scatter/gather list for the buffers that make up a packet 1000 1.1.2.2 uebayasi * and returns the SGL size in 8-byte words. The caller must size the SGL 1001 1.1.2.2 uebayasi * appropriately. 1002 1.1.2.2 uebayasi */ 1003 1.1.2.2 uebayasi static __inline void 1004 1.1.2.2 uebayasi make_sgl(struct sg_ent *sgp, bus_dma_segment_t *segs, int nsegs) 1005 1.1.2.2 uebayasi { 1006 1.1.2.2 uebayasi int i, idx; 1007 1.1.2.2 uebayasi 1008 1.1.2.2 uebayasi for (idx = 0, i = 0; i < nsegs; i++, idx ^= 1) { 1009 1.1.2.2 uebayasi if (i && idx == 0) 1010 1.1.2.2 uebayasi ++sgp; 1011 1.1.2.2 uebayasi 1012 1.1.2.2 uebayasi sgp->len[idx] = htobe32(segs[i].ds_len); 1013 1.1.2.2 uebayasi sgp->addr[idx] = htobe64(segs[i].ds_addr); 1014 1.1.2.2 uebayasi } 1015 1.1.2.2 uebayasi 1016 1.1.2.2 uebayasi if (idx) 1017 1.1.2.2 uebayasi sgp->len[idx] = 0; 1018 1.1.2.2 uebayasi } 1019 1.1.2.2 uebayasi 1020 1.1.2.2 uebayasi /** 1021 1.1.2.2 uebayasi * check_ring_tx_db - check and potentially ring a Tx queue's doorbell 1022 1.1.2.2 uebayasi * @adap: the adapter 1023 1.1.2.2 uebayasi * @q: the Tx queue 1024 1.1.2.2 uebayasi * 1025 1.1.2.2 uebayasi * Ring the doorbel if a Tx queue is asleep. There is a natural race, 1026 1.1.2.2 uebayasi * where the HW is going to sleep just after we checked, however, 1027 1.1.2.2 uebayasi * then the interrupt handler will detect the outstanding TX packet 1028 1.1.2.2 uebayasi * and ring the doorbell for us. 1029 1.1.2.2 uebayasi * 1030 1.1.2.2 uebayasi * When GTS is disabled we unconditionally ring the doorbell. 1031 1.1.2.2 uebayasi */ 1032 1.1.2.2 uebayasi static __inline void 1033 1.1.2.2 uebayasi check_ring_tx_db(adapter_t *adap, struct sge_txq *q) 1034 1.1.2.2 uebayasi { 1035 1.1.2.2 uebayasi #if USE_GTS 1036 1.1.2.2 uebayasi clear_bit(TXQ_LAST_PKT_DB, &q->flags); 1037 1.1.2.2 uebayasi if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { 1038 1.1.2.2 uebayasi set_bit(TXQ_LAST_PKT_DB, &q->flags); 1039 1.1.2.2 uebayasi #ifdef T3_TRACE 1040 1.1.2.2 uebayasi T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", 1041 1.1.2.2 uebayasi q->cntxt_id); 1042 1.1.2.2 uebayasi #endif 1043 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, 1044 1.1.2.2 uebayasi F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1045 1.1.2.2 uebayasi } 1046 1.1.2.2 uebayasi #else 1047 1.1.2.2 uebayasi wmb(); /* write descriptors before telling HW */ 1048 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, 1049 1.1.2.2 uebayasi F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1050 1.1.2.2 uebayasi #endif 1051 1.1.2.2 uebayasi } 1052 1.1.2.2 uebayasi 1053 1.1.2.2 uebayasi static __inline void 1054 1.1.2.2 uebayasi wr_gen2(struct tx_desc *d, unsigned int gen) 1055 1.1.2.2 uebayasi { 1056 1.1.2.2 uebayasi #if SGE_NUM_GENBITS == 2 1057 1.1.2.2 uebayasi d->flit[TX_DESC_FLITS - 1] = htobe64(gen); 1058 1.1.2.2 uebayasi #endif 1059 1.1.2.2 uebayasi } 1060 1.1.2.2 uebayasi 1061 1.1.2.2 uebayasi 1062 1.1.2.2 uebayasi 1063 1.1.2.2 uebayasi /** 1064 1.1.2.2 uebayasi * write_wr_hdr_sgl - write a WR header and, optionally, SGL 1065 1.1.2.2 uebayasi * @ndesc: number of Tx descriptors spanned by the SGL 1066 1.1.2.2 uebayasi * @txd: first Tx descriptor to be written 1067 1.1.2.2 uebayasi * @txqs: txq state (generation and producer index) 1068 1.1.2.2 uebayasi * @txq: the SGE Tx queue 1069 1.1.2.2 uebayasi * @sgl: the SGL 1070 1.1.2.2 uebayasi * @flits: number of flits to the start of the SGL in the first descriptor 1071 1.1.2.2 uebayasi * @sgl_flits: the SGL size in flits 1072 1.1.2.2 uebayasi * @wr_hi: top 32 bits of WR header based on WR type (big endian) 1073 1.1.2.2 uebayasi * @wr_lo: low 32 bits of WR header based on WR type (big endian) 1074 1.1.2.2 uebayasi * 1075 1.1.2.2 uebayasi * Write a work request header and an associated SGL. If the SGL is 1076 1.1.2.2 uebayasi * small enough to fit into one Tx descriptor it has already been written 1077 1.1.2.2 uebayasi * and we just need to write the WR header. Otherwise we distribute the 1078 1.1.2.2 uebayasi * SGL across the number of descriptors it spans. 1079 1.1.2.2 uebayasi */ 1080 1.1.2.2 uebayasi 1081 1.1.2.2 uebayasi static void 1082 1.1.2.2 uebayasi write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, 1083 1.1.2.2 uebayasi const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, 1084 1.1.2.2 uebayasi unsigned int sgl_flits, unsigned int wr_hi, unsigned int wr_lo) 1085 1.1.2.2 uebayasi { 1086 1.1.2.2 uebayasi 1087 1.1.2.2 uebayasi struct work_request_hdr *wrp = (struct work_request_hdr *)txd; 1088 1.1.2.2 uebayasi struct tx_sw_desc *txsd = &txq->sdesc[txqs->pidx]; 1089 1.1.2.2 uebayasi 1090 1.1.2.2 uebayasi if (__predict_true(ndesc == 1)) { 1091 1.1.2.2 uebayasi wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | 1092 1.1.2.2 uebayasi V_WR_SGLSFLT(flits)) | wr_hi; 1093 1.1.2.2 uebayasi wmb(); 1094 1.1.2.2 uebayasi wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) | 1095 1.1.2.2 uebayasi V_WR_GEN(txqs->gen)) | wr_lo; 1096 1.1.2.2 uebayasi /* XXX gen? */ 1097 1.1.2.2 uebayasi wr_gen2(txd, txqs->gen); 1098 1.1.2.2 uebayasi } else { 1099 1.1.2.2 uebayasi unsigned int ogen = txqs->gen; 1100 1.1.2.2 uebayasi const uint64_t *fp = (const uint64_t *)sgl; 1101 1.1.2.2 uebayasi struct work_request_hdr *wp = wrp; 1102 1.1.2.2 uebayasi 1103 1.1.2.2 uebayasi wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | 1104 1.1.2.2 uebayasi V_WR_SGLSFLT(flits)) | wr_hi; 1105 1.1.2.2 uebayasi 1106 1.1.2.2 uebayasi while (sgl_flits) { 1107 1.1.2.2 uebayasi unsigned int avail = WR_FLITS - flits; 1108 1.1.2.2 uebayasi 1109 1.1.2.2 uebayasi if (avail > sgl_flits) 1110 1.1.2.2 uebayasi avail = sgl_flits; 1111 1.1.2.2 uebayasi memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); 1112 1.1.2.2 uebayasi sgl_flits -= avail; 1113 1.1.2.2 uebayasi ndesc--; 1114 1.1.2.2 uebayasi if (!sgl_flits) 1115 1.1.2.2 uebayasi break; 1116 1.1.2.2 uebayasi 1117 1.1.2.2 uebayasi fp += avail; 1118 1.1.2.2 uebayasi txd++; 1119 1.1.2.2 uebayasi txsd++; 1120 1.1.2.2 uebayasi if (++txqs->pidx == txq->size) { 1121 1.1.2.2 uebayasi txqs->pidx = 0; 1122 1.1.2.2 uebayasi txqs->gen ^= 1; 1123 1.1.2.2 uebayasi txd = txq->desc; 1124 1.1.2.2 uebayasi txsd = txq->sdesc; 1125 1.1.2.2 uebayasi } 1126 1.1.2.2 uebayasi 1127 1.1.2.2 uebayasi /* 1128 1.1.2.2 uebayasi * when the head of the mbuf chain 1129 1.1.2.2 uebayasi * is freed all clusters will be freed 1130 1.1.2.2 uebayasi * with it 1131 1.1.2.2 uebayasi */ 1132 1.1.2.2 uebayasi txsd->m = NULL; 1133 1.1.2.2 uebayasi wrp = (struct work_request_hdr *)txd; 1134 1.1.2.2 uebayasi wrp->wr_hi = htonl(V_WR_DATATYPE(1) | 1135 1.1.2.2 uebayasi V_WR_SGLSFLT(1)) | wr_hi; 1136 1.1.2.2 uebayasi wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS, 1137 1.1.2.2 uebayasi sgl_flits + 1)) | 1138 1.1.2.2 uebayasi V_WR_GEN(txqs->gen)) | wr_lo; 1139 1.1.2.2 uebayasi wr_gen2(txd, txqs->gen); 1140 1.1.2.2 uebayasi flits = 1; 1141 1.1.2.2 uebayasi } 1142 1.1.2.2 uebayasi wrp->wr_hi |= htonl(F_WR_EOP); 1143 1.1.2.2 uebayasi wmb(); 1144 1.1.2.2 uebayasi wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; 1145 1.1.2.2 uebayasi wr_gen2((struct tx_desc *)wp, ogen); 1146 1.1.2.2 uebayasi } 1147 1.1.2.2 uebayasi } 1148 1.1.2.2 uebayasi 1149 1.1.2.2 uebayasi 1150 1.1.2.2 uebayasi /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ 1151 1.1.2.2 uebayasi #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) 1152 1.1.2.2 uebayasi 1153 1.1.2.2 uebayasi int 1154 1.1.2.2 uebayasi t3_encap(struct port_info *p, struct mbuf **m, int *free_it) 1155 1.1.2.2 uebayasi { 1156 1.1.2.2 uebayasi adapter_t *sc; 1157 1.1.2.2 uebayasi struct mbuf *m0; 1158 1.1.2.2 uebayasi struct sge_qset *qs; 1159 1.1.2.2 uebayasi struct sge_txq *txq; 1160 1.1.2.2 uebayasi struct tx_sw_desc *stx; 1161 1.1.2.2 uebayasi struct txq_state txqs; 1162 1.1.2.2 uebayasi unsigned int ndesc, flits, cntrl, mlen; 1163 1.1.2.2 uebayasi int err, nsegs, tso_info = 0; 1164 1.1.2.2 uebayasi 1165 1.1.2.2 uebayasi struct work_request_hdr *wrp; 1166 1.1.2.2 uebayasi struct tx_sw_desc *txsd; 1167 1.1.2.2 uebayasi struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; 1168 1.1.2.2 uebayasi bus_dma_segment_t segs[TX_MAX_SEGS]; 1169 1.1.2.2 uebayasi uint32_t wr_hi, wr_lo, sgl_flits; 1170 1.1.2.2 uebayasi 1171 1.1.2.2 uebayasi struct tx_desc *txd; 1172 1.1.2.2 uebayasi struct cpl_tx_pkt *cpl; 1173 1.1.2.2 uebayasi 1174 1.1.2.2 uebayasi m0 = *m; 1175 1.1.2.2 uebayasi sc = p->adapter; 1176 1.1.2.2 uebayasi 1177 1.1.2.2 uebayasi DPRINTF("t3_encap port_id=%d qsidx=%d ", p->port_id, p->first_qset); 1178 1.1.2.2 uebayasi 1179 1.1.2.2 uebayasi /* port_id=1 qsid=1 txpkt_intf=2 tx_chan=0 */ 1180 1.1.2.2 uebayasi 1181 1.1.2.2 uebayasi qs = &sc->sge.qs[p->first_qset]; 1182 1.1.2.2 uebayasi 1183 1.1.2.2 uebayasi txq = &qs->txq[TXQ_ETH]; 1184 1.1.2.2 uebayasi stx = &txq->sdesc[txq->pidx]; 1185 1.1.2.2 uebayasi txd = &txq->desc[txq->pidx]; 1186 1.1.2.2 uebayasi cpl = (struct cpl_tx_pkt *)txd; 1187 1.1.2.2 uebayasi mlen = m0->m_pkthdr.len; 1188 1.1.2.2 uebayasi cpl->len = htonl(mlen | 0x80000000); 1189 1.1.2.2 uebayasi 1190 1.1.2.2 uebayasi DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", mlen, p->txpkt_intf, p->tx_chan); 1191 1.1.2.2 uebayasi /* 1192 1.1.2.2 uebayasi * XXX handle checksum, TSO, and VLAN here 1193 1.1.2.2 uebayasi * 1194 1.1.2.2 uebayasi */ 1195 1.1.2.2 uebayasi cntrl = V_TXPKT_INTF(p->txpkt_intf); 1196 1.1.2.2 uebayasi 1197 1.1.2.2 uebayasi /* 1198 1.1.2.2 uebayasi * XXX need to add VLAN support for 6.x 1199 1.1.2.2 uebayasi */ 1200 1.1.2.2 uebayasi #ifdef VLAN_SUPPORTED 1201 1.1.2.2 uebayasi if (m0->m_flags & M_VLANTAG) 1202 1.1.2.2 uebayasi cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m0->m_pkthdr.ether_vtag); 1203 1.1.2.2 uebayasi if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) 1204 1.1.2.2 uebayasi tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); 1205 1.1.2.2 uebayasi #endif 1206 1.1.2.2 uebayasi if (tso_info) { 1207 1.1.2.2 uebayasi int eth_type; 1208 1.1.2.2 uebayasi struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *) cpl; 1209 1.1.2.2 uebayasi struct ip *ip; 1210 1.1.2.2 uebayasi struct tcphdr *tcp; 1211 1.1.2.2 uebayasi char *pkthdr, tmp[TCPPKTHDRSIZE]; /* is this too large for the stack? */ 1212 1.1.2.2 uebayasi 1213 1.1.2.2 uebayasi txd->flit[2] = 0; 1214 1.1.2.2 uebayasi cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); 1215 1.1.2.2 uebayasi hdr->cntrl = htonl(cntrl); 1216 1.1.2.2 uebayasi 1217 1.1.2.2 uebayasi if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { 1218 1.1.2.2 uebayasi pkthdr = &tmp[0]; 1219 1.1.2.2 uebayasi m_copydata(m0, 0, TCPPKTHDRSIZE, pkthdr); 1220 1.1.2.2 uebayasi } else { 1221 1.1.2.2 uebayasi pkthdr = mtod(m0, char *); 1222 1.1.2.2 uebayasi } 1223 1.1.2.2 uebayasi 1224 1.1.2.2 uebayasi #ifdef VLAN_SUPPORTED 1225 1.1.2.2 uebayasi if (__predict_false(m0->m_flags & M_VLANTAG)) { 1226 1.1.2.2 uebayasi eth_type = CPL_ETH_II_VLAN; 1227 1.1.2.2 uebayasi ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + 1228 1.1.2.2 uebayasi ETHER_VLAN_ENCAP_LEN); 1229 1.1.2.2 uebayasi } else { 1230 1.1.2.2 uebayasi eth_type = CPL_ETH_II; 1231 1.1.2.2 uebayasi ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); 1232 1.1.2.2 uebayasi } 1233 1.1.2.2 uebayasi #else 1234 1.1.2.2 uebayasi eth_type = CPL_ETH_II; 1235 1.1.2.2 uebayasi ip = (struct ip *)(pkthdr + ETHER_HDR_LEN); 1236 1.1.2.2 uebayasi #endif 1237 1.1.2.2 uebayasi tcp = (struct tcphdr *)((uint8_t *)ip + 1238 1.1.2.2 uebayasi sizeof(*ip)); 1239 1.1.2.2 uebayasi 1240 1.1.2.2 uebayasi tso_info |= V_LSO_ETH_TYPE(eth_type) | 1241 1.1.2.2 uebayasi V_LSO_IPHDR_WORDS(ip->ip_hl) | 1242 1.1.2.2 uebayasi V_LSO_TCPHDR_WORDS(tcp->th_off); 1243 1.1.2.2 uebayasi hdr->lso_info = htonl(tso_info); 1244 1.1.2.2 uebayasi flits = 3; 1245 1.1.2.2 uebayasi } else { 1246 1.1.2.2 uebayasi cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); 1247 1.1.2.2 uebayasi cpl->cntrl = htonl(cntrl); 1248 1.1.2.2 uebayasi 1249 1.1.2.2 uebayasi if (mlen <= WR_LEN - sizeof(*cpl)) { 1250 1.1.2.2 uebayasi txq_prod(txq, 1, &txqs); 1251 1.1.2.2 uebayasi txq->sdesc[txqs.pidx].m = NULL; 1252 1.1.2.2 uebayasi 1253 1.1.2.2 uebayasi if (m0->m_len == m0->m_pkthdr.len) 1254 1.1.2.2 uebayasi memcpy(&txd->flit[2], mtod(m0, uint8_t *), mlen); 1255 1.1.2.2 uebayasi else 1256 1.1.2.2 uebayasi m_copydata(m0, 0, mlen, (void *)&txd->flit[2]); 1257 1.1.2.2 uebayasi 1258 1.1.2.2 uebayasi *free_it = 1; 1259 1.1.2.2 uebayasi flits = (mlen + 7) / 8 + 2; 1260 1.1.2.2 uebayasi cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | 1261 1.1.2.2 uebayasi V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | 1262 1.1.2.2 uebayasi F_WR_SOP | F_WR_EOP | txqs.compl); 1263 1.1.2.2 uebayasi wmb(); 1264 1.1.2.2 uebayasi cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | 1265 1.1.2.2 uebayasi V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); 1266 1.1.2.2 uebayasi 1267 1.1.2.2 uebayasi wr_gen2(txd, txqs.gen); 1268 1.1.2.2 uebayasi check_ring_tx_db(sc, txq); 1269 1.1.2.2 uebayasi return (0); 1270 1.1.2.2 uebayasi } 1271 1.1.2.2 uebayasi flits = 2; 1272 1.1.2.2 uebayasi } 1273 1.1.2.2 uebayasi 1274 1.1.2.2 uebayasi wrp = (struct work_request_hdr *)txd; 1275 1.1.2.2 uebayasi 1276 1.1.2.2 uebayasi if ((err = busdma_map_mbufs(m, txq, stx, segs, &nsegs)) != 0) { 1277 1.1.2.2 uebayasi return (err); 1278 1.1.2.2 uebayasi } 1279 1.1.2.2 uebayasi m0 = *m; 1280 1.1.2.2 uebayasi ndesc = calc_tx_descs(m0, nsegs); 1281 1.1.2.2 uebayasi 1282 1.1.2.2 uebayasi sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; 1283 1.1.2.2 uebayasi make_sgl(sgp, segs, nsegs); 1284 1.1.2.2 uebayasi 1285 1.1.2.2 uebayasi sgl_flits = sgl_len(nsegs); 1286 1.1.2.2 uebayasi 1287 1.1.2.2 uebayasi DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc); 1288 1.1.2.2 uebayasi txq_prod(txq, ndesc, &txqs); 1289 1.1.2.2 uebayasi txsd = &txq->sdesc[txqs.pidx]; 1290 1.1.2.2 uebayasi wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); 1291 1.1.2.2 uebayasi wr_lo = htonl(V_WR_TID(txq->token)); 1292 1.1.2.2 uebayasi txsd->m = m0; 1293 1.1.2.2 uebayasi m_set_priority(m0, txqs.pidx); 1294 1.1.2.2 uebayasi 1295 1.1.2.2 uebayasi write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); 1296 1.1.2.2 uebayasi check_ring_tx_db(p->adapter, txq); 1297 1.1.2.2 uebayasi 1298 1.1.2.2 uebayasi return (0); 1299 1.1.2.2 uebayasi } 1300 1.1.2.2 uebayasi 1301 1.1.2.2 uebayasi 1302 1.1.2.2 uebayasi /** 1303 1.1.2.2 uebayasi * write_imm - write a packet into a Tx descriptor as immediate data 1304 1.1.2.2 uebayasi * @d: the Tx descriptor to write 1305 1.1.2.2 uebayasi * @m: the packet 1306 1.1.2.2 uebayasi * @len: the length of packet data to write as immediate data 1307 1.1.2.2 uebayasi * @gen: the generation bit value to write 1308 1.1.2.2 uebayasi * 1309 1.1.2.2 uebayasi * Writes a packet as immediate data into a Tx descriptor. The packet 1310 1.1.2.2 uebayasi * contains a work request at its beginning. We must write the packet 1311 1.1.2.2 uebayasi * carefully so the SGE doesn't read accidentally before it's written in 1312 1.1.2.2 uebayasi * its entirety. 1313 1.1.2.2 uebayasi */ 1314 1.1.2.2 uebayasi static __inline void 1315 1.1.2.2 uebayasi write_imm(struct tx_desc *d, struct mbuf *m, 1316 1.1.2.2 uebayasi unsigned int len, unsigned int gen) 1317 1.1.2.2 uebayasi { 1318 1.1.2.2 uebayasi struct work_request_hdr *from = mtod(m, struct work_request_hdr *); 1319 1.1.2.2 uebayasi struct work_request_hdr *to = (struct work_request_hdr *)d; 1320 1.1.2.2 uebayasi 1321 1.1.2.2 uebayasi memcpy(&to[1], &from[1], len - sizeof(*from)); 1322 1.1.2.2 uebayasi to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP | 1323 1.1.2.2 uebayasi V_WR_BCNTLFLT(len & 7)); 1324 1.1.2.2 uebayasi wmb(); 1325 1.1.2.2 uebayasi to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) | 1326 1.1.2.2 uebayasi V_WR_LEN((len + 7) / 8)); 1327 1.1.2.2 uebayasi wr_gen2(d, gen); 1328 1.1.2.2 uebayasi m_freem(m); 1329 1.1.2.2 uebayasi } 1330 1.1.2.2 uebayasi 1331 1.1.2.2 uebayasi /** 1332 1.1.2.2 uebayasi * check_desc_avail - check descriptor availability on a send queue 1333 1.1.2.2 uebayasi * @adap: the adapter 1334 1.1.2.2 uebayasi * @q: the TX queue 1335 1.1.2.2 uebayasi * @m: the packet needing the descriptors 1336 1.1.2.2 uebayasi * @ndesc: the number of Tx descriptors needed 1337 1.1.2.2 uebayasi * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL) 1338 1.1.2.2 uebayasi * 1339 1.1.2.2 uebayasi * Checks if the requested number of Tx descriptors is available on an 1340 1.1.2.2 uebayasi * SGE send queue. If the queue is already suspended or not enough 1341 1.1.2.2 uebayasi * descriptors are available the packet is queued for later transmission. 1342 1.1.2.2 uebayasi * Must be called with the Tx queue locked. 1343 1.1.2.2 uebayasi * 1344 1.1.2.2 uebayasi * Returns 0 if enough descriptors are available, 1 if there aren't 1345 1.1.2.2 uebayasi * enough descriptors and the packet has been queued, and 2 if the caller 1346 1.1.2.2 uebayasi * needs to retry because there weren't enough descriptors at the 1347 1.1.2.2 uebayasi * beginning of the call but some freed up in the mean time. 1348 1.1.2.2 uebayasi */ 1349 1.1.2.2 uebayasi static __inline int 1350 1.1.2.2 uebayasi check_desc_avail(adapter_t *adap, struct sge_txq *q, 1351 1.1.2.2 uebayasi struct mbuf *m, unsigned int ndesc, 1352 1.1.2.2 uebayasi unsigned int qid) 1353 1.1.2.2 uebayasi { 1354 1.1.2.2 uebayasi /* 1355 1.1.2.2 uebayasi * XXX We currently only use this for checking the control queue 1356 1.1.2.2 uebayasi * the control queue is only used for binding qsets which happens 1357 1.1.2.2 uebayasi * at init time so we are guaranteed enough descriptors 1358 1.1.2.2 uebayasi */ 1359 1.1.2.2 uebayasi if (__predict_false(!mbufq_empty(&q->sendq))) { 1360 1.1.2.2 uebayasi addq_exit: mbufq_tail(&q->sendq, m); 1361 1.1.2.2 uebayasi return 1; 1362 1.1.2.2 uebayasi } 1363 1.1.2.2 uebayasi if (__predict_false(q->size - q->in_use < ndesc)) { 1364 1.1.2.2 uebayasi 1365 1.1.2.2 uebayasi struct sge_qset *qs = txq_to_qset(q, qid); 1366 1.1.2.2 uebayasi 1367 1.1.2.2 uebayasi setbit(&qs->txq_stopped, qid); 1368 1.1.2.2 uebayasi smp_mb(); 1369 1.1.2.2 uebayasi 1370 1.1.2.2 uebayasi if (should_restart_tx(q) && 1371 1.1.2.2 uebayasi test_and_clear_bit(qid, &qs->txq_stopped)) 1372 1.1.2.2 uebayasi return 2; 1373 1.1.2.2 uebayasi 1374 1.1.2.2 uebayasi q->stops++; 1375 1.1.2.2 uebayasi goto addq_exit; 1376 1.1.2.2 uebayasi } 1377 1.1.2.2 uebayasi return 0; 1378 1.1.2.2 uebayasi } 1379 1.1.2.2 uebayasi 1380 1.1.2.2 uebayasi 1381 1.1.2.2 uebayasi /** 1382 1.1.2.2 uebayasi * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs 1383 1.1.2.2 uebayasi * @q: the SGE control Tx queue 1384 1.1.2.2 uebayasi * 1385 1.1.2.2 uebayasi * This is a variant of reclaim_completed_tx() that is used for Tx queues 1386 1.1.2.2 uebayasi * that send only immediate data (presently just the control queues) and 1387 1.1.2.2 uebayasi * thus do not have any mbufs 1388 1.1.2.2 uebayasi */ 1389 1.1.2.2 uebayasi static __inline void 1390 1.1.2.2 uebayasi reclaim_completed_tx_imm(struct sge_txq *q) 1391 1.1.2.2 uebayasi { 1392 1.1.2.2 uebayasi unsigned int reclaim = q->processed - q->cleaned; 1393 1.1.2.2 uebayasi 1394 1.1.2.2 uebayasi mtx_assert(&q->lock, MA_OWNED); 1395 1.1.2.2 uebayasi 1396 1.1.2.2 uebayasi q->in_use -= reclaim; 1397 1.1.2.2 uebayasi q->cleaned += reclaim; 1398 1.1.2.2 uebayasi } 1399 1.1.2.2 uebayasi 1400 1.1.2.2 uebayasi static __inline int 1401 1.1.2.2 uebayasi immediate(const struct mbuf *m) 1402 1.1.2.2 uebayasi { 1403 1.1.2.2 uebayasi return m->m_len <= WR_LEN && m->m_pkthdr.len <= WR_LEN ; 1404 1.1.2.2 uebayasi } 1405 1.1.2.2 uebayasi 1406 1.1.2.2 uebayasi /** 1407 1.1.2.2 uebayasi * ctrl_xmit - send a packet through an SGE control Tx queue 1408 1.1.2.2 uebayasi * @adap: the adapter 1409 1.1.2.2 uebayasi * @q: the control queue 1410 1.1.2.2 uebayasi * @m: the packet 1411 1.1.2.2 uebayasi * 1412 1.1.2.2 uebayasi * Send a packet through an SGE control Tx queue. Packets sent through 1413 1.1.2.2 uebayasi * a control queue must fit entirely as immediate data in a single Tx 1414 1.1.2.2 uebayasi * descriptor and have no page fragments. 1415 1.1.2.2 uebayasi */ 1416 1.1.2.2 uebayasi static int 1417 1.1.2.2 uebayasi ctrl_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 1418 1.1.2.2 uebayasi { 1419 1.1.2.2 uebayasi int ret; 1420 1.1.2.2 uebayasi struct work_request_hdr *wrp = mtod(m, struct work_request_hdr *); 1421 1.1.2.2 uebayasi 1422 1.1.2.2 uebayasi if (__predict_false(!immediate(m))) { 1423 1.1.2.2 uebayasi m_freem(m); 1424 1.1.2.2 uebayasi return 0; 1425 1.1.2.2 uebayasi } 1426 1.1.2.2 uebayasi 1427 1.1.2.2 uebayasi wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP); 1428 1.1.2.2 uebayasi wrp->wr_lo = htonl(V_WR_TID(q->token)); 1429 1.1.2.2 uebayasi 1430 1.1.2.2 uebayasi mtx_lock(&q->lock); 1431 1.1.2.2 uebayasi again: reclaim_completed_tx_imm(q); 1432 1.1.2.2 uebayasi 1433 1.1.2.2 uebayasi ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); 1434 1.1.2.2 uebayasi if (__predict_false(ret)) { 1435 1.1.2.2 uebayasi if (ret == 1) { 1436 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1437 1.1.2.2 uebayasi return (-1); 1438 1.1.2.2 uebayasi } 1439 1.1.2.2 uebayasi goto again; 1440 1.1.2.2 uebayasi } 1441 1.1.2.2 uebayasi 1442 1.1.2.2 uebayasi write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1443 1.1.2.2 uebayasi 1444 1.1.2.2 uebayasi q->in_use++; 1445 1.1.2.2 uebayasi if (++q->pidx >= q->size) { 1446 1.1.2.2 uebayasi q->pidx = 0; 1447 1.1.2.2 uebayasi q->gen ^= 1; 1448 1.1.2.2 uebayasi } 1449 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1450 1.1.2.2 uebayasi wmb(); 1451 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, 1452 1.1.2.2 uebayasi F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1453 1.1.2.2 uebayasi return (0); 1454 1.1.2.2 uebayasi } 1455 1.1.2.2 uebayasi 1456 1.1.2.2 uebayasi 1457 1.1.2.2 uebayasi /** 1458 1.1.2.2 uebayasi * restart_ctrlq - restart a suspended control queue 1459 1.1.2.2 uebayasi * @qs: the queue set cotaining the control queue 1460 1.1.2.2 uebayasi * 1461 1.1.2.2 uebayasi * Resumes transmission on a suspended Tx control queue. 1462 1.1.2.2 uebayasi */ 1463 1.1.2.2 uebayasi static void 1464 1.1.2.2 uebayasi restart_ctrlq(struct work *wk, void *data) 1465 1.1.2.2 uebayasi { 1466 1.1.2.2 uebayasi struct mbuf *m; 1467 1.1.2.2 uebayasi struct sge_qset *qs = (struct sge_qset *)data; 1468 1.1.2.2 uebayasi struct sge_txq *q = &qs->txq[TXQ_CTRL]; 1469 1.1.2.2 uebayasi adapter_t *adap = qs->port->adapter; 1470 1.1.2.2 uebayasi 1471 1.1.2.2 uebayasi mtx_lock(&q->lock); 1472 1.1.2.2 uebayasi again: reclaim_completed_tx_imm(q); 1473 1.1.2.2 uebayasi 1474 1.1.2.2 uebayasi while (q->in_use < q->size && 1475 1.1.2.2 uebayasi (m = mbufq_dequeue(&q->sendq)) != NULL) { 1476 1.1.2.2 uebayasi 1477 1.1.2.2 uebayasi write_imm(&q->desc[q->pidx], m, m->m_len, q->gen); 1478 1.1.2.2 uebayasi 1479 1.1.2.2 uebayasi if (++q->pidx >= q->size) { 1480 1.1.2.2 uebayasi q->pidx = 0; 1481 1.1.2.2 uebayasi q->gen ^= 1; 1482 1.1.2.2 uebayasi } 1483 1.1.2.2 uebayasi q->in_use++; 1484 1.1.2.2 uebayasi } 1485 1.1.2.2 uebayasi if (!mbufq_empty(&q->sendq)) { 1486 1.1.2.2 uebayasi setbit(&qs->txq_stopped, TXQ_CTRL); 1487 1.1.2.2 uebayasi smp_mb(); 1488 1.1.2.2 uebayasi 1489 1.1.2.2 uebayasi if (should_restart_tx(q) && 1490 1.1.2.2 uebayasi test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) 1491 1.1.2.2 uebayasi goto again; 1492 1.1.2.2 uebayasi q->stops++; 1493 1.1.2.2 uebayasi } 1494 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1495 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, 1496 1.1.2.2 uebayasi F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1497 1.1.2.2 uebayasi } 1498 1.1.2.2 uebayasi 1499 1.1.2.2 uebayasi 1500 1.1.2.2 uebayasi /* 1501 1.1.2.2 uebayasi * Send a management message through control queue 0 1502 1.1.2.2 uebayasi */ 1503 1.1.2.2 uebayasi int 1504 1.1.2.2 uebayasi t3_mgmt_tx(struct adapter *adap, struct mbuf *m) 1505 1.1.2.2 uebayasi { 1506 1.1.2.2 uebayasi return ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], m); 1507 1.1.2.2 uebayasi } 1508 1.1.2.2 uebayasi 1509 1.1.2.2 uebayasi /** 1510 1.1.2.2 uebayasi * free_qset - free the resources of an SGE queue set 1511 1.1.2.2 uebayasi * @sc: the controller owning the queue set 1512 1.1.2.2 uebayasi * @q: the queue set 1513 1.1.2.2 uebayasi * 1514 1.1.2.2 uebayasi * Release the HW and SW resources associated with an SGE queue set, such 1515 1.1.2.2 uebayasi * as HW contexts, packet buffers, and descriptor rings. Traffic to the 1516 1.1.2.2 uebayasi * queue set must be quiesced prior to calling this. 1517 1.1.2.2 uebayasi */ 1518 1.1.2.2 uebayasi static void 1519 1.1.2.2 uebayasi t3_free_qset(adapter_t *sc, struct sge_qset *q) 1520 1.1.2.2 uebayasi { 1521 1.1.2.2 uebayasi int i; 1522 1.1.2.2 uebayasi 1523 1.1.2.2 uebayasi for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 1524 1.1.2.2 uebayasi if (q->fl[i].desc) { 1525 1.1.2.2 uebayasi mtx_lock(&sc->sge.reg_lock); 1526 1.1.2.2 uebayasi t3_sge_disable_fl(sc, q->fl[i].cntxt_id); 1527 1.1.2.2 uebayasi mtx_unlock(&sc->sge.reg_lock); 1528 1.1.2.2 uebayasi bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); 1529 1.1.2.2 uebayasi INT3; 1530 1.1.2.2 uebayasi // bus_dmamem_free(q->fl[i].desc_tag, &q->fl[i].phys_addr, 1); 1531 1.1.2.2 uebayasi // XXXXXXXXXXX destroy DMA tags???? 1532 1.1.2.2 uebayasi } 1533 1.1.2.2 uebayasi if (q->fl[i].sdesc) { 1534 1.1.2.2 uebayasi free_rx_bufs(sc, &q->fl[i]); 1535 1.1.2.2 uebayasi free(q->fl[i].sdesc, M_DEVBUF); 1536 1.1.2.2 uebayasi } 1537 1.1.2.2 uebayasi } 1538 1.1.2.2 uebayasi 1539 1.1.2.2 uebayasi for (i = 0; i < SGE_TXQ_PER_SET; i++) { 1540 1.1.2.2 uebayasi if (q->txq[i].desc) { 1541 1.1.2.2 uebayasi mtx_lock(&sc->sge.reg_lock); 1542 1.1.2.2 uebayasi t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); 1543 1.1.2.2 uebayasi mtx_unlock(&sc->sge.reg_lock); 1544 1.1.2.2 uebayasi bus_dmamap_unload(q->txq[i].desc_tag, 1545 1.1.2.2 uebayasi q->txq[i].desc_map); 1546 1.1.2.2 uebayasi INT3; 1547 1.1.2.2 uebayasi // bus_dmamem_free(q->txq[i].desc_tag, &q->txq[i].phys_addr, 1); 1548 1.1.2.2 uebayasi // XXXXXXXXXXX destroy DMA tags???? And the lock?!??! 1549 1.1.2.2 uebayasi 1550 1.1.2.2 uebayasi } 1551 1.1.2.2 uebayasi if (q->txq[i].sdesc) { 1552 1.1.2.2 uebayasi free(q->txq[i].sdesc, M_DEVBUF); 1553 1.1.2.2 uebayasi } 1554 1.1.2.2 uebayasi } 1555 1.1.2.2 uebayasi 1556 1.1.2.2 uebayasi if (q->rspq.desc) { 1557 1.1.2.2 uebayasi mtx_lock(&sc->sge.reg_lock); 1558 1.1.2.2 uebayasi t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); 1559 1.1.2.2 uebayasi mtx_unlock(&sc->sge.reg_lock); 1560 1.1.2.2 uebayasi 1561 1.1.2.2 uebayasi bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); 1562 1.1.2.2 uebayasi INT3; 1563 1.1.2.2 uebayasi // bus_dmamem_free(q->rspq.desc_tag, &q->rspq.phys_addr, 1); 1564 1.1.2.2 uebayasi // XXXXXXXXXXX destroy DMA tags???? and the LOCK ?!?!? 1565 1.1.2.2 uebayasi } 1566 1.1.2.2 uebayasi 1567 1.1.2.2 uebayasi memset(q, 0, sizeof(*q)); 1568 1.1.2.2 uebayasi } 1569 1.1.2.2 uebayasi 1570 1.1.2.2 uebayasi /** 1571 1.1.2.2 uebayasi * t3_free_sge_resources - free SGE resources 1572 1.1.2.2 uebayasi * @sc: the adapter softc 1573 1.1.2.2 uebayasi * 1574 1.1.2.2 uebayasi * Frees resources used by the SGE queue sets. 1575 1.1.2.2 uebayasi */ 1576 1.1.2.2 uebayasi void 1577 1.1.2.2 uebayasi t3_free_sge_resources(adapter_t *sc) 1578 1.1.2.2 uebayasi { 1579 1.1.2.2 uebayasi int i, nqsets; 1580 1.1.2.2 uebayasi 1581 1.1.2.2 uebayasi for (nqsets = i = 0; i < (sc)->params.nports; i++) 1582 1.1.2.2 uebayasi nqsets += sc->port[i].nqsets; 1583 1.1.2.2 uebayasi 1584 1.1.2.2 uebayasi for (i = 0; i < nqsets; ++i) 1585 1.1.2.2 uebayasi t3_free_qset(sc, &sc->sge.qs[i]); 1586 1.1.2.2 uebayasi } 1587 1.1.2.2 uebayasi 1588 1.1.2.2 uebayasi /** 1589 1.1.2.2 uebayasi * t3_sge_start - enable SGE 1590 1.1.2.2 uebayasi * @sc: the controller softc 1591 1.1.2.2 uebayasi * 1592 1.1.2.2 uebayasi * Enables the SGE for DMAs. This is the last step in starting packet 1593 1.1.2.2 uebayasi * transfers. 1594 1.1.2.2 uebayasi */ 1595 1.1.2.2 uebayasi void 1596 1.1.2.2 uebayasi t3_sge_start(adapter_t *sc) 1597 1.1.2.2 uebayasi { 1598 1.1.2.2 uebayasi t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE); 1599 1.1.2.2 uebayasi } 1600 1.1.2.2 uebayasi 1601 1.1.2.2 uebayasi /** 1602 1.1.2.2 uebayasi * t3_sge_stop - disable SGE operation 1603 1.1.2.2 uebayasi * @sc: the adapter 1604 1.1.2.2 uebayasi * 1605 1.1.2.2 uebayasi * Disables the DMA engine. This can be called in emeregencies (e.g., 1606 1.1.2.2 uebayasi * from error interrupts) or from normal process context. In the latter 1607 1.1.2.2 uebayasi * case it also disables any pending queue restart tasklets. Note that 1608 1.1.2.2 uebayasi * if it is called in interrupt context it cannot disable the restart 1609 1.1.2.2 uebayasi * tasklets as it cannot wait, however the tasklets will have no effect 1610 1.1.2.2 uebayasi * since the doorbells are disabled and the driver will call this again 1611 1.1.2.2 uebayasi * later from process context, at which time the tasklets will be stopped 1612 1.1.2.2 uebayasi * if they are still running. 1613 1.1.2.2 uebayasi */ 1614 1.1.2.2 uebayasi void 1615 1.1.2.2 uebayasi t3_sge_stop(adapter_t *sc) 1616 1.1.2.2 uebayasi { 1617 1.1.2.2 uebayasi int i, nqsets; 1618 1.1.2.2 uebayasi 1619 1.1.2.2 uebayasi t3_set_reg_field(sc, A_SG_CONTROL, F_GLOBALENABLE, 0); 1620 1.1.2.2 uebayasi 1621 1.1.2.2 uebayasi for (nqsets = i = 0; i < (sc)->params.nports; i++) 1622 1.1.2.2 uebayasi nqsets += sc->port[i].nqsets; 1623 1.1.2.2 uebayasi 1624 1.1.2.2 uebayasi for (i = 0; i < nqsets; ++i) { 1625 1.1.2.2 uebayasi } 1626 1.1.2.2 uebayasi } 1627 1.1.2.2 uebayasi 1628 1.1.2.2 uebayasi 1629 1.1.2.2 uebayasi /** 1630 1.1.2.2 uebayasi * free_tx_desc - reclaims Tx descriptors and their buffers 1631 1.1.2.2 uebayasi * @adapter: the adapter 1632 1.1.2.2 uebayasi * @q: the Tx queue to reclaim descriptors from 1633 1.1.2.2 uebayasi * @n: the number of descriptors to reclaim 1634 1.1.2.2 uebayasi * 1635 1.1.2.2 uebayasi * Reclaims Tx descriptors from an SGE Tx queue and frees the associated 1636 1.1.2.2 uebayasi * Tx buffers. Called with the Tx queue lock held. 1637 1.1.2.2 uebayasi */ 1638 1.1.2.2 uebayasi int 1639 1.1.2.2 uebayasi free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec) 1640 1.1.2.2 uebayasi { 1641 1.1.2.2 uebayasi struct tx_sw_desc *d; 1642 1.1.2.2 uebayasi unsigned int cidx = q->cidx; 1643 1.1.2.2 uebayasi int nbufs = 0; 1644 1.1.2.2 uebayasi 1645 1.1.2.2 uebayasi #ifdef T3_TRACE 1646 1.1.2.2 uebayasi T3_TRACE2(sc->tb[q->cntxt_id & 7], 1647 1.1.2.2 uebayasi "reclaiming %u Tx descriptors at cidx %u", n, cidx); 1648 1.1.2.2 uebayasi #endif 1649 1.1.2.2 uebayasi d = &q->sdesc[cidx]; 1650 1.1.2.2 uebayasi 1651 1.1.2.2 uebayasi while (n-- > 0) { 1652 1.1.2.2 uebayasi DPRINTF("cidx=%d d=%p\n", cidx, d); 1653 1.1.2.2 uebayasi if (d->m) { 1654 1.1.2.2 uebayasi if (d->flags & TX_SW_DESC_MAPPED) { 1655 1.1.2.2 uebayasi bus_dmamap_unload(q->entry_tag, d->map); 1656 1.1.2.2 uebayasi bus_dmamap_destroy(q->entry_tag, d->map); 1657 1.1.2.2 uebayasi d->flags &= ~TX_SW_DESC_MAPPED; 1658 1.1.2.2 uebayasi } 1659 1.1.2.2 uebayasi if (m_get_priority(d->m) == cidx) { 1660 1.1.2.2 uebayasi m_vec[nbufs] = d->m; 1661 1.1.2.2 uebayasi d->m = NULL; 1662 1.1.2.2 uebayasi nbufs++; 1663 1.1.2.2 uebayasi } else { 1664 1.1.2.2 uebayasi printf("pri=%d cidx=%d\n", (int)m_get_priority(d->m), cidx); 1665 1.1.2.2 uebayasi } 1666 1.1.2.2 uebayasi } 1667 1.1.2.2 uebayasi ++d; 1668 1.1.2.2 uebayasi if (++cidx == q->size) { 1669 1.1.2.2 uebayasi cidx = 0; 1670 1.1.2.2 uebayasi d = q->sdesc; 1671 1.1.2.2 uebayasi } 1672 1.1.2.2 uebayasi } 1673 1.1.2.2 uebayasi q->cidx = cidx; 1674 1.1.2.2 uebayasi 1675 1.1.2.2 uebayasi return (nbufs); 1676 1.1.2.2 uebayasi } 1677 1.1.2.2 uebayasi 1678 1.1.2.2 uebayasi /** 1679 1.1.2.2 uebayasi * is_new_response - check if a response is newly written 1680 1.1.2.2 uebayasi * @r: the response descriptor 1681 1.1.2.2 uebayasi * @q: the response queue 1682 1.1.2.2 uebayasi * 1683 1.1.2.2 uebayasi * Returns true if a response descriptor contains a yet unprocessed 1684 1.1.2.2 uebayasi * response. 1685 1.1.2.2 uebayasi */ 1686 1.1.2.2 uebayasi static __inline int 1687 1.1.2.2 uebayasi is_new_response(const struct rsp_desc *r, 1688 1.1.2.2 uebayasi const struct sge_rspq *q) 1689 1.1.2.2 uebayasi { 1690 1.1.2.2 uebayasi return (r->intr_gen & F_RSPD_GEN2) == q->gen; 1691 1.1.2.2 uebayasi } 1692 1.1.2.2 uebayasi 1693 1.1.2.2 uebayasi #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) 1694 1.1.2.2 uebayasi #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ 1695 1.1.2.2 uebayasi V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ 1696 1.1.2.2 uebayasi V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \ 1697 1.1.2.2 uebayasi V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR)) 1698 1.1.2.2 uebayasi 1699 1.1.2.2 uebayasi /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */ 1700 1.1.2.2 uebayasi #define NOMEM_INTR_DELAY 2500 1701 1.1.2.2 uebayasi 1702 1.1.2.2 uebayasi /** 1703 1.1.2.2 uebayasi * write_ofld_wr - write an offload work request 1704 1.1.2.2 uebayasi * @adap: the adapter 1705 1.1.2.2 uebayasi * @m: the packet to send 1706 1.1.2.2 uebayasi * @q: the Tx queue 1707 1.1.2.2 uebayasi * @pidx: index of the first Tx descriptor to write 1708 1.1.2.2 uebayasi * @gen: the generation value to use 1709 1.1.2.2 uebayasi * @ndesc: number of descriptors the packet will occupy 1710 1.1.2.2 uebayasi * 1711 1.1.2.2 uebayasi * Write an offload work request to send the supplied packet. The packet 1712 1.1.2.2 uebayasi * data already carry the work request with most fields populated. 1713 1.1.2.2 uebayasi */ 1714 1.1.2.2 uebayasi static void 1715 1.1.2.2 uebayasi write_ofld_wr(adapter_t *adap, struct mbuf *m, 1716 1.1.2.2 uebayasi struct sge_txq *q, unsigned int pidx, 1717 1.1.2.2 uebayasi unsigned int gen, unsigned int ndesc, 1718 1.1.2.2 uebayasi bus_dma_segment_t *segs, unsigned int nsegs) 1719 1.1.2.2 uebayasi { 1720 1.1.2.2 uebayasi unsigned int sgl_flits, flits; 1721 1.1.2.2 uebayasi struct work_request_hdr *from; 1722 1.1.2.2 uebayasi struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; 1723 1.1.2.2 uebayasi struct tx_desc *d = &q->desc[pidx]; 1724 1.1.2.2 uebayasi struct txq_state txqs; 1725 1.1.2.2 uebayasi 1726 1.1.2.2 uebayasi if (immediate(m)) { 1727 1.1.2.2 uebayasi q->sdesc[pidx].m = NULL; 1728 1.1.2.2 uebayasi write_imm(d, m, m->m_len, gen); 1729 1.1.2.2 uebayasi return; 1730 1.1.2.2 uebayasi } 1731 1.1.2.2 uebayasi 1732 1.1.2.2 uebayasi /* Only TX_DATA builds SGLs */ 1733 1.1.2.2 uebayasi 1734 1.1.2.2 uebayasi from = mtod(m, struct work_request_hdr *); 1735 1.1.2.2 uebayasi INT3; /// DEBUG this??? 1736 1.1.2.2 uebayasi flits = 3; // XXXXXXXXXXXXXX 1737 1.1.2.2 uebayasi 1738 1.1.2.2 uebayasi sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : sgl; 1739 1.1.2.2 uebayasi 1740 1.1.2.2 uebayasi make_sgl(sgp, segs, nsegs); 1741 1.1.2.2 uebayasi sgl_flits = sgl_len(nsegs); 1742 1.1.2.2 uebayasi 1743 1.1.2.2 uebayasi txqs.gen = q->gen; 1744 1.1.2.2 uebayasi txqs.pidx = q->pidx; 1745 1.1.2.2 uebayasi txqs.compl = (q->unacked & 8) << (S_WR_COMPL - 3); 1746 1.1.2.2 uebayasi write_wr_hdr_sgl(ndesc, d, &txqs, q, sgl, flits, sgl_flits, 1747 1.1.2.2 uebayasi from->wr_hi, from->wr_lo); 1748 1.1.2.2 uebayasi } 1749 1.1.2.2 uebayasi 1750 1.1.2.2 uebayasi /** 1751 1.1.2.2 uebayasi * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet 1752 1.1.2.2 uebayasi * @m: the packet 1753 1.1.2.2 uebayasi * 1754 1.1.2.2 uebayasi * Returns the number of Tx descriptors needed for the given offload 1755 1.1.2.2 uebayasi * packet. These packets are already fully constructed. 1756 1.1.2.2 uebayasi */ 1757 1.1.2.2 uebayasi static __inline unsigned int 1758 1.1.2.2 uebayasi calc_tx_descs_ofld(struct mbuf *m, unsigned int nsegs) 1759 1.1.2.2 uebayasi { 1760 1.1.2.2 uebayasi unsigned int flits, cnt = 0; 1761 1.1.2.2 uebayasi 1762 1.1.2.2 uebayasi 1763 1.1.2.2 uebayasi if (m->m_len <= WR_LEN) 1764 1.1.2.2 uebayasi return 1; /* packet fits as immediate data */ 1765 1.1.2.2 uebayasi 1766 1.1.2.2 uebayasi if (m->m_flags & M_IOVEC) 1767 1.1.2.2 uebayasi cnt = mtomv(m)->mv_count; 1768 1.1.2.2 uebayasi 1769 1.1.2.2 uebayasi INT3; // Debug this???? 1770 1.1.2.2 uebayasi flits = 3; // XXXXXXXXX 1771 1.1.2.2 uebayasi 1772 1.1.2.2 uebayasi return flits_to_desc(flits + sgl_len(cnt)); 1773 1.1.2.2 uebayasi } 1774 1.1.2.2 uebayasi 1775 1.1.2.2 uebayasi /** 1776 1.1.2.2 uebayasi * ofld_xmit - send a packet through an offload queue 1777 1.1.2.2 uebayasi * @adap: the adapter 1778 1.1.2.2 uebayasi * @q: the Tx offload queue 1779 1.1.2.2 uebayasi * @m: the packet 1780 1.1.2.2 uebayasi * 1781 1.1.2.2 uebayasi * Send an offload packet through an SGE offload queue. 1782 1.1.2.2 uebayasi */ 1783 1.1.2.2 uebayasi static int 1784 1.1.2.2 uebayasi ofld_xmit(adapter_t *adap, struct sge_txq *q, struct mbuf *m) 1785 1.1.2.2 uebayasi { 1786 1.1.2.2 uebayasi int ret, nsegs; 1787 1.1.2.2 uebayasi unsigned int ndesc; 1788 1.1.2.2 uebayasi unsigned int pidx, gen; 1789 1.1.2.2 uebayasi struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; 1790 1.1.2.2 uebayasi bus_dma_segment_t segs[TX_MAX_SEGS]; 1791 1.1.2.2 uebayasi int i, cleaned; 1792 1.1.2.2 uebayasi struct tx_sw_desc *stx = &q->sdesc[q->pidx]; 1793 1.1.2.2 uebayasi 1794 1.1.2.2 uebayasi mtx_lock(&q->lock); 1795 1.1.2.2 uebayasi if ((ret = busdma_map_mbufs(&m, q, stx, segs, &nsegs)) != 0) { 1796 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1797 1.1.2.2 uebayasi return (ret); 1798 1.1.2.2 uebayasi } 1799 1.1.2.2 uebayasi ndesc = calc_tx_descs_ofld(m, nsegs); 1800 1.1.2.2 uebayasi again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec); 1801 1.1.2.2 uebayasi 1802 1.1.2.2 uebayasi ret = check_desc_avail(adap, q, m, ndesc, TXQ_OFLD); 1803 1.1.2.2 uebayasi if (__predict_false(ret)) { 1804 1.1.2.2 uebayasi if (ret == 1) { 1805 1.1.2.2 uebayasi m_set_priority(m, ndesc); /* save for restart */ 1806 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1807 1.1.2.2 uebayasi return EINTR; 1808 1.1.2.2 uebayasi } 1809 1.1.2.2 uebayasi goto again; 1810 1.1.2.2 uebayasi } 1811 1.1.2.2 uebayasi 1812 1.1.2.2 uebayasi gen = q->gen; 1813 1.1.2.2 uebayasi q->in_use += ndesc; 1814 1.1.2.2 uebayasi pidx = q->pidx; 1815 1.1.2.2 uebayasi q->pidx += ndesc; 1816 1.1.2.2 uebayasi if (q->pidx >= q->size) { 1817 1.1.2.2 uebayasi q->pidx -= q->size; 1818 1.1.2.2 uebayasi q->gen ^= 1; 1819 1.1.2.2 uebayasi } 1820 1.1.2.2 uebayasi #ifdef T3_TRACE 1821 1.1.2.2 uebayasi T3_TRACE5(adap->tb[q->cntxt_id & 7], 1822 1.1.2.2 uebayasi "ofld_xmit: ndesc %u, pidx %u, len %u, main %u, frags %u", 1823 1.1.2.2 uebayasi ndesc, pidx, skb->len, skb->len - skb->data_len, 1824 1.1.2.2 uebayasi skb_shinfo(skb)->nr_frags); 1825 1.1.2.2 uebayasi #endif 1826 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1827 1.1.2.2 uebayasi 1828 1.1.2.2 uebayasi write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 1829 1.1.2.2 uebayasi check_ring_tx_db(adap, q); 1830 1.1.2.2 uebayasi 1831 1.1.2.2 uebayasi for (i = 0; i < cleaned; i++) { 1832 1.1.2.2 uebayasi m_freem_vec(m_vec[i]); 1833 1.1.2.2 uebayasi } 1834 1.1.2.2 uebayasi return (0); 1835 1.1.2.2 uebayasi } 1836 1.1.2.2 uebayasi 1837 1.1.2.2 uebayasi /** 1838 1.1.2.2 uebayasi * restart_offloadq - restart a suspended offload queue 1839 1.1.2.2 uebayasi * @qs: the queue set cotaining the offload queue 1840 1.1.2.2 uebayasi * 1841 1.1.2.2 uebayasi * Resumes transmission on a suspended Tx offload queue. 1842 1.1.2.2 uebayasi */ 1843 1.1.2.2 uebayasi static void 1844 1.1.2.2 uebayasi restart_offloadq(struct work *wk, void *data) 1845 1.1.2.2 uebayasi { 1846 1.1.2.2 uebayasi 1847 1.1.2.2 uebayasi struct mbuf *m; 1848 1.1.2.2 uebayasi struct sge_qset *qs = data; 1849 1.1.2.2 uebayasi struct sge_txq *q = &qs->txq[TXQ_OFLD]; 1850 1.1.2.2 uebayasi adapter_t *adap = qs->port->adapter; 1851 1.1.2.2 uebayasi struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; 1852 1.1.2.2 uebayasi bus_dma_segment_t segs[TX_MAX_SEGS]; 1853 1.1.2.2 uebayasi int nsegs, i, cleaned; 1854 1.1.2.2 uebayasi struct tx_sw_desc *stx = &q->sdesc[q->pidx]; 1855 1.1.2.2 uebayasi 1856 1.1.2.2 uebayasi mtx_lock(&q->lock); 1857 1.1.2.2 uebayasi again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec); 1858 1.1.2.2 uebayasi 1859 1.1.2.2 uebayasi while ((m = mbufq_peek(&q->sendq)) != NULL) { 1860 1.1.2.2 uebayasi unsigned int gen, pidx; 1861 1.1.2.2 uebayasi unsigned int ndesc = m_get_priority(m); 1862 1.1.2.2 uebayasi 1863 1.1.2.2 uebayasi if (__predict_false(q->size - q->in_use < ndesc)) { 1864 1.1.2.2 uebayasi setbit(&qs->txq_stopped, TXQ_OFLD); 1865 1.1.2.2 uebayasi smp_mb(); 1866 1.1.2.2 uebayasi 1867 1.1.2.2 uebayasi if (should_restart_tx(q) && 1868 1.1.2.2 uebayasi test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) 1869 1.1.2.2 uebayasi goto again; 1870 1.1.2.2 uebayasi q->stops++; 1871 1.1.2.2 uebayasi break; 1872 1.1.2.2 uebayasi } 1873 1.1.2.2 uebayasi 1874 1.1.2.2 uebayasi gen = q->gen; 1875 1.1.2.2 uebayasi q->in_use += ndesc; 1876 1.1.2.2 uebayasi pidx = q->pidx; 1877 1.1.2.2 uebayasi q->pidx += ndesc; 1878 1.1.2.2 uebayasi if (q->pidx >= q->size) { 1879 1.1.2.2 uebayasi q->pidx -= q->size; 1880 1.1.2.2 uebayasi q->gen ^= 1; 1881 1.1.2.2 uebayasi } 1882 1.1.2.2 uebayasi 1883 1.1.2.2 uebayasi (void)mbufq_dequeue(&q->sendq); 1884 1.1.2.2 uebayasi busdma_map_mbufs(&m, q, stx, segs, &nsegs); 1885 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1886 1.1.2.2 uebayasi write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); 1887 1.1.2.2 uebayasi mtx_lock(&q->lock); 1888 1.1.2.2 uebayasi } 1889 1.1.2.2 uebayasi mtx_unlock(&q->lock); 1890 1.1.2.2 uebayasi 1891 1.1.2.2 uebayasi #if USE_GTS 1892 1.1.2.2 uebayasi set_bit(TXQ_RUNNING, &q->flags); 1893 1.1.2.2 uebayasi set_bit(TXQ_LAST_PKT_DB, &q->flags); 1894 1.1.2.2 uebayasi #endif 1895 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_KDOORBELL, 1896 1.1.2.2 uebayasi F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); 1897 1.1.2.2 uebayasi 1898 1.1.2.2 uebayasi for (i = 0; i < cleaned; i++) { 1899 1.1.2.2 uebayasi m_freem_vec(m_vec[i]); 1900 1.1.2.2 uebayasi } 1901 1.1.2.2 uebayasi } 1902 1.1.2.2 uebayasi 1903 1.1.2.2 uebayasi /** 1904 1.1.2.2 uebayasi * queue_set - return the queue set a packet should use 1905 1.1.2.2 uebayasi * @m: the packet 1906 1.1.2.2 uebayasi * 1907 1.1.2.2 uebayasi * Maps a packet to the SGE queue set it should use. The desired queue 1908 1.1.2.2 uebayasi * set is carried in bits 1-3 in the packet's priority. 1909 1.1.2.2 uebayasi */ 1910 1.1.2.2 uebayasi static __inline int 1911 1.1.2.2 uebayasi queue_set(const struct mbuf *m) 1912 1.1.2.2 uebayasi { 1913 1.1.2.2 uebayasi return m_get_priority(m) >> 1; 1914 1.1.2.2 uebayasi } 1915 1.1.2.2 uebayasi 1916 1.1.2.2 uebayasi /** 1917 1.1.2.2 uebayasi * is_ctrl_pkt - return whether an offload packet is a control packet 1918 1.1.2.2 uebayasi * @m: the packet 1919 1.1.2.2 uebayasi * 1920 1.1.2.2 uebayasi * Determines whether an offload packet should use an OFLD or a CTRL 1921 1.1.2.2 uebayasi * Tx queue. This is indicated by bit 0 in the packet's priority. 1922 1.1.2.2 uebayasi */ 1923 1.1.2.2 uebayasi static __inline int 1924 1.1.2.2 uebayasi is_ctrl_pkt(const struct mbuf *m) 1925 1.1.2.2 uebayasi { 1926 1.1.2.2 uebayasi return m_get_priority(m) & 1; 1927 1.1.2.2 uebayasi } 1928 1.1.2.2 uebayasi 1929 1.1.2.2 uebayasi /** 1930 1.1.2.2 uebayasi * t3_offload_tx - send an offload packet 1931 1.1.2.2 uebayasi * @tdev: the offload device to send to 1932 1.1.2.2 uebayasi * @m: the packet 1933 1.1.2.2 uebayasi * 1934 1.1.2.2 uebayasi * Sends an offload packet. We use the packet priority to select the 1935 1.1.2.2 uebayasi * appropriate Tx queue as follows: bit 0 indicates whether the packet 1936 1.1.2.2 uebayasi * should be sent as regular or control, bits 1-3 select the queue set. 1937 1.1.2.2 uebayasi */ 1938 1.1.2.2 uebayasi int 1939 1.1.2.2 uebayasi t3_offload_tx(struct toedev *tdev, struct mbuf *m) 1940 1.1.2.2 uebayasi { 1941 1.1.2.2 uebayasi adapter_t *adap = tdev2adap(tdev); 1942 1.1.2.2 uebayasi struct sge_qset *qs = &adap->sge.qs[queue_set(m)]; 1943 1.1.2.2 uebayasi 1944 1.1.2.2 uebayasi if (__predict_false(is_ctrl_pkt(m))) 1945 1.1.2.2 uebayasi return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], m); 1946 1.1.2.2 uebayasi 1947 1.1.2.2 uebayasi return ofld_xmit(adap, &qs->txq[TXQ_OFLD], m); 1948 1.1.2.2 uebayasi } 1949 1.1.2.2 uebayasi 1950 1.1.2.2 uebayasi static void 1951 1.1.2.2 uebayasi restart_tx(struct sge_qset *qs) 1952 1.1.2.2 uebayasi { 1953 1.1.2.2 uebayasi if (isset(&qs->txq_stopped, TXQ_OFLD) && 1954 1.1.2.2 uebayasi should_restart_tx(&qs->txq[TXQ_OFLD]) && 1955 1.1.2.2 uebayasi test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { 1956 1.1.2.2 uebayasi qs->txq[TXQ_OFLD].restarts++; 1957 1.1.2.2 uebayasi workqueue_enqueue(qs->txq[TXQ_OFLD].qresume_task.wq, &qs->txq[TXQ_OFLD].qresume_task.w, NULL); 1958 1.1.2.2 uebayasi } 1959 1.1.2.2 uebayasi if (isset(&qs->txq_stopped, TXQ_CTRL) && 1960 1.1.2.2 uebayasi should_restart_tx(&qs->txq[TXQ_CTRL]) && 1961 1.1.2.2 uebayasi test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { 1962 1.1.2.2 uebayasi qs->txq[TXQ_CTRL].restarts++; 1963 1.1.2.2 uebayasi workqueue_enqueue(qs->txq[TXQ_CTRL].qresume_task.wq, &qs->txq[TXQ_CTRL].qresume_task.w, NULL); 1964 1.1.2.2 uebayasi } 1965 1.1.2.2 uebayasi } 1966 1.1.2.2 uebayasi 1967 1.1.2.2 uebayasi /** 1968 1.1.2.2 uebayasi * t3_sge_alloc_qset - initialize an SGE queue set 1969 1.1.2.2 uebayasi * @sc: the controller softc 1970 1.1.2.2 uebayasi * @id: the queue set id 1971 1.1.2.2 uebayasi * @nports: how many Ethernet ports will be using this queue set 1972 1.1.2.2 uebayasi * @irq_vec_idx: the IRQ vector index for response queue interrupts 1973 1.1.2.2 uebayasi * @p: configuration parameters for this queue set 1974 1.1.2.2 uebayasi * @ntxq: number of Tx queues for the queue set 1975 1.1.2.2 uebayasi * @pi: port info for queue set 1976 1.1.2.2 uebayasi * 1977 1.1.2.2 uebayasi * Allocate resources and initialize an SGE queue set. A queue set 1978 1.1.2.2 uebayasi * comprises a response queue, two Rx free-buffer queues, and up to 3 1979 1.1.2.2 uebayasi * Tx queues. The Tx queues are assigned roles in the order Ethernet 1980 1.1.2.2 uebayasi * queue, offload queue, and control queue. 1981 1.1.2.2 uebayasi */ 1982 1.1.2.2 uebayasi int 1983 1.1.2.2 uebayasi t3_sge_alloc_qset(adapter_t *sc, u_int id, int nports, int irq_vec_idx, 1984 1.1.2.2 uebayasi const struct qset_params *p, int ntxq, struct port_info *pi) 1985 1.1.2.2 uebayasi { 1986 1.1.2.2 uebayasi struct sge_qset *q = &sc->sge.qs[id]; 1987 1.1.2.2 uebayasi int i, ret = 0; 1988 1.1.2.2 uebayasi 1989 1.1.2.2 uebayasi init_qset_cntxt(q, id); 1990 1.1.2.2 uebayasi 1991 1.1.2.2 uebayasi if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), 1992 1.1.2.2 uebayasi sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, 1993 1.1.2.2 uebayasi &q->fl[0].desc, &q->fl[0].sdesc, 1994 1.1.2.2 uebayasi &q->fl[0].desc_tag, &q->fl[0].desc_map, 1995 1.1.2.2 uebayasi sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { 1996 1.1.2.2 uebayasi goto err; 1997 1.1.2.2 uebayasi } 1998 1.1.2.2 uebayasi 1999 1.1.2.2 uebayasi if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), 2000 1.1.2.2 uebayasi sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, 2001 1.1.2.2 uebayasi &q->fl[1].desc, &q->fl[1].sdesc, 2002 1.1.2.2 uebayasi &q->fl[1].desc_tag, &q->fl[1].desc_map, 2003 1.1.2.2 uebayasi sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { 2004 1.1.2.2 uebayasi goto err; 2005 1.1.2.2 uebayasi } 2006 1.1.2.2 uebayasi 2007 1.1.2.2 uebayasi if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, 2008 1.1.2.2 uebayasi &q->rspq.phys_addr, &q->rspq.desc, NULL, 2009 1.1.2.2 uebayasi &q->rspq.desc_tag, &q->rspq.desc_map, 2010 1.1.2.2 uebayasi NULL, NULL)) != 0) { 2011 1.1.2.2 uebayasi goto err; 2012 1.1.2.2 uebayasi } 2013 1.1.2.2 uebayasi 2014 1.1.2.2 uebayasi for (i = 0; i < ntxq; ++i) { 2015 1.1.2.2 uebayasi /* 2016 1.1.2.2 uebayasi * The control queue always uses immediate data so does not 2017 1.1.2.2 uebayasi * need to keep track of any mbufs. 2018 1.1.2.2 uebayasi * XXX Placeholder for future TOE support. 2019 1.1.2.2 uebayasi */ 2020 1.1.2.2 uebayasi size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc); 2021 1.1.2.2 uebayasi 2022 1.1.2.2 uebayasi if ((ret = alloc_ring(sc, p->txq_size[i], 2023 1.1.2.2 uebayasi sizeof(struct tx_desc), sz, 2024 1.1.2.2 uebayasi &q->txq[i].phys_addr, &q->txq[i].desc, 2025 1.1.2.2 uebayasi &q->txq[i].sdesc, &q->txq[i].desc_tag, 2026 1.1.2.2 uebayasi &q->txq[i].desc_map, 2027 1.1.2.2 uebayasi sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { 2028 1.1.2.2 uebayasi goto err; 2029 1.1.2.2 uebayasi } 2030 1.1.2.2 uebayasi mbufq_init(&q->txq[i].sendq); 2031 1.1.2.2 uebayasi q->txq[i].gen = 1; 2032 1.1.2.2 uebayasi q->txq[i].size = p->txq_size[i]; 2033 1.1.2.2 uebayasi snprintf(q->txq[i].lockbuf, TXQ_NAME_LEN, "t3 txq lock %d:%d:%d", 2034 1.1.2.2 uebayasi 0, irq_vec_idx, i); 2035 1.1.2.2 uebayasi MTX_INIT(&q->txq[i].lock, q->txq[i].lockbuf, NULL, MTX_DEF); 2036 1.1.2.2 uebayasi } 2037 1.1.2.2 uebayasi 2038 1.1.2.2 uebayasi q->txq[TXQ_ETH].port = pi; 2039 1.1.2.2 uebayasi 2040 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qresume_task.name = "restart_offloadq"; 2041 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qresume_task.func = restart_offloadq; 2042 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qresume_task.context = q; 2043 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &q->txq[TXQ_OFLD].qresume_task, NULL, "cxgb_make_task"); 2044 1.1.2.2 uebayasi 2045 1.1.2.2 uebayasi q->txq[TXQ_CTRL].qresume_task.name = "restart_ctrlq"; 2046 1.1.2.2 uebayasi q->txq[TXQ_CTRL].qresume_task.func = restart_ctrlq; 2047 1.1.2.2 uebayasi q->txq[TXQ_CTRL].qresume_task.context = q; 2048 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &q->txq[TXQ_CTRL].qresume_task, NULL, "cxgb_make_task"); 2049 1.1.2.2 uebayasi 2050 1.1.2.2 uebayasi q->txq[TXQ_ETH].qreclaim_task.name = "sge_txq_reclaim_handler"; 2051 1.1.2.2 uebayasi q->txq[TXQ_ETH].qreclaim_task.func = sge_txq_reclaim_handler; 2052 1.1.2.2 uebayasi q->txq[TXQ_ETH].qreclaim_task.context = &q->txq[TXQ_ETH]; 2053 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &q->txq[TXQ_ETH].qreclaim_task, NULL, "cxgb_make_task"); 2054 1.1.2.2 uebayasi 2055 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qreclaim_task.name = "sge_txq_reclaim_handler"; 2056 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qreclaim_task.func = sge_txq_reclaim_handler; 2057 1.1.2.2 uebayasi q->txq[TXQ_OFLD].qreclaim_task.context = &q->txq[TXQ_OFLD]; 2058 1.1.2.2 uebayasi kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &q->txq[TXQ_OFLD].qreclaim_task, NULL, "cxgb_make_task"); 2059 1.1.2.2 uebayasi 2060 1.1.2.2 uebayasi q->fl[0].gen = q->fl[1].gen = 1; 2061 1.1.2.2 uebayasi q->fl[0].size = p->fl_size; 2062 1.1.2.2 uebayasi q->fl[1].size = p->jumbo_size; 2063 1.1.2.2 uebayasi 2064 1.1.2.2 uebayasi q->rspq.gen = 1; 2065 1.1.2.2 uebayasi q->rspq.cidx = 0; 2066 1.1.2.2 uebayasi q->rspq.size = p->rspq_size; 2067 1.1.2.2 uebayasi 2068 1.1.2.2 uebayasi q->txq[TXQ_ETH].stop_thres = nports * 2069 1.1.2.2 uebayasi flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); 2070 1.1.2.2 uebayasi 2071 1.1.2.2 uebayasi q->fl[0].buf_size = MCLBYTES; 2072 1.1.2.2 uebayasi q->fl[1].buf_size = MJUMPAGESIZE; 2073 1.1.2.2 uebayasi 2074 1.1.2.2 uebayasi q->lro.enabled = lro_default; 2075 1.1.2.2 uebayasi 2076 1.1.2.2 uebayasi mtx_lock(&sc->sge.reg_lock); 2077 1.1.2.2 uebayasi ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, 2078 1.1.2.2 uebayasi q->rspq.phys_addr, q->rspq.size, 2079 1.1.2.2 uebayasi q->fl[0].buf_size, 1, 0); 2080 1.1.2.2 uebayasi if (ret) { 2081 1.1.2.2 uebayasi printf("error %d from t3_sge_init_rspcntxt\n", ret); 2082 1.1.2.2 uebayasi goto err_unlock; 2083 1.1.2.2 uebayasi } 2084 1.1.2.2 uebayasi 2085 1.1.2.2 uebayasi for (i = 0; i < SGE_RXQ_PER_SET; ++i) { 2086 1.1.2.2 uebayasi ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, 2087 1.1.2.2 uebayasi q->fl[i].phys_addr, q->fl[i].size, 2088 1.1.2.2 uebayasi q->fl[i].buf_size, p->cong_thres, 1, 2089 1.1.2.2 uebayasi 0); 2090 1.1.2.2 uebayasi if (ret) { 2091 1.1.2.2 uebayasi printf("error %d from t3_sge_init_flcntxt for index i=%d\n", ret, i); 2092 1.1.2.2 uebayasi goto err_unlock; 2093 1.1.2.2 uebayasi } 2094 1.1.2.2 uebayasi } 2095 1.1.2.2 uebayasi 2096 1.1.2.2 uebayasi ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, 2097 1.1.2.2 uebayasi SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, 2098 1.1.2.2 uebayasi q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, 2099 1.1.2.2 uebayasi 1, 0); 2100 1.1.2.2 uebayasi if (ret) { 2101 1.1.2.2 uebayasi printf("error %d from t3_sge_init_ecntxt\n", ret); 2102 1.1.2.2 uebayasi goto err_unlock; 2103 1.1.2.2 uebayasi } 2104 1.1.2.2 uebayasi 2105 1.1.2.2 uebayasi if (ntxq > 1) { 2106 1.1.2.2 uebayasi ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, 2107 1.1.2.2 uebayasi USE_GTS, SGE_CNTXT_OFLD, id, 2108 1.1.2.2 uebayasi q->txq[TXQ_OFLD].phys_addr, 2109 1.1.2.2 uebayasi q->txq[TXQ_OFLD].size, 0, 1, 0); 2110 1.1.2.2 uebayasi if (ret) { 2111 1.1.2.2 uebayasi printf("error %d from t3_sge_init_ecntxt\n", ret); 2112 1.1.2.2 uebayasi goto err_unlock; 2113 1.1.2.2 uebayasi } 2114 1.1.2.2 uebayasi } 2115 1.1.2.2 uebayasi 2116 1.1.2.2 uebayasi if (ntxq > 2) { 2117 1.1.2.2 uebayasi ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, 2118 1.1.2.2 uebayasi SGE_CNTXT_CTRL, id, 2119 1.1.2.2 uebayasi q->txq[TXQ_CTRL].phys_addr, 2120 1.1.2.2 uebayasi q->txq[TXQ_CTRL].size, 2121 1.1.2.2 uebayasi q->txq[TXQ_CTRL].token, 1, 0); 2122 1.1.2.2 uebayasi if (ret) { 2123 1.1.2.2 uebayasi printf("error %d from t3_sge_init_ecntxt\n", ret); 2124 1.1.2.2 uebayasi goto err_unlock; 2125 1.1.2.2 uebayasi } 2126 1.1.2.2 uebayasi } 2127 1.1.2.2 uebayasi 2128 1.1.2.2 uebayasi snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", 2129 1.1.2.2 uebayasi 0, irq_vec_idx); 2130 1.1.2.2 uebayasi MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); 2131 1.1.2.2 uebayasi 2132 1.1.2.2 uebayasi mtx_unlock(&sc->sge.reg_lock); 2133 1.1.2.2 uebayasi t3_update_qset_coalesce(q, p); 2134 1.1.2.2 uebayasi q->port = pi; 2135 1.1.2.2 uebayasi 2136 1.1.2.2 uebayasi refill_fl(sc, &q->fl[0], q->fl[0].size); 2137 1.1.2.2 uebayasi refill_fl(sc, &q->fl[1], q->fl[1].size); 2138 1.1.2.2 uebayasi refill_rspq(sc, &q->rspq, q->rspq.size - 1); 2139 1.1.2.2 uebayasi 2140 1.1.2.2 uebayasi t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | 2141 1.1.2.2 uebayasi V_NEWTIMER(q->rspq.holdoff_tmr)); 2142 1.1.2.2 uebayasi 2143 1.1.2.2 uebayasi return (0); 2144 1.1.2.2 uebayasi 2145 1.1.2.2 uebayasi err_unlock: 2146 1.1.2.2 uebayasi mtx_unlock(&sc->sge.reg_lock); 2147 1.1.2.2 uebayasi err: 2148 1.1.2.2 uebayasi t3_free_qset(sc, q); 2149 1.1.2.2 uebayasi 2150 1.1.2.2 uebayasi return (ret); 2151 1.1.2.2 uebayasi } 2152 1.1.2.2 uebayasi 2153 1.1.2.2 uebayasi void 2154 1.1.2.2 uebayasi t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad) 2155 1.1.2.2 uebayasi { 2156 1.1.2.2 uebayasi struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad); 2157 1.1.2.2 uebayasi struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; 2158 1.1.2.2 uebayasi struct ifnet *ifp = pi->ifp; 2159 1.1.2.2 uebayasi 2160 1.1.2.2 uebayasi DPRINTF("rx_eth m=%p m->m_data=%p p->iff=%d\n", m, mtod(m, uint8_t *), cpl->iff); 2161 1.1.2.2 uebayasi 2162 1.1.2.2 uebayasi /* 2163 1.1.2.2 uebayasi * XXX need to add VLAN support for 6.x 2164 1.1.2.2 uebayasi */ 2165 1.1.2.2 uebayasi #ifdef VLAN_SUPPORTED 2166 1.1.2.2 uebayasi if (__predict_false(cpl->vlan_valid)) { 2167 1.1.2.2 uebayasi m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); 2168 1.1.2.2 uebayasi m->m_flags |= M_VLANTAG; 2169 1.1.2.2 uebayasi } 2170 1.1.2.2 uebayasi #endif 2171 1.1.2.2 uebayasi 2172 1.1.2.2 uebayasi m->m_pkthdr.rcvif = ifp; 2173 1.1.2.2 uebayasi m_explode(m); 2174 1.1.2.2 uebayasi /* 2175 1.1.2.2 uebayasi * adjust after conversion to mbuf chain 2176 1.1.2.2 uebayasi */ 2177 1.1.2.2 uebayasi m_adj(m, sizeof(*cpl) + ethpad); 2178 1.1.2.2 uebayasi 2179 1.1.2.2 uebayasi (*ifp->if_input)(ifp, m); 2180 1.1.2.2 uebayasi } 2181 1.1.2.2 uebayasi 2182 1.1.2.2 uebayasi /** 2183 1.1.2.2 uebayasi * get_packet - return the next ingress packet buffer from a free list 2184 1.1.2.2 uebayasi * @adap: the adapter that received the packet 2185 1.1.2.2 uebayasi * @drop_thres: # of remaining buffers before we start dropping packets 2186 1.1.2.2 uebayasi * @qs: the qset that the SGE free list holding the packet belongs to 2187 1.1.2.2 uebayasi * @mh: the mbuf header, contains a pointer to the head and tail of the mbuf chain 2188 1.1.2.2 uebayasi * @r: response descriptor 2189 1.1.2.2 uebayasi * 2190 1.1.2.2 uebayasi * Get the next packet from a free list and complete setup of the 2191 1.1.2.2 uebayasi * sk_buff. If the packet is small we make a copy and recycle the 2192 1.1.2.2 uebayasi * original buffer, otherwise we use the original buffer itself. If a 2193 1.1.2.2 uebayasi * positive drop threshold is supplied packets are dropped and their 2194 1.1.2.2 uebayasi * buffers recycled if (a) the number of remaining buffers is under the 2195 1.1.2.2 uebayasi * threshold and the packet is too big to copy, or (b) the packet should 2196 1.1.2.2 uebayasi * be copied but there is no memory for the copy. 2197 1.1.2.2 uebayasi */ 2198 1.1.2.2 uebayasi #ifdef DISABLE_MBUF_IOVEC 2199 1.1.2.2 uebayasi 2200 1.1.2.2 uebayasi static int 2201 1.1.2.2 uebayasi get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2202 1.1.2.2 uebayasi struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m) 2203 1.1.2.2 uebayasi { 2204 1.1.2.2 uebayasi 2205 1.1.2.2 uebayasi unsigned int len_cq = ntohl(r->len_cq); 2206 1.1.2.2 uebayasi struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2207 1.1.2.2 uebayasi struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2208 1.1.2.2 uebayasi uint32_t len = G_RSPD_LEN(len_cq); 2209 1.1.2.2 uebayasi uint32_t flags = ntohl(r->flags); 2210 1.1.2.2 uebayasi uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2211 1.1.2.2 uebayasi int ret = 0; 2212 1.1.2.2 uebayasi 2213 1.1.2.2 uebayasi prefetch(sd->cl); 2214 1.1.2.2 uebayasi 2215 1.1.2.2 uebayasi fl->credits--; 2216 1.1.2.2 uebayasi bus_dmamap_sync(fl->entry_tag, sd->map, 0, len, BUS_DMASYNC_POSTREAD); 2217 1.1.2.2 uebayasi bus_dmamap_unload(fl->entry_tag, sd->map); 2218 1.1.2.2 uebayasi 2219 1.1.2.2 uebayasi m->m_len = len; 2220 1.1.2.2 uebayasi m_cljset(m, sd->cl, fl->type); 2221 1.1.2.2 uebayasi 2222 1.1.2.2 uebayasi switch(sopeop) { 2223 1.1.2.2 uebayasi case RSPQ_SOP_EOP: 2224 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2225 1.1.2.2 uebayasi mh->mh_head = mh->mh_tail = m; 2226 1.1.2.2 uebayasi m->m_pkthdr.len = len; 2227 1.1.2.2 uebayasi m->m_flags |= M_PKTHDR; 2228 1.1.2.2 uebayasi ret = 1; 2229 1.1.2.2 uebayasi break; 2230 1.1.2.2 uebayasi case RSPQ_NSOP_NEOP: 2231 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2232 1.1.2.2 uebayasi m->m_flags &= ~M_PKTHDR; 2233 1.1.2.2 uebayasi if (mh->mh_tail == NULL) { 2234 1.1.2.2 uebayasi if (cxgb_debug) 2235 1.1.2.2 uebayasi printf("discarding intermediate descriptor entry\n"); 2236 1.1.2.2 uebayasi m_freem(m); 2237 1.1.2.2 uebayasi break; 2238 1.1.2.2 uebayasi } 2239 1.1.2.2 uebayasi mh->mh_tail->m_next = m; 2240 1.1.2.2 uebayasi mh->mh_tail = m; 2241 1.1.2.2 uebayasi mh->mh_head->m_pkthdr.len += len; 2242 1.1.2.2 uebayasi ret = 0; 2243 1.1.2.2 uebayasi break; 2244 1.1.2.2 uebayasi case RSPQ_SOP: 2245 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2246 1.1.2.2 uebayasi m->m_pkthdr.len = len; 2247 1.1.2.2 uebayasi mh->mh_head = mh->mh_tail = m; 2248 1.1.2.2 uebayasi m->m_flags |= M_PKTHDR; 2249 1.1.2.2 uebayasi ret = 0; 2250 1.1.2.2 uebayasi break; 2251 1.1.2.2 uebayasi case RSPQ_EOP: 2252 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2253 1.1.2.2 uebayasi m->m_flags &= ~M_PKTHDR; 2254 1.1.2.2 uebayasi mh->mh_head->m_pkthdr.len += len; 2255 1.1.2.2 uebayasi mh->mh_tail->m_next = m; 2256 1.1.2.2 uebayasi mh->mh_tail = m; 2257 1.1.2.2 uebayasi ret = 1; 2258 1.1.2.2 uebayasi break; 2259 1.1.2.2 uebayasi } 2260 1.1.2.2 uebayasi if (++fl->cidx == fl->size) 2261 1.1.2.2 uebayasi fl->cidx = 0; 2262 1.1.2.2 uebayasi 2263 1.1.2.2 uebayasi return (ret); 2264 1.1.2.2 uebayasi } 2265 1.1.2.2 uebayasi 2266 1.1.2.2 uebayasi #else 2267 1.1.2.2 uebayasi static int 2268 1.1.2.2 uebayasi get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, 2269 1.1.2.2 uebayasi struct mbuf *m, struct rsp_desc *r) 2270 1.1.2.2 uebayasi { 2271 1.1.2.2 uebayasi 2272 1.1.2.2 uebayasi unsigned int len_cq = ntohl(r->len_cq); 2273 1.1.2.2 uebayasi struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; 2274 1.1.2.2 uebayasi struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; 2275 1.1.2.2 uebayasi uint32_t len = G_RSPD_LEN(len_cq); 2276 1.1.2.2 uebayasi uint32_t flags = ntohl(r->flags); 2277 1.1.2.2 uebayasi uint8_t sopeop = G_RSPD_SOP_EOP(flags); 2278 1.1.2.2 uebayasi void *cl; 2279 1.1.2.2 uebayasi int ret = 0; 2280 1.1.2.2 uebayasi 2281 1.1.2.2 uebayasi prefetch(sd->cl); 2282 1.1.2.2 uebayasi 2283 1.1.2.2 uebayasi fl->credits--; 2284 1.1.2.2 uebayasi bus_dmamap_sync(fl->entry_tag, sd->map, 0, len, BUS_DMASYNC_POSTREAD); 2285 1.1.2.2 uebayasi 2286 1.1.2.2 uebayasi if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { 2287 1.1.2.2 uebayasi cl = mtod(m, void *); 2288 1.1.2.2 uebayasi memcpy(cl, sd->cl, len); 2289 1.1.2.2 uebayasi recycle_rx_buf(adap, fl, fl->cidx); 2290 1.1.2.2 uebayasi } else { 2291 1.1.2.2 uebayasi cl = sd->cl; 2292 1.1.2.2 uebayasi bus_dmamap_unload(fl->entry_tag, sd->map); 2293 1.1.2.2 uebayasi } 2294 1.1.2.2 uebayasi switch(sopeop) { 2295 1.1.2.2 uebayasi case RSPQ_SOP_EOP: 2296 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); 2297 1.1.2.2 uebayasi m->m_len = m->m_pkthdr.len = len; 2298 1.1.2.2 uebayasi if (cl == sd->cl) 2299 1.1.2.2 uebayasi m_cljset(m, cl, fl->type); 2300 1.1.2.2 uebayasi ret = 1; 2301 1.1.2.2 uebayasi goto done; 2302 1.1.2.2 uebayasi break; 2303 1.1.2.2 uebayasi case RSPQ_NSOP_NEOP: 2304 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); 2305 1.1.2.2 uebayasi ret = 0; 2306 1.1.2.2 uebayasi break; 2307 1.1.2.2 uebayasi case RSPQ_SOP: 2308 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); 2309 1.1.2.2 uebayasi m_iovinit(m); 2310 1.1.2.2 uebayasi ret = 0; 2311 1.1.2.2 uebayasi break; 2312 1.1.2.2 uebayasi case RSPQ_EOP: 2313 1.1.2.2 uebayasi DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); 2314 1.1.2.2 uebayasi ret = 1; 2315 1.1.2.2 uebayasi break; 2316 1.1.2.2 uebayasi } 2317 1.1.2.2 uebayasi m_iovappend(m, cl, fl->buf_size, len, 0); 2318 1.1.2.2 uebayasi 2319 1.1.2.2 uebayasi done: 2320 1.1.2.2 uebayasi if (++fl->cidx == fl->size) 2321 1.1.2.2 uebayasi fl->cidx = 0; 2322 1.1.2.2 uebayasi 2323 1.1.2.2 uebayasi return (ret); 2324 1.1.2.2 uebayasi } 2325 1.1.2.2 uebayasi #endif 2326 1.1.2.2 uebayasi /** 2327 1.1.2.2 uebayasi * handle_rsp_cntrl_info - handles control information in a response 2328 1.1.2.2 uebayasi * @qs: the queue set corresponding to the response 2329 1.1.2.2 uebayasi * @flags: the response control flags 2330 1.1.2.2 uebayasi * 2331 1.1.2.2 uebayasi * Handles the control information of an SGE response, such as GTS 2332 1.1.2.2 uebayasi * indications and completion credits for the queue set's Tx queues. 2333 1.1.2.2 uebayasi * HW coalesces credits, we don't do any extra SW coalescing. 2334 1.1.2.2 uebayasi */ 2335 1.1.2.2 uebayasi static __inline void 2336 1.1.2.2 uebayasi handle_rsp_cntrl_info(struct sge_qset *qs, uint32_t flags) 2337 1.1.2.2 uebayasi { 2338 1.1.2.2 uebayasi unsigned int credits; 2339 1.1.2.2 uebayasi 2340 1.1.2.2 uebayasi #if USE_GTS 2341 1.1.2.2 uebayasi if (flags & F_RSPD_TXQ0_GTS) 2342 1.1.2.2 uebayasi clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); 2343 1.1.2.2 uebayasi #endif 2344 1.1.2.2 uebayasi credits = G_RSPD_TXQ0_CR(flags); 2345 1.1.2.2 uebayasi if (credits) { 2346 1.1.2.2 uebayasi qs->txq[TXQ_ETH].processed += credits; 2347 1.1.2.2 uebayasi if (desc_reclaimable(&qs->txq[TXQ_ETH]) > TX_START_MAX_DESC) 2348 1.1.2.2 uebayasi workqueue_enqueue(qs->port->timer_reclaim_task.wq, 2349 1.1.2.2 uebayasi &qs->port->timer_reclaim_task.w, NULL); 2350 1.1.2.2 uebayasi } 2351 1.1.2.2 uebayasi 2352 1.1.2.2 uebayasi credits = G_RSPD_TXQ2_CR(flags); 2353 1.1.2.2 uebayasi if (credits) 2354 1.1.2.2 uebayasi qs->txq[TXQ_CTRL].processed += credits; 2355 1.1.2.2 uebayasi 2356 1.1.2.2 uebayasi # if USE_GTS 2357 1.1.2.2 uebayasi if (flags & F_RSPD_TXQ1_GTS) 2358 1.1.2.2 uebayasi clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); 2359 1.1.2.2 uebayasi # endif 2360 1.1.2.2 uebayasi credits = G_RSPD_TXQ1_CR(flags); 2361 1.1.2.2 uebayasi if (credits) 2362 1.1.2.2 uebayasi qs->txq[TXQ_OFLD].processed += credits; 2363 1.1.2.2 uebayasi } 2364 1.1.2.2 uebayasi 2365 1.1.2.2 uebayasi static void 2366 1.1.2.2 uebayasi check_ring_db(adapter_t *adap, struct sge_qset *qs, 2367 1.1.2.2 uebayasi unsigned int sleeping) 2368 1.1.2.2 uebayasi { 2369 1.1.2.2 uebayasi ; 2370 1.1.2.2 uebayasi } 2371 1.1.2.2 uebayasi 2372 1.1.2.2 uebayasi /** 2373 1.1.2.2 uebayasi * process_responses - process responses from an SGE response queue 2374 1.1.2.2 uebayasi * @adap: the adapter 2375 1.1.2.2 uebayasi * @qs: the queue set to which the response queue belongs 2376 1.1.2.2 uebayasi * @budget: how many responses can be processed in this round 2377 1.1.2.2 uebayasi * 2378 1.1.2.2 uebayasi * Process responses from an SGE response queue up to the supplied budget. 2379 1.1.2.2 uebayasi * Responses include received packets as well as credits and other events 2380 1.1.2.2 uebayasi * for the queues that belong to the response queue's queue set. 2381 1.1.2.2 uebayasi * A negative budget is effectively unlimited. 2382 1.1.2.2 uebayasi * 2383 1.1.2.2 uebayasi * Additionally choose the interrupt holdoff time for the next interrupt 2384 1.1.2.2 uebayasi * on this queue. If the system is under memory shortage use a fairly 2385 1.1.2.2 uebayasi * long delay to help recovery. 2386 1.1.2.2 uebayasi */ 2387 1.1.2.2 uebayasi static int 2388 1.1.2.2 uebayasi process_responses(adapter_t *adap, struct sge_qset *qs, int budget) 2389 1.1.2.2 uebayasi { 2390 1.1.2.2 uebayasi struct sge_rspq *rspq = &qs->rspq; 2391 1.1.2.2 uebayasi struct rsp_desc *r = &rspq->desc[rspq->cidx]; 2392 1.1.2.2 uebayasi int budget_left = budget; 2393 1.1.2.2 uebayasi unsigned int sleeping = 0; 2394 1.1.2.2 uebayasi int lro = qs->lro.enabled; 2395 1.1.2.2 uebayasi #ifdef DEBUG 2396 1.1.2.2 uebayasi static int last_holdoff = 0; 2397 1.1.2.2 uebayasi if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { 2398 1.1.2.2 uebayasi printf("next_holdoff=%d\n", rspq->holdoff_tmr); 2399 1.1.2.2 uebayasi last_holdoff = rspq->holdoff_tmr; 2400 1.1.2.2 uebayasi } 2401 1.1.2.2 uebayasi #endif 2402 1.1.2.2 uebayasi rspq->next_holdoff = rspq->holdoff_tmr; 2403 1.1.2.2 uebayasi 2404 1.1.2.2 uebayasi while (__predict_true(budget_left && is_new_response(r, rspq))) { 2405 1.1.2.2 uebayasi int eth, eop = 0, ethpad = 0; 2406 1.1.2.2 uebayasi uint32_t flags = ntohl(r->flags); 2407 1.1.2.2 uebayasi uint32_t rss_csum = *(const uint32_t *)r; 2408 1.1.2.2 uebayasi uint32_t rss_hash = r->rss_hdr.rss_hash_val; 2409 1.1.2.2 uebayasi 2410 1.1.2.2 uebayasi eth = (r->rss_hdr.opcode == CPL_RX_PKT); 2411 1.1.2.2 uebayasi 2412 1.1.2.2 uebayasi if (__predict_false(flags & F_RSPD_ASYNC_NOTIF)) { 2413 1.1.2.2 uebayasi /* XXX */ 2414 1.1.2.2 uebayasi } else if (flags & F_RSPD_IMM_DATA_VALID) { 2415 1.1.2.2 uebayasi #ifdef DISABLE_MBUF_IOVEC 2416 1.1.2.2 uebayasi if (cxgb_debug) 2417 1.1.2.2 uebayasi printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx); 2418 1.1.2.2 uebayasi 2419 1.1.2.2 uebayasi if(get_imm_packet(adap, r, &rspq->rspq_mh) == 0) { 2420 1.1.2.2 uebayasi rspq->next_holdoff = NOMEM_INTR_DELAY; 2421 1.1.2.2 uebayasi budget_left--; 2422 1.1.2.2 uebayasi break; 2423 1.1.2.2 uebayasi } else { 2424 1.1.2.2 uebayasi eop = 1; 2425 1.1.2.2 uebayasi } 2426 1.1.2.2 uebayasi #else 2427 1.1.2.2 uebayasi struct mbuf *m = NULL; 2428 1.1.2.2 uebayasi 2429 1.1.2.2 uebayasi if (rspq->rspq_mbuf == NULL) 2430 1.1.2.2 uebayasi rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA); 2431 1.1.2.2 uebayasi else 2432 1.1.2.2 uebayasi m = m_gethdr(M_DONTWAIT, MT_DATA); 2433 1.1.2.2 uebayasi 2434 1.1.2.2 uebayasi /* 2435 1.1.2.2 uebayasi * XXX revisit me 2436 1.1.2.2 uebayasi */ 2437 1.1.2.2 uebayasi if (rspq->rspq_mbuf == NULL && m == NULL) { 2438 1.1.2.2 uebayasi rspq->next_holdoff = NOMEM_INTR_DELAY; 2439 1.1.2.2 uebayasi budget_left--; 2440 1.1.2.2 uebayasi break; 2441 1.1.2.2 uebayasi } 2442 1.1.2.2 uebayasi if (get_imm_packet(adap, r, rspq->rspq_mbuf, m, flags)) 2443 1.1.2.2 uebayasi goto skip; 2444 1.1.2.2 uebayasi eop = 1; 2445 1.1.2.2 uebayasi #endif 2446 1.1.2.2 uebayasi rspq->imm_data++; 2447 1.1.2.2 uebayasi } else if (r->len_cq) { 2448 1.1.2.2 uebayasi int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; 2449 1.1.2.2 uebayasi 2450 1.1.2.2 uebayasi #ifdef DISABLE_MBUF_IOVEC 2451 1.1.2.2 uebayasi struct mbuf *m; 2452 1.1.2.2 uebayasi m = m_gethdr(M_NOWAIT, MT_DATA); 2453 1.1.2.2 uebayasi 2454 1.1.2.2 uebayasi if (m == NULL) { 2455 1.1.2.2 uebayasi log(LOG_WARNING, "failed to get mbuf for packet\n"); 2456 1.1.2.2 uebayasi break; 2457 1.1.2.2 uebayasi } 2458 1.1.2.2 uebayasi 2459 1.1.2.2 uebayasi eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m); 2460 1.1.2.2 uebayasi #else 2461 1.1.2.2 uebayasi if (rspq->rspq_mbuf == NULL) 2462 1.1.2.2 uebayasi rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA); 2463 1.1.2.2 uebayasi if (rspq->rspq_mbuf == NULL) { 2464 1.1.2.2 uebayasi log(LOG_WARNING, "failed to get mbuf for packet\n"); 2465 1.1.2.2 uebayasi break; 2466 1.1.2.2 uebayasi } 2467 1.1.2.2 uebayasi eop = get_packet(adap, drop_thresh, qs, rspq->rspq_mbuf, r); 2468 1.1.2.2 uebayasi #endif 2469 1.1.2.2 uebayasi ethpad = 2; 2470 1.1.2.2 uebayasi } else { 2471 1.1.2.2 uebayasi DPRINTF("pure response\n"); 2472 1.1.2.2 uebayasi rspq->pure_rsps++; 2473 1.1.2.2 uebayasi } 2474 1.1.2.2 uebayasi 2475 1.1.2.2 uebayasi if (flags & RSPD_CTRL_MASK) { 2476 1.1.2.2 uebayasi sleeping |= flags & RSPD_GTS_MASK; 2477 1.1.2.2 uebayasi handle_rsp_cntrl_info(qs, flags); 2478 1.1.2.2 uebayasi } 2479 1.1.2.2 uebayasi #ifndef DISABLE_MBUF_IOVEC 2480 1.1.2.2 uebayasi skip: 2481 1.1.2.2 uebayasi #endif 2482 1.1.2.2 uebayasi r++; 2483 1.1.2.2 uebayasi if (__predict_false(++rspq->cidx == rspq->size)) { 2484 1.1.2.2 uebayasi rspq->cidx = 0; 2485 1.1.2.2 uebayasi rspq->gen ^= 1; 2486 1.1.2.2 uebayasi r = rspq->desc; 2487 1.1.2.2 uebayasi } 2488 1.1.2.2 uebayasi 2489 1.1.2.2 uebayasi prefetch(r); 2490 1.1.2.2 uebayasi if (++rspq->credits >= (rspq->size / 4)) { 2491 1.1.2.2 uebayasi refill_rspq(adap, rspq, rspq->credits); 2492 1.1.2.2 uebayasi rspq->credits = 0; 2493 1.1.2.2 uebayasi } 2494 1.1.2.2 uebayasi 2495 1.1.2.2 uebayasi if (eop) { 2496 1.1.2.2 uebayasi prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *)); 2497 1.1.2.2 uebayasi prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES); 2498 1.1.2.2 uebayasi 2499 1.1.2.2 uebayasi if (eth) { 2500 1.1.2.2 uebayasi t3_rx_eth_lro(adap, rspq, rspq->rspq_mh.mh_head, ethpad, 2501 1.1.2.2 uebayasi rss_hash, rss_csum, lro); 2502 1.1.2.2 uebayasi 2503 1.1.2.2 uebayasi rspq->rspq_mh.mh_head = NULL; 2504 1.1.2.2 uebayasi } else { 2505 1.1.2.2 uebayasi rspq->rspq_mh.mh_head->m_pkthdr.csum_data = rss_csum; 2506 1.1.2.2 uebayasi /* 2507 1.1.2.2 uebayasi * XXX size mismatch 2508 1.1.2.2 uebayasi */ 2509 1.1.2.2 uebayasi m_set_priority(rspq->rspq_mh.mh_head, rss_hash); 2510 1.1.2.2 uebayasi } 2511 1.1.2.2 uebayasi __refill_fl(adap, &qs->fl[0]); 2512 1.1.2.2 uebayasi __refill_fl(adap, &qs->fl[1]); 2513 1.1.2.2 uebayasi 2514 1.1.2.2 uebayasi } 2515 1.1.2.2 uebayasi --budget_left; 2516 1.1.2.2 uebayasi } 2517 1.1.2.2 uebayasi 2518 1.1.2.2 uebayasi t3_lro_flush(adap, qs, &qs->lro); 2519 1.1.2.2 uebayasi 2520 1.1.2.2 uebayasi if (sleeping) 2521 1.1.2.2 uebayasi check_ring_db(adap, qs, sleeping); 2522 1.1.2.2 uebayasi 2523 1.1.2.2 uebayasi smp_mb(); /* commit Tx queue processed updates */ 2524 1.1.2.2 uebayasi if (__predict_false(qs->txq_stopped != 0)) 2525 1.1.2.2 uebayasi restart_tx(qs); 2526 1.1.2.2 uebayasi 2527 1.1.2.2 uebayasi budget -= budget_left; 2528 1.1.2.2 uebayasi return (budget); 2529 1.1.2.2 uebayasi } 2530 1.1.2.2 uebayasi 2531 1.1.2.2 uebayasi /* 2532 1.1.2.2 uebayasi * A helper function that processes responses and issues GTS. 2533 1.1.2.2 uebayasi */ 2534 1.1.2.2 uebayasi static __inline int 2535 1.1.2.2 uebayasi process_responses_gts(adapter_t *adap, struct sge_rspq *rq) 2536 1.1.2.2 uebayasi { 2537 1.1.2.2 uebayasi int work; 2538 1.1.2.2 uebayasi static int last_holdoff = 0; 2539 1.1.2.2 uebayasi 2540 1.1.2.2 uebayasi work = process_responses(adap, rspq_to_qset(rq), -1); 2541 1.1.2.2 uebayasi 2542 1.1.2.2 uebayasi if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { 2543 1.1.2.2 uebayasi printf("next_holdoff=%d\n", rq->next_holdoff); 2544 1.1.2.2 uebayasi last_holdoff = rq->next_holdoff; 2545 1.1.2.2 uebayasi } 2546 1.1.2.2 uebayasi if (work) 2547 1.1.2.2 uebayasi t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | 2548 1.1.2.2 uebayasi V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); 2549 1.1.2.2 uebayasi return work; 2550 1.1.2.2 uebayasi } 2551 1.1.2.2 uebayasi 2552 1.1.2.2 uebayasi 2553 1.1.2.2 uebayasi /* 2554 1.1.2.2 uebayasi * Interrupt handler for legacy INTx interrupts for T3B-based cards. 2555 1.1.2.2 uebayasi * Handles data events from SGE response queues as well as error and other 2556 1.1.2.2 uebayasi * async events as they all use the same interrupt pin. We use one SGE 2557 1.1.2.2 uebayasi * response queue per port in this mode and protect all response queues with 2558 1.1.2.2 uebayasi * queue 0's lock. 2559 1.1.2.2 uebayasi */ 2560 1.1.2.2 uebayasi int 2561 1.1.2.2 uebayasi t3b_intr(void *data) 2562 1.1.2.2 uebayasi { 2563 1.1.2.2 uebayasi uint32_t i, map; 2564 1.1.2.2 uebayasi adapter_t *adap = data; 2565 1.1.2.2 uebayasi struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 2566 1.1.2.2 uebayasi 2567 1.1.2.2 uebayasi t3_write_reg(adap, A_PL_CLI, 0); 2568 1.1.2.2 uebayasi map = t3_read_reg(adap, A_SG_DATA_INTR); 2569 1.1.2.2 uebayasi 2570 1.1.2.2 uebayasi if (!map) 2571 1.1.2.2 uebayasi return (FALSE); 2572 1.1.2.2 uebayasi 2573 1.1.2.2 uebayasi if (__predict_false(map & F_ERRINTR)) 2574 1.1.2.2 uebayasi workqueue_enqueue(adap->slow_intr_task.wq, &adap->slow_intr_task.w, NULL); 2575 1.1.2.2 uebayasi 2576 1.1.2.2 uebayasi mtx_lock(&q0->lock); 2577 1.1.2.2 uebayasi for_each_port(adap, i) 2578 1.1.2.2 uebayasi if (map & (1 << i)) 2579 1.1.2.2 uebayasi process_responses_gts(adap, &adap->sge.qs[i].rspq); 2580 1.1.2.2 uebayasi mtx_unlock(&q0->lock); 2581 1.1.2.2 uebayasi 2582 1.1.2.2 uebayasi return (TRUE); 2583 1.1.2.2 uebayasi } 2584 1.1.2.2 uebayasi 2585 1.1.2.2 uebayasi /* 2586 1.1.2.2 uebayasi * The MSI interrupt handler. This needs to handle data events from SGE 2587 1.1.2.2 uebayasi * response queues as well as error and other async events as they all use 2588 1.1.2.2 uebayasi * the same MSI vector. We use one SGE response queue per port in this mode 2589 1.1.2.2 uebayasi * and protect all response queues with queue 0's lock. 2590 1.1.2.2 uebayasi */ 2591 1.1.2.2 uebayasi int 2592 1.1.2.2 uebayasi t3_intr_msi(void *data) 2593 1.1.2.2 uebayasi { 2594 1.1.2.2 uebayasi adapter_t *adap = data; 2595 1.1.2.2 uebayasi struct sge_rspq *q0 = &adap->sge.qs[0].rspq; 2596 1.1.2.2 uebayasi int i, new_packets = 0; 2597 1.1.2.2 uebayasi 2598 1.1.2.2 uebayasi mtx_lock(&q0->lock); 2599 1.1.2.2 uebayasi 2600 1.1.2.2 uebayasi for_each_port(adap, i) 2601 1.1.2.2 uebayasi if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) 2602 1.1.2.2 uebayasi new_packets = 1; 2603 1.1.2.2 uebayasi mtx_unlock(&q0->lock); 2604 1.1.2.2 uebayasi if (new_packets == 0) 2605 1.1.2.2 uebayasi workqueue_enqueue(adap->slow_intr_task.wq, &adap->slow_intr_task.w, NULL); 2606 1.1.2.2 uebayasi 2607 1.1.2.2 uebayasi return (TRUE); 2608 1.1.2.2 uebayasi } 2609 1.1.2.2 uebayasi 2610 1.1.2.2 uebayasi int 2611 1.1.2.2 uebayasi t3_intr_msix(void *data) 2612 1.1.2.2 uebayasi { 2613 1.1.2.2 uebayasi struct sge_qset *qs = data; 2614 1.1.2.2 uebayasi adapter_t *adap = qs->port->adapter; 2615 1.1.2.2 uebayasi struct sge_rspq *rspq = &qs->rspq; 2616 1.1.2.2 uebayasi 2617 1.1.2.2 uebayasi mtx_lock(&rspq->lock); 2618 1.1.2.2 uebayasi if (process_responses_gts(adap, rspq) == 0) 2619 1.1.2.2 uebayasi rspq->unhandled_irqs++; 2620 1.1.2.2 uebayasi mtx_unlock(&rspq->lock); 2621 1.1.2.2 uebayasi 2622 1.1.2.2 uebayasi return (TRUE); 2623 1.1.2.2 uebayasi } 2624 1.1.2.2 uebayasi 2625 1.1.2.2 uebayasi /** 2626 1.1.2.2 uebayasi * t3_get_desc - dump an SGE descriptor for debugging purposes 2627 1.1.2.2 uebayasi * @qs: the queue set 2628 1.1.2.2 uebayasi * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx) 2629 1.1.2.2 uebayasi * @idx: the descriptor index in the queue 2630 1.1.2.2 uebayasi * @data: where to dump the descriptor contents 2631 1.1.2.2 uebayasi * 2632 1.1.2.2 uebayasi * Dumps the contents of a HW descriptor of an SGE queue. Returns the 2633 1.1.2.2 uebayasi * size of the descriptor. 2634 1.1.2.2 uebayasi */ 2635 1.1.2.2 uebayasi int 2636 1.1.2.2 uebayasi t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx, 2637 1.1.2.2 uebayasi unsigned char *data) 2638 1.1.2.2 uebayasi { 2639 1.1.2.2 uebayasi if (qnum >= 6) 2640 1.1.2.2 uebayasi return (EINVAL); 2641 1.1.2.2 uebayasi 2642 1.1.2.2 uebayasi if (qnum < 3) { 2643 1.1.2.2 uebayasi if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) 2644 1.1.2.2 uebayasi return -EINVAL; 2645 1.1.2.2 uebayasi memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); 2646 1.1.2.2 uebayasi return sizeof(struct tx_desc); 2647 1.1.2.2 uebayasi } 2648 1.1.2.2 uebayasi 2649 1.1.2.2 uebayasi if (qnum == 3) { 2650 1.1.2.2 uebayasi if (!qs->rspq.desc || idx >= qs->rspq.size) 2651 1.1.2.2 uebayasi return (EINVAL); 2652 1.1.2.2 uebayasi memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); 2653 1.1.2.2 uebayasi return sizeof(struct rsp_desc); 2654 1.1.2.2 uebayasi } 2655 1.1.2.2 uebayasi 2656 1.1.2.2 uebayasi qnum -= 4; 2657 1.1.2.2 uebayasi if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) 2658 1.1.2.2 uebayasi return (EINVAL); 2659 1.1.2.2 uebayasi memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); 2660 1.1.2.2 uebayasi return sizeof(struct rx_desc); 2661 1.1.2.2 uebayasi } 2662
Indexes created Tue Oct 28 12:10:06 GMT 2025