1 2 /************************************************************************** 3 4 Copyright (c) 2007, Chelsio Inc. 5 All rights reserved. 6 7 Redistribution and use in source and binary forms, with or without 8 modification, are permitted provided that the following conditions are met: 9 10 1. Redistributions of source code must retain the above copyright notice, 11 this list of conditions and the following disclaimer. 12 13 2. Neither the name of the Chelsio Corporation nor the names of its 14 contributors may be used to endorse or promote products derived from 15 this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 POSSIBILITY OF SUCH DAMAGE. 28 29 30 ***************************************************************************/ 31 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: cxgb_offload.c,v 1.7 2021/08/08 20:57:09 andvar Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/kernel.h> 39 #include <sys/conf.h> 40 #include <sys/bus.h> 41 #include <sys/ioccom.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/sockio.h> 45 #include <sys/sysctl.h> 46 #include <sys/queue.h> 47 48 #ifdef CONFIG_DEFINED 49 #include <cxgb_include.h> 50 #else 51 #include "cxgb_include.h" 52 #endif 53 54 #include <net/route.h> 55 56 /* 57 * XXX 58 */ 59 #define LOG_NOTICE 2 60 #define BUG_ON(...) 61 #define VALIDATE_TID 0 62 63 64 TAILQ_HEAD(, cxgb_client) client_list; 65 TAILQ_HEAD(, toedev) ofld_dev_list; 66 TAILQ_HEAD(, adapter) adapter_list; 67 68 static struct mtx cxgb_db_lock; 69 static struct rwlock adapter_list_lock; 70 71 72 static const unsigned int MAX_ATIDS = 64 * 1024; 73 static const unsigned int ATID_BASE = 0x100000; 74 static int inited = 0; 75 76 static inline int 77 offload_activated(struct toedev *tdev) 78 { 79 struct adapter *adapter = tdev2adap(tdev); 80 81 return (isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)); 82 } 83 84 /** 85 * cxgb_register_client - register an offload client 86 * @client: the client 87 * 88 * Add the client to the client list, 89 * and call backs the client for each activated offload device 90 */ 91 void 92 cxgb_register_client(struct cxgb_client *client) 93 { 94 struct toedev *tdev; 95 96 mtx_lock(&cxgb_db_lock); 97 TAILQ_INSERT_TAIL(&client_list, client, client_entry); 98 99 if (client->add) { 100 TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { 101 if (offload_activated(tdev)) 102 client->add(tdev); 103 } 104 } 105 mtx_unlock(&cxgb_db_lock); 106 } 107 108 /** 109 * cxgb_unregister_client - unregister an offload client 110 * @client: the client 111 * 112 * Remove the client to the client list, 113 * and call backs the client for each activated offload device. 114 */ 115 void 116 cxgb_unregister_client(struct cxgb_client *client) 117 { 118 struct toedev *tdev; 119 120 mtx_lock(&cxgb_db_lock); 121 TAILQ_REMOVE(&client_list, client, client_entry); 122 123 if (client->remove) { 124 TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { 125 if (offload_activated(tdev)) 126 client->remove(tdev); 127 } 128 } 129 mtx_unlock(&cxgb_db_lock); 130 } 131 132 /** 133 * cxgb_add_clients - activate register clients for an offload device 134 * @tdev: the offload device 135 * 136 * Call backs all registered clients once a offload device is activated 137 */ 138 void 139 cxgb_add_clients(struct toedev *tdev) 140 { 141 struct cxgb_client *client; 142 143 mtx_lock(&cxgb_db_lock); 144 TAILQ_FOREACH(client, &client_list, client_entry) { 145 if (client->add) 146 client->add(tdev); 147 } 148 mtx_unlock(&cxgb_db_lock); 149 } 150 151 /** 152 * cxgb_remove_clients - activate register clients for an offload device 153 * @tdev: the offload device 154 * 155 * Call backs all registered clients once a offload device is deactivated 156 */ 157 void 158 cxgb_remove_clients(struct toedev *tdev) 159 { 160 struct cxgb_client *client; 161 162 mtx_lock(&cxgb_db_lock); 163 TAILQ_FOREACH(client, &client_list, client_entry) { 164 if (client->remove) 165 client->remove(tdev); 166 } 167 mtx_unlock(&cxgb_db_lock); 168 } 169 170 static int 171 is_offloading(struct ifnet *ifp) 172 { 173 struct adapter *adapter; 174 int port; 175 176 rw_rlock(&adapter_list_lock); 177 TAILQ_FOREACH(adapter, &adapter_list, adapter_entry) { 178 for_each_port(adapter, port) { 179 if (ifp == adapter->port[port].ifp) { 180 rw_runlock(&adapter_list_lock); 181 return 1; 182 } 183 } 184 } 185 rw_runlock(&adapter_list_lock); 186 return 0; 187 } 188 189 static struct ifnet * 190 get_iff_from_mac(adapter_t *adapter, const uint8_t *mac, unsigned int vlan) 191 { 192 #ifdef notyet 193 int i; 194 195 for_each_port(adapter, i) { 196 const struct vlan_group *grp; 197 const struct port_info *p = &adapter->port[i]; 198 struct ifnet *ifnet = p->ifp; 199 200 if (!memcmp(p->hw_addr, mac, ETHER_ADDR_LEN)) { 201 if (vlan && vlan != EVL_VLID_MASK) { 202 grp = p->vlan_grp; 203 dev = grp ? grp->vlan_devices[vlan] : NULL; 204 } else 205 while (dev->master) 206 dev = dev->master; 207 return dev; 208 } 209 } 210 #endif 211 return NULL; 212 } 213 214 static inline void 215 failover_fixup(adapter_t *adapter, int port) 216 { 217 if (adapter->params.rev == 0) { 218 struct ifnet *ifp = adapter->port[port].ifp; 219 struct cmac *mac = &adapter->port[port].mac; 220 if (!(ifp->if_flags & IFF_UP)) { 221 /* Failover triggered by the interface ifdown */ 222 t3_write_reg(adapter, A_XGM_TX_CTRL + mac->offset, 223 F_TXEN); 224 t3_read_reg(adapter, A_XGM_TX_CTRL + mac->offset); 225 } else { 226 /* Failover triggered by the interface link down */ 227 t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 0); 228 t3_read_reg(adapter, A_XGM_RX_CTRL + mac->offset); 229 t3_write_reg(adapter, A_XGM_RX_CTRL + mac->offset, 230 F_RXEN); 231 } 232 } 233 } 234 235 static int 236 cxgb_ulp_iscsi_ctl(adapter_t *adapter, unsigned int req, void *data) 237 { 238 int ret = 0; 239 struct ulp_iscsi_info *uiip = data; 240 241 switch (req) { 242 case ULP_ISCSI_GET_PARAMS: 243 uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); 244 uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); 245 uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); 246 /* 247 * On tx, the iscsi pdu has to be <= tx page size and has to 248 * fit into the Tx PM FIFO. 249 */ 250 uiip->max_txsz = uimin(adapter->params.tp.tx_pg_size, 251 t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); 252 /* on rx, the iscsi pdu has to be < rx page size and the 253 whole pdu + cpl headers has to fit into one sge buffer */ 254 uiip->max_rxsz = 255 (unsigned int)uimin(adapter->params.tp.rx_pg_size, 256 (adapter->sge.qs[0].fl[1].buf_size - 257 sizeof(struct cpl_rx_data) * 2 - 258 sizeof(struct cpl_rx_data_ddp)) ); 259 break; 260 case ULP_ISCSI_SET_PARAMS: 261 t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); 262 break; 263 default: 264 ret = (EOPNOTSUPP); 265 } 266 return ret; 267 } 268 269 /* Response queue used for RDMA events. */ 270 #define ASYNC_NOTIF_RSPQ 0 271 272 static int 273 cxgb_rdma_ctl(adapter_t *adapter, unsigned int req, void *data) 274 { 275 int ret = 0; 276 277 switch (req) { 278 case RDMA_GET_PARAMS: { 279 struct rdma_info *req2 = data; 280 281 req2->tpt_base = t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); 282 req2->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); 283 req2->pbl_base = t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); 284 req2->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); 285 req2->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); 286 req2->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); 287 break; 288 } 289 case RDMA_CQ_OP: { 290 struct rdma_cq_op *req2 = data; 291 292 /* may be called in any context */ 293 mtx_lock(&adapter->sge.reg_lock); 294 ret = t3_sge_cqcntxt_op(adapter, req2->id, req2->op, 295 req2->credits); 296 mtx_unlock(&adapter->sge.reg_lock); 297 break; 298 } 299 case RDMA_GET_MEM: { 300 struct ch_mem_range *t = data; 301 struct mc7 *mem; 302 303 if ((t->addr & 7) || (t->len & 7)) 304 return (EINVAL); 305 if (t->mem_id == MEM_CM) 306 mem = &adapter->cm; 307 else if (t->mem_id == MEM_PMRX) 308 mem = &adapter->pmrx; 309 else if (t->mem_id == MEM_PMTX) 310 mem = &adapter->pmtx; 311 else 312 return (EINVAL); 313 314 ret = t3_mc7_bd_read(mem, t->addr/8, t->len/8, (u64 *)t->buf); 315 if (ret) 316 return (ret); 317 break; 318 } 319 case RDMA_CQ_SETUP: { 320 struct rdma_cq_setup *req2 = data; 321 322 mtx_lock(&adapter->sge.reg_lock); 323 ret = t3_sge_init_cqcntxt(adapter, req2->id, req2->base_addr, 324 req2->size, ASYNC_NOTIF_RSPQ, 325 req2->ovfl_mode, req2->credits, 326 req2->credit_thres); 327 mtx_unlock(&adapter->sge.reg_lock); 328 break; 329 } 330 case RDMA_CQ_DISABLE: 331 mtx_lock(&adapter->sge.reg_lock); 332 ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); 333 mtx_unlock(&adapter->sge.reg_lock); 334 break; 335 case RDMA_CTRL_QP_SETUP: { 336 struct rdma_ctrlqp_setup *req2 = data; 337 338 mtx_lock(&adapter->sge.reg_lock); 339 ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, 340 SGE_CNTXT_RDMA, ASYNC_NOTIF_RSPQ, 341 req2->base_addr, req2->size, 342 FW_RI_TID_START, 1, 0); 343 mtx_unlock(&adapter->sge.reg_lock); 344 break; 345 } 346 default: 347 ret = EOPNOTSUPP; 348 } 349 return (ret); 350 } 351 352 static int 353 cxgb_offload_ctl(struct toedev *tdev, unsigned int req, void *data) 354 { 355 struct adapter *adapter = tdev2adap(tdev); 356 struct tid_range *tid; 357 struct mtutab *mtup; 358 struct iff_mac *iffmacp; 359 struct ddp_params *ddpp; 360 struct adap_ports *ports; 361 int port; 362 363 switch (req) { 364 case GET_MAX_OUTSTANDING_WR: 365 *(unsigned int *)data = FW_WR_NUM; 366 break; 367 case GET_WR_LEN: 368 *(unsigned int *)data = WR_FLITS; 369 break; 370 case GET_TX_MAX_CHUNK: 371 *(unsigned int *)data = 1 << 20; /* 1MB */ 372 break; 373 case GET_TID_RANGE: 374 tid = data; 375 tid->num = t3_mc5_size(&adapter->mc5) - 376 adapter->params.mc5.nroutes - 377 adapter->params.mc5.nfilters - 378 adapter->params.mc5.nservers; 379 tid->base = 0; 380 break; 381 case GET_STID_RANGE: 382 tid = data; 383 tid->num = adapter->params.mc5.nservers; 384 tid->base = t3_mc5_size(&adapter->mc5) - tid->num - 385 adapter->params.mc5.nfilters - 386 adapter->params.mc5.nroutes; 387 break; 388 case GET_L2T_CAPACITY: 389 *(unsigned int *)data = 2048; 390 break; 391 case GET_MTUS: 392 mtup = data; 393 mtup->size = NMTUS; 394 mtup->mtus = adapter->params.mtus; 395 break; 396 case GET_IFF_FROM_MAC: 397 iffmacp = data; 398 iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, 399 iffmacp->vlan_tag & EVL_VLID_MASK); 400 break; 401 case GET_DDP_PARAMS: 402 ddpp = data; 403 ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); 404 ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); 405 ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); 406 break; 407 case GET_PORTS: 408 ports = data; 409 ports->nports = adapter->params.nports; 410 for_each_port(adapter, port) 411 ports->lldevs[port] = adapter->port[port].ifp; 412 break; 413 case FAILOVER: 414 port = *(int *)data; 415 t3_port_failover(adapter, port); 416 failover_fixup(adapter, port); 417 break; 418 case FAILOVER_DONE: 419 port = *(int *)data; 420 t3_failover_done(adapter, port); 421 break; 422 case FAILOVER_CLEAR: 423 t3_failover_clear(adapter); 424 break; 425 case ULP_ISCSI_GET_PARAMS: 426 case ULP_ISCSI_SET_PARAMS: 427 if (!offload_running(adapter)) 428 return (EAGAIN); 429 return cxgb_ulp_iscsi_ctl(adapter, req, data); 430 case RDMA_GET_PARAMS: 431 case RDMA_CQ_OP: 432 case RDMA_CQ_SETUP: 433 case RDMA_CQ_DISABLE: 434 case RDMA_CTRL_QP_SETUP: 435 case RDMA_GET_MEM: 436 if (!offload_running(adapter)) 437 return (EAGAIN); 438 return cxgb_rdma_ctl(adapter, req, data); 439 default: 440 return (EOPNOTSUPP); 441 } 442 return 0; 443 } 444 445 /* 446 * Dummy handler for Rx offload packets in case we get an offload packet before 447 * proper processing is setup. This complains and drops the packet as it isn't 448 * normal to get offload packets at this stage. 449 */ 450 static int 451 rx_offload_blackhole(struct toedev *dev, struct mbuf **m, int n) 452 { 453 CH_ERR(tdev2adap(dev), "%d unexpected offload packets, first data 0x%x\n", 454 n, *mtod(m[0], uint32_t *)); 455 while (n--) 456 m_freem(m[n]); 457 return 0; 458 } 459 460 static void 461 dummy_neigh_update(struct toedev *dev, struct rtentry *neigh) 462 { 463 } 464 465 void 466 cxgb_set_dummy_ops(struct toedev *dev) 467 { 468 dev->recv = rx_offload_blackhole; 469 dev->neigh_update = dummy_neigh_update; 470 } 471 472 /* 473 * Free an active-open TID. 474 */ 475 void * 476 cxgb_free_atid(struct toedev *tdev, int atid) 477 { 478 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 479 union active_open_entry *p = atid2entry(t, atid); 480 void *ctx = p->toe_tid.ctx; 481 482 mtx_lock(&t->atid_lock); 483 p->next = t->afree; 484 t->afree = p; 485 t->atids_in_use--; 486 mtx_lock(&t->atid_lock); 487 488 return ctx; 489 } 490 491 /* 492 * Free a server TID and return it to the free pool. 493 */ 494 void 495 cxgb_free_stid(struct toedev *tdev, int stid) 496 { 497 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 498 union listen_entry *p = stid2entry(t, stid); 499 500 mtx_lock(&t->stid_lock); 501 p->next = t->sfree; 502 t->sfree = p; 503 t->stids_in_use--; 504 mtx_unlock(&t->stid_lock); 505 } 506 507 void 508 cxgb_insert_tid(struct toedev *tdev, struct cxgb_client *client, 509 void *ctx, unsigned int tid) 510 { 511 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 512 513 t->tid_tab[tid].client = client; 514 t->tid_tab[tid].ctx = ctx; 515 atomic_add_int(&t->tids_in_use, 1); 516 } 517 518 /* 519 * Populate a TID_RELEASE WR. The mbuf must be already properly sized. 520 */ 521 static inline void 522 mk_tid_release(struct mbuf *m, unsigned int tid) 523 { 524 struct cpl_tid_release *req; 525 526 m_set_priority(m, CPL_PRIORITY_SETUP); 527 req = mtod(m, struct cpl_tid_release *); 528 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 529 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 530 } 531 532 static void 533 t3_process_tid_release_list(struct work *wk, void *data) 534 { 535 struct mbuf *m; 536 struct toedev *tdev = data; 537 struct toe_data *td = TOE_DATA(tdev); 538 539 mtx_lock(&td->tid_release_lock); 540 while (td->tid_release_list) { 541 struct toe_tid_entry *p = td->tid_release_list; 542 543 td->tid_release_list = (struct toe_tid_entry *)p->ctx; 544 mtx_unlock(&td->tid_release_lock); 545 m = m_get(M_WAIT, MT_DATA); 546 mk_tid_release(m, p - td->tid_maps.tid_tab); 547 cxgb_ofld_send(tdev, m); 548 p->ctx = NULL; 549 mtx_lock(&td->tid_release_lock); 550 } 551 mtx_unlock(&td->tid_release_lock); 552 } 553 554 /* use ctx as a next pointer in the tid release list */ 555 void 556 cxgb_queue_tid_release(struct toedev *tdev, unsigned int tid) 557 { 558 struct toe_data *td = TOE_DATA(tdev); 559 struct toe_tid_entry *p = &td->tid_maps.tid_tab[tid]; 560 561 mtx_lock(&td->tid_release_lock); 562 p->ctx = td->tid_release_list; 563 td->tid_release_list = p; 564 565 if (!p->ctx) 566 workqueue_enqueue(td->tid_release_task.wq, &td->tid_release_task.w, NULL); 567 568 mtx_unlock(&td->tid_release_lock); 569 } 570 571 /* 572 * Remove a tid from the TID table. A client may defer processing its last 573 * CPL message if it is locked at the time it arrives, and while the message 574 * sits in the client's backlog the TID may be reused for another connection. 575 * To handle this we atomically switch the TID association if it still points 576 * to the original client context. 577 */ 578 void 579 cxgb_remove_tid(struct toedev *tdev, void *ctx, unsigned int tid) 580 { 581 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 582 583 BUG_ON(tid >= t->ntids); 584 if (tdev->type == T3A) 585 atomic_cmpset_ptr((uintptr_t *)&t->tid_tab[tid].ctx, (long)NULL, (long)ctx); 586 else { 587 struct mbuf *m; 588 589 m = m_get(M_NOWAIT, MT_DATA); 590 if (__predict_true(m != NULL)) { 591 mk_tid_release(m, tid); 592 cxgb_ofld_send(tdev, m); 593 t->tid_tab[tid].ctx = NULL; 594 } else 595 cxgb_queue_tid_release(tdev, tid); 596 } 597 atomic_add_int(&t->tids_in_use, -1); 598 } 599 600 int 601 cxgb_alloc_atid(struct toedev *tdev, struct cxgb_client *client, 602 void *ctx) 603 { 604 int atid = -1; 605 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 606 607 mtx_lock(&t->atid_lock); 608 if (t->afree) { 609 union active_open_entry *p = t->afree; 610 611 atid = (p - t->atid_tab) + t->atid_base; 612 t->afree = p->next; 613 p->toe_tid.ctx = ctx; 614 p->toe_tid.client = client; 615 t->atids_in_use++; 616 } 617 mtx_unlock(&t->atid_lock); 618 return atid; 619 } 620 621 int 622 cxgb_alloc_stid(struct toedev *tdev, struct cxgb_client *client, 623 void *ctx) 624 { 625 int stid = -1; 626 struct tid_info *t = &(TOE_DATA(tdev))->tid_maps; 627 628 mtx_lock(&t->stid_lock); 629 if (t->sfree) { 630 union listen_entry *p = t->sfree; 631 632 stid = (p - t->stid_tab) + t->stid_base; 633 t->sfree = p->next; 634 p->toe_tid.ctx = ctx; 635 p->toe_tid.client = client; 636 t->stids_in_use++; 637 } 638 mtx_unlock(&t->stid_lock); 639 return stid; 640 } 641 642 static int 643 do_smt_write_rpl(struct toedev *dev, struct mbuf *m) 644 { 645 struct cpl_smt_write_rpl *rpl = cplhdr(m); 646 647 if (rpl->status != CPL_ERR_NONE) 648 log(LOG_ERR, 649 "Unexpected SMT_WRITE_RPL status %u for entry %u\n", 650 rpl->status, GET_TID(rpl)); 651 652 return CPL_RET_BUF_DONE; 653 } 654 655 static int 656 do_l2t_write_rpl(struct toedev *dev, struct mbuf *m) 657 { 658 struct cpl_l2t_write_rpl *rpl = cplhdr(m); 659 660 if (rpl->status != CPL_ERR_NONE) 661 log(LOG_ERR, 662 "Unexpected L2T_WRITE_RPL status %u for entry %u\n", 663 rpl->status, GET_TID(rpl)); 664 665 return CPL_RET_BUF_DONE; 666 } 667 668 static int 669 do_act_open_rpl(struct toedev *dev, struct mbuf *m) 670 { 671 struct cpl_act_open_rpl *rpl = cplhdr(m); 672 unsigned int atid = G_TID(ntohl(rpl->atid)); 673 struct toe_tid_entry *toe_tid; 674 675 toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid); 676 if (toe_tid->ctx && toe_tid->client && toe_tid->client->handlers && 677 toe_tid->client->handlers[CPL_ACT_OPEN_RPL]) { 678 return toe_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, m, 679 toe_tid->ctx); 680 } else { 681 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 682 dev->name, CPL_ACT_OPEN_RPL); 683 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 684 } 685 } 686 687 static int 688 do_stid_rpl(struct toedev *dev, struct mbuf *m) 689 { 690 union opcode_tid *p = cplhdr(m); 691 unsigned int stid = G_TID(ntohl(p->opcode_tid)); 692 struct toe_tid_entry *toe_tid; 693 694 toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid); 695 if (toe_tid->ctx && toe_tid->client->handlers && 696 toe_tid->client->handlers[p->opcode]) { 697 return toe_tid->client->handlers[p->opcode] (dev, m, toe_tid->ctx); 698 } else { 699 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 700 dev->name, p->opcode); 701 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 702 } 703 } 704 705 static int 706 do_hwtid_rpl(struct toedev *dev, struct mbuf *m) 707 { 708 union opcode_tid *p = cplhdr(m); 709 unsigned int hwtid; 710 struct toe_tid_entry *toe_tid; 711 712 printf("do_hwtid_rpl m=%p\n", m); 713 return (0); 714 #ifdef notyet 715 716 hwtid = G_TID(ntohl(p->opcode_tid)); 717 718 toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); 719 if (toe_tid->ctx && toe_tid->client->handlers && 720 toe_tid->client->handlers[p->opcode]) { 721 return toe_tid->client->handlers[p->opcode] 722 (dev, m, toe_tid->ctx); 723 } else { 724 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 725 dev->name, p->opcode); 726 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 727 } 728 #endif 729 } 730 731 static int 732 do_cr(struct toedev *dev, struct mbuf *m) 733 { 734 struct cpl_pass_accept_req *req = cplhdr(m); 735 unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 736 struct toe_tid_entry *toe_tid; 737 738 toe_tid = lookup_stid(&(TOE_DATA(dev))->tid_maps, stid); 739 if (toe_tid->ctx && toe_tid->client->handlers && 740 toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { 741 return toe_tid->client->handlers[CPL_PASS_ACCEPT_REQ] 742 (dev, m, toe_tid->ctx); 743 } else { 744 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 745 dev->name, CPL_PASS_ACCEPT_REQ); 746 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 747 } 748 } 749 750 static int 751 do_abort_req_rss(struct toedev *dev, struct mbuf *m) 752 { 753 union opcode_tid *p = cplhdr(m); 754 unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); 755 struct toe_tid_entry *toe_tid; 756 757 toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); 758 if (toe_tid->ctx && toe_tid->client->handlers && 759 toe_tid->client->handlers[p->opcode]) { 760 return toe_tid->client->handlers[p->opcode] 761 (dev, m, toe_tid->ctx); 762 } else { 763 struct cpl_abort_req_rss *req = cplhdr(m); 764 struct cpl_abort_rpl *rpl; 765 766 struct mbuf *m2 = m_get(M_NOWAIT, MT_DATA); 767 if (!m2) { 768 log(LOG_NOTICE, "do_abort_req_rss: couldn't get mbuf!\n"); 769 goto out; 770 } 771 772 m_set_priority(m2, CPL_PRIORITY_DATA); 773 #if 0 774 __skb_put(skb, sizeof(struct cpl_abort_rpl)); 775 #endif 776 rpl = cplhdr(m2); 777 rpl->wr.wr_hi = 778 htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 779 rpl->wr.wr_lo = htonl(V_WR_TID(GET_TID(req))); 780 OPCODE_TID(rpl) = 781 htonl(MK_OPCODE_TID(CPL_ABORT_RPL, GET_TID(req))); 782 rpl->cmd = req->status; 783 cxgb_ofld_send(dev, m2); 784 out: 785 return CPL_RET_BUF_DONE; 786 } 787 } 788 789 static int 790 do_act_establish(struct toedev *dev, struct mbuf *m) 791 { 792 struct cpl_act_establish *req = cplhdr(m); 793 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 794 struct toe_tid_entry *toe_tid; 795 796 toe_tid = lookup_atid(&(TOE_DATA(dev))->tid_maps, atid); 797 if (toe_tid->ctx && toe_tid->client->handlers && 798 toe_tid->client->handlers[CPL_ACT_ESTABLISH]) { 799 return toe_tid->client->handlers[CPL_ACT_ESTABLISH] 800 (dev, m, toe_tid->ctx); 801 } else { 802 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 803 dev->name, CPL_PASS_ACCEPT_REQ); 804 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 805 } 806 } 807 808 static int 809 do_set_tcb_rpl(struct toedev *dev, struct mbuf *m) 810 { 811 struct cpl_set_tcb_rpl *rpl = cplhdr(m); 812 813 if (rpl->status != CPL_ERR_NONE) 814 log(LOG_ERR, 815 "Unexpected SET_TCB_RPL status %u for tid %u\n", 816 rpl->status, GET_TID(rpl)); 817 return CPL_RET_BUF_DONE; 818 } 819 820 static int 821 do_trace(struct toedev *dev, struct mbuf *m) 822 { 823 #if 0 824 struct cpl_trace_pkt *p = cplhdr(m); 825 826 827 skb->protocol = 0xffff; 828 skb->dev = dev->lldev; 829 skb_pull(skb, sizeof(*p)); 830 skb->mac.raw = mtod(m, (char *)); 831 netif_receive_skb(skb); 832 #endif 833 return 0; 834 } 835 836 static int 837 do_term(struct toedev *dev, struct mbuf *m) 838 { 839 unsigned int hwtid = ntohl(m_get_priority(m)) >> 8 & 0xfffff; 840 unsigned int opcode = G_OPCODE(ntohl(m->m_pkthdr.csum_data)); 841 struct toe_tid_entry *toe_tid; 842 843 toe_tid = lookup_tid(&(TOE_DATA(dev))->tid_maps, hwtid); 844 if (toe_tid->ctx && toe_tid->client->handlers && 845 toe_tid->client->handlers[opcode]) { 846 return toe_tid->client->handlers[opcode](dev, m, toe_tid->ctx); 847 } else { 848 log(LOG_ERR, "%s: received clientless CPL command 0x%x\n", 849 dev->name, opcode); 850 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 851 } 852 return (0); 853 } 854 855 #if defined(FOO) 856 #include <linux/config.h> 857 #include <linux/kallsyms.h> 858 #include <linux/kprobes.h> 859 #include <net/arp.h> 860 861 static int (*orig_arp_constructor)(struct ifnet *); 862 863 static void 864 neigh_suspect(struct ifnet *neigh) 865 { 866 struct hh_cache *hh; 867 868 neigh->output = neigh->ops->output; 869 870 for (hh = neigh->hh; hh; hh = hh->hh_next) 871 hh->hh_output = neigh->ops->output; 872 } 873 874 static void 875 neigh_connect(struct ifnet *neigh) 876 { 877 struct hh_cache *hh; 878 879 neigh->output = neigh->ops->connected_output; 880 881 for (hh = neigh->hh; hh; hh = hh->hh_next) 882 hh->hh_output = neigh->ops->hh_output; 883 } 884 885 static inline int 886 neigh_max_probes(const struct neighbour *n) 887 { 888 const struct neigh_parms *p = n->parms; 889 return (n->nud_state & NUD_PROBE ? 890 p->ucast_probes : 891 p->ucast_probes + p->app_probes + p->mcast_probes); 892 } 893 894 static void 895 neigh_timer_handler_offload(unsigned long arg) 896 { 897 unsigned long now, next; 898 struct neighbour *neigh = (struct neighbour *)arg; 899 unsigned state; 900 int notify = 0; 901 902 write_lock(&neigh->lock); 903 904 state = neigh->nud_state; 905 now = jiffies; 906 next = now + HZ; 907 908 if (!(state & NUD_IN_TIMER)) { 909 #ifndef CONFIG_SMP 910 log(LOG_WARNING, "neigh: timer & !nud_in_timer\n"); 911 #endif 912 goto out; 913 } 914 915 if (state & NUD_REACHABLE) { 916 if (time_before_eq(now, 917 neigh->confirmed + 918 neigh->parms->reachable_time)) { 919 next = neigh->confirmed + neigh->parms->reachable_time; 920 } else if (time_before_eq(now, 921 neigh->used + 922 neigh->parms->delay_probe_time)) { 923 neigh->nud_state = NUD_DELAY; 924 neigh->updated = jiffies; 925 neigh_suspect(neigh); 926 next = now + neigh->parms->delay_probe_time; 927 } else { 928 neigh->nud_state = NUD_STALE; 929 neigh->updated = jiffies; 930 neigh_suspect(neigh); 931 cxgb_neigh_update(neigh); 932 } 933 } else if (state & NUD_DELAY) { 934 if (time_before_eq(now, 935 neigh->confirmed + 936 neigh->parms->delay_probe_time)) { 937 neigh->nud_state = NUD_REACHABLE; 938 neigh->updated = jiffies; 939 neigh_connect(neigh); 940 cxgb_neigh_update(neigh); 941 next = neigh->confirmed + neigh->parms->reachable_time; 942 } else { 943 neigh->nud_state = NUD_PROBE; 944 neigh->updated = jiffies; 945 atomic_set_int(&neigh->probes, 0); 946 next = now + neigh->parms->retrans_time; 947 } 948 } else { 949 /* NUD_PROBE|NUD_INCOMPLETE */ 950 next = now + neigh->parms->retrans_time; 951 } 952 /* 953 * Needed for read of probes 954 */ 955 mb(); 956 if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) && 957 neigh->probes >= neigh_max_probes(neigh)) { 958 struct mbuf *m; 959 960 neigh->nud_state = NUD_FAILED; 961 neigh->updated = jiffies; 962 notify = 1; 963 cxgb_neigh_update(neigh); 964 NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed); 965 966 /* It is very thin place. report_unreachable is very 967 complicated routine. Particularly, it can hit the same 968 neighbour entry! 969 So that, we try to be accurate and avoid dead loop. --ANK 970 */ 971 while (neigh->nud_state == NUD_FAILED && 972 (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) { 973 write_unlock(&neigh->lock); 974 neigh->ops->error_report(neigh, skb); 975 write_lock(&neigh->lock); 976 } 977 skb_queue_purge(&neigh->arp_queue); 978 } 979 980 if (neigh->nud_state & NUD_IN_TIMER) { 981 if (time_before(next, jiffies + HZ/2)) 982 next = jiffies + HZ/2; 983 if (!mod_timer(&neigh->timer, next)) 984 neigh_hold(neigh); 985 } 986 if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) { 987 struct mbuf *m = skb_peek(&neigh->arp_queue); 988 989 write_unlock(&neigh->lock); 990 neigh->ops->solicit(neigh, skb); 991 atomic_add_int(&neigh->probes, 1); 992 if (m) 993 m_free(m); 994 } else { 995 out: 996 write_unlock(&neigh->lock); 997 } 998 999 #ifdef CONFIG_ARPD 1000 if (notify && neigh->parms->app_probes) 1001 neigh_app_notify(neigh); 1002 #endif 1003 neigh_release(neigh); 1004 } 1005 1006 static int 1007 arp_constructor_offload(struct neighbour *neigh) 1008 { 1009 if (neigh->ifp && is_offloading(neigh->ifp)) 1010 neigh->timer.function = neigh_timer_handler_offload; 1011 return orig_arp_constructor(neigh); 1012 } 1013 1014 /* 1015 * This must match exactly the signature of neigh_update for jprobes to work. 1016 * It runs from a trap handler with interrupts off so don't disable BH. 1017 */ 1018 static int 1019 neigh_update_offload(struct neighbour *neigh, const u8 *lladdr, 1020 u8 new, u32 flags) 1021 { 1022 write_lock(&neigh->lock); 1023 cxgb_neigh_update(neigh); 1024 write_unlock(&neigh->lock); 1025 jprobe_return(); 1026 /* NOTREACHED */ 1027 return 0; 1028 } 1029 1030 static struct jprobe neigh_update_jprobe = { 1031 .entry = (kprobe_opcode_t *) neigh_update_offload, 1032 .kp.addr = (kprobe_opcode_t *) neigh_update 1033 }; 1034 1035 #ifdef MODULE_SUPPORT 1036 static int 1037 prepare_arp_with_t3core(void) 1038 { 1039 int err; 1040 1041 err = register_jprobe(&neigh_update_jprobe); 1042 if (err) { 1043 log(LOG_ERR, "Could not install neigh_update jprobe, " 1044 "error %d\n", err); 1045 return err; 1046 } 1047 1048 orig_arp_constructor = arp_tbl.constructor; 1049 arp_tbl.constructor = arp_constructor_offload; 1050 1051 return 0; 1052 } 1053 1054 static void 1055 restore_arp_sans_t3core(void) 1056 { 1057 arp_tbl.constructor = orig_arp_constructor; 1058 unregister_jprobe(&neigh_update_jprobe); 1059 } 1060 1061 #else /* Module support */ 1062 static inline int 1063 prepare_arp_with_t3core(void) 1064 { 1065 return 0; 1066 } 1067 1068 static inline void 1069 restore_arp_sans_t3core(void) 1070 {} 1071 #endif 1072 #endif 1073 /* 1074 * Process a received packet with an unknown/unexpected CPL opcode. 1075 */ 1076 static int 1077 do_bad_cpl(struct toedev *dev, struct mbuf *m) 1078 { 1079 log(LOG_ERR, "%s: received bad CPL command 0x%x\n", dev->name, 1080 *mtod(m, uint32_t *)); 1081 return (CPL_RET_BUF_DONE | CPL_RET_BAD_MSG); 1082 } 1083 1084 /* 1085 * Handlers for each CPL opcode 1086 */ 1087 static cpl_handler_func cpl_handlers[NUM_CPL_CMDS]; 1088 1089 /* 1090 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied 1091 * to unregister an existing handler. 1092 */ 1093 void 1094 t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h) 1095 { 1096 if (opcode < NUM_CPL_CMDS) 1097 cpl_handlers[opcode] = h ? h : do_bad_cpl; 1098 else 1099 log(LOG_ERR, "T3C: handler registration for " 1100 "opcode %x failed\n", opcode); 1101 } 1102 1103 /* 1104 * TOEDEV's receive method. 1105 */ 1106 int 1107 process_rx(struct toedev *dev, struct mbuf **m, int n) 1108 { 1109 while (n--) { 1110 struct mbuf *m0 = *m++; 1111 unsigned int opcode = G_OPCODE(ntohl(m0->m_pkthdr.csum_data)); 1112 int ret = cpl_handlers[opcode] (dev, m0); 1113 1114 #if VALIDATE_TID 1115 if (ret & CPL_RET_UNKNOWN_TID) { 1116 union opcode_tid *p = cplhdr(m0); 1117 1118 log(LOG_ERR, "%s: CPL message (opcode %u) had " 1119 "unknown TID %u\n", dev->name, opcode, 1120 G_TID(ntohl(p->opcode_tid))); 1121 } 1122 #endif 1123 if (ret & CPL_RET_BUF_DONE) 1124 m_freem(m0); 1125 } 1126 return 0; 1127 } 1128 1129 /* 1130 * Sends an sk_buff to a T3C driver after dealing with any active network taps. 1131 */ 1132 int 1133 cxgb_ofld_send(struct toedev *dev, struct mbuf *m) 1134 { 1135 int r; 1136 1137 critical_enter(); 1138 r = dev->send(dev, m); 1139 critical_exit(); 1140 return r; 1141 } 1142 1143 1144 /** 1145 * cxgb_ofld_recv - process n received offload packets 1146 * @dev: the offload device 1147 * @m: an array of offload packets 1148 * @n: the number of offload packets 1149 * 1150 * Process an array of ingress offload packets. Each packet is forwarded 1151 * to any active network taps and then passed to the offload device's receive 1152 * method. We optimize passing packets to the receive method by passing 1153 * it the whole array at once except when there are active taps. 1154 */ 1155 int 1156 cxgb_ofld_recv(struct toedev *dev, struct mbuf **m, int n) 1157 { 1158 1159 #if defined(CONFIG_CHELSIO_T3) 1160 if (likely(!netdev_nit)) 1161 return dev->recv(dev, skb, n); 1162 1163 for ( ; n; n--, skb++) { 1164 skb[0]->dev = dev->lldev; 1165 dev_queue_xmit_nit(skb[0], dev->lldev); 1166 skb[0]->dev = NULL; 1167 dev->recv(dev, skb, 1); 1168 } 1169 return 0; 1170 #else 1171 return dev->recv(dev, m, n); 1172 #endif 1173 } 1174 1175 void 1176 cxgb_neigh_update(struct rtentry *rt) 1177 { 1178 1179 if (is_offloading(rt->rt_ifp)) { 1180 struct toedev *tdev = TOEDEV(rt->rt_ifp); 1181 1182 BUG_ON(!tdev); 1183 t3_l2t_update(tdev, rt); 1184 } 1185 } 1186 1187 static void 1188 set_l2t_ix(struct toedev *tdev, u32 tid, struct l2t_entry *e) 1189 { 1190 struct mbuf *m; 1191 struct cpl_set_tcb_field *req; 1192 1193 m = m_gethdr(M_NOWAIT, MT_DATA); 1194 if (!m) { 1195 log(LOG_ERR, "%s: cannot allocate mbuf!\n", __func__); 1196 return; 1197 } 1198 1199 m_set_priority(m, CPL_PRIORITY_CONTROL); 1200 req = mtod(m, struct cpl_set_tcb_field *); 1201 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1202 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1203 req->reply = 0; 1204 req->cpu_idx = 0; 1205 req->word = htons(W_TCB_L2T_IX); 1206 req->mask = htobe64(V_TCB_L2T_IX(M_TCB_L2T_IX)); 1207 req->val = htobe64(V_TCB_L2T_IX(e->idx)); 1208 tdev->send(tdev, m); 1209 } 1210 1211 void 1212 cxgb_redirect(struct rtentry *old, struct rtentry *new) 1213 { 1214 struct ifnet *olddev, *newdev; 1215 struct tid_info *ti; 1216 struct toedev *tdev; 1217 u32 tid; 1218 int update_tcb; 1219 struct l2t_entry *e; 1220 struct toe_tid_entry *te; 1221 1222 olddev = old->rt_ifp; 1223 newdev = new->rt_ifp; 1224 if (!is_offloading(olddev)) 1225 return; 1226 if (!is_offloading(newdev)) { 1227 log(LOG_WARNING, "%s: Redirect to non-offload" 1228 "device ignored.\n", __func__); 1229 return; 1230 } 1231 tdev = TOEDEV(olddev); 1232 BUG_ON(!tdev); 1233 if (tdev != TOEDEV(newdev)) { 1234 log(LOG_WARNING, "%s: Redirect to different " 1235 "offload device ignored.\n", __func__); 1236 return; 1237 } 1238 1239 /* Add new L2T entry */ 1240 e = t3_l2t_get(tdev, new, ((struct port_info *)new->rt_ifp->if_softc)->port_id); 1241 if (!e) { 1242 log(LOG_ERR, "%s: couldn't allocate new l2t entry!\n", 1243 __func__); 1244 return; 1245 } 1246 1247 /* Walk tid table and notify clients of dst change. */ 1248 ti = &(TOE_DATA(tdev))->tid_maps; 1249 for (tid=0; tid < ti->ntids; tid++) { 1250 te = lookup_tid(ti, tid); 1251 BUG_ON(!te); 1252 if (te->ctx && te->client && te->client->redirect) { 1253 update_tcb = te->client->redirect(te->ctx, old, new, 1254 e); 1255 if (update_tcb) { 1256 l2t_hold(L2DATA(tdev), e); 1257 set_l2t_ix(tdev, tid, e); 1258 } 1259 } 1260 } 1261 l2t_release(L2DATA(tdev), e); 1262 } 1263 1264 /* 1265 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. 1266 * The allocated memory is cleared. 1267 */ 1268 void * 1269 cxgb_alloc_mem(unsigned long size) 1270 { 1271 return malloc(size, M_DEVBUF, M_ZERO); 1272 } 1273 1274 /* 1275 * Free memory allocated through t3_alloc_mem(). 1276 */ 1277 void 1278 cxgb_free_mem(void *addr) 1279 { 1280 free(addr, M_DEVBUF); 1281 } 1282 1283 1284 /* 1285 * Allocate and initialize the TID tables. Returns 0 on success. 1286 */ 1287 static int 1288 init_tid_tabs(struct tid_info *t, unsigned int ntids, 1289 unsigned int natids, unsigned int nstids, 1290 unsigned int atid_base, unsigned int stid_base) 1291 { 1292 unsigned long size = ntids * sizeof(*t->tid_tab) + 1293 natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); 1294 1295 t->tid_tab = cxgb_alloc_mem(size); 1296 if (!t->tid_tab) 1297 return (ENOMEM); 1298 1299 t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; 1300 t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; 1301 t->ntids = ntids; 1302 t->nstids = nstids; 1303 t->stid_base = stid_base; 1304 t->sfree = NULL; 1305 t->natids = natids; 1306 t->atid_base = atid_base; 1307 t->afree = NULL; 1308 t->stids_in_use = t->atids_in_use = 0; 1309 atomic_set_int(&t->tids_in_use, 0); 1310 mtx_init(&t->stid_lock, "stid", NULL, MTX_DEF); 1311 mtx_init(&t->atid_lock, "atid", NULL, MTX_DEF); 1312 1313 /* 1314 * Setup the free lists for stid_tab and atid_tab. 1315 */ 1316 if (nstids) { 1317 while (--nstids) 1318 t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; 1319 t->sfree = t->stid_tab; 1320 } 1321 if (natids) { 1322 while (--natids) 1323 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1324 t->afree = t->atid_tab; 1325 } 1326 return 0; 1327 } 1328 1329 static void 1330 free_tid_maps(struct tid_info *t) 1331 { 1332 cxgb_free_mem(t->tid_tab); 1333 } 1334 1335 static inline void 1336 add_adapter(adapter_t *adap) 1337 { 1338 rw_wlock(&adapter_list_lock); 1339 TAILQ_INSERT_TAIL(&adapter_list, adap, adapter_entry); 1340 rw_wunlock(&adapter_list_lock); 1341 } 1342 1343 static inline void 1344 remove_adapter(adapter_t *adap) 1345 { 1346 rw_wlock(&adapter_list_lock); 1347 TAILQ_REMOVE(&adapter_list, adap, adapter_entry); 1348 rw_wunlock(&adapter_list_lock); 1349 } 1350 1351 /* 1352 * XXX 1353 */ 1354 #define t3_free_l2t(...) 1355 1356 int 1357 cxgb_offload_activate(struct adapter *adapter) 1358 { 1359 struct toedev *dev = &adapter->tdev; 1360 int natids, err; 1361 struct toe_data *t; 1362 struct tid_range stid_range, tid_range; 1363 struct mtutab mtutab; 1364 unsigned int l2t_capacity; 1365 1366 t = malloc(sizeof(*t), M_DEVBUF, M_WAITOK); 1367 if (!t) 1368 return (ENOMEM); 1369 1370 err = (EOPNOTSUPP); 1371 if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || 1372 dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || 1373 dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || 1374 dev->ctl(dev, GET_MTUS, &mtutab) < 0 || 1375 dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || 1376 dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) 1377 goto out_free; 1378 1379 err = (ENOMEM); 1380 L2DATA(dev) = t3_init_l2t(l2t_capacity); 1381 if (!L2DATA(dev)) 1382 goto out_free; 1383 1384 natids = uimin(tid_range.num / 2, MAX_ATIDS); 1385 err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, 1386 stid_range.num, ATID_BASE, stid_range.base); 1387 if (err) 1388 goto out_free_l2t; 1389 1390 t->mtus = mtutab.mtus; 1391 t->nmtus = mtutab.size; 1392 1393 t->tid_release_task.name = "t3_process_tid_release_list"; 1394 t->tid_release_task.func = t3_process_tid_release_list; 1395 t->tid_release_task.context = adapter; 1396 kthread_create(PRI_NONE, 0, NULL, cxgb_make_task, &t->tid_release_task, NULL, "cxgb_make_task"); 1397 mtx_init(&t->tid_release_lock, "tid release", NULL, MTX_DEF); 1398 t->dev = dev; 1399 1400 TOE_DATA(dev) = t; 1401 dev->recv = process_rx; 1402 dev->neigh_update = t3_l2t_update; 1403 #if 0 1404 offload_proc_dev_setup(dev); 1405 #endif 1406 /* Register netevent handler once */ 1407 if (TAILQ_EMPTY(&adapter_list)) { 1408 #if defined(CONFIG_CHELSIO_T3_MODULE) 1409 if (prepare_arp_with_t3core()) 1410 log(LOG_ERR, "Unable to set offload capabilities\n"); 1411 #endif 1412 } 1413 add_adapter(adapter); 1414 return 0; 1415 1416 out_free_l2t: 1417 t3_free_l2t(L2DATA(dev)); 1418 L2DATA(dev) = NULL; 1419 out_free: 1420 free(t, M_DEVBUF); 1421 return err; 1422 1423 } 1424 1425 void 1426 cxgb_offload_deactivate(struct adapter *adapter) 1427 { 1428 struct toedev *tdev = &adapter->tdev; 1429 struct toe_data *t = TOE_DATA(tdev); 1430 1431 remove_adapter(adapter); 1432 if (TAILQ_EMPTY(&adapter_list)) { 1433 #if defined(CONFIG_CHELSIO_T3_MODULE) 1434 restore_arp_sans_t3core(); 1435 #endif 1436 } 1437 free_tid_maps(&t->tid_maps); 1438 TOE_DATA(tdev) = NULL; 1439 t3_free_l2t(L2DATA(tdev)); 1440 L2DATA(tdev) = NULL; 1441 free(t, M_DEVBUF); 1442 } 1443 1444 1445 static inline void 1446 register_tdev(struct toedev *tdev) 1447 { 1448 static int unit; 1449 1450 mtx_lock(&cxgb_db_lock); 1451 snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); 1452 TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry); 1453 mtx_unlock(&cxgb_db_lock); 1454 } 1455 1456 static inline void 1457 unregister_tdev(struct toedev *tdev) 1458 { 1459 mtx_lock(&cxgb_db_lock); 1460 TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry); 1461 mtx_unlock(&cxgb_db_lock); 1462 } 1463 1464 void 1465 cxgb_adapter_ofld(struct adapter *adapter) 1466 { 1467 struct toedev *tdev = &adapter->tdev; 1468 1469 cxgb_set_dummy_ops(tdev); 1470 tdev->send = t3_offload_tx; 1471 tdev->ctl = cxgb_offload_ctl; 1472 tdev->type = adapter->params.rev == 0 ? 1473 T3A : T3B; 1474 1475 register_tdev(tdev); 1476 #if 0 1477 offload_proc_dev_init(tdev); 1478 #endif 1479 } 1480 1481 void 1482 cxgb_adapter_unofld(struct adapter *adapter) 1483 { 1484 struct toedev *tdev = &adapter->tdev; 1485 #if 0 1486 offload_proc_dev_cleanup(tdev); 1487 offload_proc_dev_exit(tdev); 1488 #endif 1489 tdev->recv = NULL; 1490 tdev->neigh_update = NULL; 1491 1492 unregister_tdev(tdev); 1493 } 1494 1495 void 1496 cxgb_offload_init(void) 1497 { 1498 int i; 1499 1500 if (inited) 1501 return; 1502 else 1503 inited = 1; 1504 1505 mtx_init(&cxgb_db_lock, "ofld db", NULL, MTX_DEF); 1506 rw_init(&adapter_list_lock); 1507 TAILQ_INIT(&client_list); 1508 TAILQ_INIT(&ofld_dev_list); 1509 TAILQ_INIT(&adapter_list); 1510 1511 for (i = 0; i < NUM_CPL_CMDS; ++i) 1512 cpl_handlers[i] = do_bad_cpl; 1513 1514 t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); 1515 t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl); 1516 t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); 1517 t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); 1518 t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); 1519 t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); 1520 t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); 1521 t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); 1522 t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); 1523 t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); 1524 t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); 1525 t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); 1526 t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); 1527 t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); 1528 t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); 1529 t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); 1530 t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 1531 t3_register_cpl_handler(CPL_SET_TCB_RPL, do_set_tcb_rpl); 1532 t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); 1533 t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); 1534 t3_register_cpl_handler(CPL_TRACE_PKT, do_trace); 1535 t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); 1536 t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); 1537 t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); 1538 #if 0 1539 if (offload_proc_init()) 1540 log(LOG_WARNING, "Unable to create /proc/net/cxgb3 dir\n"); 1541 #endif 1542 } 1543 1544 void 1545 cxgb_offload_exit(void) 1546 { 1547 static int deinited = 0; 1548 1549 if (deinited) 1550 return; 1551 1552 deinited = 1; 1553 mtx_destroy(&cxgb_db_lock); 1554 rw_destroy(&adapter_list_lock); 1555 #if 0 1556 offload_proc_cleanup(); 1557 #endif 1558 } 1559 1560 #if 0 1561 static int 1562 offload_info_read_proc(char *buf, char **start, off_t offset, 1563 int length, int *eof, void *data) 1564 { 1565 struct toe_data *d = data; 1566 struct tid_info *t = &d->tid_maps; 1567 int len; 1568 1569 len = snprintf(buf, length, "TID range: 0..%d, in use: %u\n" 1570 "STID range: %d..%d, in use: %u\n" 1571 "ATID range: %d..%d, in use: %u\n" 1572 "MSS: %u\n", 1573 t->ntids - 1, atomic_read(&t->tids_in_use), t->stid_base, 1574 t->stid_base + t->nstids - 1, t->stids_in_use, 1575 t->atid_base, t->atid_base + t->natids - 1, 1576 t->atids_in_use, d->tx_max_chunk); 1577 if (len > length) 1578 len = length; 1579 *eof = 1; 1580 return len; 1581 } 1582 1583 static int 1584 offload_info_proc_setup(struct proc_dir_entry *dir, 1585 struct toe_data *d) 1586 { 1587 struct proc_dir_entry *p; 1588 1589 if (!dir) 1590 return (EINVAL); 1591 1592 p = create_proc_read_entry("info", 0, dir, offload_info_read_proc, d); 1593 if (!p) 1594 return (ENOMEM); 1595 1596 p->owner = THIS_MODULE; 1597 return 0; 1598 } 1599 1600 1601 static int 1602 offload_devices_read_proc(char *buf, char **start, off_t offset, 1603 int length, int *eof, void *data) 1604 { 1605 int len; 1606 struct toedev *dev; 1607 struct net_device *ndev; 1608 1609 len = snprintf(buf, length, "Device Interfaces\n"); 1610 1611 mtx_lock(&cxgb_db_lock); 1612 TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) { 1613 if (len >= length) 1614 break; 1615 len += snprintf(buf + len, length - len, "%-16s", dev->name); 1616 read_lock(&dev_base_lock); 1617 for (ndev = dev_base; ndev; ndev = ndev->next) { 1618 if (TOEDEV(ndev) == dev) { 1619 if (len >= length) 1620 break; 1621 len += snprintf(buf + len, length - len, " %s", ndev->name); 1622 } 1623 } 1624 read_unlock(&dev_base_lock); 1625 if (len >= length) 1626 break; 1627 len += snprintf(buf + len, length - len, "\n"); 1628 } 1629 mtx_unlock(&cxgb_db_lock); 1630 1631 if (len > length) 1632 len = length; 1633 *eof = 1; 1634 return len; 1635 } 1636 1637 #endif 1638 1639
Indexes created Mon Sep 22 05:09:51 GMT 2025