nvme_pci.c revision 1.2.2.5
1 1.2.2.5 skrll /* $NetBSD: nvme_pci.c,v 1.2.2.5 2016/12/05 10:55:03 skrll Exp $ */ 2 1.2.2.2 skrll /* $OpenBSD: nvme_pci.c,v 1.3 2016/04/14 11:18:32 dlg Exp $ */ 3 1.2.2.2 skrll 4 1.2.2.2 skrll /* 5 1.2.2.2 skrll * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org> 6 1.2.2.2 skrll * 7 1.2.2.2 skrll * Permission to use, copy, modify, and distribute this software for any 8 1.2.2.2 skrll * purpose with or without fee is hereby granted, provided that the above 9 1.2.2.2 skrll * copyright notice and this permission notice appear in all copies. 10 1.2.2.2 skrll * 11 1.2.2.2 skrll * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 1.2.2.2 skrll * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 1.2.2.2 skrll * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 1.2.2.2 skrll * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 1.2.2.2 skrll * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 1.2.2.2 skrll * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 1.2.2.2 skrll * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 1.2.2.2 skrll */ 19 1.2.2.2 skrll 20 1.2.2.2 skrll /*- 21 1.2.2.2 skrll * Copyright (C) 2016 NONAKA Kimihiro <nonaka (at) netbsd.org> 22 1.2.2.2 skrll * All rights reserved. 23 1.2.2.2 skrll * 24 1.2.2.2 skrll * Redistribution and use in source and binary forms, with or without 25 1.2.2.2 skrll * modification, are permitted provided that the following conditions 26 1.2.2.2 skrll * are met: 27 1.2.2.2 skrll * 1. Redistributions of source code must retain the above copyright 28 1.2.2.2 skrll * notice, this list of conditions and the following disclaimer. 29 1.2.2.2 skrll * 2. Redistributions in binary form must reproduce the above copyright 30 1.2.2.2 skrll * notice, this list of conditions and the following disclaimer in the 31 1.2.2.2 skrll * documentation and/or other materials provided with the distribution. 32 1.2.2.2 skrll * 33 1.2.2.2 skrll * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 34 1.2.2.2 skrll * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 35 1.2.2.2 skrll * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 36 1.2.2.2 skrll * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 37 1.2.2.2 skrll * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 38 1.2.2.2 skrll * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 39 1.2.2.2 skrll * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 40 1.2.2.2 skrll * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 41 1.2.2.2 skrll * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 42 1.2.2.2 skrll * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 1.2.2.2 skrll */ 44 1.2.2.2 skrll 45 1.2.2.2 skrll #include <sys/cdefs.h> 46 1.2.2.5 skrll __KERNEL_RCSID(0, "$NetBSD: nvme_pci.c,v 1.2.2.5 2016/12/05 10:55:03 skrll Exp $"); 47 1.2.2.2 skrll 48 1.2.2.2 skrll #include <sys/param.h> 49 1.2.2.2 skrll #include <sys/systm.h> 50 1.2.2.2 skrll #include <sys/kernel.h> 51 1.2.2.2 skrll #include <sys/device.h> 52 1.2.2.2 skrll #include <sys/bitops.h> 53 1.2.2.2 skrll #include <sys/bus.h> 54 1.2.2.2 skrll #include <sys/cpu.h> 55 1.2.2.2 skrll #include <sys/interrupt.h> 56 1.2.2.2 skrll #include <sys/kmem.h> 57 1.2.2.2 skrll #include <sys/pmf.h> 58 1.2.2.4 skrll #include <sys/module.h> 59 1.2.2.2 skrll 60 1.2.2.2 skrll #include <dev/pci/pcireg.h> 61 1.2.2.2 skrll #include <dev/pci/pcivar.h> 62 1.2.2.2 skrll 63 1.2.2.2 skrll #include <dev/ic/nvmereg.h> 64 1.2.2.2 skrll #include <dev/ic/nvmevar.h> 65 1.2.2.2 skrll 66 1.2.2.2 skrll int nvme_pci_force_intx = 0; 67 1.2.2.4 skrll int nvme_pci_mpsafe = 1; 68 1.2.2.2 skrll int nvme_pci_mq = 1; /* INTx: ioq=1, MSI/MSI-X: ioq=ncpu */ 69 1.2.2.2 skrll 70 1.2.2.2 skrll #define NVME_PCI_BAR 0x10 71 1.2.2.2 skrll 72 1.2.2.2 skrll struct nvme_pci_softc { 73 1.2.2.2 skrll struct nvme_softc psc_nvme; 74 1.2.2.2 skrll 75 1.2.2.2 skrll pci_chipset_tag_t psc_pc; 76 1.2.2.2 skrll pci_intr_handle_t *psc_intrs; 77 1.2.2.2 skrll int psc_nintrs; 78 1.2.2.2 skrll }; 79 1.2.2.2 skrll 80 1.2.2.2 skrll static int nvme_pci_match(device_t, cfdata_t, void *); 81 1.2.2.2 skrll static void nvme_pci_attach(device_t, device_t, void *); 82 1.2.2.2 skrll static int nvme_pci_detach(device_t, int); 83 1.2.2.4 skrll static int nvme_pci_rescan(device_t, const char *, const int *); 84 1.2.2.2 skrll 85 1.2.2.2 skrll CFATTACH_DECL3_NEW(nvme_pci, sizeof(struct nvme_pci_softc), 86 1.2.2.4 skrll nvme_pci_match, nvme_pci_attach, nvme_pci_detach, NULL, nvme_pci_rescan, 87 1.2.2.2 skrll nvme_childdet, DVF_DETACH_SHUTDOWN); 88 1.2.2.2 skrll 89 1.2.2.2 skrll static int nvme_pci_intr_establish(struct nvme_softc *, 90 1.2.2.2 skrll uint16_t, struct nvme_queue *); 91 1.2.2.2 skrll static int nvme_pci_intr_disestablish(struct nvme_softc *, uint16_t); 92 1.2.2.2 skrll static int nvme_pci_setup_intr(struct pci_attach_args *, 93 1.2.2.2 skrll struct nvme_pci_softc *); 94 1.2.2.2 skrll 95 1.2.2.2 skrll static int 96 1.2.2.2 skrll nvme_pci_match(device_t parent, cfdata_t match, void *aux) 97 1.2.2.2 skrll { 98 1.2.2.2 skrll struct pci_attach_args *pa = aux; 99 1.2.2.2 skrll 100 1.2.2.2 skrll if (PCI_CLASS(pa->pa_class) == PCI_CLASS_MASS_STORAGE && 101 1.2.2.2 skrll PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_NVM && 102 1.2.2.2 skrll PCI_INTERFACE(pa->pa_class) == PCI_INTERFACE_NVM_NVME) 103 1.2.2.2 skrll return 1; 104 1.2.2.2 skrll 105 1.2.2.2 skrll return 0; 106 1.2.2.2 skrll } 107 1.2.2.2 skrll 108 1.2.2.2 skrll static void 109 1.2.2.2 skrll nvme_pci_attach(device_t parent, device_t self, void *aux) 110 1.2.2.2 skrll { 111 1.2.2.2 skrll struct nvme_pci_softc *psc = device_private(self); 112 1.2.2.2 skrll struct nvme_softc *sc = &psc->psc_nvme; 113 1.2.2.2 skrll struct pci_attach_args *pa = aux; 114 1.2.2.4 skrll pcireg_t memtype, reg; 115 1.2.2.2 skrll bus_addr_t memaddr; 116 1.2.2.4 skrll int flags, error; 117 1.2.2.4 skrll int msixoff; 118 1.2.2.2 skrll 119 1.2.2.2 skrll sc->sc_dev = self; 120 1.2.2.2 skrll psc->psc_pc = pa->pa_pc; 121 1.2.2.2 skrll if (pci_dma64_available(pa)) 122 1.2.2.2 skrll sc->sc_dmat = pa->pa_dmat64; 123 1.2.2.2 skrll else 124 1.2.2.2 skrll sc->sc_dmat = pa->pa_dmat; 125 1.2.2.2 skrll 126 1.2.2.2 skrll pci_aprint_devinfo(pa, NULL); 127 1.2.2.2 skrll 128 1.2.2.4 skrll reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 129 1.2.2.4 skrll if ((reg & PCI_COMMAND_MASTER_ENABLE) == 0) { 130 1.2.2.4 skrll reg |= PCI_COMMAND_MASTER_ENABLE; 131 1.2.2.4 skrll pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, reg); 132 1.2.2.4 skrll } 133 1.2.2.4 skrll 134 1.2.2.2 skrll /* Map registers */ 135 1.2.2.2 skrll memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, NVME_PCI_BAR); 136 1.2.2.2 skrll if (PCI_MAPREG_TYPE(memtype) != PCI_MAPREG_TYPE_MEM) { 137 1.2.2.2 skrll aprint_error_dev(self, "invalid type (type=0x%x)\n", memtype); 138 1.2.2.2 skrll return; 139 1.2.2.2 skrll } 140 1.2.2.2 skrll sc->sc_iot = pa->pa_memt; 141 1.2.2.2 skrll error = pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_MAPREG_START, 142 1.2.2.2 skrll memtype, &memaddr, &sc->sc_ios, &flags); 143 1.2.2.2 skrll if (error) { 144 1.2.2.2 skrll aprint_error_dev(self, "can't get map info\n"); 145 1.2.2.2 skrll return; 146 1.2.2.2 skrll } 147 1.2.2.4 skrll 148 1.2.2.2 skrll if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &msixoff, 149 1.2.2.2 skrll NULL)) { 150 1.2.2.2 skrll pcireg_t msixtbl; 151 1.2.2.2 skrll uint32_t table_offset; 152 1.2.2.2 skrll int bir; 153 1.2.2.2 skrll 154 1.2.2.2 skrll msixtbl = pci_conf_read(pa->pa_pc, pa->pa_tag, 155 1.2.2.2 skrll msixoff + PCI_MSIX_TBLOFFSET); 156 1.2.2.2 skrll table_offset = msixtbl & PCI_MSIX_TBLOFFSET_MASK; 157 1.2.2.2 skrll bir = msixtbl & PCI_MSIX_PBABIR_MASK; 158 1.2.2.2 skrll if (bir == 0) { 159 1.2.2.2 skrll sc->sc_ios = table_offset; 160 1.2.2.2 skrll } 161 1.2.2.2 skrll } 162 1.2.2.4 skrll 163 1.2.2.2 skrll error = bus_space_map(sc->sc_iot, memaddr, sc->sc_ios, flags, 164 1.2.2.2 skrll &sc->sc_ioh); 165 1.2.2.2 skrll if (error != 0) { 166 1.2.2.2 skrll aprint_error_dev(self, "can't map mem space (error=%d)\n", 167 1.2.2.2 skrll error); 168 1.2.2.2 skrll return; 169 1.2.2.2 skrll } 170 1.2.2.2 skrll 171 1.2.2.2 skrll /* Establish interrupts */ 172 1.2.2.2 skrll if (nvme_pci_setup_intr(pa, psc) != 0) { 173 1.2.2.2 skrll aprint_error_dev(self, "unable to allocate interrupt\n"); 174 1.2.2.2 skrll goto unmap; 175 1.2.2.2 skrll } 176 1.2.2.2 skrll sc->sc_intr_establish = nvme_pci_intr_establish; 177 1.2.2.2 skrll sc->sc_intr_disestablish = nvme_pci_intr_disestablish; 178 1.2.2.2 skrll 179 1.2.2.4 skrll sc->sc_ih = kmem_zalloc(sizeof(*sc->sc_ih) * psc->psc_nintrs, KM_SLEEP); 180 1.2.2.2 skrll if (sc->sc_ih == NULL) { 181 1.2.2.2 skrll aprint_error_dev(self, "unable to allocate ih memory\n"); 182 1.2.2.2 skrll goto intr_release; 183 1.2.2.2 skrll } 184 1.2.2.2 skrll 185 1.2.2.5 skrll sc->sc_softih = kmem_zalloc( 186 1.2.2.5 skrll sizeof(*sc->sc_softih) * psc->psc_nintrs, KM_SLEEP); 187 1.2.2.5 skrll if (sc->sc_softih == NULL) { 188 1.2.2.5 skrll aprint_error_dev(self, 189 1.2.2.5 skrll "unable to allocate softih memory\n"); 190 1.2.2.5 skrll goto intr_free; 191 1.2.2.4 skrll } 192 1.2.2.4 skrll 193 1.2.2.2 skrll if (nvme_attach(sc) != 0) { 194 1.2.2.2 skrll /* error printed by nvme_attach() */ 195 1.2.2.4 skrll goto softintr_free; 196 1.2.2.2 skrll } 197 1.2.2.2 skrll 198 1.2.2.2 skrll if (!pmf_device_register(self, NULL, NULL)) 199 1.2.2.2 skrll aprint_error_dev(self, "couldn't establish power handler\n"); 200 1.2.2.2 skrll 201 1.2.2.2 skrll SET(sc->sc_flags, NVME_F_ATTACHED); 202 1.2.2.2 skrll return; 203 1.2.2.2 skrll 204 1.2.2.4 skrll softintr_free: 205 1.2.2.5 skrll kmem_free(sc->sc_softih, sizeof(*sc->sc_softih) * psc->psc_nintrs); 206 1.2.2.2 skrll intr_free: 207 1.2.2.4 skrll kmem_free(sc->sc_ih, sizeof(*sc->sc_ih) * psc->psc_nintrs); 208 1.2.2.2 skrll sc->sc_nq = 0; 209 1.2.2.2 skrll intr_release: 210 1.2.2.2 skrll pci_intr_release(pa->pa_pc, psc->psc_intrs, psc->psc_nintrs); 211 1.2.2.2 skrll psc->psc_nintrs = 0; 212 1.2.2.2 skrll unmap: 213 1.2.2.2 skrll bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios); 214 1.2.2.2 skrll sc->sc_ios = 0; 215 1.2.2.2 skrll } 216 1.2.2.2 skrll 217 1.2.2.2 skrll static int 218 1.2.2.4 skrll nvme_pci_rescan(device_t self, const char *attr, const int *flags) 219 1.2.2.4 skrll { 220 1.2.2.4 skrll 221 1.2.2.4 skrll return nvme_rescan(self, attr, flags); 222 1.2.2.4 skrll } 223 1.2.2.4 skrll 224 1.2.2.4 skrll static int 225 1.2.2.2 skrll nvme_pci_detach(device_t self, int flags) 226 1.2.2.2 skrll { 227 1.2.2.2 skrll struct nvme_pci_softc *psc = device_private(self); 228 1.2.2.2 skrll struct nvme_softc *sc = &psc->psc_nvme; 229 1.2.2.4 skrll int error; 230 1.2.2.2 skrll 231 1.2.2.2 skrll if (!ISSET(sc->sc_flags, NVME_F_ATTACHED)) 232 1.2.2.2 skrll return 0; 233 1.2.2.2 skrll 234 1.2.2.2 skrll error = nvme_detach(sc, flags); 235 1.2.2.2 skrll if (error) 236 1.2.2.2 skrll return error; 237 1.2.2.2 skrll 238 1.2.2.5 skrll kmem_free(sc->sc_softih, sizeof(*sc->sc_softih) * psc->psc_nintrs); 239 1.2.2.5 skrll sc->sc_softih = NULL; 240 1.2.2.5 skrll 241 1.2.2.4 skrll kmem_free(sc->sc_ih, sizeof(*sc->sc_ih) * psc->psc_nintrs); 242 1.2.2.2 skrll pci_intr_release(psc->psc_pc, psc->psc_intrs, psc->psc_nintrs); 243 1.2.2.2 skrll bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios); 244 1.2.2.2 skrll return 0; 245 1.2.2.2 skrll } 246 1.2.2.2 skrll 247 1.2.2.2 skrll static int 248 1.2.2.2 skrll nvme_pci_intr_establish(struct nvme_softc *sc, uint16_t qid, 249 1.2.2.2 skrll struct nvme_queue *q) 250 1.2.2.2 skrll { 251 1.2.2.2 skrll struct nvme_pci_softc *psc = (struct nvme_pci_softc *)sc; 252 1.2.2.2 skrll char intr_xname[INTRDEVNAMEBUF]; 253 1.2.2.2 skrll char intrbuf[PCI_INTRSTR_LEN]; 254 1.2.2.2 skrll const char *intrstr = NULL; 255 1.2.2.2 skrll int (*ih_func)(void *); 256 1.2.2.5 skrll void (*ih_func_soft)(void *); 257 1.2.2.2 skrll void *ih_arg; 258 1.2.2.2 skrll int error; 259 1.2.2.2 skrll 260 1.2.2.4 skrll KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q); 261 1.2.2.2 skrll KASSERT(sc->sc_ih[qid] == NULL); 262 1.2.2.2 skrll 263 1.2.2.2 skrll if (nvme_pci_mpsafe) { 264 1.2.2.2 skrll pci_intr_setattr(psc->psc_pc, &psc->psc_intrs[qid], 265 1.2.2.2 skrll PCI_INTR_MPSAFE, true); 266 1.2.2.2 skrll } 267 1.2.2.4 skrll 268 1.2.2.2 skrll if (!sc->sc_use_mq) { 269 1.2.2.2 skrll snprintf(intr_xname, sizeof(intr_xname), "%s", 270 1.2.2.2 skrll device_xname(sc->sc_dev)); 271 1.2.2.2 skrll ih_arg = sc; 272 1.2.2.2 skrll ih_func = nvme_intr; 273 1.2.2.5 skrll ih_func_soft = nvme_softintr_intx; 274 1.2.2.5 skrll } else { 275 1.2.2.4 skrll if (qid == NVME_ADMIN_Q) { 276 1.2.2.2 skrll snprintf(intr_xname, sizeof(intr_xname), "%s adminq", 277 1.2.2.2 skrll device_xname(sc->sc_dev)); 278 1.2.2.2 skrll } else { 279 1.2.2.2 skrll snprintf(intr_xname, sizeof(intr_xname), "%s ioq%d", 280 1.2.2.2 skrll device_xname(sc->sc_dev), qid); 281 1.2.2.2 skrll } 282 1.2.2.2 skrll ih_arg = q; 283 1.2.2.4 skrll ih_func = nvme_intr_msi; 284 1.2.2.5 skrll ih_func_soft = nvme_softintr_msi; 285 1.2.2.2 skrll } 286 1.2.2.4 skrll 287 1.2.2.4 skrll /* establish hardware interrupt */ 288 1.2.2.2 skrll sc->sc_ih[qid] = pci_intr_establish_xname(psc->psc_pc, 289 1.2.2.2 skrll psc->psc_intrs[qid], IPL_BIO, ih_func, ih_arg, intr_xname); 290 1.2.2.2 skrll if (sc->sc_ih[qid] == NULL) { 291 1.2.2.2 skrll aprint_error_dev(sc->sc_dev, 292 1.2.2.2 skrll "unable to establish %s interrupt\n", intr_xname); 293 1.2.2.2 skrll return 1; 294 1.2.2.2 skrll } 295 1.2.2.4 skrll 296 1.2.2.5 skrll /* establish also the software interrupt */ 297 1.2.2.5 skrll sc->sc_softih[qid] = softint_establish( 298 1.2.2.5 skrll SOFTINT_BIO|(nvme_pci_mpsafe ? SOFTINT_MPSAFE : 0), 299 1.2.2.5 skrll ih_func_soft, q); 300 1.2.2.5 skrll if (sc->sc_softih[qid] == NULL) { 301 1.2.2.5 skrll pci_intr_disestablish(psc->psc_pc, sc->sc_ih[qid]); 302 1.2.2.5 skrll sc->sc_ih[qid] = NULL; 303 1.2.2.5 skrll 304 1.2.2.5 skrll aprint_error_dev(sc->sc_dev, 305 1.2.2.5 skrll "unable to establish %s soft interrupt\n", 306 1.2.2.5 skrll intr_xname); 307 1.2.2.5 skrll return 1; 308 1.2.2.4 skrll } 309 1.2.2.4 skrll 310 1.2.2.2 skrll intrstr = pci_intr_string(psc->psc_pc, psc->psc_intrs[qid], intrbuf, 311 1.2.2.2 skrll sizeof(intrbuf)); 312 1.2.2.2 skrll if (!sc->sc_use_mq) { 313 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr); 314 1.2.2.5 skrll } else if (qid == NVME_ADMIN_Q) { 315 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, 316 1.2.2.2 skrll "for admin queue interrupting at %s\n", intrstr); 317 1.2.2.2 skrll } else if (!nvme_pci_mpsafe) { 318 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, 319 1.2.2.2 skrll "for io queue %d interrupting at %s\n", qid, intrstr); 320 1.2.2.2 skrll } else { 321 1.2.2.4 skrll kcpuset_t *affinity; 322 1.2.2.4 skrll cpuid_t affinity_to; 323 1.2.2.4 skrll 324 1.2.2.2 skrll kcpuset_create(&affinity, true); 325 1.2.2.2 skrll affinity_to = (qid - 1) % ncpu; 326 1.2.2.2 skrll kcpuset_set(affinity, affinity_to); 327 1.2.2.2 skrll error = interrupt_distribute(sc->sc_ih[qid], affinity, NULL); 328 1.2.2.2 skrll kcpuset_destroy(affinity); 329 1.2.2.2 skrll aprint_normal_dev(sc->sc_dev, 330 1.2.2.2 skrll "for io queue %d interrupting at %s", qid, intrstr); 331 1.2.2.2 skrll if (error == 0) 332 1.2.2.2 skrll aprint_normal(" affinity to cpu%lu", affinity_to); 333 1.2.2.2 skrll aprint_normal("\n"); 334 1.2.2.2 skrll } 335 1.2.2.2 skrll return 0; 336 1.2.2.2 skrll } 337 1.2.2.2 skrll 338 1.2.2.2 skrll static int 339 1.2.2.2 skrll nvme_pci_intr_disestablish(struct nvme_softc *sc, uint16_t qid) 340 1.2.2.2 skrll { 341 1.2.2.2 skrll struct nvme_pci_softc *psc = (struct nvme_pci_softc *)sc; 342 1.2.2.2 skrll 343 1.2.2.4 skrll KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q); 344 1.2.2.2 skrll KASSERT(sc->sc_ih[qid] != NULL); 345 1.2.2.2 skrll 346 1.2.2.4 skrll if (sc->sc_softih) { 347 1.2.2.4 skrll softint_disestablish(sc->sc_softih[qid]); 348 1.2.2.4 skrll sc->sc_softih[qid] = NULL; 349 1.2.2.4 skrll } 350 1.2.2.4 skrll 351 1.2.2.2 skrll pci_intr_disestablish(psc->psc_pc, sc->sc_ih[qid]); 352 1.2.2.2 skrll sc->sc_ih[qid] = NULL; 353 1.2.2.2 skrll 354 1.2.2.2 skrll return 0; 355 1.2.2.2 skrll } 356 1.2.2.2 skrll 357 1.2.2.2 skrll static int 358 1.2.2.2 skrll nvme_pci_setup_intr(struct pci_attach_args *pa, struct nvme_pci_softc *psc) 359 1.2.2.2 skrll { 360 1.2.2.2 skrll struct nvme_softc *sc = &psc->psc_nvme; 361 1.2.2.4 skrll int error; 362 1.2.2.2 skrll int counts[PCI_INTR_TYPE_SIZE], alloced_counts[PCI_INTR_TYPE_SIZE]; 363 1.2.2.4 skrll pci_intr_handle_t *ihps; 364 1.2.2.2 skrll int max_type, intr_type; 365 1.2.2.2 skrll 366 1.2.2.2 skrll if (nvme_pci_force_intx) { 367 1.2.2.2 skrll max_type = PCI_INTR_TYPE_INTX; 368 1.2.2.2 skrll goto force_intx; 369 1.2.2.2 skrll } 370 1.2.2.2 skrll 371 1.2.2.2 skrll /* MSI-X */ 372 1.2.2.2 skrll max_type = PCI_INTR_TYPE_MSIX; 373 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSIX] = min(pci_msix_count(pa->pa_pc, pa->pa_tag), 374 1.2.2.2 skrll ncpu + 1); 375 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSIX] > 0) { 376 1.2.2.2 skrll memset(alloced_counts, 0, sizeof(alloced_counts)); 377 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSIX] = counts[PCI_INTR_TYPE_MSIX]; 378 1.2.2.2 skrll if (pci_intr_alloc(pa, &ihps, alloced_counts, 379 1.2.2.2 skrll PCI_INTR_TYPE_MSIX)) { 380 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSIX] = 0; 381 1.2.2.2 skrll } else { 382 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSIX] = 383 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSIX]; 384 1.2.2.2 skrll pci_intr_release(pa->pa_pc, ihps, 385 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSIX]); 386 1.2.2.2 skrll } 387 1.2.2.2 skrll } 388 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSIX] < 2) { 389 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSIX] = 0; 390 1.2.2.2 skrll max_type = PCI_INTR_TYPE_MSI; 391 1.2.2.2 skrll } else if (!nvme_pci_mq || !nvme_pci_mpsafe) { 392 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSIX] = 2; /* adminq + 1 ioq */ 393 1.2.2.2 skrll } 394 1.2.2.2 skrll 395 1.2.2.2 skrll retry_msi: 396 1.2.2.2 skrll /* MSI */ 397 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] = pci_msi_count(pa->pa_pc, pa->pa_tag); 398 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSI] > 0) { 399 1.2.2.2 skrll while (counts[PCI_INTR_TYPE_MSI] > ncpu + 1) { 400 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSI] / 2 <= ncpu + 1) 401 1.2.2.2 skrll break; 402 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] /= 2; 403 1.2.2.2 skrll } 404 1.2.2.2 skrll memset(alloced_counts, 0, sizeof(alloced_counts)); 405 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSI] = counts[PCI_INTR_TYPE_MSI]; 406 1.2.2.2 skrll if (pci_intr_alloc(pa, &ihps, alloced_counts, 407 1.2.2.2 skrll PCI_INTR_TYPE_MSI)) { 408 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] = 0; 409 1.2.2.2 skrll } else { 410 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] = 411 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSI]; 412 1.2.2.2 skrll pci_intr_release(pa->pa_pc, ihps, 413 1.2.2.2 skrll alloced_counts[PCI_INTR_TYPE_MSI]); 414 1.2.2.2 skrll } 415 1.2.2.2 skrll } 416 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSI] < 1) { 417 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] = 0; 418 1.2.2.2 skrll if (max_type == PCI_INTR_TYPE_MSI) 419 1.2.2.2 skrll max_type = PCI_INTR_TYPE_INTX; 420 1.2.2.2 skrll } else if (!nvme_pci_mq || !nvme_pci_mpsafe) { 421 1.2.2.2 skrll if (counts[PCI_INTR_TYPE_MSI] > 2) 422 1.2.2.2 skrll counts[PCI_INTR_TYPE_MSI] = 2; /* adminq + 1 ioq */ 423 1.2.2.2 skrll } 424 1.2.2.2 skrll 425 1.2.2.2 skrll force_intx: 426 1.2.2.2 skrll /* INTx */ 427 1.2.2.2 skrll counts[PCI_INTR_TYPE_INTX] = 1; 428 1.2.2.2 skrll 429 1.2.2.2 skrll memcpy(alloced_counts, counts, sizeof(counts)); 430 1.2.2.2 skrll error = pci_intr_alloc(pa, &ihps, alloced_counts, max_type); 431 1.2.2.2 skrll if (error) { 432 1.2.2.2 skrll if (max_type != PCI_INTR_TYPE_INTX) { 433 1.2.2.2 skrll retry: 434 1.2.2.2 skrll memset(counts, 0, sizeof(counts)); 435 1.2.2.2 skrll if (max_type == PCI_INTR_TYPE_MSIX) { 436 1.2.2.2 skrll max_type = PCI_INTR_TYPE_MSI; 437 1.2.2.2 skrll goto retry_msi; 438 1.2.2.2 skrll } else { 439 1.2.2.2 skrll max_type = PCI_INTR_TYPE_INTX; 440 1.2.2.2 skrll goto force_intx; 441 1.2.2.2 skrll } 442 1.2.2.2 skrll } 443 1.2.2.2 skrll return error; 444 1.2.2.2 skrll } 445 1.2.2.2 skrll 446 1.2.2.4 skrll intr_type = pci_intr_type(pa->pa_pc, ihps[0]); 447 1.2.2.2 skrll if (alloced_counts[intr_type] < counts[intr_type]) { 448 1.2.2.2 skrll if (intr_type != PCI_INTR_TYPE_INTX) { 449 1.2.2.2 skrll pci_intr_release(pa->pa_pc, ihps, 450 1.2.2.2 skrll alloced_counts[intr_type]); 451 1.2.2.2 skrll max_type = intr_type; 452 1.2.2.2 skrll goto retry; 453 1.2.2.2 skrll } 454 1.2.2.2 skrll return EBUSY; 455 1.2.2.2 skrll } 456 1.2.2.2 skrll 457 1.2.2.2 skrll psc->psc_intrs = ihps; 458 1.2.2.2 skrll psc->psc_nintrs = alloced_counts[intr_type]; 459 1.2.2.2 skrll if (intr_type == PCI_INTR_TYPE_MSI) { 460 1.2.2.2 skrll if (alloced_counts[intr_type] > ncpu + 1) 461 1.2.2.2 skrll alloced_counts[intr_type] = ncpu + 1; 462 1.2.2.2 skrll } 463 1.2.2.2 skrll sc->sc_use_mq = alloced_counts[intr_type] > 1; 464 1.2.2.2 skrll sc->sc_nq = sc->sc_use_mq ? alloced_counts[intr_type] - 1 : 1; 465 1.2.2.4 skrll 466 1.2.2.4 skrll 467 1.2.2.2 skrll return 0; 468 1.2.2.2 skrll } 469 1.2.2.4 skrll 470 1.2.2.4 skrll MODULE(MODULE_CLASS_DRIVER, nvme, "pci,dk_subr"); 471 1.2.2.4 skrll 472 1.2.2.4 skrll #ifdef _MODULE 473 1.2.2.4 skrll #include "ioconf.c" 474 1.2.2.4 skrll #endif 475 1.2.2.4 skrll 476 1.2.2.4 skrll static int 477 1.2.2.4 skrll nvme_modcmd(modcmd_t cmd, void *opaque) 478 1.2.2.4 skrll { 479 1.2.2.4 skrll #ifdef _MODULE 480 1.2.2.4 skrll devmajor_t cmajor, bmajor; 481 1.2.2.4 skrll extern const struct cdevsw nvme_cdevsw; 482 1.2.2.4 skrll #endif 483 1.2.2.4 skrll int error = 0; 484 1.2.2.4 skrll 485 1.2.2.4 skrll #ifdef _MODULE 486 1.2.2.4 skrll switch (cmd) { 487 1.2.2.4 skrll case MODULE_CMD_INIT: 488 1.2.2.4 skrll error = config_init_component(cfdriver_ioconf_nvme_pci, 489 1.2.2.4 skrll cfattach_ioconf_nvme_pci, cfdata_ioconf_nvme_pci); 490 1.2.2.4 skrll if (error) 491 1.2.2.4 skrll break; 492 1.2.2.4 skrll 493 1.2.2.4 skrll bmajor = cmajor = NODEVMAJOR; 494 1.2.2.4 skrll error = devsw_attach(nvme_cd.cd_name, NULL, &bmajor, 495 1.2.2.4 skrll &nvme_cdevsw, &cmajor); 496 1.2.2.4 skrll if (error) { 497 1.2.2.4 skrll aprint_error("%s: unable to register devsw\n", 498 1.2.2.4 skrll nvme_cd.cd_name); 499 1.2.2.4 skrll /* do not abort, just /dev/nvme* will not work */ 500 1.2.2.4 skrll } 501 1.2.2.4 skrll break; 502 1.2.2.4 skrll case MODULE_CMD_FINI: 503 1.2.2.4 skrll devsw_detach(NULL, &nvme_cdevsw); 504 1.2.2.4 skrll 505 1.2.2.4 skrll error = config_fini_component(cfdriver_ioconf_nvme_pci, 506 1.2.2.4 skrll cfattach_ioconf_nvme_pci, cfdata_ioconf_nvme_pci); 507 1.2.2.4 skrll break; 508 1.2.2.4 skrll default: 509 1.2.2.4 skrll break; 510 1.2.2.4 skrll } 511 1.2.2.4 skrll #endif 512 1.2.2.4 skrll return error; 513 1.2.2.4 skrll } 514
Indexes created Fri Oct 03 14:09:59 GMT 2025