nvme.c revision 1.30.2.1 1 1.30.2.1 martin /* $NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $ */
2 1.1 nonaka /* $OpenBSD: nvme.c,v 1.49 2016/04/18 05:59:50 dlg Exp $ */
3 1.1 nonaka
4 1.1 nonaka /*
5 1.1 nonaka * Copyright (c) 2014 David Gwynne <dlg (at) openbsd.org>
6 1.1 nonaka *
7 1.1 nonaka * Permission to use, copy, modify, and distribute this software for any
8 1.1 nonaka * purpose with or without fee is hereby granted, provided that the above
9 1.1 nonaka * copyright notice and this permission notice appear in all copies.
10 1.1 nonaka *
11 1.1 nonaka * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12 1.1 nonaka * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13 1.1 nonaka * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14 1.1 nonaka * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15 1.1 nonaka * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16 1.1 nonaka * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17 1.1 nonaka * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18 1.1 nonaka */
19 1.1 nonaka
20 1.1 nonaka #include <sys/cdefs.h>
21 1.30.2.1 martin __KERNEL_RCSID(0, "$NetBSD: nvme.c,v 1.30.2.1 2018/03/17 08:11:18 martin Exp $");
22 1.1 nonaka
23 1.1 nonaka #include <sys/param.h>
24 1.1 nonaka #include <sys/systm.h>
25 1.1 nonaka #include <sys/kernel.h>
26 1.1 nonaka #include <sys/atomic.h>
27 1.1 nonaka #include <sys/bus.h>
28 1.1 nonaka #include <sys/buf.h>
29 1.3 nonaka #include <sys/conf.h>
30 1.1 nonaka #include <sys/device.h>
31 1.1 nonaka #include <sys/kmem.h>
32 1.1 nonaka #include <sys/once.h>
33 1.3 nonaka #include <sys/proc.h>
34 1.1 nonaka #include <sys/queue.h>
35 1.1 nonaka #include <sys/mutex.h>
36 1.1 nonaka
37 1.3 nonaka #include <uvm/uvm_extern.h>
38 1.3 nonaka
39 1.1 nonaka #include <dev/ic/nvmereg.h>
40 1.1 nonaka #include <dev/ic/nvmevar.h>
41 1.3 nonaka #include <dev/ic/nvmeio.h>
42 1.1 nonaka
43 1.22 jdolecek int nvme_adminq_size = 32;
44 1.9 jdolecek int nvme_ioq_size = 1024;
45 1.1 nonaka
46 1.1 nonaka static int nvme_print(void *, const char *);
47 1.1 nonaka
48 1.1 nonaka static int nvme_ready(struct nvme_softc *, uint32_t);
49 1.1 nonaka static int nvme_enable(struct nvme_softc *, u_int);
50 1.1 nonaka static int nvme_disable(struct nvme_softc *);
51 1.1 nonaka static int nvme_shutdown(struct nvme_softc *);
52 1.1 nonaka
53 1.1 nonaka #ifdef NVME_DEBUG
54 1.1 nonaka static void nvme_dumpregs(struct nvme_softc *);
55 1.1 nonaka #endif
56 1.1 nonaka static int nvme_identify(struct nvme_softc *, u_int);
57 1.1 nonaka static void nvme_fill_identify(struct nvme_queue *, struct nvme_ccb *,
58 1.1 nonaka void *);
59 1.1 nonaka
60 1.20 jdolecek static int nvme_ccbs_alloc(struct nvme_queue *, uint16_t);
61 1.1 nonaka static void nvme_ccbs_free(struct nvme_queue *);
62 1.1 nonaka
63 1.1 nonaka static struct nvme_ccb *
64 1.1 nonaka nvme_ccb_get(struct nvme_queue *);
65 1.1 nonaka static void nvme_ccb_put(struct nvme_queue *, struct nvme_ccb *);
66 1.1 nonaka
67 1.1 nonaka static int nvme_poll(struct nvme_softc *, struct nvme_queue *,
68 1.1 nonaka struct nvme_ccb *, void (*)(struct nvme_queue *,
69 1.7 jdolecek struct nvme_ccb *, void *), int);
70 1.1 nonaka static void nvme_poll_fill(struct nvme_queue *, struct nvme_ccb *, void *);
71 1.1 nonaka static void nvme_poll_done(struct nvme_queue *, struct nvme_ccb *,
72 1.1 nonaka struct nvme_cqe *);
73 1.1 nonaka static void nvme_sqe_fill(struct nvme_queue *, struct nvme_ccb *, void *);
74 1.1 nonaka static void nvme_empty_done(struct nvme_queue *, struct nvme_ccb *,
75 1.1 nonaka struct nvme_cqe *);
76 1.1 nonaka
77 1.1 nonaka static struct nvme_queue *
78 1.1 nonaka nvme_q_alloc(struct nvme_softc *, uint16_t, u_int, u_int);
79 1.1 nonaka static int nvme_q_create(struct nvme_softc *, struct nvme_queue *);
80 1.1 nonaka static int nvme_q_delete(struct nvme_softc *, struct nvme_queue *);
81 1.1 nonaka static void nvme_q_submit(struct nvme_softc *, struct nvme_queue *,
82 1.1 nonaka struct nvme_ccb *, void (*)(struct nvme_queue *,
83 1.1 nonaka struct nvme_ccb *, void *));
84 1.1 nonaka static int nvme_q_complete(struct nvme_softc *, struct nvme_queue *q);
85 1.1 nonaka static void nvme_q_free(struct nvme_softc *, struct nvme_queue *);
86 1.1 nonaka
87 1.19 jdolecek static struct nvme_dmamem *
88 1.19 jdolecek nvme_dmamem_alloc(struct nvme_softc *, size_t);
89 1.1 nonaka static void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *);
90 1.19 jdolecek static void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *,
91 1.19 jdolecek int);
92 1.1 nonaka
93 1.1 nonaka static void nvme_ns_io_fill(struct nvme_queue *, struct nvme_ccb *,
94 1.1 nonaka void *);
95 1.1 nonaka static void nvme_ns_io_done(struct nvme_queue *, struct nvme_ccb *,
96 1.1 nonaka struct nvme_cqe *);
97 1.1 nonaka static void nvme_ns_sync_fill(struct nvme_queue *, struct nvme_ccb *,
98 1.1 nonaka void *);
99 1.1 nonaka static void nvme_ns_sync_done(struct nvme_queue *, struct nvme_ccb *,
100 1.1 nonaka struct nvme_cqe *);
101 1.25 jdolecek static void nvme_getcache_fill(struct nvme_queue *, struct nvme_ccb *,
102 1.25 jdolecek void *);
103 1.25 jdolecek static void nvme_getcache_done(struct nvme_queue *, struct nvme_ccb *,
104 1.25 jdolecek struct nvme_cqe *);
105 1.1 nonaka
106 1.3 nonaka static void nvme_pt_fill(struct nvme_queue *, struct nvme_ccb *,
107 1.3 nonaka void *);
108 1.3 nonaka static void nvme_pt_done(struct nvme_queue *, struct nvme_ccb *,
109 1.3 nonaka struct nvme_cqe *);
110 1.3 nonaka static int nvme_command_passthrough(struct nvme_softc *,
111 1.3 nonaka struct nvme_pt_command *, uint16_t, struct lwp *, bool);
112 1.3 nonaka
113 1.23 nonaka static int nvme_get_number_of_queues(struct nvme_softc *, u_int *);
114 1.23 nonaka
115 1.7 jdolecek #define NVME_TIMO_QOP 5 /* queue create and delete timeout */
116 1.7 jdolecek #define NVME_TIMO_IDENT 10 /* probe identify timeout */
117 1.7 jdolecek #define NVME_TIMO_PT -1 /* passthrough cmd timeout */
118 1.13 jdolecek #define NVME_TIMO_SY 60 /* sync cache timeout */
119 1.7 jdolecek
120 1.1 nonaka #define nvme_read4(_s, _r) \
121 1.1 nonaka bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r))
122 1.1 nonaka #define nvme_write4(_s, _r, _v) \
123 1.1 nonaka bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v))
124 1.28 nonaka /*
125 1.28 nonaka * Some controllers, at least Apple NVMe, always require split
126 1.28 nonaka * transfers, so don't use bus_space_{read,write}_8() on LP64.
127 1.28 nonaka */
128 1.1 nonaka static inline uint64_t
129 1.1 nonaka nvme_read8(struct nvme_softc *sc, bus_size_t r)
130 1.1 nonaka {
131 1.1 nonaka uint64_t v;
132 1.1 nonaka uint32_t *a = (uint32_t *)&v;
133 1.1 nonaka
134 1.1 nonaka #if _BYTE_ORDER == _LITTLE_ENDIAN
135 1.1 nonaka a[0] = nvme_read4(sc, r);
136 1.1 nonaka a[1] = nvme_read4(sc, r + 4);
137 1.1 nonaka #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
138 1.1 nonaka a[1] = nvme_read4(sc, r);
139 1.1 nonaka a[0] = nvme_read4(sc, r + 4);
140 1.1 nonaka #endif
141 1.1 nonaka
142 1.1 nonaka return v;
143 1.1 nonaka }
144 1.1 nonaka
145 1.1 nonaka static inline void
146 1.1 nonaka nvme_write8(struct nvme_softc *sc, bus_size_t r, uint64_t v)
147 1.1 nonaka {
148 1.1 nonaka uint32_t *a = (uint32_t *)&v;
149 1.1 nonaka
150 1.1 nonaka #if _BYTE_ORDER == _LITTLE_ENDIAN
151 1.1 nonaka nvme_write4(sc, r, a[0]);
152 1.1 nonaka nvme_write4(sc, r + 4, a[1]);
153 1.1 nonaka #else /* _BYTE_ORDER == _LITTLE_ENDIAN */
154 1.1 nonaka nvme_write4(sc, r, a[1]);
155 1.1 nonaka nvme_write4(sc, r + 4, a[0]);
156 1.1 nonaka #endif
157 1.1 nonaka }
158 1.1 nonaka #define nvme_barrier(_s, _r, _l, _f) \
159 1.1 nonaka bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f))
160 1.1 nonaka
161 1.1 nonaka #ifdef NVME_DEBUG
162 1.6 jdolecek static __used void
163 1.1 nonaka nvme_dumpregs(struct nvme_softc *sc)
164 1.1 nonaka {
165 1.1 nonaka uint64_t r8;
166 1.1 nonaka uint32_t r4;
167 1.1 nonaka
168 1.1 nonaka #define DEVNAME(_sc) device_xname((_sc)->sc_dev)
169 1.1 nonaka r8 = nvme_read8(sc, NVME_CAP);
170 1.8 jdolecek printf("%s: cap 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP));
171 1.1 nonaka printf("%s: mpsmax %u (%u)\n", DEVNAME(sc),
172 1.1 nonaka (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8)));
173 1.1 nonaka printf("%s: mpsmin %u (%u)\n", DEVNAME(sc),
174 1.1 nonaka (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8)));
175 1.8 jdolecek printf("%s: css %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CSS(r8));
176 1.8 jdolecek printf("%s: nssrs %"PRIu64"\n", DEVNAME(sc), NVME_CAP_NSSRS(r8));
177 1.8 jdolecek printf("%s: dstrd %"PRIu64"\n", DEVNAME(sc), NVME_CAP_DSTRD(r8));
178 1.8 jdolecek printf("%s: to %"PRIu64" msec\n", DEVNAME(sc), NVME_CAP_TO(r8));
179 1.8 jdolecek printf("%s: ams %"PRIu64"\n", DEVNAME(sc), NVME_CAP_AMS(r8));
180 1.8 jdolecek printf("%s: cqr %"PRIu64"\n", DEVNAME(sc), NVME_CAP_CQR(r8));
181 1.8 jdolecek printf("%s: mqes %"PRIu64"\n", DEVNAME(sc), NVME_CAP_MQES(r8));
182 1.1 nonaka
183 1.1 nonaka printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS));
184 1.1 nonaka
185 1.1 nonaka r4 = nvme_read4(sc, NVME_CC);
186 1.1 nonaka printf("%s: cc 0x%04x\n", DEVNAME(sc), r4);
187 1.8 jdolecek printf("%s: iocqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4),
188 1.8 jdolecek (1 << NVME_CC_IOCQES_R(r4)));
189 1.8 jdolecek printf("%s: iosqes %u (%u)\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4),
190 1.8 jdolecek (1 << NVME_CC_IOSQES_R(r4)));
191 1.1 nonaka printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4));
192 1.1 nonaka printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4));
193 1.8 jdolecek printf("%s: mps %u (%u)\n", DEVNAME(sc), NVME_CC_MPS_R(r4),
194 1.8 jdolecek (1 << NVME_CC_MPS_R(r4)));
195 1.1 nonaka printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4));
196 1.6 jdolecek printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN) ? 1 : 0);
197 1.1 nonaka
198 1.8 jdolecek r4 = nvme_read4(sc, NVME_CSTS);
199 1.8 jdolecek printf("%s: csts 0x%08x\n", DEVNAME(sc), r4);
200 1.8 jdolecek printf("%s: rdy %u\n", DEVNAME(sc), r4 & NVME_CSTS_RDY);
201 1.8 jdolecek printf("%s: cfs %u\n", DEVNAME(sc), r4 & NVME_CSTS_CFS);
202 1.8 jdolecek printf("%s: shst %x\n", DEVNAME(sc), r4 & NVME_CSTS_SHST_MASK);
203 1.8 jdolecek
204 1.8 jdolecek r4 = nvme_read4(sc, NVME_AQA);
205 1.8 jdolecek printf("%s: aqa 0x%08x\n", DEVNAME(sc), r4);
206 1.8 jdolecek printf("%s: acqs %u\n", DEVNAME(sc), NVME_AQA_ACQS_R(r4));
207 1.8 jdolecek printf("%s: asqs %u\n", DEVNAME(sc), NVME_AQA_ASQS_R(r4));
208 1.8 jdolecek
209 1.8 jdolecek printf("%s: asq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ));
210 1.8 jdolecek printf("%s: acq 0x%016"PRIx64"\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ));
211 1.1 nonaka #undef DEVNAME
212 1.1 nonaka }
213 1.1 nonaka #endif /* NVME_DEBUG */
214 1.1 nonaka
215 1.1 nonaka static int
216 1.1 nonaka nvme_ready(struct nvme_softc *sc, uint32_t rdy)
217 1.1 nonaka {
218 1.1 nonaka u_int i = 0;
219 1.8 jdolecek uint32_t cc;
220 1.8 jdolecek
221 1.8 jdolecek cc = nvme_read4(sc, NVME_CC);
222 1.8 jdolecek if (((cc & NVME_CC_EN) != 0) != (rdy != 0)) {
223 1.8 jdolecek aprint_error_dev(sc->sc_dev,
224 1.8 jdolecek "controller enabled status expected %d, found to be %d\n",
225 1.8 jdolecek (rdy != 0), ((cc & NVME_CC_EN) != 0));
226 1.8 jdolecek return ENXIO;
227 1.8 jdolecek }
228 1.1 nonaka
229 1.1 nonaka while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) {
230 1.1 nonaka if (i++ > sc->sc_rdy_to)
231 1.8 jdolecek return ENXIO;
232 1.1 nonaka
233 1.1 nonaka delay(1000);
234 1.1 nonaka nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ);
235 1.1 nonaka }
236 1.1 nonaka
237 1.1 nonaka return 0;
238 1.1 nonaka }
239 1.1 nonaka
240 1.1 nonaka static int
241 1.1 nonaka nvme_enable(struct nvme_softc *sc, u_int mps)
242 1.1 nonaka {
243 1.8 jdolecek uint32_t cc, csts;
244 1.1 nonaka
245 1.1 nonaka cc = nvme_read4(sc, NVME_CC);
246 1.8 jdolecek csts = nvme_read4(sc, NVME_CSTS);
247 1.8 jdolecek
248 1.7 jdolecek if (ISSET(cc, NVME_CC_EN)) {
249 1.7 jdolecek aprint_error_dev(sc->sc_dev, "controller unexpectedly enabled, failed to stay disabled\n");
250 1.8 jdolecek
251 1.8 jdolecek if (ISSET(csts, NVME_CSTS_RDY))
252 1.8 jdolecek return 1;
253 1.8 jdolecek
254 1.8 jdolecek goto waitready;
255 1.7 jdolecek }
256 1.1 nonaka
257 1.1 nonaka nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem));
258 1.1 nonaka nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
259 1.8 jdolecek delay(5000);
260 1.1 nonaka nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem));
261 1.1 nonaka nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
262 1.8 jdolecek delay(5000);
263 1.8 jdolecek
264 1.8 jdolecek nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) |
265 1.8 jdolecek NVME_AQA_ASQS(sc->sc_admin_q->q_entries));
266 1.8 jdolecek nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE);
267 1.8 jdolecek delay(5000);
268 1.1 nonaka
269 1.1 nonaka CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK |
270 1.1 nonaka NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK);
271 1.1 nonaka SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1));
272 1.1 nonaka SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE));
273 1.1 nonaka SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM));
274 1.1 nonaka SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR));
275 1.1 nonaka SET(cc, NVME_CC_MPS(mps));
276 1.1 nonaka SET(cc, NVME_CC_EN);
277 1.1 nonaka
278 1.1 nonaka nvme_write4(sc, NVME_CC, cc);
279 1.1 nonaka nvme_barrier(sc, 0, sc->sc_ios,
280 1.1 nonaka BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
281 1.8 jdolecek delay(5000);
282 1.1 nonaka
283 1.8 jdolecek waitready:
284 1.1 nonaka return nvme_ready(sc, NVME_CSTS_RDY);
285 1.1 nonaka }
286 1.1 nonaka
287 1.1 nonaka static int
288 1.1 nonaka nvme_disable(struct nvme_softc *sc)
289 1.1 nonaka {
290 1.1 nonaka uint32_t cc, csts;
291 1.1 nonaka
292 1.1 nonaka cc = nvme_read4(sc, NVME_CC);
293 1.8 jdolecek csts = nvme_read4(sc, NVME_CSTS);
294 1.8 jdolecek
295 1.8 jdolecek if (ISSET(cc, NVME_CC_EN) && !ISSET(csts, NVME_CSTS_RDY))
296 1.8 jdolecek nvme_ready(sc, NVME_CSTS_RDY);
297 1.1 nonaka
298 1.1 nonaka CLR(cc, NVME_CC_EN);
299 1.1 nonaka
300 1.1 nonaka nvme_write4(sc, NVME_CC, cc);
301 1.8 jdolecek nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_READ);
302 1.8 jdolecek
303 1.8 jdolecek delay(5000);
304 1.1 nonaka
305 1.1 nonaka return nvme_ready(sc, 0);
306 1.1 nonaka }
307 1.1 nonaka
308 1.1 nonaka int
309 1.1 nonaka nvme_attach(struct nvme_softc *sc)
310 1.1 nonaka {
311 1.1 nonaka uint64_t cap;
312 1.1 nonaka uint32_t reg;
313 1.1 nonaka u_int dstrd;
314 1.1 nonaka u_int mps = PAGE_SHIFT;
315 1.23 nonaka u_int ioq_allocated;
316 1.20 jdolecek uint16_t adminq_entries = nvme_adminq_size;
317 1.20 jdolecek uint16_t ioq_entries = nvme_ioq_size;
318 1.1 nonaka int i;
319 1.1 nonaka
320 1.1 nonaka reg = nvme_read4(sc, NVME_VS);
321 1.1 nonaka if (reg == 0xffffffff) {
322 1.1 nonaka aprint_error_dev(sc->sc_dev, "invalid mapping\n");
323 1.1 nonaka return 1;
324 1.1 nonaka }
325 1.1 nonaka
326 1.27 nonaka if (NVME_VS_TER(reg) == 0)
327 1.27 nonaka aprint_normal_dev(sc->sc_dev, "NVMe %d.%d\n", NVME_VS_MJR(reg),
328 1.27 nonaka NVME_VS_MNR(reg));
329 1.27 nonaka else
330 1.27 nonaka aprint_normal_dev(sc->sc_dev, "NVMe %d.%d.%d\n", NVME_VS_MJR(reg),
331 1.27 nonaka NVME_VS_MNR(reg), NVME_VS_TER(reg));
332 1.1 nonaka
333 1.1 nonaka cap = nvme_read8(sc, NVME_CAP);
334 1.1 nonaka dstrd = NVME_CAP_DSTRD(cap);
335 1.1 nonaka if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) {
336 1.1 nonaka aprint_error_dev(sc->sc_dev, "NVMe minimum page size %u "
337 1.1 nonaka "is greater than CPU page size %u\n",
338 1.1 nonaka 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT);
339 1.1 nonaka return 1;
340 1.1 nonaka }
341 1.1 nonaka if (NVME_CAP_MPSMAX(cap) < mps)
342 1.1 nonaka mps = NVME_CAP_MPSMAX(cap);
343 1.15 nonaka if (ioq_entries > NVME_CAP_MQES(cap))
344 1.15 nonaka ioq_entries = NVME_CAP_MQES(cap);
345 1.1 nonaka
346 1.8 jdolecek /* set initial values to be used for admin queue during probe */
347 1.1 nonaka sc->sc_rdy_to = NVME_CAP_TO(cap);
348 1.1 nonaka sc->sc_mps = 1 << mps;
349 1.1 nonaka sc->sc_mdts = MAXPHYS;
350 1.1 nonaka sc->sc_max_sgl = 2;
351 1.1 nonaka
352 1.1 nonaka if (nvme_disable(sc) != 0) {
353 1.1 nonaka aprint_error_dev(sc->sc_dev, "unable to disable controller\n");
354 1.1 nonaka return 1;
355 1.1 nonaka }
356 1.1 nonaka
357 1.1 nonaka sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, adminq_entries, dstrd);
358 1.1 nonaka if (sc->sc_admin_q == NULL) {
359 1.1 nonaka aprint_error_dev(sc->sc_dev,
360 1.1 nonaka "unable to allocate admin queue\n");
361 1.1 nonaka return 1;
362 1.1 nonaka }
363 1.1 nonaka if (sc->sc_intr_establish(sc, NVME_ADMIN_Q, sc->sc_admin_q))
364 1.1 nonaka goto free_admin_q;
365 1.1 nonaka
366 1.1 nonaka if (nvme_enable(sc, mps) != 0) {
367 1.1 nonaka aprint_error_dev(sc->sc_dev, "unable to enable controller\n");
368 1.1 nonaka goto disestablish_admin_q;
369 1.1 nonaka }
370 1.1 nonaka
371 1.1 nonaka if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) {
372 1.1 nonaka aprint_error_dev(sc->sc_dev, "unable to identify controller\n");
373 1.1 nonaka goto disable;
374 1.1 nonaka }
375 1.1 nonaka
376 1.1 nonaka /* we know how big things are now */
377 1.1 nonaka sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps;
378 1.1 nonaka
379 1.1 nonaka /* reallocate ccbs of admin queue with new max sgl. */
380 1.1 nonaka nvme_ccbs_free(sc->sc_admin_q);
381 1.1 nonaka nvme_ccbs_alloc(sc->sc_admin_q, sc->sc_admin_q->q_entries);
382 1.1 nonaka
383 1.23 nonaka if (sc->sc_use_mq) {
384 1.23 nonaka /* Limit the number of queues to the number allocated in HW */
385 1.23 nonaka if (nvme_get_number_of_queues(sc, &ioq_allocated) != 0) {
386 1.23 nonaka aprint_error_dev(sc->sc_dev,
387 1.23 nonaka "unable to get number of queues\n");
388 1.23 nonaka goto disable;
389 1.23 nonaka }
390 1.23 nonaka if (sc->sc_nq > ioq_allocated)
391 1.23 nonaka sc->sc_nq = ioq_allocated;
392 1.23 nonaka }
393 1.23 nonaka
394 1.1 nonaka sc->sc_q = kmem_zalloc(sizeof(*sc->sc_q) * sc->sc_nq, KM_SLEEP);
395 1.1 nonaka for (i = 0; i < sc->sc_nq; i++) {
396 1.1 nonaka sc->sc_q[i] = nvme_q_alloc(sc, i + 1, ioq_entries, dstrd);
397 1.1 nonaka if (sc->sc_q[i] == NULL) {
398 1.1 nonaka aprint_error_dev(sc->sc_dev,
399 1.1 nonaka "unable to allocate io queue\n");
400 1.1 nonaka goto free_q;
401 1.1 nonaka }
402 1.1 nonaka if (nvme_q_create(sc, sc->sc_q[i]) != 0) {
403 1.1 nonaka aprint_error_dev(sc->sc_dev,
404 1.1 nonaka "unable to create io queue\n");
405 1.1 nonaka nvme_q_free(sc, sc->sc_q[i]);
406 1.1 nonaka goto free_q;
407 1.1 nonaka }
408 1.1 nonaka }
409 1.1 nonaka
410 1.1 nonaka if (!sc->sc_use_mq)
411 1.1 nonaka nvme_write4(sc, NVME_INTMC, 1);
412 1.1 nonaka
413 1.9 jdolecek /* probe subdevices */
414 1.1 nonaka sc->sc_namespaces = kmem_zalloc(sizeof(*sc->sc_namespaces) * sc->sc_nn,
415 1.1 nonaka KM_SLEEP);
416 1.14 pgoyette nvme_rescan(sc->sc_dev, "nvme", &i);
417 1.1 nonaka
418 1.1 nonaka return 0;
419 1.1 nonaka
420 1.1 nonaka free_q:
421 1.1 nonaka while (--i >= 0) {
422 1.1 nonaka nvme_q_delete(sc, sc->sc_q[i]);
423 1.1 nonaka nvme_q_free(sc, sc->sc_q[i]);
424 1.1 nonaka }
425 1.1 nonaka disable:
426 1.1 nonaka nvme_disable(sc);
427 1.1 nonaka disestablish_admin_q:
428 1.1 nonaka sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
429 1.1 nonaka free_admin_q:
430 1.1 nonaka nvme_q_free(sc, sc->sc_admin_q);
431 1.1 nonaka
432 1.1 nonaka return 1;
433 1.1 nonaka }
434 1.1 nonaka
435 1.14 pgoyette int
436 1.14 pgoyette nvme_rescan(device_t self, const char *attr, const int *flags)
437 1.14 pgoyette {
438 1.14 pgoyette struct nvme_softc *sc = device_private(self);
439 1.14 pgoyette struct nvme_attach_args naa;
440 1.15 nonaka uint64_t cap;
441 1.15 nonaka int ioq_entries = nvme_ioq_size;
442 1.15 nonaka int i;
443 1.15 nonaka
444 1.15 nonaka cap = nvme_read8(sc, NVME_CAP);
445 1.15 nonaka if (ioq_entries > NVME_CAP_MQES(cap))
446 1.15 nonaka ioq_entries = NVME_CAP_MQES(cap);
447 1.14 pgoyette
448 1.14 pgoyette for (i = 0; i < sc->sc_nn; i++) {
449 1.14 pgoyette if (sc->sc_namespaces[i].dev)
450 1.14 pgoyette continue;
451 1.14 pgoyette memset(&naa, 0, sizeof(naa));
452 1.14 pgoyette naa.naa_nsid = i + 1;
453 1.21 jdolecek naa.naa_qentries = (ioq_entries - 1) * sc->sc_nq;
454 1.21 jdolecek naa.naa_maxphys = sc->sc_mdts;
455 1.14 pgoyette sc->sc_namespaces[i].dev = config_found(sc->sc_dev, &naa,
456 1.14 pgoyette nvme_print);
457 1.14 pgoyette }
458 1.14 pgoyette return 0;
459 1.14 pgoyette }
460 1.14 pgoyette
461 1.1 nonaka static int
462 1.1 nonaka nvme_print(void *aux, const char *pnp)
463 1.1 nonaka {
464 1.1 nonaka struct nvme_attach_args *naa = aux;
465 1.1 nonaka
466 1.1 nonaka if (pnp)
467 1.1 nonaka aprint_normal("at %s", pnp);
468 1.1 nonaka
469 1.1 nonaka if (naa->naa_nsid > 0)
470 1.1 nonaka aprint_normal(" nsid %d", naa->naa_nsid);
471 1.1 nonaka
472 1.1 nonaka return UNCONF;
473 1.1 nonaka }
474 1.1 nonaka
475 1.1 nonaka int
476 1.1 nonaka nvme_detach(struct nvme_softc *sc, int flags)
477 1.1 nonaka {
478 1.1 nonaka int i, error;
479 1.1 nonaka
480 1.1 nonaka error = config_detach_children(sc->sc_dev, flags);
481 1.1 nonaka if (error)
482 1.1 nonaka return error;
483 1.1 nonaka
484 1.1 nonaka error = nvme_shutdown(sc);
485 1.1 nonaka if (error)
486 1.1 nonaka return error;
487 1.1 nonaka
488 1.9 jdolecek /* from now on we are committed to detach, following will never fail */
489 1.1 nonaka for (i = 0; i < sc->sc_nq; i++)
490 1.1 nonaka nvme_q_free(sc, sc->sc_q[i]);
491 1.1 nonaka kmem_free(sc->sc_q, sizeof(*sc->sc_q) * sc->sc_nq);
492 1.1 nonaka nvme_q_free(sc, sc->sc_admin_q);
493 1.1 nonaka
494 1.1 nonaka return 0;
495 1.1 nonaka }
496 1.1 nonaka
497 1.1 nonaka static int
498 1.1 nonaka nvme_shutdown(struct nvme_softc *sc)
499 1.1 nonaka {
500 1.1 nonaka uint32_t cc, csts;
501 1.1 nonaka bool disabled = false;
502 1.1 nonaka int i;
503 1.1 nonaka
504 1.1 nonaka if (!sc->sc_use_mq)
505 1.1 nonaka nvme_write4(sc, NVME_INTMS, 1);
506 1.1 nonaka
507 1.1 nonaka for (i = 0; i < sc->sc_nq; i++) {
508 1.1 nonaka if (nvme_q_delete(sc, sc->sc_q[i]) != 0) {
509 1.1 nonaka aprint_error_dev(sc->sc_dev,
510 1.1 nonaka "unable to delete io queue %d, disabling\n", i + 1);
511 1.1 nonaka disabled = true;
512 1.1 nonaka }
513 1.1 nonaka }
514 1.1 nonaka sc->sc_intr_disestablish(sc, NVME_ADMIN_Q);
515 1.1 nonaka if (disabled)
516 1.1 nonaka goto disable;
517 1.1 nonaka
518 1.1 nonaka cc = nvme_read4(sc, NVME_CC);
519 1.1 nonaka CLR(cc, NVME_CC_SHN_MASK);
520 1.1 nonaka SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL));
521 1.1 nonaka nvme_write4(sc, NVME_CC, cc);
522 1.1 nonaka
523 1.1 nonaka for (i = 0; i < 4000; i++) {
524 1.1 nonaka nvme_barrier(sc, 0, sc->sc_ios,
525 1.1 nonaka BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE);
526 1.1 nonaka csts = nvme_read4(sc, NVME_CSTS);
527 1.1 nonaka if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE)
528 1.1 nonaka return 0;
529 1.1 nonaka
530 1.1 nonaka delay(1000);
531 1.1 nonaka }
532 1.1 nonaka
533 1.1 nonaka aprint_error_dev(sc->sc_dev, "unable to shudown, disabling\n");
534 1.1 nonaka
535 1.1 nonaka disable:
536 1.1 nonaka nvme_disable(sc);
537 1.1 nonaka return 0;
538 1.1 nonaka }
539 1.1 nonaka
540 1.1 nonaka void
541 1.1 nonaka nvme_childdet(device_t self, device_t child)
542 1.1 nonaka {
543 1.1 nonaka struct nvme_softc *sc = device_private(self);
544 1.1 nonaka int i;
545 1.1 nonaka
546 1.1 nonaka for (i = 0; i < sc->sc_nn; i++) {
547 1.1 nonaka if (sc->sc_namespaces[i].dev == child) {
548 1.1 nonaka /* Already freed ns->ident. */
549 1.1 nonaka sc->sc_namespaces[i].dev = NULL;
550 1.1 nonaka break;
551 1.1 nonaka }
552 1.1 nonaka }
553 1.1 nonaka }
554 1.1 nonaka
555 1.1 nonaka int
556 1.1 nonaka nvme_ns_identify(struct nvme_softc *sc, uint16_t nsid)
557 1.1 nonaka {
558 1.1 nonaka struct nvme_sqe sqe;
559 1.1 nonaka struct nvm_identify_namespace *identify;
560 1.19 jdolecek struct nvme_dmamem *mem;
561 1.1 nonaka struct nvme_ccb *ccb;
562 1.1 nonaka struct nvme_namespace *ns;
563 1.19 jdolecek int rv;
564 1.1 nonaka
565 1.1 nonaka KASSERT(nsid > 0);
566 1.1 nonaka
567 1.1 nonaka ccb = nvme_ccb_get(sc->sc_admin_q);
568 1.11 jdolecek KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
569 1.1 nonaka
570 1.19 jdolecek mem = nvme_dmamem_alloc(sc, sizeof(*identify));
571 1.19 jdolecek if (mem == NULL)
572 1.19 jdolecek return ENOMEM;
573 1.1 nonaka
574 1.1 nonaka memset(&sqe, 0, sizeof(sqe));
575 1.1 nonaka sqe.opcode = NVM_ADMIN_IDENTIFY;
576 1.1 nonaka htolem32(&sqe.nsid, nsid);
577 1.1 nonaka htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem));
578 1.1 nonaka htolem32(&sqe.cdw10, 0);
579 1.1 nonaka
580 1.1 nonaka ccb->ccb_done = nvme_empty_done;
581 1.1 nonaka ccb->ccb_cookie = &sqe;
582 1.1 nonaka
583 1.1 nonaka nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
584 1.19 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT);
585 1.1 nonaka nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
586 1.1 nonaka
587 1.1 nonaka nvme_ccb_put(sc->sc_admin_q, ccb);
588 1.1 nonaka
589 1.19 jdolecek if (rv != 0) {
590 1.19 jdolecek rv = EIO;
591 1.1 nonaka goto done;
592 1.1 nonaka }
593 1.1 nonaka
594 1.1 nonaka /* commit */
595 1.1 nonaka
596 1.1 nonaka identify = kmem_zalloc(sizeof(*identify), KM_SLEEP);
597 1.19 jdolecek *identify = *((volatile struct nvm_identify_namespace *)NVME_DMA_KVA(mem));
598 1.19 jdolecek //memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify));
599 1.1 nonaka
600 1.1 nonaka ns = nvme_ns_get(sc, nsid);
601 1.1 nonaka KASSERT(ns);
602 1.1 nonaka ns->ident = identify;
603 1.1 nonaka
604 1.1 nonaka done:
605 1.19 jdolecek nvme_dmamem_free(sc, mem);
606 1.1 nonaka
607 1.19 jdolecek return rv;
608 1.1 nonaka }
609 1.1 nonaka
610 1.1 nonaka int
611 1.11 jdolecek nvme_ns_dobio(struct nvme_softc *sc, uint16_t nsid, void *cookie,
612 1.11 jdolecek struct buf *bp, void *data, size_t datasize,
613 1.11 jdolecek int secsize, daddr_t blkno, int flags, nvme_nnc_done nnc_done)
614 1.1 nonaka {
615 1.1 nonaka struct nvme_queue *q = nvme_get_q(sc);
616 1.1 nonaka struct nvme_ccb *ccb;
617 1.1 nonaka bus_dmamap_t dmap;
618 1.1 nonaka int i, error;
619 1.1 nonaka
620 1.1 nonaka ccb = nvme_ccb_get(q);
621 1.1 nonaka if (ccb == NULL)
622 1.1 nonaka return EAGAIN;
623 1.1 nonaka
624 1.1 nonaka ccb->ccb_done = nvme_ns_io_done;
625 1.11 jdolecek ccb->ccb_cookie = cookie;
626 1.11 jdolecek
627 1.11 jdolecek /* namespace context */
628 1.11 jdolecek ccb->nnc_nsid = nsid;
629 1.11 jdolecek ccb->nnc_flags = flags;
630 1.11 jdolecek ccb->nnc_buf = bp;
631 1.11 jdolecek ccb->nnc_datasize = datasize;
632 1.11 jdolecek ccb->nnc_secsize = secsize;
633 1.11 jdolecek ccb->nnc_blkno = blkno;
634 1.11 jdolecek ccb->nnc_done = nnc_done;
635 1.1 nonaka
636 1.1 nonaka dmap = ccb->ccb_dmamap;
637 1.11 jdolecek error = bus_dmamap_load(sc->sc_dmat, dmap, data,
638 1.11 jdolecek datasize, NULL,
639 1.11 jdolecek (ISSET(flags, NVME_NS_CTX_F_POLL) ?
640 1.1 nonaka BUS_DMA_NOWAIT : BUS_DMA_WAITOK) |
641 1.11 jdolecek (ISSET(flags, NVME_NS_CTX_F_READ) ?
642 1.1 nonaka BUS_DMA_READ : BUS_DMA_WRITE));
643 1.1 nonaka if (error) {
644 1.1 nonaka nvme_ccb_put(q, ccb);
645 1.1 nonaka return error;
646 1.1 nonaka }
647 1.1 nonaka
648 1.1 nonaka bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
649 1.11 jdolecek ISSET(flags, NVME_NS_CTX_F_READ) ?
650 1.1 nonaka BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
651 1.1 nonaka
652 1.1 nonaka if (dmap->dm_nsegs > 2) {
653 1.1 nonaka for (i = 1; i < dmap->dm_nsegs; i++) {
654 1.1 nonaka htolem64(&ccb->ccb_prpl[i - 1],
655 1.1 nonaka dmap->dm_segs[i].ds_addr);
656 1.1 nonaka }
657 1.1 nonaka bus_dmamap_sync(sc->sc_dmat,
658 1.1 nonaka NVME_DMA_MAP(q->q_ccb_prpls),
659 1.1 nonaka ccb->ccb_prpl_off,
660 1.16 nonaka sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
661 1.1 nonaka BUS_DMASYNC_PREWRITE);
662 1.1 nonaka }
663 1.1 nonaka
664 1.11 jdolecek if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
665 1.7 jdolecek if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0)
666 1.1 nonaka return EIO;
667 1.1 nonaka return 0;
668 1.1 nonaka }
669 1.1 nonaka
670 1.1 nonaka nvme_q_submit(sc, q, ccb, nvme_ns_io_fill);
671 1.1 nonaka return 0;
672 1.1 nonaka }
673 1.1 nonaka
674 1.1 nonaka static void
675 1.1 nonaka nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
676 1.1 nonaka {
677 1.1 nonaka struct nvme_sqe_io *sqe = slot;
678 1.1 nonaka bus_dmamap_t dmap = ccb->ccb_dmamap;
679 1.1 nonaka
680 1.11 jdolecek sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
681 1.1 nonaka NVM_CMD_READ : NVM_CMD_WRITE;
682 1.11 jdolecek htolem32(&sqe->nsid, ccb->nnc_nsid);
683 1.1 nonaka
684 1.1 nonaka htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
685 1.1 nonaka switch (dmap->dm_nsegs) {
686 1.1 nonaka case 1:
687 1.1 nonaka break;
688 1.1 nonaka case 2:
689 1.1 nonaka htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
690 1.1 nonaka break;
691 1.1 nonaka default:
692 1.1 nonaka /* the prp list is already set up and synced */
693 1.1 nonaka htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
694 1.1 nonaka break;
695 1.1 nonaka }
696 1.1 nonaka
697 1.11 jdolecek htolem64(&sqe->slba, ccb->nnc_blkno);
698 1.11 jdolecek
699 1.26 jdolecek if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA))
700 1.26 jdolecek htolem16(&sqe->ioflags, NVM_SQE_IO_FUA);
701 1.26 jdolecek
702 1.11 jdolecek /* guaranteed by upper layers, but check just in case */
703 1.11 jdolecek KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0);
704 1.11 jdolecek htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1);
705 1.1 nonaka }
706 1.1 nonaka
707 1.1 nonaka static void
708 1.1 nonaka nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb,
709 1.1 nonaka struct nvme_cqe *cqe)
710 1.1 nonaka {
711 1.1 nonaka struct nvme_softc *sc = q->q_sc;
712 1.1 nonaka bus_dmamap_t dmap = ccb->ccb_dmamap;
713 1.11 jdolecek void *nnc_cookie = ccb->ccb_cookie;
714 1.11 jdolecek nvme_nnc_done nnc_done = ccb->nnc_done;
715 1.11 jdolecek struct buf *bp = ccb->nnc_buf;
716 1.1 nonaka
717 1.1 nonaka if (dmap->dm_nsegs > 2) {
718 1.1 nonaka bus_dmamap_sync(sc->sc_dmat,
719 1.1 nonaka NVME_DMA_MAP(q->q_ccb_prpls),
720 1.1 nonaka ccb->ccb_prpl_off,
721 1.16 nonaka sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
722 1.1 nonaka BUS_DMASYNC_POSTWRITE);
723 1.1 nonaka }
724 1.1 nonaka
725 1.1 nonaka bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
726 1.11 jdolecek ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ?
727 1.1 nonaka BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
728 1.1 nonaka
729 1.1 nonaka bus_dmamap_unload(sc->sc_dmat, dmap);
730 1.1 nonaka nvme_ccb_put(q, ccb);
731 1.1 nonaka
732 1.25 jdolecek nnc_done(nnc_cookie, bp, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
733 1.25 jdolecek }
734 1.25 jdolecek
735 1.25 jdolecek /*
736 1.25 jdolecek * If there is no volatile write cache, it makes no sense to issue
737 1.25 jdolecek * flush commands or query for the status.
738 1.25 jdolecek */
739 1.25 jdolecek bool
740 1.25 jdolecek nvme_has_volatile_write_cache(struct nvme_softc *sc)
741 1.25 jdolecek {
742 1.25 jdolecek /* sc_identify is filled during attachment */
743 1.25 jdolecek return ((sc->sc_identify.vwc & NVME_ID_CTRLR_VWC_PRESENT) != 0);
744 1.1 nonaka }
745 1.1 nonaka
746 1.1 nonaka int
747 1.11 jdolecek nvme_ns_sync(struct nvme_softc *sc, uint16_t nsid, void *cookie,
748 1.11 jdolecek int flags, nvme_nnc_done nnc_done)
749 1.1 nonaka {
750 1.1 nonaka struct nvme_queue *q = nvme_get_q(sc);
751 1.1 nonaka struct nvme_ccb *ccb;
752 1.1 nonaka
753 1.1 nonaka ccb = nvme_ccb_get(q);
754 1.1 nonaka if (ccb == NULL)
755 1.1 nonaka return EAGAIN;
756 1.1 nonaka
757 1.1 nonaka ccb->ccb_done = nvme_ns_sync_done;
758 1.11 jdolecek ccb->ccb_cookie = cookie;
759 1.1 nonaka
760 1.11 jdolecek /* namespace context */
761 1.11 jdolecek ccb->nnc_nsid = nsid;
762 1.11 jdolecek ccb->nnc_flags = flags;
763 1.11 jdolecek ccb->nnc_done = nnc_done;
764 1.11 jdolecek
765 1.11 jdolecek if (ISSET(flags, NVME_NS_CTX_F_POLL)) {
766 1.7 jdolecek if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0)
767 1.1 nonaka return EIO;
768 1.1 nonaka return 0;
769 1.1 nonaka }
770 1.1 nonaka
771 1.1 nonaka nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill);
772 1.1 nonaka return 0;
773 1.1 nonaka }
774 1.1 nonaka
775 1.1 nonaka static void
776 1.1 nonaka nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
777 1.1 nonaka {
778 1.1 nonaka struct nvme_sqe *sqe = slot;
779 1.1 nonaka
780 1.1 nonaka sqe->opcode = NVM_CMD_FLUSH;
781 1.11 jdolecek htolem32(&sqe->nsid, ccb->nnc_nsid);
782 1.1 nonaka }
783 1.1 nonaka
784 1.1 nonaka static void
785 1.1 nonaka nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb,
786 1.1 nonaka struct nvme_cqe *cqe)
787 1.1 nonaka {
788 1.11 jdolecek void *cookie = ccb->ccb_cookie;
789 1.11 jdolecek nvme_nnc_done nnc_done = ccb->nnc_done;
790 1.1 nonaka
791 1.1 nonaka nvme_ccb_put(q, ccb);
792 1.1 nonaka
793 1.25 jdolecek nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
794 1.25 jdolecek }
795 1.25 jdolecek
796 1.25 jdolecek /*
797 1.25 jdolecek * Get status of volatile write cache. Always asynchronous.
798 1.25 jdolecek */
799 1.25 jdolecek int
800 1.25 jdolecek nvme_admin_getcache(struct nvme_softc *sc, void *cookie, nvme_nnc_done nnc_done)
801 1.25 jdolecek {
802 1.25 jdolecek struct nvme_ccb *ccb;
803 1.25 jdolecek struct nvme_queue *q = sc->sc_admin_q;
804 1.25 jdolecek
805 1.25 jdolecek ccb = nvme_ccb_get(q);
806 1.25 jdolecek if (ccb == NULL)
807 1.25 jdolecek return EAGAIN;
808 1.25 jdolecek
809 1.25 jdolecek ccb->ccb_done = nvme_getcache_done;
810 1.25 jdolecek ccb->ccb_cookie = cookie;
811 1.25 jdolecek
812 1.25 jdolecek /* namespace context */
813 1.25 jdolecek ccb->nnc_flags = 0;
814 1.25 jdolecek ccb->nnc_done = nnc_done;
815 1.25 jdolecek
816 1.25 jdolecek nvme_q_submit(sc, q, ccb, nvme_getcache_fill);
817 1.25 jdolecek return 0;
818 1.25 jdolecek }
819 1.25 jdolecek
820 1.25 jdolecek static void
821 1.25 jdolecek nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
822 1.25 jdolecek {
823 1.25 jdolecek struct nvme_sqe *sqe = slot;
824 1.25 jdolecek
825 1.25 jdolecek sqe->opcode = NVM_ADMIN_GET_FEATURES;
826 1.25 jdolecek sqe->cdw10 = NVM_FEATURE_VOLATILE_WRITE_CACHE;
827 1.25 jdolecek }
828 1.25 jdolecek
829 1.25 jdolecek static void
830 1.25 jdolecek nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb,
831 1.25 jdolecek struct nvme_cqe *cqe)
832 1.25 jdolecek {
833 1.25 jdolecek void *cookie = ccb->ccb_cookie;
834 1.25 jdolecek nvme_nnc_done nnc_done = ccb->nnc_done;
835 1.25 jdolecek
836 1.25 jdolecek nvme_ccb_put(q, ccb);
837 1.25 jdolecek
838 1.25 jdolecek nnc_done(cookie, NULL, lemtoh16(&cqe->flags), lemtoh32(&cqe->cdw0));
839 1.1 nonaka }
840 1.1 nonaka
841 1.1 nonaka void
842 1.1 nonaka nvme_ns_free(struct nvme_softc *sc, uint16_t nsid)
843 1.1 nonaka {
844 1.1 nonaka struct nvme_namespace *ns;
845 1.1 nonaka struct nvm_identify_namespace *identify;
846 1.1 nonaka
847 1.1 nonaka ns = nvme_ns_get(sc, nsid);
848 1.1 nonaka KASSERT(ns);
849 1.1 nonaka
850 1.1 nonaka identify = ns->ident;
851 1.1 nonaka ns->ident = NULL;
852 1.1 nonaka if (identify != NULL)
853 1.1 nonaka kmem_free(identify, sizeof(*identify));
854 1.1 nonaka }
855 1.1 nonaka
856 1.1 nonaka static void
857 1.3 nonaka nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
858 1.3 nonaka {
859 1.3 nonaka struct nvme_softc *sc = q->q_sc;
860 1.3 nonaka struct nvme_sqe *sqe = slot;
861 1.3 nonaka struct nvme_pt_command *pt = ccb->ccb_cookie;
862 1.3 nonaka bus_dmamap_t dmap = ccb->ccb_dmamap;
863 1.3 nonaka int i;
864 1.3 nonaka
865 1.3 nonaka sqe->opcode = pt->cmd.opcode;
866 1.3 nonaka htolem32(&sqe->nsid, pt->cmd.nsid);
867 1.3 nonaka
868 1.3 nonaka if (pt->buf != NULL && pt->len > 0) {
869 1.3 nonaka htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr);
870 1.3 nonaka switch (dmap->dm_nsegs) {
871 1.3 nonaka case 1:
872 1.3 nonaka break;
873 1.3 nonaka case 2:
874 1.3 nonaka htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr);
875 1.3 nonaka break;
876 1.3 nonaka default:
877 1.3 nonaka for (i = 1; i < dmap->dm_nsegs; i++) {
878 1.3 nonaka htolem64(&ccb->ccb_prpl[i - 1],
879 1.3 nonaka dmap->dm_segs[i].ds_addr);
880 1.3 nonaka }
881 1.3 nonaka bus_dmamap_sync(sc->sc_dmat,
882 1.3 nonaka NVME_DMA_MAP(q->q_ccb_prpls),
883 1.3 nonaka ccb->ccb_prpl_off,
884 1.16 nonaka sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
885 1.3 nonaka BUS_DMASYNC_PREWRITE);
886 1.3 nonaka htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva);
887 1.3 nonaka break;
888 1.3 nonaka }
889 1.3 nonaka }
890 1.3 nonaka
891 1.3 nonaka htolem32(&sqe->cdw10, pt->cmd.cdw10);
892 1.3 nonaka htolem32(&sqe->cdw11, pt->cmd.cdw11);
893 1.3 nonaka htolem32(&sqe->cdw12, pt->cmd.cdw12);
894 1.3 nonaka htolem32(&sqe->cdw13, pt->cmd.cdw13);
895 1.3 nonaka htolem32(&sqe->cdw14, pt->cmd.cdw14);
896 1.3 nonaka htolem32(&sqe->cdw15, pt->cmd.cdw15);
897 1.3 nonaka }
898 1.3 nonaka
899 1.3 nonaka static void
900 1.3 nonaka nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe)
901 1.3 nonaka {
902 1.3 nonaka struct nvme_softc *sc = q->q_sc;
903 1.3 nonaka struct nvme_pt_command *pt = ccb->ccb_cookie;
904 1.3 nonaka bus_dmamap_t dmap = ccb->ccb_dmamap;
905 1.3 nonaka
906 1.3 nonaka if (pt->buf != NULL && pt->len > 0) {
907 1.3 nonaka if (dmap->dm_nsegs > 2) {
908 1.3 nonaka bus_dmamap_sync(sc->sc_dmat,
909 1.3 nonaka NVME_DMA_MAP(q->q_ccb_prpls),
910 1.3 nonaka ccb->ccb_prpl_off,
911 1.16 nonaka sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1),
912 1.3 nonaka BUS_DMASYNC_POSTWRITE);
913 1.3 nonaka }
914 1.3 nonaka
915 1.3 nonaka bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize,
916 1.3 nonaka pt->is_read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
917 1.3 nonaka bus_dmamap_unload(sc->sc_dmat, dmap);
918 1.3 nonaka }
919 1.3 nonaka
920 1.23 nonaka pt->cpl.cdw0 = lemtoh32(&cqe->cdw0);
921 1.23 nonaka pt->cpl.flags = lemtoh16(&cqe->flags) & ~NVME_CQE_PHASE;
922 1.3 nonaka }
923 1.3 nonaka
924 1.3 nonaka static int
925 1.3 nonaka nvme_command_passthrough(struct nvme_softc *sc, struct nvme_pt_command *pt,
926 1.3 nonaka uint16_t nsid, struct lwp *l, bool is_adminq)
927 1.3 nonaka {
928 1.3 nonaka struct nvme_queue *q;
929 1.3 nonaka struct nvme_ccb *ccb;
930 1.3 nonaka void *buf = NULL;
931 1.3 nonaka int error;
932 1.3 nonaka
933 1.9 jdolecek /* limit command size to maximum data transfer size */
934 1.3 nonaka if ((pt->buf == NULL && pt->len > 0) ||
935 1.9 jdolecek (pt->buf != NULL && (pt->len == 0 || pt->len > sc->sc_mdts)))
936 1.3 nonaka return EINVAL;
937 1.3 nonaka
938 1.3 nonaka q = is_adminq ? sc->sc_admin_q : nvme_get_q(sc);
939 1.3 nonaka ccb = nvme_ccb_get(q);
940 1.3 nonaka if (ccb == NULL)
941 1.3 nonaka return EBUSY;
942 1.3 nonaka
943 1.9 jdolecek if (pt->buf != NULL) {
944 1.9 jdolecek KASSERT(pt->len > 0);
945 1.3 nonaka buf = kmem_alloc(pt->len, KM_SLEEP);
946 1.3 nonaka if (!pt->is_read) {
947 1.3 nonaka error = copyin(pt->buf, buf, pt->len);
948 1.3 nonaka if (error)
949 1.3 nonaka goto kmem_free;
950 1.3 nonaka }
951 1.3 nonaka error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf,
952 1.3 nonaka pt->len, NULL,
953 1.3 nonaka BUS_DMA_WAITOK |
954 1.3 nonaka (pt->is_read ? BUS_DMA_READ : BUS_DMA_WRITE));
955 1.3 nonaka if (error)
956 1.3 nonaka goto kmem_free;
957 1.3 nonaka bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap,
958 1.3 nonaka 0, ccb->ccb_dmamap->dm_mapsize,
959 1.3 nonaka pt->is_read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
960 1.3 nonaka }
961 1.3 nonaka
962 1.3 nonaka ccb->ccb_done = nvme_pt_done;
963 1.3 nonaka ccb->ccb_cookie = pt;
964 1.3 nonaka
965 1.3 nonaka pt->cmd.nsid = nsid;
966 1.7 jdolecek if (nvme_poll(sc, q, ccb, nvme_pt_fill, NVME_TIMO_PT)) {
967 1.3 nonaka error = EIO;
968 1.3 nonaka goto out;
969 1.3 nonaka }
970 1.3 nonaka
971 1.3 nonaka error = 0;
972 1.3 nonaka out:
973 1.3 nonaka if (buf != NULL) {
974 1.3 nonaka if (error == 0 && pt->is_read)
975 1.3 nonaka error = copyout(buf, pt->buf, pt->len);
976 1.3 nonaka kmem_free:
977 1.3 nonaka kmem_free(buf, pt->len);
978 1.3 nonaka }
979 1.3 nonaka nvme_ccb_put(q, ccb);
980 1.3 nonaka return error;
981 1.3 nonaka }
982 1.3 nonaka
983 1.3 nonaka static void
984 1.1 nonaka nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
985 1.1 nonaka void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *))
986 1.1 nonaka {
987 1.1 nonaka struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem);
988 1.1 nonaka uint32_t tail;
989 1.1 nonaka
990 1.1 nonaka mutex_enter(&q->q_sq_mtx);
991 1.1 nonaka tail = q->q_sq_tail;
992 1.1 nonaka if (++q->q_sq_tail >= q->q_entries)
993 1.1 nonaka q->q_sq_tail = 0;
994 1.1 nonaka
995 1.1 nonaka sqe += tail;
996 1.1 nonaka
997 1.1 nonaka bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
998 1.1 nonaka sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE);
999 1.1 nonaka memset(sqe, 0, sizeof(*sqe));
1000 1.1 nonaka (*fill)(q, ccb, sqe);
1001 1.1 nonaka sqe->cid = ccb->ccb_id;
1002 1.1 nonaka bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem),
1003 1.1 nonaka sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE);
1004 1.1 nonaka
1005 1.1 nonaka nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail);
1006 1.1 nonaka mutex_exit(&q->q_sq_mtx);
1007 1.1 nonaka }
1008 1.1 nonaka
1009 1.1 nonaka struct nvme_poll_state {
1010 1.1 nonaka struct nvme_sqe s;
1011 1.1 nonaka struct nvme_cqe c;
1012 1.1 nonaka };
1013 1.1 nonaka
1014 1.1 nonaka static int
1015 1.1 nonaka nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb,
1016 1.7 jdolecek void (*fill)(struct nvme_queue *, struct nvme_ccb *, void *), int timo_sec)
1017 1.1 nonaka {
1018 1.1 nonaka struct nvme_poll_state state;
1019 1.1 nonaka void (*done)(struct nvme_queue *, struct nvme_ccb *, struct nvme_cqe *);
1020 1.1 nonaka void *cookie;
1021 1.1 nonaka uint16_t flags;
1022 1.7 jdolecek int step = 10;
1023 1.7 jdolecek int maxloop = timo_sec * 1000000 / step;
1024 1.7 jdolecek int error = 0;
1025 1.1 nonaka
1026 1.1 nonaka memset(&state, 0, sizeof(state));
1027 1.1 nonaka (*fill)(q, ccb, &state.s);
1028 1.1 nonaka
1029 1.1 nonaka done = ccb->ccb_done;
1030 1.1 nonaka cookie = ccb->ccb_cookie;
1031 1.1 nonaka
1032 1.1 nonaka ccb->ccb_done = nvme_poll_done;
1033 1.1 nonaka ccb->ccb_cookie = &state;
1034 1.1 nonaka
1035 1.1 nonaka nvme_q_submit(sc, q, ccb, nvme_poll_fill);
1036 1.1 nonaka while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) {
1037 1.1 nonaka if (nvme_q_complete(sc, q) == 0)
1038 1.7 jdolecek delay(step);
1039 1.1 nonaka
1040 1.7 jdolecek if (timo_sec >= 0 && --maxloop <= 0) {
1041 1.7 jdolecek error = ETIMEDOUT;
1042 1.7 jdolecek break;
1043 1.7 jdolecek }
1044 1.1 nonaka }
1045 1.1 nonaka
1046 1.1 nonaka ccb->ccb_cookie = cookie;
1047 1.1 nonaka done(q, ccb, &state.c);
1048 1.1 nonaka
1049 1.7 jdolecek if (error == 0) {
1050 1.7 jdolecek flags = lemtoh16(&state.c.flags);
1051 1.7 jdolecek return flags & ~NVME_CQE_PHASE;
1052 1.7 jdolecek } else {
1053 1.7 jdolecek return 1;
1054 1.7 jdolecek }
1055 1.1 nonaka }
1056 1.1 nonaka
1057 1.1 nonaka static void
1058 1.1 nonaka nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1059 1.1 nonaka {
1060 1.1 nonaka struct nvme_sqe *sqe = slot;
1061 1.1 nonaka struct nvme_poll_state *state = ccb->ccb_cookie;
1062 1.1 nonaka
1063 1.1 nonaka *sqe = state->s;
1064 1.1 nonaka }
1065 1.1 nonaka
1066 1.1 nonaka static void
1067 1.1 nonaka nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1068 1.1 nonaka struct nvme_cqe *cqe)
1069 1.1 nonaka {
1070 1.1 nonaka struct nvme_poll_state *state = ccb->ccb_cookie;
1071 1.1 nonaka
1072 1.1 nonaka SET(cqe->flags, htole16(NVME_CQE_PHASE));
1073 1.1 nonaka state->c = *cqe;
1074 1.1 nonaka }
1075 1.1 nonaka
1076 1.1 nonaka static void
1077 1.1 nonaka nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1078 1.1 nonaka {
1079 1.1 nonaka struct nvme_sqe *src = ccb->ccb_cookie;
1080 1.1 nonaka struct nvme_sqe *dst = slot;
1081 1.1 nonaka
1082 1.1 nonaka *dst = *src;
1083 1.1 nonaka }
1084 1.1 nonaka
1085 1.1 nonaka static void
1086 1.1 nonaka nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb,
1087 1.1 nonaka struct nvme_cqe *cqe)
1088 1.1 nonaka {
1089 1.1 nonaka }
1090 1.1 nonaka
1091 1.1 nonaka static int
1092 1.1 nonaka nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q)
1093 1.1 nonaka {
1094 1.1 nonaka struct nvme_ccb *ccb;
1095 1.1 nonaka struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe;
1096 1.1 nonaka uint16_t flags;
1097 1.1 nonaka int rv = 0;
1098 1.1 nonaka
1099 1.9 jdolecek mutex_enter(&q->q_cq_mtx);
1100 1.1 nonaka
1101 1.1 nonaka nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1102 1.1 nonaka for (;;) {
1103 1.9 jdolecek cqe = &ring[q->q_cq_head];
1104 1.1 nonaka flags = lemtoh16(&cqe->flags);
1105 1.1 nonaka if ((flags & NVME_CQE_PHASE) != q->q_cq_phase)
1106 1.1 nonaka break;
1107 1.1 nonaka
1108 1.1 nonaka ccb = &q->q_ccbs[cqe->cid];
1109 1.1 nonaka
1110 1.9 jdolecek if (++q->q_cq_head >= q->q_entries) {
1111 1.9 jdolecek q->q_cq_head = 0;
1112 1.1 nonaka q->q_cq_phase ^= NVME_CQE_PHASE;
1113 1.1 nonaka }
1114 1.1 nonaka
1115 1.18 jdolecek #ifdef DEBUG
1116 1.18 jdolecek /*
1117 1.18 jdolecek * If we get spurious completion notification, something
1118 1.18 jdolecek * is seriously hosed up. Very likely DMA to some random
1119 1.18 jdolecek * memory place happened, so just bail out.
1120 1.18 jdolecek */
1121 1.18 jdolecek if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) {
1122 1.18 jdolecek panic("%s: invalid ccb detected",
1123 1.18 jdolecek device_xname(sc->sc_dev));
1124 1.18 jdolecek /* NOTREACHED */
1125 1.18 jdolecek }
1126 1.18 jdolecek #endif
1127 1.20 jdolecek
1128 1.20 jdolecek rv++;
1129 1.9 jdolecek
1130 1.9 jdolecek /*
1131 1.10 jdolecek * Unlock the mutex before calling the ccb_done callback
1132 1.9 jdolecek * and re-lock afterwards. The callback triggers lddone()
1133 1.9 jdolecek * which schedules another i/o, and also calls nvme_ccb_put().
1134 1.9 jdolecek * Unlock/relock avoids possibility of deadlock.
1135 1.9 jdolecek */
1136 1.9 jdolecek mutex_exit(&q->q_cq_mtx);
1137 1.9 jdolecek ccb->ccb_done(q, ccb, cqe);
1138 1.9 jdolecek mutex_enter(&q->q_cq_mtx);
1139 1.1 nonaka }
1140 1.1 nonaka nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1141 1.1 nonaka
1142 1.1 nonaka if (rv)
1143 1.9 jdolecek nvme_write4(sc, q->q_cqhdbl, q->q_cq_head);
1144 1.9 jdolecek
1145 1.1 nonaka mutex_exit(&q->q_cq_mtx);
1146 1.1 nonaka
1147 1.1 nonaka return rv;
1148 1.1 nonaka }
1149 1.1 nonaka
1150 1.1 nonaka static int
1151 1.1 nonaka nvme_identify(struct nvme_softc *sc, u_int mps)
1152 1.1 nonaka {
1153 1.1 nonaka char sn[41], mn[81], fr[17];
1154 1.1 nonaka struct nvm_identify_controller *identify;
1155 1.19 jdolecek struct nvme_dmamem *mem;
1156 1.1 nonaka struct nvme_ccb *ccb;
1157 1.1 nonaka u_int mdts;
1158 1.19 jdolecek int rv = 1;
1159 1.1 nonaka
1160 1.1 nonaka ccb = nvme_ccb_get(sc->sc_admin_q);
1161 1.11 jdolecek KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
1162 1.1 nonaka
1163 1.19 jdolecek mem = nvme_dmamem_alloc(sc, sizeof(*identify));
1164 1.19 jdolecek if (mem == NULL)
1165 1.19 jdolecek return 1;
1166 1.1 nonaka
1167 1.1 nonaka ccb->ccb_done = nvme_empty_done;
1168 1.19 jdolecek ccb->ccb_cookie = mem;
1169 1.1 nonaka
1170 1.1 nonaka nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD);
1171 1.19 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify,
1172 1.7 jdolecek NVME_TIMO_IDENT);
1173 1.1 nonaka nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD);
1174 1.1 nonaka
1175 1.1 nonaka nvme_ccb_put(sc->sc_admin_q, ccb);
1176 1.1 nonaka
1177 1.19 jdolecek if (rv != 0)
1178 1.1 nonaka goto done;
1179 1.1 nonaka
1180 1.1 nonaka identify = NVME_DMA_KVA(mem);
1181 1.1 nonaka
1182 1.2 christos strnvisx(sn, sizeof(sn), (const char *)identify->sn,
1183 1.2 christos sizeof(identify->sn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1184 1.2 christos strnvisx(mn, sizeof(mn), (const char *)identify->mn,
1185 1.2 christos sizeof(identify->mn), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1186 1.2 christos strnvisx(fr, sizeof(fr), (const char *)identify->fr,
1187 1.2 christos sizeof(identify->fr), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
1188 1.1 nonaka aprint_normal_dev(sc->sc_dev, "%s, firmware %s, serial %s\n", mn, fr,
1189 1.1 nonaka sn);
1190 1.1 nonaka
1191 1.1 nonaka if (identify->mdts > 0) {
1192 1.1 nonaka mdts = (1 << identify->mdts) * (1 << mps);
1193 1.1 nonaka if (mdts < sc->sc_mdts)
1194 1.1 nonaka sc->sc_mdts = mdts;
1195 1.1 nonaka }
1196 1.1 nonaka
1197 1.1 nonaka sc->sc_nn = lemtoh32(&identify->nn);
1198 1.1 nonaka
1199 1.1 nonaka memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify));
1200 1.1 nonaka
1201 1.1 nonaka done:
1202 1.19 jdolecek nvme_dmamem_free(sc, mem);
1203 1.1 nonaka
1204 1.19 jdolecek return rv;
1205 1.1 nonaka }
1206 1.1 nonaka
1207 1.1 nonaka static int
1208 1.1 nonaka nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q)
1209 1.1 nonaka {
1210 1.1 nonaka struct nvme_sqe_q sqe;
1211 1.1 nonaka struct nvme_ccb *ccb;
1212 1.1 nonaka int rv;
1213 1.1 nonaka
1214 1.9 jdolecek if (sc->sc_use_mq && sc->sc_intr_establish(sc, q->q_id, q) != 0)
1215 1.1 nonaka return 1;
1216 1.1 nonaka
1217 1.1 nonaka ccb = nvme_ccb_get(sc->sc_admin_q);
1218 1.1 nonaka KASSERT(ccb != NULL);
1219 1.1 nonaka
1220 1.1 nonaka ccb->ccb_done = nvme_empty_done;
1221 1.1 nonaka ccb->ccb_cookie = &sqe;
1222 1.1 nonaka
1223 1.1 nonaka memset(&sqe, 0, sizeof(sqe));
1224 1.1 nonaka sqe.opcode = NVM_ADMIN_ADD_IOCQ;
1225 1.1 nonaka htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem));
1226 1.1 nonaka htolem16(&sqe.qsize, q->q_entries - 1);
1227 1.1 nonaka htolem16(&sqe.qid, q->q_id);
1228 1.1 nonaka sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC;
1229 1.1 nonaka if (sc->sc_use_mq)
1230 1.1 nonaka htolem16(&sqe.cqid, q->q_id); /* qid == vector */
1231 1.1 nonaka
1232 1.7 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1233 1.1 nonaka if (rv != 0)
1234 1.1 nonaka goto fail;
1235 1.1 nonaka
1236 1.1 nonaka ccb->ccb_done = nvme_empty_done;
1237 1.1 nonaka ccb->ccb_cookie = &sqe;
1238 1.1 nonaka
1239 1.1 nonaka memset(&sqe, 0, sizeof(sqe));
1240 1.1 nonaka sqe.opcode = NVM_ADMIN_ADD_IOSQ;
1241 1.1 nonaka htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem));
1242 1.1 nonaka htolem16(&sqe.qsize, q->q_entries - 1);
1243 1.1 nonaka htolem16(&sqe.qid, q->q_id);
1244 1.1 nonaka htolem16(&sqe.cqid, q->q_id);
1245 1.1 nonaka sqe.qflags = NVM_SQE_Q_PC;
1246 1.1 nonaka
1247 1.7 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1248 1.1 nonaka if (rv != 0)
1249 1.1 nonaka goto fail;
1250 1.1 nonaka
1251 1.1 nonaka fail:
1252 1.1 nonaka nvme_ccb_put(sc->sc_admin_q, ccb);
1253 1.1 nonaka return rv;
1254 1.1 nonaka }
1255 1.1 nonaka
1256 1.1 nonaka static int
1257 1.1 nonaka nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q)
1258 1.1 nonaka {
1259 1.1 nonaka struct nvme_sqe_q sqe;
1260 1.1 nonaka struct nvme_ccb *ccb;
1261 1.1 nonaka int rv;
1262 1.1 nonaka
1263 1.1 nonaka ccb = nvme_ccb_get(sc->sc_admin_q);
1264 1.1 nonaka KASSERT(ccb != NULL);
1265 1.1 nonaka
1266 1.1 nonaka ccb->ccb_done = nvme_empty_done;
1267 1.1 nonaka ccb->ccb_cookie = &sqe;
1268 1.1 nonaka
1269 1.1 nonaka memset(&sqe, 0, sizeof(sqe));
1270 1.1 nonaka sqe.opcode = NVM_ADMIN_DEL_IOSQ;
1271 1.1 nonaka htolem16(&sqe.qid, q->q_id);
1272 1.1 nonaka
1273 1.7 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1274 1.1 nonaka if (rv != 0)
1275 1.1 nonaka goto fail;
1276 1.1 nonaka
1277 1.1 nonaka ccb->ccb_done = nvme_empty_done;
1278 1.1 nonaka ccb->ccb_cookie = &sqe;
1279 1.1 nonaka
1280 1.1 nonaka memset(&sqe, 0, sizeof(sqe));
1281 1.1 nonaka sqe.opcode = NVM_ADMIN_DEL_IOCQ;
1282 1.1 nonaka htolem16(&sqe.qid, q->q_id);
1283 1.1 nonaka
1284 1.7 jdolecek rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP);
1285 1.1 nonaka if (rv != 0)
1286 1.1 nonaka goto fail;
1287 1.1 nonaka
1288 1.1 nonaka fail:
1289 1.1 nonaka nvme_ccb_put(sc->sc_admin_q, ccb);
1290 1.1 nonaka
1291 1.1 nonaka if (rv == 0 && sc->sc_use_mq) {
1292 1.1 nonaka if (sc->sc_intr_disestablish(sc, q->q_id))
1293 1.1 nonaka rv = 1;
1294 1.1 nonaka }
1295 1.1 nonaka
1296 1.1 nonaka return rv;
1297 1.1 nonaka }
1298 1.1 nonaka
1299 1.1 nonaka static void
1300 1.1 nonaka nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot)
1301 1.1 nonaka {
1302 1.1 nonaka struct nvme_sqe *sqe = slot;
1303 1.1 nonaka struct nvme_dmamem *mem = ccb->ccb_cookie;
1304 1.1 nonaka
1305 1.1 nonaka sqe->opcode = NVM_ADMIN_IDENTIFY;
1306 1.19 jdolecek htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem));
1307 1.1 nonaka htolem32(&sqe->cdw10, 1);
1308 1.1 nonaka }
1309 1.1 nonaka
1310 1.1 nonaka static int
1311 1.23 nonaka nvme_get_number_of_queues(struct nvme_softc *sc, u_int *nqap)
1312 1.23 nonaka {
1313 1.23 nonaka struct nvme_pt_command pt;
1314 1.23 nonaka struct nvme_ccb *ccb;
1315 1.23 nonaka uint16_t ncqa, nsqa;
1316 1.23 nonaka int rv;
1317 1.23 nonaka
1318 1.23 nonaka ccb = nvme_ccb_get(sc->sc_admin_q);
1319 1.23 nonaka KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */
1320 1.23 nonaka
1321 1.23 nonaka memset(&pt, 0, sizeof(pt));
1322 1.23 nonaka pt.cmd.opcode = NVM_ADMIN_GET_FEATURES;
1323 1.25 jdolecek pt.cmd.cdw10 = NVM_FEATURE_NUMBER_OF_QUEUES;
1324 1.23 nonaka
1325 1.23 nonaka ccb->ccb_done = nvme_pt_done;
1326 1.23 nonaka ccb->ccb_cookie = &pt;
1327 1.23 nonaka
1328 1.23 nonaka rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP);
1329 1.23 nonaka
1330 1.23 nonaka nvme_ccb_put(sc->sc_admin_q, ccb);
1331 1.23 nonaka
1332 1.23 nonaka if (rv != 0) {
1333 1.23 nonaka *nqap = 0;
1334 1.23 nonaka return EIO;
1335 1.23 nonaka }
1336 1.23 nonaka
1337 1.23 nonaka ncqa = pt.cpl.cdw0 >> 16;
1338 1.23 nonaka nsqa = pt.cpl.cdw0 & 0xffff;
1339 1.23 nonaka *nqap = MIN(ncqa, nsqa) + 1;
1340 1.23 nonaka
1341 1.23 nonaka return 0;
1342 1.23 nonaka }
1343 1.23 nonaka
1344 1.23 nonaka static int
1345 1.20 jdolecek nvme_ccbs_alloc(struct nvme_queue *q, uint16_t nccbs)
1346 1.1 nonaka {
1347 1.1 nonaka struct nvme_softc *sc = q->q_sc;
1348 1.1 nonaka struct nvme_ccb *ccb;
1349 1.1 nonaka bus_addr_t off;
1350 1.1 nonaka uint64_t *prpl;
1351 1.1 nonaka u_int i;
1352 1.1 nonaka
1353 1.1 nonaka mutex_init(&q->q_ccb_mtx, MUTEX_DEFAULT, IPL_BIO);
1354 1.1 nonaka SIMPLEQ_INIT(&q->q_ccb_list);
1355 1.1 nonaka
1356 1.1 nonaka q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP);
1357 1.1 nonaka
1358 1.1 nonaka q->q_nccbs = nccbs;
1359 1.19 jdolecek q->q_ccb_prpls = nvme_dmamem_alloc(sc,
1360 1.19 jdolecek sizeof(*prpl) * sc->sc_max_sgl * nccbs);
1361 1.1 nonaka
1362 1.1 nonaka prpl = NVME_DMA_KVA(q->q_ccb_prpls);
1363 1.1 nonaka off = 0;
1364 1.1 nonaka
1365 1.1 nonaka for (i = 0; i < nccbs; i++) {
1366 1.1 nonaka ccb = &q->q_ccbs[i];
1367 1.1 nonaka
1368 1.1 nonaka if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts,
1369 1.1 nonaka sc->sc_max_sgl + 1 /* we get a free prp in the sqe */,
1370 1.1 nonaka sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
1371 1.1 nonaka &ccb->ccb_dmamap) != 0)
1372 1.1 nonaka goto free_maps;
1373 1.1 nonaka
1374 1.1 nonaka ccb->ccb_id = i;
1375 1.1 nonaka ccb->ccb_prpl = prpl;
1376 1.1 nonaka ccb->ccb_prpl_off = off;
1377 1.1 nonaka ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off;
1378 1.1 nonaka
1379 1.1 nonaka SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry);
1380 1.1 nonaka
1381 1.1 nonaka prpl += sc->sc_max_sgl;
1382 1.1 nonaka off += sizeof(*prpl) * sc->sc_max_sgl;
1383 1.1 nonaka }
1384 1.1 nonaka
1385 1.1 nonaka return 0;
1386 1.1 nonaka
1387 1.1 nonaka free_maps:
1388 1.1 nonaka nvme_ccbs_free(q);
1389 1.1 nonaka return 1;
1390 1.1 nonaka }
1391 1.1 nonaka
1392 1.1 nonaka static struct nvme_ccb *
1393 1.1 nonaka nvme_ccb_get(struct nvme_queue *q)
1394 1.1 nonaka {
1395 1.20 jdolecek struct nvme_ccb *ccb = NULL;
1396 1.1 nonaka
1397 1.1 nonaka mutex_enter(&q->q_ccb_mtx);
1398 1.30.2.1 martin ccb = SIMPLEQ_FIRST(&q->q_ccb_list);
1399 1.30.2.1 martin if (ccb != NULL) {
1400 1.1 nonaka SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1401 1.18 jdolecek #ifdef DEBUG
1402 1.18 jdolecek ccb->ccb_cookie = NULL;
1403 1.18 jdolecek #endif
1404 1.18 jdolecek }
1405 1.1 nonaka mutex_exit(&q->q_ccb_mtx);
1406 1.1 nonaka
1407 1.1 nonaka return ccb;
1408 1.1 nonaka }
1409 1.1 nonaka
1410 1.1 nonaka static void
1411 1.1 nonaka nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb)
1412 1.1 nonaka {
1413 1.1 nonaka
1414 1.1 nonaka mutex_enter(&q->q_ccb_mtx);
1415 1.18 jdolecek #ifdef DEBUG
1416 1.18 jdolecek ccb->ccb_cookie = (void *)NVME_CCB_FREE;
1417 1.18 jdolecek #endif
1418 1.1 nonaka SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry);
1419 1.1 nonaka mutex_exit(&q->q_ccb_mtx);
1420 1.1 nonaka }
1421 1.1 nonaka
1422 1.1 nonaka static void
1423 1.1 nonaka nvme_ccbs_free(struct nvme_queue *q)
1424 1.1 nonaka {
1425 1.1 nonaka struct nvme_softc *sc = q->q_sc;
1426 1.1 nonaka struct nvme_ccb *ccb;
1427 1.1 nonaka
1428 1.1 nonaka mutex_enter(&q->q_ccb_mtx);
1429 1.1 nonaka while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) {
1430 1.1 nonaka SIMPLEQ_REMOVE_HEAD(&q->q_ccb_list, ccb_entry);
1431 1.1 nonaka bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap);
1432 1.1 nonaka }
1433 1.1 nonaka mutex_exit(&q->q_ccb_mtx);
1434 1.1 nonaka
1435 1.19 jdolecek nvme_dmamem_free(sc, q->q_ccb_prpls);
1436 1.1 nonaka kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs);
1437 1.1 nonaka q->q_ccbs = NULL;
1438 1.1 nonaka mutex_destroy(&q->q_ccb_mtx);
1439 1.1 nonaka }
1440 1.1 nonaka
1441 1.1 nonaka static struct nvme_queue *
1442 1.1 nonaka nvme_q_alloc(struct nvme_softc *sc, uint16_t id, u_int entries, u_int dstrd)
1443 1.1 nonaka {
1444 1.1 nonaka struct nvme_queue *q;
1445 1.1 nonaka
1446 1.1 nonaka q = kmem_alloc(sizeof(*q), KM_SLEEP);
1447 1.1 nonaka q->q_sc = sc;
1448 1.19 jdolecek q->q_sq_dmamem = nvme_dmamem_alloc(sc,
1449 1.19 jdolecek sizeof(struct nvme_sqe) * entries);
1450 1.19 jdolecek if (q->q_sq_dmamem == NULL)
1451 1.1 nonaka goto free;
1452 1.1 nonaka
1453 1.19 jdolecek q->q_cq_dmamem = nvme_dmamem_alloc(sc,
1454 1.19 jdolecek sizeof(struct nvme_cqe) * entries);
1455 1.19 jdolecek if (q->q_cq_dmamem == NULL)
1456 1.1 nonaka goto free_sq;
1457 1.1 nonaka
1458 1.1 nonaka memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem));
1459 1.1 nonaka memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem));
1460 1.1 nonaka
1461 1.1 nonaka mutex_init(&q->q_sq_mtx, MUTEX_DEFAULT, IPL_BIO);
1462 1.1 nonaka mutex_init(&q->q_cq_mtx, MUTEX_DEFAULT, IPL_BIO);
1463 1.1 nonaka q->q_sqtdbl = NVME_SQTDBL(id, dstrd);
1464 1.1 nonaka q->q_cqhdbl = NVME_CQHDBL(id, dstrd);
1465 1.1 nonaka q->q_id = id;
1466 1.1 nonaka q->q_entries = entries;
1467 1.1 nonaka q->q_sq_tail = 0;
1468 1.1 nonaka q->q_cq_head = 0;
1469 1.1 nonaka q->q_cq_phase = NVME_CQE_PHASE;
1470 1.1 nonaka
1471 1.1 nonaka nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE);
1472 1.1 nonaka nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD);
1473 1.1 nonaka
1474 1.20 jdolecek /*
1475 1.20 jdolecek * Due to definition of full and empty queue (queue is empty
1476 1.20 jdolecek * when head == tail, full when tail is one less then head),
1477 1.20 jdolecek * we can actually only have (entries - 1) in-flight commands.
1478 1.20 jdolecek */
1479 1.20 jdolecek if (nvme_ccbs_alloc(q, entries - 1) != 0) {
1480 1.1 nonaka aprint_error_dev(sc->sc_dev, "unable to allocate ccbs\n");
1481 1.1 nonaka goto free_cq;
1482 1.1 nonaka }
1483 1.1 nonaka
1484 1.1 nonaka return q;
1485 1.1 nonaka
1486 1.1 nonaka free_cq:
1487 1.19 jdolecek nvme_dmamem_free(sc, q->q_cq_dmamem);
1488 1.1 nonaka free_sq:
1489 1.19 jdolecek nvme_dmamem_free(sc, q->q_sq_dmamem);
1490 1.1 nonaka free:
1491 1.1 nonaka kmem_free(q, sizeof(*q));
1492 1.1 nonaka
1493 1.1 nonaka return NULL;
1494 1.1 nonaka }
1495 1.1 nonaka
1496 1.1 nonaka static void
1497 1.1 nonaka nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q)
1498 1.1 nonaka {
1499 1.1 nonaka nvme_ccbs_free(q);
1500 1.9 jdolecek mutex_destroy(&q->q_sq_mtx);
1501 1.9 jdolecek mutex_destroy(&q->q_cq_mtx);
1502 1.1 nonaka nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD);
1503 1.1 nonaka nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE);
1504 1.19 jdolecek nvme_dmamem_free(sc, q->q_cq_dmamem);
1505 1.19 jdolecek nvme_dmamem_free(sc, q->q_sq_dmamem);
1506 1.1 nonaka kmem_free(q, sizeof(*q));
1507 1.1 nonaka }
1508 1.1 nonaka
1509 1.1 nonaka int
1510 1.1 nonaka nvme_intr(void *xsc)
1511 1.1 nonaka {
1512 1.1 nonaka struct nvme_softc *sc = xsc;
1513 1.1 nonaka
1514 1.10 jdolecek /*
1515 1.10 jdolecek * INTx is level triggered, controller deasserts the interrupt only
1516 1.10 jdolecek * when we advance command queue head via write to the doorbell.
1517 1.17 jdolecek * Tell the controller to block the interrupts while we process
1518 1.17 jdolecek * the queue(s).
1519 1.10 jdolecek */
1520 1.17 jdolecek nvme_write4(sc, NVME_INTMS, 1);
1521 1.17 jdolecek
1522 1.17 jdolecek softint_schedule(sc->sc_softih[0]);
1523 1.17 jdolecek
1524 1.17 jdolecek /* don't know, might not have been for us */
1525 1.17 jdolecek return 1;
1526 1.17 jdolecek }
1527 1.17 jdolecek
1528 1.17 jdolecek void
1529 1.17 jdolecek nvme_softintr_intx(void *xq)
1530 1.17 jdolecek {
1531 1.17 jdolecek struct nvme_queue *q = xq;
1532 1.17 jdolecek struct nvme_softc *sc = q->q_sc;
1533 1.17 jdolecek
1534 1.17 jdolecek nvme_q_complete(sc, sc->sc_admin_q);
1535 1.1 nonaka if (sc->sc_q != NULL)
1536 1.17 jdolecek nvme_q_complete(sc, sc->sc_q[0]);
1537 1.1 nonaka
1538 1.17 jdolecek /*
1539 1.17 jdolecek * Processing done, tell controller to issue interrupts again. There
1540 1.17 jdolecek * is no race, as NVMe spec requires the controller to maintain state,
1541 1.17 jdolecek * and assert the interrupt whenever there are unacknowledged
1542 1.17 jdolecek * completion queue entries.
1543 1.17 jdolecek */
1544 1.17 jdolecek nvme_write4(sc, NVME_INTMC, 1);
1545 1.1 nonaka }
1546 1.1 nonaka
1547 1.1 nonaka int
1548 1.9 jdolecek nvme_intr_msi(void *xq)
1549 1.1 nonaka {
1550 1.1 nonaka struct nvme_queue *q = xq;
1551 1.1 nonaka
1552 1.9 jdolecek KASSERT(q && q->q_sc && q->q_sc->sc_softih
1553 1.9 jdolecek && q->q_sc->sc_softih[q->q_id]);
1554 1.1 nonaka
1555 1.17 jdolecek /*
1556 1.17 jdolecek * MSI/MSI-X are edge triggered, so can handover processing to softint
1557 1.17 jdolecek * without masking the interrupt.
1558 1.17 jdolecek */
1559 1.9 jdolecek softint_schedule(q->q_sc->sc_softih[q->q_id]);
1560 1.1 nonaka
1561 1.9 jdolecek return 1;
1562 1.1 nonaka }
1563 1.1 nonaka
1564 1.9 jdolecek void
1565 1.9 jdolecek nvme_softintr_msi(void *xq)
1566 1.1 nonaka {
1567 1.1 nonaka struct nvme_queue *q = xq;
1568 1.9 jdolecek struct nvme_softc *sc = q->q_sc;
1569 1.1 nonaka
1570 1.9 jdolecek nvme_q_complete(sc, q);
1571 1.1 nonaka }
1572 1.1 nonaka
1573 1.19 jdolecek static struct nvme_dmamem *
1574 1.19 jdolecek nvme_dmamem_alloc(struct nvme_softc *sc, size_t size)
1575 1.1 nonaka {
1576 1.19 jdolecek struct nvme_dmamem *ndm;
1577 1.1 nonaka int nsegs;
1578 1.1 nonaka
1579 1.19 jdolecek ndm = kmem_zalloc(sizeof(*ndm), KM_SLEEP);
1580 1.19 jdolecek if (ndm == NULL)
1581 1.19 jdolecek return NULL;
1582 1.19 jdolecek
1583 1.1 nonaka ndm->ndm_size = size;
1584 1.1 nonaka
1585 1.1 nonaka if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
1586 1.1 nonaka BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0)
1587 1.1 nonaka goto ndmfree;
1588 1.1 nonaka
1589 1.1 nonaka if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg,
1590 1.1 nonaka 1, &nsegs, BUS_DMA_WAITOK) != 0)
1591 1.1 nonaka goto destroy;
1592 1.1 nonaka
1593 1.1 nonaka if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size,
1594 1.1 nonaka &ndm->ndm_kva, BUS_DMA_WAITOK) != 0)
1595 1.1 nonaka goto free;
1596 1.1 nonaka memset(ndm->ndm_kva, 0, size);
1597 1.1 nonaka
1598 1.1 nonaka if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size,
1599 1.1 nonaka NULL, BUS_DMA_WAITOK) != 0)
1600 1.1 nonaka goto unmap;
1601 1.1 nonaka
1602 1.19 jdolecek return ndm;
1603 1.1 nonaka
1604 1.1 nonaka unmap:
1605 1.1 nonaka bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size);
1606 1.1 nonaka free:
1607 1.1 nonaka bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
1608 1.1 nonaka destroy:
1609 1.1 nonaka bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
1610 1.1 nonaka ndmfree:
1611 1.19 jdolecek kmem_free(ndm, sizeof(*ndm));
1612 1.19 jdolecek return NULL;
1613 1.19 jdolecek }
1614 1.19 jdolecek
1615 1.19 jdolecek static void
1616 1.19 jdolecek nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops)
1617 1.19 jdolecek {
1618 1.19 jdolecek bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem),
1619 1.19 jdolecek 0, NVME_DMA_LEN(mem), ops);
1620 1.1 nonaka }
1621 1.1 nonaka
1622 1.1 nonaka void
1623 1.1 nonaka nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm)
1624 1.1 nonaka {
1625 1.1 nonaka bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map);
1626 1.1 nonaka bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size);
1627 1.1 nonaka bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1);
1628 1.1 nonaka bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map);
1629 1.19 jdolecek kmem_free(ndm, sizeof(*ndm));
1630 1.1 nonaka }
1631 1.3 nonaka
1632 1.3 nonaka /*
1633 1.3 nonaka * ioctl
1634 1.3 nonaka */
1635 1.3 nonaka
1636 1.3 nonaka dev_type_open(nvmeopen);
1637 1.3 nonaka dev_type_close(nvmeclose);
1638 1.3 nonaka dev_type_ioctl(nvmeioctl);
1639 1.3 nonaka
1640 1.3 nonaka const struct cdevsw nvme_cdevsw = {
1641 1.3 nonaka .d_open = nvmeopen,
1642 1.3 nonaka .d_close = nvmeclose,
1643 1.3 nonaka .d_read = noread,
1644 1.3 nonaka .d_write = nowrite,
1645 1.3 nonaka .d_ioctl = nvmeioctl,
1646 1.3 nonaka .d_stop = nostop,
1647 1.3 nonaka .d_tty = notty,
1648 1.3 nonaka .d_poll = nopoll,
1649 1.3 nonaka .d_mmap = nommap,
1650 1.3 nonaka .d_kqfilter = nokqfilter,
1651 1.3 nonaka .d_discard = nodiscard,
1652 1.3 nonaka .d_flag = D_OTHER,
1653 1.3 nonaka };
1654 1.3 nonaka
1655 1.3 nonaka extern struct cfdriver nvme_cd;
1656 1.3 nonaka
1657 1.3 nonaka /*
1658 1.3 nonaka * Accept an open operation on the control device.
1659 1.3 nonaka */
1660 1.3 nonaka int
1661 1.3 nonaka nvmeopen(dev_t dev, int flag, int mode, struct lwp *l)
1662 1.3 nonaka {
1663 1.3 nonaka struct nvme_softc *sc;
1664 1.3 nonaka int unit = minor(dev) / 0x10000;
1665 1.3 nonaka int nsid = minor(dev) & 0xffff;
1666 1.3 nonaka int nsidx;
1667 1.3 nonaka
1668 1.3 nonaka if ((sc = device_lookup_private(&nvme_cd, unit)) == NULL)
1669 1.3 nonaka return ENXIO;
1670 1.3 nonaka if ((sc->sc_flags & NVME_F_ATTACHED) == 0)
1671 1.3 nonaka return ENXIO;
1672 1.3 nonaka
1673 1.5 nonaka if (nsid == 0) {
1674 1.5 nonaka /* controller */
1675 1.5 nonaka if (ISSET(sc->sc_flags, NVME_F_OPEN))
1676 1.5 nonaka return EBUSY;
1677 1.5 nonaka SET(sc->sc_flags, NVME_F_OPEN);
1678 1.5 nonaka } else {
1679 1.5 nonaka /* namespace */
1680 1.5 nonaka nsidx = nsid - 1;
1681 1.5 nonaka if (nsidx >= sc->sc_nn || sc->sc_namespaces[nsidx].dev == NULL)
1682 1.5 nonaka return ENXIO;
1683 1.5 nonaka if (ISSET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN))
1684 1.5 nonaka return EBUSY;
1685 1.5 nonaka SET(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
1686 1.5 nonaka }
1687 1.3 nonaka return 0;
1688 1.3 nonaka }
1689 1.3 nonaka
1690 1.3 nonaka /*
1691 1.3 nonaka * Accept the last close on the control device.
1692 1.3 nonaka */
1693 1.3 nonaka int
1694 1.5 nonaka nvmeclose(dev_t dev, int flag, int mode, struct lwp *l)
1695 1.3 nonaka {
1696 1.3 nonaka struct nvme_softc *sc;
1697 1.3 nonaka int unit = minor(dev) / 0x10000;
1698 1.3 nonaka int nsid = minor(dev) & 0xffff;
1699 1.3 nonaka int nsidx;
1700 1.3 nonaka
1701 1.3 nonaka sc = device_lookup_private(&nvme_cd, unit);
1702 1.3 nonaka if (sc == NULL)
1703 1.3 nonaka return ENXIO;
1704 1.3 nonaka
1705 1.5 nonaka if (nsid == 0) {
1706 1.5 nonaka /* controller */
1707 1.5 nonaka CLR(sc->sc_flags, NVME_F_OPEN);
1708 1.5 nonaka } else {
1709 1.5 nonaka /* namespace */
1710 1.5 nonaka nsidx = nsid - 1;
1711 1.5 nonaka if (nsidx >= sc->sc_nn)
1712 1.5 nonaka return ENXIO;
1713 1.5 nonaka CLR(sc->sc_namespaces[nsidx].flags, NVME_NS_F_OPEN);
1714 1.5 nonaka }
1715 1.3 nonaka
1716 1.3 nonaka return 0;
1717 1.3 nonaka }
1718 1.3 nonaka
1719 1.3 nonaka /*
1720 1.3 nonaka * Handle control operations.
1721 1.3 nonaka */
1722 1.3 nonaka int
1723 1.5 nonaka nvmeioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
1724 1.3 nonaka {
1725 1.3 nonaka struct nvme_softc *sc;
1726 1.3 nonaka int unit = minor(dev) / 0x10000;
1727 1.3 nonaka int nsid = minor(dev) & 0xffff;
1728 1.5 nonaka struct nvme_pt_command *pt;
1729 1.3 nonaka
1730 1.3 nonaka sc = device_lookup_private(&nvme_cd, unit);
1731 1.3 nonaka if (sc == NULL)
1732 1.3 nonaka return ENXIO;
1733 1.3 nonaka
1734 1.3 nonaka switch (cmd) {
1735 1.3 nonaka case NVME_PASSTHROUGH_CMD:
1736 1.5 nonaka pt = data;
1737 1.5 nonaka return nvme_command_passthrough(sc, data,
1738 1.5 nonaka nsid == 0 ? pt->cmd.nsid : nsid, l, nsid == 0);
1739 1.3 nonaka }
1740 1.3 nonaka
1741 1.3 nonaka return ENOTTY;
1742 1.3 nonaka }
1743