1 | #include <machine/rtems-bsd-kernel-space.h> |
---|
2 | |
---|
3 | /*- |
---|
4 | * SPDX-License-Identifier: BSD-2-Clause-FreeBSD |
---|
5 | * |
---|
6 | * Copyright (C) 2012-2014 Intel Corporation |
---|
7 | * All rights reserved. |
---|
8 | * |
---|
9 | * Redistribution and use in source and binary forms, with or without |
---|
10 | * modification, are permitted provided that the following conditions |
---|
11 | * are met: |
---|
12 | * 1. Redistributions of source code must retain the above copyright |
---|
13 | * notice, this list of conditions and the following disclaimer. |
---|
14 | * 2. Redistributions in binary form must reproduce the above copyright |
---|
15 | * notice, this list of conditions and the following disclaimer in the |
---|
16 | * documentation and/or other materials provided with the distribution. |
---|
17 | * |
---|
18 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
---|
19 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
---|
20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
---|
21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
---|
22 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
---|
23 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
---|
24 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
---|
25 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
---|
26 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
---|
27 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
---|
28 | * SUCH DAMAGE. |
---|
29 | */ |
---|
30 | |
---|
31 | #include <sys/cdefs.h> |
---|
32 | __FBSDID("$FreeBSD$"); |
---|
33 | |
---|
34 | #include <sys/param.h> |
---|
35 | #include <sys/bus.h> |
---|
36 | #include <sys/conf.h> |
---|
37 | #include <sys/domainset.h> |
---|
38 | #include <sys/proc.h> |
---|
39 | |
---|
40 | #include <dev/pci/pcivar.h> |
---|
41 | |
---|
42 | #include "nvme_private.h" |
---|
43 | |
---|
44 | typedef enum error_print { ERROR_PRINT_NONE, ERROR_PRINT_NO_RETRY, ERROR_PRINT_ALL } error_print_t; |
---|
45 | #define DO_NOT_RETRY 1 |
---|
46 | |
---|
47 | static void _nvme_qpair_submit_request(struct nvme_qpair *qpair, |
---|
48 | struct nvme_request *req); |
---|
49 | static void nvme_qpair_destroy(struct nvme_qpair *qpair); |
---|
50 | |
---|
51 | struct nvme_opcode_string { |
---|
52 | |
---|
53 | uint16_t opc; |
---|
54 | const char * str; |
---|
55 | }; |
---|
56 | |
---|
57 | static struct nvme_opcode_string admin_opcode[] = { |
---|
58 | { NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, |
---|
59 | { NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, |
---|
60 | { NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, |
---|
61 | { NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, |
---|
62 | { NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, |
---|
63 | { NVME_OPC_IDENTIFY, "IDENTIFY" }, |
---|
64 | { NVME_OPC_ABORT, "ABORT" }, |
---|
65 | { NVME_OPC_SET_FEATURES, "SET FEATURES" }, |
---|
66 | { NVME_OPC_GET_FEATURES, "GET FEATURES" }, |
---|
67 | { NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, |
---|
68 | { NVME_OPC_FIRMWARE_ACTIVATE, "FIRMWARE ACTIVATE" }, |
---|
69 | { NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, |
---|
70 | { NVME_OPC_DEVICE_SELF_TEST, "DEVICE SELF-TEST" }, |
---|
71 | { NVME_OPC_NAMESPACE_ATTACHMENT, "NAMESPACE ATTACHMENT" }, |
---|
72 | { NVME_OPC_KEEP_ALIVE, "KEEP ALIVE" }, |
---|
73 | { NVME_OPC_DIRECTIVE_SEND, "DIRECTIVE SEND" }, |
---|
74 | { NVME_OPC_DIRECTIVE_RECEIVE, "DIRECTIVE RECEIVE" }, |
---|
75 | { NVME_OPC_VIRTUALIZATION_MANAGEMENT, "VIRTUALIZATION MANAGEMENT" }, |
---|
76 | { NVME_OPC_NVME_MI_SEND, "NVME-MI SEND" }, |
---|
77 | { NVME_OPC_NVME_MI_RECEIVE, "NVME-MI RECEIVE" }, |
---|
78 | { NVME_OPC_DOORBELL_BUFFER_CONFIG, "DOORBELL BUFFER CONFIG" }, |
---|
79 | { NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, |
---|
80 | { NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, |
---|
81 | { NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, |
---|
82 | { NVME_OPC_SANITIZE, "SANITIZE" }, |
---|
83 | { NVME_OPC_GET_LBA_STATUS, "GET LBA STATUS" }, |
---|
84 | { 0xFFFF, "ADMIN COMMAND" } |
---|
85 | }; |
---|
86 | |
---|
87 | static struct nvme_opcode_string io_opcode[] = { |
---|
88 | { NVME_OPC_FLUSH, "FLUSH" }, |
---|
89 | { NVME_OPC_WRITE, "WRITE" }, |
---|
90 | { NVME_OPC_READ, "READ" }, |
---|
91 | { NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, |
---|
92 | { NVME_OPC_COMPARE, "COMPARE" }, |
---|
93 | { NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" }, |
---|
94 | { NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, |
---|
95 | { NVME_OPC_VERIFY, "VERIFY" }, |
---|
96 | { NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" }, |
---|
97 | { NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" }, |
---|
98 | { NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" }, |
---|
99 | { NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" }, |
---|
100 | { 0xFFFF, "IO COMMAND" } |
---|
101 | }; |
---|
102 | |
---|
103 | static const char * |
---|
104 | get_admin_opcode_string(uint16_t opc) |
---|
105 | { |
---|
106 | struct nvme_opcode_string *entry; |
---|
107 | |
---|
108 | entry = admin_opcode; |
---|
109 | |
---|
110 | while (entry->opc != 0xFFFF) { |
---|
111 | if (entry->opc == opc) |
---|
112 | return (entry->str); |
---|
113 | entry++; |
---|
114 | } |
---|
115 | return (entry->str); |
---|
116 | } |
---|
117 | |
---|
118 | static const char * |
---|
119 | get_io_opcode_string(uint16_t opc) |
---|
120 | { |
---|
121 | struct nvme_opcode_string *entry; |
---|
122 | |
---|
123 | entry = io_opcode; |
---|
124 | |
---|
125 | while (entry->opc != 0xFFFF) { |
---|
126 | if (entry->opc == opc) |
---|
127 | return (entry->str); |
---|
128 | entry++; |
---|
129 | } |
---|
130 | return (entry->str); |
---|
131 | } |
---|
132 | |
---|
133 | |
---|
134 | static void |
---|
135 | nvme_admin_qpair_print_command(struct nvme_qpair *qpair, |
---|
136 | struct nvme_command *cmd) |
---|
137 | { |
---|
138 | |
---|
139 | nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%x " |
---|
140 | "cdw10:%08x cdw11:%08x\n", |
---|
141 | get_admin_opcode_string(cmd->opc), cmd->opc, qpair->id, cmd->cid, |
---|
142 | le32toh(cmd->nsid), le32toh(cmd->cdw10), le32toh(cmd->cdw11)); |
---|
143 | } |
---|
144 | |
---|
145 | static void |
---|
146 | nvme_io_qpair_print_command(struct nvme_qpair *qpair, |
---|
147 | struct nvme_command *cmd) |
---|
148 | { |
---|
149 | |
---|
150 | switch (cmd->opc) { |
---|
151 | case NVME_OPC_WRITE: |
---|
152 | case NVME_OPC_READ: |
---|
153 | case NVME_OPC_WRITE_UNCORRECTABLE: |
---|
154 | case NVME_OPC_COMPARE: |
---|
155 | case NVME_OPC_WRITE_ZEROES: |
---|
156 | case NVME_OPC_VERIFY: |
---|
157 | nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d " |
---|
158 | "lba:%llu len:%d\n", |
---|
159 | get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, le32toh(cmd->nsid), |
---|
160 | ((unsigned long long)le32toh(cmd->cdw11) << 32) + le32toh(cmd->cdw10), |
---|
161 | (le32toh(cmd->cdw12) & 0xFFFF) + 1); |
---|
162 | break; |
---|
163 | case NVME_OPC_FLUSH: |
---|
164 | case NVME_OPC_DATASET_MANAGEMENT: |
---|
165 | case NVME_OPC_RESERVATION_REGISTER: |
---|
166 | case NVME_OPC_RESERVATION_REPORT: |
---|
167 | case NVME_OPC_RESERVATION_ACQUIRE: |
---|
168 | case NVME_OPC_RESERVATION_RELEASE: |
---|
169 | nvme_printf(qpair->ctrlr, "%s sqid:%d cid:%d nsid:%d\n", |
---|
170 | get_io_opcode_string(cmd->opc), qpair->id, cmd->cid, le32toh(cmd->nsid)); |
---|
171 | break; |
---|
172 | default: |
---|
173 | nvme_printf(qpair->ctrlr, "%s (%02x) sqid:%d cid:%d nsid:%d\n", |
---|
174 | get_io_opcode_string(cmd->opc), cmd->opc, qpair->id, |
---|
175 | cmd->cid, le32toh(cmd->nsid)); |
---|
176 | break; |
---|
177 | } |
---|
178 | } |
---|
179 | |
---|
180 | static void |
---|
181 | nvme_qpair_print_command(struct nvme_qpair *qpair, struct nvme_command *cmd) |
---|
182 | { |
---|
183 | if (qpair->id == 0) |
---|
184 | nvme_admin_qpair_print_command(qpair, cmd); |
---|
185 | else |
---|
186 | nvme_io_qpair_print_command(qpair, cmd); |
---|
187 | if (nvme_verbose_cmd_dump) { |
---|
188 | nvme_printf(qpair->ctrlr, |
---|
189 | "nsid:%#x rsvd2:%#x rsvd3:%#x mptr:%#jx prp1:%#jx prp2:%#jx\n", |
---|
190 | cmd->nsid, cmd->rsvd2, cmd->rsvd3, (uintmax_t)cmd->mptr, |
---|
191 | (uintmax_t)cmd->prp1, (uintmax_t)cmd->prp2); |
---|
192 | nvme_printf(qpair->ctrlr, |
---|
193 | "cdw10: %#x cdw11:%#x cdw12:%#x cdw13:%#x cdw14:%#x cdw15:%#x\n", |
---|
194 | cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, |
---|
195 | cmd->cdw15); |
---|
196 | } |
---|
197 | } |
---|
198 | |
---|
199 | struct nvme_status_string { |
---|
200 | |
---|
201 | uint16_t sc; |
---|
202 | const char * str; |
---|
203 | }; |
---|
204 | |
---|
205 | static struct nvme_status_string generic_status[] = { |
---|
206 | { NVME_SC_SUCCESS, "SUCCESS" }, |
---|
207 | { NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, |
---|
208 | { NVME_SC_INVALID_FIELD, "INVALID_FIELD" }, |
---|
209 | { NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, |
---|
210 | { NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, |
---|
211 | { NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, |
---|
212 | { NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, |
---|
213 | { NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, |
---|
214 | { NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, |
---|
215 | { NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, |
---|
216 | { NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, |
---|
217 | { NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, |
---|
218 | { NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, |
---|
219 | { NVME_SC_INVALID_SGL_SEGMENT_DESCR, "INVALID SGL SEGMENT DESCRIPTOR" }, |
---|
220 | { NVME_SC_INVALID_NUMBER_OF_SGL_DESCR, "INVALID NUMBER OF SGL DESCRIPTORS" }, |
---|
221 | { NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" }, |
---|
222 | { NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" }, |
---|
223 | { NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" }, |
---|
224 | { NVME_SC_INVALID_USE_OF_CMB, "INVALID USE OF CONTROLLER MEMORY BUFFER" }, |
---|
225 | { NVME_SC_PRP_OFFET_INVALID, "PRP OFFET INVALID" }, |
---|
226 | { NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" }, |
---|
227 | { NVME_SC_OPERATION_DENIED, "OPERATION DENIED" }, |
---|
228 | { NVME_SC_SGL_OFFSET_INVALID, "SGL OFFSET INVALID" }, |
---|
229 | { NVME_SC_HOST_ID_INCONSISTENT_FORMAT, "HOST IDENTIFIER INCONSISTENT FORMAT" }, |
---|
230 | { NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED, "KEEP ALIVE TIMEOUT EXPIRED" }, |
---|
231 | { NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID, "KEEP ALIVE TIMEOUT INVALID" }, |
---|
232 | { NVME_SC_ABORTED_DUE_TO_PREEMPT, "COMMAND ABORTED DUE TO PREEMPT AND ABORT" }, |
---|
233 | { NVME_SC_SANITIZE_FAILED, "SANITIZE FAILED" }, |
---|
234 | { NVME_SC_SANITIZE_IN_PROGRESS, "SANITIZE IN PROGRESS" }, |
---|
235 | { NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID, "SGL_DATA_BLOCK_GRANULARITY_INVALID" }, |
---|
236 | { NVME_SC_NOT_SUPPORTED_IN_CMB, "COMMAND NOT SUPPORTED FOR QUEUE IN CMB" }, |
---|
237 | { NVME_SC_NAMESPACE_IS_WRITE_PROTECTED, "NAMESPACE IS WRITE PROTECTED" }, |
---|
238 | { NVME_SC_COMMAND_INTERRUPTED, "COMMAND INTERRUPTED" }, |
---|
239 | { NVME_SC_TRANSIENT_TRANSPORT_ERROR, "TRANSIENT TRANSPORT ERROR" }, |
---|
240 | |
---|
241 | { NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, |
---|
242 | { NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, |
---|
243 | { NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, |
---|
244 | { NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" }, |
---|
245 | { NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" }, |
---|
246 | { 0xFFFF, "GENERIC" } |
---|
247 | }; |
---|
248 | |
---|
249 | static struct nvme_status_string command_specific_status[] = { |
---|
250 | { NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, |
---|
251 | { NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, |
---|
252 | { NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" }, |
---|
253 | { NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, |
---|
254 | { NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, |
---|
255 | { NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, |
---|
256 | { NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, |
---|
257 | { NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, |
---|
258 | { NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, |
---|
259 | { NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, |
---|
260 | { NVME_SC_FIRMWARE_REQUIRES_RESET, "FIRMWARE REQUIRES RESET" }, |
---|
261 | { NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" }, |
---|
262 | { NVME_SC_FEATURE_NOT_SAVEABLE, "FEATURE IDENTIFIER NOT SAVEABLE" }, |
---|
263 | { NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" }, |
---|
264 | { NVME_SC_FEATURE_NOT_NS_SPECIFIC, "FEATURE NOT NAMESPACE SPECIFIC" }, |
---|
265 | { NVME_SC_FW_ACT_REQUIRES_NVMS_RESET, "FIRMWARE ACTIVATION REQUIRES NVM SUBSYSTEM RESET" }, |
---|
266 | { NVME_SC_FW_ACT_REQUIRES_RESET, "FIRMWARE ACTIVATION REQUIRES RESET" }, |
---|
267 | { NVME_SC_FW_ACT_REQUIRES_TIME, "FIRMWARE ACTIVATION REQUIRES MAXIMUM TIME VIOLATION" }, |
---|
268 | { NVME_SC_FW_ACT_PROHIBITED, "FIRMWARE ACTIVATION PROHIBITED" }, |
---|
269 | { NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" }, |
---|
270 | { NVME_SC_NS_INSUFFICIENT_CAPACITY, "NAMESPACE INSUFFICIENT CAPACITY" }, |
---|
271 | { NVME_SC_NS_ID_UNAVAILABLE, "NAMESPACE IDENTIFIER UNAVAILABLE" }, |
---|
272 | { NVME_SC_NS_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" }, |
---|
273 | { NVME_SC_NS_IS_PRIVATE, "NAMESPACE IS PRIVATE" }, |
---|
274 | { NVME_SC_NS_NOT_ATTACHED, "NS NOT ATTACHED" }, |
---|
275 | { NVME_SC_THIN_PROV_NOT_SUPPORTED, "THIN PROVISIONING NOT SUPPORTED" }, |
---|
276 | { NVME_SC_CTRLR_LIST_INVALID, "CONTROLLER LIST INVALID" }, |
---|
277 | { NVME_SC_SELT_TEST_IN_PROGRESS, "DEVICE SELT-TEST IN PROGRESS" }, |
---|
278 | { NVME_SC_BOOT_PART_WRITE_PROHIB, "BOOT PARTITION WRITE PROHIBITED" }, |
---|
279 | { NVME_SC_INVALID_CTRLR_ID, "INVALID CONTROLLER IDENTIFIER" }, |
---|
280 | { NVME_SC_INVALID_SEC_CTRLR_STATE, "INVALID SECONDARY CONTROLLER STATE" }, |
---|
281 | { NVME_SC_INVALID_NUM_OF_CTRLR_RESRC, "INVALID NUMBER OF CONTROLLER RESOURCES" }, |
---|
282 | { NVME_SC_INVALID_RESOURCE_ID, "INVALID RESOURCE IDENTIFIER" }, |
---|
283 | { NVME_SC_SANITIZE_PROHIBITED_WPMRE, "SANITIZE PROHIBITED WRITE PERSISTENT MEMORY REGION ENABLED" }, |
---|
284 | { NVME_SC_ANA_GROUP_ID_INVALID, "ANA GROUP IDENTIFIED INVALID" }, |
---|
285 | { NVME_SC_ANA_ATTACH_FAILED, "ANA ATTACH FAILED" }, |
---|
286 | |
---|
287 | { NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, |
---|
288 | { NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, |
---|
289 | { NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE, "WRITE TO RO PAGE" }, |
---|
290 | { 0xFFFF, "COMMAND SPECIFIC" } |
---|
291 | }; |
---|
292 | |
---|
293 | static struct nvme_status_string media_error_status[] = { |
---|
294 | { NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, |
---|
295 | { NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, |
---|
296 | { NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, |
---|
297 | { NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, |
---|
298 | { NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, |
---|
299 | { NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, |
---|
300 | { NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, |
---|
301 | { NVME_SC_DEALLOCATED_OR_UNWRITTEN, "DEALLOCATED OR UNWRITTEN LOGICAL BLOCK" }, |
---|
302 | { 0xFFFF, "MEDIA ERROR" } |
---|
303 | }; |
---|
304 | |
---|
305 | static struct nvme_status_string path_related_status[] = { |
---|
306 | { NVME_SC_INTERNAL_PATH_ERROR, "INTERNAL PATH ERROR" }, |
---|
307 | { NVME_SC_ASYMMETRIC_ACCESS_PERSISTENT_LOSS, "ASYMMETRIC ACCESS PERSISTENT LOSS" }, |
---|
308 | { NVME_SC_ASYMMETRIC_ACCESS_INACCESSIBLE, "ASYMMETRIC ACCESS INACCESSIBLE" }, |
---|
309 | { NVME_SC_ASYMMETRIC_ACCESS_TRANSITION, "ASYMMETRIC ACCESS TRANSITION" }, |
---|
310 | { NVME_SC_CONTROLLER_PATHING_ERROR, "CONTROLLER PATHING ERROR" }, |
---|
311 | { NVME_SC_HOST_PATHING_ERROR, "HOST PATHING ERROR" }, |
---|
312 | { NVME_SC_COMMAND_ABOTHED_BY_HOST, "COMMAND ABOTHED BY HOST" }, |
---|
313 | { 0xFFFF, "PATH RELATED" }, |
---|
314 | }; |
---|
315 | |
---|
316 | static const char * |
---|
317 | get_status_string(uint16_t sct, uint16_t sc) |
---|
318 | { |
---|
319 | struct nvme_status_string *entry; |
---|
320 | |
---|
321 | switch (sct) { |
---|
322 | case NVME_SCT_GENERIC: |
---|
323 | entry = generic_status; |
---|
324 | break; |
---|
325 | case NVME_SCT_COMMAND_SPECIFIC: |
---|
326 | entry = command_specific_status; |
---|
327 | break; |
---|
328 | case NVME_SCT_MEDIA_ERROR: |
---|
329 | entry = media_error_status; |
---|
330 | break; |
---|
331 | case NVME_SCT_PATH_RELATED: |
---|
332 | entry = path_related_status; |
---|
333 | break; |
---|
334 | case NVME_SCT_VENDOR_SPECIFIC: |
---|
335 | return ("VENDOR SPECIFIC"); |
---|
336 | default: |
---|
337 | return ("RESERVED"); |
---|
338 | } |
---|
339 | |
---|
340 | while (entry->sc != 0xFFFF) { |
---|
341 | if (entry->sc == sc) |
---|
342 | return (entry->str); |
---|
343 | entry++; |
---|
344 | } |
---|
345 | return (entry->str); |
---|
346 | } |
---|
347 | |
---|
348 | static void |
---|
349 | nvme_qpair_print_completion(struct nvme_qpair *qpair, |
---|
350 | struct nvme_completion *cpl) |
---|
351 | { |
---|
352 | uint16_t sct, sc; |
---|
353 | |
---|
354 | sct = NVME_STATUS_GET_SCT(cpl->status); |
---|
355 | sc = NVME_STATUS_GET_SC(cpl->status); |
---|
356 | |
---|
357 | nvme_printf(qpair->ctrlr, "%s (%02x/%02x) sqid:%d cid:%d cdw0:%x\n", |
---|
358 | get_status_string(sct, sc), sct, sc, cpl->sqid, cpl->cid, |
---|
359 | cpl->cdw0); |
---|
360 | } |
---|
361 | |
---|
362 | static boolean_t |
---|
363 | nvme_completion_is_retry(const struct nvme_completion *cpl) |
---|
364 | { |
---|
365 | uint8_t sct, sc, dnr; |
---|
366 | |
---|
367 | sct = NVME_STATUS_GET_SCT(cpl->status); |
---|
368 | sc = NVME_STATUS_GET_SC(cpl->status); |
---|
369 | dnr = NVME_STATUS_GET_DNR(cpl->status); /* Do Not Retry Bit */ |
---|
370 | |
---|
371 | /* |
---|
372 | * TODO: spec is not clear how commands that are aborted due |
---|
373 | * to TLER will be marked. So for now, it seems |
---|
374 | * NAMESPACE_NOT_READY is the only case where we should |
---|
375 | * look at the DNR bit. Requests failed with ABORTED_BY_REQUEST |
---|
376 | * set the DNR bit correctly since the driver controls that. |
---|
377 | */ |
---|
378 | switch (sct) { |
---|
379 | case NVME_SCT_GENERIC: |
---|
380 | switch (sc) { |
---|
381 | case NVME_SC_ABORTED_BY_REQUEST: |
---|
382 | case NVME_SC_NAMESPACE_NOT_READY: |
---|
383 | if (dnr) |
---|
384 | return (0); |
---|
385 | else |
---|
386 | return (1); |
---|
387 | case NVME_SC_INVALID_OPCODE: |
---|
388 | case NVME_SC_INVALID_FIELD: |
---|
389 | case NVME_SC_COMMAND_ID_CONFLICT: |
---|
390 | case NVME_SC_DATA_TRANSFER_ERROR: |
---|
391 | case NVME_SC_ABORTED_POWER_LOSS: |
---|
392 | case NVME_SC_INTERNAL_DEVICE_ERROR: |
---|
393 | case NVME_SC_ABORTED_SQ_DELETION: |
---|
394 | case NVME_SC_ABORTED_FAILED_FUSED: |
---|
395 | case NVME_SC_ABORTED_MISSING_FUSED: |
---|
396 | case NVME_SC_INVALID_NAMESPACE_OR_FORMAT: |
---|
397 | case NVME_SC_COMMAND_SEQUENCE_ERROR: |
---|
398 | case NVME_SC_LBA_OUT_OF_RANGE: |
---|
399 | case NVME_SC_CAPACITY_EXCEEDED: |
---|
400 | default: |
---|
401 | return (0); |
---|
402 | } |
---|
403 | case NVME_SCT_COMMAND_SPECIFIC: |
---|
404 | case NVME_SCT_MEDIA_ERROR: |
---|
405 | return (0); |
---|
406 | case NVME_SCT_PATH_RELATED: |
---|
407 | switch (sc) { |
---|
408 | case NVME_SC_INTERNAL_PATH_ERROR: |
---|
409 | if (dnr) |
---|
410 | return (0); |
---|
411 | else |
---|
412 | return (1); |
---|
413 | default: |
---|
414 | return (0); |
---|
415 | } |
---|
416 | case NVME_SCT_VENDOR_SPECIFIC: |
---|
417 | default: |
---|
418 | return (0); |
---|
419 | } |
---|
420 | } |
---|
421 | |
---|
422 | static void |
---|
423 | nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, |
---|
424 | struct nvme_completion *cpl, error_print_t print_on_error) |
---|
425 | { |
---|
426 | struct nvme_request *req; |
---|
427 | boolean_t retry, error, retriable; |
---|
428 | |
---|
429 | req = tr->req; |
---|
430 | error = nvme_completion_is_error(cpl); |
---|
431 | retriable = nvme_completion_is_retry(cpl); |
---|
432 | retry = error && retriable && req->retries < nvme_retry_count; |
---|
433 | if (retry) |
---|
434 | qpair->num_retries++; |
---|
435 | if (error && req->retries >= nvme_retry_count && retriable) |
---|
436 | qpair->num_failures++; |
---|
437 | |
---|
438 | if (error && (print_on_error == ERROR_PRINT_ALL || |
---|
439 | (!retry && print_on_error == ERROR_PRINT_NO_RETRY))) { |
---|
440 | nvme_qpair_print_command(qpair, &req->cmd); |
---|
441 | nvme_qpair_print_completion(qpair, cpl); |
---|
442 | } |
---|
443 | |
---|
444 | qpair->act_tr[cpl->cid] = NULL; |
---|
445 | |
---|
446 | KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n")); |
---|
447 | |
---|
448 | if (req->cb_fn && !retry) |
---|
449 | req->cb_fn(req->cb_arg, cpl); |
---|
450 | |
---|
451 | mtx_lock(&qpair->lock); |
---|
452 | callout_stop(&tr->timer); |
---|
453 | |
---|
454 | if (retry) { |
---|
455 | req->retries++; |
---|
456 | nvme_qpair_submit_tracker(qpair, tr); |
---|
457 | } else { |
---|
458 | if (req->type != NVME_REQUEST_NULL) { |
---|
459 | bus_dmamap_sync(qpair->dma_tag_payload, |
---|
460 | tr->payload_dma_map, |
---|
461 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); |
---|
462 | bus_dmamap_unload(qpair->dma_tag_payload, |
---|
463 | tr->payload_dma_map); |
---|
464 | } |
---|
465 | |
---|
466 | nvme_free_request(req); |
---|
467 | tr->req = NULL; |
---|
468 | |
---|
469 | TAILQ_REMOVE(&qpair->outstanding_tr, tr, tailq); |
---|
470 | TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); |
---|
471 | |
---|
472 | /* |
---|
473 | * If the controller is in the middle of resetting, don't |
---|
474 | * try to submit queued requests here - let the reset logic |
---|
475 | * handle that instead. |
---|
476 | */ |
---|
477 | if (!STAILQ_EMPTY(&qpair->queued_req) && |
---|
478 | !qpair->ctrlr->is_resetting) { |
---|
479 | req = STAILQ_FIRST(&qpair->queued_req); |
---|
480 | STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); |
---|
481 | _nvme_qpair_submit_request(qpair, req); |
---|
482 | } |
---|
483 | } |
---|
484 | |
---|
485 | mtx_unlock(&qpair->lock); |
---|
486 | } |
---|
487 | |
---|
488 | static void |
---|
489 | nvme_qpair_manual_complete_tracker(struct nvme_qpair *qpair, |
---|
490 | struct nvme_tracker *tr, uint32_t sct, uint32_t sc, uint32_t dnr, |
---|
491 | error_print_t print_on_error) |
---|
492 | { |
---|
493 | struct nvme_completion cpl; |
---|
494 | |
---|
495 | memset(&cpl, 0, sizeof(cpl)); |
---|
496 | cpl.sqid = qpair->id; |
---|
497 | cpl.cid = tr->cid; |
---|
498 | cpl.status |= (sct & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT; |
---|
499 | cpl.status |= (sc & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; |
---|
500 | cpl.status |= (dnr & NVME_STATUS_DNR_MASK) << NVME_STATUS_DNR_SHIFT; |
---|
501 | nvme_qpair_complete_tracker(qpair, tr, &cpl, print_on_error); |
---|
502 | } |
---|
503 | |
---|
504 | void |
---|
505 | nvme_qpair_manual_complete_request(struct nvme_qpair *qpair, |
---|
506 | struct nvme_request *req, uint32_t sct, uint32_t sc) |
---|
507 | { |
---|
508 | struct nvme_completion cpl; |
---|
509 | boolean_t error; |
---|
510 | |
---|
511 | memset(&cpl, 0, sizeof(cpl)); |
---|
512 | cpl.sqid = qpair->id; |
---|
513 | cpl.status |= (sct & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT; |
---|
514 | cpl.status |= (sc & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT; |
---|
515 | |
---|
516 | error = nvme_completion_is_error(&cpl); |
---|
517 | |
---|
518 | if (error) { |
---|
519 | nvme_qpair_print_command(qpair, &req->cmd); |
---|
520 | nvme_qpair_print_completion(qpair, &cpl); |
---|
521 | } |
---|
522 | |
---|
523 | if (req->cb_fn) |
---|
524 | req->cb_fn(req->cb_arg, &cpl); |
---|
525 | |
---|
526 | nvme_free_request(req); |
---|
527 | } |
---|
528 | |
---|
529 | bool |
---|
530 | nvme_qpair_process_completions(struct nvme_qpair *qpair) |
---|
531 | { |
---|
532 | struct nvme_tracker *tr; |
---|
533 | struct nvme_completion cpl; |
---|
534 | int done = 0; |
---|
535 | bool in_panic = dumping || SCHEDULER_STOPPED(); |
---|
536 | |
---|
537 | qpair->num_intr_handler_calls++; |
---|
538 | |
---|
539 | /* |
---|
540 | * qpair is not enabled, likely because a controller reset is is in |
---|
541 | * progress. Ignore the interrupt - any I/O that was associated with |
---|
542 | * this interrupt will get retried when the reset is complete. |
---|
543 | */ |
---|
544 | if (!qpair->is_enabled) |
---|
545 | return (false); |
---|
546 | |
---|
547 | /* |
---|
548 | * A panic can stop the CPU this routine is running on at any point. If |
---|
549 | * we're called during a panic, complete the sq_head wrap protocol for |
---|
550 | * the case where we are interrupted just after the increment at 1 |
---|
551 | * below, but before we can reset cq_head to zero at 2. Also cope with |
---|
552 | * the case where we do the zero at 2, but may or may not have done the |
---|
553 | * phase adjustment at step 3. The panic machinery flushes all pending |
---|
554 | * memory writes, so we can make these strong ordering assumptions |
---|
555 | * that would otherwise be unwise if we were racing in real time. |
---|
556 | */ |
---|
557 | if (__predict_false(in_panic)) { |
---|
558 | if (qpair->cq_head == qpair->num_entries) { |
---|
559 | /* |
---|
560 | * Here we know that we need to zero cq_head and then negate |
---|
561 | * the phase, which hasn't been assigned if cq_head isn't |
---|
562 | * zero due to the atomic_store_rel. |
---|
563 | */ |
---|
564 | qpair->cq_head = 0; |
---|
565 | qpair->phase = !qpair->phase; |
---|
566 | } else if (qpair->cq_head == 0) { |
---|
567 | /* |
---|
568 | * In this case, we know that the assignment at 2 |
---|
569 | * happened below, but we don't know if it 3 happened or |
---|
570 | * not. To do this, we look at the last completion |
---|
571 | * entry and set the phase to the opposite phase |
---|
572 | * that it has. This gets us back in sync |
---|
573 | */ |
---|
574 | cpl = qpair->cpl[qpair->num_entries - 1]; |
---|
575 | nvme_completion_swapbytes(&cpl); |
---|
576 | qpair->phase = !NVME_STATUS_GET_P(cpl.status); |
---|
577 | } |
---|
578 | } |
---|
579 | |
---|
580 | bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, |
---|
581 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); |
---|
582 | while (1) { |
---|
583 | cpl = qpair->cpl[qpair->cq_head]; |
---|
584 | |
---|
585 | /* Convert to host endian */ |
---|
586 | nvme_completion_swapbytes(&cpl); |
---|
587 | |
---|
588 | if (NVME_STATUS_GET_P(cpl.status) != qpair->phase) |
---|
589 | break; |
---|
590 | |
---|
591 | tr = qpair->act_tr[cpl.cid]; |
---|
592 | |
---|
593 | if (tr != NULL) { |
---|
594 | nvme_qpair_complete_tracker(qpair, tr, &cpl, ERROR_PRINT_ALL); |
---|
595 | qpair->sq_head = cpl.sqhd; |
---|
596 | done++; |
---|
597 | } else if (!in_panic) { |
---|
598 | /* |
---|
599 | * A missing tracker is normally an error. However, a |
---|
600 | * panic can stop the CPU this routine is running on |
---|
601 | * after completing an I/O but before updating |
---|
602 | * qpair->cq_head at 1 below. Later, we re-enter this |
---|
603 | * routine to poll I/O associated with the kernel |
---|
604 | * dump. We find that the tr has been set to null before |
---|
605 | * calling the completion routine. If it hasn't |
---|
606 | * completed (or it triggers a panic), then '1' below |
---|
607 | * won't have updated cq_head. Rather than panic again, |
---|
608 | * ignore this condition because it's not unexpected. |
---|
609 | */ |
---|
610 | nvme_printf(qpair->ctrlr, |
---|
611 | "cpl does not map to outstanding cmd\n"); |
---|
612 | /* nvme_dump_completion expects device endianess */ |
---|
613 | nvme_dump_completion(&qpair->cpl[qpair->cq_head]); |
---|
614 | KASSERT(0, ("received completion for unknown cmd")); |
---|
615 | } |
---|
616 | |
---|
617 | /* |
---|
618 | * There's a number of races with the following (see above) when |
---|
619 | * the system panics. We compensate for each one of them by |
---|
620 | * using the atomic store to force strong ordering (at least when |
---|
621 | * viewed in the aftermath of a panic). |
---|
622 | */ |
---|
623 | if (++qpair->cq_head == qpair->num_entries) { /* 1 */ |
---|
624 | atomic_store_rel_int(&qpair->cq_head, 0); /* 2 */ |
---|
625 | qpair->phase = !qpair->phase; /* 3 */ |
---|
626 | } |
---|
627 | |
---|
628 | bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, |
---|
629 | qpair->cq_hdbl_off, qpair->cq_head); |
---|
630 | } |
---|
631 | return (done != 0); |
---|
632 | } |
---|
633 | |
---|
634 | static void |
---|
635 | nvme_qpair_msix_handler(void *arg) |
---|
636 | { |
---|
637 | struct nvme_qpair *qpair = arg; |
---|
638 | |
---|
639 | nvme_qpair_process_completions(qpair); |
---|
640 | } |
---|
641 | |
---|
642 | int |
---|
643 | nvme_qpair_construct(struct nvme_qpair *qpair, |
---|
644 | uint32_t num_entries, uint32_t num_trackers, |
---|
645 | struct nvme_controller *ctrlr) |
---|
646 | { |
---|
647 | struct nvme_tracker *tr; |
---|
648 | size_t cmdsz, cplsz, prpsz, allocsz, prpmemsz; |
---|
649 | uint64_t queuemem_phys, prpmem_phys, list_phys; |
---|
650 | uint8_t *queuemem, *prpmem, *prp_list; |
---|
651 | int i, err; |
---|
652 | |
---|
653 | qpair->vector = ctrlr->msix_enabled ? qpair->id : 0; |
---|
654 | qpair->num_entries = num_entries; |
---|
655 | qpair->num_trackers = num_trackers; |
---|
656 | qpair->ctrlr = ctrlr; |
---|
657 | |
---|
658 | if (ctrlr->msix_enabled) { |
---|
659 | |
---|
660 | /* |
---|
661 | * MSI-X vector resource IDs start at 1, so we add one to |
---|
662 | * the queue's vector to get the corresponding rid to use. |
---|
663 | */ |
---|
664 | qpair->rid = qpair->vector + 1; |
---|
665 | |
---|
666 | qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, |
---|
667 | &qpair->rid, RF_ACTIVE); |
---|
668 | bus_setup_intr(ctrlr->dev, qpair->res, |
---|
669 | INTR_TYPE_MISC | INTR_MPSAFE, NULL, |
---|
670 | nvme_qpair_msix_handler, qpair, &qpair->tag); |
---|
671 | if (qpair->id == 0) { |
---|
672 | bus_describe_intr(ctrlr->dev, qpair->res, qpair->tag, |
---|
673 | "admin"); |
---|
674 | } else { |
---|
675 | bus_describe_intr(ctrlr->dev, qpair->res, qpair->tag, |
---|
676 | "io%d", qpair->id - 1); |
---|
677 | } |
---|
678 | } |
---|
679 | |
---|
680 | mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF); |
---|
681 | |
---|
682 | /* Note: NVMe PRP format is restricted to 4-byte alignment. */ |
---|
683 | err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), |
---|
684 | 4, PAGE_SIZE, BUS_SPACE_MAXADDR, |
---|
685 | BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE, |
---|
686 | (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0, |
---|
687 | NULL, NULL, &qpair->dma_tag_payload); |
---|
688 | if (err != 0) { |
---|
689 | nvme_printf(ctrlr, "payload tag create failed %d\n", err); |
---|
690 | goto out; |
---|
691 | } |
---|
692 | |
---|
693 | /* |
---|
694 | * Each component must be page aligned, and individual PRP lists |
---|
695 | * cannot cross a page boundary. |
---|
696 | */ |
---|
697 | cmdsz = qpair->num_entries * sizeof(struct nvme_command); |
---|
698 | cmdsz = roundup2(cmdsz, PAGE_SIZE); |
---|
699 | cplsz = qpair->num_entries * sizeof(struct nvme_completion); |
---|
700 | cplsz = roundup2(cplsz, PAGE_SIZE); |
---|
701 | prpsz = sizeof(uint64_t) * NVME_MAX_PRP_LIST_ENTRIES;; |
---|
702 | prpmemsz = qpair->num_trackers * prpsz; |
---|
703 | allocsz = cmdsz + cplsz + prpmemsz; |
---|
704 | |
---|
705 | err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev), |
---|
706 | PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, |
---|
707 | allocsz, 1, allocsz, 0, NULL, NULL, &qpair->dma_tag); |
---|
708 | if (err != 0) { |
---|
709 | nvme_printf(ctrlr, "tag create failed %d\n", err); |
---|
710 | goto out; |
---|
711 | } |
---|
712 | #ifndef __rtems__ |
---|
713 | bus_dma_tag_set_domain(qpair->dma_tag, qpair->domain); |
---|
714 | #endif /* __rtems__ */ |
---|
715 | |
---|
716 | if (bus_dmamem_alloc(qpair->dma_tag, (void **)&queuemem, |
---|
717 | BUS_DMA_NOWAIT, &qpair->queuemem_map)) { |
---|
718 | nvme_printf(ctrlr, "failed to alloc qpair memory\n"); |
---|
719 | goto out; |
---|
720 | } |
---|
721 | |
---|
722 | if (bus_dmamap_load(qpair->dma_tag, qpair->queuemem_map, |
---|
723 | queuemem, allocsz, nvme_single_map, &queuemem_phys, 0) != 0) { |
---|
724 | nvme_printf(ctrlr, "failed to load qpair memory\n"); |
---|
725 | goto out; |
---|
726 | } |
---|
727 | |
---|
728 | qpair->num_cmds = 0; |
---|
729 | qpair->num_intr_handler_calls = 0; |
---|
730 | qpair->num_retries = 0; |
---|
731 | qpair->num_failures = 0; |
---|
732 | qpair->cmd = (struct nvme_command *)queuemem; |
---|
733 | qpair->cpl = (struct nvme_completion *)(queuemem + cmdsz); |
---|
734 | prpmem = (uint8_t *)(queuemem + cmdsz + cplsz); |
---|
735 | qpair->cmd_bus_addr = queuemem_phys; |
---|
736 | qpair->cpl_bus_addr = queuemem_phys + cmdsz; |
---|
737 | prpmem_phys = queuemem_phys + cmdsz + cplsz; |
---|
738 | |
---|
739 | /* |
---|
740 | * Calcuate the stride of the doorbell register. Many emulators set this |
---|
741 | * value to correspond to a cache line. However, some hardware has set |
---|
742 | * it to various small values. |
---|
743 | */ |
---|
744 | qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) + |
---|
745 | (qpair->id << (ctrlr->dstrd + 1)); |
---|
746 | qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) + |
---|
747 | (qpair->id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd); |
---|
748 | |
---|
749 | TAILQ_INIT(&qpair->free_tr); |
---|
750 | TAILQ_INIT(&qpair->outstanding_tr); |
---|
751 | STAILQ_INIT(&qpair->queued_req); |
---|
752 | |
---|
753 | list_phys = prpmem_phys; |
---|
754 | prp_list = prpmem; |
---|
755 | for (i = 0; i < qpair->num_trackers; i++) { |
---|
756 | |
---|
757 | if (list_phys + prpsz > prpmem_phys + prpmemsz) { |
---|
758 | qpair->num_trackers = i; |
---|
759 | break; |
---|
760 | } |
---|
761 | |
---|
762 | /* |
---|
763 | * Make sure that the PRP list for this tracker doesn't |
---|
764 | * overflow to another page. |
---|
765 | */ |
---|
766 | if (trunc_page(list_phys) != |
---|
767 | trunc_page(list_phys + prpsz - 1)) { |
---|
768 | list_phys = roundup2(list_phys, PAGE_SIZE); |
---|
769 | prp_list = |
---|
770 | (uint8_t *)roundup2((uintptr_t)prp_list, PAGE_SIZE); |
---|
771 | } |
---|
772 | |
---|
773 | tr = malloc_domainset(sizeof(*tr), M_NVME, |
---|
774 | DOMAINSET_PREF(qpair->domain), M_ZERO | M_WAITOK); |
---|
775 | bus_dmamap_create(qpair->dma_tag_payload, 0, |
---|
776 | &tr->payload_dma_map); |
---|
777 | callout_init(&tr->timer, 1); |
---|
778 | tr->cid = i; |
---|
779 | tr->qpair = qpair; |
---|
780 | tr->prp = (uint64_t *)prp_list; |
---|
781 | tr->prp_bus_addr = list_phys; |
---|
782 | TAILQ_INSERT_HEAD(&qpair->free_tr, tr, tailq); |
---|
783 | list_phys += prpsz; |
---|
784 | prp_list += prpsz; |
---|
785 | } |
---|
786 | |
---|
787 | if (qpair->num_trackers == 0) { |
---|
788 | nvme_printf(ctrlr, "failed to allocate enough trackers\n"); |
---|
789 | goto out; |
---|
790 | } |
---|
791 | |
---|
792 | qpair->act_tr = malloc_domainset(sizeof(struct nvme_tracker *) * |
---|
793 | qpair->num_entries, M_NVME, DOMAINSET_PREF(qpair->domain), |
---|
794 | M_ZERO | M_WAITOK); |
---|
795 | return (0); |
---|
796 | |
---|
797 | out: |
---|
798 | nvme_qpair_destroy(qpair); |
---|
799 | return (ENOMEM); |
---|
800 | } |
---|
801 | |
---|
802 | static void |
---|
803 | nvme_qpair_destroy(struct nvme_qpair *qpair) |
---|
804 | { |
---|
805 | struct nvme_tracker *tr; |
---|
806 | |
---|
807 | if (qpair->tag) |
---|
808 | bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag); |
---|
809 | |
---|
810 | if (mtx_initialized(&qpair->lock)) |
---|
811 | mtx_destroy(&qpair->lock); |
---|
812 | |
---|
813 | if (qpair->res) |
---|
814 | bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ, |
---|
815 | rman_get_rid(qpair->res), qpair->res); |
---|
816 | |
---|
817 | if (qpair->cmd != NULL) { |
---|
818 | bus_dmamap_unload(qpair->dma_tag, qpair->queuemem_map); |
---|
819 | bus_dmamem_free(qpair->dma_tag, qpair->cmd, |
---|
820 | qpair->queuemem_map); |
---|
821 | } |
---|
822 | |
---|
823 | if (qpair->act_tr) |
---|
824 | free_domain(qpair->act_tr, M_NVME); |
---|
825 | |
---|
826 | while (!TAILQ_EMPTY(&qpair->free_tr)) { |
---|
827 | tr = TAILQ_FIRST(&qpair->free_tr); |
---|
828 | TAILQ_REMOVE(&qpair->free_tr, tr, tailq); |
---|
829 | bus_dmamap_destroy(qpair->dma_tag_payload, |
---|
830 | tr->payload_dma_map); |
---|
831 | free_domain(tr, M_NVME); |
---|
832 | } |
---|
833 | |
---|
834 | if (qpair->dma_tag) |
---|
835 | bus_dma_tag_destroy(qpair->dma_tag); |
---|
836 | |
---|
837 | if (qpair->dma_tag_payload) |
---|
838 | bus_dma_tag_destroy(qpair->dma_tag_payload); |
---|
839 | } |
---|
840 | |
---|
841 | static void |
---|
842 | nvme_admin_qpair_abort_aers(struct nvme_qpair *qpair) |
---|
843 | { |
---|
844 | struct nvme_tracker *tr; |
---|
845 | |
---|
846 | tr = TAILQ_FIRST(&qpair->outstanding_tr); |
---|
847 | while (tr != NULL) { |
---|
848 | if (tr->req->cmd.opc == NVME_OPC_ASYNC_EVENT_REQUEST) { |
---|
849 | nvme_qpair_manual_complete_tracker(qpair, tr, |
---|
850 | NVME_SCT_GENERIC, NVME_SC_ABORTED_SQ_DELETION, 0, |
---|
851 | ERROR_PRINT_NONE); |
---|
852 | tr = TAILQ_FIRST(&qpair->outstanding_tr); |
---|
853 | } else { |
---|
854 | tr = TAILQ_NEXT(tr, tailq); |
---|
855 | } |
---|
856 | } |
---|
857 | } |
---|
858 | |
---|
859 | void |
---|
860 | nvme_admin_qpair_destroy(struct nvme_qpair *qpair) |
---|
861 | { |
---|
862 | |
---|
863 | nvme_admin_qpair_abort_aers(qpair); |
---|
864 | nvme_qpair_destroy(qpair); |
---|
865 | } |
---|
866 | |
---|
867 | void |
---|
868 | nvme_io_qpair_destroy(struct nvme_qpair *qpair) |
---|
869 | { |
---|
870 | |
---|
871 | nvme_qpair_destroy(qpair); |
---|
872 | } |
---|
873 | |
---|
874 | static void |
---|
875 | nvme_abort_complete(void *arg, const struct nvme_completion *status) |
---|
876 | { |
---|
877 | struct nvme_tracker *tr = arg; |
---|
878 | |
---|
879 | /* |
---|
880 | * If cdw0 == 1, the controller was not able to abort the command |
---|
881 | * we requested. We still need to check the active tracker array, |
---|
882 | * to cover race where I/O timed out at same time controller was |
---|
883 | * completing the I/O. |
---|
884 | */ |
---|
885 | if (status->cdw0 == 1 && tr->qpair->act_tr[tr->cid] != NULL) { |
---|
886 | /* |
---|
887 | * An I/O has timed out, and the controller was unable to |
---|
888 | * abort it for some reason. Construct a fake completion |
---|
889 | * status, and then complete the I/O's tracker manually. |
---|
890 | */ |
---|
891 | nvme_printf(tr->qpair->ctrlr, |
---|
892 | "abort command failed, aborting command manually\n"); |
---|
893 | nvme_qpair_manual_complete_tracker(tr->qpair, tr, |
---|
894 | NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST, 0, ERROR_PRINT_ALL); |
---|
895 | } |
---|
896 | } |
---|
897 | |
---|
898 | static void |
---|
899 | nvme_timeout(void *arg) |
---|
900 | { |
---|
901 | struct nvme_tracker *tr = arg; |
---|
902 | struct nvme_qpair *qpair = tr->qpair; |
---|
903 | struct nvme_controller *ctrlr = qpair->ctrlr; |
---|
904 | uint32_t csts; |
---|
905 | uint8_t cfs; |
---|
906 | |
---|
907 | /* |
---|
908 | * Read csts to get value of cfs - controller fatal status. |
---|
909 | * If no fatal status, try to call the completion routine, and |
---|
910 | * if completes transactions, report a missed interrupt and |
---|
911 | * return (this may need to be rate limited). Otherwise, if |
---|
912 | * aborts are enabled and the controller is not reporting |
---|
913 | * fatal status, abort the command. Otherwise, just reset the |
---|
914 | * controller and hope for the best. |
---|
915 | */ |
---|
916 | csts = nvme_mmio_read_4(ctrlr, csts); |
---|
917 | cfs = (csts >> NVME_CSTS_REG_CFS_SHIFT) & NVME_CSTS_REG_CFS_MASK; |
---|
918 | if (cfs == 0 && nvme_qpair_process_completions(qpair)) { |
---|
919 | nvme_printf(ctrlr, "Missing interrupt\n"); |
---|
920 | return; |
---|
921 | } |
---|
922 | if (ctrlr->enable_aborts && cfs == 0) { |
---|
923 | nvme_printf(ctrlr, "Aborting command due to a timeout.\n"); |
---|
924 | nvme_ctrlr_cmd_abort(ctrlr, tr->cid, qpair->id, |
---|
925 | nvme_abort_complete, tr); |
---|
926 | } else { |
---|
927 | nvme_printf(ctrlr, "Resetting controller due to a timeout%s.\n", |
---|
928 | (csts == 0xffffffff) ? " and possible hot unplug" : |
---|
929 | (cfs ? " and fatal error status" : "")); |
---|
930 | nvme_ctrlr_reset(ctrlr); |
---|
931 | } |
---|
932 | } |
---|
933 | |
---|
934 | void |
---|
935 | nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) |
---|
936 | { |
---|
937 | struct nvme_request *req; |
---|
938 | struct nvme_controller *ctrlr; |
---|
939 | |
---|
940 | mtx_assert(&qpair->lock, MA_OWNED); |
---|
941 | |
---|
942 | req = tr->req; |
---|
943 | req->cmd.cid = tr->cid; |
---|
944 | qpair->act_tr[tr->cid] = tr; |
---|
945 | ctrlr = qpair->ctrlr; |
---|
946 | |
---|
947 | if (req->timeout) |
---|
948 | #ifndef __rtems__ |
---|
949 | callout_reset_on(&tr->timer, ctrlr->timeout_period * hz, |
---|
950 | nvme_timeout, tr, qpair->cpu); |
---|
951 | #else /* __rtems__ */ |
---|
952 | callout_reset_on(&tr->timer, ctrlr->timeout_period * hz, |
---|
953 | nvme_timeout, tr, -1); |
---|
954 | #endif /* __rtems__ */ |
---|
955 | |
---|
956 | /* Copy the command from the tracker to the submission queue. */ |
---|
957 | memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); |
---|
958 | |
---|
959 | if (++qpair->sq_tail == qpair->num_entries) |
---|
960 | qpair->sq_tail = 0; |
---|
961 | |
---|
962 | bus_dmamap_sync(qpair->dma_tag, qpair->queuemem_map, |
---|
963 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
---|
964 | #ifndef __powerpc__ |
---|
965 | /* |
---|
966 | * powerpc's bus_dmamap_sync() already includes a heavyweight sync, but |
---|
967 | * no other archs do. |
---|
968 | */ |
---|
969 | wmb(); |
---|
970 | #endif |
---|
971 | |
---|
972 | bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, |
---|
973 | qpair->sq_tdbl_off, qpair->sq_tail); |
---|
974 | qpair->num_cmds++; |
---|
975 | } |
---|
976 | |
---|
977 | static void |
---|
978 | nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) |
---|
979 | { |
---|
980 | struct nvme_tracker *tr = arg; |
---|
981 | uint32_t cur_nseg; |
---|
982 | |
---|
983 | /* |
---|
984 | * If the mapping operation failed, return immediately. The caller |
---|
985 | * is responsible for detecting the error status and failing the |
---|
986 | * tracker manually. |
---|
987 | */ |
---|
988 | if (error != 0) { |
---|
989 | nvme_printf(tr->qpair->ctrlr, |
---|
990 | "nvme_payload_map err %d\n", error); |
---|
991 | return; |
---|
992 | } |
---|
993 | |
---|
994 | /* |
---|
995 | * Note that we specified PAGE_SIZE for alignment and max |
---|
996 | * segment size when creating the bus dma tags. So here |
---|
997 | * we can safely just transfer each segment to its |
---|
998 | * associated PRP entry. |
---|
999 | */ |
---|
1000 | tr->req->cmd.prp1 = htole64(seg[0].ds_addr); |
---|
1001 | |
---|
1002 | if (nseg == 2) { |
---|
1003 | tr->req->cmd.prp2 = htole64(seg[1].ds_addr); |
---|
1004 | } else if (nseg > 2) { |
---|
1005 | cur_nseg = 1; |
---|
1006 | tr->req->cmd.prp2 = htole64((uint64_t)tr->prp_bus_addr); |
---|
1007 | while (cur_nseg < nseg) { |
---|
1008 | tr->prp[cur_nseg-1] = |
---|
1009 | htole64((uint64_t)seg[cur_nseg].ds_addr); |
---|
1010 | cur_nseg++; |
---|
1011 | } |
---|
1012 | } else { |
---|
1013 | /* |
---|
1014 | * prp2 should not be used by the controller |
---|
1015 | * since there is only one segment, but set |
---|
1016 | * to 0 just to be safe. |
---|
1017 | */ |
---|
1018 | tr->req->cmd.prp2 = 0; |
---|
1019 | } |
---|
1020 | |
---|
1021 | bus_dmamap_sync(tr->qpair->dma_tag_payload, tr->payload_dma_map, |
---|
1022 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
---|
1023 | nvme_qpair_submit_tracker(tr->qpair, tr); |
---|
1024 | } |
---|
1025 | |
---|
1026 | static void |
---|
1027 | _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) |
---|
1028 | { |
---|
1029 | struct nvme_tracker *tr; |
---|
1030 | int err = 0; |
---|
1031 | |
---|
1032 | mtx_assert(&qpair->lock, MA_OWNED); |
---|
1033 | |
---|
1034 | tr = TAILQ_FIRST(&qpair->free_tr); |
---|
1035 | req->qpair = qpair; |
---|
1036 | |
---|
1037 | if (tr == NULL || !qpair->is_enabled) { |
---|
1038 | /* |
---|
1039 | * No tracker is available, or the qpair is disabled due to |
---|
1040 | * an in-progress controller-level reset or controller |
---|
1041 | * failure. |
---|
1042 | */ |
---|
1043 | |
---|
1044 | if (qpair->ctrlr->is_failed) { |
---|
1045 | /* |
---|
1046 | * The controller has failed. Post the request to a |
---|
1047 | * task where it will be aborted, so that we do not |
---|
1048 | * invoke the request's callback in the context |
---|
1049 | * of the submission. |
---|
1050 | */ |
---|
1051 | nvme_ctrlr_post_failed_request(qpair->ctrlr, req); |
---|
1052 | } else { |
---|
1053 | /* |
---|
1054 | * Put the request on the qpair's request queue to be |
---|
1055 | * processed when a tracker frees up via a command |
---|
1056 | * completion or when the controller reset is |
---|
1057 | * completed. |
---|
1058 | */ |
---|
1059 | STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); |
---|
1060 | } |
---|
1061 | return; |
---|
1062 | } |
---|
1063 | |
---|
1064 | TAILQ_REMOVE(&qpair->free_tr, tr, tailq); |
---|
1065 | TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq); |
---|
1066 | tr->req = req; |
---|
1067 | |
---|
1068 | switch (req->type) { |
---|
1069 | case NVME_REQUEST_VADDR: |
---|
1070 | KASSERT(req->payload_size <= qpair->ctrlr->max_xfer_size, |
---|
1071 | ("payload_size (%d) exceeds max_xfer_size (%d)\n", |
---|
1072 | req->payload_size, qpair->ctrlr->max_xfer_size)); |
---|
1073 | err = bus_dmamap_load(tr->qpair->dma_tag_payload, |
---|
1074 | tr->payload_dma_map, req->u.payload, req->payload_size, |
---|
1075 | nvme_payload_map, tr, 0); |
---|
1076 | if (err != 0) |
---|
1077 | nvme_printf(qpair->ctrlr, |
---|
1078 | "bus_dmamap_load returned 0x%x!\n", err); |
---|
1079 | break; |
---|
1080 | case NVME_REQUEST_NULL: |
---|
1081 | nvme_qpair_submit_tracker(tr->qpair, tr); |
---|
1082 | break; |
---|
1083 | #ifndef __rtems__ |
---|
1084 | case NVME_REQUEST_BIO: |
---|
1085 | KASSERT(req->u.bio->bio_bcount <= qpair->ctrlr->max_xfer_size, |
---|
1086 | ("bio->bio_bcount (%jd) exceeds max_xfer_size (%d)\n", |
---|
1087 | (intmax_t)req->u.bio->bio_bcount, |
---|
1088 | qpair->ctrlr->max_xfer_size)); |
---|
1089 | err = bus_dmamap_load_bio(tr->qpair->dma_tag_payload, |
---|
1090 | tr->payload_dma_map, req->u.bio, nvme_payload_map, tr, 0); |
---|
1091 | if (err != 0) |
---|
1092 | nvme_printf(qpair->ctrlr, |
---|
1093 | "bus_dmamap_load_bio returned 0x%x!\n", err); |
---|
1094 | break; |
---|
1095 | case NVME_REQUEST_CCB: |
---|
1096 | err = bus_dmamap_load_ccb(tr->qpair->dma_tag_payload, |
---|
1097 | tr->payload_dma_map, req->u.payload, |
---|
1098 | nvme_payload_map, tr, 0); |
---|
1099 | if (err != 0) |
---|
1100 | nvme_printf(qpair->ctrlr, |
---|
1101 | "bus_dmamap_load_ccb returned 0x%x!\n", err); |
---|
1102 | break; |
---|
1103 | #endif /* __rtems__ */ |
---|
1104 | default: |
---|
1105 | panic("unknown nvme request type 0x%x\n", req->type); |
---|
1106 | break; |
---|
1107 | } |
---|
1108 | |
---|
1109 | if (err != 0) { |
---|
1110 | /* |
---|
1111 | * The dmamap operation failed, so we manually fail the |
---|
1112 | * tracker here with DATA_TRANSFER_ERROR status. |
---|
1113 | * |
---|
1114 | * nvme_qpair_manual_complete_tracker must not be called |
---|
1115 | * with the qpair lock held. |
---|
1116 | */ |
---|
1117 | mtx_unlock(&qpair->lock); |
---|
1118 | nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, |
---|
1119 | NVME_SC_DATA_TRANSFER_ERROR, DO_NOT_RETRY, ERROR_PRINT_ALL); |
---|
1120 | mtx_lock(&qpair->lock); |
---|
1121 | } |
---|
1122 | } |
---|
1123 | |
---|
1124 | void |
---|
1125 | nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) |
---|
1126 | { |
---|
1127 | |
---|
1128 | mtx_lock(&qpair->lock); |
---|
1129 | _nvme_qpair_submit_request(qpair, req); |
---|
1130 | mtx_unlock(&qpair->lock); |
---|
1131 | } |
---|
1132 | |
---|
1133 | static void |
---|
1134 | nvme_qpair_enable(struct nvme_qpair *qpair) |
---|
1135 | { |
---|
1136 | |
---|
1137 | qpair->is_enabled = TRUE; |
---|
1138 | } |
---|
1139 | |
---|
1140 | void |
---|
1141 | nvme_qpair_reset(struct nvme_qpair *qpair) |
---|
1142 | { |
---|
1143 | |
---|
1144 | qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; |
---|
1145 | |
---|
1146 | /* |
---|
1147 | * First time through the completion queue, HW will set phase |
---|
1148 | * bit on completions to 1. So set this to 1 here, indicating |
---|
1149 | * we're looking for a 1 to know which entries have completed. |
---|
1150 | * we'll toggle the bit each time when the completion queue |
---|
1151 | * rolls over. |
---|
1152 | */ |
---|
1153 | qpair->phase = 1; |
---|
1154 | |
---|
1155 | memset(qpair->cmd, 0, |
---|
1156 | qpair->num_entries * sizeof(struct nvme_command)); |
---|
1157 | memset(qpair->cpl, 0, |
---|
1158 | qpair->num_entries * sizeof(struct nvme_completion)); |
---|
1159 | } |
---|
1160 | |
---|
1161 | void |
---|
1162 | nvme_admin_qpair_enable(struct nvme_qpair *qpair) |
---|
1163 | { |
---|
1164 | struct nvme_tracker *tr; |
---|
1165 | struct nvme_tracker *tr_temp; |
---|
1166 | |
---|
1167 | /* |
---|
1168 | * Manually abort each outstanding admin command. Do not retry |
---|
1169 | * admin commands found here, since they will be left over from |
---|
1170 | * a controller reset and its likely the context in which the |
---|
1171 | * command was issued no longer applies. |
---|
1172 | */ |
---|
1173 | TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { |
---|
1174 | nvme_printf(qpair->ctrlr, |
---|
1175 | "aborting outstanding admin command\n"); |
---|
1176 | nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, |
---|
1177 | NVME_SC_ABORTED_BY_REQUEST, DO_NOT_RETRY, ERROR_PRINT_ALL); |
---|
1178 | } |
---|
1179 | |
---|
1180 | nvme_qpair_enable(qpair); |
---|
1181 | } |
---|
1182 | |
---|
1183 | void |
---|
1184 | nvme_io_qpair_enable(struct nvme_qpair *qpair) |
---|
1185 | { |
---|
1186 | STAILQ_HEAD(, nvme_request) temp; |
---|
1187 | struct nvme_tracker *tr; |
---|
1188 | struct nvme_tracker *tr_temp; |
---|
1189 | struct nvme_request *req; |
---|
1190 | |
---|
1191 | /* |
---|
1192 | * Manually abort each outstanding I/O. This normally results in a |
---|
1193 | * retry, unless the retry count on the associated request has |
---|
1194 | * reached its limit. |
---|
1195 | */ |
---|
1196 | TAILQ_FOREACH_SAFE(tr, &qpair->outstanding_tr, tailq, tr_temp) { |
---|
1197 | nvme_printf(qpair->ctrlr, "aborting outstanding i/o\n"); |
---|
1198 | nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, |
---|
1199 | NVME_SC_ABORTED_BY_REQUEST, 0, ERROR_PRINT_NO_RETRY); |
---|
1200 | } |
---|
1201 | |
---|
1202 | mtx_lock(&qpair->lock); |
---|
1203 | |
---|
1204 | nvme_qpair_enable(qpair); |
---|
1205 | |
---|
1206 | STAILQ_INIT(&temp); |
---|
1207 | STAILQ_SWAP(&qpair->queued_req, &temp, nvme_request); |
---|
1208 | |
---|
1209 | while (!STAILQ_EMPTY(&temp)) { |
---|
1210 | req = STAILQ_FIRST(&temp); |
---|
1211 | STAILQ_REMOVE_HEAD(&temp, stailq); |
---|
1212 | nvme_printf(qpair->ctrlr, "resubmitting queued i/o\n"); |
---|
1213 | nvme_qpair_print_command(qpair, &req->cmd); |
---|
1214 | _nvme_qpair_submit_request(qpair, req); |
---|
1215 | } |
---|
1216 | |
---|
1217 | mtx_unlock(&qpair->lock); |
---|
1218 | } |
---|
1219 | |
---|
1220 | static void |
---|
1221 | nvme_qpair_disable(struct nvme_qpair *qpair) |
---|
1222 | { |
---|
1223 | struct nvme_tracker *tr; |
---|
1224 | |
---|
1225 | qpair->is_enabled = FALSE; |
---|
1226 | mtx_lock(&qpair->lock); |
---|
1227 | TAILQ_FOREACH(tr, &qpair->outstanding_tr, tailq) |
---|
1228 | callout_stop(&tr->timer); |
---|
1229 | mtx_unlock(&qpair->lock); |
---|
1230 | } |
---|
1231 | |
---|
1232 | void |
---|
1233 | nvme_admin_qpair_disable(struct nvme_qpair *qpair) |
---|
1234 | { |
---|
1235 | |
---|
1236 | nvme_qpair_disable(qpair); |
---|
1237 | nvme_admin_qpair_abort_aers(qpair); |
---|
1238 | } |
---|
1239 | |
---|
1240 | void |
---|
1241 | nvme_io_qpair_disable(struct nvme_qpair *qpair) |
---|
1242 | { |
---|
1243 | |
---|
1244 | nvme_qpair_disable(qpair); |
---|
1245 | } |
---|
1246 | |
---|
1247 | void |
---|
1248 | nvme_qpair_fail(struct nvme_qpair *qpair) |
---|
1249 | { |
---|
1250 | struct nvme_tracker *tr; |
---|
1251 | struct nvme_request *req; |
---|
1252 | |
---|
1253 | if (!mtx_initialized(&qpair->lock)) |
---|
1254 | return; |
---|
1255 | |
---|
1256 | mtx_lock(&qpair->lock); |
---|
1257 | |
---|
1258 | while (!STAILQ_EMPTY(&qpair->queued_req)) { |
---|
1259 | req = STAILQ_FIRST(&qpair->queued_req); |
---|
1260 | STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); |
---|
1261 | nvme_printf(qpair->ctrlr, "failing queued i/o\n"); |
---|
1262 | mtx_unlock(&qpair->lock); |
---|
1263 | nvme_qpair_manual_complete_request(qpair, req, NVME_SCT_GENERIC, |
---|
1264 | NVME_SC_ABORTED_BY_REQUEST); |
---|
1265 | mtx_lock(&qpair->lock); |
---|
1266 | } |
---|
1267 | |
---|
1268 | /* Manually abort each outstanding I/O. */ |
---|
1269 | while (!TAILQ_EMPTY(&qpair->outstanding_tr)) { |
---|
1270 | tr = TAILQ_FIRST(&qpair->outstanding_tr); |
---|
1271 | /* |
---|
1272 | * Do not remove the tracker. The abort_tracker path will |
---|
1273 | * do that for us. |
---|
1274 | */ |
---|
1275 | nvme_printf(qpair->ctrlr, "failing outstanding i/o\n"); |
---|
1276 | mtx_unlock(&qpair->lock); |
---|
1277 | nvme_qpair_manual_complete_tracker(qpair, tr, NVME_SCT_GENERIC, |
---|
1278 | NVME_SC_ABORTED_BY_REQUEST, DO_NOT_RETRY, ERROR_PRINT_ALL); |
---|
1279 | mtx_lock(&qpair->lock); |
---|
1280 | } |
---|
1281 | |
---|
1282 | mtx_unlock(&qpair->lock); |
---|
1283 | } |
---|
1284 | |
---|