@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 10,696 lines 308 KB
Raw
1 /*
2 * QEMU NVM Express Controller
3 *
4 * Copyright (c) 2012, Intel Corporation
5 *
6 * Written by Keith Busch <keith.busch@intel.com>
7 *
8 * This code is licensed under the GNU GPL v2 or later.
9 */
10
11 /**
12 * Reference Specs: http://www.nvmexpress.org, 1.4, 1.3, 1.2, 1.1, 1.0e
13 *
14 * https://nvmexpress.org/developers/nvme-specification/
15 *
16 *
17 * Notes on coding style
18 * ---------------------
19 * While QEMU coding style prefers lowercase hexadecimals in constants, the
20 * NVMe subsystem use this format from the NVMe specifications in the comments
21 * (i.e. 'h' suffix instead of '0x' prefix).
22 *
23 * Usage
24 * -----
25 * See docs/system/devices/nvme.rst for extensive documentation.
26 *
27 * Add options:
28 * -drive file=<file>,if=none,id=<drive_id>
29 * -device nvme-subsys,id=<subsys_id>,nqn=<nqn_id>
30 * -device nvme,serial=<serial>,id=<bus_name>, \
31 * cmb_size_mb=<cmb_size_mb[optional]>, \
32 * [pmrdev=<mem_backend_file_id>,] \
33 * max_ioqpairs=<N[optional]>, \
34 * aerl=<N[optional]>,aer_max_queued=<N[optional]>, \
35 * mdts=<N[optional]>,vsl=<N[optional]>, \
36 * zoned.zasl=<N[optional]>, \
37 * zoned.auto_transition=<on|off[optional]>, \
38 * sriov_max_vfs=<N[optional]> \
39 * sriov_vq_flexible=<N[optional]> \
40 * sriov_vi_flexible=<N[optional]> \
41 * sriov_max_vi_per_vf=<N[optional]> \
42 * sriov_max_vq_per_vf=<N[optional]> \
43 * atomic.dn=<on|off[optional]>, \
44 * atomic.awun<N[optional]>, \
45 * atomic.awupf<N[optional]>, \
46 * subsys=<subsys_id>, \
47 * model=<model-str>, \
48 * firmware-version=<version-str>
49 * -device nvme-ns,drive=<drive_id>,bus=<bus_name>,nsid=<nsid>,\
50 * zoned=<true|false[optional]>, \
51 * subsys=<subsys_id>,shared=<true|false[optional]>, \
52 * detached=<true|false[optional]>, \
53 * zoned.zone_size=<N[optional]>, \
54 * zoned.zone_capacity=<N[optional]>, \
55 * zoned.descr_ext_size=<N[optional]>, \
56 * zoned.max_active=<N[optional]>, \
57 * zoned.max_open=<N[optional]>, \
58 * zoned.cross_read=<true|false[optional]>
59 *
60 * Note cmb_size_mb denotes size of CMB in MB. CMB is assumed to be at
61 * offset 0 in BAR2 and supports only WDS, RDS and SQS for now. By default, the
62 * device will use the "v1.4 CMB scheme" - use the `legacy-cmb` parameter to
63 * always enable the CMBLOC and CMBSZ registers (v1.3 behavior).
64 *
65 * Enabling pmr emulation can be achieved by pointing to memory-backend-file.
66 * For example:
67 * -object memory-backend-file,id=<mem_id>,share=on,mem-path=<file_path>, \
68 * size=<size> .... -device nvme,...,pmrdev=<mem_id>
69 *
70 * The PMR will use BAR 4/5 exclusively.
71 *
72 * To place controller(s) and namespace(s) to a subsystem, then provide
73 * nvme-subsys device as above.
74 *
75 * nvme subsystem device parameters
76 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
77 * - `nqn`
78 * This parameter provides the `<nqn_id>` part of the string
79 * `nqn.2019-08.org.qemu:<nqn_id>` which will be reported in the SUBNQN field
80 * of subsystem controllers. Note that `<nqn_id>` should be unique per
81 * subsystem, but this is not enforced by QEMU. If not specified, it will
82 * default to the value of the `id` parameter (`<subsys_id>`).
83 *
84 * nvme device parameters
85 * ~~~~~~~~~~~~~~~~~~~~~~
86 * - `subsys`
87 * Specifying this parameter attaches the controller to the subsystem and
88 * the SUBNQN field in the controller will report the NQN of the subsystem
89 * device. This also enables multi controller capability represented in
90 * Identify Controller data structure in CMIC (Controller Multi-path I/O and
91 * Namespace Sharing Capabilities).
92 *
93 * - `aerl`
94 * The Asynchronous Event Request Limit (AERL). Indicates the maximum number
95 * of concurrently outstanding Asynchronous Event Request commands support
96 * by the controller. This is a 0's based value.
97 *
98 * - `aer_max_queued`
99 * This is the maximum number of events that the device will enqueue for
100 * completion when there are no outstanding AERs. When the maximum number of
101 * enqueued events are reached, subsequent events will be dropped.
102 *
103 * - `mdts`
104 * Indicates the maximum data transfer size for a command that transfers data
105 * between host-accessible memory and the controller. The value is specified
106 * as a power of two (2^n) and is in units of the minimum memory page size
107 * (CAP.MPSMIN). The default value is 7 (i.e. 512 KiB).
108 *
109 * - `vsl`
110 * Indicates the maximum data size limit for the Verify command. Like `mdts`,
111 * this value is specified as a power of two (2^n) and is in units of the
112 * minimum memory page size (CAP.MPSMIN). The default value is 7 (i.e. 512
113 * KiB).
114 *
115 * - `zoned.zasl`
116 * Indicates the maximum data transfer size for the Zone Append command. Like
117 * `mdts`, the value is specified as a power of two (2^n) and is in units of
118 * the minimum memory page size (CAP.MPSMIN). The default value is 0 (i.e.
119 * defaulting to the value of `mdts`).
120 *
121 * - `zoned.auto_transition`
122 * Indicates if zones in zone state implicitly opened can be automatically
123 * transitioned to zone state closed for resource management purposes.
124 * Defaults to 'on'.
125 *
126 * - `sriov_max_vfs`
127 * Indicates the maximum number of PCIe virtual functions supported
128 * by the controller. The default value is 0. Specifying a non-zero value
129 * enables reporting of both SR-IOV and ARI capabilities by the NVMe device.
130 * Virtual function controllers will not report SR-IOV capability.
131 *
132 * NOTE: Single Root I/O Virtualization support is experimental.
133 * All the related parameters may be subject to change.
134 *
135 * - `sriov_vq_flexible`
136 * Indicates the total number of flexible queue resources assignable to all
137 * the secondary controllers. Implicitly sets the number of primary
138 * controller's private resources to `(max_ioqpairs - sriov_vq_flexible)`.
139 *
140 * - `sriov_vi_flexible`
141 * Indicates the total number of flexible interrupt resources assignable to
142 * all the secondary controllers. Implicitly sets the number of primary
143 * controller's private resources to `(msix_qsize - sriov_vi_flexible)`.
144 *
145 * - `sriov_max_vi_per_vf`
146 * Indicates the maximum number of virtual interrupt resources assignable
147 * to a secondary controller. The default 0 resolves to
148 * `(sriov_vi_flexible / sriov_max_vfs)`.
149 *
150 * - `sriov_max_vq_per_vf`
151 * Indicates the maximum number of virtual queue resources assignable to
152 * a secondary controller. The default 0 resolves to
153 * `(sriov_vq_flexible / sriov_max_vfs)`.
154 *
155 * nvme namespace device parameters
156 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
157 * - `shared`
158 * When the parent nvme device (as defined explicitly by the 'bus' parameter
159 * or implicitly by the most recently defined NvmeBus) is linked to an
160 * nvme-subsys device, the namespace will be attached to all controllers in
161 * the subsystem. If set to 'off' (the default), the namespace will remain a
162 * private namespace and may only be attached to a single controller at a
163 * time.
164 *
165 * - `detached`
166 * This parameter is only valid together with the `subsys` parameter. If left
167 * at the default value (`false/off`), the namespace will be attached to all
168 * controllers in the NVMe subsystem at boot-up. If set to `true/on`, the
169 * namespace will be available in the subsystem but not attached to any
170 * controllers.
171 *
172 * Setting `zoned` to true selects Zoned Command Set at the namespace.
173 * In this case, the following namespace properties are available to configure
174 * zoned operation:
175 * zoned.zone_size=<zone size in bytes, default: 128MiB>
176 * The number may be followed by K, M, G as in kilo-, mega- or giga-.
177 *
178 * zoned.zone_capacity=<zone capacity in bytes, default: zone size>
179 * The value 0 (default) forces zone capacity to be the same as zone
180 * size. The value of this property may not exceed zone size.
181 *
182 * zoned.descr_ext_size=<zone descriptor extension size, default 0>
183 * This value needs to be specified in 64B units. If it is zero,
184 * namespace(s) will not support zone descriptor extensions.
185 *
186 * zoned.max_active=<Maximum Active Resources (zones), default: 0>
187 * The default value means there is no limit to the number of
188 * concurrently active zones.
189 *
190 * zoned.max_open=<Maximum Open Resources (zones), default: 0>
191 * The default value means there is no limit to the number of
192 * concurrently open zones.
193 *
194 * zoned.cross_read=<enable RAZB, default: false>
195 * Setting this property to true enables Read Across Zone Boundaries.
196 */
197
198 #include "qemu/osdep.h"
199 #include "qemu/bitops.h"
200 #include "qemu/cutils.h"
201 #include "qemu/error-report.h"
202 #include "qemu/log.h"
203 #include "qemu/units.h"
204 #include "qemu/range.h"
205 #include "qapi/error.h"
206 #include "qapi/visitor.h"
207 #include "system/system.h"
208 #include "system/block-backend.h"
209 #include "system/hostmem.h"
210 #include "hw/pci/msix.h"
211 #include "hw/pci/pcie_sriov.h"
212 #include "system/spdm-socket.h"
213 #include "migration/blocker.h"
214 #include "migration/qemu-file-types.h"
215 #include "migration/vmstate.h"
216
217 #include "nvme.h"
218 #include "dif.h"
219 #include "trace.h"
220
221 #define NVME_MAX_IOQPAIRS 0xffff
222 #define NVME_DB_SIZE 4
223 #define NVME_SPEC_VER 0x00010400
224 #define NVME_CMB_BIR 2
225 #define NVME_PMR_BIR 4
226 #define NVME_TEMPERATURE 0x143
227 #define NVME_TEMPERATURE_WARNING 0x157
228 #define NVME_TEMPERATURE_CRITICAL 0x175
229 #define NVME_NUM_FW_SLOTS 1
230 #define NVME_DEFAULT_MAX_ZA_SIZE (128 * KiB)
231 #define NVME_VF_RES_GRANULARITY 1
232 #define NVME_VF_OFFSET 0x1
233 #define NVME_VF_STRIDE 1
234
235 #define NVME_GUEST_ERR(trace, fmt, ...) \
236 do { \
237 (trace_##trace)(__VA_ARGS__); \
238 qemu_log_mask(LOG_GUEST_ERROR, #trace \
239 " in %s: " fmt "\n", __func__, ## __VA_ARGS__); \
240 } while (0)
241
242 static const bool nvme_feature_support[NVME_FID_MAX] = {
243 [NVME_ARBITRATION] = true,
244 [NVME_POWER_MANAGEMENT] = true,
245 [NVME_TEMPERATURE_THRESHOLD] = true,
246 [NVME_ERROR_RECOVERY] = true,
247 [NVME_VOLATILE_WRITE_CACHE] = true,
248 [NVME_NUMBER_OF_QUEUES] = true,
249 [NVME_INTERRUPT_COALESCING] = true,
250 [NVME_INTERRUPT_VECTOR_CONF] = true,
251 [NVME_WRITE_ATOMICITY] = true,
252 [NVME_ASYNCHRONOUS_EVENT_CONF] = true,
253 [NVME_TIMESTAMP] = true,
254 [NVME_HOST_BEHAVIOR_SUPPORT] = true,
255 [NVME_COMMAND_SET_PROFILE] = true,
256 [NVME_FDP_MODE] = true,
257 [NVME_FDP_EVENTS] = true,
258 /* if you add something here, please update nvme_set_migration_blockers() */
259 };
260
261 static const uint32_t nvme_feature_cap[NVME_FID_MAX] = {
262 [NVME_TEMPERATURE_THRESHOLD] = NVME_FEAT_CAP_CHANGE,
263 [NVME_ERROR_RECOVERY] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
264 [NVME_VOLATILE_WRITE_CACHE] = NVME_FEAT_CAP_CHANGE,
265 [NVME_NUMBER_OF_QUEUES] = NVME_FEAT_CAP_CHANGE,
266 [NVME_WRITE_ATOMICITY] = NVME_FEAT_CAP_CHANGE,
267 [NVME_ASYNCHRONOUS_EVENT_CONF] = NVME_FEAT_CAP_CHANGE,
268 [NVME_TIMESTAMP] = NVME_FEAT_CAP_CHANGE,
269 [NVME_HOST_BEHAVIOR_SUPPORT] = NVME_FEAT_CAP_CHANGE,
270 [NVME_COMMAND_SET_PROFILE] = NVME_FEAT_CAP_CHANGE,
271 [NVME_FDP_MODE] = NVME_FEAT_CAP_CHANGE,
272 [NVME_FDP_EVENTS] = NVME_FEAT_CAP_CHANGE | NVME_FEAT_CAP_NS,
273 };
274
275 static const uint32_t nvme_cse_acs_default[256] = {
276 [NVME_ADM_CMD_DELETE_SQ] = NVME_CMD_EFF_CSUPP,
277 [NVME_ADM_CMD_CREATE_SQ] = NVME_CMD_EFF_CSUPP,
278 [NVME_ADM_CMD_GET_LOG_PAGE] = NVME_CMD_EFF_CSUPP,
279 [NVME_ADM_CMD_DELETE_CQ] = NVME_CMD_EFF_CSUPP,
280 [NVME_ADM_CMD_CREATE_CQ] = NVME_CMD_EFF_CSUPP,
281 [NVME_ADM_CMD_IDENTIFY] = NVME_CMD_EFF_CSUPP,
282 [NVME_ADM_CMD_ABORT] = NVME_CMD_EFF_CSUPP,
283 [NVME_ADM_CMD_SET_FEATURES] = NVME_CMD_EFF_CSUPP,
284 [NVME_ADM_CMD_GET_FEATURES] = NVME_CMD_EFF_CSUPP,
285 [NVME_ADM_CMD_ASYNC_EV_REQ] = NVME_CMD_EFF_CSUPP,
286 [NVME_ADM_CMD_NS_ATTACHMENT] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_NIC |
287 NVME_CMD_EFF_CCC,
288 [NVME_ADM_CMD_FORMAT_NVM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
289 [NVME_ADM_CMD_DIRECTIVE_RECV] = NVME_CMD_EFF_CSUPP,
290 [NVME_ADM_CMD_DIRECTIVE_SEND] = NVME_CMD_EFF_CSUPP,
291 [NVME_ADM_CMD_SECURITY_SEND] = NVME_CMD_EFF_CSUPP,
292 [NVME_ADM_CMD_SECURITY_RECV] = NVME_CMD_EFF_CSUPP,
293 };
294
295 static const uint32_t nvme_cse_iocs_nvm_default[256] = {
296 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
297 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
298 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
299 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
300 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
301 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
302 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
303 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
304 [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
305 [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
306 };
307
308 static const uint32_t nvme_cse_iocs_zoned_default[256] = {
309 [NVME_CMD_FLUSH] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
310 [NVME_CMD_WRITE_ZEROES] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
311 [NVME_CMD_WRITE] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
312 [NVME_CMD_READ] = NVME_CMD_EFF_CSUPP,
313 [NVME_CMD_DSM] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
314 [NVME_CMD_VERIFY] = NVME_CMD_EFF_CSUPP,
315 [NVME_CMD_COPY] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
316 [NVME_CMD_COMPARE] = NVME_CMD_EFF_CSUPP,
317 [NVME_CMD_IO_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
318 [NVME_CMD_IO_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
319
320 [NVME_CMD_ZONE_APPEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
321 [NVME_CMD_ZONE_MGMT_SEND] = NVME_CMD_EFF_CSUPP | NVME_CMD_EFF_LBCC,
322 [NVME_CMD_ZONE_MGMT_RECV] = NVME_CMD_EFF_CSUPP,
323 };
324
325 static void nvme_process_sq(void *opaque);
326 static void nvme_ctrl_reset(NvmeCtrl *n, NvmeResetType rst);
327 static inline uint64_t nvme_get_timestamp(const NvmeCtrl *n);
328
329 static uint16_t nvme_sqid(NvmeRequest *req)
330 {
331 return le16_to_cpu(req->sq->sqid);
332 }
333
334 static inline uint16_t nvme_make_pid(NvmeNamespace *ns, uint16_t rg,
335 uint16_t ph)
336 {
337 uint16_t rgif = ns->endgrp->fdp.rgif;
338
339 if (!rgif) {
340 return ph;
341 }
342
343 return (rg << (16 - rgif)) | ph;
344 }
345
346 static inline bool nvme_ph_valid(NvmeNamespace *ns, uint16_t ph)
347 {
348 return ph < ns->fdp.nphs;
349 }
350
351 static inline bool nvme_rg_valid(NvmeEnduranceGroup *endgrp, uint16_t rg)
352 {
353 return rg < endgrp->fdp.nrg;
354 }
355
356 static inline uint16_t nvme_pid2ph(NvmeNamespace *ns, uint16_t pid)
357 {
358 uint16_t rgif = ns->endgrp->fdp.rgif;
359
360 if (!rgif) {
361 return pid;
362 }
363
364 return pid & ((1 << (15 - rgif)) - 1);
365 }
366
367 static inline uint16_t nvme_pid2rg(NvmeNamespace *ns, uint16_t pid)
368 {
369 uint16_t rgif = ns->endgrp->fdp.rgif;
370
371 if (!rgif) {
372 return 0;
373 }
374
375 return pid >> (16 - rgif);
376 }
377
378 static inline bool nvme_parse_pid(NvmeNamespace *ns, uint16_t pid,
379 uint16_t *ph, uint16_t *rg)
380 {
381 *rg = nvme_pid2rg(ns, pid);
382 *ph = nvme_pid2ph(ns, pid);
383
384 return nvme_ph_valid(ns, *ph) && nvme_rg_valid(ns->endgrp, *rg);
385 }
386
387 static void nvme_assign_zone_state(NvmeNamespace *ns, NvmeZone *zone,
388 NvmeZoneState state)
389 {
390 if (QTAILQ_IN_USE(zone, entry)) {
391 switch (nvme_get_zone_state(zone)) {
392 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
393 QTAILQ_REMOVE(&ns->exp_open_zones, zone, entry);
394 break;
395 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
396 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
397 break;
398 case NVME_ZONE_STATE_CLOSED:
399 QTAILQ_REMOVE(&ns->closed_zones, zone, entry);
400 break;
401 case NVME_ZONE_STATE_FULL:
402 QTAILQ_REMOVE(&ns->full_zones, zone, entry);
403 default:
404 ;
405 }
406 }
407
408 nvme_set_zone_state(zone, state);
409
410 switch (state) {
411 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
412 QTAILQ_INSERT_TAIL(&ns->exp_open_zones, zone, entry);
413 break;
414 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
415 QTAILQ_INSERT_TAIL(&ns->imp_open_zones, zone, entry);
416 break;
417 case NVME_ZONE_STATE_CLOSED:
418 QTAILQ_INSERT_TAIL(&ns->closed_zones, zone, entry);
419 break;
420 case NVME_ZONE_STATE_FULL:
421 QTAILQ_INSERT_TAIL(&ns->full_zones, zone, entry);
422 case NVME_ZONE_STATE_READ_ONLY:
423 break;
424 default:
425 zone->d.za = 0;
426 }
427 }
428
429 static uint16_t nvme_zns_check_resources(NvmeNamespace *ns, uint32_t act,
430 uint32_t opn, uint32_t zrwa)
431 {
432 if (ns->params.max_active_zones != 0 &&
433 ns->nr_active_zones + act > ns->params.max_active_zones) {
434 trace_pci_nvme_err_insuff_active_res(ns->params.max_active_zones);
435 return NVME_ZONE_TOO_MANY_ACTIVE | NVME_DNR;
436 }
437
438 if (ns->params.max_open_zones != 0 &&
439 ns->nr_open_zones + opn > ns->params.max_open_zones) {
440 trace_pci_nvme_err_insuff_open_res(ns->params.max_open_zones);
441 return NVME_ZONE_TOO_MANY_OPEN | NVME_DNR;
442 }
443
444 if (zrwa > ns->zns.numzrwa) {
445 return NVME_NOZRWA | NVME_DNR;
446 }
447
448 return NVME_SUCCESS;
449 }
450
451 /*
452 * Check if we can open a zone without exceeding open/active limits.
453 * AOR stands for "Active and Open Resources" (see TP 4053 section 2.5).
454 */
455 static uint16_t nvme_aor_check(NvmeNamespace *ns, uint32_t act, uint32_t opn)
456 {
457 return nvme_zns_check_resources(ns, act, opn, 0);
458 }
459
460 static NvmeFdpEvent *nvme_fdp_alloc_event(NvmeCtrl *n, NvmeFdpEventBuffer *ebuf)
461 {
462 NvmeFdpEvent *ret = NULL;
463 bool is_full = ebuf->next == ebuf->start && ebuf->nelems;
464
465 ret = &ebuf->events[ebuf->next++];
466 if (unlikely(ebuf->next == NVME_FDP_MAX_EVENTS)) {
467 ebuf->next = 0;
468 }
469 if (is_full) {
470 ebuf->start = ebuf->next;
471 } else {
472 ebuf->nelems++;
473 }
474
475 memset(ret, 0, sizeof(NvmeFdpEvent));
476 ret->timestamp = nvme_get_timestamp(n);
477
478 return ret;
479 }
480
481 static inline int log_event(NvmeRuHandle *ruh, uint8_t event_type)
482 {
483 return (ruh->event_filter >> nvme_fdp_evf_shifts[event_type]) & 0x1;
484 }
485
486 static bool nvme_update_ruh(NvmeCtrl *n, NvmeNamespace *ns, uint16_t pid)
487 {
488 NvmeEnduranceGroup *endgrp = ns->endgrp;
489 NvmeRuHandle *ruh;
490 NvmeReclaimUnit *ru;
491 NvmeFdpEvent *e = NULL;
492 uint16_t ph, rg, ruhid;
493
494 if (!nvme_parse_pid(ns, pid, &ph, &rg)) {
495 return false;
496 }
497
498 ruhid = ns->fdp.phs[ph];
499
500 ruh = &endgrp->fdp.ruhs[ruhid];
501 ru = &ruh->rus[rg];
502
503 if (ru->ruamw) {
504 if (log_event(ruh, FDP_EVT_RU_NOT_FULLY_WRITTEN)) {
505 e = nvme_fdp_alloc_event(n, &endgrp->fdp.host_events);
506 e->type = FDP_EVT_RU_NOT_FULLY_WRITTEN;
507 e->flags = FDPEF_PIV | FDPEF_NSIDV | FDPEF_LV;
508 e->pid = cpu_to_le16(pid);
509 e->nsid = cpu_to_le32(ns->params.nsid);
510 e->rgid = cpu_to_le16(rg);
511 e->ruhid = cpu_to_le16(ruhid);
512 }
513
514 /* log (eventual) GC overhead of prematurely swapping the RU */
515 nvme_fdp_stat_inc(&endgrp->fdp.mbmw, nvme_l2b(ns, ru->ruamw));
516 }
517
518 ru->ruamw = ruh->ruamw;
519
520 return true;
521 }
522
523 static bool nvme_addr_is_cmb(NvmeCtrl *n, hwaddr addr)
524 {
525 hwaddr hi, lo;
526
527 if (!n->cmb.cmse) {
528 return false;
529 }
530
531 lo = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
532 hi = lo + int128_get64(n->cmb.mem.size);
533
534 return addr >= lo && addr < hi;
535 }
536
537 static inline void *nvme_addr_to_cmb(NvmeCtrl *n, hwaddr addr)
538 {
539 hwaddr base = n->params.legacy_cmb ? n->cmb.mem.addr : n->cmb.cba;
540 return &n->cmb.buf[addr - base];
541 }
542
543 static bool nvme_addr_is_pmr(NvmeCtrl *n, hwaddr addr)
544 {
545 hwaddr hi;
546
547 if (!n->pmr.cmse) {
548 return false;
549 }
550
551 hi = n->pmr.cba + int128_get64(n->pmr.dev->mr.size);
552
553 return addr >= n->pmr.cba && addr < hi;
554 }
555
556 static inline void *nvme_addr_to_pmr(NvmeCtrl *n, hwaddr addr)
557 {
558 return memory_region_get_ram_ptr(&n->pmr.dev->mr) + (addr - n->pmr.cba);
559 }
560
561 static inline bool nvme_addr_is_iomem(NvmeCtrl *n, hwaddr addr)
562 {
563 hwaddr hi, lo;
564
565 /*
566 * The purpose of this check is to guard against invalid "local" access to
567 * the iomem (i.e. controller registers). Thus, we check against the range
568 * covered by the 'bar0' MemoryRegion since that is currently composed of
569 * two subregions (the NVMe "MBAR" and the MSI-X table/pba). Note, however,
570 * that if the device model is ever changed to allow the CMB to be located
571 * in BAR0 as well, then this must be changed.
572 */
573 lo = n->bar0.addr;
574 hi = lo + int128_get64(n->bar0.size);
575
576 return addr >= lo && addr < hi;
577 }
578
579 static int nvme_addr_read(NvmeCtrl *n, hwaddr addr, void *buf, int size)
580 {
581 hwaddr hi = addr + size - 1;
582 if (hi < addr) {
583 return 1;
584 }
585
586 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
587 memcpy(buf, nvme_addr_to_cmb(n, addr), size);
588 return 0;
589 }
590
591 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
592 memcpy(buf, nvme_addr_to_pmr(n, addr), size);
593 return 0;
594 }
595
596 return pci_dma_read(PCI_DEVICE(n), addr, buf, size);
597 }
598
599 static int nvme_addr_write(NvmeCtrl *n, hwaddr addr, const void *buf, int size)
600 {
601 hwaddr hi = addr + size - 1;
602 if (hi < addr) {
603 return 1;
604 }
605
606 if (n->bar.cmbsz && nvme_addr_is_cmb(n, addr) && nvme_addr_is_cmb(n, hi)) {
607 memcpy(nvme_addr_to_cmb(n, addr), buf, size);
608 return 0;
609 }
610
611 if (nvme_addr_is_pmr(n, addr) && nvme_addr_is_pmr(n, hi)) {
612 memcpy(nvme_addr_to_pmr(n, addr), buf, size);
613 return 0;
614 }
615
616 return pci_dma_write(PCI_DEVICE(n), addr, buf, size);
617 }
618
619 static bool nvme_nsid_valid(NvmeCtrl *n, uint32_t nsid)
620 {
621 return nsid &&
622 (nsid == NVME_NSID_BROADCAST || nsid <= NVME_MAX_NAMESPACES);
623 }
624
625 static int nvme_check_sqid(NvmeCtrl *n, uint16_t sqid)
626 {
627 return sqid < n->conf_ioqpairs + 1 && n->sq[sqid] != NULL ? 0 : -1;
628 }
629
630 static int nvme_check_cqid(NvmeCtrl *n, uint16_t cqid)
631 {
632 return cqid < n->conf_ioqpairs + 1 && n->cq[cqid] != NULL ? 0 : -1;
633 }
634
635 static void nvme_inc_cq_tail(NvmeCQueue *cq)
636 {
637 cq->tail++;
638 if (cq->tail >= cq->size) {
639 cq->tail = 0;
640 cq->phase = !cq->phase;
641 }
642 }
643
644 static void nvme_inc_sq_head(NvmeSQueue *sq)
645 {
646 sq->head = (sq->head + 1) % sq->size;
647 }
648
649 static uint8_t nvme_cq_full(NvmeCQueue *cq)
650 {
651 return (cq->tail + 1) % cq->size == cq->head;
652 }
653
654 static uint8_t nvme_sq_empty(NvmeSQueue *sq)
655 {
656 return sq->head == sq->tail;
657 }
658
659 static void nvme_irq_check(NvmeCtrl *n)
660 {
661 PCIDevice *pci = PCI_DEVICE(n);
662 uint32_t intms = ldl_le_p(&n->bar.intms);
663
664 if (msix_enabled(pci)) {
665 return;
666 }
667
668 /* vfs does not implement intx */
669 if (pci_is_vf(pci)) {
670 return;
671 }
672
673 if (~intms & n->irq_status) {
674 pci_irq_assert(pci);
675 } else {
676 pci_irq_deassert(pci);
677 }
678 }
679
680 static void nvme_irq_assert(NvmeCtrl *n, NvmeCQueue *cq)
681 {
682 PCIDevice *pci = PCI_DEVICE(n);
683
684 if (cq->irq_enabled) {
685 if (msix_enabled(pci)) {
686 trace_pci_nvme_irq_msix(cq->vector);
687 msix_notify(pci, cq->vector);
688 } else {
689 trace_pci_nvme_irq_pin();
690 assert(cq->vector < 32);
691 n->irq_status |= 1 << cq->vector;
692 nvme_irq_check(n);
693 }
694 } else {
695 trace_pci_nvme_irq_masked();
696 }
697 }
698
699 static void nvme_irq_deassert(NvmeCtrl *n, NvmeCQueue *cq)
700 {
701 if (cq->irq_enabled) {
702 if (msix_enabled(PCI_DEVICE(n))) {
703 return;
704 } else {
705 assert(cq->vector < 32);
706 if (!n->cq_pending) {
707 n->irq_status &= ~(1 << cq->vector);
708 }
709 nvme_irq_check(n);
710 }
711 }
712 }
713
714 static void nvme_req_clear(NvmeRequest *req)
715 {
716 req->ns = NULL;
717 req->opaque = NULL;
718 req->aiocb = NULL;
719 memset(&req->cqe, 0x0, sizeof(req->cqe));
720 req->status = NVME_SUCCESS;
721 }
722
723 static inline void nvme_sg_init(NvmeCtrl *n, NvmeSg *sg, bool dma)
724 {
725 if (dma) {
726 pci_dma_sglist_init(&sg->qsg, PCI_DEVICE(n), 0);
727 sg->flags = NVME_SG_DMA;
728 } else {
729 qemu_iovec_init(&sg->iov, 0);
730 }
731
732 sg->flags |= NVME_SG_ALLOC;
733 }
734
735 static inline void nvme_sg_unmap(NvmeSg *sg)
736 {
737 if (!(sg->flags & NVME_SG_ALLOC)) {
738 return;
739 }
740
741 if (sg->flags & NVME_SG_DMA) {
742 qemu_sglist_destroy(&sg->qsg);
743 } else {
744 qemu_iovec_destroy(&sg->iov);
745 }
746
747 memset(sg, 0x0, sizeof(*sg));
748 }
749
750 /*
751 * When metadata is transferred as extended LBAs, the DPTR mapped into `sg`
752 * holds both data and metadata. This function splits the data and metadata
753 * into two separate QSG/IOVs.
754 */
755 static void nvme_sg_split(NvmeSg *sg, NvmeNamespace *ns, NvmeSg *data,
756 NvmeSg *mdata)
757 {
758 NvmeSg *dst = data;
759 uint32_t trans_len, count = ns->lbasz;
760 uint64_t offset = 0;
761 bool dma = sg->flags & NVME_SG_DMA;
762 size_t sge_len;
763 size_t sg_len = dma ? sg->qsg.size : sg->iov.size;
764 int sg_idx = 0;
765
766 assert(sg->flags & NVME_SG_ALLOC);
767
768 while (sg_len) {
769 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
770
771 trans_len = MIN(sg_len, count);
772 trans_len = MIN(trans_len, sge_len - offset);
773
774 if (dst) {
775 if (dma) {
776 qemu_sglist_add(&dst->qsg, sg->qsg.sg[sg_idx].base + offset,
777 trans_len);
778 } else {
779 qemu_iovec_add(&dst->iov,
780 sg->iov.iov[sg_idx].iov_base + offset,
781 trans_len);
782 }
783 }
784
785 sg_len -= trans_len;
786 count -= trans_len;
787 offset += trans_len;
788
789 if (count == 0) {
790 dst = (dst == data) ? mdata : data;
791 count = (dst == data) ? ns->lbasz : ns->lbaf.ms;
792 }
793
794 if (sge_len == offset) {
795 offset = 0;
796 sg_idx++;
797 }
798 }
799 }
800
801 static uint16_t nvme_map_addr_cmb(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
802 size_t len)
803 {
804 if (!len) {
805 return NVME_SUCCESS;
806 }
807
808 trace_pci_nvme_map_addr_cmb(addr, len);
809
810 if (!nvme_addr_is_cmb(n, addr) || !nvme_addr_is_cmb(n, addr + len - 1)) {
811 return NVME_DATA_TRAS_ERROR;
812 }
813
814 qemu_iovec_add(iov, nvme_addr_to_cmb(n, addr), len);
815
816 return NVME_SUCCESS;
817 }
818
819 static uint16_t nvme_map_addr_pmr(NvmeCtrl *n, QEMUIOVector *iov, hwaddr addr,
820 size_t len)
821 {
822 if (!len) {
823 return NVME_SUCCESS;
824 }
825
826 if (!nvme_addr_is_pmr(n, addr) || !nvme_addr_is_pmr(n, addr + len - 1)) {
827 return NVME_DATA_TRAS_ERROR;
828 }
829
830 qemu_iovec_add(iov, nvme_addr_to_pmr(n, addr), len);
831
832 return NVME_SUCCESS;
833 }
834
835 static uint16_t nvme_map_addr(NvmeCtrl *n, NvmeSg *sg, hwaddr addr, size_t len)
836 {
837 bool cmb = false, pmr = false;
838
839 if (!len) {
840 return NVME_SUCCESS;
841 }
842
843 trace_pci_nvme_map_addr(addr, len);
844
845 if (nvme_addr_is_iomem(n, addr)) {
846 return NVME_DATA_TRAS_ERROR;
847 }
848
849 if (nvme_addr_is_cmb(n, addr)) {
850 cmb = true;
851 } else if (nvme_addr_is_pmr(n, addr)) {
852 pmr = true;
853 }
854
855 if (cmb || pmr) {
856 if (sg->flags & NVME_SG_DMA) {
857 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
858 }
859
860 if (sg->iov.niov + 1 > IOV_MAX) {
861 goto max_mappings_exceeded;
862 }
863
864 if (cmb) {
865 return nvme_map_addr_cmb(n, &sg->iov, addr, len);
866 } else {
867 return nvme_map_addr_pmr(n, &sg->iov, addr, len);
868 }
869 }
870
871 if (!(sg->flags & NVME_SG_DMA)) {
872 return NVME_INVALID_USE_OF_CMB | NVME_DNR;
873 }
874
875 if (sg->qsg.nsg + 1 > IOV_MAX) {
876 goto max_mappings_exceeded;
877 }
878
879 qemu_sglist_add(&sg->qsg, addr, len);
880
881 return NVME_SUCCESS;
882
883 max_mappings_exceeded:
884 NVME_GUEST_ERR(pci_nvme_ub_too_many_mappings,
885 "number of mappings exceed 1024");
886 return NVME_INTERNAL_DEV_ERROR | NVME_DNR;
887 }
888
889 static inline bool nvme_addr_is_dma(NvmeCtrl *n, hwaddr addr)
890 {
891 return !(nvme_addr_is_cmb(n, addr) || nvme_addr_is_pmr(n, addr));
892 }
893
894 static uint16_t nvme_map_prp(NvmeCtrl *n, NvmeSg *sg, uint64_t prp1,
895 uint64_t prp2, uint32_t len)
896 {
897 hwaddr trans_len = n->page_size - (prp1 % n->page_size);
898 trans_len = MIN(len, trans_len);
899 int num_prps = (len >> n->page_bits) + 1;
900 uint16_t status;
901 int ret;
902
903 trace_pci_nvme_map_prp(trans_len, len, prp1, prp2, num_prps);
904
905 nvme_sg_init(n, sg, nvme_addr_is_dma(n, prp1));
906
907 status = nvme_map_addr(n, sg, prp1, trans_len);
908 if (status) {
909 goto unmap;
910 }
911
912 len -= trans_len;
913 if (len) {
914 if (len > n->page_size) {
915 g_autofree uint64_t *prp_list = g_new(uint64_t, n->max_prp_ents);
916 uint32_t nents, prp_trans;
917 int i = 0;
918
919 /*
920 * The first PRP list entry, pointed to by PRP2 may contain offset.
921 * Hence, we need to calculate the number of entries in based on
922 * that offset.
923 */
924 nents = (n->page_size - (prp2 & (n->page_size - 1))) >> 3;
925 prp_trans = MIN(n->max_prp_ents, nents) * sizeof(uint64_t);
926 ret = nvme_addr_read(n, prp2, (void *)prp_list, prp_trans);
927 if (ret) {
928 trace_pci_nvme_err_addr_read(prp2);
929 status = NVME_DATA_TRAS_ERROR;
930 goto unmap;
931 }
932 while (len != 0) {
933 uint64_t prp_ent = le64_to_cpu(prp_list[i]);
934
935 if (i == nents - 1 && len > n->page_size) {
936 if (unlikely(prp_ent & (n->page_size - 1))) {
937 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
938 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
939 goto unmap;
940 }
941
942 i = 0;
943 nents = (len + n->page_size - 1) >> n->page_bits;
944 nents = MIN(nents, n->max_prp_ents);
945 prp_trans = nents * sizeof(uint64_t);
946 ret = nvme_addr_read(n, prp_ent, (void *)prp_list,
947 prp_trans);
948 if (ret) {
949 trace_pci_nvme_err_addr_read(prp_ent);
950 status = NVME_DATA_TRAS_ERROR;
951 goto unmap;
952 }
953 prp_ent = le64_to_cpu(prp_list[i]);
954 }
955
956 if (unlikely(prp_ent & (n->page_size - 1))) {
957 trace_pci_nvme_err_invalid_prplist_ent(prp_ent);
958 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
959 goto unmap;
960 }
961
962 trans_len = MIN(len, n->page_size);
963 status = nvme_map_addr(n, sg, prp_ent, trans_len);
964 if (status) {
965 goto unmap;
966 }
967
968 len -= trans_len;
969 i++;
970 }
971 } else {
972 if (unlikely(prp2 & (n->page_size - 1))) {
973 trace_pci_nvme_err_invalid_prp2_align(prp2);
974 status = NVME_INVALID_PRP_OFFSET | NVME_DNR;
975 goto unmap;
976 }
977 status = nvme_map_addr(n, sg, prp2, len);
978 if (status) {
979 goto unmap;
980 }
981 }
982 }
983
984 return NVME_SUCCESS;
985
986 unmap:
987 nvme_sg_unmap(sg);
988 return status;
989 }
990
991 /*
992 * Map 'nsgld' data descriptors from 'segment'. The function will subtract the
993 * number of bytes mapped in len.
994 */
995 static uint16_t nvme_map_sgl_data(NvmeCtrl *n, NvmeSg *sg,
996 NvmeSglDescriptor *segment, uint64_t nsgld,
997 size_t *len, NvmeCmd *cmd)
998 {
999 dma_addr_t addr, trans_len;
1000 uint32_t dlen;
1001 uint16_t status;
1002
1003 for (int i = 0; i < nsgld; i++) {
1004 uint8_t type = NVME_SGL_TYPE(segment[i].type);
1005
1006 switch (type) {
1007 case NVME_SGL_DESCR_TYPE_DATA_BLOCK:
1008 break;
1009 case NVME_SGL_DESCR_TYPE_SEGMENT:
1010 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1011 return NVME_INVALID_NUM_SGL_DESCRS | NVME_DNR;
1012 default:
1013 return NVME_SGL_DESCR_TYPE_INVALID | NVME_DNR;
1014 }
1015
1016 dlen = le32_to_cpu(segment[i].len);
1017
1018 if (!dlen) {
1019 continue;
1020 }
1021
1022 if (*len == 0) {
1023 /*
1024 * All data has been mapped, but the SGL contains additional
1025 * segments and/or descriptors. The controller might accept
1026 * ignoring the rest of the SGL.
1027 */
1028 uint32_t sgls = le32_to_cpu(n->id_ctrl.sgls);
1029 if (sgls & NVME_CTRL_SGLS_EXCESS_LENGTH) {
1030 break;
1031 }
1032
1033 trace_pci_nvme_err_invalid_sgl_excess_length(dlen);
1034 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1035 }
1036
1037 trans_len = MIN(*len, dlen);
1038
1039 addr = le64_to_cpu(segment[i].addr);
1040
1041 if (UINT64_MAX - addr < dlen) {
1042 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1043 }
1044
1045 status = nvme_map_addr(n, sg, addr, trans_len);
1046 if (status) {
1047 return status;
1048 }
1049
1050 *len -= trans_len;
1051 }
1052
1053 return NVME_SUCCESS;
1054 }
1055
1056 static uint16_t nvme_map_sgl(NvmeCtrl *n, NvmeSg *sg, NvmeSglDescriptor sgl,
1057 size_t len, NvmeCmd *cmd)
1058 {
1059 /*
1060 * Read the segment in chunks of 256 descriptors (one 4k page) to avoid
1061 * dynamically allocating a potentially huge SGL. The spec allows the SGL
1062 * to be larger (as in number of bytes required to describe the SGL
1063 * descriptors and segment chain) than the command transfer size, so it is
1064 * not bounded by MDTS.
1065 */
1066 #define SEG_CHUNK_SIZE 256
1067
1068 QEMU_UNINITIALIZED NvmeSglDescriptor segment[SEG_CHUNK_SIZE];
1069 NvmeSglDescriptor *sgld, *last_sgld;
1070 uint64_t nsgld;
1071 uint32_t seg_len;
1072 uint16_t status;
1073 hwaddr addr;
1074 int ret;
1075
1076 sgld = &sgl;
1077 addr = le64_to_cpu(sgl.addr);
1078
1079 trace_pci_nvme_map_sgl(NVME_SGL_TYPE(sgl.type), len);
1080
1081 nvme_sg_init(n, sg, nvme_addr_is_dma(n, addr));
1082
1083 /*
1084 * If the entire transfer can be described with a single data block it can
1085 * be mapped directly.
1086 */
1087 if (NVME_SGL_TYPE(sgl.type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1088 status = nvme_map_sgl_data(n, sg, sgld, 1, &len, cmd);
1089 if (status) {
1090 goto unmap;
1091 }
1092
1093 goto out;
1094 }
1095
1096 for (;;) {
1097 size_t prev_len = len;
1098
1099 switch (NVME_SGL_TYPE(sgld->type)) {
1100 case NVME_SGL_DESCR_TYPE_SEGMENT:
1101 case NVME_SGL_DESCR_TYPE_LAST_SEGMENT:
1102 break;
1103 default:
1104 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1105 }
1106
1107 seg_len = le32_to_cpu(sgld->len);
1108
1109 /* check the length of the (Last) Segment descriptor */
1110 if (!seg_len || seg_len & 0xf) {
1111 return NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1112 }
1113
1114 if (UINT64_MAX - addr < seg_len) {
1115 return NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1116 }
1117
1118 nsgld = seg_len / sizeof(NvmeSglDescriptor);
1119
1120 while (nsgld > SEG_CHUNK_SIZE) {
1121 if (nvme_addr_read(n, addr, segment, sizeof(segment))) {
1122 trace_pci_nvme_err_addr_read(addr);
1123 status = NVME_DATA_TRAS_ERROR;
1124 goto unmap;
1125 }
1126
1127 status = nvme_map_sgl_data(n, sg, segment, SEG_CHUNK_SIZE,
1128 &len, cmd);
1129 if (status) {
1130 goto unmap;
1131 }
1132
1133 nsgld -= SEG_CHUNK_SIZE;
1134 addr += SEG_CHUNK_SIZE * sizeof(NvmeSglDescriptor);
1135 }
1136
1137 ret = nvme_addr_read(n, addr, segment, nsgld *
1138 sizeof(NvmeSglDescriptor));
1139 if (ret) {
1140 trace_pci_nvme_err_addr_read(addr);
1141 status = NVME_DATA_TRAS_ERROR;
1142 goto unmap;
1143 }
1144
1145 last_sgld = &segment[nsgld - 1];
1146
1147 /*
1148 * If the segment ends with a Data Block, then we are done.
1149 */
1150 if (NVME_SGL_TYPE(last_sgld->type) == NVME_SGL_DESCR_TYPE_DATA_BLOCK) {
1151 status = nvme_map_sgl_data(n, sg, segment, nsgld, &len, cmd);
1152 if (status) {
1153 goto unmap;
1154 }
1155
1156 goto out;
1157 }
1158
1159 /*
1160 * If the last descriptor was not a Data Block, then the current
1161 * segment must not be a Last Segment.
1162 */
1163 if (NVME_SGL_TYPE(sgld->type) == NVME_SGL_DESCR_TYPE_LAST_SEGMENT) {
1164 status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1165 goto unmap;
1166 }
1167
1168 sgld = last_sgld;
1169 addr = le64_to_cpu(sgld->addr);
1170
1171 /*
1172 * Do not map the last descriptor; it will be a Segment or Last Segment
1173 * descriptor and is handled by the next iteration.
1174 */
1175 status = nvme_map_sgl_data(n, sg, segment, nsgld - 1, &len, cmd);
1176 if (status) {
1177 goto unmap;
1178 }
1179
1180 /*
1181 * Reject if this segment made no forward progress. The host should
1182 * have skipped linking an empty segment. While not strictly spec
1183 * compliant, allowing this makes it easy for a pathological host to
1184 * create an infinite loop.
1185 */
1186 if (len == prev_len) {
1187 status = NVME_INVALID_SGL_SEG_DESCR | NVME_DNR;
1188 goto unmap;
1189 }
1190 }
1191
1192 out:
1193 /* if there is any residual left in len, the SGL was too short */
1194 if (len) {
1195 status = NVME_DATA_SGL_LEN_INVALID | NVME_DNR;
1196 goto unmap;
1197 }
1198
1199 return NVME_SUCCESS;
1200
1201 unmap:
1202 nvme_sg_unmap(sg);
1203 return status;
1204 }
1205
1206 uint16_t nvme_map_dptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1207 NvmeCmd *cmd)
1208 {
1209 uint64_t prp1, prp2;
1210
1211 switch (NVME_CMD_FLAGS_PSDT(cmd->flags)) {
1212 case NVME_PSDT_PRP:
1213 prp1 = le64_to_cpu(cmd->dptr.prp1);
1214 prp2 = le64_to_cpu(cmd->dptr.prp2);
1215
1216 return nvme_map_prp(n, sg, prp1, prp2, len);
1217 case NVME_PSDT_SGL_MPTR_CONTIGUOUS:
1218 case NVME_PSDT_SGL_MPTR_SGL:
1219 return nvme_map_sgl(n, sg, cmd->dptr.sgl, len, cmd);
1220 default:
1221 return NVME_INVALID_FIELD;
1222 }
1223 }
1224
1225 static uint16_t nvme_map_mptr(NvmeCtrl *n, NvmeSg *sg, size_t len,
1226 NvmeCmd *cmd)
1227 {
1228 int psdt = NVME_CMD_FLAGS_PSDT(cmd->flags);
1229 hwaddr mptr = le64_to_cpu(cmd->mptr);
1230 uint16_t status;
1231
1232 if (psdt == NVME_PSDT_SGL_MPTR_SGL) {
1233 NvmeSglDescriptor sgl;
1234
1235 if (nvme_addr_read(n, mptr, &sgl, sizeof(sgl))) {
1236 return NVME_DATA_TRAS_ERROR;
1237 }
1238
1239 status = nvme_map_sgl(n, sg, sgl, len, cmd);
1240 if (status && (status & 0x7ff) == NVME_DATA_SGL_LEN_INVALID) {
1241 status = NVME_MD_SGL_LEN_INVALID | NVME_DNR;
1242 }
1243
1244 return status;
1245 }
1246
1247 nvme_sg_init(n, sg, nvme_addr_is_dma(n, mptr));
1248 status = nvme_map_addr(n, sg, mptr, len);
1249 if (status) {
1250 nvme_sg_unmap(sg);
1251 }
1252
1253 return status;
1254 }
1255
1256 static uint16_t nvme_map_data(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1257 {
1258 NvmeNamespace *ns = req->ns;
1259 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1260 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1261 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1262 size_t len = nvme_l2b(ns, nlb);
1263 uint16_t status;
1264
1265 if (nvme_ns_ext(ns) &&
1266 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1267 NvmeSg sg;
1268
1269 len += nvme_m2b(ns, nlb);
1270
1271 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1272 if (status) {
1273 return status;
1274 }
1275
1276 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1277 nvme_sg_split(&sg, ns, &req->sg, NULL);
1278 nvme_sg_unmap(&sg);
1279
1280 return NVME_SUCCESS;
1281 }
1282
1283 return nvme_map_dptr(n, &req->sg, len, &req->cmd);
1284 }
1285
1286 static uint16_t nvme_map_mdata(NvmeCtrl *n, uint32_t nlb, NvmeRequest *req)
1287 {
1288 NvmeNamespace *ns = req->ns;
1289 size_t len = nvme_m2b(ns, nlb);
1290 uint16_t status;
1291
1292 if (nvme_ns_ext(ns)) {
1293 NvmeSg sg;
1294
1295 len += nvme_l2b(ns, nlb);
1296
1297 status = nvme_map_dptr(n, &sg, len, &req->cmd);
1298 if (status) {
1299 return status;
1300 }
1301
1302 nvme_sg_init(n, &req->sg, sg.flags & NVME_SG_DMA);
1303 nvme_sg_split(&sg, ns, NULL, &req->sg);
1304 nvme_sg_unmap(&sg);
1305
1306 return NVME_SUCCESS;
1307 }
1308
1309 return nvme_map_mptr(n, &req->sg, len, &req->cmd);
1310 }
1311
1312 static uint16_t nvme_tx_interleaved(NvmeCtrl *n, NvmeSg *sg, uint8_t *ptr,
1313 uint32_t len, uint32_t bytes,
1314 int32_t skip_bytes, int64_t offset,
1315 NvmeTxDirection dir)
1316 {
1317 hwaddr addr;
1318 uint32_t trans_len, count = bytes;
1319 bool dma = sg->flags & NVME_SG_DMA;
1320 int64_t sge_len;
1321 int sg_idx = 0;
1322 int ret;
1323
1324 assert(sg->flags & NVME_SG_ALLOC);
1325
1326 while (len) {
1327 sge_len = dma ? sg->qsg.sg[sg_idx].len : sg->iov.iov[sg_idx].iov_len;
1328
1329 if (sge_len - offset < 0) {
1330 offset -= sge_len;
1331 sg_idx++;
1332 continue;
1333 }
1334
1335 if (sge_len == offset) {
1336 offset = 0;
1337 sg_idx++;
1338 continue;
1339 }
1340
1341 trans_len = MIN(len, count);
1342 trans_len = MIN(trans_len, sge_len - offset);
1343
1344 if (dma) {
1345 addr = sg->qsg.sg[sg_idx].base + offset;
1346 } else {
1347 addr = (hwaddr)(uintptr_t)sg->iov.iov[sg_idx].iov_base + offset;
1348 }
1349
1350 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1351 ret = nvme_addr_read(n, addr, ptr, trans_len);
1352 } else {
1353 ret = nvme_addr_write(n, addr, ptr, trans_len);
1354 }
1355
1356 if (ret) {
1357 return NVME_DATA_TRAS_ERROR;
1358 }
1359
1360 ptr += trans_len;
1361 len -= trans_len;
1362 count -= trans_len;
1363 offset += trans_len;
1364
1365 if (count == 0) {
1366 count = bytes;
1367 offset += skip_bytes;
1368 }
1369 }
1370
1371 return NVME_SUCCESS;
1372 }
1373
1374 static uint16_t nvme_tx(NvmeCtrl *n, NvmeSg *sg, void *ptr, uint32_t len,
1375 NvmeTxDirection dir)
1376 {
1377 assert(sg->flags & NVME_SG_ALLOC);
1378
1379 if (sg->flags & NVME_SG_DMA) {
1380 const MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
1381 dma_addr_t residual;
1382
1383 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1384 dma_buf_write(ptr, len, &residual, &sg->qsg, attrs);
1385 } else {
1386 dma_buf_read(ptr, len, &residual, &sg->qsg, attrs);
1387 }
1388
1389 if (unlikely(residual)) {
1390 trace_pci_nvme_err_invalid_dma();
1391 return NVME_INVALID_FIELD | NVME_DNR;
1392 }
1393 } else {
1394 size_t bytes;
1395
1396 if (dir == NVME_TX_DIRECTION_TO_DEVICE) {
1397 bytes = qemu_iovec_to_buf(&sg->iov, 0, ptr, len);
1398 } else {
1399 bytes = qemu_iovec_from_buf(&sg->iov, 0, ptr, len);
1400 }
1401
1402 if (unlikely(bytes != len)) {
1403 trace_pci_nvme_err_invalid_dma();
1404 return NVME_INVALID_FIELD | NVME_DNR;
1405 }
1406 }
1407
1408 return NVME_SUCCESS;
1409 }
1410
1411 static inline uint16_t nvme_c2h(NvmeCtrl *n, void *ptr, uint32_t len,
1412 NvmeRequest *req)
1413 {
1414 uint16_t status;
1415
1416 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1417 if (status) {
1418 return status;
1419 }
1420
1421 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_FROM_DEVICE);
1422 }
1423
1424 static inline uint16_t nvme_h2c(NvmeCtrl *n, void *ptr, uint32_t len,
1425 NvmeRequest *req)
1426 {
1427 uint16_t status;
1428
1429 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
1430 if (status) {
1431 return status;
1432 }
1433
1434 return nvme_tx(n, &req->sg, ptr, len, NVME_TX_DIRECTION_TO_DEVICE);
1435 }
1436
1437 uint16_t nvme_bounce_data(NvmeCtrl *n, void *ptr, uint32_t len,
1438 NvmeTxDirection dir, NvmeRequest *req)
1439 {
1440 NvmeNamespace *ns = req->ns;
1441 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
1442 bool pi = !!NVME_ID_NS_DPS_TYPE(ns->id_ns.dps);
1443 bool pract = !!(le16_to_cpu(rw->control) & NVME_RW_PRINFO_PRACT);
1444
1445 if (nvme_ns_ext(ns) &&
1446 !(pi && pract && ns->lbaf.ms == nvme_pi_tuple_size(ns))) {
1447 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbasz,
1448 ns->lbaf.ms, 0, dir);
1449 }
1450
1451 return nvme_tx(n, &req->sg, ptr, len, dir);
1452 }
1453
1454 uint16_t nvme_bounce_mdata(NvmeCtrl *n, void *ptr, uint32_t len,
1455 NvmeTxDirection dir, NvmeRequest *req)
1456 {
1457 NvmeNamespace *ns = req->ns;
1458 uint16_t status;
1459
1460 if (nvme_ns_ext(ns)) {
1461 return nvme_tx_interleaved(n, &req->sg, ptr, len, ns->lbaf.ms,
1462 ns->lbasz, ns->lbasz, dir);
1463 }
1464
1465 nvme_sg_unmap(&req->sg);
1466
1467 status = nvme_map_mptr(n, &req->sg, len, &req->cmd);
1468 if (status) {
1469 return status;
1470 }
1471
1472 return nvme_tx(n, &req->sg, ptr, len, dir);
1473 }
1474
1475 static inline void nvme_blk_read(BlockBackend *blk, int64_t offset,
1476 uint32_t align, BlockCompletionFunc *cb,
1477 NvmeRequest *req)
1478 {
1479 assert(req->sg.flags & NVME_SG_ALLOC);
1480
1481 if (req->sg.flags & NVME_SG_DMA) {
1482 req->aiocb = dma_blk_read(blk, &req->sg.qsg, offset, align, cb, req);
1483 } else {
1484 req->aiocb = blk_aio_preadv(blk, offset, &req->sg.iov, 0, cb, req);
1485 }
1486 }
1487
1488 static inline void nvme_blk_write(BlockBackend *blk, int64_t offset,
1489 uint32_t align, BlockCompletionFunc *cb,
1490 NvmeRequest *req)
1491 {
1492 assert(req->sg.flags & NVME_SG_ALLOC);
1493
1494 if (req->sg.flags & NVME_SG_DMA) {
1495 req->aiocb = dma_blk_write(blk, &req->sg.qsg, offset, align, cb, req);
1496 } else {
1497 req->aiocb = blk_aio_pwritev(blk, offset, &req->sg.iov, 0, cb, req);
1498 }
1499 }
1500
1501 static void nvme_update_cq_eventidx(const NvmeCQueue *cq)
1502 {
1503 trace_pci_nvme_update_cq_eventidx(cq->cqid, cq->head);
1504
1505 stl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->ei_addr, cq->head,
1506 MEMTXATTRS_UNSPECIFIED);
1507 }
1508
1509 static void nvme_update_cq_head(NvmeCQueue *cq)
1510 {
1511 ldl_le_pci_dma(PCI_DEVICE(cq->ctrl), cq->db_addr, &cq->head,
1512 MEMTXATTRS_UNSPECIFIED);
1513
1514 trace_pci_nvme_update_cq_head(cq->cqid, cq->head);
1515 }
1516
1517 static void nvme_post_cqes(void *opaque)
1518 {
1519 NvmeCQueue *cq = opaque;
1520 NvmeCtrl *n = cq->ctrl;
1521 NvmeRequest *req, *next;
1522 bool pending = cq->head != cq->tail;
1523 int ret;
1524
1525 QTAILQ_FOREACH_SAFE(req, &cq->req_list, entry, next) {
1526 NvmeSQueue *sq;
1527 hwaddr addr;
1528
1529 if (n->dbbuf_enabled) {
1530 nvme_update_cq_eventidx(cq);
1531 nvme_update_cq_head(cq);
1532 }
1533
1534 if (nvme_cq_full(cq)) {
1535 break;
1536 }
1537
1538 /*
1539 * Here we take the following fields from NvmeRequest structure
1540 * and write cqe to the guest RAM based on them:
1541 * - req->sq
1542 * - req->status
1543 * - req->cqe
1544 *
1545 * If you change this code and more fields from NvmeRequest are
1546 * used, please make sure that you have handled this in:
1547 * nvme_vmstate_request and nvme_ctrl_pre_save().
1548 */
1549
1550 sq = req->sq;
1551 req->cqe.status = cpu_to_le16((req->status << 1) | cq->phase);
1552 req->cqe.sq_head = cpu_to_le16(sq->head);
1553 addr = cq->dma_addr + (cq->tail << NVME_CQES);
1554 ret = pci_dma_write(PCI_DEVICE(n), addr, (void *)&req->cqe,
1555 sizeof(req->cqe));
1556 if (ret) {
1557 trace_pci_nvme_err_addr_write(addr);
1558 trace_pci_nvme_err_cfs();
1559 stl_le_p(&n->bar.csts, NVME_CSTS_FAILED);
1560 break;
1561 }
1562
1563 QTAILQ_REMOVE(&cq->req_list, req, entry);
1564
1565 nvme_inc_cq_tail(cq);
1566
1567 if (QTAILQ_EMPTY(&sq->req_list) && !nvme_sq_empty(sq)) {
1568 qemu_bh_schedule(sq->bh);
1569 }
1570
1571 QTAILQ_INSERT_TAIL(&sq->req_list, req, entry);
1572 }
1573 if (cq->tail != cq->head) {
1574 if (cq->irq_enabled && !pending) {
1575 n->cq_pending++;
1576 }
1577
1578 nvme_irq_assert(n, cq);
1579 }
1580 }
1581
1582 static void nvme_enqueue_req_completion(NvmeCQueue *cq, NvmeRequest *req)
1583 {
1584 assert(cq->cqid == req->sq->cqid);
1585 trace_pci_nvme_enqueue_req_completion(nvme_cid(req), cq->cqid,
1586 le32_to_cpu(req->cqe.result),
1587 le32_to_cpu(req->cqe.dw1),
1588 req->status);
1589
1590 if (req->status) {
1591 trace_pci_nvme_err_req_status(nvme_cid(req), nvme_nsid(req->ns),
1592 req->status, req->cmd.opcode);
1593 }
1594
1595 nvme_sg_unmap(&req->sg);
1596
1597 QTAILQ_REMOVE(&req->sq->out_req_list, req, entry);
1598 QTAILQ_INSERT_TAIL(&cq->req_list, req, entry);
1599
1600 qemu_bh_schedule(cq->bh);
1601 }
1602
1603 static void nvme_process_aers(void *opaque)
1604 {
1605 NvmeCtrl *n = opaque;
1606 NvmeAsyncEvent *event, *next;
1607
1608 trace_pci_nvme_process_aers(n->aer_queued);
1609
1610 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1611 NvmeRequest *req;
1612 NvmeAerResult *result;
1613
1614 /* can't post cqe if there is nothing to complete */
1615 if (!n->outstanding_aers) {
1616 trace_pci_nvme_no_outstanding_aers();
1617 break;
1618 }
1619
1620 /* ignore if masked (cqe posted, but event not cleared) */
1621 if (n->aer_mask & (1 << event->result.event_type)) {
1622 trace_pci_nvme_aer_masked(event->result.event_type, n->aer_mask);
1623 continue;
1624 }
1625
1626 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1627 n->aer_queued--;
1628
1629 n->aer_mask |= 1 << event->result.event_type;
1630 n->outstanding_aers--;
1631
1632 req = n->aer_reqs[n->outstanding_aers];
1633
1634 result = (NvmeAerResult *) &req->cqe.result;
1635 result->event_type = event->result.event_type;
1636 result->event_info = event->result.event_info;
1637 result->log_page = event->result.log_page;
1638 g_free(event);
1639
1640 trace_pci_nvme_aer_post_cqe(result->event_type, result->event_info,
1641 result->log_page);
1642
1643 nvme_enqueue_req_completion(&n->admin_cq, req);
1644 }
1645 }
1646
1647 static void nvme_enqueue_event(NvmeCtrl *n, uint8_t event_type,
1648 uint8_t event_info, uint8_t log_page)
1649 {
1650 NvmeAsyncEvent *event;
1651
1652 trace_pci_nvme_enqueue_event(event_type, event_info, log_page);
1653
1654 if (n->aer_queued == n->params.aer_max_queued) {
1655 trace_pci_nvme_enqueue_event_noqueue(n->aer_queued);
1656 return;
1657 }
1658
1659 event = g_new(NvmeAsyncEvent, 1);
1660 event->result = (NvmeAerResult) {
1661 .event_type = event_type,
1662 .event_info = event_info,
1663 .log_page = log_page,
1664 };
1665
1666 QTAILQ_INSERT_TAIL(&n->aer_queue, event, entry);
1667 n->aer_queued++;
1668
1669 nvme_process_aers(n);
1670 }
1671
1672 static void nvme_smart_event(NvmeCtrl *n, uint8_t event)
1673 {
1674 uint8_t aer_info;
1675
1676 /* Ref SPEC <Asynchronous Event Information 0x2013 SMART / Health Status> */
1677 if (!(NVME_AEC_SMART(n->features.async_config) & event)) {
1678 return;
1679 }
1680
1681 switch (event) {
1682 case NVME_SMART_SPARE:
1683 aer_info = NVME_AER_INFO_SMART_SPARE_THRESH;
1684 break;
1685 case NVME_SMART_TEMPERATURE:
1686 aer_info = NVME_AER_INFO_SMART_TEMP_THRESH;
1687 break;
1688 case NVME_SMART_RELIABILITY:
1689 case NVME_SMART_MEDIA_READ_ONLY:
1690 case NVME_SMART_FAILED_VOLATILE_MEDIA:
1691 case NVME_SMART_PMR_UNRELIABLE:
1692 aer_info = NVME_AER_INFO_SMART_RELIABILITY;
1693 break;
1694 default:
1695 return;
1696 }
1697
1698 nvme_enqueue_event(n, NVME_AER_TYPE_SMART, aer_info, NVME_LOG_SMART_INFO);
1699 }
1700
1701 static void nvme_clear_events(NvmeCtrl *n, uint8_t event_type)
1702 {
1703 NvmeAsyncEvent *event, *next;
1704
1705 n->aer_mask &= ~(1 << event_type);
1706
1707 QTAILQ_FOREACH_SAFE(event, &n->aer_queue, entry, next) {
1708 if (event->result.event_type == event_type) {
1709 QTAILQ_REMOVE(&n->aer_queue, event, entry);
1710 n->aer_queued--;
1711 g_free(event);
1712 }
1713 }
1714 }
1715
1716 static inline uint16_t nvme_check_mdts(NvmeCtrl *n, size_t len)
1717 {
1718 uint8_t mdts = n->params.mdts;
1719
1720 if (mdts && len > n->page_size << mdts) {
1721 trace_pci_nvme_err_mdts(len);
1722 return NVME_INVALID_FIELD | NVME_DNR;
1723 }
1724
1725 return NVME_SUCCESS;
1726 }
1727
1728 static inline uint16_t nvme_check_bounds(NvmeNamespace *ns, uint64_t slba,
1729 uint32_t nlb)
1730 {
1731 uint64_t nsze = le64_to_cpu(ns->id_ns.nsze);
1732
1733 if (unlikely(UINT64_MAX - slba < nlb || slba + nlb > nsze)) {
1734 trace_pci_nvme_err_invalid_lba_range(slba, nlb, nsze);
1735 return NVME_LBA_RANGE | NVME_DNR;
1736 }
1737
1738 return NVME_SUCCESS;
1739 }
1740
1741 static int nvme_block_status_all(NvmeNamespace *ns, uint64_t slba,
1742 uint32_t nlb, int flags)
1743 {
1744 BlockDriverState *bs = blk_bs(ns->blkconf.blk);
1745
1746 int64_t pnum = 0, bytes = nvme_l2b(ns, nlb);
1747 int64_t offset = nvme_l2b(ns, slba);
1748 int ret;
1749
1750 /*
1751 * `pnum` holds the number of bytes after offset that shares the same
1752 * allocation status as the byte at offset. If `pnum` is different from
1753 * `bytes`, we should check the allocation status of the next range and
1754 * continue this until all bytes have been checked.
1755 */
1756 do {
1757 bytes -= pnum;
1758
1759 ret = bdrv_block_status(bs, offset, bytes, &pnum, NULL, NULL);
1760 if (ret < 0) {
1761 return ret;
1762 }
1763
1764
1765 trace_pci_nvme_block_status(offset, bytes, pnum, ret,
1766 !!(ret & BDRV_BLOCK_ZERO));
1767
1768 if (!(ret & flags)) {
1769 return 1;
1770 }
1771
1772 offset += pnum;
1773 } while (pnum != bytes);
1774
1775 return 0;
1776 }
1777
1778 static uint16_t nvme_check_dulbe(NvmeNamespace *ns, uint64_t slba,
1779 uint32_t nlb)
1780 {
1781 int ret;
1782 Error *err = NULL;
1783
1784 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_DATA);
1785 if (ret) {
1786 if (ret < 0) {
1787 error_setg_errno(&err, -ret, "unable to get block status");
1788 error_report_err(err);
1789
1790 return NVME_INTERNAL_DEV_ERROR;
1791 }
1792
1793 return NVME_DULB;
1794 }
1795
1796 return NVME_SUCCESS;
1797 }
1798
1799 static inline uint32_t nvme_zone_idx(NvmeNamespace *ns, uint64_t slba)
1800 {
1801 return ns->zone_size_log2 > 0 ? slba >> ns->zone_size_log2 :
1802 slba / ns->zone_size;
1803 }
1804
1805 static inline NvmeZone *nvme_get_zone_by_slba(NvmeNamespace *ns, uint64_t slba)
1806 {
1807 uint32_t zone_idx = nvme_zone_idx(ns, slba);
1808
1809 if (zone_idx >= ns->num_zones) {
1810 return NULL;
1811 }
1812
1813 return &ns->zone_array[zone_idx];
1814 }
1815
1816 static uint16_t nvme_check_zone_state_for_write(NvmeZone *zone)
1817 {
1818 uint64_t zslba = zone->d.zslba;
1819
1820 switch (nvme_get_zone_state(zone)) {
1821 case NVME_ZONE_STATE_EMPTY:
1822 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1823 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1824 case NVME_ZONE_STATE_CLOSED:
1825 return NVME_SUCCESS;
1826 case NVME_ZONE_STATE_FULL:
1827 trace_pci_nvme_err_zone_is_full(zslba);
1828 return NVME_ZONE_FULL;
1829 case NVME_ZONE_STATE_OFFLINE:
1830 trace_pci_nvme_err_zone_is_offline(zslba);
1831 return NVME_ZONE_OFFLINE;
1832 case NVME_ZONE_STATE_READ_ONLY:
1833 trace_pci_nvme_err_zone_is_read_only(zslba);
1834 return NVME_ZONE_READ_ONLY;
1835 default:
1836 g_assert_not_reached();
1837 }
1838
1839 return NVME_INTERNAL_DEV_ERROR;
1840 }
1841
1842 static uint16_t nvme_check_zone_write(NvmeNamespace *ns, NvmeZone *zone,
1843 uint64_t slba, uint32_t nlb)
1844 {
1845 uint64_t zcap = nvme_zone_wr_boundary(zone);
1846 uint16_t status;
1847
1848 status = nvme_check_zone_state_for_write(zone);
1849 if (status) {
1850 return status;
1851 }
1852
1853 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1854 uint64_t ezrwa = zone->w_ptr + 2 * ns->zns.zrwas;
1855
1856 if (slba < zone->w_ptr || slba + nlb > ezrwa) {
1857 trace_pci_nvme_err_zone_invalid_write(slba, zone->w_ptr);
1858 return NVME_ZONE_INVALID_WRITE;
1859 }
1860 } else {
1861 if (unlikely(slba != zone->w_ptr)) {
1862 trace_pci_nvme_err_write_not_at_wp(slba, zone->d.zslba,
1863 zone->w_ptr);
1864 return NVME_ZONE_INVALID_WRITE;
1865 }
1866 }
1867
1868 if (unlikely((slba + nlb) > zcap)) {
1869 trace_pci_nvme_err_zone_boundary(slba, nlb, zcap);
1870 return NVME_ZONE_BOUNDARY_ERROR;
1871 }
1872
1873 return NVME_SUCCESS;
1874 }
1875
1876 static uint16_t nvme_check_zone_state_for_read(NvmeZone *zone)
1877 {
1878 switch (nvme_get_zone_state(zone)) {
1879 case NVME_ZONE_STATE_EMPTY:
1880 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1881 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1882 case NVME_ZONE_STATE_FULL:
1883 case NVME_ZONE_STATE_CLOSED:
1884 case NVME_ZONE_STATE_READ_ONLY:
1885 return NVME_SUCCESS;
1886 case NVME_ZONE_STATE_OFFLINE:
1887 trace_pci_nvme_err_zone_is_offline(zone->d.zslba);
1888 return NVME_ZONE_OFFLINE;
1889 default:
1890 g_assert_not_reached();
1891 }
1892
1893 return NVME_INTERNAL_DEV_ERROR;
1894 }
1895
1896 static uint16_t nvme_check_zone_read(NvmeNamespace *ns, uint64_t slba,
1897 uint32_t nlb)
1898 {
1899 NvmeZone *zone;
1900 uint64_t bndry, end;
1901 uint16_t status;
1902
1903 zone = nvme_get_zone_by_slba(ns, slba);
1904 assert(zone);
1905
1906 bndry = nvme_zone_rd_boundary(ns, zone);
1907 end = slba + nlb;
1908
1909 status = nvme_check_zone_state_for_read(zone);
1910 if (status) {
1911 ;
1912 } else if (unlikely(end > bndry)) {
1913 if (!ns->params.cross_zone_read) {
1914 status = NVME_ZONE_BOUNDARY_ERROR;
1915 } else {
1916 /*
1917 * Read across zone boundary - check that all subsequent
1918 * zones that are being read have an appropriate state.
1919 */
1920 do {
1921 zone++;
1922 status = nvme_check_zone_state_for_read(zone);
1923 if (status) {
1924 break;
1925 }
1926 } while (end > nvme_zone_rd_boundary(ns, zone));
1927 }
1928 }
1929
1930 return status;
1931 }
1932
1933 static uint16_t nvme_zrm_finish(NvmeNamespace *ns, NvmeZone *zone)
1934 {
1935 switch (nvme_get_zone_state(zone)) {
1936 case NVME_ZONE_STATE_FULL:
1937 return NVME_SUCCESS;
1938
1939 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1940 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1941 nvme_aor_dec_open(ns);
1942 /* fallthrough */
1943 case NVME_ZONE_STATE_CLOSED:
1944 nvme_aor_dec_active(ns);
1945
1946 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1947 zone->d.za &= ~NVME_ZA_ZRWA_VALID;
1948 if (ns->params.numzrwa) {
1949 ns->zns.numzrwa++;
1950 }
1951 }
1952
1953 /* fallthrough */
1954 case NVME_ZONE_STATE_EMPTY:
1955 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_FULL);
1956 return NVME_SUCCESS;
1957
1958 default:
1959 return NVME_ZONE_INVAL_TRANSITION;
1960 }
1961 }
1962
1963 static uint16_t nvme_zrm_close(NvmeNamespace *ns, NvmeZone *zone)
1964 {
1965 switch (nvme_get_zone_state(zone)) {
1966 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1967 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1968 nvme_aor_dec_open(ns);
1969 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
1970 /* fall through */
1971 case NVME_ZONE_STATE_CLOSED:
1972 return NVME_SUCCESS;
1973
1974 default:
1975 return NVME_ZONE_INVAL_TRANSITION;
1976 }
1977 }
1978
1979 static uint16_t nvme_zrm_reset(NvmeNamespace *ns, NvmeZone *zone)
1980 {
1981 switch (nvme_get_zone_state(zone)) {
1982 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
1983 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
1984 nvme_aor_dec_open(ns);
1985 /* fallthrough */
1986 case NVME_ZONE_STATE_CLOSED:
1987 nvme_aor_dec_active(ns);
1988
1989 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
1990 if (ns->params.numzrwa) {
1991 ns->zns.numzrwa++;
1992 }
1993 }
1994
1995 /* fallthrough */
1996 case NVME_ZONE_STATE_FULL:
1997 zone->w_ptr = zone->d.zslba;
1998 zone->d.wp = zone->w_ptr;
1999 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EMPTY);
2000 /* fallthrough */
2001 case NVME_ZONE_STATE_EMPTY:
2002 return NVME_SUCCESS;
2003
2004 default:
2005 return NVME_ZONE_INVAL_TRANSITION;
2006 }
2007 }
2008
2009 static void nvme_zrm_auto_transition_zone(NvmeNamespace *ns)
2010 {
2011 NvmeZone *zone;
2012
2013 if (ns->params.max_open_zones &&
2014 ns->nr_open_zones == ns->params.max_open_zones) {
2015 zone = QTAILQ_FIRST(&ns->imp_open_zones);
2016 if (zone) {
2017 /*
2018 * Automatically close this implicitly open zone.
2019 */
2020 QTAILQ_REMOVE(&ns->imp_open_zones, zone, entry);
2021 nvme_zrm_close(ns, zone);
2022 }
2023 }
2024 }
2025
2026 enum {
2027 NVME_ZRM_AUTO = 1 << 0,
2028 NVME_ZRM_ZRWA = 1 << 1,
2029 };
2030
2031 static uint16_t nvme_zrm_open_flags(NvmeCtrl *n, NvmeNamespace *ns,
2032 NvmeZone *zone, int flags)
2033 {
2034 int act = 0;
2035 uint16_t status;
2036
2037 switch (nvme_get_zone_state(zone)) {
2038 case NVME_ZONE_STATE_EMPTY:
2039 act = 1;
2040
2041 /* fallthrough */
2042
2043 case NVME_ZONE_STATE_CLOSED:
2044 if (n->params.auto_transition_zones) {
2045 nvme_zrm_auto_transition_zone(ns);
2046 }
2047 status = nvme_zns_check_resources(ns, act, 1,
2048 (flags & NVME_ZRM_ZRWA) ? 1 : 0);
2049 if (status) {
2050 return status;
2051 }
2052
2053 if (act) {
2054 nvme_aor_inc_active(ns);
2055 }
2056
2057 nvme_aor_inc_open(ns);
2058
2059 if (flags & NVME_ZRM_AUTO) {
2060 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_IMPLICITLY_OPEN);
2061 return NVME_SUCCESS;
2062 }
2063
2064 /* fallthrough */
2065
2066 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
2067 if (flags & NVME_ZRM_AUTO) {
2068 return NVME_SUCCESS;
2069 }
2070
2071 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_EXPLICITLY_OPEN);
2072
2073 /* fallthrough */
2074
2075 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
2076 if (flags & NVME_ZRM_ZRWA) {
2077 ns->zns.numzrwa--;
2078
2079 zone->d.za |= NVME_ZA_ZRWA_VALID;
2080 }
2081
2082 return NVME_SUCCESS;
2083
2084 default:
2085 return NVME_ZONE_INVAL_TRANSITION;
2086 }
2087 }
2088
2089 static inline uint16_t nvme_zrm_auto(NvmeCtrl *n, NvmeNamespace *ns,
2090 NvmeZone *zone)
2091 {
2092 return nvme_zrm_open_flags(n, ns, zone, NVME_ZRM_AUTO);
2093 }
2094
2095 static void nvme_advance_zone_wp(NvmeNamespace *ns, NvmeZone *zone,
2096 uint32_t nlb)
2097 {
2098 zone->d.wp += nlb;
2099
2100 if (zone->d.wp == nvme_zone_wr_boundary(zone)) {
2101 nvme_zrm_finish(ns, zone);
2102 }
2103 }
2104
2105 static void nvme_zoned_zrwa_implicit_flush(NvmeNamespace *ns, NvmeZone *zone,
2106 uint32_t nlbc)
2107 {
2108 uint16_t nzrwafgs = DIV_ROUND_UP(nlbc, ns->zns.zrwafg);
2109
2110 nlbc = nzrwafgs * ns->zns.zrwafg;
2111
2112 trace_pci_nvme_zoned_zrwa_implicit_flush(zone->d.zslba, nlbc);
2113
2114 zone->w_ptr += nlbc;
2115
2116 nvme_advance_zone_wp(ns, zone, nlbc);
2117 }
2118
2119 static void nvme_finalize_zoned_write(NvmeNamespace *ns, NvmeRequest *req)
2120 {
2121 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2122 NvmeZone *zone;
2123 uint64_t slba;
2124 uint32_t nlb;
2125
2126 slba = le64_to_cpu(rw->slba);
2127 nlb = le16_to_cpu(rw->nlb) + 1;
2128 zone = nvme_get_zone_by_slba(ns, slba);
2129 assert(zone);
2130
2131 if (zone->d.za & NVME_ZA_ZRWA_VALID) {
2132 uint64_t ezrwa = zone->w_ptr + ns->zns.zrwas - 1;
2133 uint64_t elba = slba + nlb - 1;
2134
2135 if (elba > ezrwa) {
2136 nvme_zoned_zrwa_implicit_flush(ns, zone, elba - ezrwa);
2137 }
2138
2139 return;
2140 }
2141
2142 nvme_advance_zone_wp(ns, zone, nlb);
2143 }
2144
2145 static inline bool nvme_is_write(NvmeRequest *req)
2146 {
2147 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2148
2149 return rw->opcode == NVME_CMD_WRITE ||
2150 rw->opcode == NVME_CMD_ZONE_APPEND ||
2151 rw->opcode == NVME_CMD_WRITE_ZEROES;
2152 }
2153
2154 static void nvme_misc_cb(void *opaque, int ret)
2155 {
2156 NvmeRequest *req = opaque;
2157 uint16_t cid = nvme_cid(req);
2158
2159 trace_pci_nvme_misc_cb(cid);
2160
2161 if (ret) {
2162 if (!req->status) {
2163 req->status = NVME_INTERNAL_DEV_ERROR;
2164 }
2165
2166 trace_pci_nvme_err_aio(cid, strerror(-ret), req->status);
2167 }
2168
2169 nvme_enqueue_req_completion(nvme_cq(req), req);
2170 }
2171
2172 void nvme_rw_complete_cb(void *opaque, int ret)
2173 {
2174 NvmeRequest *req = opaque;
2175 NvmeNamespace *ns = req->ns;
2176 BlockBackend *blk = ns->blkconf.blk;
2177 BlockAcctCookie *acct = &req->acct;
2178 BlockAcctStats *stats = blk_get_stats(blk);
2179
2180 trace_pci_nvme_rw_complete_cb(nvme_cid(req), blk_name(blk));
2181
2182 if (ret) {
2183 Error *err = NULL;
2184
2185 block_acct_failed(stats, acct);
2186
2187 switch (req->cmd.opcode) {
2188 case NVME_CMD_READ:
2189 req->status = NVME_UNRECOVERED_READ;
2190 break;
2191
2192 case NVME_CMD_WRITE:
2193 case NVME_CMD_WRITE_ZEROES:
2194 case NVME_CMD_ZONE_APPEND:
2195 req->status = NVME_WRITE_FAULT;
2196 break;
2197
2198 default:
2199 req->status = NVME_INTERNAL_DEV_ERROR;
2200 break;
2201 }
2202
2203 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2204
2205 error_setg_errno(&err, -ret, "aio failed");
2206 error_report_err(err);
2207 } else {
2208 block_acct_done(stats, acct);
2209 }
2210
2211 if (ns->params.zoned && nvme_is_write(req)) {
2212 nvme_finalize_zoned_write(ns, req);
2213 }
2214
2215 nvme_enqueue_req_completion(nvme_cq(req), req);
2216 }
2217
2218 static void nvme_rw_cb(void *opaque, int ret)
2219 {
2220 NvmeRequest *req = opaque;
2221 NvmeNamespace *ns = req->ns;
2222
2223 BlockBackend *blk = ns->blkconf.blk;
2224
2225 trace_pci_nvme_rw_cb(nvme_cid(req), blk_name(blk));
2226
2227 if (ret) {
2228 goto out;
2229 }
2230
2231 if (ns->lbaf.ms) {
2232 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2233 uint64_t slba = le64_to_cpu(rw->slba);
2234 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
2235 uint64_t offset = nvme_moff(ns, slba);
2236
2237 if (req->cmd.opcode == NVME_CMD_WRITE_ZEROES) {
2238 size_t mlen = nvme_m2b(ns, nlb);
2239
2240 req->aiocb = blk_aio_pwrite_zeroes(blk, offset, mlen,
2241 BDRV_REQ_MAY_UNMAP,
2242 nvme_rw_complete_cb, req);
2243 return;
2244 }
2245
2246 if (nvme_ns_ext(ns) || req->cmd.mptr) {
2247 uint16_t status;
2248
2249 nvme_sg_unmap(&req->sg);
2250 status = nvme_map_mdata(nvme_ctrl(req), nlb, req);
2251 if (status) {
2252 ret = -EFAULT;
2253 goto out;
2254 }
2255
2256 if (req->cmd.opcode == NVME_CMD_READ) {
2257 return nvme_blk_read(blk, offset, 1, nvme_rw_complete_cb, req);
2258 }
2259
2260 return nvme_blk_write(blk, offset, 1, nvme_rw_complete_cb, req);
2261 }
2262 }
2263
2264 out:
2265 nvme_rw_complete_cb(req, ret);
2266 }
2267
2268 static void nvme_verify_cb(void *opaque, int ret)
2269 {
2270 NvmeBounceContext *ctx = opaque;
2271 NvmeRequest *req = ctx->req;
2272 NvmeNamespace *ns = req->ns;
2273 BlockBackend *blk = ns->blkconf.blk;
2274 BlockAcctCookie *acct = &req->acct;
2275 BlockAcctStats *stats = blk_get_stats(blk);
2276 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2277 uint64_t slba = le64_to_cpu(rw->slba);
2278 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2279 uint16_t apptag = le16_to_cpu(rw->apptag);
2280 uint16_t appmask = le16_to_cpu(rw->appmask);
2281 uint64_t reftag = le32_to_cpu(rw->reftag);
2282 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2283 uint16_t status;
2284
2285 reftag |= cdw3 << 32;
2286
2287 trace_pci_nvme_verify_cb(nvme_cid(req), prinfo, apptag, appmask, reftag);
2288
2289 if (ret) {
2290 block_acct_failed(stats, acct);
2291 req->status = NVME_UNRECOVERED_READ;
2292
2293 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2294
2295 goto out;
2296 }
2297
2298 block_acct_done(stats, acct);
2299
2300 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2301 status = nvme_dif_mangle_mdata(ns, ctx->mdata.bounce,
2302 ctx->mdata.iov.size, slba);
2303 if (status) {
2304 req->status = status;
2305 goto out;
2306 }
2307
2308 req->status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2309 ctx->mdata.bounce, ctx->mdata.iov.size,
2310 prinfo, slba, apptag, appmask, &reftag);
2311 }
2312
2313 out:
2314 qemu_iovec_destroy(&ctx->data.iov);
2315 g_free(ctx->data.bounce);
2316
2317 qemu_iovec_destroy(&ctx->mdata.iov);
2318 g_free(ctx->mdata.bounce);
2319
2320 g_free(ctx);
2321
2322 nvme_enqueue_req_completion(nvme_cq(req), req);
2323 }
2324
2325
2326 static void nvme_verify_mdata_in_cb(void *opaque, int ret)
2327 {
2328 NvmeBounceContext *ctx = opaque;
2329 NvmeRequest *req = ctx->req;
2330 NvmeNamespace *ns = req->ns;
2331 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2332 uint64_t slba = le64_to_cpu(rw->slba);
2333 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2334 size_t mlen = nvme_m2b(ns, nlb);
2335 uint64_t offset = nvme_moff(ns, slba);
2336 BlockBackend *blk = ns->blkconf.blk;
2337
2338 trace_pci_nvme_verify_mdata_in_cb(nvme_cid(req), blk_name(blk));
2339
2340 if (ret) {
2341 goto out;
2342 }
2343
2344 ctx->mdata.bounce = g_malloc(mlen);
2345
2346 qemu_iovec_reset(&ctx->mdata.iov);
2347 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2348
2349 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2350 nvme_verify_cb, ctx);
2351 return;
2352
2353 out:
2354 nvme_verify_cb(ctx, ret);
2355 }
2356
2357 struct nvme_compare_ctx {
2358 struct {
2359 QEMUIOVector iov;
2360 uint8_t *bounce;
2361 } data;
2362
2363 struct {
2364 QEMUIOVector iov;
2365 uint8_t *bounce;
2366 } mdata;
2367 };
2368
2369 static void nvme_compare_mdata_cb(void *opaque, int ret)
2370 {
2371 NvmeRequest *req = opaque;
2372 NvmeNamespace *ns = req->ns;
2373 NvmeCtrl *n = nvme_ctrl(req);
2374 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2375 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2376 uint16_t apptag = le16_to_cpu(rw->apptag);
2377 uint16_t appmask = le16_to_cpu(rw->appmask);
2378 uint64_t reftag = le32_to_cpu(rw->reftag);
2379 uint64_t cdw3 = le32_to_cpu(rw->cdw3);
2380 struct nvme_compare_ctx *ctx = req->opaque;
2381 g_autofree uint8_t *buf = NULL;
2382 BlockBackend *blk = ns->blkconf.blk;
2383 BlockAcctCookie *acct = &req->acct;
2384 BlockAcctStats *stats = blk_get_stats(blk);
2385 uint16_t status = NVME_SUCCESS;
2386
2387 reftag |= cdw3 << 32;
2388
2389 trace_pci_nvme_compare_mdata_cb(nvme_cid(req));
2390
2391 if (ret) {
2392 block_acct_failed(stats, acct);
2393 req->status = NVME_UNRECOVERED_READ;
2394
2395 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2396
2397 goto out;
2398 }
2399
2400 buf = g_malloc(ctx->mdata.iov.size);
2401
2402 status = nvme_bounce_mdata(n, buf, ctx->mdata.iov.size,
2403 NVME_TX_DIRECTION_TO_DEVICE, req);
2404 if (status) {
2405 req->status = status;
2406 goto out;
2407 }
2408
2409 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2410 uint64_t slba = le64_to_cpu(rw->slba);
2411 uint8_t *bufp;
2412 uint8_t *mbufp = ctx->mdata.bounce;
2413 uint8_t *end = mbufp + ctx->mdata.iov.size;
2414 int16_t pil = 0;
2415
2416 status = nvme_dif_check(ns, ctx->data.bounce, ctx->data.iov.size,
2417 ctx->mdata.bounce, ctx->mdata.iov.size, prinfo,
2418 slba, apptag, appmask, &reftag);
2419 if (status) {
2420 req->status = status;
2421 goto out;
2422 }
2423
2424 /*
2425 * When formatted with protection information, do not compare the DIF
2426 * tuple.
2427 */
2428 if (!(ns->id_ns.dps & NVME_ID_NS_DPS_FIRST_EIGHT)) {
2429 pil = ns->lbaf.ms - nvme_pi_tuple_size(ns);
2430 }
2431
2432 for (bufp = buf; mbufp < end; bufp += ns->lbaf.ms, mbufp += ns->lbaf.ms) {
2433 if (memcmp(bufp + pil, mbufp + pil, ns->lbaf.ms - pil)) {
2434 req->status = NVME_CMP_FAILURE | NVME_DNR;
2435 goto out;
2436 }
2437 }
2438
2439 goto out;
2440 }
2441
2442 if (memcmp(buf, ctx->mdata.bounce, ctx->mdata.iov.size)) {
2443 req->status = NVME_CMP_FAILURE | NVME_DNR;
2444 goto out;
2445 }
2446
2447 block_acct_done(stats, acct);
2448
2449 out:
2450 qemu_iovec_destroy(&ctx->data.iov);
2451 g_free(ctx->data.bounce);
2452
2453 qemu_iovec_destroy(&ctx->mdata.iov);
2454 g_free(ctx->mdata.bounce);
2455
2456 g_free(ctx);
2457
2458 nvme_enqueue_req_completion(nvme_cq(req), req);
2459 }
2460
2461 static void nvme_compare_data_cb(void *opaque, int ret)
2462 {
2463 NvmeRequest *req = opaque;
2464 NvmeCtrl *n = nvme_ctrl(req);
2465 NvmeNamespace *ns = req->ns;
2466 BlockBackend *blk = ns->blkconf.blk;
2467 BlockAcctCookie *acct = &req->acct;
2468 BlockAcctStats *stats = blk_get_stats(blk);
2469
2470 struct nvme_compare_ctx *ctx = req->opaque;
2471 g_autofree uint8_t *buf = NULL;
2472 uint16_t status;
2473
2474 trace_pci_nvme_compare_data_cb(nvme_cid(req));
2475
2476 if (ret) {
2477 block_acct_failed(stats, acct);
2478 req->status = NVME_UNRECOVERED_READ;
2479
2480 trace_pci_nvme_err_aio(nvme_cid(req), strerror(-ret), req->status);
2481
2482 goto out;
2483 }
2484
2485 buf = g_malloc(ctx->data.iov.size);
2486
2487 status = nvme_bounce_data(n, buf, ctx->data.iov.size,
2488 NVME_TX_DIRECTION_TO_DEVICE, req);
2489 if (status) {
2490 req->status = status;
2491 goto out;
2492 }
2493
2494 if (memcmp(buf, ctx->data.bounce, ctx->data.iov.size)) {
2495 req->status = NVME_CMP_FAILURE | NVME_DNR;
2496 goto out;
2497 }
2498
2499 if (ns->lbaf.ms) {
2500 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2501 uint64_t slba = le64_to_cpu(rw->slba);
2502 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2503 size_t mlen = nvme_m2b(ns, nlb);
2504 uint64_t offset = nvme_moff(ns, slba);
2505
2506 ctx->mdata.bounce = g_malloc(mlen);
2507
2508 qemu_iovec_init(&ctx->mdata.iov, 1);
2509 qemu_iovec_add(&ctx->mdata.iov, ctx->mdata.bounce, mlen);
2510
2511 req->aiocb = blk_aio_preadv(blk, offset, &ctx->mdata.iov, 0,
2512 nvme_compare_mdata_cb, req);
2513 return;
2514 }
2515
2516 block_acct_done(stats, acct);
2517
2518 out:
2519 qemu_iovec_destroy(&ctx->data.iov);
2520 g_free(ctx->data.bounce);
2521 g_free(ctx);
2522
2523 nvme_enqueue_req_completion(nvme_cq(req), req);
2524 }
2525
2526 typedef struct NvmeDSMAIOCB {
2527 BlockAIOCB common;
2528 BlockAIOCB *aiocb;
2529 NvmeRequest *req;
2530 int ret;
2531
2532 NvmeDsmRange *range;
2533 unsigned int nr;
2534 unsigned int idx;
2535 } NvmeDSMAIOCB;
2536
2537 static void nvme_dsm_cancel(BlockAIOCB *aiocb)
2538 {
2539 NvmeDSMAIOCB *iocb = container_of(aiocb, NvmeDSMAIOCB, common);
2540
2541 /* break nvme_dsm_cb loop */
2542 iocb->idx = iocb->nr;
2543 iocb->ret = -ECANCELED;
2544
2545 if (iocb->aiocb) {
2546 blk_aio_cancel_async(iocb->aiocb);
2547 iocb->aiocb = NULL;
2548 } else {
2549 /*
2550 * We only reach this if nvme_dsm_cancel() has already been called or
2551 * the command ran to completion.
2552 */
2553 assert(iocb->idx == iocb->nr);
2554 }
2555 }
2556
2557 static const AIOCBInfo nvme_dsm_aiocb_info = {
2558 .aiocb_size = sizeof(NvmeDSMAIOCB),
2559 .cancel_async = nvme_dsm_cancel,
2560 };
2561
2562 static void nvme_dsm_cb(void *opaque, int ret);
2563
2564 static void nvme_dsm_md_cb(void *opaque, int ret)
2565 {
2566 NvmeDSMAIOCB *iocb = opaque;
2567 NvmeRequest *req = iocb->req;
2568 NvmeNamespace *ns = req->ns;
2569 NvmeDsmRange *range;
2570 uint64_t slba;
2571 uint32_t nlb;
2572
2573 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
2574 goto done;
2575 }
2576
2577 range = &iocb->range[iocb->idx - 1];
2578 slba = le64_to_cpu(range->slba);
2579 nlb = le32_to_cpu(range->nlb);
2580
2581 /*
2582 * Check that all block were discarded (zeroed); otherwise we do not zero
2583 * the metadata.
2584 */
2585
2586 ret = nvme_block_status_all(ns, slba, nlb, BDRV_BLOCK_ZERO);
2587 if (ret) {
2588 if (ret < 0) {
2589 goto done;
2590 }
2591
2592 nvme_dsm_cb(iocb, 0);
2593 return;
2594 }
2595
2596 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, nvme_moff(ns, slba),
2597 nvme_m2b(ns, nlb), BDRV_REQ_MAY_UNMAP,
2598 nvme_dsm_cb, iocb);
2599 return;
2600
2601 done:
2602 nvme_dsm_cb(iocb, ret);
2603 }
2604
2605 static void nvme_dsm_cb(void *opaque, int ret)
2606 {
2607 NvmeDSMAIOCB *iocb = opaque;
2608 NvmeRequest *req = iocb->req;
2609 NvmeCtrl *n = nvme_ctrl(req);
2610 NvmeNamespace *ns = req->ns;
2611 NvmeDsmRange *range;
2612 uint64_t slba;
2613 uint32_t nlb;
2614
2615 if (iocb->ret < 0) {
2616 goto done;
2617 } else if (ret < 0) {
2618 iocb->ret = ret;
2619 goto done;
2620 }
2621
2622 next:
2623 if (iocb->idx == iocb->nr) {
2624 goto done;
2625 }
2626
2627 range = &iocb->range[iocb->idx++];
2628 slba = le64_to_cpu(range->slba);
2629 nlb = le32_to_cpu(range->nlb);
2630
2631 trace_pci_nvme_dsm_deallocate(slba, nlb);
2632
2633 if (nlb > n->dmrsl) {
2634 trace_pci_nvme_dsm_single_range_limit_exceeded(nlb, n->dmrsl);
2635 goto next;
2636 }
2637
2638 if (nvme_check_bounds(ns, slba, nlb)) {
2639 trace_pci_nvme_err_invalid_lba_range(slba, nlb,
2640 ns->id_ns.nsze);
2641 goto next;
2642 }
2643
2644 iocb->aiocb = blk_aio_pdiscard(ns->blkconf.blk, nvme_l2b(ns, slba),
2645 nvme_l2b(ns, nlb),
2646 nvme_dsm_md_cb, iocb);
2647 return;
2648
2649 done:
2650 iocb->aiocb = NULL;
2651 iocb->common.cb(iocb->common.opaque, iocb->ret);
2652 g_free(iocb->range);
2653 qemu_aio_unref(iocb);
2654 }
2655
2656 static uint16_t nvme_dsm(NvmeCtrl *n, NvmeRequest *req)
2657 {
2658 NvmeNamespace *ns = req->ns;
2659 NvmeDsmCmd *dsm = (NvmeDsmCmd *) &req->cmd;
2660 uint32_t attr = le32_to_cpu(dsm->attributes);
2661 uint32_t nr = (le32_to_cpu(dsm->nr) & 0xff) + 1;
2662 uint16_t status = NVME_SUCCESS;
2663
2664 trace_pci_nvme_dsm(nr, attr);
2665
2666 if (attr & NVME_DSMGMT_AD) {
2667 NvmeDSMAIOCB *iocb = blk_aio_get(&nvme_dsm_aiocb_info, ns->blkconf.blk,
2668 nvme_misc_cb, req);
2669
2670 iocb->req = req;
2671 iocb->ret = 0;
2672 iocb->range = g_new(NvmeDsmRange, nr);
2673 iocb->nr = nr;
2674 iocb->idx = 0;
2675
2676 status = nvme_h2c(n, (uint8_t *)iocb->range, sizeof(NvmeDsmRange) * nr,
2677 req);
2678 if (status) {
2679 g_free(iocb->range);
2680 qemu_aio_unref(iocb);
2681
2682 return status;
2683 }
2684
2685 req->aiocb = &iocb->common;
2686 nvme_dsm_cb(iocb, 0);
2687
2688 return NVME_NO_COMPLETE;
2689 }
2690
2691 return status;
2692 }
2693
2694 static uint16_t nvme_verify(NvmeCtrl *n, NvmeRequest *req)
2695 {
2696 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
2697 NvmeNamespace *ns = req->ns;
2698 BlockBackend *blk = ns->blkconf.blk;
2699 uint64_t slba = le64_to_cpu(rw->slba);
2700 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
2701 size_t len = nvme_l2b(ns, nlb);
2702 size_t data_len = len;
2703 int64_t offset = nvme_l2b(ns, slba);
2704 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
2705 uint32_t reftag = le32_to_cpu(rw->reftag);
2706 NvmeBounceContext *ctx = NULL;
2707 uint16_t status;
2708
2709 trace_pci_nvme_verify(nvme_cid(req), nvme_nsid(ns), slba, nlb);
2710
2711 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
2712 status = nvme_check_prinfo(ns, prinfo, slba, reftag);
2713 if (status) {
2714 return status;
2715 }
2716
2717 if (prinfo & NVME_PRINFO_PRACT) {
2718 return NVME_INVALID_PROT_INFO | NVME_DNR;
2719 }
2720 }
2721
2722 if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
2723 data_len += nvme_m2b(ns, nlb);
2724 }
2725
2726 if (data_len > (n->page_size << n->params.vsl)) {
2727 return NVME_INVALID_FIELD | NVME_DNR;
2728 }
2729
2730 status = nvme_check_bounds(ns, slba, nlb);
2731 if (status) {
2732 return status;
2733 }
2734
2735 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
2736 status = nvme_check_dulbe(ns, slba, nlb);
2737 if (status) {
2738 return status;
2739 }
2740 }
2741
2742 ctx = g_new0(NvmeBounceContext, 1);
2743 ctx->req = req;
2744
2745 ctx->data.bounce = g_malloc(len);
2746
2747 qemu_iovec_init(&ctx->data.iov, 1);
2748 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, len);
2749
2750 block_acct_start(blk_get_stats(blk), &req->acct, ctx->data.iov.size,
2751 BLOCK_ACCT_READ);
2752
2753 req->aiocb = blk_aio_preadv(ns->blkconf.blk, offset, &ctx->data.iov, 0,
2754 nvme_verify_mdata_in_cb, ctx);
2755 return NVME_NO_COMPLETE;
2756 }
2757
2758 typedef struct NvmeCopyAIOCB {
2759 BlockAIOCB common;
2760 BlockAIOCB *aiocb;
2761 NvmeRequest *req;
2762 NvmeCtrl *n;
2763 int ret;
2764
2765 void *ranges;
2766 unsigned int format;
2767 int nr;
2768 int idx;
2769
2770 uint8_t *bounce;
2771 QEMUIOVector iov;
2772 struct {
2773 BlockAcctCookie read;
2774 BlockAcctCookie write;
2775 } acct;
2776
2777 uint64_t reftag;
2778 uint64_t slba;
2779
2780 NvmeZone *zone;
2781 NvmeNamespace *sns;
2782 uint32_t tcl;
2783 } NvmeCopyAIOCB;
2784
2785 static void nvme_copy_cancel(BlockAIOCB *aiocb)
2786 {
2787 NvmeCopyAIOCB *iocb = container_of(aiocb, NvmeCopyAIOCB, common);
2788
2789 iocb->ret = -ECANCELED;
2790
2791 if (iocb->aiocb) {
2792 blk_aio_cancel_async(iocb->aiocb);
2793 iocb->aiocb = NULL;
2794 }
2795 }
2796
2797 static const AIOCBInfo nvme_copy_aiocb_info = {
2798 .aiocb_size = sizeof(NvmeCopyAIOCB),
2799 .cancel_async = nvme_copy_cancel,
2800 };
2801
2802 static void nvme_copy_done(NvmeCopyAIOCB *iocb)
2803 {
2804 NvmeRequest *req = iocb->req;
2805 NvmeNamespace *ns = req->ns;
2806 BlockAcctStats *stats = blk_get_stats(ns->blkconf.blk);
2807
2808 if (iocb->idx != iocb->nr) {
2809 req->cqe.result = cpu_to_le32(iocb->idx);
2810 }
2811
2812 qemu_iovec_destroy(&iocb->iov);
2813 g_free(iocb->bounce);
2814 g_free(iocb->ranges);
2815
2816 if (iocb->ret < 0) {
2817 block_acct_failed(stats, &iocb->acct.read);
2818 block_acct_failed(stats, &iocb->acct.write);
2819 } else {
2820 block_acct_done(stats, &iocb->acct.read);
2821 block_acct_done(stats, &iocb->acct.write);
2822 }
2823
2824 iocb->common.cb(iocb->common.opaque, iocb->ret);
2825 qemu_aio_unref(iocb);
2826 }
2827
2828 static void nvme_do_copy(NvmeCopyAIOCB *iocb);
2829
2830 static void nvme_copy_source_range_parse_format0_2(void *ranges,
2831 int idx, uint64_t *slba,
2832 uint32_t *nlb,
2833 uint32_t *snsid,
2834 uint16_t *apptag,
2835 uint16_t *appmask,
2836 uint64_t *reftag)
2837 {
2838 NvmeCopySourceRangeFormat0_2 *_ranges = ranges;
2839
2840 if (snsid) {
2841 *snsid = le32_to_cpu(_ranges[idx].sparams);
2842 }
2843
2844 if (slba) {
2845 *slba = le64_to_cpu(_ranges[idx].slba);
2846 }
2847
2848 if (nlb) {
2849 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2850 }
2851
2852 if (apptag) {
2853 *apptag = le16_to_cpu(_ranges[idx].apptag);
2854 }
2855
2856 if (appmask) {
2857 *appmask = le16_to_cpu(_ranges[idx].appmask);
2858 }
2859
2860 if (reftag) {
2861 *reftag = le32_to_cpu(_ranges[idx].reftag);
2862 }
2863 }
2864
2865 static void nvme_copy_source_range_parse_format1_3(void *ranges, int idx,
2866 uint64_t *slba,
2867 uint32_t *nlb,
2868 uint32_t *snsid,
2869 uint16_t *apptag,
2870 uint16_t *appmask,
2871 uint64_t *reftag)
2872 {
2873 NvmeCopySourceRangeFormat1_3 *_ranges = ranges;
2874
2875 if (snsid) {
2876 *snsid = le32_to_cpu(_ranges[idx].sparams);
2877 }
2878
2879 if (slba) {
2880 *slba = le64_to_cpu(_ranges[idx].slba);
2881 }
2882
2883 if (nlb) {
2884 *nlb = le16_to_cpu(_ranges[idx].nlb) + 1;
2885 }
2886
2887 if (apptag) {
2888 *apptag = le16_to_cpu(_ranges[idx].apptag);
2889 }
2890
2891 if (appmask) {
2892 *appmask = le16_to_cpu(_ranges[idx].appmask);
2893 }
2894
2895 if (reftag) {
2896 *reftag = 0;
2897
2898 *reftag |= (uint64_t)_ranges[idx].sr[4] << 40;
2899 *reftag |= (uint64_t)_ranges[idx].sr[5] << 32;
2900 *reftag |= (uint64_t)_ranges[idx].sr[6] << 24;
2901 *reftag |= (uint64_t)_ranges[idx].sr[7] << 16;
2902 *reftag |= (uint64_t)_ranges[idx].sr[8] << 8;
2903 *reftag |= (uint64_t)_ranges[idx].sr[9];
2904 }
2905 }
2906
2907 static void nvme_copy_source_range_parse(void *ranges, int idx, uint8_t format,
2908 uint64_t *slba, uint32_t *nlb,
2909 uint32_t *snsid, uint16_t *apptag,
2910 uint16_t *appmask, uint64_t *reftag)
2911 {
2912 switch (format) {
2913 case NVME_COPY_FORMAT_0:
2914 case NVME_COPY_FORMAT_2:
2915 nvme_copy_source_range_parse_format0_2(ranges, idx, slba, nlb, snsid,
2916 apptag, appmask, reftag);
2917 break;
2918
2919 case NVME_COPY_FORMAT_1:
2920 case NVME_COPY_FORMAT_3:
2921 nvme_copy_source_range_parse_format1_3(ranges, idx, slba, nlb, snsid,
2922 apptag, appmask, reftag);
2923 break;
2924
2925 default:
2926 abort();
2927 }
2928 }
2929
2930 static inline uint16_t nvme_check_copy_mcl(NvmeNamespace *ns,
2931 NvmeCopyAIOCB *iocb, uint16_t nr)
2932 {
2933 uint32_t copy_len = 0;
2934
2935 for (int idx = 0; idx < nr; idx++) {
2936 uint32_t nlb;
2937 nvme_copy_source_range_parse(iocb->ranges, idx, iocb->format, NULL,
2938 &nlb, NULL, NULL, NULL, NULL);
2939 copy_len += nlb;
2940 }
2941 iocb->tcl = copy_len;
2942 if (copy_len > ns->id_ns.mcl) {
2943 return NVME_CMD_SIZE_LIMIT | NVME_DNR;
2944 }
2945
2946 return NVME_SUCCESS;
2947 }
2948
2949 static void nvme_copy_out_completed_cb(void *opaque, int ret)
2950 {
2951 NvmeCopyAIOCB *iocb = opaque;
2952 NvmeRequest *req = iocb->req;
2953 NvmeNamespace *dns = req->ns;
2954 uint32_t nlb;
2955
2956 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2957 &nlb, NULL, NULL, NULL, NULL);
2958
2959 if (ret < 0) {
2960 iocb->ret = ret;
2961 req->status = NVME_WRITE_FAULT;
2962 goto out;
2963 } else if (iocb->ret < 0) {
2964 goto out;
2965 }
2966
2967 if (dns->params.zoned) {
2968 nvme_advance_zone_wp(dns, iocb->zone, nlb);
2969 }
2970
2971 iocb->idx++;
2972 iocb->slba += nlb;
2973 out:
2974 nvme_do_copy(iocb);
2975 }
2976
2977 static void nvme_copy_out_cb(void *opaque, int ret)
2978 {
2979 NvmeCopyAIOCB *iocb = opaque;
2980 NvmeRequest *req = iocb->req;
2981 NvmeNamespace *dns = req->ns;
2982 uint32_t nlb;
2983 size_t mlen;
2984 uint8_t *mbounce;
2985
2986 if (ret < 0 || iocb->ret < 0 || !dns->lbaf.ms) {
2987 goto out;
2988 }
2989
2990 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, NULL,
2991 &nlb, NULL, NULL, NULL, NULL);
2992
2993 mlen = nvme_m2b(dns, nlb);
2994 mbounce = iocb->bounce + nvme_l2b(dns, nlb);
2995
2996 qemu_iovec_reset(&iocb->iov);
2997 qemu_iovec_add(&iocb->iov, mbounce, mlen);
2998
2999 iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_moff(dns, iocb->slba),
3000 &iocb->iov, 0, nvme_copy_out_completed_cb,
3001 iocb);
3002
3003 return;
3004
3005 out:
3006 nvme_copy_out_completed_cb(iocb, ret);
3007 }
3008
3009 static void nvme_copy_in_completed_cb(void *opaque, int ret)
3010 {
3011 NvmeCopyAIOCB *iocb = opaque;
3012 NvmeRequest *req = iocb->req;
3013 NvmeNamespace *sns = iocb->sns;
3014 NvmeNamespace *dns = req->ns;
3015 NvmeCopyCmd *copy = NULL;
3016 uint8_t *mbounce = NULL;
3017 uint32_t nlb;
3018 uint64_t slba;
3019 uint16_t apptag, appmask;
3020 uint64_t reftag;
3021 size_t len, mlen;
3022 uint16_t status;
3023
3024 if (ret < 0) {
3025 iocb->ret = ret;
3026 req->status = NVME_UNRECOVERED_READ;
3027 goto out;
3028 } else if (iocb->ret < 0) {
3029 goto out;
3030 }
3031
3032 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3033 &nlb, NULL, &apptag, &appmask, &reftag);
3034
3035 trace_pci_nvme_copy_out(iocb->slba, nlb);
3036
3037 len = nvme_l2b(sns, nlb);
3038
3039 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps)) {
3040 copy = (NvmeCopyCmd *)&req->cmd;
3041
3042 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3043
3044 mlen = nvme_m2b(sns, nlb);
3045 mbounce = iocb->bounce + nvme_l2b(sns, nlb);
3046
3047 status = nvme_dif_mangle_mdata(sns, mbounce, mlen, slba);
3048 if (status) {
3049 goto invalid;
3050 }
3051 status = nvme_dif_check(sns, iocb->bounce, len, mbounce, mlen, prinfor,
3052 slba, apptag, appmask, &reftag);
3053 if (status) {
3054 goto invalid;
3055 }
3056 }
3057
3058 if (NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3059 copy = (NvmeCopyCmd *)&req->cmd;
3060 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3061
3062 mlen = nvme_m2b(dns, nlb);
3063 mbounce = iocb->bounce + nvme_l2b(dns, nlb);
3064
3065 apptag = le16_to_cpu(copy->apptag);
3066 appmask = le16_to_cpu(copy->appmask);
3067
3068 if (prinfow & NVME_PRINFO_PRACT) {
3069 status = nvme_check_prinfo(dns, prinfow, iocb->slba, iocb->reftag);
3070 if (status) {
3071 goto invalid;
3072 }
3073
3074 nvme_dif_pract_generate_dif(dns, iocb->bounce, len, mbounce, mlen,
3075 apptag, &iocb->reftag);
3076 } else {
3077 status = nvme_dif_check(dns, iocb->bounce, len, mbounce, mlen,
3078 prinfow, iocb->slba, apptag, appmask,
3079 &iocb->reftag);
3080 if (status) {
3081 goto invalid;
3082 }
3083 }
3084 }
3085
3086 status = nvme_check_bounds(dns, iocb->slba, nlb);
3087 if (status) {
3088 goto invalid;
3089 }
3090
3091 if (dns->params.zoned) {
3092 status = nvme_check_zone_write(dns, iocb->zone, iocb->slba, nlb);
3093 if (status) {
3094 goto invalid;
3095 }
3096
3097 if (!(iocb->zone->d.za & NVME_ZA_ZRWA_VALID)) {
3098 iocb->zone->w_ptr += nlb;
3099 }
3100 }
3101
3102 qemu_iovec_reset(&iocb->iov);
3103 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3104
3105 block_acct_start(blk_get_stats(dns->blkconf.blk), &iocb->acct.write, 0,
3106 BLOCK_ACCT_WRITE);
3107
3108 iocb->aiocb = blk_aio_pwritev(dns->blkconf.blk, nvme_l2b(dns, iocb->slba),
3109 &iocb->iov, 0, nvme_copy_out_cb, iocb);
3110
3111 return;
3112
3113 invalid:
3114 req->status = status;
3115 iocb->ret = -1;
3116 out:
3117 nvme_do_copy(iocb);
3118 }
3119
3120 static void nvme_copy_in_cb(void *opaque, int ret)
3121 {
3122 NvmeCopyAIOCB *iocb = opaque;
3123 NvmeNamespace *sns = iocb->sns;
3124 uint64_t slba;
3125 uint32_t nlb;
3126
3127 if (ret < 0 || iocb->ret < 0 || !sns->lbaf.ms) {
3128 goto out;
3129 }
3130
3131 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format, &slba,
3132 &nlb, NULL, NULL, NULL, NULL);
3133
3134 qemu_iovec_reset(&iocb->iov);
3135 qemu_iovec_add(&iocb->iov, iocb->bounce + nvme_l2b(sns, nlb),
3136 nvme_m2b(sns, nlb));
3137
3138 iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_moff(sns, slba),
3139 &iocb->iov, 0, nvme_copy_in_completed_cb,
3140 iocb);
3141 return;
3142
3143 out:
3144 nvme_copy_in_completed_cb(iocb, ret);
3145 }
3146
3147 static inline bool nvme_csi_supports_copy(uint8_t csi)
3148 {
3149 return csi == NVME_CSI_NVM || csi == NVME_CSI_ZONED;
3150 }
3151
3152 static inline bool nvme_copy_ns_format_match(NvmeNamespace *sns,
3153 NvmeNamespace *dns)
3154 {
3155 return sns->lbaf.ds == dns->lbaf.ds && sns->lbaf.ms == dns->lbaf.ms;
3156 }
3157
3158 static bool nvme_copy_matching_ns_format(NvmeNamespace *sns, NvmeNamespace *dns,
3159 bool pi_enable)
3160 {
3161 if (!nvme_csi_supports_copy(sns->csi) ||
3162 !nvme_csi_supports_copy(dns->csi)) {
3163 return false;
3164 }
3165
3166 if (!pi_enable && !nvme_copy_ns_format_match(sns, dns)) {
3167 return false;
3168 }
3169
3170 if (pi_enable && (!nvme_copy_ns_format_match(sns, dns) ||
3171 sns->id_ns.dps != dns->id_ns.dps)) {
3172 return false;
3173 }
3174
3175 return true;
3176 }
3177
3178 static inline bool nvme_copy_corresp_pi_match(NvmeNamespace *sns,
3179 NvmeNamespace *dns)
3180 {
3181 return sns->lbaf.ms == 0 &&
3182 ((dns->lbaf.ms == 8 && dns->pif == 0) ||
3183 (dns->lbaf.ms == 16 && dns->pif == 1));
3184 }
3185
3186 static bool nvme_copy_corresp_pi_format(NvmeNamespace *sns, NvmeNamespace *dns,
3187 bool sns_pi_en)
3188 {
3189 if (!nvme_csi_supports_copy(sns->csi) ||
3190 !nvme_csi_supports_copy(dns->csi)) {
3191 return false;
3192 }
3193
3194 if (!sns_pi_en && !nvme_copy_corresp_pi_match(sns, dns)) {
3195 return false;
3196 }
3197
3198 if (sns_pi_en && !nvme_copy_corresp_pi_match(dns, sns)) {
3199 return false;
3200 }
3201
3202 return true;
3203 }
3204
3205 static void nvme_do_copy(NvmeCopyAIOCB *iocb)
3206 {
3207 NvmeRequest *req = iocb->req;
3208 NvmeNamespace *sns;
3209 NvmeNamespace *dns = req->ns;
3210 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3211 uint16_t prinfor = ((copy->control[0] >> 4) & 0xf);
3212 uint16_t prinfow = ((copy->control[2] >> 2) & 0xf);
3213 uint64_t slba;
3214 uint32_t nlb;
3215 size_t len, blen;
3216 uint16_t status;
3217 uint32_t dnsid = le32_to_cpu(req->cmd.nsid);
3218 uint32_t snsid = dnsid;
3219
3220 if (iocb->ret < 0) {
3221 goto done;
3222 }
3223
3224 if (iocb->idx == iocb->nr) {
3225 goto done;
3226 }
3227
3228 if (iocb->format == 2 || iocb->format == 3) {
3229 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3230 &slba, &nlb, &snsid, NULL, NULL, NULL);
3231 if (snsid != dnsid) {
3232 if (snsid == NVME_NSID_BROADCAST ||
3233 !nvme_nsid_valid(iocb->n, snsid)) {
3234 status = NVME_INVALID_NSID | NVME_DNR;
3235 goto invalid;
3236 }
3237 iocb->sns = nvme_ns(iocb->n, snsid);
3238 if (unlikely(!iocb->sns)) {
3239 status = NVME_INVALID_FIELD | NVME_DNR;
3240 goto invalid;
3241 }
3242 } else {
3243 if (((slba + nlb) > iocb->slba) &&
3244 ((slba + nlb) < (iocb->slba + iocb->tcl))) {
3245 status = NVME_CMD_OVERLAP_IO_RANGE | NVME_DNR;
3246 goto invalid;
3247 }
3248 }
3249 } else {
3250 nvme_copy_source_range_parse(iocb->ranges, iocb->idx, iocb->format,
3251 &slba, &nlb, NULL, NULL, NULL, NULL);
3252 }
3253
3254 sns = iocb->sns;
3255 if ((snsid == dnsid) && NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3256 ((prinfor & NVME_PRINFO_PRACT) != (prinfow & NVME_PRINFO_PRACT))) {
3257 status = NVME_INVALID_FIELD | NVME_DNR;
3258 goto invalid;
3259 } else if (snsid != dnsid) {
3260 if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3261 !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3262 if (!nvme_copy_matching_ns_format(sns, dns, false)) {
3263 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3264 goto invalid;
3265 }
3266 }
3267 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3268 NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3269 if ((prinfor & NVME_PRINFO_PRACT) !=
3270 (prinfow & NVME_PRINFO_PRACT)) {
3271 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3272 goto invalid;
3273 } else {
3274 if (!nvme_copy_matching_ns_format(sns, dns, true)) {
3275 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3276 goto invalid;
3277 }
3278 }
3279 }
3280
3281 if (!NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3282 NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3283 if (!(prinfow & NVME_PRINFO_PRACT)) {
3284 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3285 goto invalid;
3286 } else {
3287 if (!nvme_copy_corresp_pi_format(sns, dns, false)) {
3288 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3289 goto invalid;
3290 }
3291 }
3292 }
3293
3294 if (NVME_ID_NS_DPS_TYPE(sns->id_ns.dps) &&
3295 !NVME_ID_NS_DPS_TYPE(dns->id_ns.dps)) {
3296 if (!(prinfor & NVME_PRINFO_PRACT)) {
3297 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3298 goto invalid;
3299 } else {
3300 if (!nvme_copy_corresp_pi_format(sns, dns, true)) {
3301 status = NVME_CMD_INCOMP_NS_OR_FMT | NVME_DNR;
3302 goto invalid;
3303 }
3304 }
3305 }
3306 }
3307 len = nvme_l2b(sns, nlb);
3308
3309 trace_pci_nvme_copy_source_range(slba, nlb);
3310
3311 if (nlb > le16_to_cpu(sns->id_ns.mssrl)) {
3312 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3313 goto invalid;
3314 }
3315
3316 status = nvme_check_bounds(sns, slba, nlb);
3317 if (status) {
3318 goto invalid;
3319 }
3320
3321 if (NVME_ERR_REC_DULBE(sns->features.err_rec)) {
3322 status = nvme_check_dulbe(sns, slba, nlb);
3323 if (status) {
3324 goto invalid;
3325 }
3326 }
3327
3328 if (sns->params.zoned) {
3329 status = nvme_check_zone_read(sns, slba, nlb);
3330 if (status) {
3331 goto invalid;
3332 }
3333 }
3334
3335 g_free(iocb->bounce);
3336 assert(g_size_checked_mul(&blen, le16_to_cpu(sns->id_ns.mssrl),
3337 sns->lbasz + MAX(sns->lbaf.ms, dns->lbaf.ms)));
3338
3339 iocb->bounce = g_malloc(blen);
3340
3341 qemu_iovec_reset(&iocb->iov);
3342 assert(len <= blen);
3343 qemu_iovec_add(&iocb->iov, iocb->bounce, len);
3344
3345 block_acct_start(blk_get_stats(sns->blkconf.blk), &iocb->acct.read, 0,
3346 BLOCK_ACCT_READ);
3347
3348 iocb->aiocb = blk_aio_preadv(sns->blkconf.blk, nvme_l2b(sns, slba),
3349 &iocb->iov, 0, nvme_copy_in_cb, iocb);
3350 return;
3351
3352 invalid:
3353 req->status = status;
3354 iocb->ret = -1;
3355 done:
3356 nvme_copy_done(iocb);
3357 }
3358
3359 static uint16_t nvme_copy(NvmeCtrl *n, NvmeRequest *req)
3360 {
3361 NvmeNamespace *ns = req->ns;
3362 NvmeCopyCmd *copy = (NvmeCopyCmd *)&req->cmd;
3363 NvmeCopyAIOCB *iocb = blk_aio_get(&nvme_copy_aiocb_info, ns->blkconf.blk,
3364 nvme_misc_cb, req);
3365 uint16_t nr = copy->nr + 1;
3366 uint8_t format = copy->control[0] & 0xf;
3367 size_t len = sizeof(NvmeCopySourceRangeFormat0_2);
3368
3369 uint16_t status;
3370
3371 trace_pci_nvme_copy(nvme_cid(req), nvme_nsid(ns), nr, format);
3372
3373 iocb->ranges = NULL;
3374 iocb->zone = NULL;
3375
3376 if (!(n->id_ctrl.ocfs & (1 << format)) ||
3377 ((format == 2 || format == 3) &&
3378 !(n->features.hbs.cdfe & (1 << format)))) {
3379 trace_pci_nvme_err_copy_invalid_format(format);
3380 status = NVME_INVALID_FIELD | NVME_DNR;
3381 goto invalid;
3382 }
3383
3384 if (nr > ns->id_ns.msrc + 1) {
3385 status = NVME_CMD_SIZE_LIMIT | NVME_DNR;
3386 goto invalid;
3387 }
3388
3389 if ((ns->pif == 0x0 && (format != 0x0 && format != 0x2)) ||
3390 (ns->pif != 0x0 && (format != 0x1 && format != 0x3))) {
3391 status = NVME_INVALID_FORMAT | NVME_DNR;
3392 goto invalid;
3393 }
3394
3395 if (ns->pif) {
3396 len = sizeof(NvmeCopySourceRangeFormat1_3);
3397 }
3398
3399 iocb->format = format;
3400 iocb->ranges = g_malloc_n(nr, len);
3401 status = nvme_h2c(n, (uint8_t *)iocb->ranges, len * nr, req);
3402 if (status) {
3403 goto invalid;
3404 }
3405
3406 iocb->slba = le64_to_cpu(copy->sdlba);
3407
3408 if (ns->params.zoned) {
3409 iocb->zone = nvme_get_zone_by_slba(ns, iocb->slba);
3410 if (!iocb->zone) {
3411 status = NVME_LBA_RANGE | NVME_DNR;
3412 goto invalid;
3413 }
3414
3415 status = nvme_zrm_auto(n, ns, iocb->zone);
3416 if (status) {
3417 goto invalid;
3418 }
3419 }
3420
3421 status = nvme_check_copy_mcl(ns, iocb, nr);
3422 if (status) {
3423 goto invalid;
3424 }
3425
3426 iocb->req = req;
3427 iocb->ret = 0;
3428 iocb->nr = nr;
3429 iocb->idx = 0;
3430 iocb->reftag = le32_to_cpu(copy->reftag);
3431 iocb->reftag |= (uint64_t)le32_to_cpu(copy->cdw3) << 32;
3432
3433 qemu_iovec_init(&iocb->iov, 1);
3434
3435 req->aiocb = &iocb->common;
3436 iocb->sns = req->ns;
3437 iocb->n = n;
3438 iocb->bounce = NULL;
3439 nvme_do_copy(iocb);
3440
3441 return NVME_NO_COMPLETE;
3442
3443 invalid:
3444 g_free(iocb->ranges);
3445 qemu_aio_unref(iocb);
3446 return status;
3447 }
3448
3449 static uint16_t nvme_compare(NvmeCtrl *n, NvmeRequest *req)
3450 {
3451 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3452 NvmeNamespace *ns = req->ns;
3453 BlockBackend *blk = ns->blkconf.blk;
3454 uint64_t slba = le64_to_cpu(rw->slba);
3455 uint32_t nlb = le16_to_cpu(rw->nlb) + 1;
3456 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3457 size_t data_len = nvme_l2b(ns, nlb);
3458 size_t len = data_len;
3459 int64_t offset = nvme_l2b(ns, slba);
3460 struct nvme_compare_ctx *ctx = NULL;
3461 uint16_t status;
3462
3463 trace_pci_nvme_compare(nvme_cid(req), nvme_nsid(ns), slba, nlb);
3464
3465 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps) && (prinfo & NVME_PRINFO_PRACT)) {
3466 return NVME_INVALID_PROT_INFO | NVME_DNR;
3467 }
3468
3469 if (nvme_ns_ext(ns)) {
3470 len += nvme_m2b(ns, nlb);
3471 }
3472
3473 if (NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt)) {
3474 status = nvme_check_mdts(n, data_len);
3475 } else {
3476 status = nvme_check_mdts(n, len);
3477 }
3478 if (status) {
3479 return status;
3480 }
3481
3482 status = nvme_check_bounds(ns, slba, nlb);
3483 if (status) {
3484 return status;
3485 }
3486
3487 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3488 status = nvme_check_dulbe(ns, slba, nlb);
3489 if (status) {
3490 return status;
3491 }
3492 }
3493
3494 status = nvme_map_dptr(n, &req->sg, len, &req->cmd);
3495 if (status) {
3496 return status;
3497 }
3498
3499 ctx = g_new(struct nvme_compare_ctx, 1);
3500 ctx->data.bounce = g_malloc(data_len);
3501
3502 req->opaque = ctx;
3503
3504 qemu_iovec_init(&ctx->data.iov, 1);
3505 qemu_iovec_add(&ctx->data.iov, ctx->data.bounce, data_len);
3506
3507 block_acct_start(blk_get_stats(blk), &req->acct, data_len,
3508 BLOCK_ACCT_READ);
3509 req->aiocb = blk_aio_preadv(blk, offset, &ctx->data.iov, 0,
3510 nvme_compare_data_cb, req);
3511
3512 return NVME_NO_COMPLETE;
3513 }
3514
3515 typedef struct NvmeFlushAIOCB {
3516 BlockAIOCB common;
3517 BlockAIOCB *aiocb;
3518 NvmeRequest *req;
3519 int ret;
3520
3521 NvmeNamespace *ns;
3522 uint32_t nsid;
3523 bool broadcast;
3524 } NvmeFlushAIOCB;
3525
3526 static void nvme_flush_cancel(BlockAIOCB *acb)
3527 {
3528 NvmeFlushAIOCB *iocb = container_of(acb, NvmeFlushAIOCB, common);
3529
3530 iocb->ret = -ECANCELED;
3531
3532 if (iocb->aiocb) {
3533 blk_aio_cancel_async(iocb->aiocb);
3534 iocb->aiocb = NULL;
3535 }
3536 }
3537
3538 static const AIOCBInfo nvme_flush_aiocb_info = {
3539 .aiocb_size = sizeof(NvmeFlushAIOCB),
3540 .cancel_async = nvme_flush_cancel,
3541 };
3542
3543 static void nvme_do_flush(NvmeFlushAIOCB *iocb);
3544
3545 static void nvme_flush_ns_cb(void *opaque, int ret)
3546 {
3547 NvmeFlushAIOCB *iocb = opaque;
3548 NvmeNamespace *ns = iocb->ns;
3549
3550 if (ret < 0) {
3551 iocb->ret = ret;
3552 iocb->req->status = NVME_WRITE_FAULT;
3553 goto out;
3554 } else if (iocb->ret < 0) {
3555 goto out;
3556 }
3557
3558 if (ns) {
3559 trace_pci_nvme_flush_ns(iocb->nsid);
3560
3561 iocb->ns = NULL;
3562 iocb->aiocb = blk_aio_flush(ns->blkconf.blk, nvme_flush_ns_cb, iocb);
3563 return;
3564 }
3565
3566 out:
3567 nvme_do_flush(iocb);
3568 }
3569
3570 static void nvme_do_flush(NvmeFlushAIOCB *iocb)
3571 {
3572 NvmeRequest *req = iocb->req;
3573 NvmeCtrl *n = nvme_ctrl(req);
3574 int i;
3575
3576 if (iocb->ret < 0) {
3577 goto done;
3578 }
3579
3580 if (iocb->broadcast) {
3581 for (i = iocb->nsid + 1; i <= NVME_MAX_NAMESPACES; i++) {
3582 iocb->ns = nvme_ns(n, i);
3583 if (iocb->ns) {
3584 iocb->nsid = i;
3585 break;
3586 }
3587 }
3588 }
3589
3590 if (!iocb->ns) {
3591 goto done;
3592 }
3593
3594 nvme_flush_ns_cb(iocb, 0);
3595 return;
3596
3597 done:
3598 iocb->common.cb(iocb->common.opaque, iocb->ret);
3599 qemu_aio_unref(iocb);
3600 }
3601
3602 static uint16_t nvme_flush(NvmeCtrl *n, NvmeRequest *req)
3603 {
3604 NvmeFlushAIOCB *iocb;
3605 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
3606 uint16_t status;
3607
3608 iocb = qemu_aio_get(&nvme_flush_aiocb_info, NULL, nvme_misc_cb, req);
3609
3610 iocb->req = req;
3611 iocb->ret = 0;
3612 iocb->ns = NULL;
3613 iocb->nsid = 0;
3614 iocb->broadcast = (nsid == NVME_NSID_BROADCAST);
3615
3616 if (!iocb->broadcast) {
3617 if (!nvme_nsid_valid(n, nsid)) {
3618 status = NVME_INVALID_NSID | NVME_DNR;
3619 goto out;
3620 }
3621
3622 iocb->ns = nvme_ns(n, nsid);
3623 if (!iocb->ns) {
3624 status = NVME_INVALID_FIELD | NVME_DNR;
3625 goto out;
3626 }
3627
3628 iocb->nsid = nsid;
3629 }
3630
3631 req->aiocb = &iocb->common;
3632 nvme_do_flush(iocb);
3633
3634 return NVME_NO_COMPLETE;
3635
3636 out:
3637 qemu_aio_unref(iocb);
3638
3639 return status;
3640 }
3641
3642 static uint16_t nvme_read(NvmeCtrl *n, NvmeRequest *req)
3643 {
3644 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3645 NvmeNamespace *ns = req->ns;
3646 uint64_t slba = le64_to_cpu(rw->slba);
3647 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3648 uint8_t prinfo = NVME_RW_PRINFO(le16_to_cpu(rw->control));
3649 uint64_t data_size = nvme_l2b(ns, nlb);
3650 uint64_t mapped_size = data_size;
3651 uint64_t data_offset;
3652 BlockBackend *blk = ns->blkconf.blk;
3653 uint16_t status;
3654
3655 if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
3656 mapped_size += nvme_m2b(ns, nlb);
3657
3658 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3659 bool pract = prinfo & NVME_PRINFO_PRACT;
3660
3661 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3662 mapped_size = data_size;
3663 }
3664 }
3665 }
3666
3667 trace_pci_nvme_read(nvme_cid(req), nvme_nsid(ns), nlb, mapped_size, slba);
3668
3669 status = nvme_check_mdts(n, mapped_size);
3670 if (status) {
3671 goto invalid;
3672 }
3673
3674 status = nvme_check_bounds(ns, slba, nlb);
3675 if (status) {
3676 goto invalid;
3677 }
3678
3679 if (ns->params.zoned) {
3680 status = nvme_check_zone_read(ns, slba, nlb);
3681 if (status) {
3682 trace_pci_nvme_err_zone_read_not_ok(slba, nlb, status);
3683 goto invalid;
3684 }
3685 }
3686
3687 if (NVME_ERR_REC_DULBE(ns->features.err_rec)) {
3688 status = nvme_check_dulbe(ns, slba, nlb);
3689 if (status) {
3690 goto invalid;
3691 }
3692 }
3693
3694 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3695 return nvme_dif_rw(n, req);
3696 }
3697
3698 status = nvme_map_data(n, nlb, req);
3699 if (status) {
3700 goto invalid;
3701 }
3702
3703 data_offset = nvme_l2b(ns, slba);
3704
3705 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3706 BLOCK_ACCT_READ);
3707 nvme_blk_read(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3708 return NVME_NO_COMPLETE;
3709
3710 invalid:
3711 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
3712 return status | NVME_DNR;
3713 }
3714
3715 static void nvme_do_write_fdp(NvmeCtrl *n, NvmeRequest *req, uint64_t slba,
3716 uint32_t nlb)
3717 {
3718 NvmeNamespace *ns = req->ns;
3719 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3720 uint64_t data_size = nvme_l2b(ns, nlb);
3721 uint32_t dw12 = le32_to_cpu(req->cmd.cdw12);
3722 uint8_t dtype = (dw12 >> 20) & 0xf;
3723 uint16_t pid = le16_to_cpu(rw->dspec);
3724 uint16_t ph, rg, ruhid;
3725 NvmeReclaimUnit *ru;
3726
3727 if (dtype != NVME_DIRECTIVE_DATA_PLACEMENT ||
3728 !nvme_parse_pid(ns, pid, &ph, &rg)) {
3729 ph = 0;
3730 rg = 0;
3731 }
3732
3733 ruhid = ns->fdp.phs[ph];
3734 ru = &ns->endgrp->fdp.ruhs[ruhid].rus[rg];
3735
3736 nvme_fdp_stat_inc(&ns->endgrp->fdp.hbmw, data_size);
3737 nvme_fdp_stat_inc(&ns->endgrp->fdp.mbmw, data_size);
3738
3739 while (nlb) {
3740 if (nlb < ru->ruamw) {
3741 ru->ruamw -= nlb;
3742 break;
3743 }
3744
3745 nlb -= ru->ruamw;
3746 nvme_update_ruh(n, ns, pid);
3747 }
3748 }
3749
3750 static uint16_t nvme_do_write(NvmeCtrl *n, NvmeRequest *req, bool append,
3751 bool wrz)
3752 {
3753 NvmeRwCmd *rw = (NvmeRwCmd *)&req->cmd;
3754 NvmeNamespace *ns = req->ns;
3755 uint64_t slba = le64_to_cpu(rw->slba);
3756 uint32_t nlb = (uint32_t)le16_to_cpu(rw->nlb) + 1;
3757 uint16_t ctrl = le16_to_cpu(rw->control);
3758 uint8_t prinfo = NVME_RW_PRINFO(ctrl);
3759 uint64_t data_size = nvme_l2b(ns, nlb);
3760 uint64_t mapped_size = data_size;
3761 uint64_t data_offset;
3762 NvmeZone *zone;
3763 NvmeZonedResult *res = (NvmeZonedResult *)&req->cqe;
3764 BlockBackend *blk = ns->blkconf.blk;
3765 uint16_t status;
3766
3767 if (nvme_ns_ext(ns) && !(NVME_ID_CTRL_CTRATT_MEM(n->id_ctrl.ctratt))) {
3768 mapped_size += nvme_m2b(ns, nlb);
3769
3770 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3771 bool pract = prinfo & NVME_PRINFO_PRACT;
3772
3773 if (pract && ns->lbaf.ms == nvme_pi_tuple_size(ns)) {
3774 mapped_size -= nvme_m2b(ns, nlb);
3775 }
3776 }
3777 }
3778
3779 trace_pci_nvme_write(nvme_cid(req), nvme_io_opc_str(rw->opcode),
3780 nvme_nsid(ns), nlb, mapped_size, slba);
3781
3782 if (!wrz) {
3783 status = nvme_check_mdts(n, mapped_size);
3784 if (status) {
3785 goto invalid;
3786 }
3787 }
3788
3789 status = nvme_check_bounds(ns, slba, nlb);
3790 if (status) {
3791 goto invalid;
3792 }
3793
3794 if (ns->params.zoned) {
3795 zone = nvme_get_zone_by_slba(ns, slba);
3796 assert(zone);
3797
3798 if (append) {
3799 bool piremap = !!(ctrl & NVME_RW_PIREMAP);
3800
3801 if (unlikely(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3802 return NVME_INVALID_ZONE_OP | NVME_DNR;
3803 }
3804
3805 if (unlikely(slba != zone->d.zslba)) {
3806 trace_pci_nvme_err_append_not_at_start(slba, zone->d.zslba);
3807 status = NVME_INVALID_FIELD;
3808 goto invalid;
3809 }
3810
3811 if (n->params.zasl &&
3812 data_size > (uint64_t)n->page_size << n->params.zasl) {
3813 trace_pci_nvme_err_zasl(data_size);
3814 return NVME_INVALID_FIELD | NVME_DNR;
3815 }
3816
3817 slba = zone->w_ptr;
3818 rw->slba = cpu_to_le64(slba);
3819 res->slba = cpu_to_le64(slba);
3820
3821 switch (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3822 case NVME_ID_NS_DPS_TYPE_1:
3823 if (!piremap) {
3824 return NVME_INVALID_PROT_INFO | NVME_DNR;
3825 }
3826
3827 /* fallthrough */
3828
3829 case NVME_ID_NS_DPS_TYPE_2:
3830 if (piremap) {
3831 uint32_t reftag = le32_to_cpu(rw->reftag);
3832 rw->reftag = cpu_to_le32(reftag + (slba - zone->d.zslba));
3833 }
3834
3835 break;
3836
3837 case NVME_ID_NS_DPS_TYPE_3:
3838 if (piremap) {
3839 return NVME_INVALID_PROT_INFO | NVME_DNR;
3840 }
3841
3842 break;
3843 }
3844 }
3845
3846 status = nvme_check_zone_write(ns, zone, slba, nlb);
3847 if (status) {
3848 goto invalid;
3849 }
3850
3851 status = nvme_zrm_auto(n, ns, zone);
3852 if (status) {
3853 goto invalid;
3854 }
3855
3856 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
3857 zone->w_ptr += nlb;
3858 }
3859 } else if (ns->endgrp && ns->endgrp->fdp.enabled) {
3860 nvme_do_write_fdp(n, req, slba, nlb);
3861 }
3862
3863 data_offset = nvme_l2b(ns, slba);
3864
3865 if (NVME_ID_NS_DPS_TYPE(ns->id_ns.dps)) {
3866 return nvme_dif_rw(n, req);
3867 }
3868
3869 if (!wrz) {
3870 status = nvme_map_data(n, nlb, req);
3871 if (status) {
3872 goto invalid;
3873 }
3874
3875 block_acct_start(blk_get_stats(blk), &req->acct, data_size,
3876 BLOCK_ACCT_WRITE);
3877 nvme_blk_write(blk, data_offset, BDRV_SECTOR_SIZE, nvme_rw_cb, req);
3878 } else {
3879 req->aiocb = blk_aio_pwrite_zeroes(blk, data_offset, data_size,
3880 BDRV_REQ_MAY_UNMAP, nvme_rw_cb,
3881 req);
3882 }
3883
3884 return NVME_NO_COMPLETE;
3885
3886 invalid:
3887 block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
3888 return status | NVME_DNR;
3889 }
3890
3891 static inline uint16_t nvme_write(NvmeCtrl *n, NvmeRequest *req)
3892 {
3893 return nvme_do_write(n, req, false, false);
3894 }
3895
3896 static inline uint16_t nvme_write_zeroes(NvmeCtrl *n, NvmeRequest *req)
3897 {
3898 return nvme_do_write(n, req, false, true);
3899 }
3900
3901 static inline uint16_t nvme_zone_append(NvmeCtrl *n, NvmeRequest *req)
3902 {
3903 return nvme_do_write(n, req, true, false);
3904 }
3905
3906 static uint16_t nvme_get_mgmt_zone_slba_idx(NvmeNamespace *ns, NvmeCmd *c,
3907 uint64_t *slba, uint32_t *zone_idx)
3908 {
3909 uint32_t dw10 = le32_to_cpu(c->cdw10);
3910 uint32_t dw11 = le32_to_cpu(c->cdw11);
3911
3912 if (!ns->params.zoned) {
3913 trace_pci_nvme_err_invalid_opc(c->opcode);
3914 return NVME_INVALID_OPCODE | NVME_DNR;
3915 }
3916
3917 *slba = ((uint64_t)dw11) << 32 | dw10;
3918 if (unlikely(*slba >= ns->id_ns.nsze)) {
3919 trace_pci_nvme_err_invalid_lba_range(*slba, 0, ns->id_ns.nsze);
3920 *slba = 0;
3921 return NVME_LBA_RANGE | NVME_DNR;
3922 }
3923
3924 *zone_idx = nvme_zone_idx(ns, *slba);
3925 assert(*zone_idx < ns->num_zones);
3926
3927 return NVME_SUCCESS;
3928 }
3929
3930 typedef uint16_t (*op_handler_t)(NvmeNamespace *, NvmeZone *, NvmeZoneState,
3931 NvmeRequest *);
3932
3933 enum NvmeZoneProcessingMask {
3934 NVME_PROC_CURRENT_ZONE = 0,
3935 NVME_PROC_OPENED_ZONES = 1 << 0,
3936 NVME_PROC_CLOSED_ZONES = 1 << 1,
3937 NVME_PROC_READ_ONLY_ZONES = 1 << 2,
3938 NVME_PROC_FULL_ZONES = 1 << 3,
3939 };
3940
3941 static uint16_t nvme_open_zone(NvmeNamespace *ns, NvmeZone *zone,
3942 NvmeZoneState state, NvmeRequest *req)
3943 {
3944 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
3945 int flags = 0;
3946
3947 if (cmd->zsflags & NVME_ZSFLAG_ZRWA_ALLOC) {
3948 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
3949
3950 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
3951 return NVME_INVALID_ZONE_OP | NVME_DNR;
3952 }
3953
3954 if (zone->w_ptr % ns->zns.zrwafg) {
3955 return NVME_NOZRWA | NVME_DNR;
3956 }
3957
3958 flags = NVME_ZRM_ZRWA;
3959 }
3960
3961 return nvme_zrm_open_flags(nvme_ctrl(req), ns, zone, flags);
3962 }
3963
3964 static uint16_t nvme_close_zone(NvmeNamespace *ns, NvmeZone *zone,
3965 NvmeZoneState state, NvmeRequest *req)
3966 {
3967 return nvme_zrm_close(ns, zone);
3968 }
3969
3970 static uint16_t nvme_finish_zone(NvmeNamespace *ns, NvmeZone *zone,
3971 NvmeZoneState state, NvmeRequest *req)
3972 {
3973 return nvme_zrm_finish(ns, zone);
3974 }
3975
3976 static uint16_t nvme_offline_zone(NvmeNamespace *ns, NvmeZone *zone,
3977 NvmeZoneState state, NvmeRequest *req)
3978 {
3979 switch (state) {
3980 case NVME_ZONE_STATE_READ_ONLY:
3981 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_OFFLINE);
3982 /* fall through */
3983 case NVME_ZONE_STATE_OFFLINE:
3984 return NVME_SUCCESS;
3985 default:
3986 return NVME_ZONE_INVAL_TRANSITION;
3987 }
3988 }
3989
3990 static uint16_t nvme_set_zd_ext(NvmeNamespace *ns, NvmeZone *zone)
3991 {
3992 uint16_t status;
3993 uint8_t state = nvme_get_zone_state(zone);
3994
3995 if (state == NVME_ZONE_STATE_EMPTY) {
3996 status = nvme_aor_check(ns, 1, 0);
3997 if (status) {
3998 return status;
3999 }
4000 nvme_aor_inc_active(ns);
4001 zone->d.za |= NVME_ZA_ZD_EXT_VALID;
4002 nvme_assign_zone_state(ns, zone, NVME_ZONE_STATE_CLOSED);
4003 return NVME_SUCCESS;
4004 }
4005
4006 return NVME_ZONE_INVAL_TRANSITION;
4007 }
4008
4009 static uint16_t nvme_bulk_proc_zone(NvmeNamespace *ns, NvmeZone *zone,
4010 enum NvmeZoneProcessingMask proc_mask,
4011 op_handler_t op_hndlr, NvmeRequest *req)
4012 {
4013 uint16_t status = NVME_SUCCESS;
4014 NvmeZoneState zs = nvme_get_zone_state(zone);
4015 bool proc_zone;
4016
4017 switch (zs) {
4018 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
4019 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
4020 proc_zone = proc_mask & NVME_PROC_OPENED_ZONES;
4021 break;
4022 case NVME_ZONE_STATE_CLOSED:
4023 proc_zone = proc_mask & NVME_PROC_CLOSED_ZONES;
4024 break;
4025 case NVME_ZONE_STATE_READ_ONLY:
4026 proc_zone = proc_mask & NVME_PROC_READ_ONLY_ZONES;
4027 break;
4028 case NVME_ZONE_STATE_FULL:
4029 proc_zone = proc_mask & NVME_PROC_FULL_ZONES;
4030 break;
4031 default:
4032 proc_zone = false;
4033 }
4034
4035 if (proc_zone) {
4036 status = op_hndlr(ns, zone, zs, req);
4037 }
4038
4039 return status;
4040 }
4041
4042 static uint16_t nvme_do_zone_op(NvmeNamespace *ns, NvmeZone *zone,
4043 enum NvmeZoneProcessingMask proc_mask,
4044 op_handler_t op_hndlr, NvmeRequest *req)
4045 {
4046 NvmeZone *next;
4047 uint16_t status = NVME_SUCCESS;
4048 int i;
4049
4050 if (!proc_mask) {
4051 status = op_hndlr(ns, zone, nvme_get_zone_state(zone), req);
4052 } else {
4053 if (proc_mask & NVME_PROC_CLOSED_ZONES) {
4054 QTAILQ_FOREACH_SAFE(zone, &ns->closed_zones, entry, next) {
4055 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4056 req);
4057 if (status && status != NVME_NO_COMPLETE) {
4058 goto out;
4059 }
4060 }
4061 }
4062 if (proc_mask & NVME_PROC_OPENED_ZONES) {
4063 QTAILQ_FOREACH_SAFE(zone, &ns->imp_open_zones, entry, next) {
4064 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4065 req);
4066 if (status && status != NVME_NO_COMPLETE) {
4067 goto out;
4068 }
4069 }
4070
4071 QTAILQ_FOREACH_SAFE(zone, &ns->exp_open_zones, entry, next) {
4072 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4073 req);
4074 if (status && status != NVME_NO_COMPLETE) {
4075 goto out;
4076 }
4077 }
4078 }
4079 if (proc_mask & NVME_PROC_FULL_ZONES) {
4080 QTAILQ_FOREACH_SAFE(zone, &ns->full_zones, entry, next) {
4081 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4082 req);
4083 if (status && status != NVME_NO_COMPLETE) {
4084 goto out;
4085 }
4086 }
4087 }
4088
4089 if (proc_mask & NVME_PROC_READ_ONLY_ZONES) {
4090 for (i = 0; i < ns->num_zones; i++, zone++) {
4091 status = nvme_bulk_proc_zone(ns, zone, proc_mask, op_hndlr,
4092 req);
4093 if (status && status != NVME_NO_COMPLETE) {
4094 goto out;
4095 }
4096 }
4097 }
4098 }
4099
4100 out:
4101 return status;
4102 }
4103
4104 typedef struct NvmeZoneResetAIOCB {
4105 BlockAIOCB common;
4106 BlockAIOCB *aiocb;
4107 NvmeRequest *req;
4108 int ret;
4109
4110 bool all;
4111 int idx;
4112 NvmeZone *zone;
4113 } NvmeZoneResetAIOCB;
4114
4115 static void nvme_zone_reset_cancel(BlockAIOCB *aiocb)
4116 {
4117 NvmeZoneResetAIOCB *iocb = container_of(aiocb, NvmeZoneResetAIOCB, common);
4118 NvmeRequest *req = iocb->req;
4119 NvmeNamespace *ns = req->ns;
4120
4121 iocb->idx = ns->num_zones;
4122
4123 iocb->ret = -ECANCELED;
4124
4125 if (iocb->aiocb) {
4126 blk_aio_cancel_async(iocb->aiocb);
4127 iocb->aiocb = NULL;
4128 }
4129 }
4130
4131 static const AIOCBInfo nvme_zone_reset_aiocb_info = {
4132 .aiocb_size = sizeof(NvmeZoneResetAIOCB),
4133 .cancel_async = nvme_zone_reset_cancel,
4134 };
4135
4136 static void nvme_zone_reset_cb(void *opaque, int ret);
4137
4138 static void nvme_zone_reset_epilogue_cb(void *opaque, int ret)
4139 {
4140 NvmeZoneResetAIOCB *iocb = opaque;
4141 NvmeRequest *req = iocb->req;
4142 NvmeNamespace *ns = req->ns;
4143 int64_t moff;
4144 int count;
4145
4146 if (ret < 0 || iocb->ret < 0 || !ns->lbaf.ms) {
4147 goto out;
4148 }
4149
4150 moff = nvme_moff(ns, iocb->zone->d.zslba);
4151 count = nvme_m2b(ns, ns->zone_size);
4152
4153 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk, moff, count,
4154 BDRV_REQ_MAY_UNMAP,
4155 nvme_zone_reset_cb, iocb);
4156 return;
4157
4158 out:
4159 nvme_zone_reset_cb(iocb, ret);
4160 }
4161
4162 static void nvme_zone_reset_cb(void *opaque, int ret)
4163 {
4164 NvmeZoneResetAIOCB *iocb = opaque;
4165 NvmeRequest *req = iocb->req;
4166 NvmeNamespace *ns = req->ns;
4167
4168 if (iocb->ret < 0) {
4169 goto done;
4170 } else if (ret < 0) {
4171 iocb->ret = ret;
4172 goto done;
4173 }
4174
4175 if (iocb->zone) {
4176 nvme_zrm_reset(ns, iocb->zone);
4177
4178 if (!iocb->all) {
4179 goto done;
4180 }
4181 }
4182
4183 while (iocb->idx < ns->num_zones) {
4184 NvmeZone *zone = &ns->zone_array[iocb->idx++];
4185
4186 switch (nvme_get_zone_state(zone)) {
4187 case NVME_ZONE_STATE_EMPTY:
4188 if (!iocb->all) {
4189 goto done;
4190 }
4191
4192 continue;
4193
4194 case NVME_ZONE_STATE_EXPLICITLY_OPEN:
4195 case NVME_ZONE_STATE_IMPLICITLY_OPEN:
4196 case NVME_ZONE_STATE_CLOSED:
4197 case NVME_ZONE_STATE_FULL:
4198 iocb->zone = zone;
4199 break;
4200
4201 default:
4202 continue;
4203 }
4204
4205 trace_pci_nvme_zns_zone_reset(zone->d.zslba);
4206
4207 iocb->aiocb = blk_aio_pwrite_zeroes(ns->blkconf.blk,
4208 nvme_l2b(ns, zone->d.zslba),
4209 nvme_l2b(ns, ns->zone_size),
4210 BDRV_REQ_MAY_UNMAP,
4211 nvme_zone_reset_epilogue_cb,
4212 iocb);
4213 return;
4214 }
4215
4216 done:
4217 iocb->aiocb = NULL;
4218
4219 iocb->common.cb(iocb->common.opaque, iocb->ret);
4220 qemu_aio_unref(iocb);
4221 }
4222
4223 static uint16_t nvme_zone_mgmt_send_zrwa_flush(NvmeCtrl *n, NvmeZone *zone,
4224 uint64_t elba, NvmeRequest *req)
4225 {
4226 NvmeNamespace *ns = req->ns;
4227 uint16_t ozcs = le16_to_cpu(ns->id_ns_zoned->ozcs);
4228 uint64_t wp = zone->d.wp;
4229 uint32_t nlb = elba - wp + 1;
4230 uint16_t status;
4231
4232
4233 if (!(ozcs & NVME_ID_NS_ZONED_OZCS_ZRWASUP)) {
4234 return NVME_INVALID_ZONE_OP | NVME_DNR;
4235 }
4236
4237 if (!(zone->d.za & NVME_ZA_ZRWA_VALID)) {
4238 return NVME_INVALID_FIELD | NVME_DNR;
4239 }
4240
4241 if (elba < wp || elba > wp + ns->zns.zrwas) {
4242 return NVME_ZONE_BOUNDARY_ERROR | NVME_DNR;
4243 }
4244
4245 if (nlb % ns->zns.zrwafg) {
4246 return NVME_INVALID_FIELD | NVME_DNR;
4247 }
4248
4249 status = nvme_zrm_auto(n, ns, zone);
4250 if (status) {
4251 return status;
4252 }
4253
4254 zone->w_ptr += nlb;
4255
4256 nvme_advance_zone_wp(ns, zone, nlb);
4257
4258 return NVME_SUCCESS;
4259 }
4260
4261 static uint16_t nvme_zone_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4262 {
4263 NvmeZoneSendCmd *cmd = (NvmeZoneSendCmd *)&req->cmd;
4264 NvmeNamespace *ns = req->ns;
4265 NvmeZone *zone;
4266 NvmeZoneResetAIOCB *iocb;
4267 uint8_t *zd_ext;
4268 uint64_t slba = 0;
4269 uint32_t zone_idx = 0;
4270 uint16_t status;
4271 uint8_t action = cmd->zsa;
4272 bool all;
4273 enum NvmeZoneProcessingMask proc_mask = NVME_PROC_CURRENT_ZONE;
4274
4275 all = cmd->zsflags & NVME_ZSFLAG_SELECT_ALL;
4276
4277 req->status = NVME_SUCCESS;
4278
4279 if (!all) {
4280 status = nvme_get_mgmt_zone_slba_idx(ns, &req->cmd, &slba, &zone_idx);
4281 if (status) {
4282 return status;
4283 }
4284 }
4285
4286 zone = &ns->zone_array[zone_idx];
4287 if (slba != zone->d.zslba && action != NVME_ZONE_ACTION_ZRWA_FLUSH) {
4288 trace_pci_nvme_err_unaligned_zone_cmd(action, slba, zone->d.zslba);
4289 return NVME_INVALID_FIELD | NVME_DNR;
4290 }
4291
4292 switch (action) {
4293
4294 case NVME_ZONE_ACTION_OPEN:
4295 if (all) {
4296 proc_mask = NVME_PROC_CLOSED_ZONES;
4297 }
4298 trace_pci_nvme_open_zone(slba, zone_idx, all);
4299 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_open_zone, req);
4300 break;
4301
4302 case NVME_ZONE_ACTION_CLOSE:
4303 if (all) {
4304 proc_mask = NVME_PROC_OPENED_ZONES;
4305 }
4306 trace_pci_nvme_close_zone(slba, zone_idx, all);
4307 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_close_zone, req);
4308 break;
4309
4310 case NVME_ZONE_ACTION_FINISH:
4311 if (all) {
4312 proc_mask = NVME_PROC_OPENED_ZONES | NVME_PROC_CLOSED_ZONES;
4313 }
4314 trace_pci_nvme_finish_zone(slba, zone_idx, all);
4315 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_finish_zone, req);
4316 break;
4317
4318 case NVME_ZONE_ACTION_RESET:
4319 trace_pci_nvme_reset_zone(slba, zone_idx, all);
4320
4321 iocb = blk_aio_get(&nvme_zone_reset_aiocb_info, ns->blkconf.blk,
4322 nvme_misc_cb, req);
4323
4324 iocb->req = req;
4325 iocb->ret = 0;
4326 iocb->all = all;
4327 iocb->idx = zone_idx;
4328 iocb->zone = NULL;
4329
4330 req->aiocb = &iocb->common;
4331 nvme_zone_reset_cb(iocb, 0);
4332
4333 return NVME_NO_COMPLETE;
4334
4335 case NVME_ZONE_ACTION_OFFLINE:
4336 if (all) {
4337 proc_mask = NVME_PROC_READ_ONLY_ZONES;
4338 }
4339 trace_pci_nvme_offline_zone(slba, zone_idx, all);
4340 status = nvme_do_zone_op(ns, zone, proc_mask, nvme_offline_zone, req);
4341 break;
4342
4343 case NVME_ZONE_ACTION_SET_ZD_EXT:
4344 trace_pci_nvme_set_descriptor_extension(slba, zone_idx);
4345 if (all || !ns->params.zd_extension_size) {
4346 return NVME_INVALID_FIELD | NVME_DNR;
4347 }
4348 zd_ext = nvme_get_zd_extension(ns, zone_idx);
4349 status = nvme_h2c(n, zd_ext, ns->params.zd_extension_size, req);
4350 if (status) {
4351 trace_pci_nvme_err_zd_extension_map_error(zone_idx);
4352 return status;
4353 }
4354
4355 status = nvme_set_zd_ext(ns, zone);
4356 if (status == NVME_SUCCESS) {
4357 trace_pci_nvme_zd_extension_set(zone_idx);
4358 return status;
4359 }
4360 break;
4361
4362 case NVME_ZONE_ACTION_ZRWA_FLUSH:
4363 if (all) {
4364 return NVME_INVALID_FIELD | NVME_DNR;
4365 }
4366
4367 return nvme_zone_mgmt_send_zrwa_flush(n, zone, slba, req);
4368
4369 default:
4370 trace_pci_nvme_err_invalid_mgmt_action(action);
4371 status = NVME_INVALID_FIELD;
4372 }
4373
4374 if (status == NVME_ZONE_INVAL_TRANSITION) {
4375 trace_pci_nvme_err_invalid_zone_state_transition(action, slba,
4376 zone->d.za);
4377 }
4378 if (status) {
4379 status |= NVME_DNR;
4380 }
4381
4382 return status;
4383 }
4384
4385 static bool nvme_zone_matches_filter(uint32_t zafs, NvmeZone *zl)
4386 {
4387 NvmeZoneState zs = nvme_get_zone_state(zl);
4388
4389 switch (zafs) {
4390 case NVME_ZONE_REPORT_ALL:
4391 return true;
4392 case NVME_ZONE_REPORT_EMPTY:
4393 return zs == NVME_ZONE_STATE_EMPTY;
4394 case NVME_ZONE_REPORT_IMPLICITLY_OPEN:
4395 return zs == NVME_ZONE_STATE_IMPLICITLY_OPEN;
4396 case NVME_ZONE_REPORT_EXPLICITLY_OPEN:
4397 return zs == NVME_ZONE_STATE_EXPLICITLY_OPEN;
4398 case NVME_ZONE_REPORT_CLOSED:
4399 return zs == NVME_ZONE_STATE_CLOSED;
4400 case NVME_ZONE_REPORT_FULL:
4401 return zs == NVME_ZONE_STATE_FULL;
4402 case NVME_ZONE_REPORT_READ_ONLY:
4403 return zs == NVME_ZONE_STATE_READ_ONLY;
4404 case NVME_ZONE_REPORT_OFFLINE:
4405 return zs == NVME_ZONE_STATE_OFFLINE;
4406 default:
4407 return false;
4408 }
4409 }
4410
4411 static uint16_t nvme_zone_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4412 {
4413 NvmeCmd *cmd = &req->cmd;
4414 NvmeNamespace *ns = req->ns;
4415 /* cdw12 is zero-based number of dwords to return. Convert to bytes */
4416 uint32_t data_size = (le32_to_cpu(cmd->cdw12) + 1) << 2;
4417 uint32_t dw13 = le32_to_cpu(cmd->cdw13);
4418 uint32_t zone_idx, zra, zrasf, partial;
4419 uint64_t max_zones, nr_zones = 0;
4420 uint16_t status;
4421 uint64_t slba;
4422 NvmeZoneDescr *z;
4423 NvmeZone *zone;
4424 NvmeZoneReportHeader *header;
4425 void *buf, *buf_p;
4426 size_t zone_entry_sz;
4427 int i;
4428
4429 req->status = NVME_SUCCESS;
4430
4431 status = nvme_get_mgmt_zone_slba_idx(ns, cmd, &slba, &zone_idx);
4432 if (status) {
4433 return status;
4434 }
4435
4436 zra = dw13 & 0xff;
4437 if (zra != NVME_ZONE_REPORT && zra != NVME_ZONE_REPORT_EXTENDED) {
4438 return NVME_INVALID_FIELD | NVME_DNR;
4439 }
4440 if (zra == NVME_ZONE_REPORT_EXTENDED && !ns->params.zd_extension_size) {
4441 return NVME_INVALID_FIELD | NVME_DNR;
4442 }
4443
4444 zrasf = (dw13 >> 8) & 0xff;
4445 if (zrasf > NVME_ZONE_REPORT_OFFLINE) {
4446 return NVME_INVALID_FIELD | NVME_DNR;
4447 }
4448
4449 if (data_size < sizeof(NvmeZoneReportHeader)) {
4450 return NVME_INVALID_FIELD | NVME_DNR;
4451 }
4452
4453 status = nvme_check_mdts(n, data_size);
4454 if (status) {
4455 return status;
4456 }
4457
4458 partial = (dw13 >> 16) & 0x01;
4459
4460 zone_entry_sz = sizeof(NvmeZoneDescr);
4461 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4462 zone_entry_sz += ns->params.zd_extension_size;
4463 }
4464
4465 max_zones = (data_size - sizeof(NvmeZoneReportHeader)) / zone_entry_sz;
4466 buf = g_malloc0(data_size);
4467
4468 zone = &ns->zone_array[zone_idx];
4469 for (i = zone_idx; i < ns->num_zones; i++) {
4470 if (partial && nr_zones >= max_zones) {
4471 break;
4472 }
4473 if (nvme_zone_matches_filter(zrasf, zone++)) {
4474 nr_zones++;
4475 }
4476 }
4477 header = buf;
4478 header->nr_zones = cpu_to_le64(nr_zones);
4479
4480 buf_p = buf + sizeof(NvmeZoneReportHeader);
4481 for (; zone_idx < ns->num_zones && max_zones > 0; zone_idx++) {
4482 zone = &ns->zone_array[zone_idx];
4483 if (nvme_zone_matches_filter(zrasf, zone)) {
4484 z = buf_p;
4485 buf_p += sizeof(NvmeZoneDescr);
4486
4487 z->zt = zone->d.zt;
4488 z->zs = zone->d.zs;
4489 z->zcap = cpu_to_le64(zone->d.zcap);
4490 z->zslba = cpu_to_le64(zone->d.zslba);
4491 z->za = zone->d.za;
4492
4493 if (nvme_wp_is_valid(zone)) {
4494 z->wp = cpu_to_le64(zone->d.wp);
4495 } else {
4496 z->wp = cpu_to_le64(~0ULL);
4497 }
4498
4499 if (zra == NVME_ZONE_REPORT_EXTENDED) {
4500 if (zone->d.za & NVME_ZA_ZD_EXT_VALID) {
4501 memcpy(buf_p, nvme_get_zd_extension(ns, zone_idx),
4502 ns->params.zd_extension_size);
4503 }
4504 buf_p += ns->params.zd_extension_size;
4505 }
4506
4507 max_zones--;
4508 }
4509 }
4510
4511 status = nvme_c2h(n, (uint8_t *)buf, data_size, req);
4512
4513 g_free(buf);
4514
4515 return status;
4516 }
4517
4518 static uint16_t nvme_io_mgmt_recv_ruhs(NvmeCtrl *n, NvmeRequest *req,
4519 size_t len)
4520 {
4521 NvmeNamespace *ns = req->ns;
4522 NvmeEnduranceGroup *endgrp;
4523 NvmeRuhStatus *hdr;
4524 NvmeRuhStatusDescr *ruhsd;
4525 unsigned int nruhsd;
4526 uint16_t rg, ph, *ruhid;
4527 size_t trans_len;
4528 g_autofree uint8_t *buf = NULL;
4529
4530 if (!n->subsys) {
4531 return NVME_INVALID_FIELD | NVME_DNR;
4532 }
4533
4534 if (ns->params.nsid == 0 || ns->params.nsid == 0xffffffff) {
4535 return NVME_INVALID_NSID | NVME_DNR;
4536 }
4537
4538 if (!n->subsys->endgrp.fdp.enabled) {
4539 return NVME_FDP_DISABLED | NVME_DNR;
4540 }
4541
4542 endgrp = ns->endgrp;
4543
4544 nruhsd = ns->fdp.nphs * endgrp->fdp.nrg;
4545 trans_len = sizeof(NvmeRuhStatus) + nruhsd * sizeof(NvmeRuhStatusDescr);
4546 buf = g_malloc0(trans_len);
4547
4548 trans_len = MIN(trans_len, len);
4549
4550 hdr = (NvmeRuhStatus *)buf;
4551 ruhsd = (NvmeRuhStatusDescr *)(buf + sizeof(NvmeRuhStatus));
4552
4553 hdr->nruhsd = cpu_to_le16(nruhsd);
4554
4555 ruhid = ns->fdp.phs;
4556
4557 for (ph = 0; ph < ns->fdp.nphs; ph++, ruhid++) {
4558 NvmeRuHandle *ruh = &endgrp->fdp.ruhs[*ruhid];
4559
4560 for (rg = 0; rg < endgrp->fdp.nrg; rg++, ruhsd++) {
4561 uint16_t pid = nvme_make_pid(ns, rg, ph);
4562
4563 ruhsd->pid = cpu_to_le16(pid);
4564 ruhsd->ruhid = *ruhid;
4565 ruhsd->earutr = 0;
4566 ruhsd->ruamw = cpu_to_le64(ruh->rus[rg].ruamw);
4567 }
4568 }
4569
4570 return nvme_c2h(n, buf, trans_len, req);
4571 }
4572
4573 static uint16_t nvme_io_mgmt_recv(NvmeCtrl *n, NvmeRequest *req)
4574 {
4575 NvmeCmd *cmd = &req->cmd;
4576 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4577 uint32_t numd = le32_to_cpu(cmd->cdw11);
4578 uint8_t mo = (cdw10 & 0xff);
4579 size_t len = (numd + 1) << 2;
4580
4581 switch (mo) {
4582 case NVME_IOMR_MO_NOP:
4583 return 0;
4584 case NVME_IOMR_MO_RUH_STATUS:
4585 return nvme_io_mgmt_recv_ruhs(n, req, len);
4586 default:
4587 return NVME_INVALID_FIELD | NVME_DNR;
4588 };
4589 }
4590
4591 static uint16_t nvme_io_mgmt_send_ruh_update(NvmeCtrl *n, NvmeRequest *req)
4592 {
4593 NvmeCmd *cmd = &req->cmd;
4594 NvmeNamespace *ns = req->ns;
4595 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4596 uint16_t ret = NVME_SUCCESS;
4597 uint32_t npid = (cdw10 >> 16) + 1;
4598 unsigned int i = 0;
4599 g_autofree uint16_t *pids = NULL;
4600 uint32_t maxnpid;
4601
4602 if (!ns->endgrp || !ns->endgrp->fdp.enabled) {
4603 return NVME_FDP_DISABLED | NVME_DNR;
4604 }
4605
4606 maxnpid = n->subsys->endgrp.fdp.nrg * n->subsys->endgrp.fdp.nruh;
4607
4608 if (unlikely(npid >= MIN(NVME_FDP_MAXPIDS, maxnpid))) {
4609 return NVME_INVALID_FIELD | NVME_DNR;
4610 }
4611
4612 pids = g_new(uint16_t, npid);
4613
4614 ret = nvme_h2c(n, pids, npid * sizeof(uint16_t), req);
4615 if (ret) {
4616 return ret;
4617 }
4618
4619 for (; i < npid; i++) {
4620 if (!nvme_update_ruh(n, ns, pids[i])) {
4621 return NVME_INVALID_FIELD | NVME_DNR;
4622 }
4623 }
4624
4625 return ret;
4626 }
4627
4628 static uint16_t nvme_io_mgmt_send(NvmeCtrl *n, NvmeRequest *req)
4629 {
4630 NvmeCmd *cmd = &req->cmd;
4631 uint32_t cdw10 = le32_to_cpu(cmd->cdw10);
4632 uint8_t mo = (cdw10 & 0xff);
4633
4634 switch (mo) {
4635 case NVME_IOMS_MO_NOP:
4636 return 0;
4637 case NVME_IOMS_MO_RUH_UPDATE:
4638 return nvme_io_mgmt_send_ruh_update(n, req);
4639 /* if you add something here, please update nvme_set_migration_blockers() */
4640 default:
4641 return NVME_INVALID_FIELD | NVME_DNR;
4642 };
4643 }
4644
4645 static uint16_t __nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req)
4646 {
4647 switch (req->cmd.opcode) {
4648 case NVME_CMD_WRITE:
4649 return nvme_write(n, req);
4650 case NVME_CMD_READ:
4651 return nvme_read(n, req);
4652 case NVME_CMD_COMPARE:
4653 return nvme_compare(n, req);
4654 case NVME_CMD_WRITE_ZEROES:
4655 return nvme_write_zeroes(n, req);
4656 case NVME_CMD_DSM:
4657 return nvme_dsm(n, req);
4658 case NVME_CMD_VERIFY:
4659 return nvme_verify(n, req);
4660 case NVME_CMD_COPY:
4661 return nvme_copy(n, req);
4662 case NVME_CMD_IO_MGMT_RECV:
4663 return nvme_io_mgmt_recv(n, req);
4664 case NVME_CMD_IO_MGMT_SEND:
4665 return nvme_io_mgmt_send(n, req);
4666 }
4667
4668 g_assert_not_reached();
4669 }
4670
4671 static uint16_t nvme_io_cmd_nvm(NvmeCtrl *n, NvmeRequest *req)
4672 {
4673 if (!(n->cse.iocs.nvm[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4674 trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4675 return NVME_INVALID_OPCODE | NVME_DNR;
4676 }
4677
4678 return __nvme_io_cmd_nvm(n, req);
4679 }
4680
4681 static uint16_t nvme_io_cmd_zoned(NvmeCtrl *n, NvmeRequest *req)
4682 {
4683 if (!(n->cse.iocs.zoned[req->cmd.opcode] & NVME_CMD_EFF_CSUPP)) {
4684 trace_pci_nvme_err_invalid_opc(req->cmd.opcode);
4685 return NVME_INVALID_OPCODE | NVME_DNR;
4686 }
4687
4688 switch (req->cmd.opcode) {
4689 case NVME_CMD_ZONE_APPEND:
4690 return nvme_zone_append(n, req);
4691 case NVME_CMD_ZONE_MGMT_SEND:
4692 return nvme_zone_mgmt_send(n, req);
4693 case NVME_CMD_ZONE_MGMT_RECV:
4694 return nvme_zone_mgmt_recv(n, req);
4695 }
4696
4697 return __nvme_io_cmd_nvm(n, req);
4698 }
4699
4700 static uint16_t nvme_io_cmd(NvmeCtrl *n, NvmeRequest *req)
4701 {
4702 NvmeNamespace *ns;
4703 uint32_t nsid = le32_to_cpu(req->cmd.nsid);
4704
4705 trace_pci_nvme_io_cmd(nvme_cid(req), nsid, nvme_sqid(req),
4706 req->cmd.opcode, nvme_io_opc_str(req->cmd.opcode));
4707
4708 /*
4709 * In the base NVM command set, Flush may apply to all namespaces
4710 * (indicated by NSID being set to FFFFFFFFh). But if that feature is used
4711 * along with TP 4056 (Namespace Types), it may be pretty screwed up.
4712 *
4713 * If NSID is indeed set to FFFFFFFFh, we simply cannot associate the
4714 * opcode with a specific command since we cannot determine a unique I/O
4715 * command set. Opcode 0h could have any other meaning than something
4716 * equivalent to flushing and say it DOES have completely different
4717 * semantics in some other command set - does an NSID of FFFFFFFFh then
4718 * mean "for all namespaces, apply whatever command set specific command
4719 * that uses the 0h opcode?" Or does it mean "for all namespaces, apply
4720 * whatever command that uses the 0h opcode if, and only if, it allows NSID
4721 * to be FFFFFFFFh"?
4722 *
4723 * Anyway (and luckily), for now, we do not care about this since the
4724 * device only supports namespace types that includes the NVM Flush command
4725 * (NVM and Zoned), so always do an NVM Flush.
4726 */
4727
4728 if (req->cmd.opcode == NVME_CMD_FLUSH) {
4729 return nvme_flush(n, req);
4730 }
4731
4732 if (!nvme_nsid_valid(n, nsid) || nsid == NVME_NSID_BROADCAST) {
4733 return NVME_INVALID_NSID | NVME_DNR;
4734 }
4735
4736 ns = nvme_ns(n, nsid);
4737 if (unlikely(!ns)) {
4738 return NVME_INVALID_FIELD | NVME_DNR;
4739 }
4740
4741 if (ns->status) {
4742 return ns->status;
4743 }
4744
4745 if (NVME_CMD_FLAGS_FUSE(req->cmd.flags)) {
4746 return NVME_INVALID_FIELD;
4747 }
4748
4749 req->ns = ns;
4750
4751 switch (ns->csi) {
4752 case NVME_CSI_NVM:
4753 return nvme_io_cmd_nvm(n, req);
4754 case NVME_CSI_ZONED:
4755 return nvme_io_cmd_zoned(n, req);
4756 }
4757
4758 g_assert_not_reached();
4759 }
4760
4761 static void nvme_cq_notifier(EventNotifier *e)
4762 {
4763 NvmeCQueue *cq = container_of(e, NvmeCQueue, notifier);
4764 NvmeCtrl *n = cq->ctrl;
4765
4766 if (!event_notifier_test_and_clear(e)) {
4767 return;
4768 }
4769
4770 nvme_update_cq_head(cq);
4771
4772 if (cq->tail == cq->head) {
4773 if (cq->irq_enabled) {
4774 n->cq_pending--;
4775 }
4776
4777 nvme_irq_deassert(n, cq);
4778 }
4779
4780 qemu_bh_schedule(cq->bh);
4781 }
4782
4783 static int nvme_init_cq_ioeventfd(NvmeCQueue *cq)
4784 {
4785 NvmeCtrl *n = cq->ctrl;
4786 uint16_t offset = (cq->cqid << 3) + (1 << 2);
4787 int ret;
4788
4789 ret = event_notifier_init(&cq->notifier, 0);
4790 if (ret < 0) {
4791 return ret;
4792 }
4793
4794 event_notifier_set_handler(&cq->notifier, nvme_cq_notifier);
4795 memory_region_add_eventfd(&n->iomem,
4796 0x1000 + offset, 4, false, 0, &cq->notifier);
4797
4798 return 0;
4799 }
4800
4801 static void nvme_sq_notifier(EventNotifier *e)
4802 {
4803 NvmeSQueue *sq = container_of(e, NvmeSQueue, notifier);
4804
4805 if (!event_notifier_test_and_clear(e)) {
4806 return;
4807 }
4808
4809 nvme_process_sq(sq);
4810 }
4811
4812 static int nvme_init_sq_ioeventfd(NvmeSQueue *sq)
4813 {
4814 NvmeCtrl *n = sq->ctrl;
4815 uint16_t offset = sq->sqid << 3;
4816 int ret;
4817
4818 ret = event_notifier_init(&sq->notifier, 0);
4819 if (ret < 0) {
4820 return ret;
4821 }
4822
4823 event_notifier_set_handler(&sq->notifier, nvme_sq_notifier);
4824 memory_region_add_eventfd(&n->iomem,
4825 0x1000 + offset, 4, false, 0, &sq->notifier);
4826
4827 return 0;
4828 }
4829
4830 /*
4831 * A pending Async Event Request has no aiocb (nvme_aer() parks it without
4832 * issuing any block I/O), so there is nothing to cancel; just drop it.
4833 */
4834 static void nvme_sq_cancel_inflight(NvmeSQueue *sq, uint16_t status)
4835 {
4836 NvmeRequest *r;
4837
4838 while (!QTAILQ_EMPTY(&sq->out_req_list)) {
4839 r = QTAILQ_FIRST(&sq->out_req_list);
4840 r->status = status;
4841
4842 if (r->aiocb) {
4843 blk_aio_cancel(r->aiocb);
4844 } else {
4845 QTAILQ_REMOVE(&sq->out_req_list, r, entry);
4846 }
4847 }
4848 }
4849
4850 static void nvme_free_sq(NvmeSQueue *sq, NvmeCtrl *n)
4851 {
4852 uint16_t offset = sq->sqid << 3;
4853
4854 n->sq[sq->sqid] = NULL;
4855 qemu_bh_delete(sq->bh);
4856 if (sq->ioeventfd_enabled) {
4857 memory_region_del_eventfd(&n->iomem,
4858 0x1000 + offset, 4, false, 0, &sq->notifier);
4859 event_notifier_set_handler(&sq->notifier, NULL);
4860 event_notifier_cleanup(&sq->notifier);
4861 }
4862 g_free(sq->io_req);
4863 if (sq->sqid) {
4864 g_free(sq);
4865 }
4866 }
4867
4868 static uint16_t nvme_del_sq(NvmeCtrl *n, NvmeRequest *req)
4869 {
4870 NvmeDeleteQ *c = (NvmeDeleteQ *)&req->cmd;
4871 NvmeRequest *r, *next;
4872 NvmeSQueue *sq;
4873 NvmeCQueue *cq;
4874 uint16_t qid = le16_to_cpu(c->qid);
4875
4876 if (unlikely(!qid || nvme_check_sqid(n, qid))) {
4877 trace_pci_nvme_err_invalid_del_sq(qid);
4878 return NVME_INVALID_QID | NVME_DNR;
4879 }
4880
4881 trace_pci_nvme_del_sq(qid);
4882
4883 sq = n->sq[qid];
4884 nvme_sq_cancel_inflight(sq, NVME_CMD_ABORT_SQ_DEL);
4885
4886 if (!nvme_check_cqid(n, sq->cqid)) {
4887 cq = n->cq[sq->cqid];
4888 QTAILQ_REMOVE(&cq->sq_list, sq, entry);
4889
4890 nvme_post_cqes(cq);
4891 QTAILQ_FOREACH_SAFE(r, &cq->req_list, entry, next) {
4892 if (r->sq == sq) {
4893 QTAILQ_REMOVE(&cq->req_list, r, entry);
4894 QTAILQ_INSERT_TAIL(&sq->req_list, r, entry);
4895 }
4896 }
4897 }
4898
4899 nvme_free_sq(sq, n);
4900 return NVME_SUCCESS;
4901 }
4902
4903 static void __nvme_init_sq(NvmeSQueue *sq)
4904 {
4905 NvmeCtrl *n = sq->ctrl;
4906 uint16_t sqid = sq->sqid;
4907 uint16_t cqid = sq->cqid;
4908 int i;
4909 NvmeCQueue *cq;
4910
4911 sq->io_req = g_new0(NvmeRequest, sq->size);
4912
4913 QTAILQ_INIT(&sq->req_list);
4914 QTAILQ_INIT(&sq->out_req_list);
4915 for (i = 0; i < sq->size; i++) {
4916 sq->io_req[i].sq = sq;
4917 QTAILQ_INSERT_TAIL(&(sq->req_list), &sq->io_req[i], entry);
4918 }
4919
4920 sq->bh = qemu_bh_new_guarded(nvme_process_sq, sq,
4921 &DEVICE(sq->ctrl)->mem_reentrancy_guard);
4922
4923 if (n->dbbuf_enabled) {
4924 sq->db_addr = n->dbbuf_dbs + (sqid << 3);
4925 sq->ei_addr = n->dbbuf_eis + (sqid << 3);
4926
4927 if (n->params.ioeventfd && sq->sqid != 0) {
4928 if (!nvme_init_sq_ioeventfd(sq)) {
4929 sq->ioeventfd_enabled = true;
4930 }
4931 }
4932 }
4933
4934 assert(n->cq[cqid]);
4935 cq = n->cq[cqid];
4936 QTAILQ_INSERT_TAIL(&(cq->sq_list), sq, entry);
4937 n->sq[sqid] = sq;
4938 }
4939
4940 static void nvme_init_sq(NvmeSQueue *sq, NvmeCtrl *n, uint64_t dma_addr,
4941 uint16_t sqid, uint16_t cqid, uint16_t size)
4942 {
4943 sq->ctrl = n;
4944 sq->dma_addr = dma_addr;
4945 sq->sqid = sqid;
4946 sq->size = size;
4947 sq->cqid = cqid;
4948 sq->head = sq->tail = 0;
4949 __nvme_init_sq(sq);
4950 }
4951
4952 static void nvme_restore_sq(NvmeSQueue *sq_from)
4953 {
4954 NvmeCtrl *n = sq_from->ctrl;
4955 NvmeSQueue *sq = sq_from;
4956
4957 if (sq_from->sqid == 0) {
4958 sq = &n->admin_sq;
4959 sq->ctrl = n;
4960 sq->dma_addr = sq_from->dma_addr;
4961 sq->sqid = sq_from->sqid;
4962 sq->size = sq_from->size;
4963 sq->cqid = sq_from->cqid;
4964 sq->head = sq_from->head;
4965 sq->tail = sq_from->tail;
4966 }
4967
4968 __nvme_init_sq(sq);
4969 }
4970
4971 static uint16_t nvme_create_sq(NvmeCtrl *n, NvmeRequest *req)
4972 {
4973 NvmeSQueue *sq;
4974 NvmeCreateSq *c = (NvmeCreateSq *)&req->cmd;
4975
4976 uint16_t cqid = le16_to_cpu(c->cqid);
4977 uint16_t sqid = le16_to_cpu(c->sqid);
4978 uint16_t qsize = le16_to_cpu(c->qsize);
4979 uint16_t qflags = le16_to_cpu(c->sq_flags);
4980 uint64_t prp1 = le64_to_cpu(c->prp1);
4981
4982 trace_pci_nvme_create_sq(prp1, sqid, cqid, qsize, qflags);
4983
4984 if (unlikely(!cqid || nvme_check_cqid(n, cqid))) {
4985 trace_pci_nvme_err_invalid_create_sq_cqid(cqid);
4986 return NVME_INVALID_CQID | NVME_DNR;
4987 }
4988 if (unlikely(!sqid || sqid > n->conf_ioqpairs || n->sq[sqid] != NULL)) {
4989 trace_pci_nvme_err_invalid_create_sq_sqid(sqid);
4990 return NVME_INVALID_QID | NVME_DNR;
4991 }
4992 if (unlikely(!qsize || qsize > NVME_CAP_MQES(ldq_le_p(&n->bar.cap)))) {
4993 trace_pci_nvme_err_invalid_create_sq_size(qsize);
4994 return NVME_MAX_QSIZE_EXCEEDED | NVME_DNR;
4995 }
4996 if (unlikely(prp1 & (n->page_size - 1))) {
4997 trace_pci_nvme_err_invalid_create_sq_addr(prp1);
4998 return NVME_INVALID_PRP_OFFSET | NVME_DNR;
4999 }
5000 if (unlikely(!(NVME_SQ_FLAGS_PC(qflags)))) {
Showing first 5,000 of 10,696 lines. View raw