diff --git a/libnvme/src/libnvme.ld b/libnvme/src/libnvme.ld index 971ba33ac2..18f578be11 100644 --- a/libnvme/src/libnvme.ld +++ b/libnvme/src/libnvme.ld @@ -27,6 +27,8 @@ LIBNVME_3 { libnvme_dump_config; libnvme_dump_tree; libnvme_errno_to_string; + libnvme_exec_admin_passthru; + libnvme_exec_io_passthru; libnvme_export_tls_key; libnvme_export_tls_key_versioned; libnvme_filter_ctrls; @@ -149,6 +151,7 @@ LIBNVME_3 { libnvme_read_hostnqn; libnvme_read_key; libnvme_realloc; + libnvme_reap_passthru_async; libnvme_refresh_topology; libnvme_rescan_ctrl; libnvme_rescan_ns; @@ -176,7 +179,9 @@ LIBNVME_3 { libnvme_status_to_string; libnvme_strerror; libnvme_submit_admin_passthru; + libnvme_submit_admin_passthru_async; libnvme_submit_io_passthru; + libnvme_submit_io_passthru_async; libnvme_subsystem_first_ctrl; libnvme_subsystem_first_ns; libnvme_subsystem_get_host; @@ -201,8 +206,7 @@ LIBNVME_3 { libnvme_update_key; libnvme_uuid_from_string; libnvme_uuid_to_string; - libnvme_wait_admin_passthru; - libnvme_wait_io_passthru; + libnvme_wait_passthru; local: *; }; diff --git a/libnvme/src/nvme/ioctl-linux.c b/libnvme/src/nvme/ioctl-linux.c index b8b2545cda..2d19623451 100644 --- a/libnvme/src/nvme/ioctl-linux.c +++ b/libnvme/src/nvme/ioctl-linux.c @@ -242,9 +242,6 @@ __libnvme_public int libnvme_submit_admin_passthru( if (!hdl) return -ENODEV; - if (hdl->uring_enabled) - return libnvme_submit_admin_passthru_async(hdl, cmd); - if (!cmd->timeout_ms && hdl->timeout) cmd->timeout_ms = hdl->timeout; @@ -259,3 +256,65 @@ __libnvme_public int libnvme_submit_admin_passthru( return -ENOTSUP; } + +__libnvme_public int libnvme_exec_admin_passthru( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd) +{ + struct libnvme_passthru_completion completion; + int err; + + if (!hdl) + return -ENODEV; + + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + goto no_uring; + + err = libnvme_submit_admin_passthru_async(hdl, cmd, NULL); + if (err) { + if (err == -ENOTSUP) + goto no_uring; + + return err; + } + + err = libnvme_reap_passthru_async(hdl, &completion); + if (err) + return err; + + return completion.status; + +no_uring: + return libnvme_submit_admin_passthru(hdl, cmd); +} + +__libnvme_public int libnvme_exec_io_passthru( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd) +{ + struct libnvme_passthru_completion completion; + int err; + + if (!hdl) + return -ENODEV; + + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + goto no_uring; + + err = libnvme_submit_io_passthru_async(hdl, cmd, NULL); + if (err) { + if (err == -ENOTSUP) + goto no_uring; + + return err; + } + + err = libnvme_reap_passthru_async(hdl, &completion); + if (err) + return err; + + return completion.status; + +no_uring: + return libnvme_submit_io_passthru(hdl, cmd); +} diff --git a/libnvme/src/nvme/ioctl-win.c b/libnvme/src/nvme/ioctl-win.c index dfa6444079..92d3a02543 100644 --- a/libnvme/src/nvme/ioctl-win.c +++ b/libnvme/src/nvme/ioctl-win.c @@ -1728,3 +1728,17 @@ __libnvme_public int libnvme_submit_admin_passthru(struct libnvme_transport_hand return -ENOTSUP; } } + +__libnvme_public int libnvme_exec_admin_passthru( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd) +{ + return libnvme_submit_admin_passthru(hdl, cmd); +} + +__libnvme_public int libnvme_exec_io_passthru( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd) +{ + return libnvme_submit_io_passthru(hdl, cmd); +} diff --git a/libnvme/src/nvme/ioctl.h b/libnvme/src/nvme/ioctl.h index 30d9c9b841..36c8b3cdba 100644 --- a/libnvme/src/nvme/ioctl.h +++ b/libnvme/src/nvme/ioctl.h @@ -40,39 +40,45 @@ int libnvme_submit_admin_passthru(struct libnvme_transport_handle *hdl, struct libnvme_passthru_cmd *cmd); /** - * libnvme_wait_admin_passthru() - Wait for pending admin passthru completions - * @hdl: Transport handle + * struct libnvme_passthru_completion - Async passthru completion record + * @cmd: Command that completed + * @cookie: User cookie provided to libnvme_submit_*_passthru_async() + * @status: Completion status (NVMe status or negative errno) * - * When io_uring is enabled, libnvme_submit_admin_passthru() queues commands - * asynchronously. Call this function after one or more submits to drain all - * pending completions before inspecting response data. + * Used for both admin and IO passthru command completions. + */ +struct libnvme_passthru_completion { + struct libnvme_passthru_cmd *cmd; + void *cookie; + int status; +}; + +/** + * libnvme_submit_admin_passthru_async() - Queue admin passthru command + * @hdl: Transport handle + * @cmd: The nvme admin command to send + * @cookie: User-defined opaque value returned at completion * - * This is a no-op when io_uring is not available. + * Queues @cmd for asynchronous execution. Completion is reported via + * libnvme_reap_admin_passthru_async(). * - * Return: 0 on success, negative error code otherwise. + * Return: 0 on successful queueing, negative error code otherwise. */ -int libnvme_wait_admin_passthru(struct libnvme_transport_handle *hdl); +int libnvme_submit_admin_passthru_async(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd, void *cookie); /** * libnvme_exec_admin_passthru() - Submit an admin passthru command and wait * @hdl: Transport handle * @cmd: The nvme admin command to send * - * Convenience wrapper that combines libnvme_submit_admin_passthru() and - * libnvme_wait_admin_passthru() into a single synchronous call. Use this - * for the common case where commands are sent one at a time. Use the - * split-phase API directly when batching multiple commands with io_uring. + * Synchronous command execution. * * Return: The nvme command status if a response was received (see * &enum nvme_status_field), or negative error code otherwise. */ -static inline int libnvme_exec_admin_passthru( - struct libnvme_transport_handle *hdl, - struct libnvme_passthru_cmd *cmd) -{ - int err = libnvme_submit_admin_passthru(hdl, cmd); - return err ? err : libnvme_wait_admin_passthru(hdl); -} +int libnvme_exec_admin_passthru(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd); /** * libnvme_submit_io_passthru() - Submit an nvme passthrough command @@ -88,34 +94,59 @@ int libnvme_submit_io_passthru(struct libnvme_transport_handle *hdl, struct libnvme_passthru_cmd *cmd); /** - * libnvme_wait_io_passthru() - Wait for pending IO passthru completions + * libnvme_submit_io_passthru_async() - Queue IO passthru command * @hdl: Transport handle + * @cmd: The nvme IO command to send + * @cookie: User-defined opaque value returned at completion * - * Counterpart to libnvme_submit_io_passthru() for the split-phase API. - * Currently a no-op as the IO passthru path does not yet use io_uring. + * Queues @cmd for asynchronous execution. Completion is reported via + * libnvme_reap_io_passthru_async(). * - * Return: 0 on success, negative error code otherwise. + * Return: 0 on successful queueing, negative error code otherwise. */ -int libnvme_wait_io_passthru(struct libnvme_transport_handle *hdl); +int libnvme_submit_io_passthru_async(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd, void *cookie); /** * libnvme_exec_io_passthru() - Submit an IO passthru command and wait * @hdl: Transport handle * @cmd: The nvme IO command to send * - * Convenience wrapper combining libnvme_submit_io_passthru() and - * libnvme_wait_io_passthru() into a single synchronous call. + * Synchronous command execution. Note: when io_uring is enabled, this shares + * the async queue. Avoid mixing this with direct async API usage on the same + * handle. For batching, use the async API exclusively. * * Return: The nvme command status if a response was received (see * &enum nvme_status_field), or negative error code otherwise. */ -static inline int libnvme_exec_io_passthru( - struct libnvme_transport_handle *hdl, - struct libnvme_passthru_cmd *cmd) -{ - int err = libnvme_submit_io_passthru(hdl, cmd); - return err ? err : libnvme_wait_io_passthru(hdl); -} +int libnvme_exec_io_passthru(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd); + +/** + * libnvme_reap_passthru_async() - Reap one async completion + * @hdl: Transport handle + * @completion: Completion output structure + * + * Waits for one queued passthru command to complete and stores the + * completed command pointer, associated cookie, and completion status in + * @completion. + * + * Return: 0 on success, negative error code otherwise. + */ +int libnvme_reap_passthru_async(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_completion *completion); + +/** + * libnvme_wait_passthru() - Wait for all pending passthru completions + * @hdl: Transport handle + * + * Drains all pending passthru commands from the async queue. Use this + * after batching multiple libnvme_submit_admin_passthru() calls when io_uring + * is enabled. + * + * Return: 0 on success, or the first non-zero status encountered. + */ +int libnvme_wait_passthru(struct libnvme_transport_handle *hdl); /** * libnvme_reset_subsystem() - Initiate a subsystem reset diff --git a/libnvme/src/nvme/lib.c b/libnvme/src/nvme/lib.c index 6fc01f60e9..83e125ae3a 100644 --- a/libnvme/src/nvme/lib.c +++ b/libnvme/src/nvme/lib.c @@ -96,7 +96,6 @@ __libnvme_public void libnvme_free_global_ctx(struct libnvme_global_ctx *ctx) #endif free(ctx->config_file); free(ctx->application); - libnvme_close_uring(ctx); free(ctx); } @@ -197,6 +196,7 @@ static int __nvme_transport_handle_open_direct( void __libnvme_transport_handle_close_direct( struct libnvme_transport_handle *hdl) { + libnvme_close_uring(hdl); close(hdl->fd); free(hdl); } @@ -316,4 +316,3 @@ __libnvme_public bool libnvme_transport_handle_is_mi( { return hdl->type == LIBNVME_TRANSPORT_HANDLE_TYPE_MI; } - diff --git a/libnvme/src/nvme/no-uring.c b/libnvme/src/nvme/no-uring.c index 981f0eb543..e8a0abace3 100644 --- a/libnvme/src/nvme/no-uring.c +++ b/libnvme/src/nvme/no-uring.c @@ -14,34 +14,52 @@ #include "private.h" #include "compiler-attributes.h" -int libnvme_open_uring(struct libnvme_global_ctx *ctx) +int libnvme_open_uring(__libnvme_unused struct libnvme_transport_handle *hdl) { return -ENOTSUP; } -void libnvme_close_uring(struct libnvme_global_ctx *ctx) +void libnvme_close_uring(__libnvme_unused struct libnvme_transport_handle *hdl) { } int __libnvme_transport_handle_open_uring(struct libnvme_transport_handle *hdl) { - hdl->ctx->uring_state = LIBNVME_IO_URING_STATE_NOT_AVAILABLE; + hdl->uring_state = LIBNVME_IO_URING_STATE_NOT_AVAILABLE; + return -ENOTSUP; } -int libnvme_submit_admin_passthru_async(struct libnvme_transport_handle *hdl, - struct libnvme_passthru_cmd *cmd) +__libnvme_public int libnvme_submit_admin_passthru_async( + __libnvme_unused struct libnvme_transport_handle *hdl, + __libnvme_unused struct libnvme_passthru_cmd *cmd, + __libnvme_unused void *cookie) { + if (hdl->uring_state == LIBNVME_IO_URING_STATE_UNKNOWN) + return __libnvme_transport_handle_open_uring(hdl); + return -ENOTSUP; } -__libnvme_public int libnvme_wait_admin_passthru( - __libnvme_unused struct libnvme_transport_handle *hdl) +__libnvme_public int libnvme_submit_io_passthru_async( + __libnvme_unused struct libnvme_transport_handle *hdl, + __libnvme_unused struct libnvme_passthru_cmd *cmd, + __libnvme_unused void *cookie) +{ + if (hdl->uring_state == LIBNVME_IO_URING_STATE_UNKNOWN) + return __libnvme_transport_handle_open_uring(hdl); + + return -ENOTSUP; +} + +__libnvme_public int libnvme_reap_passthru_async( + __libnvme_unused struct libnvme_transport_handle *hdl, + __libnvme_unused struct libnvme_passthru_completion *completion) { - return 0; + return -ENOTSUP; } -__libnvme_public int libnvme_wait_io_passthru( +__libnvme_public int libnvme_wait_passthru( __libnvme_unused struct libnvme_transport_handle *hdl) { - return 0; + return -ENOTSUP; } diff --git a/libnvme/src/nvme/nvme-cmds.c b/libnvme/src/nvme/nvme-cmds.c index d0bc1b6479..3e1ded9ff0 100644 --- a/libnvme/src/nvme/nvme-cmds.c +++ b/libnvme/src/nvme/nvme-cmds.c @@ -32,6 +32,32 @@ static void nvme_init_env(void) force_4k = true; } +static int submit_get_log_cmd(struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd) +{ + int err; + + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + goto no_uring; + + err = libnvme_submit_admin_passthru_async(hdl, cmd, NULL); + if (err && err == -ENOTSUP) + goto no_uring; + + return err; + +no_uring: + return libnvme_submit_admin_passthru(hdl, cmd); +} + +static int wait_get_log_cmd(struct libnvme_transport_handle *hdl) +{ + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + return 0; + + return libnvme_wait_passthru(hdl); +} + __libnvme_public int libnvme_get_log(struct libnvme_transport_handle *hdl, struct libnvme_passthru_cmd *cmd, bool rae, __u32 xfer_len) @@ -85,7 +111,7 @@ __libnvme_public int libnvme_get_log(struct libnvme_transport_handle *hdl, cmd->data_len = xfer; cmd->addr = (__u64)(uintptr_t)ptr; - ret = libnvme_submit_admin_passthru(hdl, cmd); + ret = submit_get_log_cmd(hdl, cmd); if (ret) return ret; @@ -93,7 +119,7 @@ __libnvme_public int libnvme_get_log(struct libnvme_transport_handle *hdl, ptr += xfer; } while (offset < data_len); - return libnvme_wait_admin_passthru(hdl); + return wait_get_log_cmd(hdl); } static int read_ana_chunk(struct libnvme_transport_handle *hdl, diff --git a/libnvme/src/nvme/private.h b/libnvme/src/nvme/private.h index ac7962a123..3e2bc51e63 100644 --- a/libnvme/src/nvme/private.h +++ b/libnvme/src/nvme/private.h @@ -22,6 +22,9 @@ #include +struct libnvme_passthru_completion; +struct libnvme_async_req; + const char *libnvme_subsys_sysfs_dir(void); const char *libnvme_ctrl_sysfs_dir(void); const char *libnvme_ns_sysfs_dir(void); @@ -173,6 +176,12 @@ enum ioctl_state { IOCTL_STATE_IOCTL64 = 2, }; +enum libnvme_io_uring_state { + LIBNVME_IO_URING_STATE_UNKNOWN = 0, + LIBNVME_IO_URING_STATE_NOT_AVAILABLE, + LIBNVME_IO_URING_STATE_AVAILABLE, +}; + struct libnvme_transport_handle { struct libnvme_global_ctx *ctx; enum libnvme_transport_handle_type type; @@ -194,7 +203,13 @@ struct libnvme_transport_handle { struct stat stat; enum ioctl_state ioctl_admin_state; enum ioctl_state ioctl_io_state; - bool uring_enabled; + enum libnvme_io_uring_state uring_state; +#ifdef CONFIG_LIBURING + unsigned int uring_pending; + struct io_uring *ring; + struct libnvme_async_req *dry_run_head; + struct libnvme_async_req *dry_run_tail; +#endif #ifdef CONFIG_MI /* mi */ @@ -416,12 +431,6 @@ struct libnvme_fabric_options { // !generate-accessors bool trsvcid; }; -enum libnvme_io_uring_state { - LIBNVME_IO_URING_STATE_UNKNOWN = 0, - LIBNVME_IO_URING_STATE_NOT_AVAILABLE, - LIBNVME_IO_URING_STATE_AVAILABLE, -}; - struct libnvme_global_ctx { // !generate-python:alias=GlobalCtx char *config_file; char *application; @@ -436,12 +445,6 @@ struct libnvme_global_ctx { // !generate-python:alias=GlobalCtx struct libnvme_fabric_options *options; struct ifaddrs *ifaddrs_cache; /* init with libnvmf_getifaddrs() */ #endif - - enum libnvme_io_uring_state uring_state; -#ifdef CONFIG_LIBURING - int ring_cmds; - struct io_uring *ring; -#endif }; int libnvme_set_attr(const char *dir, const char *attr, const char *value); @@ -686,9 +689,6 @@ void libnvme_ns_release_transport_handle(struct libnvme_ns *n); int libnvme_mi_admin_admin_passthru(struct libnvme_transport_handle *hdl, struct libnvme_passthru_cmd *cmd); -int libnvme_open_uring(struct libnvme_global_ctx *ctx); -void libnvme_close_uring(struct libnvme_global_ctx *ctx); +int libnvme_open_uring(struct libnvme_transport_handle *hdl); +void libnvme_close_uring(struct libnvme_transport_handle *hdl); int __libnvme_transport_handle_open_uring(struct libnvme_transport_handle *hdl); -int libnvme_submit_admin_passthru_async(struct libnvme_transport_handle *hdl, - struct libnvme_passthru_cmd *cmd); - diff --git a/libnvme/src/nvme/uring.c b/libnvme/src/nvme/uring.c index aa43af14be..ce4a591180 100644 --- a/libnvme/src/nvme/uring.c +++ b/libnvme/src/nvme/uring.c @@ -7,6 +7,9 @@ * Chaitanya Kulkarni * Daniel Wagner */ +#include +#include + #include #include @@ -19,17 +22,36 @@ */ #define NVME_URING_ENTRIES 16 -int libnvme_open_uring(struct libnvme_global_ctx *ctx) +struct libnvme_async_req { + struct libnvme_passthru_cmd *cmd; + void *cookie; + void *user_data; + unsigned int cmd_op; + struct libnvme_async_req *next; +}; + +static int libnvme_probe_uring(void) { struct io_uring_probe *probe; - struct io_uring *ring; + int err = 0; probe = io_uring_get_probe(); if (!probe) return -ENOTSUP; if (!io_uring_opcode_supported(probe, IORING_OP_URING_CMD)) - return -ENOTSUP; + err = -ENOTSUP; + + io_uring_free_probe(probe); + return err; +} + +int libnvme_open_uring(struct libnvme_transport_handle *hdl) +{ + struct io_uring *ring; + + if (hdl->ring) + return 0; ring = calloc(1, sizeof(*ring)); if (!ring) @@ -38,119 +60,250 @@ int libnvme_open_uring(struct libnvme_global_ctx *ctx) if (io_uring_queue_init(NVME_URING_ENTRIES, ring, IORING_SETUP_SQE128 | IORING_SETUP_CQE32)) { free(ring); - return -errno; + return -ENOTSUP; } - ctx->ring = ring; + hdl->ring = ring; + return 0; +} + +static int nvme_submit_uring_cmd(struct libnvme_transport_handle *hdl, + struct libnvme_async_req *req) +{ + struct io_uring_sqe *sqe; + int ret; + + sqe = io_uring_get_sqe(hdl->ring); + if (!sqe) + return -EAGAIN; + + memcpy(&sqe->cmd, req->cmd, sizeof(*req->cmd)); + + sqe->fd = hdl->fd; + sqe->opcode = IORING_OP_URING_CMD; + sqe->cmd_op = req->cmd_op; + io_uring_sqe_set_data(sqe, req); + + ret = io_uring_submit(hdl->ring); + if (ret < 0) + return -errno; + return 0; } -void libnvme_close_uring(struct libnvme_global_ctx *ctx) +static struct libnvme_async_req *nvme_dequeue_dry_run_req( + struct libnvme_transport_handle *hdl) +{ + struct libnvme_async_req *req = hdl->dry_run_head; + + if (!req) + return NULL; + + hdl->dry_run_head = req->next; + if (!hdl->dry_run_head) + hdl->dry_run_tail = NULL; + + return req; +} + +void libnvme_close_uring(struct libnvme_transport_handle *hdl) { - if (!ctx->ring) + struct libnvme_passthru_completion completion; + + if (!hdl->ring) return; - io_uring_queue_exit(ctx->ring); - free(ctx->ring); + /* Drain any pending completions */ + while (hdl->uring_pending) { + if (libnvme_reap_passthru_async(hdl, &completion)) + break; + } + + io_uring_queue_exit(hdl->ring); + free(hdl->ring); + hdl->ring = NULL; + hdl->uring_pending = 0; } int __libnvme_transport_handle_open_uring(struct libnvme_transport_handle *hdl) { int err; - switch (hdl->ctx->uring_state) { + switch (hdl->uring_state) { case LIBNVME_IO_URING_STATE_NOT_AVAILABLE: return -ENOTSUP; - case LIBNVME_IO_URING_STATE_AVAILABLE: - goto uring_enabled; case LIBNVME_IO_URING_STATE_UNKNOWN: + err = libnvme_probe_uring(); + if (err) + goto no_uring; + break; + case LIBNVME_IO_URING_STATE_AVAILABLE: break; } - err = libnvme_open_uring(hdl->ctx); - if (err) { - hdl->ctx->uring_state = LIBNVME_IO_URING_STATE_NOT_AVAILABLE; - return err; - } - hdl->ctx->uring_state = LIBNVME_IO_URING_STATE_AVAILABLE; + err = libnvme_open_uring(hdl); + if (err) + goto no_uring; -uring_enabled: - hdl->uring_enabled = true; + hdl->uring_state = LIBNVME_IO_URING_STATE_AVAILABLE; return 0; + +no_uring: + hdl->uring_state = LIBNVME_IO_URING_STATE_NOT_AVAILABLE; + return err; } -static int nvme_submit_uring_cmd(struct io_uring *ring, int fd, - struct libnvme_passthru_cmd *cmd) +static int libnvme_submit_passthru_async( + struct libnvme_transport_handle *hdl, + unsigned long ioctl_cmd, + struct libnvme_passthru_cmd *cmd, void *cookie) { - struct io_uring_sqe *sqe; - int ret; + struct libnvme_async_req *req; + int err; - sqe = io_uring_get_sqe(ring); - if (!sqe) - return -1; + if (!hdl) + return -ENODEV; - memcpy(&sqe->cmd, cmd, sizeof(*cmd)); + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + return -ENOTSUP; - sqe->fd = fd; - sqe->opcode = IORING_OP_URING_CMD; - sqe->cmd_op = LIBNVME_URING_CMD_ADMIN; + if (hdl->uring_state == LIBNVME_IO_URING_STATE_UNKNOWN) { + err = __libnvme_transport_handle_open_uring(hdl); + if (err) + return err; + } - ret = io_uring_submit(ring); - if (ret < 0) - return -errno; + if (hdl->uring_pending >= NVME_URING_ENTRIES) + return -EAGAIN; + + if (!cmd->timeout_ms && hdl->timeout) + cmd->timeout_ms = hdl->timeout; + + req = calloc(1, sizeof(*req)); + if (!req) + return -ENOMEM; + + req->cmd = cmd; + req->cookie = cookie; + req->cmd_op = ioctl_cmd; + req->user_data = hdl->submit_entry(hdl, cmd); + hdl->uring_pending += 1; + + if (hdl->ctx->dry_run) { + req->next = NULL; + if (hdl->dry_run_tail) + hdl->dry_run_tail->next = req; + else + hdl->dry_run_head = req; + + hdl->dry_run_tail = req; + return 0; + } + + err = nvme_submit_uring_cmd(hdl, req); + if (err) { + hdl->uring_pending -= 1; + hdl->submit_exit(hdl, cmd, err, req->user_data); + free(req); + return err; + } return 0; } -__libnvme_public int libnvme_wait_admin_passthru( - struct libnvme_transport_handle *hdl) +__libnvme_public int libnvme_submit_admin_passthru_async( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd, void *cookie) +{ + return libnvme_submit_passthru_async(hdl, LIBNVME_URING_CMD_ADMIN, + cmd, cookie); +} + +__libnvme_public int libnvme_submit_io_passthru_async( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_cmd *cmd, void *cookie) +{ + return libnvme_submit_passthru_async(hdl, LIBNVME_URING_CMD_IO, + cmd, cookie); +} + +__libnvme_public int libnvme_reap_passthru_async( + struct libnvme_transport_handle *hdl, + struct libnvme_passthru_completion *completion) { + struct libnvme_async_req *req; struct io_uring_cqe *cqe; - struct io_uring *ring; - int err, ret = 0; + int err; if (!hdl) return -ENODEV; - ring = hdl->ctx->ring; + if (!completion) + return -EINVAL; - for (int i = 0; i < hdl->ctx->ring_cmds; i++) { - err = io_uring_wait_cqe(ring, &cqe); - if (err < 0) { - hdl->ctx->ring_cmds = 0; + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + return -ENOTSUP; + + req = nvme_dequeue_dry_run_req(hdl); + if (req) { + err = 0; + goto complete; + } + + if (!hdl->uring_pending) + return -EAGAIN; + + for (;;) { + err = io_uring_wait_cqe(hdl->ring, &cqe); + if (err < 0) return -errno; + + req = io_uring_cqe_get_data(cqe); + err = cqe->res; + io_uring_cqe_seen(hdl->ring, cqe); + if (!req) + return -EIO; + + if (err < 0 && hdl->decide_retry(hdl, req->cmd, err)) { + err = nvme_submit_uring_cmd(hdl, req); + if (!err) + continue; } - if (!ret && cqe->res) - ret = cqe->res; - io_uring_cqe_seen(ring, cqe); + + break; } - hdl->ctx->ring_cmds = 0; - return ret; +complete: + hdl->uring_pending -= 1; + hdl->submit_exit(hdl, req->cmd, err, req->user_data); + completion->cmd = req->cmd; + completion->cookie = req->cookie; + completion->status = err; + free(req); + + return 0; } -int libnvme_submit_admin_passthru_async(struct libnvme_transport_handle *hdl, - struct libnvme_passthru_cmd *cmd) +__libnvme_public int libnvme_wait_passthru( + struct libnvme_transport_handle *hdl) { - int err; + struct libnvme_passthru_completion completion; + int err, ret = 0; + + if (!hdl) + return -ENODEV; + + if (hdl->uring_state == LIBNVME_IO_URING_STATE_NOT_AVAILABLE) + return -ENOTSUP; - if (hdl->ctx->ring_cmds >= NVME_URING_ENTRIES) { - err = libnvme_wait_admin_passthru(hdl); + while (hdl->uring_pending) { + err = libnvme_reap_passthru_async(hdl, &completion); if (err) return err; + if (!ret && completion.status) + ret = completion.status; } - err = nvme_submit_uring_cmd(hdl->ctx->ring, hdl->fd, cmd); - if (err) - return err; - - hdl->ctx->ring_cmds += 1; - return 0; -} - -__libnvme_public int libnvme_wait_io_passthru( - __libnvme_unused struct libnvme_transport_handle *hdl) -{ - return 0; + return ret; } diff --git a/libnvme/test/ioctl/async.c b/libnvme/test/ioctl/async.c new file mode 100644 index 0000000000..57f461422e --- /dev/null +++ b/libnvme/test/ioctl/async.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/** + * This file is part of libnvme. + * Copyright (c) 2026 Daniel Wagner, SUSE Software Solutions + */ + +#include +#include +#include + +#include + +#include "mock.h" +#include "util.h" + +static struct libnvme_transport_handle *test_hdl; + +static void test_admin_async_submit_not_supported(void) +{ + struct nvme_id_ctrl id = {}; + struct libnvme_passthru_cmd cmd; + int err; + + nvme_init_identify_ctrl(&cmd, &id); + err = libnvme_submit_admin_passthru_async(test_hdl, &cmd, + (void *)(uintptr_t)0x1234); + check(err == -ENOTSUP, "submit async returned %d, expected %d", + err, -ENOTSUP); +} + +static void test_io_async_submit_not_supported(void) +{ + struct libnvme_passthru_cmd cmd = {}; + int err; + + err = libnvme_submit_io_passthru_async(test_hdl, &cmd, + (void *)(uintptr_t)0x5678); + check(err == -ENOTSUP, "IO submit async returned %d, expected %d", + err, -ENOTSUP); +} + +static void test_async_reap_not_supported(void) +{ + struct libnvme_passthru_completion completion = {}; + int err; + + err = libnvme_reap_passthru_async(test_hdl, &completion); + check(err == -ENOTSUP, "reap async returned %d, expected %d", + err, -ENOTSUP); +} + +static void test_sync_path_fallback(void) +{ + struct nvme_id_ctrl expected_id = {}, id = {}; + struct mock_cmd mock_admin_cmd = { + .opcode = nvme_admin_identify, + .data_len = sizeof(expected_id), + .cdw10 = NVME_IDENTIFY_CNS_CTRL, + .out_data = &expected_id, + }; + struct libnvme_passthru_cmd cmd; + int err; + + arbitrary(&expected_id, sizeof(expected_id)); + set_mock_admin_cmds(&mock_admin_cmd, 1); + nvme_init_identify_ctrl(&cmd, &id); + err = libnvme_exec_admin_passthru(test_hdl, &cmd); + end_mock_cmds(); + check(err == 0, "sync fallback returned %d", err); + cmp(&id, &expected_id, sizeof(id), "incorrect identify data"); +} + +static void test_io_sync_path_fallback(void) +{ + struct libnvme_passthru_cmd cmd = {}; + struct mock_cmd mock_io_cmd = { + .opcode = nvme_cmd_read, + .data_len = 0, + }; + int err; + + set_mock_io_cmds(&mock_io_cmd, 1); + cmd.opcode = nvme_cmd_read; + err = libnvme_exec_io_passthru(test_hdl, &cmd); + end_mock_cmds(); + check(err == 0, "IO sync fallback returned %d", err); +} + +static void test_batched_submit_wait_pattern(void) +{ + struct nvme_id_ctrl id1 = {}, id2 = {}; + struct mock_cmd mock_cmds[] = { + { + .opcode = nvme_admin_identify, + .data_len = sizeof(id1), + .cdw10 = NVME_IDENTIFY_CNS_CTRL, + .out_data = &id1, + }, + { + .opcode = nvme_admin_identify, + .data_len = sizeof(id2), + .cdw10 = NVME_IDENTIFY_CNS_CTRL, + .out_data = &id2, + }, + }; + struct libnvme_passthru_cmd cmd1, cmd2; + int err; + + /* Test the batching pattern: submit multiple, then wait */ + arbitrary(&id1, sizeof(id1)); + arbitrary(&id2, sizeof(id2)); + set_mock_admin_cmds(mock_cmds, 2); + + nvme_init_identify_ctrl(&cmd1, &id1); + nvme_init_identify_ctrl(&cmd2, &id2); + + /* When io_uring is unavailable, submit falls back to exec */ + err = libnvme_submit_admin_passthru(test_hdl, &cmd1); + check(err == 0, "batched submit 1 returned %d", err); + + err = libnvme_submit_admin_passthru(test_hdl, &cmd2); + check(err == 0, "batched submit 2 returned %d", err); + + /* wait is a no-op when io_uring is not available */ + err = libnvme_wait_passthru(test_hdl); + check(err == -ENOTSUP, "batched wait returned %d", err); + + end_mock_cmds(); +} + +static void run_test(const char *test_name, void (*test_fn)(void)) +{ + printf("Running test %s...", test_name); + fflush(stdout); + test_fn(); + puts(" OK"); +} + +#define RUN_TEST(name) run_test(#name, test_ ## name) + +int main(void) +{ + struct libnvme_global_ctx *ctx = + libnvme_create_global_ctx(stdout, LIBNVME_DEFAULT_LOGLEVEL); + + set_mock_fd(LIBNVME_TEST_FD); + check(!libnvme_open(ctx, "NVME_TEST_FD", &test_hdl), + "opening test link failed"); + + RUN_TEST(admin_async_submit_not_supported); + RUN_TEST(io_async_submit_not_supported); + RUN_TEST(async_reap_not_supported); + RUN_TEST(sync_path_fallback); + RUN_TEST(io_sync_path_fallback); + RUN_TEST(batched_submit_wait_pattern); + + libnvme_free_global_ctx(ctx); +} diff --git a/libnvme/test/ioctl/meson.build b/libnvme/test/ioctl/meson.build index eb99025ac9..e99c464fd6 100644 --- a/libnvme/test/ioctl/meson.build +++ b/libnvme/test/ioctl/meson.build @@ -52,6 +52,18 @@ ana = executable( ) test('libnvme - ana', ana, env: mock_ioctl_env) +async = executable( + 'test-async', + 'async.c', + dependencies: [ + config_dep, + ccan_dep, + libnvme_dep, + ], + link_with: mock_ioctl, +) +test('libnvme - async', async, env: mock_ioctl_env) + if want_fabrics discovery = executable( 'test-discovery',