| 1 | /* |
| 2 | * QEMU Block driver for NBD |
| 3 | * |
| 4 | * Copyright (c) 2021 Virtuozzo International GmbH. |
| 5 | * |
| 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | * of this software and associated documentation files (the "Software"), to deal |
| 8 | * in the Software without restriction, including without limitation the rights |
| 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | * copies of the Software, and to permit persons to whom the Software is |
| 11 | * furnished to do so, subject to the following conditions: |
| 12 | * |
| 13 | * The above copyright notice and this permission notice shall be included in |
| 14 | * all copies or substantial portions of the Software. |
| 15 | * |
| 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 22 | * THE SOFTWARE. |
| 23 | */ |
| 24 | |
| 25 | #include "qemu/osdep.h" |
| 26 | #include "trace.h" |
| 27 | |
| 28 | #include "block/nbd.h" |
| 29 | |
| 30 | #include "qapi/qapi-visit-sockets.h" |
| 31 | #include "qapi/clone-visitor.h" |
| 32 | #include "qemu/coroutine.h" |
| 33 | |
| 34 | #include "nbd/nbd-internal.h" |
| 35 | |
| 36 | struct NBDClientConnection { |
| 37 | /* Initialization constants, never change */ |
| 38 | SocketAddress *saddr; /* address to connect to */ |
| 39 | QCryptoTLSCreds *tlscreds; |
| 40 | char *tlshostname; |
| 41 | NBDExportInfo initial_info; |
| 42 | bool do_negotiation; |
| 43 | bool do_retry; |
| 44 | |
| 45 | QemuMutex mutex; |
| 46 | |
| 47 | NBDExportInfo updated_info; |
| 48 | /* |
| 49 | * @sioc represents a successful result. While thread is running, @sioc is |
| 50 | * used only by thread and not protected by mutex. When thread is not |
| 51 | * running, @sioc is stolen by nbd_co_establish_connection() under mutex. |
| 52 | */ |
| 53 | QIOChannelSocket *sioc; |
| 54 | QIOChannel *ioc; |
| 55 | /* |
| 56 | * @err represents previous attempt. It may be copied by |
| 57 | * nbd_co_establish_connection() when it reports failure. |
| 58 | */ |
| 59 | Error *err; |
| 60 | |
| 61 | /* All further fields are accessed only under mutex */ |
| 62 | bool running; /* thread is running now */ |
| 63 | bool detached; /* thread is detached and should cleanup the state */ |
| 64 | |
| 65 | /* |
| 66 | * wait_co: if non-NULL, which coroutine to wake in |
| 67 | * nbd_co_establish_connection() after yield() |
| 68 | */ |
| 69 | Coroutine *wait_co; |
| 70 | }; |
| 71 | |
| 72 | /* |
| 73 | * The function isn't protected by any mutex, only call it when the client |
| 74 | * connection attempt has not yet started. |
| 75 | */ |
| 76 | void nbd_client_connection_enable_retry(NBDClientConnection *conn) |
| 77 | { |
| 78 | conn->do_retry = true; |
| 79 | } |
| 80 | |
| 81 | NBDClientConnection *nbd_client_connection_new(const SocketAddress *saddr, |
| 82 | bool do_negotiation, |
| 83 | const char *export_name, |
| 84 | const char *x_dirty_bitmap, |
| 85 | QCryptoTLSCreds *tlscreds, |
| 86 | const char *tlshostname) |
| 87 | { |
| 88 | NBDClientConnection *conn = g_new(NBDClientConnection, 1); |
| 89 | |
| 90 | object_ref(OBJECT(tlscreds)); |
| 91 | *conn = (NBDClientConnection) { |
| 92 | .saddr = QAPI_CLONE(SocketAddress, saddr), |
| 93 | .tlscreds = tlscreds, |
| 94 | .tlshostname = g_strdup(tlshostname), |
| 95 | .do_negotiation = do_negotiation, |
| 96 | |
| 97 | .initial_info.request_sizes = true, |
| 98 | .initial_info.mode = NBD_MODE_EXTENDED, |
| 99 | .initial_info.base_allocation = true, |
| 100 | .initial_info.x_dirty_bitmap = g_strdup(x_dirty_bitmap), |
| 101 | .initial_info.name = g_strdup(export_name ?: "") |
| 102 | }; |
| 103 | |
| 104 | qemu_mutex_init(&conn->mutex); |
| 105 | |
| 106 | return conn; |
| 107 | } |
| 108 | |
| 109 | static void nbd_client_connection_do_free(NBDClientConnection *conn) |
| 110 | { |
| 111 | if (conn->sioc) { |
| 112 | qio_channel_close(QIO_CHANNEL(conn->sioc), NULL); |
| 113 | object_unref(OBJECT(conn->sioc)); |
| 114 | } |
| 115 | error_free(conn->err); |
| 116 | qapi_free_SocketAddress(conn->saddr); |
| 117 | g_free(conn->tlshostname); |
| 118 | object_unref(OBJECT(conn->tlscreds)); |
| 119 | g_free(conn->initial_info.x_dirty_bitmap); |
| 120 | g_free(conn->initial_info.name); |
| 121 | g_free(conn); |
| 122 | } |
| 123 | |
| 124 | /* |
| 125 | * Connect to @addr and do NBD negotiation if @info is not null. If @tlscreds |
| 126 | * are given @outioc is returned. @outioc is provided only on success. The call |
| 127 | * may be cancelled from other thread by simply qio_channel_shutdown(sioc). |
| 128 | */ |
| 129 | static int nbd_connect(QIOChannelSocket *sioc, SocketAddress *addr, |
| 130 | NBDExportInfo *info, QCryptoTLSCreds *tlscreds, |
| 131 | const char *tlshostname, |
| 132 | QIOChannel **outioc, Error **errp) |
| 133 | { |
| 134 | int ret; |
| 135 | |
| 136 | if (outioc) { |
| 137 | *outioc = NULL; |
| 138 | } |
| 139 | |
| 140 | ret = qio_channel_socket_connect_sync(sioc, addr, errp); |
| 141 | if (ret < 0) { |
| 142 | return ret; |
| 143 | } |
| 144 | |
| 145 | nbd_set_socket_send_buffer(sioc); |
| 146 | qio_channel_set_delay(QIO_CHANNEL(sioc), false); |
| 147 | |
| 148 | if (!info) { |
| 149 | return 0; |
| 150 | } |
| 151 | |
| 152 | ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), tlscreds, tlshostname, |
| 153 | outioc, info, errp); |
| 154 | if (ret < 0) { |
| 155 | /* |
| 156 | * nbd_receive_negotiate() may setup tls ioc and return it even on |
| 157 | * failure path. In this case we should use it instead of original |
| 158 | * channel. |
| 159 | */ |
| 160 | if (outioc && *outioc) { |
| 161 | qio_channel_close(*outioc, NULL); |
| 162 | object_unref(OBJECT(*outioc)); |
| 163 | *outioc = NULL; |
| 164 | } else { |
| 165 | qio_channel_close(QIO_CHANNEL(sioc), NULL); |
| 166 | } |
| 167 | |
| 168 | return ret; |
| 169 | } |
| 170 | |
| 171 | return 0; |
| 172 | } |
| 173 | |
| 174 | static void *connect_thread_func(void *opaque) |
| 175 | { |
| 176 | NBDClientConnection *conn = opaque; |
| 177 | int ret; |
| 178 | bool do_free; |
| 179 | uint64_t timeout = 1; |
| 180 | uint64_t max_timeout = 16; |
| 181 | |
| 182 | qemu_mutex_lock(&conn->mutex); |
| 183 | while (!conn->detached) { |
| 184 | Error *local_err = NULL; |
| 185 | |
| 186 | assert(!conn->sioc); |
| 187 | conn->sioc = qio_channel_socket_new(); |
| 188 | |
| 189 | qemu_mutex_unlock(&conn->mutex); |
| 190 | |
| 191 | conn->updated_info = conn->initial_info; |
| 192 | |
| 193 | ret = nbd_connect(conn->sioc, conn->saddr, |
| 194 | conn->do_negotiation ? &conn->updated_info : NULL, |
| 195 | conn->tlscreds, conn->tlshostname, |
| 196 | &conn->ioc, &local_err); |
| 197 | |
| 198 | /* |
| 199 | * conn->updated_info will finally be returned to the user. Clear the |
| 200 | * pointers to our internally allocated strings, which are IN parameters |
| 201 | * of nbd_receive_negotiate() and therefore nbd_connect(). Caller |
| 202 | * shouldn't be interested in these fields. |
| 203 | */ |
| 204 | conn->updated_info.x_dirty_bitmap = NULL; |
| 205 | conn->updated_info.name = NULL; |
| 206 | |
| 207 | qemu_mutex_lock(&conn->mutex); |
| 208 | |
| 209 | error_free(conn->err); |
| 210 | conn->err = local_err; |
| 211 | |
| 212 | if (ret < 0) { |
| 213 | object_unref(OBJECT(conn->sioc)); |
| 214 | conn->sioc = NULL; |
| 215 | if (conn->do_retry && !conn->detached) { |
| 216 | trace_nbd_connect_thread_sleep(timeout); |
| 217 | qemu_mutex_unlock(&conn->mutex); |
| 218 | |
| 219 | sleep(timeout); |
| 220 | if (timeout < max_timeout) { |
| 221 | timeout *= 2; |
| 222 | } |
| 223 | |
| 224 | qemu_mutex_lock(&conn->mutex); |
| 225 | continue; |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | break; |
| 230 | } |
| 231 | |
| 232 | /* mutex is locked */ |
| 233 | |
| 234 | assert(conn->running); |
| 235 | conn->running = false; |
| 236 | if (conn->wait_co) { |
| 237 | aio_co_wake(conn->wait_co); |
| 238 | conn->wait_co = NULL; |
| 239 | } |
| 240 | do_free = conn->detached; |
| 241 | |
| 242 | qemu_mutex_unlock(&conn->mutex); |
| 243 | |
| 244 | if (do_free) { |
| 245 | nbd_client_connection_do_free(conn); |
| 246 | } |
| 247 | |
| 248 | return NULL; |
| 249 | } |
| 250 | |
| 251 | void nbd_client_connection_release(NBDClientConnection *conn) |
| 252 | { |
| 253 | bool do_free = false; |
| 254 | |
| 255 | if (!conn) { |
| 256 | return; |
| 257 | } |
| 258 | |
| 259 | WITH_QEMU_LOCK_GUARD(&conn->mutex) { |
| 260 | assert(!conn->detached); |
| 261 | if (conn->running) { |
| 262 | conn->detached = true; |
| 263 | } else { |
| 264 | do_free = true; |
| 265 | } |
| 266 | if (conn->sioc) { |
| 267 | qio_channel_shutdown(QIO_CHANNEL(conn->sioc), |
| 268 | QIO_CHANNEL_SHUTDOWN_BOTH, NULL); |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | if (do_free) { |
| 273 | nbd_client_connection_do_free(conn); |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | /* |
| 278 | * Get a new connection in context of @conn: |
| 279 | * if the thread is running, wait for completion |
| 280 | * if the thread already succeeded in the background, and user didn't get the |
| 281 | * result, just return it now |
| 282 | * otherwise the thread is not running, so start a thread and wait for |
| 283 | * completion |
| 284 | * |
| 285 | * If @blocking is false, don't wait for the thread, return immediately. |
| 286 | * |
| 287 | * If @info is not NULL, also do nbd-negotiation after successful connection. |
| 288 | * In this case info is used only as out parameter, and is fully initialized by |
| 289 | * nbd_co_establish_connection(). "IN" fields of info as well as related only to |
| 290 | * nbd_receive_export_list() would be zero (see description of NBDExportInfo in |
| 291 | * include/block/nbd.h). |
| 292 | */ |
| 293 | QIOChannel *coroutine_fn |
| 294 | nbd_co_establish_connection(NBDClientConnection *conn, NBDExportInfo *info, |
| 295 | bool blocking, Error **errp) |
| 296 | { |
| 297 | QemuThread thread; |
| 298 | |
| 299 | if (conn->do_negotiation) { |
| 300 | assert(info); |
| 301 | } |
| 302 | |
| 303 | WITH_QEMU_LOCK_GUARD(&conn->mutex) { |
| 304 | /* |
| 305 | * Don't call nbd_co_establish_connection() in several coroutines in |
| 306 | * parallel. Only one call at once is supported. |
| 307 | */ |
| 308 | assert(!conn->wait_co); |
| 309 | |
| 310 | if (!conn->running) { |
| 311 | if (conn->sioc) { |
| 312 | /* Previous attempt finally succeeded in background */ |
| 313 | if (conn->do_negotiation) { |
| 314 | memcpy(info, &conn->updated_info, sizeof(*info)); |
| 315 | if (conn->ioc) { |
| 316 | /* TLS channel now has own reference to parent */ |
| 317 | object_unref(OBJECT(conn->sioc)); |
| 318 | conn->sioc = NULL; |
| 319 | |
| 320 | return g_steal_pointer(&conn->ioc); |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | assert(!conn->ioc); |
| 325 | |
| 326 | return QIO_CHANNEL(g_steal_pointer(&conn->sioc)); |
| 327 | } |
| 328 | |
| 329 | conn->running = true; |
| 330 | qemu_thread_create(&thread, "nbd-connect", |
| 331 | connect_thread_func, conn, QEMU_THREAD_DETACHED); |
| 332 | } |
| 333 | |
| 334 | if (!blocking) { |
| 335 | if (conn->err) { |
| 336 | error_propagate(errp, error_copy(conn->err)); |
| 337 | } else { |
| 338 | error_setg(errp, "No connection at the moment"); |
| 339 | } |
| 340 | |
| 341 | return NULL; |
| 342 | } |
| 343 | |
| 344 | conn->wait_co = qemu_coroutine_self(); |
| 345 | } |
| 346 | |
| 347 | /* |
| 348 | * We are going to wait for connect-thread finish, but |
| 349 | * nbd_co_establish_connection_cancel() can interrupt. |
| 350 | */ |
| 351 | qemu_coroutine_yield(); |
| 352 | |
| 353 | WITH_QEMU_LOCK_GUARD(&conn->mutex) { |
| 354 | if (conn->running) { |
| 355 | /* |
| 356 | * The connection attempt was canceled and the coroutine resumed |
| 357 | * before the connection thread finished its job. Report the |
| 358 | * attempt as failed, but leave the connection thread running, |
| 359 | * to reuse it for the next connection attempt. |
| 360 | */ |
| 361 | if (conn->err) { |
| 362 | error_propagate(errp, error_copy(conn->err)); |
| 363 | } else { |
| 364 | /* |
| 365 | * The only possible case here is cancelling by open_timer |
| 366 | * during nbd_open(). So, the error message is for that case. |
| 367 | * If we have more use cases, we can refactor |
| 368 | * nbd_co_establish_connection_cancel() to take an additional |
| 369 | * parameter cancel_reason, that would be passed than to the |
| 370 | * caller of cancelled nbd_co_establish_connection(). |
| 371 | */ |
| 372 | error_setg(errp, "Connection attempt cancelled by timeout"); |
| 373 | } |
| 374 | |
| 375 | return NULL; |
| 376 | } else { |
| 377 | /* Thread finished. There must be either error or sioc */ |
| 378 | assert(!conn->err != !conn->sioc); |
| 379 | |
| 380 | if (conn->err) { |
| 381 | error_propagate(errp, error_copy(conn->err)); |
| 382 | return NULL; |
| 383 | } |
| 384 | |
| 385 | if (conn->do_negotiation) { |
| 386 | memcpy(info, &conn->updated_info, sizeof(*info)); |
| 387 | if (conn->ioc) { |
| 388 | /* TLS channel now has own reference to parent */ |
| 389 | object_unref(OBJECT(conn->sioc)); |
| 390 | conn->sioc = NULL; |
| 391 | |
| 392 | return g_steal_pointer(&conn->ioc); |
| 393 | } |
| 394 | } |
| 395 | |
| 396 | assert(!conn->ioc); |
| 397 | |
| 398 | return QIO_CHANNEL(g_steal_pointer(&conn->sioc)); |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | abort(); /* unreachable */ |
| 403 | } |
| 404 | |
| 405 | /* |
| 406 | * nbd_co_establish_connection_cancel |
| 407 | * Cancel nbd_co_establish_connection() asynchronously. |
| 408 | * |
| 409 | * Note that this function neither directly stops the thread nor closes the |
| 410 | * socket, but rather safely wakes nbd_co_establish_connection() which is |
| 411 | * sleeping in yield() |
| 412 | */ |
| 413 | void nbd_co_establish_connection_cancel(NBDClientConnection *conn) |
| 414 | { |
| 415 | Coroutine *wait_co = NULL; |
| 416 | |
| 417 | WITH_QEMU_LOCK_GUARD(&conn->mutex) { |
| 418 | wait_co = g_steal_pointer(&conn->wait_co); |
| 419 | } |
| 420 | |
| 421 | if (wait_co) { |
| 422 | aio_co_wake(wait_co); |
| 423 | } |
| 424 | } |