block/curl: add support for S3 presigned URLs
S3 presigned URLs are signed for a specific HTTP method (typically GET for our use cases). The curl block driver currently issues a HEAD request to discover the web server features and the file size, which fails with 'HTTP 403' (forbidden). Add a 'force-range' option that skips the HEAD request and instead issues a minimal GET request (querying 1 byte from the server) to extract the file size from the 'Content-Range' response header. To achieve this the 'curl_header_cb' is redesigned to generically parse HTTP headers. $ $QEMU -drive driver=https,\ 'url=https://s3.example.com/some.img?X-Amz-Security-Token=XXX', force-range=true Enabling the 'force-range' option without the web server specified with @url supporting it might cause the server to respond successfully with 'HTTP 200' and attempt to send the whole file body. With the 'CURLOPT_NOBODY' option set the libcurl will skip reading after the headers and close the connection. QEMU still gracefully detects the missing feature. This might waste a small number of TCP packets but is otherwise transparent to the user. Acked-by: Markus Armbruster <armbru@redhat.com> Signed-off-by: Antoine Damhet <adamhet@scaleway.com> Message-ID: <20260227-fix-curl-v3-v3-3-eb8a4d88feef@scaleway.com> Reviewed-by: Kevin Wolf <kwolf@redhat.com> Signed-off-by: Kevin Wolf <kwolf@redhat.com>
Antoine Damhet committed
Feb 27, 2026 at 13:45 UTC
7b13fc97d7235006d2ccc7a132ecb70802ba258f
4 files changed
+85
-34
block/curl.c
+71
-33
@@ -62,10 +62,12 @@
62
#define CURL_BLOCK_OPT_PASSWORD_SECRET "password-secret"
63
#define CURL_BLOCK_OPT_PROXY_USERNAME "proxy-username"
64
#define CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET "proxy-password-secret"
65
+#define CURL_BLOCK_OPT_FORCE_RANGE "force-range"
66
67
#define CURL_BLOCK_OPT_READAHEAD_DEFAULT (256 * 1024)
68
#define CURL_BLOCK_OPT_SSLVERIFY_DEFAULT true
69
#define CURL_BLOCK_OPT_TIMEOUT_DEFAULT 5
70
+#define CURL_BLOCK_OPT_FORCE_RANGE_DEFAULT false
71
72
struct BDRVCURLState;
73
struct CURLState;
@@ -206,27 +208,33 @@ static size_t curl_header_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
208
{
209
BDRVCURLState *s = opaque;
210
size_t realsize = size * nmemb;
209
- const char *p = ptr;
210
- const char *end = p + realsize;
211
- const char *t = "accept-ranges : bytes "; /* A lowercase template */
211
+ g_autofree char *header = g_strstrip(g_strndup(ptr, realsize));
212
+ char *val = strchr(header, ':');
213
213
- /* check if header matches the "t" template */
214
- for (;;) {
215
- if (*t == ' ') { /* space in t matches any amount of isspace in p */
216
- if (p < end && g_ascii_isspace(*p)) {
217
- ++p;
218
- } else {
219
- ++t;
220
- }
221
- } else if (*t && p < end && *t == g_ascii_tolower(*p)) {
222
- ++p, ++t;
223
- } else {
224
- break;
225
- }
214
+ if (!val) {
215
+ return realsize;
216
}
217
228
- if (!*t && p == end) { /* if we managed to reach ends of both strings */
229
- s->accept_range = true;
218
+ *val++ = '\0';
219
+ g_strchomp(header);
220
+ while (g_ascii_isspace(*val)) {
221
+ ++val;
222
+ }
223
+
224
+ trace_curl_header_cb(header, val);
225
+
226
+ if (!g_ascii_strcasecmp(header, "accept-ranges")) {
227
+ if (!g_ascii_strcasecmp(val, "bytes")) {
228
+ s->accept_range = true;
229
+ }
230
+ } else if (!g_ascii_strcasecmp(header, "Content-Range")) {
231
+ /* Content-Range fmt is `bytes begin-end/full_size` */
232
+ val = strchr(val, '/');
233
+ if (val) {
234
+ if (qemu_strtou64(val + 1, NULL, 10, &s->len) < 0) {
235
+ s->len = UINT64_MAX;
236
+ }
237
+ }
238
}
239
240
return realsize;
@@ -668,6 +676,11 @@ static QemuOptsList runtime_opts = {
676
.type = QEMU_OPT_STRING,
677
.help = "ID of secret used as password for HTTP proxy auth",
678
},
679
+ {
680
+ .name = CURL_BLOCK_OPT_FORCE_RANGE,
681
+ .type = QEMU_OPT_BOOL,
682
+ .help = "Assume HTTP range requests are supported",
683
+ },
684
{ /* end of list */ }
685
},
686
};
@@ -690,6 +703,7 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
703
#endif
704
const char *secretid;
705
const char *protocol_delimiter;
706
+ bool force_range;
707
int ret;
708
709
bdrv_graph_rdlock_main_loop();
@@ -807,35 +821,56 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
821
}
822
823
s->accept_range = false;
824
+ s->len = UINT64_MAX;
825
+ force_range = qemu_opt_get_bool(opts, CURL_BLOCK_OPT_FORCE_RANGE,
826
+ CURL_BLOCK_OPT_FORCE_RANGE_DEFAULT);
827
+ /*
828
+ * When minimal CURL will be bumped to `7.83`, the header callback + manual
829
+ * parsing can be replaced by `curl_easy_header` calls
830
+ */
831
if (curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L) ||
832
curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, curl_header_cb) ||
833
curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, s)) {
813
- pstrcpy(state->errmsg, CURL_ERROR_SIZE,
814
- "curl library initialization failed.");
815
- goto out;
834
+ goto out_init;
835
+ }
836
+ if (force_range) {
837
+ if (curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, "GET") ||
838
+ curl_easy_setopt(state->curl, CURLOPT_RANGE, "0-0")) {
839
+ goto out_init;
840
+ }
841
}
842
+
843
if (curl_easy_perform(state->curl))
844
goto out;
819
- /* CURL 7.55.0 deprecates CURLINFO_CONTENT_LENGTH_DOWNLOAD in favour of
820
- * the *_T version which returns a more sensible type for content length.
821
- */
845
+
846
+ if (!force_range) {
847
+ /*
848
+ * CURL 7.55.0 deprecates CURLINFO_CONTENT_LENGTH_DOWNLOAD in favour of
849
+ * the *_T version which returns a more sensible type for content
850
+ * length.
851
+ */
852
#if LIBCURL_VERSION_NUM >= 0x073700
823
- if (curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &cl)) {
824
- goto out;
825
- }
853
+ if (curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T,
854
+ &cl)) {
855
+ goto out;
856
+ }
857
#else
827
- if (curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &cl)) {
828
- goto out;
829
- }
858
+ if (curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD,
859
+ &cl)) {
860
+ goto out;
861
+ }
862
#endif
831
- if (cl < 0) {
863
+ if (cl >= 0) {
864
+ s->len = cl;
865
+ }
866
+ }
867
+
868
+ if (s->len == UINT64_MAX) {
869
pstrcpy(state->errmsg, CURL_ERROR_SIZE,
870
"Server didn't report file size.");
871
goto out;
872
}
873
837
- s->len = cl;
838
-
874
if ((!strncasecmp(s->url, "http://", strlen("http://"))
875
|| !strncasecmp(s->url, "https://", strlen("https://")))
876
&& !s->accept_range) {
@@ -856,6 +891,9 @@ static int curl_open(BlockDriverState *bs, QDict *options, int flags,
891
qemu_opts_del(opts);
892
return 0;
893
894
+out_init:
895
+ pstrcpy(state->errmsg, CURL_ERROR_SIZE,
896
+ "curl library initialization failed.");
897
out:
898
error_setg(errp, "CURL: Error opening file: %s", state->errmsg);
899
curl_easy_cleanup(state->curl);
block/trace-events
+1
@@ -191,6 +191,7 @@ ssh_server_status(int status) "server status=%d"
191
curl_timer_cb(long timeout_ms) "timer callback timeout_ms %ld"
192
curl_sock_cb(int action, int fd) "sock action %d on fd %d"
193
curl_read_cb(size_t realsize) "just reading %zu bytes"
194
+curl_header_cb(const char *key, const char *val) "looking at %s: %s"
195
curl_open(const char *file) "opening %s"
196
curl_open_size(uint64_t size) "size = %" PRIu64
197
curl_setup_preadv(uint64_t bytes, uint64_t start, const char *range) "reading %" PRIu64 " at %" PRIu64 " (%s)"
docs/system/device-url-syntax.rst.inc
+6
@@ -179,6 +179,12 @@ These are specified using a special URL syntax.
179
get the size of the image to be downloaded. If not set, the
180
default timeout of 5 seconds is used.
181
182
+ ``force-range``
183
+ Don't issue a HEAD HTTP request to discover if the http server
184
+ server supports range requests and rely only on GET requests. This
185
+ is especially useful for S3 presigned URLs where HEAD requests
186
+ are unauthorized. It defaults to 'false'.
187
+
188
Note that when passing options to qemu explicitly, ``driver`` is the
189
value of <protocol>.
190
qapi/block-core.json
+7
-1
@@ -4587,12 +4587,18 @@
4587
# @cookie-secret: ID of a QCryptoSecret object providing the cookie
4588
# data in a secure way. See @cookie for the format. (since 2.10)
4589
#
4590
+# @force-range: Don't issue a HEAD HTTP request to discover if the
4591
+# http server supports range requests and rely only on GET
4592
+# requests. This is especially useful for S3 presigned URLs where
4593
+# HEAD requests are unauthorized. (default: false; since 11.0)
4594
+#
4595
# Since: 2.9
4596
##
4597
{ 'struct': 'BlockdevOptionsCurlHttp',
4598
'base': 'BlockdevOptionsCurlBase',
4599
'data': { '*cookie': 'str',
4595
- '*cookie-secret': 'str'} }
4600
+ '*cookie-secret': 'str',
4601
+ '*force-range': 'bool'} }
4602
4603
##
4604
# @BlockdevOptionsCurlHttps: