A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 12:45 UTC
[Feature]Supports Anthropic Thinking Block (#33671)
Signed-off-by: mariohong <mariohong128@gmail.com> Co-authored-by: zetaohong <i-hongzetao@stepfun.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Mario Hong committed
Feb 28, 2026 at 17:02 UTC
0892d1ab1f9b3476f31811e851d7b3705dfeaefe
2 files changed
+236
-97
vllm/entrypoints/anthropic/protocol.py
+10
-2
@@ -34,7 +34,7 @@ class AnthropicUsage(BaseModel):
34
class AnthropicContentBlock(BaseModel):
35
"""Content block in message"""
36
37
- type: Literal["text", "image", "tool_use", "tool_result"]
37
+ type: Literal["text", "image", "tool_use", "tool_result", "thinking"]
38
text: str | None = None
39
# For image content
40
source: dict[str, Any] | None = None
@@ -45,6 +45,9 @@ class AnthropicContentBlock(BaseModel):
45
input: dict[str, Any] | None = None
46
content: str | list[dict[str, Any]] | None = None
47
is_error: bool | None = None
48
+ # For thinking content
49
+ thinking: str | None = None
50
+ signature: str | None = None
51
52
53
class AnthropicMessage(BaseModel):
@@ -118,9 +121,14 @@ class AnthropicMessagesRequest(BaseModel):
121
class AnthropicDelta(BaseModel):
122
"""Delta for streaming responses"""
123
121
- type: Literal["text_delta", "input_json_delta"] | None = None
124
+ type: (
125
+ Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
126
+ | None
127
+ ) = None
128
text: str | None = None
129
+ thinking: str | None = None
130
partial_json: str | None = None
131
+ signature: str | None = None
132
133
# Message delta
134
stop_reason: (
vllm/entrypoints/anthropic/serving.py
+226
-95
@@ -8,6 +8,7 @@
8
import json
9
import logging
10
import time
11
+import uuid
12
from collections.abc import AsyncGenerator
13
from typing import Any
14
@@ -112,6 +113,7 @@ class AnthropicServingMessages(OpenAIServingChat):
113
# Handle complex content blocks
114
content_parts: list[dict[str, Any]] = []
115
tool_calls: list[dict[str, Any]] = []
116
+ reasoning_parts: list[str] = []
117
118
for block in msg.content:
119
if block.type == "text" and block.text:
@@ -123,6 +125,8 @@ class AnthropicServingMessages(OpenAIServingChat):
125
"image_url": {"url": block.source.get("data", "")},
126
}
127
)
128
+ elif block.type == "thinking" and block.thinking is not None:
129
+ reasoning_parts.append(block.thinking)
130
elif block.type == "tool_use":
131
# Convert tool use to function call format
132
tool_call = {
@@ -157,6 +161,9 @@ class AnthropicServingMessages(OpenAIServingChat):
161
}
162
)
163
164
+ if reasoning_parts:
165
+ openai_msg["reasoning"] = "".join(reasoning_parts)
166
+
167
# Add tool calls to the message if any
168
if tool_calls:
169
openai_msg["tool_calls"] = tool_calls # type: ignore
@@ -167,7 +174,7 @@ class AnthropicServingMessages(OpenAIServingChat):
174
openai_msg["content"] = content_parts[0]["text"]
175
else:
176
openai_msg["content"] = content_parts # type: ignore
170
- elif not tool_calls:
177
+ elif not tool_calls and not reasoning_parts:
178
continue
179
180
openai_messages.append(openai_msg)
@@ -263,23 +270,32 @@ class AnthropicServingMessages(OpenAIServingChat):
270
output_tokens=generator.usage.completion_tokens,
271
),
272
)
266
- if generator.choices[0].finish_reason == "stop":
273
+ choice = generator.choices[0]
274
+ if choice.finish_reason == "stop":
275
result.stop_reason = "end_turn"
268
- elif generator.choices[0].finish_reason == "length":
276
+ elif choice.finish_reason == "length":
277
result.stop_reason = "max_tokens"
270
- elif generator.choices[0].finish_reason == "tool_calls":
278
+ elif choice.finish_reason == "tool_calls":
279
result.stop_reason = "tool_use"
280
273
- content: list[AnthropicContentBlock] = [
274
- AnthropicContentBlock(
275
- type="text",
276
- text=generator.choices[0].message.content
277
- if generator.choices[0].message.content
278
- else "",
281
+ content: list[AnthropicContentBlock] = []
282
+ if choice.message.reasoning:
283
+ content.append(
284
+ AnthropicContentBlock(
285
+ type="thinking",
286
+ thinking=choice.message.reasoning,
287
+ signature=uuid.uuid4().hex,
288
+ )
289
+ )
290
+ if choice.message.content:
291
+ content.append(
292
+ AnthropicContentBlock(
293
+ type="text",
294
+ text=choice.message.content,
295
+ )
296
)
280
- ]
297
282
- for tool_call in generator.choices[0].message.tool_calls:
298
+ for tool_call in choice.message.tool_calls:
299
anthropic_tool_call = AnthropicContentBlock(
300
type="tool_use",
301
id=tool_call.id,
@@ -297,10 +313,85 @@ class AnthropicServingMessages(OpenAIServingChat):
313
generator: AsyncGenerator[str, None],
314
) -> AsyncGenerator[str, None]:
315
try:
316
+
317
+ class _ActiveBlockState:
318
+ def __init__(self) -> None:
319
+ self.content_block_index = 0
320
+ self.block_type: str | None = None
321
+ self.block_index: int | None = None
322
+ self.block_signature: str | None = None
323
+ self.signature_emitted: bool = False
324
+ self.tool_use_id: str | None = None
325
+
326
+ def reset(self) -> None:
327
+ self.block_type = None
328
+ self.block_index = None
329
+ self.block_signature = None
330
+ self.signature_emitted = False
331
+ self.tool_use_id = None
332
+
333
+ def start(self, block: AnthropicContentBlock) -> None:
334
+ self.block_type = block.type
335
+ self.block_index = self.content_block_index
336
+ if block.type == "thinking":
337
+ self.block_signature = uuid.uuid4().hex
338
+ self.signature_emitted = False
339
+ self.tool_use_id = None
340
+ elif block.type == "tool_use":
341
+ self.block_signature = None
342
+ self.signature_emitted = True
343
+ self.tool_use_id = block.id
344
+ else:
345
+ self.block_signature = None
346
+ self.signature_emitted = True
347
+ self.tool_use_id = None
348
+
349
first_item = True
350
finish_reason = None
302
- content_block_index = 0
303
- content_block_started = False
351
+ state = _ActiveBlockState()
352
+ # Map from tool call index to tool_use_id
353
+ tool_index_to_id: dict[int, str] = {}
354
+
355
+ def stop_active_block():
356
+ events: list[str] = []
357
+ if state.block_type is None:
358
+ return events
359
+ if (
360
+ state.block_type == "thinking"
361
+ and state.block_signature is not None
362
+ and not state.signature_emitted
363
+ ):
364
+ chunk = AnthropicStreamEvent(
365
+ index=state.block_index,
366
+ type="content_block_delta",
367
+ delta=AnthropicDelta(
368
+ type="signature_delta",
369
+ signature=state.block_signature,
370
+ ),
371
+ )
372
+ data = chunk.model_dump_json(exclude_unset=True)
373
+ events.append(wrap_data_with_event(data, "content_block_delta"))
374
+ state.signature_emitted = True
375
+ stop_chunk = AnthropicStreamEvent(
376
+ index=state.block_index,
377
+ type="content_block_stop",
378
+ )
379
+ data = stop_chunk.model_dump_json(exclude_unset=True)
380
+ events.append(wrap_data_with_event(data, "content_block_stop"))
381
+ state.reset()
382
+ state.content_block_index += 1
383
+ return events
384
+
385
+ def start_block(block: AnthropicContentBlock):
386
+ chunk = AnthropicStreamEvent(
387
+ index=state.content_block_index,
388
+ type="content_block_start",
389
+ content_block=block,
390
+ )
391
+ data = chunk.model_dump_json(exclude_unset=True)
392
+ event = wrap_data_with_event(data, "content_block_start")
393
+ state.start(block)
394
+ return event
395
396
async for item in generator:
397
if item.startswith("data:"):
@@ -326,6 +417,8 @@ class AnthropicServingMessages(OpenAIServingChat):
417
id=origin_chunk.id,
418
content=[],
419
model=origin_chunk.model,
420
+ stop_reason=None,
421
+ stop_sequence=None,
422
usage=AnthropicUsage(
423
input_tokens=origin_chunk.usage.prompt_tokens
424
if origin_chunk.usage
@@ -341,13 +434,8 @@ class AnthropicServingMessages(OpenAIServingChat):
434
435
# last chunk including usage info
436
if len(origin_chunk.choices) == 0:
344
- if content_block_started:
345
- stop_chunk = AnthropicStreamEvent(
346
- index=content_block_index,
347
- type="content_block_stop",
348
- )
349
- data = stop_chunk.model_dump_json(exclude_unset=True)
350
- yield wrap_data_with_event(data, "content_block_stop")
437
+ for event in stop_active_block():
438
+ yield event
439
stop_reason = self.stop_reason_map.get(
440
finish_reason or "stop"
441
)
@@ -369,96 +457,139 @@ class AnthropicServingMessages(OpenAIServingChat):
457
458
if origin_chunk.choices[0].finish_reason is not None:
459
finish_reason = origin_chunk.choices[0].finish_reason
372
- continue
460
+ # continue
461
374
- # content
375
- if origin_chunk.choices[0].delta.content is not None:
376
- if not content_block_started:
462
+ # thinking / text content
463
+ reasoning_delta = origin_chunk.choices[0].delta.reasoning
464
+ if reasoning_delta is not None:
465
+ if reasoning_delta == "":
466
+ pass
467
+ else:
468
+ if state.block_type != "thinking":
469
+ for event in stop_active_block():
470
+ yield event
471
+ start_event = start_block(
472
+ AnthropicContentBlock(
473
+ type="thinking", thinking=""
474
+ )
475
+ )
476
+ yield start_event
477
chunk = AnthropicStreamEvent(
378
- index=content_block_index,
379
- type="content_block_start",
380
- content_block=AnthropicContentBlock(
381
- type="text", text=""
478
+ index=(
479
+ state.block_index
480
+ if state.block_index is not None
481
+ else state.content_block_index
482
+ ),
483
+ type="content_block_delta",
484
+ delta=AnthropicDelta(
485
+ type="thinking_delta",
486
+ thinking=reasoning_delta,
487
),
488
)
489
data = chunk.model_dump_json(exclude_unset=True)
385
- yield wrap_data_with_event(data, "content_block_start")
386
- content_block_started = True
490
+ yield wrap_data_with_event(data, "content_block_delta")
491
492
+ if origin_chunk.choices[0].delta.content is not None:
493
if origin_chunk.choices[0].delta.content == "":
389
- continue
390
- chunk = AnthropicStreamEvent(
391
- index=content_block_index,
392
- type="content_block_delta",
393
- delta=AnthropicDelta(
394
- type="text_delta",
395
- text=origin_chunk.choices[0].delta.content,
396
- ),
397
- )
398
- data = chunk.model_dump_json(exclude_unset=True)
399
- yield wrap_data_with_event(data, "content_block_delta")
400
- continue
401
-
402
- # tool calls
403
- elif len(origin_chunk.choices[0].delta.tool_calls) > 0:
404
- tool_call = origin_chunk.choices[0].delta.tool_calls[0]
405
- if tool_call.id is not None:
406
- if content_block_started:
407
- stop_chunk = AnthropicStreamEvent(
408
- index=content_block_index,
409
- type="content_block_stop",
410
- )
411
- data = stop_chunk.model_dump_json(
412
- exclude_unset=True
413
- )
414
- yield wrap_data_with_event(
415
- data, "content_block_stop"
494
+ pass
495
+ else:
496
+ if state.block_type != "text":
497
+ for event in stop_active_block():
498
+ yield event
499
+ start_event = start_block(
500
+ AnthropicContentBlock(type="text", text="")
501
)
417
- content_block_started = False
418
- content_block_index += 1
419
-
502
+ yield start_event
503
chunk = AnthropicStreamEvent(
421
- index=content_block_index,
422
- type="content_block_start",
423
- content_block=AnthropicContentBlock(
424
- type="tool_use",
425
- id=tool_call.id,
426
- name=tool_call.function.name
427
- if tool_call.function
428
- else None,
429
- input={},
504
+ index=(
505
+ state.block_index
506
+ if state.block_index is not None
507
+ else state.content_block_index
508
),
431
- )
432
- data = chunk.model_dump_json(exclude_unset=True)
433
- yield wrap_data_with_event(data, "content_block_start")
434
- content_block_started = True
435
- if tool_call.function and tool_call.function.arguments:
436
- chunk = AnthropicStreamEvent(
437
- index=content_block_index,
438
- type="content_block_delta",
439
- delta=AnthropicDelta(
440
- type="input_json_delta",
441
- partial_json=tool_call.function.arguments,
442
- ),
443
- )
444
- data = chunk.model_dump_json(exclude_unset=True)
445
- yield wrap_data_with_event(
446
- data, "content_block_delta"
447
- )
448
-
449
- else:
450
- chunk = AnthropicStreamEvent(
451
- index=content_block_index,
509
type="content_block_delta",
510
delta=AnthropicDelta(
454
- type="input_json_delta",
455
- partial_json=tool_call.function.arguments
456
- if tool_call.function
457
- else None,
511
+ type="text_delta",
512
+ text=origin_chunk.choices[0].delta.content,
513
),
514
)
515
data = chunk.model_dump_json(exclude_unset=True)
516
yield wrap_data_with_event(data, "content_block_delta")
517
+
518
+ # tool calls - process all tool calls in the delta
519
+ if len(origin_chunk.choices[0].delta.tool_calls) > 0:
520
+ for tool_call in origin_chunk.choices[0].delta.tool_calls:
521
+ if tool_call.id is not None:
522
+ # Update mapping for incremental updates
523
+ tool_index_to_id[tool_call.index] = tool_call.id
524
+ # Only create new block if different tool call
525
+ # AND has a name
526
+ tool_name = (
527
+ tool_call.function.name
528
+ if tool_call.function
529
+ else None
530
+ )
531
+ if (
532
+ state.tool_use_id != tool_call.id
533
+ and tool_name is not None
534
+ ):
535
+ for event in stop_active_block():
536
+ yield event
537
+ start_event = start_block(
538
+ AnthropicContentBlock(
539
+ type="tool_use",
540
+ id=tool_call.id,
541
+ name=tool_name,
542
+ input={},
543
+ )
544
+ )
545
+ yield start_event
546
+ # Handle initial arguments if present
547
+ if (
548
+ tool_call.function
549
+ and tool_call.function.arguments
550
+ and state.tool_use_id == tool_call.id
551
+ ):
552
+ chunk = AnthropicStreamEvent(
553
+ index=(
554
+ state.block_index
555
+ if state.block_index is not None
556
+ else state.content_block_index
557
+ ),
558
+ type="content_block_delta",
559
+ delta=AnthropicDelta(
560
+ type="input_json_delta",
561
+ partial_json=tool_call.function.arguments,
562
+ ),
563
+ )
564
+ data = chunk.model_dump_json(exclude_unset=True)
565
+ yield wrap_data_with_event(
566
+ data, "content_block_delta"
567
+ )
568
+ else:
569
+ # Incremental update - use index to find tool_use_id
570
+ tool_use_id = tool_index_to_id.get(tool_call.index)
571
+ if (
572
+ tool_use_id is not None
573
+ and tool_call.function
574
+ and tool_call.function.arguments
575
+ and state.tool_use_id == tool_use_id
576
+ ):
577
+ chunk = AnthropicStreamEvent(
578
+ index=(
579
+ state.block_index
580
+ if state.block_index is not None
581
+ else state.content_block_index
582
+ ),
583
+ type="content_block_delta",
584
+ delta=AnthropicDelta(
585
+ type="input_json_delta",
586
+ partial_json=tool_call.function.arguments,
587
+ ),
588
+ )
589
+ data = chunk.model_dump_json(exclude_unset=True)
590
+ yield wrap_data_with_event(
591
+ data, "content_block_delta"
592
+ )
593
continue
594
else:
595
error_response = AnthropicStreamEvent(