[Feature][Frontend] Add DeepSeek-V4 FIM completion rendering (#44229)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
jack committed
Sep 23, 2026 at 21:02 UTC
d95a896ea612a072ba986a708d7efb4de1547019
5 files changed
+243
-7
tests/entrypoints/openai/completion/test_completion_error.py
+178
@@ -14,8 +14,10 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
from vllm.entrypoints.scale_out.render.serving import ServingRender
17
+from vllm.entrypoints.serve.engine.protocol import ErrorResponse
18
from vllm.exceptions import GenerationError, VLLMValidationError
19
from vllm.outputs import CompletionOutput, RequestOutput
20
+from vllm.renderers.deepseek_v4 import DeepseekV4Renderer
21
from vllm.renderers.hf import HfRenderer
22
from vllm.renderers.online_renderer import OnlineRenderer
23
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -24,6 +26,9 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
26
27
MODEL_NAME = "openai-community/gpt2"
28
MODEL_NAME_SHORT = "gpt2"
29
+DEEPSEEK_V4_FIM_BEGIN = "<｜fim▁begin｜>"
30
+DEEPSEEK_V4_FIM_HOLE = "<｜fim▁hole｜>"
31
+DEEPSEEK_V4_FIM_END = "<｜fim▁end｜>"
32
_PER_REQUEST_STATS = RequestStateStats(
33
queued_ts=1.0,
34
scheduled_ts=1.5,
@@ -302,6 +307,179 @@ def test_completion_metrics_carries_both_timing_and_spec_decode():
307
assert response.metrics.speculative_decoding.num_spec_steps == 2
308
309
310
+def _build_online_renderer_for_completion(
311
+ model_config: MockModelConfig,
312
+ renderer: Any | None = None,
313
+) -> OnlineRenderer:
314
+ if renderer is None:
315
+ renderer = MagicMock()
316
+ renderer.render_completion_suffix.return_value = None
317
+
318
+ return OnlineRenderer(
319
+ model_config=model_config,
320
+ renderer=renderer,
321
+ request_logger=None,
322
+ chat_template=None,
323
+ chat_template_content_format="auto",
324
+ )
325
+
326
+
327
+def _deepseek_v4_suffix_renderer(
328
+ model_config: MockModelConfig,
329
+) -> DeepseekV4Renderer:
330
+ return DeepseekV4Renderer(
331
+ MockVllmConfig(model_config, parallel_config=MockParallelConfig()),
332
+ None,
333
+ )
334
+
335
+
336
+@pytest.mark.asyncio
337
+@pytest.mark.skip_global_cleanup
338
+async def test_completion_suffix_uses_renderer_capability():
339
+ model_config = MockModelConfig()
340
+ online_renderer = _build_online_renderer_for_completion(
341
+ model_config,
342
+ renderer=_deepseek_v4_suffix_renderer(model_config),
343
+ )
344
+ online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
345
+
346
+ request = CompletionRequest(
347
+ model=MODEL_NAME,
348
+ prompt="def fib(n):\n return ",
349
+ suffix="\n\nprint(fib(10))",
350
+ max_tokens=64,
351
+ )
352
+
353
+ result = await online_renderer.render_completion(request)
354
+
355
+ assert result == [{"ok": True}]
356
+ online_renderer.preprocess_completion.assert_awaited_once()
357
+ assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == (
358
+ f"{DEEPSEEK_V4_FIM_BEGIN}def fib(n):\n return "
359
+ f"{DEEPSEEK_V4_FIM_HOLE}\n\nprint(fib(10))"
360
+ f"{DEEPSEEK_V4_FIM_END}"
361
+ )
362
+
363
+
364
+@pytest.mark.asyncio
365
+@pytest.mark.skip_global_cleanup
366
+async def test_completion_suffix_supports_text_prompt_list():
367
+ model_config = MockModelConfig()
368
+ online_renderer = _build_online_renderer_for_completion(
369
+ model_config,
370
+ renderer=_deepseek_v4_suffix_renderer(model_config),
371
+ )
372
+ online_renderer.preprocess_completion = AsyncMock(return_value=[{"ok": True}])
373
+
374
+ request = CompletionRequest(
375
+ model=MODEL_NAME,
376
+ prompt=["prefix A", "prefix B"],
377
+ suffix=" suffix",
378
+ )
379
+
380
+ result = await online_renderer.render_completion(request)
381
+
382
+ assert result == [{"ok": True}]
383
+ assert online_renderer.preprocess_completion.call_args.kwargs["prompt_input"] == [
384
+ f"{DEEPSEEK_V4_FIM_BEGIN}prefix A"
385
+ f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
386
+ f"{DEEPSEEK_V4_FIM_BEGIN}prefix B"
387
+ f"{DEEPSEEK_V4_FIM_HOLE} suffix{DEEPSEEK_V4_FIM_END}",
388
+ ]
389
+
390
+
391
+@pytest.mark.asyncio
392
+@pytest.mark.skip_global_cleanup
393
+async def test_completion_suffix_rejects_renderer_without_fim_support():
394
+ model_config = MockModelConfig()
395
+ online_renderer = _build_online_renderer_for_completion(model_config)
396
+ online_renderer.preprocess_completion = AsyncMock()
397
+
398
+ request = CompletionRequest(
399
+ model=MODEL_NAME,
400
+ prompt="prefix",
401
+ suffix="suffix",
402
+ )
403
+
404
+ result = await online_renderer.render_completion(request)
405
+
406
+ assert isinstance(result, ErrorResponse)
407
+ assert "FIM completion rendering" in result.error.message
408
+ online_renderer.preprocess_completion.assert_not_awaited()
409
+
410
+
411
+@pytest.mark.asyncio
412
+@pytest.mark.skip_global_cleanup
413
+async def test_completion_suffix_rejects_echo():
414
+ model_config = MockModelConfig()
415
+ online_renderer = _build_online_renderer_for_completion(
416
+ model_config,
417
+ renderer=_deepseek_v4_suffix_renderer(model_config),
418
+ )
419
+ online_renderer.preprocess_completion = AsyncMock()
420
+
421
+ request = CompletionRequest(
422
+ model=MODEL_NAME,
423
+ prompt="prefix",
424
+ suffix="suffix",
425
+ echo=True,
426
+ )
427
+
428
+ result = await online_renderer.render_completion(request)
429
+
430
+ assert isinstance(result, ErrorResponse)
431
+ assert "Echo is unsupported with suffix" in result.error.message
432
+ online_renderer.preprocess_completion.assert_not_awaited()
433
+
434
+
435
+@pytest.mark.asyncio
436
+@pytest.mark.skip_global_cleanup
437
+async def test_completion_suffix_rejects_prompt_embeds():
438
+ model_config = MockModelConfig()
439
+ online_renderer = _build_online_renderer_for_completion(
440
+ model_config,
441
+ renderer=_deepseek_v4_suffix_renderer(model_config),
442
+ )
443
+ online_renderer.preprocess_completion = AsyncMock()
444
+
445
+ request = CompletionRequest(
446
+ model=MODEL_NAME,
447
+ prompt="prefix",
448
+ suffix="suffix",
449
+ prompt_embeds=b"embeds",
450
+ )
451
+
452
+ result = await online_renderer.render_completion(request)
453
+
454
+ assert isinstance(result, ErrorResponse)
455
+ assert "suffix is not supported with prompt_embeds" in result.error.message
456
+ online_renderer.preprocess_completion.assert_not_awaited()
457
+
458
+
459
+@pytest.mark.asyncio
460
+@pytest.mark.skip_global_cleanup
461
+@pytest.mark.parametrize("prompt", ([1, 2, 3], [[1, 2, 3]]))
462
+async def test_completion_suffix_rejects_token_prompt(prompt):
463
+ model_config = MockModelConfig()
464
+ online_renderer = _build_online_renderer_for_completion(
465
+ model_config,
466
+ renderer=_deepseek_v4_suffix_renderer(model_config),
467
+ )
468
+ online_renderer.preprocess_completion = AsyncMock()
469
+
470
+ request = CompletionRequest(
471
+ model=MODEL_NAME,
472
+ prompt=prompt,
473
+ suffix="suffix",
474
+ )
475
+
476
+ result = await online_renderer.render_completion(request)
477
+
478
+ assert isinstance(result, ErrorResponse)
479
+ assert "requires text prompt input" in result.error.message
480
+ online_renderer.preprocess_completion.assert_not_awaited()
481
+
482
+
483
@pytest.mark.asyncio
484
async def test_completion_error_non_stream():
485
"""Test finish_reason='error' returns 500 InternalServerError (non-streaming)."""
vllm/entrypoints/openai/completion/serving.py
+1
-3
@@ -116,9 +116,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
116
See https://platform.openai.com/docs/api-reference/completions/create
117
for the API specification. This API mimics the OpenAI Completion API.
118
119
- NOTE: Currently we do not support the following feature:
120
- - suffix (the language models we currently support do not support
121
- suffix)
119
+ NOTE: suffix is only supported by models that implement FIM rendering.
120
"""
121
return await self._with_kv_transfer_rejection_cleanup(
122
self._create_completion(request, raw_request), request, raw_request
vllm/renderers/base.py
+4
@@ -193,6 +193,10 @@ class BaseRenderer(ABC, Generic[_T]):
193
194
return tokenizer
195
196
+ def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
197
+ """Render OpenAI completion suffix input when the renderer supports FIM."""
198
+ return None
199
+
200
def _decode(self, *args, **kwargs):
201
return self.get_tokenizer().decode(*args, **kwargs)
202
vllm/renderers/deepseek_v4.py
+7
@@ -16,6 +16,10 @@ from .inputs import DictPrompt
16
from .inputs.preprocess import parse_dec_only_prompt
17
from .params import ChatParams
18
19
+_FIM_BEGIN = "<｜fim▁begin｜>"
20
+_FIM_HOLE = "<｜fim▁hole｜>"
21
+_FIM_END = "<｜fim▁end｜>"
22
+
23
24
class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
25
def __init__(
@@ -32,6 +36,9 @@ class DeepseekV4Renderer(BaseRenderer[DeepseekV4Tokenizer]):
36
def _apply_chat_template(self, *args, **kwargs):
37
return self.get_tokenizer().apply_chat_template(*args, **kwargs)
38
39
+ def render_completion_suffix(self, prompt: str, suffix: str) -> str | None:
40
+ return f"{_FIM_BEGIN}{prompt}{_FIM_HOLE}{suffix}{_FIM_END}"
41
+
42
def render_messages(
43
self,
44
messages: list[ChatCompletionMessageParam],
vllm/renderers/online_renderer.py
+53
-4
@@ -4,7 +4,7 @@ import time
4
from collections.abc import Sequence
5
from dataclasses import dataclass
6
from http import HTTPStatus
7
-from typing import TYPE_CHECKING, Any, TypeAlias
7
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
8
9
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
10
from openai.types.responses.tool import Mcp, Tool
@@ -548,9 +548,58 @@ class OnlineRenderer:
548
Called directly by render_completion_request and delegated to by
549
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
550
"""
551
- # Return error for unsupported features.
551
+ prompt_input = request.prompt
552
if request.suffix is not None:
553
- return self.create_error_response("suffix is not currently supported")
553
+ if request.echo:
554
+ return self.create_error_response(
555
+ "Echo is unsupported with suffix.",
556
+ param="suffix",
557
+ )
558
+
559
+ if request.prompt_embeds is not None:
560
+ return self.create_error_response(
561
+ "suffix is not supported with prompt_embeds",
562
+ param="suffix",
563
+ )
564
+
565
+ if request.truncate_prompt_tokens is not None:
566
+ return self.create_error_response(
567
+ "suffix is not supported with truncate_prompt_tokens",
568
+ param="suffix",
569
+ )
570
+
571
+ if isinstance(request.prompt, str):
572
+ rendered_prompt = self.renderer.render_completion_suffix(
573
+ request.prompt, request.suffix
574
+ )
575
+ if rendered_prompt is None:
576
+ return self.create_error_response(
577
+ "suffix is only supported for models with FIM completion "
578
+ "rendering",
579
+ param="suffix",
580
+ )
581
+ prompt_input = rendered_prompt
582
+ elif isinstance(request.prompt, list) and all(
583
+ isinstance(prompt, str) for prompt in request.prompt
584
+ ):
585
+ rendered_prompts = []
586
+ for prompt in cast(list[str], request.prompt):
587
+ rendered_prompt = self.renderer.render_completion_suffix(
588
+ prompt, request.suffix
589
+ )
590
+ if rendered_prompt is None:
591
+ return self.create_error_response(
592
+ "suffix is only supported for models with FIM completion "
593
+ "rendering",
594
+ param="suffix",
595
+ )
596
+ rendered_prompts.append(rendered_prompt)
597
+ prompt_input = rendered_prompts
598
+ else:
599
+ return self.create_error_response(
600
+ "suffix requires text prompt input for FIM completion rendering",
601
+ param="suffix",
602
+ )
603
604
if request.echo and request.prompt_embeds is not None:
605
return self.create_error_response("Echo is unsupported with prompt embeds.")
@@ -562,7 +611,7 @@ class OnlineRenderer:
611
612
engine_inputs = await self.preprocess_completion(
613
request,
565
- prompt_input=request.prompt,
614
+ prompt_input=prompt_input,
615
prompt_embeds=request.prompt_embeds,
616
skip_mm_cache=skip_mm_cache,
617
)