A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[V1][Structured Output] Add `supports_structured_output()` method to Platform (#16148)
Signed-off-by: shen-shanshan <467638484@qq.com>
Shanshan Shen committed
Apr 7, 2025 at 19:06 UTC
e9ba99f296e77038f59f59e33579f4011ed56411
9 files changed
+41
-3
vllm/platforms/cpu.py
+4
@@ -180,3 +180,7 @@ class CpuPlatform(Platform):
180
Get device specific communicator class for distributed communication.
181
"""
182
return "vllm.distributed.device_communicators.cpu_communicator.CpuCommunicator" # noqa
183
+
184
+ @classmethod
185
+ def supports_structured_output(cls) -> bool:
186
+ return True
vllm/platforms/cuda.py
+4
@@ -308,6 +308,10 @@ class CudaPlatformBase(Platform):
308
def supports_v1(cls, model_config: ModelConfig) -> bool:
309
return True
310
311
+ @classmethod
312
+ def supports_structured_output(cls) -> bool:
313
+ return True
314
+
315
@classmethod
316
def use_custom_allreduce(cls) -> bool:
317
return True
vllm/platforms/hpu.py
+4
@@ -92,3 +92,7 @@ class HpuPlatform(Platform):
92
@classmethod
93
def get_device_communicator_cls(cls) -> str:
94
return "vllm.distributed.device_communicators.hpu_communicator.HpuCommunicator" # noqa
95
+
96
+ @classmethod
97
+ def supports_structured_output(cls) -> bool:
98
+ return True
vllm/platforms/interface.py
+7
@@ -379,6 +379,13 @@ class Platform:
379
"""
380
return False
381
382
+ @classmethod
383
+ def supports_structured_output(cls) -> bool:
384
+ """
385
+ Returns whether the current platform can support structured output.
386
+ """
387
+ return False
388
+
389
@classmethod
390
def use_custom_allreduce(cls) -> bool:
391
"""
vllm/platforms/neuron.py
+4
@@ -67,3 +67,7 @@ class NeuronPlatform(Platform):
67
@classmethod
68
def use_all_gather(cls) -> bool:
69
return True
70
+
71
+ @classmethod
72
+ def supports_structured_output(cls) -> bool:
73
+ return True
vllm/platforms/rocm.py
+4
@@ -303,6 +303,10 @@ class RocmPlatform(Platform):
303
# V1 support on AMD gpus is experimental
304
return True
305
306
+ @classmethod
307
+ def supports_structured_output(cls) -> bool:
308
+ return True
309
+
310
@classmethod
311
def use_custom_allreduce(cls) -> bool:
312
# We only enable custom allreduce for MI300 series
vllm/platforms/tpu.py
+5
@@ -133,3 +133,8 @@ class TpuPlatform(Platform):
133
def supports_v1(cls, model_config: ModelConfig) -> bool:
134
# V1 support on TPU is experimental
135
return True
136
+
137
+ @classmethod
138
+ def supports_structured_output(cls) -> bool:
139
+ # Structured output is not supported on TPU.
140
+ return False
vllm/platforms/xpu.py
+4
@@ -140,3 +140,7 @@ class XPUPlatform(Platform):
140
@classmethod
141
def get_device_communicator_cls(cls) -> str:
142
return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa
143
+
144
+ @classmethod
145
+ def supports_structured_output(cls) -> bool:
146
+ return True
vllm/v1/engine/processor.py
+5
-3
@@ -136,9 +136,11 @@ class Processor:
136
f" != {engine_level_backend}")
137
else:
138
params.guided_decoding.backend = engine_level_backend
139
- import vllm.platforms
140
- if vllm.platforms.current_platform.is_tpu():
141
- raise ValueError("Structured output is not supported on TPU.")
139
+
140
+ from vllm.platforms import current_platform
141
+ if not current_platform.supports_structured_output():
142
+ raise ValueError("Structured output is not supported on "
143
+ f"{current_platform.device_name}.")
144
145
# Request content validation
146
if engine_level_backend.startswith("xgrammar"):