A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[ROCm][bugfix] scpecilative decoding worker class (#11035)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Dec 9, 2024 at 17:00 UTC
b63ba848323efd88207b12d7582501d525503b8a
1 file changed
+2
vllm/platforms/rocm.py
+2
@@ -93,6 +93,8 @@ class RocmPlatform(Platform):
93
elif vllm_config.speculative_config:
94
parallel_config.worker_cls = \
95
"vllm.spec_decode.spec_decode_worker.create_spec_worker"
96
+ parallel_config.sd_worker_cls = \
97
+ "vllm.worker.worker.Worker"
98
else:
99
parallel_config.worker_cls = "vllm.worker.worker.Worker"
100