@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2094062b4e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[4.5/N] bugfix for quant config in speculative decode (#10007)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 4, 2024 at 15:11 UTC 2094062b4eafe465826e936fbd5cbd8f099d7762
1 file changed +4
vllm/spec_decode/spec_decode_worker.py
+4
@@ -61,6 +61,10 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
61
62 draft_worker_config = copy.deepcopy(vllm_config)
63 draft_worker_config.model_config = speculative_config.draft_model_config
64 + draft_worker_config.quant_config = VllmConfig._get_quantization_config(
65 + draft_worker_config.model_config,
66 + vllm_config.load_config,
67 + )
68 draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
69 # TODO allow draft-model specific load config.
70