@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2110557dab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[BugFix] Fix cuda graph for MLPSpeculator (#5875)

Co-authored-by: Abhinav Goyal <abhinav.goyal@flipkart.com>

Nick Hill committed Jun 26, 2024 at 21:12 UTC 2110557dabe8a18b811116c1ae9fdf75fbe27df6
2 files changed +6 -4
examples/offline_inference_mlpspeculator.py
-1
@@ -52,7 +52,6 @@ if __name__ == "__main__":
52 speculative_model="ibm-fms/llama-13b-accelerator",
53 # These are currently required for MLPSpeculator decoding
54 use_v2_block_manager=True,
55 - enforce_eager=True,
55 )
56
57 print("With speculation")
vllm/worker/model_runner.py
+6 -3
@@ -1020,10 +1020,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1020
1021 if self.return_hidden_states:
1022 # we only need to pass hidden states of most recent token
1023 + assert model_input.sampling_metadata is not None
1024 + indices = model_input.sampling_metadata.selected_token_indices
1025 if model_input.is_prompt:
1024 - assert model_input.sampling_metadata is not None
1025 - hidden_states = hidden_states.index_select(
1026 - 0, model_input.sampling_metadata.selected_token_indices)
1026 + hidden_states = hidden_states.index_select(0, indices)
1027 + elif decode_meta.use_cuda_graph:
1028 + hidden_states = hidden_states[:len(indices)]
1029 +
1030 output.hidden_states = hidden_states
1031
1032 return output