A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] Fix cuda graph for MLPSpeculator (#5875)
Co-authored-by: Abhinav Goyal <abhinav.goyal@flipkart.com>
Nick Hill committed
Jun 26, 2024 at 21:12 UTC
2110557dabe8a18b811116c1ae9fdf75fbe27df6
2 files changed
+6
-4
examples/offline_inference_mlpspeculator.py
-1
@@ -52,7 +52,6 @@ if __name__ == "__main__":
52
speculative_model="ibm-fms/llama-13b-accelerator",
53
# These are currently required for MLPSpeculator decoding
54
use_v2_block_manager=True,
55
- enforce_eager=True,
55
)
56
57
print("With speculation")
vllm/worker/model_runner.py
+6
-3
@@ -1020,10 +1020,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1020
1021
if self.return_hidden_states:
1022
# we only need to pass hidden states of most recent token
1023
+ assert model_input.sampling_metadata is not None
1024
+ indices = model_input.sampling_metadata.selected_token_indices
1025
if model_input.is_prompt:
1024
- assert model_input.sampling_metadata is not None
1025
- hidden_states = hidden_states.index_select(
1026
- 0, model_input.sampling_metadata.selected_token_indices)
1026
+ hidden_states = hidden_states.index_select(0, indices)
1027
+ elif decode_meta.use_cuda_graph:
1028
+ hidden_states = hidden_states[:len(indices)]
1029
+
1030
output.hidden_states = hidden_states
1031
1032
return output