A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix] Fix profiling.py (#16202)
Signed-off-by: zh Wang <rekind133@outlook.com>
zh Wang committed
Apr 10, 2025 at 01:03 UTC
a25866ac8d8f6eb50e18db65e856b8330854c878
1 file changed
+2
-3
examples/offline_inference/profiling.py
+2
-3
@@ -234,9 +234,8 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
234
sampling_params.max_tokens = next(output_len_generator)
235
assert isinstance(sampling_params.max_tokens, int)
236
237
- prompt_token_ids = torch.randint(
238
- llm.llm_engine.model_config.get_vocab_size(),
239
- size=(prompt_len, )).tolist()
237
+ prompt_token_ids = torch.randint(llm.get_tokenizer().vocab_size,
238
+ size=(prompt_len, )).tolist()
239
240
llm.llm_engine.add_request(
241
request_id=f"seq{i}",