@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4816d20aa4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Fix torch profiling for offline inference (#11125)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Dec 12, 2024 at 07:51 UTC 4816d20aa43fdc4abf66c28f6690a1953d8adbe9
2 files changed +21 -14
examples/offline_inference_with_profiler.py
+19 -12
@@ -1,4 +1,5 @@
1 import os
2 +import time
3
4 from vllm import LLM, SamplingParams
5
@@ -15,19 +16,25 @@ prompts = [
16 # Create a sampling params object.
17 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
18
18 -# Create an LLM.
19 -llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
19 +if __name__ == "__main__":
20
21 -llm.start_profile()
21 + # Create an LLM.
22 + llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
23
23 -# Generate texts from the prompts. The output is a list of RequestOutput objects
24 -# that contain the prompt, generated text, and other information.
25 -outputs = llm.generate(prompts, sampling_params)
24 + llm.start_profile()
25
27 -llm.stop_profile()
26 + # Generate texts from the prompts. The output is a list of RequestOutput
27 + # objects that contain the prompt, generated text, and other information.
28 + outputs = llm.generate(prompts, sampling_params)
29
29 -# Print the outputs.
30 -for output in outputs:
31 - prompt = output.prompt
32 - generated_text = output.outputs[0].text
33 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
30 + llm.stop_profile()
31 +
32 + # Print the outputs.
33 + for output in outputs:
34 + prompt = output.prompt
35 + generated_text = output.outputs[0].text
36 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37 +
38 + # Add a buffer to wait for profiler in the background process
39 + # (in case MP is on) to finish writing profiling output.
40 + time.sleep(10)
vllm/v1/engine/core_client.py
+2 -2
@@ -105,7 +105,7 @@ class InprocClient(EngineCoreClient):
105 def __del__(self):
106 self.shutdown()
107
108 - async def profile(self, is_start=True) -> None:
108 + def profile(self, is_start=True) -> None:
109 self.engine_core.profile(is_start)
110
111
@@ -212,7 +212,7 @@ class SyncMPClient(MPClient):
212 def abort_requests(self, request_ids: List[str]) -> None:
213 self._send_input(EngineCoreRequestType.ABORT, request_ids)
214
215 - async def profile(self, is_start=True) -> None:
215 + def profile(self, is_start=True) -> None:
216 self._send_input(EngineCoreRequestType.PROFILE,
217 EngineCoreProfile(is_start))
218