A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Fix torch profiling for offline inference (#11125)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Dec 12, 2024 at 07:51 UTC
4816d20aa43fdc4abf66c28f6690a1953d8adbe9
2 files changed
+21
-14
examples/offline_inference_with_profiler.py
+19
-12
@@ -1,4 +1,5 @@
1
import os
2
+import time
3
4
from vllm import LLM, SamplingParams
5
@@ -15,19 +16,25 @@ prompts = [
16
# Create a sampling params object.
17
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
18
18
-# Create an LLM.
19
-llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
19
+if __name__ == "__main__":
20
21
-llm.start_profile()
21
+ # Create an LLM.
22
+ llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
23
23
-# Generate texts from the prompts. The output is a list of RequestOutput objects
24
-# that contain the prompt, generated text, and other information.
25
-outputs = llm.generate(prompts, sampling_params)
24
+ llm.start_profile()
25
27
-llm.stop_profile()
26
+ # Generate texts from the prompts. The output is a list of RequestOutput
27
+ # objects that contain the prompt, generated text, and other information.
28
+ outputs = llm.generate(prompts, sampling_params)
29
29
-# Print the outputs.
30
-for output in outputs:
31
- prompt = output.prompt
32
- generated_text = output.outputs[0].text
33
- print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
30
+ llm.stop_profile()
31
+
32
+ # Print the outputs.
33
+ for output in outputs:
34
+ prompt = output.prompt
35
+ generated_text = output.outputs[0].text
36
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
37
+
38
+ # Add a buffer to wait for profiler in the background process
39
+ # (in case MP is on) to finish writing profiling output.
40
+ time.sleep(10)
vllm/v1/engine/core_client.py
+2
-2
@@ -105,7 +105,7 @@ class InprocClient(EngineCoreClient):
105
def __del__(self):
106
self.shutdown()
107
108
- async def profile(self, is_start=True) -> None:
108
+ def profile(self, is_start=True) -> None:
109
self.engine_core.profile(is_start)
110
111
@@ -212,7 +212,7 @@ class SyncMPClient(MPClient):
212
def abort_requests(self, request_ids: List[str]) -> None:
213
self._send_input(EngineCoreRequestType.ABORT, request_ids)
214
215
- async def profile(self, is_start=True) -> None:
215
+ def profile(self, is_start=True) -> None:
216
self._send_input(EngineCoreRequestType.PROFILE,
217
EngineCoreProfile(is_start))
218