A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[misc] [doc] [frontend] LLM torch profiler support (#7943)
William Lin committed
Sep 6, 2024 at 17:48 UTC
12dd715807ccbd7fafbb64d42571792db1cc6497
6 files changed
+74
-3
docs/source/dev/profiling/profiling_index.rst
+17
-3
@@ -17,14 +17,28 @@ Traces can be visualized using https://ui.perfetto.dev/.
17
.. tip::
18
19
Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
20
-
21
-Example commands:
20
+
21
+.. tip::
22
+
23
+ To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
24
+ Set the env variable VLLM_RPC_GET_DATA_TIMEOUT_MS to a big number before you start the server. Say something like 30 minutes.
25
+ ``export VLLM_RPC_GET_DATA_TIMEOUT_MS=1800000``
26
+
27
+Example commands and usage:
28
+===========================
29
+
30
+Offline Inference:
31
+------------------
32
+
33
+Refer to `examples/offline_inference_with_profiler.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_with_profiler.py>`_ for an example.
34
+
35
36
OpenAI Server:
37
+--------------
38
39
.. code-block:: bash
40
27
- VLLM_TORCH_PROFILER_DIR=/mnt/traces/ python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
41
+ VLLM_TORCH_PROFILER_DIR=./vllm_profile python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
42
43
benchmark_serving.py:
44
examples/offline_inference_with_profiler.py
new
+33
@@ -0,0 +1,33 @@
1
+import os
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+# enable torch profiler, can also be set on cmd line
6
+os.environ["VLLM_TORCH_PROFILER_DIR"] = "./vllm_profile"
7
+
8
+# Sample prompts.
9
+prompts = [
10
+ "Hello, my name is",
11
+ "The president of the United States is",
12
+ "The capital of France is",
13
+ "The future of AI is",
14
+]
15
+# Create a sampling params object.
16
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
+
18
+# Create an LLM.
19
+llm = LLM(model="facebook/opt-125m")
20
+
21
+llm.start_profile()
22
+
23
+# Generate texts from the prompts. The output is a list of RequestOutput objects
24
+# that contain the prompt, generated text, and other information.
25
+outputs = llm.generate(prompts, sampling_params)
26
+
27
+llm.stop_profile()
28
+
29
+# Print the outputs.
30
+for output in outputs:
31
+ prompt = output.prompt
32
+ generated_text = output.outputs[0].text
33
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/engine/llm_engine.py
+6
@@ -1914,6 +1914,12 @@ class LLMEngine:
1914
self.tokenizer.check_health()
1915
self.model_executor.check_health()
1916
1917
+ def start_profile(self) -> None:
1918
+ self.model_executor.start_profile()
1919
+
1920
+ def stop_profile(self) -> None:
1921
+ self.model_executor.stop_profile()
1922
+
1923
def is_tracing_enabled(self) -> bool:
1924
return self.tracer is not None
1925
vllm/entrypoints/llm.py
+6
@@ -560,6 +560,12 @@ class LLM:
560
outputs = self._run_engine(use_tqdm=use_tqdm)
561
return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
562
563
+ def start_profile(self) -> None:
564
+ self.llm_engine.start_profile()
565
+
566
+ def stop_profile(self) -> None:
567
+ self.llm_engine.stop_profile()
568
+
569
# LEGACY
570
def _convert_v1_inputs(
571
self,
vllm/executor/cpu_executor.py
+6
@@ -296,6 +296,12 @@ class CPUExecutor(ExecutorBase):
296
for result in parallel_worker_tasks:
297
result.get()
298
299
+ def start_profile(self) -> None:
300
+ self.driver_method_invoker(self.driver_worker, "start_profile")
301
+
302
+ def stop_profile(self) -> None:
303
+ self.driver_method_invoker(self.driver_worker, "stop_profile")
304
+
305
306
class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
307
vllm/executor/gpu_executor.py
+6
@@ -169,6 +169,12 @@ class GPUExecutor(ExecutorBase):
169
# it's running.
170
return
171
172
+ def start_profile(self) -> None:
173
+ self.driver_worker.start_profile()
174
+
175
+ def stop_profile(self) -> None:
176
+ self.driver_worker.stop_profile()
177
+
178
179
class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
180