@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 12dd715807
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[misc] [doc] [frontend] LLM torch profiler support (#7943)

William Lin committed Sep 6, 2024 at 17:48 UTC 12dd715807ccbd7fafbb64d42571792db1cc6497
6 files changed +74 -3
docs/source/dev/profiling/profiling_index.rst
+17 -3
@@ -17,14 +17,28 @@ Traces can be visualized using https://ui.perfetto.dev/.
17 .. tip::
18
19 Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
20 -
21 -Example commands:
20 +
21 +.. tip::
22 +
23 + To stop the profiler - it flushes out all the profile trace files to the directory. This takes time, for example for about 100 requests worth of data for a llama 70b, it takes about 10 minutes to flush out on a H100.
24 + Set the env variable VLLM_RPC_GET_DATA_TIMEOUT_MS to a big number before you start the server. Say something like 30 minutes.
25 + ``export VLLM_RPC_GET_DATA_TIMEOUT_MS=1800000``
26 +
27 +Example commands and usage:
28 +===========================
29 +
30 +Offline Inference:
31 +------------------
32 +
33 +Refer to `examples/offline_inference_with_profiler.py <https://github.com/vllm-project/vllm/blob/main/examples/offline_inference_with_profiler.py>`_ for an example.
34 +
35
36 OpenAI Server:
37 +--------------
38
39 .. code-block:: bash
40
27 - VLLM_TORCH_PROFILER_DIR=/mnt/traces/ python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
41 + VLLM_TORCH_PROFILER_DIR=./vllm_profile python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
42
43 benchmark_serving.py:
44
examples/offline_inference_with_profiler.py new
+33
@@ -0,0 +1,33 @@
1 +import os
2 +
3 +from vllm import LLM, SamplingParams
4 +
5 +# enable torch profiler, can also be set on cmd line
6 +os.environ["VLLM_TORCH_PROFILER_DIR"] = "./vllm_profile"
7 +
8 +# Sample prompts.
9 +prompts = [
10 + "Hello, my name is",
11 + "The president of the United States is",
12 + "The capital of France is",
13 + "The future of AI is",
14 +]
15 +# Create a sampling params object.
16 +sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17 +
18 +# Create an LLM.
19 +llm = LLM(model="facebook/opt-125m")
20 +
21 +llm.start_profile()
22 +
23 +# Generate texts from the prompts. The output is a list of RequestOutput objects
24 +# that contain the prompt, generated text, and other information.
25 +outputs = llm.generate(prompts, sampling_params)
26 +
27 +llm.stop_profile()
28 +
29 +# Print the outputs.
30 +for output in outputs:
31 + prompt = output.prompt
32 + generated_text = output.outputs[0].text
33 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
vllm/engine/llm_engine.py
+6
@@ -1914,6 +1914,12 @@ class LLMEngine:
1914 self.tokenizer.check_health()
1915 self.model_executor.check_health()
1916
1917 + def start_profile(self) -> None:
1918 + self.model_executor.start_profile()
1919 +
1920 + def stop_profile(self) -> None:
1921 + self.model_executor.stop_profile()
1922 +
1923 def is_tracing_enabled(self) -> bool:
1924 return self.tracer is not None
1925
vllm/entrypoints/llm.py
+6
@@ -560,6 +560,12 @@ class LLM:
560 outputs = self._run_engine(use_tqdm=use_tqdm)
561 return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
562
563 + def start_profile(self) -> None:
564 + self.llm_engine.start_profile()
565 +
566 + def stop_profile(self) -> None:
567 + self.llm_engine.stop_profile()
568 +
569 # LEGACY
570 def _convert_v1_inputs(
571 self,
vllm/executor/cpu_executor.py
+6
@@ -296,6 +296,12 @@ class CPUExecutor(ExecutorBase):
296 for result in parallel_worker_tasks:
297 result.get()
298
299 + def start_profile(self) -> None:
300 + self.driver_method_invoker(self.driver_worker, "start_profile")
301 +
302 + def stop_profile(self) -> None:
303 + self.driver_method_invoker(self.driver_worker, "stop_profile")
304 +
305
306 class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
307
vllm/executor/gpu_executor.py
+6
@@ -169,6 +169,12 @@ class GPUExecutor(ExecutorBase):
169 # it's running.
170 return
171
172 + def start_profile(self) -> None:
173 + self.driver_worker.start_profile()
174 +
175 + def stop_profile(self) -> None:
176 + self.driver_worker.stop_profile()
177 +
178
179 class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
180