@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dd53c4b023
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc] Add Torch profiler support (#7451)

Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

William Lin committed Aug 21, 2024 at 15:39 UTC dd53c4b023056cda6174cc32dc3d31bc01e8646a
12 files changed +191 -2
benchmarks/backend_request_func.py
+2 -2
@@ -225,8 +225,8 @@ async def async_request_openai_completions(
225 ) -> RequestFuncOutput:
226 api_url = request_func_input.api_url
227 assert api_url.endswith(
228 - "completions"
229 - ), "OpenAI Completions API URL must end with 'completions'."
228 + ("completions", "profile")
229 + ), "OpenAI Completions API URL must end with 'completions' or 'profile'."
230
231 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
232 assert not request_func_input.use_beam_search
benchmarks/benchmark_serving.py
+43
@@ -295,6 +295,7 @@ def calculate_metrics(
295 async def benchmark(
296 backend: str,
297 api_url: str,
298 + base_url: str,
299 model_id: str,
300 tokenizer: PreTrainedTokenizerBase,
301 input_requests: List[Tuple[str, int, int]],
@@ -302,6 +303,7 @@ async def benchmark(
303 use_beam_search: bool,
304 request_rate: float,
305 disable_tqdm: bool,
306 + profile: bool,
307 ):
308 if backend in ASYNC_REQUEST_FUNCS:
309 request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -326,6 +328,22 @@ async def benchmark(
328 f"are correctly specified. Error: {test_output.error}")
329 else:
330 print("Initial test run completed. Starting main benchmark run...")
331 +
332 + if profile:
333 + print("Starting profiler...")
334 + profile_input = RequestFuncInput(
335 + model=model_id,
336 + prompt=test_prompt,
337 + api_url=base_url + "/start_profile",
338 + prompt_len=test_prompt_len,
339 + output_len=test_output_len,
340 + best_of=best_of,
341 + use_beam_search=use_beam_search,
342 + )
343 + profile_output = await request_func(request_func_input=profile_input)
344 + if profile_output.success:
345 + print("Profiler started")
346 +
347 print(f"Traffic request rate: {request_rate}")
348
349 pbar = None if disable_tqdm else tqdm(total=len(input_requests))
@@ -349,6 +367,21 @@ async def benchmark(
367 pbar=pbar)))
368 outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
369
370 + if profile:
371 + print("Stopping profiler...")
372 + profile_input = RequestFuncInput(
373 + model=model_id,
374 + prompt=test_prompt,
375 + api_url=base_url + "/stop_profile",
376 + prompt_len=test_prompt_len,
377 + output_len=test_output_len,
378 + best_of=best_of,
379 + use_beam_search=use_beam_search,
380 + )
381 + profile_output = await request_func(request_func_input=profile_input)
382 + if profile_output.success:
383 + print("Profiler stopped")
384 +
385 if pbar is not None:
386 pbar.close()
387
@@ -433,8 +466,10 @@ def main(args: argparse.Namespace):
466
467 if args.base_url is not None:
468 api_url = f"{args.base_url}{args.endpoint}"
469 + base_url = f"{args.base_url}"
470 else:
471 api_url = f"http://{args.host}:{args.port}{args.endpoint}"
472 + base_url = f"http://{args.host}:{args.port}"
473
474 tokenizer = get_tokenizer(tokenizer_id,
475 trust_remote_code=args.trust_remote_code)
@@ -506,6 +541,7 @@ def main(args: argparse.Namespace):
541 benchmark(
542 backend=backend,
543 api_url=api_url,
544 + base_url=base_url,
545 model_id=model_id,
546 tokenizer=tokenizer,
547 input_requests=input_requests,
@@ -513,6 +549,7 @@ def main(args: argparse.Namespace):
549 use_beam_search=args.use_beam_search,
550 request_rate=args.request_rate,
551 disable_tqdm=args.disable_tqdm,
552 + profile=args.profile,
553 ))
554
555 # Save config and results to json
@@ -693,6 +730,12 @@ if __name__ == "__main__":
730 action="store_true",
731 help="Specify to disable tqdm progress bar.",
732 )
733 + parser.add_argument(
734 + "--profile",
735 + action="store_true",
736 + help="Use Torch Profiler. The endpoint must be launched with "
737 + "VLLM_TORCH_PROFILER_DIR to enable profiler.",
738 + )
739 parser.add_argument(
740 "--save-result",
741 action="store_true",
docs/source/dev/profiling/profiling_index.rst new
+33
@@ -0,0 +1,33 @@
1 +Profiling vLLM
2 +=================================
3 +
4 +We support tracing vLLM workers using the ``torch.profiler`` module. You can enable tracing by setting the ``VLLM_TORCH_PROFILER_DIR`` environment variable to the directory where you want to save the traces: ``VLLM_TORCH_PROFILER_DIR=/mnt/traces/``
5 +
6 +The OpenAI server also needs to be started with the ``VLLM_TORCH_PROFILER_DIR`` environment variable set.
7 +
8 +When using ``benchmarks/benchmark_serving.py``, you can enable profiling by passing the ``--profile`` flag.
9 +
10 +.. warning::
11 +
12 + Only enable profiling in a development environment.
13 +
14 +
15 +Traces can be visualized using https://ui.perfetto.dev/.
16 +
17 +.. tip::
18 +
19 + Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
20 +
21 +Example commands:
22 +
23 +OpenAI Server:
24 +
25 +.. code-block:: bash
26 +
27 + VLLM_TORCH_PROFILER_DIR=/mnt/traces/ python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
28 +
29 +benchmark_serving.py:
30 +
31 +.. code-block:: bash
32 +
33 + python benchmarks/benchmark_serving.py --backend vllm --model meta-llama/Meta-Llama-3-70B --dataset-name sharegpt --dataset-path sharegpt.json --profile --num-prompts 2
\ No newline at end of file
docs/source/index.rst
+1
@@ -136,6 +136,7 @@ Documentation
136 dev/input_processing/model_inputs_index
137 dev/multimodal/multimodal_index
138 dev/dockerfile/dockerfile
139 + dev/profiling/profiling_index
140
141 .. toctree::
142 :maxdepth: 1
vllm/engine/async_llm_engine.py
+6
@@ -1266,3 +1266,9 @@ class AsyncLLMEngine:
1266 logger_name=logger_name))
1267 else:
1268 self.engine.remove_logger(logger_name=logger_name)
1269 +
1270 + async def start_profile(self) -> None:
1271 + self.engine.model_executor._run_workers("start_profile")
1272 +
1273 + async def stop_profile(self) -> None:
1274 + self.engine.model_executor._run_workers("stop_profile")
vllm/engine/protocol.py
+8
@@ -91,3 +91,11 @@ class AsyncEngineClient(Protocol):
91 async def check_health(self) -> None:
92 """Raise if unhealthy"""
93 ...
94 +
95 + async def start_profile(self) -> None:
96 + """Start profiling the engine"""
97 + ...
98 +
99 + async def stop_profile(self) -> None:
100 + """Start profiling the engine"""
101 + ...
vllm/entrypoints/openai/api_server.py
+20
@@ -305,6 +305,26 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
305 assert_never(generator)
306
307
308 +if envs.VLLM_TORCH_PROFILER_DIR:
309 + logger.warning(
310 + "Torch Profiler is enabled in the API server. This should ONLY be "
311 + "used for local development!")
312 +
313 + @router.post("/start_profile")
314 + async def start_profile():
315 + logger.info("Starting profiler...")
316 + await async_engine_client.start_profile()
317 + logger.info("Profiler started.")
318 + return Response(status_code=200)
319 +
320 + @router.post("/stop_profile")
321 + async def stop_profile():
322 + logger.info("Stopping profiler...")
323 + await async_engine_client.stop_profile()
324 + logger.info("Profiler stopped.")
325 + return Response(status_code=200)
326 +
327 +
328 def build_app(args: Namespace) -> FastAPI:
329 app = FastAPI(lifespan=lifespan)
330 app.include_router(router)
vllm/entrypoints/openai/rpc/__init__.py
+2
@@ -46,6 +46,8 @@ class RPCUtilityRequest(Enum):
46 DO_LOG_STATS = 7
47 IS_SERVER_HEALTHY = 8
48 IS_TRACING_ENABLED = 9
49 + START_PROFILE = 10
50 + STOP_PROFILE = 11
51
52
53 RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
vllm/entrypoints/openai/rpc/client.py
+14
@@ -400,3 +400,17 @@ class AsyncEngineRPCClient:
400 **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
401 raise NotImplementedError(
402 "Embeddings not supported with multiprocessing backend")
403 +
404 + async def start_profile(self) -> None:
405 + """Start profiling the engine"""
406 +
407 + await self._send_one_way_rpc_request(
408 + request=RPCUtilityRequest.START_PROFILE,
409 + error_message="RPCRequest START_PROFILE failed.")
410 +
411 + async def stop_profile(self) -> None:
412 + """Stop profiling the engine"""
413 +
414 + await self._send_one_way_rpc_request(
415 + request=RPCUtilityRequest.STOP_PROFILE,
416 + error_message="RPCRequest STOP_PROFILE failed.")
\ No newline at end of file
vllm/entrypoints/openai/rpc/server.py
+24
@@ -124,6 +124,26 @@ class AsyncEngineRPCServer:
124 except Exception as e:
125 await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
126
127 + async def start_profile(self, identity):
128 + logger.info("Starting profiler...")
129 + await self.engine.start_profile()
130 + logger.info("Profiler started.")
131 +
132 + await self.socket.send_multipart([
133 + identity,
134 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
135 + ])
136 +
137 + async def stop_profile(self, identity):
138 + logger.info("Stopping profiler...")
139 + await self.engine.stop_profile()
140 + logger.info("Profiler stopped.")
141 +
142 + await self.socket.send_multipart([
143 + identity,
144 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
145 + ])
146 +
147 def _make_handler_coro(self, identity,
148 message) -> Coroutine[Any, Any, Never]:
149 """Route the zmq message to the handler coroutine."""
@@ -153,6 +173,10 @@ class AsyncEngineRPCServer:
173 return self.check_health(identity)
174 elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
175 return self.is_tracing_enabled(identity)
176 + elif request == RPCUtilityRequest.START_PROFILE:
177 + return self.start_profile(identity)
178 + elif request == RPCUtilityRequest.STOP_PROFILE:
179 + return self.stop_profile(identity)
180 else:
181 raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
182
vllm/envs.py
+7
@@ -58,6 +58,7 @@ if TYPE_CHECKING:
58 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
59 VLLM_ALLOW_ENGINE_USE_RAY: bool = False
60 VLLM_PLUGINS: Optional[List[str]] = None
61 + VLLM_TORCH_PROFILER_DIR: Optional[str] = None
62
63
64 def get_default_cache_root():
@@ -384,6 +385,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
385 "VLLM_PLUGINS":
386 lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
387 "VLLM_PLUGINS"].split(","),
388 +
389 + # Enables torch profiler if set. Path to the directory where torch profiler
390 + # traces are saved. Note that it must be an absolute path.
391 + "VLLM_TORCH_PROFILER_DIR":
392 + lambda: (None if os.getenv("VLLM_TORCH_PROFILER_DIR", None) is None else os
393 + .path.expanduser(os.getenv("VLLM_TORCH_PROFILER_DIR", "."))),
394 }
395
396 # end-env-vars-definition
vllm/worker/worker.py
+31
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple, Type, Union
6 import torch
7 import torch.distributed
8
9 +import vllm.envs as envs
10 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 ModelConfig, ObservabilityConfig, ParallelConfig,
12 PromptAdapterConfig, SchedulerConfig,
@@ -13,6 +14,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14 from vllm.distributed import (ensure_model_parallel_initialized,
15 init_distributed_environment,
16 set_custom_all_reduce)
17 +from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
19 from vllm.model_executor import set_random_seed
20 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -27,6 +29,8 @@ from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
29 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
30 from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
31
32 +logger = init_logger(__name__)
33 +
34
35 class Worker(LocalOrDistributedWorkerBase):
36 """A worker class that executes (a partition of) the model on a GPU.
@@ -113,6 +117,33 @@ class Worker(LocalOrDistributedWorkerBase):
117 self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
118 self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
119
120 + # Torch profiler. Enabled and configured through env vars:
121 + # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
122 + if envs.VLLM_TORCH_PROFILER_DIR:
123 + torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
124 + logger.info("Profiling enabled. Traces will be saved to: %s",
125 + torch_profiler_trace_dir)
126 + self.profiler = torch.profiler.profile(
127 + activities=[
128 + torch.profiler.ProfilerActivity.CPU,
129 + torch.profiler.ProfilerActivity.CUDA,
130 + ],
131 + with_stack=True,
132 + on_trace_ready=torch.profiler.tensorboard_trace_handler(
133 + torch_profiler_trace_dir, use_gzip=True))
134 + else:
135 + self.profiler = None
136 +
137 + def start_profile(self):
138 + if self.profiler is None:
139 + raise RuntimeError("Profiler is not enabled.")
140 + self.profiler.start()
141 +
142 + def stop_profile(self):
143 + if self.profiler is None:
144 + raise RuntimeError("Profiler is not enabled.")
145 + self.profiler.stop()
146 +
147 def _is_encoder_decoder_model(self):
148 return self.model_config.is_encoder_decoder_model
149