A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc] Add Torch profiler support (#7451)
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
William Lin committed
Aug 21, 2024 at 15:39 UTC
dd53c4b023056cda6174cc32dc3d31bc01e8646a
12 files changed
+191
-2
benchmarks/backend_request_func.py
+2
-2
@@ -225,8 +225,8 @@ async def async_request_openai_completions(
225
) -> RequestFuncOutput:
226
api_url = request_func_input.api_url
227
assert api_url.endswith(
228
- "completions"
229
- ), "OpenAI Completions API URL must end with 'completions'."
228
+ ("completions", "profile")
229
+ ), "OpenAI Completions API URL must end with 'completions' or 'profile'."
230
231
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
232
assert not request_func_input.use_beam_search
benchmarks/benchmark_serving.py
+43
@@ -295,6 +295,7 @@ def calculate_metrics(
295
async def benchmark(
296
backend: str,
297
api_url: str,
298
+ base_url: str,
299
model_id: str,
300
tokenizer: PreTrainedTokenizerBase,
301
input_requests: List[Tuple[str, int, int]],
@@ -302,6 +303,7 @@ async def benchmark(
303
use_beam_search: bool,
304
request_rate: float,
305
disable_tqdm: bool,
306
+ profile: bool,
307
):
308
if backend in ASYNC_REQUEST_FUNCS:
309
request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -326,6 +328,22 @@ async def benchmark(
328
f"are correctly specified. Error: {test_output.error}")
329
else:
330
print("Initial test run completed. Starting main benchmark run...")
331
+
332
+ if profile:
333
+ print("Starting profiler...")
334
+ profile_input = RequestFuncInput(
335
+ model=model_id,
336
+ prompt=test_prompt,
337
+ api_url=base_url + "/start_profile",
338
+ prompt_len=test_prompt_len,
339
+ output_len=test_output_len,
340
+ best_of=best_of,
341
+ use_beam_search=use_beam_search,
342
+ )
343
+ profile_output = await request_func(request_func_input=profile_input)
344
+ if profile_output.success:
345
+ print("Profiler started")
346
+
347
print(f"Traffic request rate: {request_rate}")
348
349
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
@@ -349,6 +367,21 @@ async def benchmark(
367
pbar=pbar)))
368
outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
369
370
+ if profile:
371
+ print("Stopping profiler...")
372
+ profile_input = RequestFuncInput(
373
+ model=model_id,
374
+ prompt=test_prompt,
375
+ api_url=base_url + "/stop_profile",
376
+ prompt_len=test_prompt_len,
377
+ output_len=test_output_len,
378
+ best_of=best_of,
379
+ use_beam_search=use_beam_search,
380
+ )
381
+ profile_output = await request_func(request_func_input=profile_input)
382
+ if profile_output.success:
383
+ print("Profiler stopped")
384
+
385
if pbar is not None:
386
pbar.close()
387
@@ -433,8 +466,10 @@ def main(args: argparse.Namespace):
466
467
if args.base_url is not None:
468
api_url = f"{args.base_url}{args.endpoint}"
469
+ base_url = f"{args.base_url}"
470
else:
471
api_url = f"http://{args.host}:{args.port}{args.endpoint}"
472
+ base_url = f"http://{args.host}:{args.port}"
473
474
tokenizer = get_tokenizer(tokenizer_id,
475
trust_remote_code=args.trust_remote_code)
@@ -506,6 +541,7 @@ def main(args: argparse.Namespace):
541
benchmark(
542
backend=backend,
543
api_url=api_url,
544
+ base_url=base_url,
545
model_id=model_id,
546
tokenizer=tokenizer,
547
input_requests=input_requests,
@@ -513,6 +549,7 @@ def main(args: argparse.Namespace):
549
use_beam_search=args.use_beam_search,
550
request_rate=args.request_rate,
551
disable_tqdm=args.disable_tqdm,
552
+ profile=args.profile,
553
))
554
555
# Save config and results to json
@@ -693,6 +730,12 @@ if __name__ == "__main__":
730
action="store_true",
731
help="Specify to disable tqdm progress bar.",
732
)
733
+ parser.add_argument(
734
+ "--profile",
735
+ action="store_true",
736
+ help="Use Torch Profiler. The endpoint must be launched with "
737
+ "VLLM_TORCH_PROFILER_DIR to enable profiler.",
738
+ )
739
parser.add_argument(
740
"--save-result",
741
action="store_true",
docs/source/dev/profiling/profiling_index.rst
new
+33
@@ -0,0 +1,33 @@
1
+Profiling vLLM
2
+=================================
3
+
4
+We support tracing vLLM workers using the ``torch.profiler`` module. You can enable tracing by setting the ``VLLM_TORCH_PROFILER_DIR`` environment variable to the directory where you want to save the traces: ``VLLM_TORCH_PROFILER_DIR=/mnt/traces/``
5
+
6
+The OpenAI server also needs to be started with the ``VLLM_TORCH_PROFILER_DIR`` environment variable set.
7
+
8
+When using ``benchmarks/benchmark_serving.py``, you can enable profiling by passing the ``--profile`` flag.
9
+
10
+.. warning::
11
+
12
+ Only enable profiling in a development environment.
13
+
14
+
15
+Traces can be visualized using https://ui.perfetto.dev/.
16
+
17
+.. tip::
18
+
19
+ Only send a few requests through vLLM when profiling, as the traces can get quite large. Also, no need to untar the traces, they can be viewed directly.
20
+
21
+Example commands:
22
+
23
+OpenAI Server:
24
+
25
+.. code-block:: bash
26
+
27
+ VLLM_TORCH_PROFILER_DIR=/mnt/traces/ python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B
28
+
29
+benchmark_serving.py:
30
+
31
+.. code-block:: bash
32
+
33
+ python benchmarks/benchmark_serving.py --backend vllm --model meta-llama/Meta-Llama-3-70B --dataset-name sharegpt --dataset-path sharegpt.json --profile --num-prompts 2
\ No newline at end of file
docs/source/index.rst
+1
@@ -136,6 +136,7 @@ Documentation
136
dev/input_processing/model_inputs_index
137
dev/multimodal/multimodal_index
138
dev/dockerfile/dockerfile
139
+ dev/profiling/profiling_index
140
141
.. toctree::
142
:maxdepth: 1
vllm/engine/async_llm_engine.py
+6
@@ -1266,3 +1266,9 @@ class AsyncLLMEngine:
1266
logger_name=logger_name))
1267
else:
1268
self.engine.remove_logger(logger_name=logger_name)
1269
+
1270
+ async def start_profile(self) -> None:
1271
+ self.engine.model_executor._run_workers("start_profile")
1272
+
1273
+ async def stop_profile(self) -> None:
1274
+ self.engine.model_executor._run_workers("stop_profile")
vllm/engine/protocol.py
+8
@@ -91,3 +91,11 @@ class AsyncEngineClient(Protocol):
91
async def check_health(self) -> None:
92
"""Raise if unhealthy"""
93
...
94
+
95
+ async def start_profile(self) -> None:
96
+ """Start profiling the engine"""
97
+ ...
98
+
99
+ async def stop_profile(self) -> None:
100
+ """Start profiling the engine"""
101
+ ...
vllm/entrypoints/openai/api_server.py
+20
@@ -305,6 +305,26 @@ async def create_embedding(request: EmbeddingRequest, raw_request: Request):
305
assert_never(generator)
306
307
308
+if envs.VLLM_TORCH_PROFILER_DIR:
309
+ logger.warning(
310
+ "Torch Profiler is enabled in the API server. This should ONLY be "
311
+ "used for local development!")
312
+
313
+ @router.post("/start_profile")
314
+ async def start_profile():
315
+ logger.info("Starting profiler...")
316
+ await async_engine_client.start_profile()
317
+ logger.info("Profiler started.")
318
+ return Response(status_code=200)
319
+
320
+ @router.post("/stop_profile")
321
+ async def stop_profile():
322
+ logger.info("Stopping profiler...")
323
+ await async_engine_client.stop_profile()
324
+ logger.info("Profiler stopped.")
325
+ return Response(status_code=200)
326
+
327
+
328
def build_app(args: Namespace) -> FastAPI:
329
app = FastAPI(lifespan=lifespan)
330
app.include_router(router)
vllm/entrypoints/openai/rpc/__init__.py
+2
@@ -46,6 +46,8 @@ class RPCUtilityRequest(Enum):
46
DO_LOG_STATS = 7
47
IS_SERVER_HEALTHY = 8
48
IS_TRACING_ENABLED = 9
49
+ START_PROFILE = 10
50
+ STOP_PROFILE = 11
51
52
53
RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
vllm/entrypoints/openai/rpc/client.py
+14
@@ -400,3 +400,17 @@ class AsyncEngineRPCClient:
400
**kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
401
raise NotImplementedError(
402
"Embeddings not supported with multiprocessing backend")
403
+
404
+ async def start_profile(self) -> None:
405
+ """Start profiling the engine"""
406
+
407
+ await self._send_one_way_rpc_request(
408
+ request=RPCUtilityRequest.START_PROFILE,
409
+ error_message="RPCRequest START_PROFILE failed.")
410
+
411
+ async def stop_profile(self) -> None:
412
+ """Stop profiling the engine"""
413
+
414
+ await self._send_one_way_rpc_request(
415
+ request=RPCUtilityRequest.STOP_PROFILE,
416
+ error_message="RPCRequest STOP_PROFILE failed.")
\ No newline at end of file
vllm/entrypoints/openai/rpc/server.py
+24
@@ -124,6 +124,26 @@ class AsyncEngineRPCServer:
124
except Exception as e:
125
await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
126
127
+ async def start_profile(self, identity):
128
+ logger.info("Starting profiler...")
129
+ await self.engine.start_profile()
130
+ logger.info("Profiler started.")
131
+
132
+ await self.socket.send_multipart([
133
+ identity,
134
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
135
+ ])
136
+
137
+ async def stop_profile(self, identity):
138
+ logger.info("Stopping profiler...")
139
+ await self.engine.stop_profile()
140
+ logger.info("Profiler stopped.")
141
+
142
+ await self.socket.send_multipart([
143
+ identity,
144
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
145
+ ])
146
+
147
def _make_handler_coro(self, identity,
148
message) -> Coroutine[Any, Any, Never]:
149
"""Route the zmq message to the handler coroutine."""
@@ -153,6 +173,10 @@ class AsyncEngineRPCServer:
173
return self.check_health(identity)
174
elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
175
return self.is_tracing_enabled(identity)
176
+ elif request == RPCUtilityRequest.START_PROFILE:
177
+ return self.start_profile(identity)
178
+ elif request == RPCUtilityRequest.STOP_PROFILE:
179
+ return self.stop_profile(identity)
180
else:
181
raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
182
vllm/envs.py
+7
@@ -58,6 +58,7 @@ if TYPE_CHECKING:
58
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
59
VLLM_ALLOW_ENGINE_USE_RAY: bool = False
60
VLLM_PLUGINS: Optional[List[str]] = None
61
+ VLLM_TORCH_PROFILER_DIR: Optional[str] = None
62
63
64
def get_default_cache_root():
@@ -384,6 +385,12 @@ environment_variables: Dict[str, Callable[[], Any]] = {
385
"VLLM_PLUGINS":
386
lambda: None if "VLLM_PLUGINS" not in os.environ else os.environ[
387
"VLLM_PLUGINS"].split(","),
388
+
389
+ # Enables torch profiler if set. Path to the directory where torch profiler
390
+ # traces are saved. Note that it must be an absolute path.
391
+ "VLLM_TORCH_PROFILER_DIR":
392
+ lambda: (None if os.getenv("VLLM_TORCH_PROFILER_DIR", None) is None else os
393
+ .path.expanduser(os.getenv("VLLM_TORCH_PROFILER_DIR", "."))),
394
}
395
396
# end-env-vars-definition
vllm/worker/worker.py
+31
@@ -6,6 +6,7 @@ from typing import Dict, List, Optional, Set, Tuple, Type, Union
6
import torch
7
import torch.distributed
8
9
+import vllm.envs as envs
10
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
ModelConfig, ObservabilityConfig, ParallelConfig,
12
PromptAdapterConfig, SchedulerConfig,
@@ -13,6 +14,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
14
from vllm.distributed import (ensure_model_parallel_initialized,
15
init_distributed_environment,
16
set_custom_all_reduce)
17
+from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
from vllm.model_executor import set_random_seed
20
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -27,6 +29,8 @@ from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
29
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
30
from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
31
32
+logger = init_logger(__name__)
33
+
34
35
class Worker(LocalOrDistributedWorkerBase):
36
"""A worker class that executes (a partition of) the model on a GPU.
@@ -113,6 +117,33 @@ class Worker(LocalOrDistributedWorkerBase):
117
self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
118
self._seq_group_metadata_cache: Dict[str, SequenceGroupMetadata] = {}
119
120
+ # Torch profiler. Enabled and configured through env vars:
121
+ # VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
122
+ if envs.VLLM_TORCH_PROFILER_DIR:
123
+ torch_profiler_trace_dir = envs.VLLM_TORCH_PROFILER_DIR
124
+ logger.info("Profiling enabled. Traces will be saved to: %s",
125
+ torch_profiler_trace_dir)
126
+ self.profiler = torch.profiler.profile(
127
+ activities=[
128
+ torch.profiler.ProfilerActivity.CPU,
129
+ torch.profiler.ProfilerActivity.CUDA,
130
+ ],
131
+ with_stack=True,
132
+ on_trace_ready=torch.profiler.tensorboard_trace_handler(
133
+ torch_profiler_trace_dir, use_gzip=True))
134
+ else:
135
+ self.profiler = None
136
+
137
+ def start_profile(self):
138
+ if self.profiler is None:
139
+ raise RuntimeError("Profiler is not enabled.")
140
+ self.profiler.start()
141
+
142
+ def stop_profile(self):
143
+ if self.profiler is None:
144
+ raise RuntimeError("Profiler is not enabled.")
145
+ self.profiler.stop()
146
+
147
def _is_encoder_decoder_model(self):
148
return self.model_config.is_encoder_decoder_model
149