A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Make it easy to profile workers with nsight (#3162)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
Philipp Moritz committed
Mar 3, 2024 at 16:19 UTC
17c3103c562e748686a3fa4bd9b43ebe98aae3d9
4 files changed
+34
-2
benchmarks/benchmark_latency.py
+6
@@ -26,6 +26,7 @@ def main(args: argparse.Namespace):
26
enforce_eager=args.enforce_eager,
27
kv_cache_dtype=args.kv_cache_dtype,
28
device=args.device,
29
+ ray_workers_use_nsight=args.ray_workers_use_nsight,
30
)
31
32
sampling_params = SamplingParams(
@@ -145,5 +146,10 @@ if __name__ == '__main__':
146
default="cuda",
147
choices=["cuda"],
148
help='device type for vLLM execution, supporting CUDA only currently.')
149
+ parser.add_argument(
150
+ "--ray-workers-use-nsight",
151
+ action='store_true',
152
+ help="If specified, use nsight to profile ray workers",
153
+ )
154
args = parser.parse_args()
155
main(args)
vllm/config.py
+7
@@ -382,6 +382,8 @@ class ParallelConfig:
382
parallel and large models.
383
disable_custom_all_reduce: Disable the custom all-reduce kernel and
384
fall back to NCCL.
385
+ ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
386
+ https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
387
"""
388
389
def __init__(
@@ -391,6 +393,7 @@ class ParallelConfig:
393
worker_use_ray: bool,
394
max_parallel_loading_workers: Optional[int] = None,
395
disable_custom_all_reduce: bool = False,
396
+ ray_workers_use_nsight: bool = False,
397
) -> None:
398
self.pipeline_parallel_size = pipeline_parallel_size
399
if is_neuron():
@@ -404,6 +407,7 @@ class ParallelConfig:
407
self.worker_use_ray = worker_use_ray
408
self.max_parallel_loading_workers = max_parallel_loading_workers
409
self.disable_custom_all_reduce = disable_custom_all_reduce
410
+ self.ray_workers_use_nsight = ray_workers_use_nsight
411
412
self.world_size = pipeline_parallel_size * self.tensor_parallel_size
413
# Ray worker is not supported for Neuron backend.
@@ -426,6 +430,9 @@ class ParallelConfig:
430
logger.info(
431
"Disabled the custom all-reduce kernel because it is not "
432
"supported with pipeline parallelism.")
433
+ if self.ray_workers_use_nsight and not self.worker_use_ray:
434
+ raise ValueError("Unable to use nsight profiling unless workers "
435
+ "run with Ray.")
436
437
# FIXME(woosuk): Fix the stability issues and re-enable the custom
438
# all-reduce kernel.
vllm/engine/arg_utils.py
+7
-1
@@ -46,6 +46,7 @@ class EngineArgs:
46
lora_dtype = 'auto'
47
max_cpu_loras: Optional[int] = None
48
device: str = 'auto'
49
+ ray_workers_use_nsight: bool = False
50
51
def __post_init__(self):
52
if self.tokenizer is None:
@@ -168,6 +169,10 @@ class EngineArgs:
169
help='load model sequentially in multiple batches, '
170
'to avoid RAM OOM when using tensor '
171
'parallel and large models')
172
+ parser.add_argument(
173
+ '--ray-workers-use-nsight',
174
+ action='store_true',
175
+ help='If specified, use nsight to profile ray workers')
176
# KV cache arguments
177
parser.add_argument('--block-size',
178
type=int,
@@ -305,7 +310,8 @@ class EngineArgs:
310
self.tensor_parallel_size,
311
self.worker_use_ray,
312
self.max_parallel_loading_workers,
308
- self.disable_custom_all_reduce)
313
+ self.disable_custom_all_reduce,
314
+ self.ray_workers_use_nsight)
315
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
316
self.max_num_seqs,
317
model_config.max_model_len,
vllm/engine/llm_engine.py
+14
-1
@@ -124,7 +124,20 @@ class LLMEngine:
124
ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
125
if ray_usage != "1":
126
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
127
- self._init_workers_ray(placement_group)
127
+ # Pass additional arguments to initialize the worker
128
+ additional_ray_args = {}
129
+ if self.parallel_config.ray_workers_use_nsight:
130
+ logger.info("Configuring Ray workers to use nsight.")
131
+ additional_ray_args = {
132
+ "runtime_env": {
133
+ "nsight": {
134
+ "t": "cuda,cudnn,cublas",
135
+ "o": "'worker_process_%p'",
136
+ "cuda-graph-trace": "node",
137
+ }
138
+ }
139
+ }
140
+ self._init_workers_ray(placement_group, **additional_ray_args)
141
else:
142
self._init_workers()
143