@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 17c3103c56
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Make it easy to profile workers with nsight (#3162)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>

Philipp Moritz committed Mar 3, 2024 at 16:19 UTC 17c3103c562e748686a3fa4bd9b43ebe98aae3d9
4 files changed +34 -2
benchmarks/benchmark_latency.py
+6
@@ -26,6 +26,7 @@ def main(args: argparse.Namespace):
26 enforce_eager=args.enforce_eager,
27 kv_cache_dtype=args.kv_cache_dtype,
28 device=args.device,
29 + ray_workers_use_nsight=args.ray_workers_use_nsight,
30 )
31
32 sampling_params = SamplingParams(
@@ -145,5 +146,10 @@ if __name__ == '__main__':
146 default="cuda",
147 choices=["cuda"],
148 help='device type for vLLM execution, supporting CUDA only currently.')
149 + parser.add_argument(
150 + "--ray-workers-use-nsight",
151 + action='store_true',
152 + help="If specified, use nsight to profile ray workers",
153 + )
154 args = parser.parse_args()
155 main(args)
vllm/config.py
+7
@@ -382,6 +382,8 @@ class ParallelConfig:
382 parallel and large models.
383 disable_custom_all_reduce: Disable the custom all-reduce kernel and
384 fall back to NCCL.
385 + ray_workers_use_nsight: Whether to profile Ray workers with nsight, see
386 + https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler.
387 """
388
389 def __init__(
@@ -391,6 +393,7 @@ class ParallelConfig:
393 worker_use_ray: bool,
394 max_parallel_loading_workers: Optional[int] = None,
395 disable_custom_all_reduce: bool = False,
396 + ray_workers_use_nsight: bool = False,
397 ) -> None:
398 self.pipeline_parallel_size = pipeline_parallel_size
399 if is_neuron():
@@ -404,6 +407,7 @@ class ParallelConfig:
407 self.worker_use_ray = worker_use_ray
408 self.max_parallel_loading_workers = max_parallel_loading_workers
409 self.disable_custom_all_reduce = disable_custom_all_reduce
410 + self.ray_workers_use_nsight = ray_workers_use_nsight
411
412 self.world_size = pipeline_parallel_size * self.tensor_parallel_size
413 # Ray worker is not supported for Neuron backend.
@@ -426,6 +430,9 @@ class ParallelConfig:
430 logger.info(
431 "Disabled the custom all-reduce kernel because it is not "
432 "supported with pipeline parallelism.")
433 + if self.ray_workers_use_nsight and not self.worker_use_ray:
434 + raise ValueError("Unable to use nsight profiling unless workers "
435 + "run with Ray.")
436
437 # FIXME(woosuk): Fix the stability issues and re-enable the custom
438 # all-reduce kernel.
vllm/engine/arg_utils.py
+7 -1
@@ -46,6 +46,7 @@ class EngineArgs:
46 lora_dtype = 'auto'
47 max_cpu_loras: Optional[int] = None
48 device: str = 'auto'
49 + ray_workers_use_nsight: bool = False
50
51 def __post_init__(self):
52 if self.tokenizer is None:
@@ -168,6 +169,10 @@ class EngineArgs:
169 help='load model sequentially in multiple batches, '
170 'to avoid RAM OOM when using tensor '
171 'parallel and large models')
172 + parser.add_argument(
173 + '--ray-workers-use-nsight',
174 + action='store_true',
175 + help='If specified, use nsight to profile ray workers')
176 # KV cache arguments
177 parser.add_argument('--block-size',
178 type=int,
@@ -305,7 +310,8 @@ class EngineArgs:
310 self.tensor_parallel_size,
311 self.worker_use_ray,
312 self.max_parallel_loading_workers,
308 - self.disable_custom_all_reduce)
313 + self.disable_custom_all_reduce,
314 + self.ray_workers_use_nsight)
315 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
316 self.max_num_seqs,
317 model_config.max_model_len,
vllm/engine/llm_engine.py
+14 -1
@@ -124,7 +124,20 @@ class LLMEngine:
124 ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
125 if ray_usage != "1":
126 os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
127 - self._init_workers_ray(placement_group)
127 + # Pass additional arguments to initialize the worker
128 + additional_ray_args = {}
129 + if self.parallel_config.ray_workers_use_nsight:
130 + logger.info("Configuring Ray workers to use nsight.")
131 + additional_ray_args = {
132 + "runtime_env": {
133 + "nsight": {
134 + "t": "cuda,cudnn,cublas",
135 + "o": "'worker_process_%p'",
136 + "cuda-graph-trace": "node",
137 + }
138 + }
139 + }
140 + self._init_workers_ray(placement_group, **additional_ray_args)
141 else:
142 self._init_workers()
143