@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ba77527955
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[bugfix] torch profiler bug for single gpu with GPUExecutor (#8354)

William Lin committed Sep 12, 2024 at 21:30 UTC ba7752795567e3f2bfcc1dca340d107e003d32ad
3 files changed +27 -5
examples/offline_inference_with_profiler.py
+1 -1
@@ -16,7 +16,7 @@ prompts = [
16 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
18 # Create an LLM.
19 -llm = LLM(model="facebook/opt-125m")
19 +llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
20
21 llm.start_profile()
22
vllm/engine/async_llm_engine.py
+13 -2
@@ -13,6 +13,7 @@ from vllm.engine.async_timeout import asyncio_timeout
13 from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
14 from vllm.engine.metrics_types import StatLoggerBase
15 from vllm.executor.executor_base import ExecutorAsyncBase
16 +from vllm.executor.gpu_executor import GPUExecutorAsync
17 from vllm.executor.ray_utils import initialize_ray_cluster
18 from vllm.inputs import PromptInputs
19 from vllm.logger import init_logger
@@ -1019,7 +1020,17 @@ class AsyncLLMEngine:
1020 self.engine.remove_logger(logger_name=logger_name)
1021
1022 async def start_profile(self) -> None:
1022 - self.engine.model_executor._run_workers("start_profile")
1023 + # using type instead of isinstance to check to avoid capturing
1024 + # inherited classes
1025 + if type(self.engine.model_executor) == GPUExecutorAsync:
1026 + self.engine.model_executor.start_profile()
1027 + else:
1028 + self.engine.model_executor._run_workers("start_profile")
1029
1030 async def stop_profile(self) -> None:
1025 - self.engine.model_executor._run_workers("stop_profile")
1031 + # using type instead of isinstance to check to avoid capturing
1032 + # inherited classes
1033 + if type(self.engine.model_executor) == GPUExecutorAsync:
1034 + self.engine.model_executor.stop_profile()
1035 + else:
1036 + self.engine.model_executor._run_workers("stop_profile")
vllm/engine/llm_engine.py
+13 -2
@@ -26,6 +26,7 @@ from vllm.engine.output_processor.interfaces import (
26 from vllm.engine.output_processor.stop_checker import StopChecker
27 from vllm.engine.output_processor.util import create_output_by_sequence_group
28 from vllm.executor.executor_base import ExecutorBase
29 +from vllm.executor.gpu_executor import GPUExecutor
30 from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32 InputRegistry, LLMInputs, PromptInputs)
@@ -1597,10 +1598,20 @@ class LLMEngine:
1598 self.model_executor.check_health()
1599
1600 def start_profile(self) -> None:
1600 - self.model_executor.start_profile()
1601 + # using type instead of isinstance to check to avoid capturing
1602 + # inherited classes (MultiprocessingGPUExecutor)
1603 + if type(self.model_executor) == GPUExecutor:
1604 + self.model_executor.start_profile()
1605 + else:
1606 + self.model_executor._run_workers("start_profile")
1607
1608 def stop_profile(self) -> None:
1603 - self.model_executor.stop_profile()
1609 + # using type instead of isinstance to check to avoid capturing
1610 + # inherited classes (MultiprocessingGPUExecutor)
1611 + if type(self.model_executor) == GPUExecutor:
1612 + self.model_executor.stop_profile()
1613 + else:
1614 + self.model_executor._run_workers("stop_profile")
1615
1616 def is_tracing_enabled(self) -> bool:
1617 return self.tracer is not None