A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[bugfix] torch profiler bug for single gpu with GPUExecutor (#8354)
William Lin committed
Sep 12, 2024 at 21:30 UTC
ba7752795567e3f2bfcc1dca340d107e003d32ad
3 files changed
+27
-5
examples/offline_inference_with_profiler.py
+1
-1
@@ -16,7 +16,7 @@ prompts = [
16
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
18
# Create an LLM.
19
-llm = LLM(model="facebook/opt-125m")
19
+llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
20
21
llm.start_profile()
22
vllm/engine/async_llm_engine.py
+13
-2
@@ -13,6 +13,7 @@ from vllm.engine.async_timeout import asyncio_timeout
13
from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
14
from vllm.engine.metrics_types import StatLoggerBase
15
from vllm.executor.executor_base import ExecutorAsyncBase
16
+from vllm.executor.gpu_executor import GPUExecutorAsync
17
from vllm.executor.ray_utils import initialize_ray_cluster
18
from vllm.inputs import PromptInputs
19
from vllm.logger import init_logger
@@ -1019,7 +1020,17 @@ class AsyncLLMEngine:
1020
self.engine.remove_logger(logger_name=logger_name)
1021
1022
async def start_profile(self) -> None:
1022
- self.engine.model_executor._run_workers("start_profile")
1023
+ # using type instead of isinstance to check to avoid capturing
1024
+ # inherited classes
1025
+ if type(self.engine.model_executor) == GPUExecutorAsync:
1026
+ self.engine.model_executor.start_profile()
1027
+ else:
1028
+ self.engine.model_executor._run_workers("start_profile")
1029
1030
async def stop_profile(self) -> None:
1025
- self.engine.model_executor._run_workers("stop_profile")
1031
+ # using type instead of isinstance to check to avoid capturing
1032
+ # inherited classes
1033
+ if type(self.engine.model_executor) == GPUExecutorAsync:
1034
+ self.engine.model_executor.stop_profile()
1035
+ else:
1036
+ self.engine.model_executor._run_workers("stop_profile")
vllm/engine/llm_engine.py
+13
-2
@@ -26,6 +26,7 @@ from vllm.engine.output_processor.interfaces import (
26
from vllm.engine.output_processor.stop_checker import StopChecker
27
from vllm.engine.output_processor.util import create_output_by_sequence_group
28
from vllm.executor.executor_base import ExecutorBase
29
+from vllm.executor.gpu_executor import GPUExecutor
30
from vllm.executor.ray_utils import initialize_ray_cluster
31
from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32
InputRegistry, LLMInputs, PromptInputs)
@@ -1597,10 +1598,20 @@ class LLMEngine:
1598
self.model_executor.check_health()
1599
1600
def start_profile(self) -> None:
1600
- self.model_executor.start_profile()
1601
+ # using type instead of isinstance to check to avoid capturing
1602
+ # inherited classes (MultiprocessingGPUExecutor)
1603
+ if type(self.model_executor) == GPUExecutor:
1604
+ self.model_executor.start_profile()
1605
+ else:
1606
+ self.model_executor._run_workers("start_profile")
1607
1608
def stop_profile(self) -> None:
1603
- self.model_executor.stop_profile()
1609
+ # using type instead of isinstance to check to avoid capturing
1610
+ # inherited classes (MultiprocessingGPUExecutor)
1611
+ if type(self.model_executor) == GPUExecutor:
1612
+ self.model_executor.stop_profile()
1613
+ else:
1614
+ self.model_executor._run_workers("stop_profile")
1615
1616
def is_tracing_enabled(self) -> bool:
1617
return self.tracer is not None