@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 933790c209
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Add span metrics for model_forward, scheduler and sampler time (#7089)

Mahesh Keralapura committed Aug 9, 2024 at 13:55 UTC 933790c209fe73398a542d9fb31a138103ee3ccb
17 files changed +189 -21
tests/tracing/test_tracing.py
+2
@@ -114,3 +114,5 @@ def test_traces(trace_service):
114 SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115 e2e_time = metrics.finished_time - metrics.arrival_time
116 assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117 + assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER
118 + ) == metrics.scheduler_time
tests/worker/test_model_runner.py
+1
@@ -24,6 +24,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
24 load_config=engine_config.load_config,
25 lora_config=engine_config.lora_config,
26 prompt_adapter_config=engine_config.prompt_adapter_config,
27 + observability_config=engine_config.observability_config,
28 is_driver_worker=True,
29 )
30 return model_runner
vllm/config.py
+15
@@ -1656,11 +1656,26 @@ class ObservabilityConfig:
1656 """Configuration for observability."""
1657 otlp_traces_endpoint: Optional[str] = None
1658
1659 + # Collecting detailed timing information for each request can be expensive.
1660 +
1661 + # If set, collects the model forward time for the request.
1662 + collect_model_forward_time: bool = False
1663 +
1664 + # If set, collects the model execute time for the request.
1665 + collect_model_execute_time: bool = False
1666 +
1667 def __post_init__(self):
1668 if not is_otel_installed() and self.otlp_traces_endpoint is not None:
1669 raise ValueError("OpenTelemetry packages must be installed before "
1670 "configuring 'otlp_traces_endpoint'")
1671
1672 + if ((self.collect_model_forward_time
1673 + or self.collect_model_execute_time)
1674 + and self.otlp_traces_endpoint is None):
1675 + raise ValueError(
1676 + "collect_model_forward_time or collect_model_execute_time "
1677 + "requires --otlp-traces-endpoint to be set.")
1678 +
1679
1680 @dataclass(frozen=True)
1681 class EngineConfig:
vllm/core/scheduler.py
+12
@@ -1032,6 +1032,7 @@ class Scheduler:
1032 # such as self.running, self.swapped, and self.waiting.
1033 scheduler_outputs = self._schedule()
1034 now = time.time()
1035 + scheduler_start_time = time.perf_counter()
1036
1037 if not self.cache_config.enable_prefix_caching:
1038 common_computed_block_nums = []
@@ -1127,6 +1128,17 @@ class Scheduler:
1128
1129 self._seq_group_metadata_cache.reset()
1130
1131 + scheduler_time = time.perf_counter() - scheduler_start_time
1132 + # Add this to scheduler time to all the sequences that are currently
1133 + # running. This will help estimate if the scheduler is a significant
1134 + # component in the e2e latency.
1135 + for seq_group in self.running:
1136 + if seq_group is not None and seq_group.metrics is not None:
1137 + if seq_group.metrics.scheduler_time is not None:
1138 + seq_group.metrics.scheduler_time += scheduler_time
1139 + else:
1140 + seq_group.metrics.scheduler_time = scheduler_time
1141 +
1142 return seq_group_metadata_list, scheduler_outputs
1143
1144 def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
vllm/engine/arg_utils.py
+32 -1
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
20
21 logger = init_logger(__name__)
22
23 +ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
24 +
25
26 def nullable_str(val: str):
27 if not val or val == "None":
@@ -117,6 +119,7 @@ class EngineArgs:
119 disable_logprobs_during_spec_decoding: Optional[bool] = None
120
121 otlp_traces_endpoint: Optional[str] = None
122 + collect_detailed_traces: Optional[str] = None
123
124 def __post_init__(self):
125 if self.tokenizer is None:
@@ -660,6 +663,16 @@ class EngineArgs:
663 type=str,
664 default=None,
665 help='Target URL to which OpenTelemetry traces will be sent.')
666 + parser.add_argument(
667 + '--collect-detailed-traces',
668 + type=str,
669 + default=None,
670 + help="Valid choices are " +
671 + ",".join(ALLOWED_DETAILED_TRACE_MODULES) +
672 + ". It makes sense to set this only if --otlp-traces-endpoint is"
673 + " set. If set, it will collect detailed traces for the specified "
674 + "modules. This involves use of possibly costly and or blocking "
675 + "operations and hence might have a performance impact.")
676
677 return parser
678
@@ -852,8 +865,26 @@ class EngineArgs:
865 decoding_config = DecodingConfig(
866 guided_decoding_backend=self.guided_decoding_backend)
867
868 + detailed_trace_modules = []
869 + if self.collect_detailed_traces is not None:
870 + detailed_trace_modules = self.collect_detailed_traces.split(",")
871 + for m in detailed_trace_modules:
872 + if m not in ALLOWED_DETAILED_TRACE_MODULES:
873 + raise ValueError(
874 + f"Invalid module {m} in collect_detailed_traces. "
875 + f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}")
876 + if (m == "model"
877 + or m == "all") and self.pipeline_parallel_size > 1:
878 + raise ValueError(
879 + "Collection of detailed traces for the 'model' module is "
880 + "not yet supported with pipeline parallelism.")
881 observability_config = ObservabilityConfig(
856 - otlp_traces_endpoint=self.otlp_traces_endpoint)
882 + otlp_traces_endpoint=self.otlp_traces_endpoint,
883 + collect_model_forward_time="model" in detailed_trace_modules
884 + or "all" in detailed_trace_modules,
885 + collect_model_execute_time="worker" in detailed_trace_modules
886 + or "all" in detailed_trace_modules,
887 + )
888
889 if (model_config.get_sliding_window() is not None
890 and scheduler_config.chunked_prefill_enabled
vllm/engine/llm_engine.py
+29
@@ -267,6 +267,7 @@ class LLMEngine:
267 speculative_config=speculative_config,
268 load_config=load_config,
269 prompt_adapter_config=prompt_adapter_config,
270 + observability_config=self.observability_config,
271 )
272
273 if not self.model_config.embedding_mode:
@@ -1183,6 +1184,22 @@ class LLMEngine:
1184 seq_group = scheduled_seq_group.seq_group
1185 seq_group.update_num_computed_tokens(
1186 scheduled_seq_group.token_chunk_size)
1187 + if output is not None and len(output) > 0:
1188 + for o in output:
1189 + if (isinstance(o, SamplerOutput)
1190 + and seq_group.metrics is not None):
1191 + if seq_group.metrics.model_forward_time is not None:
1192 + seq_group.metrics.model_forward_time += (
1193 + o.model_forward_time)
1194 + else:
1195 + seq_group.metrics.model_forward_time = (
1196 + o.model_forward_time)
1197 + if seq_group.metrics.model_execute_time is not None:
1198 + seq_group.metrics.model_execute_time += (
1199 + o.model_execute_time)
1200 + else:
1201 + seq_group.metrics.model_execute_time = (
1202 + o.model_execute_time)
1203 if self.model_config.embedding_mode:
1204 self._process_sequence_group_outputs(seq_group, outputs)
1205 continue
@@ -1575,6 +1592,18 @@ class LLMEngine:
1592 seq_span.set_attribute(
1593 SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1594 seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1595 + if metrics.scheduler_time is not None:
1596 + seq_span.set_attribute(
1597 + SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER,
1598 + metrics.scheduler_time)
1599 + if metrics.model_forward_time is not None:
1600 + seq_span.set_attribute(
1601 + SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD,
1602 + metrics.model_forward_time / 1000.0)
1603 + if metrics.model_execute_time is not None:
1604 + seq_span.set_attribute(
1605 + SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
1606 + metrics.model_execute_time)
1607
1608 def is_encoder_decoder_model(self):
1609 return self.model_config.is_encoder_decoder_model
vllm/executor/executor_base.py
+4 -3
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 - ModelConfig, MultiModalConfig, ParallelConfig,
6 - PromptAdapterConfig, SchedulerConfig,
5 + ModelConfig, MultiModalConfig, ObservabilityConfig,
6 + ParallelConfig, PromptAdapterConfig, SchedulerConfig,
7 SpeculativeConfig)
8 from vllm.lora.request import LoRARequest
9 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -32,6 +32,7 @@ class ExecutorBase(ABC):
32 multimodal_config: Optional[MultiModalConfig],
33 speculative_config: Optional[SpeculativeConfig],
34 prompt_adapter_config: Optional[PromptAdapterConfig],
35 + observability_config: Optional[ObservabilityConfig],
36 ) -> None:
37 self.model_config = model_config
38 self.cache_config = cache_config
@@ -43,7 +44,7 @@ class ExecutorBase(ABC):
44 self.multimodal_config = multimodal_config
45 self.speculative_config = speculative_config
46 self.prompt_adapter_config = prompt_adapter_config
46 -
47 + self.observability_config = observability_config
48 self._init_executor()
49
50 @abstractmethod
vllm/executor/gpu_executor.py
+1
@@ -60,6 +60,7 @@ class GPUExecutor(ExecutorBase):
60 prompt_adapter_config=self.prompt_adapter_config,
61 is_driver_worker=(not self.parallel_config)
62 or (rank % self.parallel_config.tensor_parallel_size == 0),
63 + observability_config=self.observability_config,
64 )
65
66 def _get_create_worker_kwargs(
vllm/sequence.py
+17
@@ -92,6 +92,13 @@ class RequestMetrics:
92 first_token_time: The time when the first token was generated.
93 time_in_queue: The time the request spent in the queue.
94 finished_time: The time when the request was finished.
95 + scheduler_time: The time spent in the scheduler when this request was
96 + being considered by the scheduler.
97 + model_forward_time: The time spent in the model forward pass when this
98 + request was in the batch.
99 + model_execute_time: The time spent in the model execute function. This
100 + will include model forward, block/sync across
101 + workers, cpu-gpu sync time and sampling time.
102 """
103 arrival_time: float
104 last_token_time: float
@@ -99,6 +106,9 @@ class RequestMetrics:
106 first_token_time: Optional[float]
107 time_in_queue: Optional[float]
108 finished_time: Optional[float] = None
109 + scheduler_time: Optional[float] = None
110 + model_forward_time: Optional[float] = None
111 + model_execute_time: Optional[float] = None
112
113
114 class SequenceData:
@@ -968,6 +978,13 @@ class SamplerOutput:
978 # Optional last hidden states from the model.
979 hidden_states: Optional[torch.Tensor] = None
980
981 + # Time taken in the forward pass for this across all workers
982 + model_forward_time: Optional[float] = None
983 +
984 + # Time taken in the model execute function. This will include model forward,
985 + # block/sync across workers, cpu-gpu sync time and sampling time.
986 + model_execute_time: Optional[float] = None
987 +
988 def __getitem__(self, idx: int):
989 return self.outputs[idx]
990
vllm/spec_decode/draft_model_runner.py
+4 -2
@@ -23,8 +23,8 @@ except ImportError:
23 FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24
25 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26 - ModelConfig, MultiModalConfig, ParallelConfig,
27 - PromptAdapterConfig, SchedulerConfig)
26 + ModelConfig, MultiModalConfig, ObservabilityConfig,
27 + ParallelConfig, PromptAdapterConfig, SchedulerConfig)
28 from vllm.logger import init_logger
29 from vllm.multimodal import MultiModalInputs
30 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
@@ -69,6 +69,7 @@ class TP1DraftModelRunner(ModelRunner):
69 multimodal_config: Optional[MultiModalConfig] = None,
70 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
71 return_hidden_states: bool = False,
72 + observability_config: Optional[ObservabilityConfig] = None,
73 ):
74 if return_hidden_states:
75 raise ValueError(
@@ -88,6 +89,7 @@ class TP1DraftModelRunner(ModelRunner):
89 multimodal_config=multimodal_config,
90 prompt_adapter_config=prompt_adapter_config,
91 return_hidden_states=return_hidden_states,
92 + observability_config=observability_config,
93 )
94
95 self.flashinfer_decode_workspace_buffer = None
vllm/spec_decode/target_model_runner.py
+5 -3
@@ -1,8 +1,8 @@
1 from typing import List, Optional
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4 - ModelConfig, MultiModalConfig, ParallelConfig,
5 - PromptAdapterConfig, SchedulerConfig)
4 + ModelConfig, MultiModalConfig, ObservabilityConfig,
5 + ParallelConfig, PromptAdapterConfig, SchedulerConfig)
6 from vllm.sequence import SequenceGroupMetadata
7 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8 ModelRunner)
@@ -32,7 +32,8 @@ class TargetModelRunner(ModelRunner):
32 is_driver_worker: bool = False,
33 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34 multimodal_config: Optional[MultiModalConfig] = None,
35 - return_hidden_states: bool = False):
35 + return_hidden_states: bool = False,
36 + observability_config: Optional[ObservabilityConfig] = None):
37 # An internal boolean member variable to indicate if token log
38 # probabilities are needed or not.
39 self.disable_logprobs = True
@@ -49,6 +50,7 @@ class TargetModelRunner(ModelRunner):
50 multimodal_config=multimodal_config,
51 prompt_adapter_config=prompt_adapter_config,
52 return_hidden_states=return_hidden_states,
53 + observability_config=observability_config,
54 )
55
56 def prepare_model_input(
vllm/tracing.py
+6
@@ -92,6 +92,12 @@ class SpanAttributes(BaseSpanAttributes):
92 LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
93 LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
94 LLM_LATENCY_E2E = "gen_ai.latency.e2e"
95 + LLM_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
96 + # Time taken in the forward pass for this across all workers
97 + LLM_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward"
98 + # Time taken in the model execute function. This will include model
99 + # forward, block/sync across workers, cpu-gpu sync time and sampling time.
100 + LLM_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute"
101
102
103 def contains_trace_headers(headers: Mapping[str, str]) -> bool:
vllm/worker/embedding_model_runner.py
+5 -3
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4 import torch
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 - ModelConfig, MultiModalConfig, ParallelConfig,
8 - PromptAdapterConfig, SchedulerConfig)
7 + ModelConfig, MultiModalConfig, ObservabilityConfig,
8 + ParallelConfig, PromptAdapterConfig, SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.multimodal import MultiModalInputs
@@ -45,6 +45,7 @@ class EmbeddingModelRunner(
45 is_driver_worker: bool = False,
46 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47 multimodal_config: Optional[MultiModalConfig] = None,
48 + observability_config: Optional[ObservabilityConfig] = None,
49 ):
50 super().__init__(model_config,
51 parallel_config,
@@ -56,7 +57,8 @@ class EmbeddingModelRunner(
57 kv_cache_dtype=kv_cache_dtype,
58 is_driver_worker=is_driver_worker,
59 prompt_adapter_config=prompt_adapter_config,
59 - multimodal_config=multimodal_config)
60 + multimodal_config=multimodal_config,
61 + observability_config=observability_config)
62
63 @torch.inference_mode()
64 def execute_model(
vllm/worker/enc_dec_model_runner.py
+3 -2
@@ -10,8 +10,8 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
10 get_global_forced_attn_backend,
11 global_force_attn_backend)
12 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13 - ModelConfig, MultiModalConfig, ParallelConfig,
14 - PromptAdapterConfig, SchedulerConfig)
13 + ModelConfig, MultiModalConfig, ObservabilityConfig,
14 + ParallelConfig, PromptAdapterConfig, SchedulerConfig)
15 from vllm.inputs import INPUT_REGISTRY
16 from vllm.logger import init_logger
17 from vllm.model_executor import SamplingMetadata
@@ -82,6 +82,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
82 is_driver_worker: bool = False,
83 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84 multimodal_config: Optional[MultiModalConfig] = None,
85 + observability_config: Optional[ObservabilityConfig] = None,
86 ):
87 '''
88 EncoderDecoderModelRunner constructor.
vllm/worker/model_runner.py
+28 -4
@@ -27,8 +27,8 @@ except ImportError:
27 import vllm.envs as envs
28 from vllm.attention import AttentionMetadata, get_attn_backend
29 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
30 - ModelConfig, MultiModalConfig, ParallelConfig,
31 - PromptAdapterConfig, SchedulerConfig)
30 + ModelConfig, MultiModalConfig, ObservabilityConfig,
31 + ParallelConfig, PromptAdapterConfig, SchedulerConfig)
32 from vllm.distributed import get_pp_group
33 from vllm.distributed.parallel_state import graph_capture
34 from vllm.inputs import INPUT_REGISTRY
@@ -806,6 +806,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
806 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
807 multimodal_config: Optional[MultiModalConfig] = None,
808 return_hidden_states: bool = False,
809 + observability_config: Optional[ObservabilityConfig] = None,
810 ):
811 self.model_config = model_config
812 self.parallel_config = parallel_config
@@ -818,6 +819,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
819 self.prompt_adapter_config = prompt_adapter_config
820 self.multimodal_config = multimodal_config
821 self.return_hidden_states = return_hidden_states
822 + self.observability_config = observability_config
823
824 self.device = self.device_config.device
825 self.pin_memory = is_pin_memory_available()
@@ -1527,6 +1529,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1529 "finished_requests_ids": model_input.finished_requests_ids,
1530 "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1531 } if self.has_seqlen_agnostic else {}
1532 + if (self.observability_config is not None
1533 + and self.observability_config.collect_model_forward_time):
1534 + model_forward_start = torch.cuda.Event(enable_timing=True)
1535 + model_forward_end = torch.cuda.Event(enable_timing=True)
1536 + model_forward_start.record()
1537 +
1538 hidden_or_intermediate_states = model_executable(
1539 input_ids=model_input.input_tokens,
1540 positions=model_input.input_positions,
@@ -1537,6 +1545,10 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1545 device=self.device),
1546 **seqlen_agnostic_kwargs)
1547
1548 + if (self.observability_config is not None
1549 + and self.observability_config.collect_model_forward_time):
1550 + model_forward_end.record()
1551 +
1552 # Compute the logits in the last pipeline stage.
1553 if not get_pp_group().is_last_rank:
1554 return hidden_or_intermediate_states
@@ -1552,6 +1564,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1564 logits=logits,
1565 sampling_metadata=model_input.sampling_metadata,
1566 )
1567 + if (self.observability_config is not None
1568 + and self.observability_config.collect_model_forward_time
1569 + and output is not None):
1570 + model_forward_end.synchronize()
1571 + model_forward_time = model_forward_start.elapsed_time(
1572 + model_forward_end)
1573 + # If there are multiple workers, we are still tracking the latency
1574 + # from the start time of the driver worker to the end time of the
1575 + # driver worker. The model forward time will then end up covering
1576 + # the communication time as well.
1577 + output.model_forward_time = model_forward_time
1578
1579 if self.return_hidden_states:
1580 # we only need to pass hidden states of most recent token
@@ -1709,8 +1732,9 @@ class CUDAGraphRunner:
1732 **kwargs)
1733 if intermediate_tensors is not None:
1734 for key in intermediate_tensors.tensors:
1712 - self.input_buffers[key].copy_(intermediate_tensors[key],
1713 - non_blocking=True)
1735 + if key != "model_execute_time":
1736 + self.input_buffers[key].copy_(intermediate_tensors[key],
1737 + non_blocking=True)
1738 # Run the graph.
1739 self.graph.replay()
1740 # Return the output tensor.
vllm/worker/worker.py
+5 -2
@@ -7,8 +7,8 @@ import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, MultiModalConfig, ParallelConfig,
11 - PromptAdapterConfig, SchedulerConfig,
10 + ModelConfig, MultiModalConfig, ObservabilityConfig,
11 + ParallelConfig, PromptAdapterConfig, SchedulerConfig,
12 SpeculativeConfig)
13 from vllm.distributed import (ensure_model_parallel_initialized,
14 init_distributed_environment,
@@ -51,6 +51,7 @@ class Worker(LocalOrDistributedWorkerBase):
51 prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52 is_driver_worker: bool = False,
53 model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
54 + observability_config: Optional[ObservabilityConfig] = None,
55 ) -> None:
56 self.model_config = model_config
57 self.parallel_config = parallel_config
@@ -73,6 +74,7 @@ class Worker(LocalOrDistributedWorkerBase):
74 from vllm.utils import init_cached_hf_modules
75 init_cached_hf_modules()
76 self.multimodal_config = multimodal_config
77 + self.observability_config = observability_config
78
79 # Return hidden states from target model if the draft model is an
80 # mlp_speculator
@@ -102,6 +104,7 @@ class Worker(LocalOrDistributedWorkerBase):
104 is_driver_worker=is_driver_worker,
105 prompt_adapter_config=prompt_adapter_config,
106 multimodal_config=multimodal_config,
107 + observability_config=observability_config,
108 **speculative_args,
109 )
110 # Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+20 -1
@@ -1,11 +1,13 @@
1 import dataclasses
2 import importlib
3 import os
4 +import time
5 from abc import ABC, abstractmethod
6 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
7
8 import torch
9
10 +from vllm.config import ObservabilityConfig
11 from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
@@ -172,6 +174,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
174 """
175 is_driver_worker: bool
176 model_runner: ModelRunnerBase
177 + observability_config: Optional[ObservabilityConfig] = None
178
179 @property
180 @abstractmethod
@@ -219,6 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222 ) -> Optional[List[SamplerOutput]]:
223 """Executes at least one model step on the given sequences, unless no
224 sequences are provided."""
225 + start_time = time.perf_counter()
226 if self.is_driver_worker:
227 if execute_model_req is None:
228 if self.do_metadata_broadcast:
@@ -265,21 +269,36 @@ class LocalOrDistributedWorkerBase(WorkerBase):
269 return []
270
271 intermediate_tensors = None
272 + orig_model_execute_time = 0.0
273 if not get_pp_group().is_first_rank:
274 intermediate_tensors = IntermediateTensors(
275 get_pp_group().recv_tensor_dict(
276 all_gather_group=get_tp_group()))
277 + if (self.observability_config is not None
278 + and self.observability_config.collect_model_execute_time):
279 + orig_model_execute_time = intermediate_tensors.tensors.get(
280 + "model_execute_time", torch.tensor(0)).item()
281
282 output = self.model_runner.execute_model(
283 model_input, self.kv_cache[worker_input.virtual_engine]
284 if self.kv_cache is not None else None, intermediate_tensors,
285 num_steps)
277 -
286 + model_execute_time = time.perf_counter() - start_time
287 if not get_pp_group().is_last_rank:
288 # output is IntermediateTensors
289 + if (self.observability_config is not None
290 + and self.observability_config.collect_model_execute_time):
291 + output.tensors["model_execute_time"] = torch.tensor(
292 + model_execute_time + orig_model_execute_time)
293 get_pp_group().send_tensor_dict(output.tensors,
294 all_gather_group=get_tp_group())
295 return [None]
296 + if (self.observability_config is not None
297 + and self.observability_config.collect_model_execute_time
298 + and output is not None):
299 + for o in output:
300 + o.model_execute_time = (orig_model_execute_time +
301 + model_execute_time)
302
303 # output is List[SamplerOutput]
304 return output