A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Add span metrics for model_forward, scheduler and sampler time (#7089)
Mahesh Keralapura committed
Aug 9, 2024 at 13:55 UTC
933790c209fe73398a542d9fb31a138103ee3ccb
17 files changed
+189
-21
tests/tracing/test_tracing.py
+2
@@ -114,3 +114,5 @@ def test_traces(trace_service):
114
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115
e2e_time = metrics.finished_time - metrics.arrival_time
116
assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117
+ assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER
118
+ ) == metrics.scheduler_time
tests/worker/test_model_runner.py
+1
@@ -24,6 +24,7 @@ def _create_model_runner(model: str, *args, **kwargs) -> ModelRunner:
24
load_config=engine_config.load_config,
25
lora_config=engine_config.lora_config,
26
prompt_adapter_config=engine_config.prompt_adapter_config,
27
+ observability_config=engine_config.observability_config,
28
is_driver_worker=True,
29
)
30
return model_runner
vllm/config.py
+15
@@ -1656,11 +1656,26 @@ class ObservabilityConfig:
1656
"""Configuration for observability."""
1657
otlp_traces_endpoint: Optional[str] = None
1658
1659
+ # Collecting detailed timing information for each request can be expensive.
1660
+
1661
+ # If set, collects the model forward time for the request.
1662
+ collect_model_forward_time: bool = False
1663
+
1664
+ # If set, collects the model execute time for the request.
1665
+ collect_model_execute_time: bool = False
1666
+
1667
def __post_init__(self):
1668
if not is_otel_installed() and self.otlp_traces_endpoint is not None:
1669
raise ValueError("OpenTelemetry packages must be installed before "
1670
"configuring 'otlp_traces_endpoint'")
1671
1672
+ if ((self.collect_model_forward_time
1673
+ or self.collect_model_execute_time)
1674
+ and self.otlp_traces_endpoint is None):
1675
+ raise ValueError(
1676
+ "collect_model_forward_time or collect_model_execute_time "
1677
+ "requires --otlp-traces-endpoint to be set.")
1678
+
1679
1680
@dataclass(frozen=True)
1681
class EngineConfig:
vllm/core/scheduler.py
+12
@@ -1032,6 +1032,7 @@ class Scheduler:
1032
# such as self.running, self.swapped, and self.waiting.
1033
scheduler_outputs = self._schedule()
1034
now = time.time()
1035
+ scheduler_start_time = time.perf_counter()
1036
1037
if not self.cache_config.enable_prefix_caching:
1038
common_computed_block_nums = []
@@ -1127,6 +1128,17 @@ class Scheduler:
1128
1129
self._seq_group_metadata_cache.reset()
1130
1131
+ scheduler_time = time.perf_counter() - scheduler_start_time
1132
+ # Add this to scheduler time to all the sequences that are currently
1133
+ # running. This will help estimate if the scheduler is a significant
1134
+ # component in the e2e latency.
1135
+ for seq_group in self.running:
1136
+ if seq_group is not None and seq_group.metrics is not None:
1137
+ if seq_group.metrics.scheduler_time is not None:
1138
+ seq_group.metrics.scheduler_time += scheduler_time
1139
+ else:
1140
+ seq_group.metrics.scheduler_time = scheduler_time
1141
+
1142
return seq_group_metadata_list, scheduler_outputs
1143
1144
def fork_seq(self, parent_seq: Sequence, child_seq: Sequence) -> None:
vllm/engine/arg_utils.py
+32
-1
@@ -20,6 +20,8 @@ if TYPE_CHECKING:
20
21
logger = init_logger(__name__)
22
23
+ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
24
+
25
26
def nullable_str(val: str):
27
if not val or val == "None":
@@ -117,6 +119,7 @@ class EngineArgs:
119
disable_logprobs_during_spec_decoding: Optional[bool] = None
120
121
otlp_traces_endpoint: Optional[str] = None
122
+ collect_detailed_traces: Optional[str] = None
123
124
def __post_init__(self):
125
if self.tokenizer is None:
@@ -660,6 +663,16 @@ class EngineArgs:
663
type=str,
664
default=None,
665
help='Target URL to which OpenTelemetry traces will be sent.')
666
+ parser.add_argument(
667
+ '--collect-detailed-traces',
668
+ type=str,
669
+ default=None,
670
+ help="Valid choices are " +
671
+ ",".join(ALLOWED_DETAILED_TRACE_MODULES) +
672
+ ". It makes sense to set this only if --otlp-traces-endpoint is"
673
+ " set. If set, it will collect detailed traces for the specified "
674
+ "modules. This involves use of possibly costly and or blocking "
675
+ "operations and hence might have a performance impact.")
676
677
return parser
678
@@ -852,8 +865,26 @@ class EngineArgs:
865
decoding_config = DecodingConfig(
866
guided_decoding_backend=self.guided_decoding_backend)
867
868
+ detailed_trace_modules = []
869
+ if self.collect_detailed_traces is not None:
870
+ detailed_trace_modules = self.collect_detailed_traces.split(",")
871
+ for m in detailed_trace_modules:
872
+ if m not in ALLOWED_DETAILED_TRACE_MODULES:
873
+ raise ValueError(
874
+ f"Invalid module {m} in collect_detailed_traces. "
875
+ f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}")
876
+ if (m == "model"
877
+ or m == "all") and self.pipeline_parallel_size > 1:
878
+ raise ValueError(
879
+ "Collection of detailed traces for the 'model' module is "
880
+ "not yet supported with pipeline parallelism.")
881
observability_config = ObservabilityConfig(
856
- otlp_traces_endpoint=self.otlp_traces_endpoint)
882
+ otlp_traces_endpoint=self.otlp_traces_endpoint,
883
+ collect_model_forward_time="model" in detailed_trace_modules
884
+ or "all" in detailed_trace_modules,
885
+ collect_model_execute_time="worker" in detailed_trace_modules
886
+ or "all" in detailed_trace_modules,
887
+ )
888
889
if (model_config.get_sliding_window() is not None
890
and scheduler_config.chunked_prefill_enabled
vllm/engine/llm_engine.py
+29
@@ -267,6 +267,7 @@ class LLMEngine:
267
speculative_config=speculative_config,
268
load_config=load_config,
269
prompt_adapter_config=prompt_adapter_config,
270
+ observability_config=self.observability_config,
271
)
272
273
if not self.model_config.embedding_mode:
@@ -1183,6 +1184,22 @@ class LLMEngine:
1184
seq_group = scheduled_seq_group.seq_group
1185
seq_group.update_num_computed_tokens(
1186
scheduled_seq_group.token_chunk_size)
1187
+ if output is not None and len(output) > 0:
1188
+ for o in output:
1189
+ if (isinstance(o, SamplerOutput)
1190
+ and seq_group.metrics is not None):
1191
+ if seq_group.metrics.model_forward_time is not None:
1192
+ seq_group.metrics.model_forward_time += (
1193
+ o.model_forward_time)
1194
+ else:
1195
+ seq_group.metrics.model_forward_time = (
1196
+ o.model_forward_time)
1197
+ if seq_group.metrics.model_execute_time is not None:
1198
+ seq_group.metrics.model_execute_time += (
1199
+ o.model_execute_time)
1200
+ else:
1201
+ seq_group.metrics.model_execute_time = (
1202
+ o.model_execute_time)
1203
if self.model_config.embedding_mode:
1204
self._process_sequence_group_outputs(seq_group, outputs)
1205
continue
@@ -1575,6 +1592,18 @@ class LLMEngine:
1592
seq_span.set_attribute(
1593
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN, ttft)
1594
seq_span.set_attribute(SpanAttributes.LLM_LATENCY_E2E, e2e_time)
1595
+ if metrics.scheduler_time is not None:
1596
+ seq_span.set_attribute(
1597
+ SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER,
1598
+ metrics.scheduler_time)
1599
+ if metrics.model_forward_time is not None:
1600
+ seq_span.set_attribute(
1601
+ SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD,
1602
+ metrics.model_forward_time / 1000.0)
1603
+ if metrics.model_execute_time is not None:
1604
+ seq_span.set_attribute(
1605
+ SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE,
1606
+ metrics.model_execute_time)
1607
1608
def is_encoder_decoder_model(self):
1609
return self.model_config.is_encoder_decoder_model
vllm/executor/executor_base.py
+4
-3
@@ -2,8 +2,8 @@ from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
- ModelConfig, MultiModalConfig, ParallelConfig,
6
- PromptAdapterConfig, SchedulerConfig,
5
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
6
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig,
7
SpeculativeConfig)
8
from vllm.lora.request import LoRARequest
9
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -32,6 +32,7 @@ class ExecutorBase(ABC):
32
multimodal_config: Optional[MultiModalConfig],
33
speculative_config: Optional[SpeculativeConfig],
34
prompt_adapter_config: Optional[PromptAdapterConfig],
35
+ observability_config: Optional[ObservabilityConfig],
36
) -> None:
37
self.model_config = model_config
38
self.cache_config = cache_config
@@ -43,7 +44,7 @@ class ExecutorBase(ABC):
44
self.multimodal_config = multimodal_config
45
self.speculative_config = speculative_config
46
self.prompt_adapter_config = prompt_adapter_config
46
-
47
+ self.observability_config = observability_config
48
self._init_executor()
49
50
@abstractmethod
vllm/executor/gpu_executor.py
+1
@@ -60,6 +60,7 @@ class GPUExecutor(ExecutorBase):
60
prompt_adapter_config=self.prompt_adapter_config,
61
is_driver_worker=(not self.parallel_config)
62
or (rank % self.parallel_config.tensor_parallel_size == 0),
63
+ observability_config=self.observability_config,
64
)
65
66
def _get_create_worker_kwargs(
vllm/sequence.py
+17
@@ -92,6 +92,13 @@ class RequestMetrics:
92
first_token_time: The time when the first token was generated.
93
time_in_queue: The time the request spent in the queue.
94
finished_time: The time when the request was finished.
95
+ scheduler_time: The time spent in the scheduler when this request was
96
+ being considered by the scheduler.
97
+ model_forward_time: The time spent in the model forward pass when this
98
+ request was in the batch.
99
+ model_execute_time: The time spent in the model execute function. This
100
+ will include model forward, block/sync across
101
+ workers, cpu-gpu sync time and sampling time.
102
"""
103
arrival_time: float
104
last_token_time: float
@@ -99,6 +106,9 @@ class RequestMetrics:
106
first_token_time: Optional[float]
107
time_in_queue: Optional[float]
108
finished_time: Optional[float] = None
109
+ scheduler_time: Optional[float] = None
110
+ model_forward_time: Optional[float] = None
111
+ model_execute_time: Optional[float] = None
112
113
114
class SequenceData:
@@ -968,6 +978,13 @@ class SamplerOutput:
978
# Optional last hidden states from the model.
979
hidden_states: Optional[torch.Tensor] = None
980
981
+ # Time taken in the forward pass for this across all workers
982
+ model_forward_time: Optional[float] = None
983
+
984
+ # Time taken in the model execute function. This will include model forward,
985
+ # block/sync across workers, cpu-gpu sync time and sampling time.
986
+ model_execute_time: Optional[float] = None
987
+
988
def __getitem__(self, idx: int):
989
return self.outputs[idx]
990
vllm/spec_decode/draft_model_runner.py
+4
-2
@@ -23,8 +23,8 @@ except ImportError:
23
FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24
25
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26
- ModelConfig, MultiModalConfig, ParallelConfig,
27
- PromptAdapterConfig, SchedulerConfig)
26
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
27
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig)
28
from vllm.logger import init_logger
29
from vllm.multimodal import MultiModalInputs
30
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
@@ -69,6 +69,7 @@ class TP1DraftModelRunner(ModelRunner):
69
multimodal_config: Optional[MultiModalConfig] = None,
70
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
71
return_hidden_states: bool = False,
72
+ observability_config: Optional[ObservabilityConfig] = None,
73
):
74
if return_hidden_states:
75
raise ValueError(
@@ -88,6 +89,7 @@ class TP1DraftModelRunner(ModelRunner):
89
multimodal_config=multimodal_config,
90
prompt_adapter_config=prompt_adapter_config,
91
return_hidden_states=return_hidden_states,
92
+ observability_config=observability_config,
93
)
94
95
self.flashinfer_decode_workspace_buffer = None
vllm/spec_decode/target_model_runner.py
+5
-3
@@ -1,8 +1,8 @@
1
from typing import List, Optional
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
4
- ModelConfig, MultiModalConfig, ParallelConfig,
5
- PromptAdapterConfig, SchedulerConfig)
4
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
5
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig)
6
from vllm.sequence import SequenceGroupMetadata
7
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
8
ModelRunner)
@@ -32,7 +32,8 @@ class TargetModelRunner(ModelRunner):
32
is_driver_worker: bool = False,
33
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
34
multimodal_config: Optional[MultiModalConfig] = None,
35
- return_hidden_states: bool = False):
35
+ return_hidden_states: bool = False,
36
+ observability_config: Optional[ObservabilityConfig] = None):
37
# An internal boolean member variable to indicate if token log
38
# probabilities are needed or not.
39
self.disable_logprobs = True
@@ -49,6 +50,7 @@ class TargetModelRunner(ModelRunner):
50
multimodal_config=multimodal_config,
51
prompt_adapter_config=prompt_adapter_config,
52
return_hidden_states=return_hidden_states,
53
+ observability_config=observability_config,
54
)
55
56
def prepare_model_input(
vllm/tracing.py
+6
@@ -92,6 +92,12 @@ class SpanAttributes(BaseSpanAttributes):
92
LLM_LATENCY_TIME_IN_QUEUE = "gen_ai.latency.time_in_queue"
93
LLM_LATENCY_TIME_TO_FIRST_TOKEN = "gen_ai.latency.time_to_first_token"
94
LLM_LATENCY_E2E = "gen_ai.latency.e2e"
95
+ LLM_LATENCY_TIME_IN_SCHEDULER = "gen_ai.latency.time_in_scheduler"
96
+ # Time taken in the forward pass for this across all workers
97
+ LLM_LATENCY_TIME_IN_MODEL_FORWARD = "gen_ai.latency.time_in_model_forward"
98
+ # Time taken in the model execute function. This will include model
99
+ # forward, block/sync across workers, cpu-gpu sync time and sampling time.
100
+ LLM_LATENCY_TIME_IN_MODEL_EXECUTE = "gen_ai.latency.time_in_model_execute"
101
102
103
def contains_trace_headers(headers: Mapping[str, str]) -> bool:
vllm/worker/embedding_model_runner.py
+5
-3
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4
import torch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
- ModelConfig, MultiModalConfig, ParallelConfig,
8
- PromptAdapterConfig, SchedulerConfig)
7
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
8
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.multimodal import MultiModalInputs
@@ -45,6 +45,7 @@ class EmbeddingModelRunner(
45
is_driver_worker: bool = False,
46
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
47
multimodal_config: Optional[MultiModalConfig] = None,
48
+ observability_config: Optional[ObservabilityConfig] = None,
49
):
50
super().__init__(model_config,
51
parallel_config,
@@ -56,7 +57,8 @@ class EmbeddingModelRunner(
57
kv_cache_dtype=kv_cache_dtype,
58
is_driver_worker=is_driver_worker,
59
prompt_adapter_config=prompt_adapter_config,
59
- multimodal_config=multimodal_config)
60
+ multimodal_config=multimodal_config,
61
+ observability_config=observability_config)
62
63
@torch.inference_mode()
64
def execute_model(
vllm/worker/enc_dec_model_runner.py
+3
-2
@@ -10,8 +10,8 @@ from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
10
get_global_forced_attn_backend,
11
global_force_attn_backend)
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
- ModelConfig, MultiModalConfig, ParallelConfig,
14
- PromptAdapterConfig, SchedulerConfig)
13
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
14
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig)
15
from vllm.inputs import INPUT_REGISTRY
16
from vllm.logger import init_logger
17
from vllm.model_executor import SamplingMetadata
@@ -82,6 +82,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
82
is_driver_worker: bool = False,
83
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
84
multimodal_config: Optional[MultiModalConfig] = None,
85
+ observability_config: Optional[ObservabilityConfig] = None,
86
):
87
'''
88
EncoderDecoderModelRunner constructor.
vllm/worker/model_runner.py
+28
-4
@@ -27,8 +27,8 @@ except ImportError:
27
import vllm.envs as envs
28
from vllm.attention import AttentionMetadata, get_attn_backend
29
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
30
- ModelConfig, MultiModalConfig, ParallelConfig,
31
- PromptAdapterConfig, SchedulerConfig)
30
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
31
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig)
32
from vllm.distributed import get_pp_group
33
from vllm.distributed.parallel_state import graph_capture
34
from vllm.inputs import INPUT_REGISTRY
@@ -806,6 +806,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
806
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
807
multimodal_config: Optional[MultiModalConfig] = None,
808
return_hidden_states: bool = False,
809
+ observability_config: Optional[ObservabilityConfig] = None,
810
):
811
self.model_config = model_config
812
self.parallel_config = parallel_config
@@ -818,6 +819,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
819
self.prompt_adapter_config = prompt_adapter_config
820
self.multimodal_config = multimodal_config
821
self.return_hidden_states = return_hidden_states
822
+ self.observability_config = observability_config
823
824
self.device = self.device_config.device
825
self.pin_memory = is_pin_memory_available()
@@ -1527,6 +1529,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1529
"finished_requests_ids": model_input.finished_requests_ids,
1530
"request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1531
} if self.has_seqlen_agnostic else {}
1532
+ if (self.observability_config is not None
1533
+ and self.observability_config.collect_model_forward_time):
1534
+ model_forward_start = torch.cuda.Event(enable_timing=True)
1535
+ model_forward_end = torch.cuda.Event(enable_timing=True)
1536
+ model_forward_start.record()
1537
+
1538
hidden_or_intermediate_states = model_executable(
1539
input_ids=model_input.input_tokens,
1540
positions=model_input.input_positions,
@@ -1537,6 +1545,10 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1545
device=self.device),
1546
**seqlen_agnostic_kwargs)
1547
1548
+ if (self.observability_config is not None
1549
+ and self.observability_config.collect_model_forward_time):
1550
+ model_forward_end.record()
1551
+
1552
# Compute the logits in the last pipeline stage.
1553
if not get_pp_group().is_last_rank:
1554
return hidden_or_intermediate_states
@@ -1552,6 +1564,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1564
logits=logits,
1565
sampling_metadata=model_input.sampling_metadata,
1566
)
1567
+ if (self.observability_config is not None
1568
+ and self.observability_config.collect_model_forward_time
1569
+ and output is not None):
1570
+ model_forward_end.synchronize()
1571
+ model_forward_time = model_forward_start.elapsed_time(
1572
+ model_forward_end)
1573
+ # If there are multiple workers, we are still tracking the latency
1574
+ # from the start time of the driver worker to the end time of the
1575
+ # driver worker. The model forward time will then end up covering
1576
+ # the communication time as well.
1577
+ output.model_forward_time = model_forward_time
1578
1579
if self.return_hidden_states:
1580
# we only need to pass hidden states of most recent token
@@ -1709,8 +1732,9 @@ class CUDAGraphRunner:
1732
**kwargs)
1733
if intermediate_tensors is not None:
1734
for key in intermediate_tensors.tensors:
1712
- self.input_buffers[key].copy_(intermediate_tensors[key],
1713
- non_blocking=True)
1735
+ if key != "model_execute_time":
1736
+ self.input_buffers[key].copy_(intermediate_tensors[key],
1737
+ non_blocking=True)
1738
# Run the graph.
1739
self.graph.replay()
1740
# Return the output tensor.
vllm/worker/worker.py
+5
-2
@@ -7,8 +7,8 @@ import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, MultiModalConfig, ParallelConfig,
11
- PromptAdapterConfig, SchedulerConfig,
10
+ ModelConfig, MultiModalConfig, ObservabilityConfig,
11
+ ParallelConfig, PromptAdapterConfig, SchedulerConfig,
12
SpeculativeConfig)
13
from vllm.distributed import (ensure_model_parallel_initialized,
14
init_distributed_environment,
@@ -51,6 +51,7 @@ class Worker(LocalOrDistributedWorkerBase):
51
prompt_adapter_config: Optional[PromptAdapterConfig] = None,
52
is_driver_worker: bool = False,
53
model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
54
+ observability_config: Optional[ObservabilityConfig] = None,
55
) -> None:
56
self.model_config = model_config
57
self.parallel_config = parallel_config
@@ -73,6 +74,7 @@ class Worker(LocalOrDistributedWorkerBase):
74
from vllm.utils import init_cached_hf_modules
75
init_cached_hf_modules()
76
self.multimodal_config = multimodal_config
77
+ self.observability_config = observability_config
78
79
# Return hidden states from target model if the draft model is an
80
# mlp_speculator
@@ -102,6 +104,7 @@ class Worker(LocalOrDistributedWorkerBase):
104
is_driver_worker=is_driver_worker,
105
prompt_adapter_config=prompt_adapter_config,
106
multimodal_config=multimodal_config,
107
+ observability_config=observability_config,
108
**speculative_args,
109
)
110
# Uninitialized cache engine. Will be initialized by
vllm/worker/worker_base.py
+20
-1
@@ -1,11 +1,13 @@
1
import dataclasses
2
import importlib
3
import os
4
+import time
5
from abc import ABC, abstractmethod
6
from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
7
8
import torch
9
10
+from vllm.config import ObservabilityConfig
11
from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
@@ -172,6 +174,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
174
"""
175
is_driver_worker: bool
176
model_runner: ModelRunnerBase
177
+ observability_config: Optional[ObservabilityConfig] = None
178
179
@property
180
@abstractmethod
@@ -219,6 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
222
) -> Optional[List[SamplerOutput]]:
223
"""Executes at least one model step on the given sequences, unless no
224
sequences are provided."""
225
+ start_time = time.perf_counter()
226
if self.is_driver_worker:
227
if execute_model_req is None:
228
if self.do_metadata_broadcast:
@@ -265,21 +269,36 @@ class LocalOrDistributedWorkerBase(WorkerBase):
269
return []
270
271
intermediate_tensors = None
272
+ orig_model_execute_time = 0.0
273
if not get_pp_group().is_first_rank:
274
intermediate_tensors = IntermediateTensors(
275
get_pp_group().recv_tensor_dict(
276
all_gather_group=get_tp_group()))
277
+ if (self.observability_config is not None
278
+ and self.observability_config.collect_model_execute_time):
279
+ orig_model_execute_time = intermediate_tensors.tensors.get(
280
+ "model_execute_time", torch.tensor(0)).item()
281
282
output = self.model_runner.execute_model(
283
model_input, self.kv_cache[worker_input.virtual_engine]
284
if self.kv_cache is not None else None, intermediate_tensors,
285
num_steps)
277
-
286
+ model_execute_time = time.perf_counter() - start_time
287
if not get_pp_group().is_last_rank:
288
# output is IntermediateTensors
289
+ if (self.observability_config is not None
290
+ and self.observability_config.collect_model_execute_time):
291
+ output.tensors["model_execute_time"] = torch.tensor(
292
+ model_execute_time + orig_model_execute_time)
293
get_pp_group().send_tensor_dict(output.tensors,
294
all_gather_group=get_tp_group())
295
return [None]
296
+ if (self.observability_config is not None
297
+ and self.observability_config.collect_model_execute_time
298
+ and output is not None):
299
+ for o in output:
300
+ o.model_execute_time = (orig_model_execute_time +
301
+ model_execute_time)
302
303
# output is List[SamplerOutput]
304
return output