A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Fix tracking of model forward time in case of PP>1 (#7440)
[Core] Fix tracking of model forward time to the span traces in case of PP>1 (#7440)
Mahesh Keralapura committed
Aug 16, 2024 at 13:46 UTC
93478b63d2c40a88e213125f11833ffb05d45da0
5 files changed
+94
-11
.buildkite/test-pipeline.yaml
+5
-3
@@ -101,10 +101,9 @@ steps:
101
- pytest -v -s distributed/test_pynccl.py
102
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
103
104
-##### fast check tests #####
105
-##### 1 GPU test #####
106
-
104
- label: Metrics, Tracing Test # 10min
105
+ num_gpus: 2
106
+ fast_check: true
107
source_file_dependencies:
108
- vllm/
109
- tests/metrics
@@ -118,6 +117,9 @@ steps:
117
opentelemetry-semantic-conventions-ai"
118
- pytest -v -s tracing
119
120
+##### fast check tests #####
121
+##### 1 GPU test #####
122
+
123
- label: Regression Test # 5min
124
mirror_hardwares: [amd]
125
source_file_dependencies:
tests/tracing/test_tracing.py
+66
@@ -114,5 +114,71 @@ def test_traces(trace_service):
114
SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115
e2e_time = metrics.finished_time - metrics.arrival_time
116
assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117
+ assert metrics.scheduler_time > 0
118
assert attributes.get(
119
SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
120
+ # Model forward and model execute should be none, since detailed traces is
121
+ # not enabled.
122
+ assert metrics.model_forward_time is None
123
+ assert metrics.model_execute_time is None
124
+
125
+
126
+def test_traces_with_detailed_steps(trace_service):
127
+ os.environ[OTEL_EXPORTER_OTLP_TRACES_INSECURE] = "true"
128
+
129
+ sampling_params = SamplingParams(temperature=0.01,
130
+ top_p=0.1,
131
+ max_tokens=256)
132
+ model = "facebook/opt-125m"
133
+ llm = LLM(
134
+ model=model,
135
+ otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
136
+ collect_detailed_traces="all",
137
+ )
138
+ prompts = ["This is a short prompt"]
139
+ outputs = llm.generate(prompts, sampling_params=sampling_params)
140
+
141
+ timeout = 5
142
+ if not trace_service.evt.wait(timeout):
143
+ raise TimeoutError(
144
+ f"The fake trace service didn't receive a trace within "
145
+ f"the {timeout} seconds timeout")
146
+
147
+ attributes = decode_attributes(trace_service.request.resource_spans[0].
148
+ scope_spans[0].spans[0].attributes)
149
+ assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
150
+ assert attributes.get(
151
+ SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
152
+ assert attributes.get(
153
+ SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
154
+ assert attributes.get(
155
+ SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
156
+ assert attributes.get(
157
+ SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
158
+ assert attributes.get(
159
+ SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
160
+ assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
161
+ assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
162
+ outputs[0].prompt_token_ids)
163
+ completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
164
+ assert attributes.get(
165
+ SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
166
+ metrics = outputs[0].metrics
167
+ assert attributes.get(
168
+ SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
169
+ ttft = metrics.first_token_time - metrics.arrival_time
170
+ assert attributes.get(
171
+ SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
172
+ e2e_time = metrics.finished_time - metrics.arrival_time
173
+ assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
174
+ assert metrics.scheduler_time > 0
175
+ assert attributes.get(
176
+ SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
177
+ assert metrics.model_forward_time > 0
178
+ assert attributes.get(
179
+ SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD) == pytest.approx(
180
+ metrics.model_forward_time / 1000)
181
+ assert metrics.model_execute_time > 0
182
+ assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE
183
+ ) == metrics.model_execute_time
184
+ assert metrics.model_forward_time < 1000 * metrics.model_execute_time
vllm/core/scheduler.py
+1
-1
@@ -1033,9 +1033,9 @@ class Scheduler:
1033
# Schedule sequence groups.
1034
# This function call changes the internal states of the scheduler
1035
# such as self.running, self.swapped, and self.waiting.
1036
+ scheduler_start_time = time.perf_counter()
1037
scheduler_outputs = self._schedule()
1038
now = time.time()
1038
- scheduler_start_time = time.perf_counter()
1039
1040
if not self.cache_config.enable_prefix_caching:
1041
common_computed_block_nums = []
vllm/engine/arg_utils.py
-5
@@ -948,11 +948,6 @@ class EngineArgs:
948
raise ValueError(
949
f"Invalid module {m} in collect_detailed_traces. "
950
f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}")
951
- if (m == "model"
952
- or m == "all") and self.pipeline_parallel_size > 1:
953
- raise ValueError(
954
- "Collection of detailed traces for the 'model' module is "
955
- "not yet supported with pipeline parallelism.")
951
observability_config = ObservabilityConfig(
952
otlp_traces_endpoint=self.otlp_traces_endpoint,
953
collect_model_forward_time="model" in detailed_trace_modules
vllm/worker/model_runner.py
+22
-2
@@ -1554,6 +1554,21 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1554
1555
# Compute the logits in the last pipeline stage.
1556
if not get_pp_group().is_last_rank:
1557
+ if (self.is_driver_worker
1558
+ and hidden_or_intermediate_states is not None
1559
+ and isinstance(hidden_or_intermediate_states,
1560
+ IntermediateTensors)
1561
+ and self.observability_config is not None
1562
+ and self.observability_config.collect_model_forward_time):
1563
+ model_forward_end.synchronize()
1564
+ model_forward_time = model_forward_start.elapsed_time(
1565
+ model_forward_end)
1566
+ orig_model_forward_time = 0.0
1567
+ if intermediate_tensors is not None:
1568
+ orig_model_forward_time = intermediate_tensors.tensors.get(
1569
+ "model_forward_time", torch.tensor(0.0)).item()
1570
+ hidden_or_intermediate_states.tensors["model_forward_time"] = (
1571
+ torch.tensor(model_forward_time + orig_model_forward_time))
1572
return hidden_or_intermediate_states
1573
1574
logits = self.model.compute_logits(hidden_or_intermediate_states,
@@ -1573,11 +1588,16 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1588
model_forward_end.synchronize()
1589
model_forward_time = model_forward_start.elapsed_time(
1590
model_forward_end)
1591
+ orig_model_forward_time = 0.0
1592
+ if intermediate_tensors is not None:
1593
+ orig_model_forward_time = intermediate_tensors.tensors.get(
1594
+ "model_forward_time", torch.tensor(0.0)).item()
1595
# If there are multiple workers, we are still tracking the latency
1596
# from the start time of the driver worker to the end time of the
1597
# driver worker. The model forward time will then end up covering
1598
# the communication time as well.
1580
- output.model_forward_time = model_forward_time
1599
+ output.model_forward_time = (orig_model_forward_time +
1600
+ model_forward_time)
1601
1602
if self.return_hidden_states:
1603
# we only need to pass hidden states of most recent token
@@ -1735,7 +1755,7 @@ class CUDAGraphRunner:
1755
**kwargs)
1756
if intermediate_tensors is not None:
1757
for key in intermediate_tensors.tensors:
1738
- if key != "model_execute_time":
1758
+ if key != "model_execute_time" and key != "model_forward_time":
1759
self.input_buffers[key].copy_(intermediate_tensors[key],
1760
non_blocking=True)
1761
# Run the graph.