@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93478b63d2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Fix tracking of model forward time in case of PP>1 (#7440)

[Core] Fix tracking of model forward time to the span traces in case of PP>1 (#7440)

Mahesh Keralapura committed Aug 16, 2024 at 13:46 UTC 93478b63d2c40a88e213125f11833ffb05d45da0
5 files changed +94 -11
.buildkite/test-pipeline.yaml
+5 -3
@@ -101,10 +101,9 @@ steps:
101 - pytest -v -s distributed/test_pynccl.py
102 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
103
104 -##### fast check tests #####
105 -##### 1 GPU test #####
106 -
104 - label: Metrics, Tracing Test # 10min
105 + num_gpus: 2
106 + fast_check: true
107 source_file_dependencies:
108 - vllm/
109 - tests/metrics
@@ -118,6 +117,9 @@ steps:
117 opentelemetry-semantic-conventions-ai"
118 - pytest -v -s tracing
119
120 +##### fast check tests #####
121 +##### 1 GPU test #####
122 +
123 - label: Regression Test # 5min
124 mirror_hardwares: [amd]
125 source_file_dependencies:
tests/tracing/test_tracing.py
+66
@@ -114,5 +114,71 @@ def test_traces(trace_service):
114 SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
115 e2e_time = metrics.finished_time - metrics.arrival_time
116 assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
117 + assert metrics.scheduler_time > 0
118 assert attributes.get(
119 SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
120 + # Model forward and model execute should be none, since detailed traces is
121 + # not enabled.
122 + assert metrics.model_forward_time is None
123 + assert metrics.model_execute_time is None
124 +
125 +
126 +def test_traces_with_detailed_steps(trace_service):
127 + os.environ[OTEL_EXPORTER_OTLP_TRACES_INSECURE] = "true"
128 +
129 + sampling_params = SamplingParams(temperature=0.01,
130 + top_p=0.1,
131 + max_tokens=256)
132 + model = "facebook/opt-125m"
133 + llm = LLM(
134 + model=model,
135 + otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
136 + collect_detailed_traces="all",
137 + )
138 + prompts = ["This is a short prompt"]
139 + outputs = llm.generate(prompts, sampling_params=sampling_params)
140 +
141 + timeout = 5
142 + if not trace_service.evt.wait(timeout):
143 + raise TimeoutError(
144 + f"The fake trace service didn't receive a trace within "
145 + f"the {timeout} seconds timeout")
146 +
147 + attributes = decode_attributes(trace_service.request.resource_spans[0].
148 + scope_spans[0].spans[0].attributes)
149 + assert attributes.get(SpanAttributes.LLM_RESPONSE_MODEL) == model
150 + assert attributes.get(
151 + SpanAttributes.LLM_REQUEST_ID) == outputs[0].request_id
152 + assert attributes.get(
153 + SpanAttributes.LLM_REQUEST_TEMPERATURE) == sampling_params.temperature
154 + assert attributes.get(
155 + SpanAttributes.LLM_REQUEST_TOP_P) == sampling_params.top_p
156 + assert attributes.get(
157 + SpanAttributes.LLM_REQUEST_MAX_TOKENS) == sampling_params.max_tokens
158 + assert attributes.get(
159 + SpanAttributes.LLM_REQUEST_BEST_OF) == sampling_params.best_of
160 + assert attributes.get(SpanAttributes.LLM_REQUEST_N) == sampling_params.n
161 + assert attributes.get(SpanAttributes.LLM_USAGE_PROMPT_TOKENS) == len(
162 + outputs[0].prompt_token_ids)
163 + completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
164 + assert attributes.get(
165 + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS) == completion_tokens
166 + metrics = outputs[0].metrics
167 + assert attributes.get(
168 + SpanAttributes.LLM_LATENCY_TIME_IN_QUEUE) == metrics.time_in_queue
169 + ttft = metrics.first_token_time - metrics.arrival_time
170 + assert attributes.get(
171 + SpanAttributes.LLM_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
172 + e2e_time = metrics.finished_time - metrics.arrival_time
173 + assert attributes.get(SpanAttributes.LLM_LATENCY_E2E) == e2e_time
174 + assert metrics.scheduler_time > 0
175 + assert attributes.get(
176 + SpanAttributes.LLM_LATENCY_TIME_IN_SCHEDULER) == metrics.scheduler_time
177 + assert metrics.model_forward_time > 0
178 + assert attributes.get(
179 + SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_FORWARD) == pytest.approx(
180 + metrics.model_forward_time / 1000)
181 + assert metrics.model_execute_time > 0
182 + assert attributes.get(SpanAttributes.LLM_LATENCY_TIME_IN_MODEL_EXECUTE
183 + ) == metrics.model_execute_time
184 + assert metrics.model_forward_time < 1000 * metrics.model_execute_time
vllm/core/scheduler.py
+1 -1
@@ -1033,9 +1033,9 @@ class Scheduler:
1033 # Schedule sequence groups.
1034 # This function call changes the internal states of the scheduler
1035 # such as self.running, self.swapped, and self.waiting.
1036 + scheduler_start_time = time.perf_counter()
1037 scheduler_outputs = self._schedule()
1038 now = time.time()
1038 - scheduler_start_time = time.perf_counter()
1039
1040 if not self.cache_config.enable_prefix_caching:
1041 common_computed_block_nums = []
vllm/engine/arg_utils.py
-5
@@ -948,11 +948,6 @@ class EngineArgs:
948 raise ValueError(
949 f"Invalid module {m} in collect_detailed_traces. "
950 f"Valid modules are {ALLOWED_DETAILED_TRACE_MODULES}")
951 - if (m == "model"
952 - or m == "all") and self.pipeline_parallel_size > 1:
953 - raise ValueError(
954 - "Collection of detailed traces for the 'model' module is "
955 - "not yet supported with pipeline parallelism.")
951 observability_config = ObservabilityConfig(
952 otlp_traces_endpoint=self.otlp_traces_endpoint,
953 collect_model_forward_time="model" in detailed_trace_modules
vllm/worker/model_runner.py
+22 -2
@@ -1554,6 +1554,21 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1554
1555 # Compute the logits in the last pipeline stage.
1556 if not get_pp_group().is_last_rank:
1557 + if (self.is_driver_worker
1558 + and hidden_or_intermediate_states is not None
1559 + and isinstance(hidden_or_intermediate_states,
1560 + IntermediateTensors)
1561 + and self.observability_config is not None
1562 + and self.observability_config.collect_model_forward_time):
1563 + model_forward_end.synchronize()
1564 + model_forward_time = model_forward_start.elapsed_time(
1565 + model_forward_end)
1566 + orig_model_forward_time = 0.0
1567 + if intermediate_tensors is not None:
1568 + orig_model_forward_time = intermediate_tensors.tensors.get(
1569 + "model_forward_time", torch.tensor(0.0)).item()
1570 + hidden_or_intermediate_states.tensors["model_forward_time"] = (
1571 + torch.tensor(model_forward_time + orig_model_forward_time))
1572 return hidden_or_intermediate_states
1573
1574 logits = self.model.compute_logits(hidden_or_intermediate_states,
@@ -1573,11 +1588,16 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1588 model_forward_end.synchronize()
1589 model_forward_time = model_forward_start.elapsed_time(
1590 model_forward_end)
1591 + orig_model_forward_time = 0.0
1592 + if intermediate_tensors is not None:
1593 + orig_model_forward_time = intermediate_tensors.tensors.get(
1594 + "model_forward_time", torch.tensor(0.0)).item()
1595 # If there are multiple workers, we are still tracking the latency
1596 # from the start time of the driver worker to the end time of the
1597 # driver worker. The model forward time will then end up covering
1598 # the communication time as well.
1580 - output.model_forward_time = model_forward_time
1599 + output.model_forward_time = (orig_model_forward_time +
1600 + model_forward_time)
1601
1602 if self.return_hidden_states:
1603 # we only need to pass hidden states of most recent token
@@ -1735,7 +1755,7 @@ class CUDAGraphRunner:
1755 **kwargs)
1756 if intermediate_tensors is not None:
1757 for key in intermediate_tensors.tensors:
1738 - if key != "model_execute_time":
1758 + if key != "model_execute_time" and key != "model_forward_time":
1759 self.input_buffers[key].copy_(intermediate_tensors[key],
1760 non_blocking=True)
1761 # Run the graph.