A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[CI/Build] Fix pre-commit errors (#13696)
Cyrus Leung committed
Feb 22, 2025 at 16:31 UTC
7f6bae561c210da06af5d40e8861b0d2ddfe339c
6 files changed
+24
-17
benchmarks/benchmark_latency.py
+3
-3
@@ -43,9 +43,9 @@ def main(args: argparse.Namespace):
43
# the engine will automatically process the request in multiple batches.
44
llm = LLM(**dataclasses.asdict(engine_args))
45
assert llm.llm_engine.model_config.max_model_len >= (
46
- args.input_len + args.output_len), (
47
- "Please ensure that max_model_len is greater than"
48
- " the sum of input_len and output_len.")
46
+ args.input_len +
47
+ args.output_len), ("Please ensure that max_model_len is greater than"
48
+ " the sum of input_len and output_len.")
49
50
sampling_params = SamplingParams(
51
n=args.n,
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -523,7 +523,7 @@ class OpenAIServing:
523
return logprob.decoded_token
524
return tokenizer.decode(token_id)
525
526
- def _is_model_supported(self, model_name) -> bool:
526
+ def _is_model_supported(self, model_name: Optional[str]) -> bool:
527
if not model_name:
528
return True
529
return self.models.is_base_model(model_name)
vllm/entrypoints/openai/serving_score.py
+6
-1
@@ -358,7 +358,12 @@ class ServingScores(OpenAIServing):
358
request.truncate_prompt_tokens,
359
)
360
return self.request_output_to_rerank_response(
361
- final_res_batch, request_id, self._get_model_name(request.model), documents, top_n)
361
+ final_res_batch,
362
+ request_id,
363
+ self._get_model_name(request.model),
364
+ documents,
365
+ top_n,
366
+ )
367
except asyncio.CancelledError:
368
return self.create_error_response("Client disconnected")
369
except ValueError as e:
vllm/model_executor/layers/mamba/mamba_mixer2.py
+5
-8
@@ -134,7 +134,7 @@ def extra_groups_for_head_shards(ngroups: int, tp_size: int):
134
return 0
135
136
# for n_groups == 1, this is exactly tp_size - n_groups
137
- return tp_size - ngroups
137
+ return tp_size - ngroups
138
139
140
def mamba_v2_sharded_weight_loader(
@@ -168,12 +168,9 @@ def mamba_v2_sharded_weight_loader(
168
# - compute the rank into the loaded shard.
169
# - if there is replication, different TP shards will
170
# take from the same rank.
171
- if duplicate_groups:
172
- # NOTE: currently we only support duplication
173
- # in the case where num_groups == 1
174
- rank = 0
175
- else:
176
- rank = tp_rank
171
+ # NOTE: currently we only support duplication
172
+ # in the case where num_groups == 1
173
+ rank = 0 if duplicate_groups else tp_rank
174
175
# - leftmost boundary index into loaded weight.
176
loaded_skip = rank * shard_size
@@ -247,7 +244,7 @@ class MambaMixer2(CustomOp):
244
assert num_heads % self.tp_size == 0, \
245
"Tensor parallel world size must divide num heads."
246
250
-
247
+
248
assert (n_groups % self.tp_size) == 0 or n_groups == 1, \
249
(
250
"If tensor parallel world size does not divide num_heads, "
vllm/utils.py
+4
-2
@@ -1198,10 +1198,12 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
1198
try:
1199
value = int(value)
1200
except ValueError:
1201
- raise argparse.ArgumentTypeError("Port must be an integer")
1201
+ msg = "Port must be an integer"
1202
+ raise argparse.ArgumentTypeError(msg) from None
1203
1204
if not (1024 <= value <= 65535):
1204
- raise argparse.ArgumentTypeError("Port must be between 1024 and 65535")
1205
+ raise argparse.ArgumentTypeError(
1206
+ "Port must be between 1024 and 65535")
1207
1208
return value
1209
vllm/v1/worker/gpu_model_runner.py
+5
-2
@@ -1319,13 +1319,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1319
generators={},
1320
max_num_logprobs=None,
1321
no_penalties=True,
1322
- prompt_token_ids=torch.ones_like(logits, dtype=torch.int64),
1322
+ prompt_token_ids=torch.ones_like(logits,
1323
+ dtype=torch.int64),
1324
frequency_penalties=dummy_tensors(0.1),
1325
presence_penalties=dummy_tensors(0.1),
1326
repetition_penalties=dummy_tensors(0.1),
1327
output_token_ids=[[] for _ in range(num_reqs)],
1328
min_tokens={},
1328
- logit_bias=[None for _ in range(num_reqs)])
1329
+ logit_bias=[None for _ in range(num_reqs)],
1330
+ allowed_token_ids_mask=None,
1331
+ )
1332
sampler_output = self.model.sample(
1333
logits=logits, sampling_metadata=dummy_metadata)
1334
else: