A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[mypy][5/N] Support all typing on model executor (#4427)
SangBin Cho committed
Apr 29, 2024 at 11:01 UTC
df29793dc73a83f3c86c19de967adffda1a28a93
10 files changed
+61
-34
.github/workflows/mypy.yaml
+1
-1
@@ -43,8 +43,8 @@ jobs:
43
mypy vllm/worker --config-file pyproject.toml
44
mypy vllm/spec_decode --config-file pyproject.toml
45
mypy vllm/lora --config-file pyproject.toml
46
+ mypy vllm/model_executor --config-file pyproject.toml
47
48
# TODO(sang): Fix nested dir
48
- mypy vllm/model_executor/*.py --config-file pyproject.toml
49
mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
50
format.sh
+1
-1
@@ -105,7 +105,7 @@ mypy vllm/transformers_utils --config-file pyproject.toml
105
mypy vllm/engine --config-file pyproject.toml
106
mypy vllm/worker --config-file pyproject.toml
107
mypy vllm/spec_decode --config-file pyproject.toml
108
-mypy vllm/model_executor/*.py --config-file pyproject.toml
108
+mypy vllm/model_executor --config-file pyproject.toml
109
mypy vllm/lora --config-file pyproject.toml
110
111
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+1
@@ -61,6 +61,7 @@ def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
61
return schema
62
if isinstance(schema, BaseModel):
63
return schema.model_json_schema()
64
+ raise AssertionError(f"Unsupported schema type {schema}")
65
66
67
@lru_cache
vllm/model_executor/layers/linear.py
+11
-1
@@ -128,7 +128,8 @@ class LinearBase(torch.nn.Module):
128
params_dtype = torch.get_default_dtype()
129
self.params_dtype = params_dtype
130
if quant_config is None:
131
- self.quant_method = UnquantizedLinearMethod()
131
+ self.quant_method: Optional[
132
+ QuantizeMethodBase] = UnquantizedLinearMethod()
133
else:
134
self.quant_method = quant_config.get_quant_method(self)
135
@@ -160,6 +161,8 @@ class ReplicatedLinear(LinearBase):
161
super().__init__(input_size, output_size, skip_bias_add, params_dtype,
162
quant_config)
163
164
+ # All the linear layer supports quant method.
165
+ assert self.quant_method is not None
166
self.quant_method.create_weights(self, self.input_size,
167
[self.output_size], self.input_size,
168
self.output_size, self.params_dtype)
@@ -173,6 +176,7 @@ class ReplicatedLinear(LinearBase):
176
177
def forward(self, x: torch.Tensor) -> torch.Tensor:
178
bias = self.bias if not self.skip_bias_add else None
179
+ assert self.quant_method is not None
180
output = self.quant_method.apply(self, x, bias)
181
output_bias = self.bias if self.skip_bias_add else None
182
return output, output_bias
@@ -221,6 +225,8 @@ class ColumnParallelLinear(LinearBase):
225
self.output_size_per_partition = divide(output_size, tp_size)
226
if output_sizes is None:
227
output_sizes = [output_size]
228
+ # All the linear layer supports quant method.
229
+ assert self.quant_method is not None
230
self.quant_method.create_weights(self,
231
self.input_size,
232
[x // tp_size for x in output_sizes],
@@ -255,6 +261,7 @@ class ColumnParallelLinear(LinearBase):
261
bias = self.bias if not self.skip_bias_add else None
262
263
# Matrix multiply.
264
+ assert self.quant_method is not None
265
output_parallel = self.quant_method.apply(self, input_, bias)
266
if self.gather_output:
267
# All-gather across the partitions.
@@ -579,6 +586,8 @@ class RowParallelLinear(LinearBase):
586
# Divide the weight matrix along the last dimension.
587
self.tp_size = get_tensor_model_parallel_world_size()
588
self.input_size_per_partition = divide(input_size, self.tp_size)
589
+ # All the linear layer supports quant method.
590
+ assert self.quant_method is not None
591
self.quant_method.create_weights(self,
592
self.input_size_per_partition,
593
[self.output_size],
@@ -624,6 +633,7 @@ class RowParallelLinear(LinearBase):
633
input_parallel = splitted_input[tp_rank].contiguous()
634
635
# Matrix multiply.
636
+ assert self.quant_method is not None
637
output_parallel = self.quant_method.apply(self, input_parallel)
638
if self.reduce_results and self.tp_size > 1:
639
output_ = tensor_model_parallel_all_reduce(output_parallel)
vllm/model_executor/layers/quantization/__init__.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import Type
1
+from typing import Dict, Type
2
3
from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4
from vllm.model_executor.layers.quantization.awq import AWQConfig
@@ -9,7 +9,7 @@ from vllm.model_executor.layers.quantization.gptq import GPTQConfig
9
from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10
from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
11
12
-QUANTIZATION_METHODS = {
12
+QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
13
"aqlm": AQLMConfig,
14
"awq": AWQConfig,
15
"fp8": Fp8Config,
vllm/model_executor/layers/quantization/base_config.py
+11
-3
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Any, Dict, List
2
+from typing import Any, Dict, List, Optional
3
4
import torch
5
from torch import nn
@@ -76,8 +76,16 @@ class QuantizationConfig(ABC):
76
"quantization config.")
77
78
@abstractmethod
79
- def get_quant_method(self, layer: torch.nn.Module) -> QuantizeMethodBase:
80
- """Get the quantize method to use for the quantized layer."""
79
+ def get_quant_method(
80
+ self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
81
+ """Get the quantize method to use for the quantized layer.
82
+
83
+ Args:
84
+ layer: The layer for the quant method.
85
+ Returns:
86
+ The quantize method. None if the given layer doesn't support quant
87
+ method.
88
+ """
89
raise NotImplementedError
90
91
@abstractmethod
vllm/model_executor/layers/quantization/squeezellm.py
+2
-3
@@ -52,11 +52,10 @@ class SqueezeLLMConfig(QuantizationConfig):
52
return cls(weight_bits)
53
54
def get_quant_method(
55
- self,
56
- layer: torch.nn.Module) -> Optional["SqueezeLLMLinearMethod"]:
55
+ self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
56
if isinstance(layer, LinearBase):
57
return SqueezeLLMLinearMethod(self)
59
- return
58
+ return None
59
60
def get_scaled_act_names(self) -> List[str]:
61
return []
vllm/model_executor/layers/rotary_embedding.py
+2
-2
@@ -431,8 +431,8 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
431
torch.full_like(positions, k)).long()
432
idx = (torch.add(positions, long_prompt_offset)
433
if long_prompt_offset is not None else positions)
434
- self.long_short_cos_sin_cache = self.long_short_cos_sin_cache.to(
435
- idx.device)
434
+ self.long_short_cos_sin_cache: torch.Tensor = (
435
+ self.long_short_cos_sin_cache.to(idx.device))
436
idx = torch.add(idx, offsets) if offsets is not None else idx
437
cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
438
vllm/model_executor/layers/sampler.py
+27
-20
@@ -13,6 +13,9 @@ from vllm.sampling_params import SamplingType
13
from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
14
SamplerOutput, SequenceGroupOutput, SequenceOutput)
15
16
+# (num_token_ids, num_parent_ids) per sequence group.
17
+SampleResultType = List[Tuple[List[int], List[int]]]
18
+
19
20
class Sampler(nn.Module):
21
"""Samples the next tokens from the model's outputs.
@@ -155,7 +158,7 @@ def _apply_min_tokens_penalty(
158
have not been generated yet
159
"""
160
# list of indices in logits that will be set to -inf
158
- logits_to_penalize = []
161
+ logits_to_penalize: List[Tuple[int, int]] = []
162
logits_applied = 0
163
for seq_group in sampling_metadata.seq_groups:
164
seq_ids = seq_group.seq_ids
@@ -269,7 +272,7 @@ def _apply_min_p(
272
def _greedy_sample(
273
selected_seq_groups: List[SequenceGroupToSample],
274
samples: torch.Tensor,
272
-) -> List[Tuple[List[int], List[int]]]:
275
+) -> SampleResultType:
276
"""Run greedy sampling on a given samples.
277
278
Args:
@@ -284,7 +287,7 @@ def _greedy_sample(
287
"""
288
samples = samples.tolist()
289
sample_idx = 0
287
- results = []
290
+ results: SampleResultType = []
291
for seq_group in selected_seq_groups:
292
if not seq_group.do_sample:
293
results.append(([], []))
@@ -304,7 +307,7 @@ def _greedy_sample(
307
def _random_sample(
308
selected_seq_groups: List[SequenceGroupToSample],
309
random_samples: torch.Tensor,
307
-) -> List[Tuple[List[int], List[int]]]:
310
+) -> SampleResultType:
311
"""Run random sampling on a given samples.
312
313
Args:
@@ -320,7 +323,7 @@ def _random_sample(
323
# Find the maximum best_of value of the prompt phase requests.
324
random_samples = random_samples.cpu()
325
sample_idx = 0
323
- results = []
326
+ results: SampleResultType = []
327
for seq_group in selected_seq_groups:
328
if not seq_group.do_sample:
329
results.append(([], []))
@@ -348,7 +351,7 @@ def _random_sample(
351
def _beam_search_sample(
352
selected_seq_groups: List[SequenceGroupToSample],
353
logprobs: torch.Tensor,
351
-) -> List[Tuple[List[int], List[int]]]:
354
+) -> SampleResultType:
355
"""Run beam sampling on a given samples.
356
357
Args:
@@ -370,7 +373,7 @@ def _beam_search_sample(
373
# NOTE: Beam search is not vectorized, so its speed can be slower than
374
# other sampling methods.
375
sample_idx = 0
373
- results = []
376
+ results: SampleResultType = []
377
for seq_group in selected_seq_groups:
378
if not seq_group.do_sample:
379
results.append(([], []))
@@ -391,16 +394,16 @@ def _beam_search_sample(
394
next_token_ids = next_token_ids.tolist()
395
else:
396
# Generation phase.
394
- cumulative_logprobs = [
397
+ cumulative_logprobs: List[int] = [
398
seq_group.seq_data[seq_id].cumulative_logprob
399
for seq_id in seq_ids
400
]
398
- cumulative_logprobs = torch.tensor(
401
+ cumulative_logprobs_tensor = torch.tensor(
402
cumulative_logprobs,
403
dtype=torch.float,
404
device=seq_group_logprobs.device)
405
seq_group_logprobs = (seq_group_logprobs +
403
- cumulative_logprobs.unsqueeze(dim=1))
406
+ cumulative_logprobs_tensor.unsqueeze(dim=1))
407
_, topk_ids = torch.topk(seq_group_logprobs.flatten(),
408
2 * beam_width)
409
topk_ids = topk_ids.tolist()
@@ -452,8 +455,10 @@ def _sample_with_torch(
455
sampling_metadata: SamplingMetadata,
456
include_gpu_probs_tensor: bool,
457
modify_greedy_probs: bool,
455
-) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
456
- categorized_seq_group_ids = {t: [] for t in SamplingType}
458
+) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
459
+ categorized_seq_group_ids: Dict[SamplingType,
460
+ List[int]] = {t: []
461
+ for t in SamplingType}
462
categorized_sample_indices = sampling_metadata.categorized_sample_indices
463
for i, seq_group in enumerate(sampling_metadata.seq_groups):
464
sampling_params = seq_group.sampling_params
@@ -555,8 +560,10 @@ def _sample_with_triton_kernel(
560
logprobs: torch.Tensor,
561
sampling_metadata: SamplingMetadata,
562
sampling_tensors: SamplingTensors,
558
-) -> List[Tuple[List[int], List[int]]]:
559
- categorized_seq_group_ids = {t: [] for t in SamplingType}
563
+) -> SampleResultType:
564
+ categorized_seq_group_ids: Dict[SamplingType,
565
+ List[int]] = {t: []
566
+ for t in SamplingType}
567
categorized_sample_indices = sampling_metadata.categorized_sample_indices
568
for i, seq_group in enumerate(sampling_metadata.seq_groups):
569
sampling_params = seq_group.sampling_params
@@ -632,7 +639,7 @@ def _sample(
639
probs: torch.Tensor, logprobs: torch.Tensor,
640
sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
641
include_gpu_probs_tensor: bool, modify_greedy_probs: bool
635
-) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
642
+) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
643
"""
644
Args:
645
probs: (num_query_tokens_in_batch, num_vocab)
@@ -680,7 +687,7 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
687
def _get_logprobs(
688
logprobs: torch.Tensor,
689
sampling_metadata: SamplingMetadata,
683
- sample_results: List[Tuple[List[int], List[int]]],
690
+ sample_results: SampleResultType,
691
) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
692
"""Return sample lobprobs and prompt logprobs.
693
@@ -751,8 +758,8 @@ def _get_logprobs(
758
assert len(next_token_ids) == len(query_indices)
759
760
if len(query_indices) == 0:
754
- empty_sampled_logprob = []
755
- empty_prompt_logprob = None
761
+ empty_sampled_logprob: SampleLogprobs = []
762
+ empty_prompt_logprob: Optional[PromptLogprobs] = None
763
return [empty_prompt_logprob], [empty_sampled_logprob]
764
765
query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
@@ -965,7 +972,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
972
973
974
def _build_sampler_output(
968
- sample_results: List[Tuple[List[int], List[int]]],
975
+ sample_results: SampleResultType,
976
sampling_metadata: SamplingMetadata,
977
prompt_logprobs: List[Optional[PromptLogprobs]],
978
sample_logprobs: List[SampleLogprobs],
@@ -1009,7 +1016,7 @@ def _build_sampler_output(
1016
)
1017
1018
1012
-def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[str]:
1019
+def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1020
"""Get a list of next prompt tokens to compute logprob from a
1021
given sequence group.
1022
vllm/model_executor/model_loader/tensorizer.py
+3
-1
@@ -64,7 +64,7 @@ class TensorizerConfig:
64
"s3_secret_access_key": self.s3_secret_access_key,
65
"s3_endpoint": self.s3_endpoint,
66
}
67
- return TensorizerArgs(**tensorizer_args)
67
+ return TensorizerArgs(**tensorizer_args) # type: ignore
68
69
def verify_with_parallel_config(
70
self,
@@ -270,8 +270,10 @@ class TensorizerAgent:
270
self.model = self._init_model()
271
272
def _init_model(self):
273
+ assert self.tensorizer_config.hf_config is not None
274
model_args = self.tensorizer_config.hf_config
275
model_args.torch_dtype = self.tensorizer_config.dtype
276
+ assert self.tensorizer_config.model_class is not None
277
with no_init_or_tensor():
278
return self.tensorizer_config.model_class(
279
config=model_args,