@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df29793dc7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[mypy][5/N] Support all typing on model executor (#4427)

SangBin Cho committed Apr 29, 2024 at 11:01 UTC df29793dc73a83f3c86c19de967adffda1a28a93
10 files changed +61 -34
.github/workflows/mypy.yaml
+1 -1
@@ -43,8 +43,8 @@ jobs:
43 mypy vllm/worker --config-file pyproject.toml
44 mypy vllm/spec_decode --config-file pyproject.toml
45 mypy vllm/lora --config-file pyproject.toml
46 + mypy vllm/model_executor --config-file pyproject.toml
47
48 # TODO(sang): Fix nested dir
48 - mypy vllm/model_executor/*.py --config-file pyproject.toml
49 mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
50
format.sh
+1 -1
@@ -105,7 +105,7 @@ mypy vllm/transformers_utils --config-file pyproject.toml
105 mypy vllm/engine --config-file pyproject.toml
106 mypy vllm/worker --config-file pyproject.toml
107 mypy vllm/spec_decode --config-file pyproject.toml
108 -mypy vllm/model_executor/*.py --config-file pyproject.toml
108 +mypy vllm/model_executor --config-file pyproject.toml
109 mypy vllm/lora --config-file pyproject.toml
110
111
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+1
@@ -61,6 +61,7 @@ def _normalize_json_schema_object(schema: Union[str, dict, BaseModel]) -> dict:
61 return schema
62 if isinstance(schema, BaseModel):
63 return schema.model_json_schema()
64 + raise AssertionError(f"Unsupported schema type {schema}")
65
66
67 @lru_cache
vllm/model_executor/layers/linear.py
+11 -1
@@ -128,7 +128,8 @@ class LinearBase(torch.nn.Module):
128 params_dtype = torch.get_default_dtype()
129 self.params_dtype = params_dtype
130 if quant_config is None:
131 - self.quant_method = UnquantizedLinearMethod()
131 + self.quant_method: Optional[
132 + QuantizeMethodBase] = UnquantizedLinearMethod()
133 else:
134 self.quant_method = quant_config.get_quant_method(self)
135
@@ -160,6 +161,8 @@ class ReplicatedLinear(LinearBase):
161 super().__init__(input_size, output_size, skip_bias_add, params_dtype,
162 quant_config)
163
164 + # All the linear layer supports quant method.
165 + assert self.quant_method is not None
166 self.quant_method.create_weights(self, self.input_size,
167 [self.output_size], self.input_size,
168 self.output_size, self.params_dtype)
@@ -173,6 +176,7 @@ class ReplicatedLinear(LinearBase):
176
177 def forward(self, x: torch.Tensor) -> torch.Tensor:
178 bias = self.bias if not self.skip_bias_add else None
179 + assert self.quant_method is not None
180 output = self.quant_method.apply(self, x, bias)
181 output_bias = self.bias if self.skip_bias_add else None
182 return output, output_bias
@@ -221,6 +225,8 @@ class ColumnParallelLinear(LinearBase):
225 self.output_size_per_partition = divide(output_size, tp_size)
226 if output_sizes is None:
227 output_sizes = [output_size]
228 + # All the linear layer supports quant method.
229 + assert self.quant_method is not None
230 self.quant_method.create_weights(self,
231 self.input_size,
232 [x // tp_size for x in output_sizes],
@@ -255,6 +261,7 @@ class ColumnParallelLinear(LinearBase):
261 bias = self.bias if not self.skip_bias_add else None
262
263 # Matrix multiply.
264 + assert self.quant_method is not None
265 output_parallel = self.quant_method.apply(self, input_, bias)
266 if self.gather_output:
267 # All-gather across the partitions.
@@ -579,6 +586,8 @@ class RowParallelLinear(LinearBase):
586 # Divide the weight matrix along the last dimension.
587 self.tp_size = get_tensor_model_parallel_world_size()
588 self.input_size_per_partition = divide(input_size, self.tp_size)
589 + # All the linear layer supports quant method.
590 + assert self.quant_method is not None
591 self.quant_method.create_weights(self,
592 self.input_size_per_partition,
593 [self.output_size],
@@ -624,6 +633,7 @@ class RowParallelLinear(LinearBase):
633 input_parallel = splitted_input[tp_rank].contiguous()
634
635 # Matrix multiply.
636 + assert self.quant_method is not None
637 output_parallel = self.quant_method.apply(self, input_parallel)
638 if self.reduce_results and self.tp_size > 1:
639 output_ = tensor_model_parallel_all_reduce(output_parallel)
vllm/model_executor/layers/quantization/__init__.py
+2 -2
@@ -1,4 +1,4 @@
1 -from typing import Type
1 +from typing import Dict, Type
2
3 from vllm.model_executor.layers.quantization.aqlm import AQLMConfig
4 from vllm.model_executor.layers.quantization.awq import AWQConfig
@@ -9,7 +9,7 @@ from vllm.model_executor.layers.quantization.gptq import GPTQConfig
9 from vllm.model_executor.layers.quantization.marlin import MarlinConfig
10 from vllm.model_executor.layers.quantization.squeezellm import SqueezeLLMConfig
11
12 -QUANTIZATION_METHODS = {
12 +QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = {
13 "aqlm": AQLMConfig,
14 "awq": AWQConfig,
15 "fp8": Fp8Config,
vllm/model_executor/layers/quantization/base_config.py
+11 -3
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Any, Dict, List
2 +from typing import Any, Dict, List, Optional
3
4 import torch
5 from torch import nn
@@ -76,8 +76,16 @@ class QuantizationConfig(ABC):
76 "quantization config.")
77
78 @abstractmethod
79 - def get_quant_method(self, layer: torch.nn.Module) -> QuantizeMethodBase:
80 - """Get the quantize method to use for the quantized layer."""
79 + def get_quant_method(
80 + self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
81 + """Get the quantize method to use for the quantized layer.
82 +
83 + Args:
84 + layer: The layer for the quant method.
85 + Returns:
86 + The quantize method. None if the given layer doesn't support quant
87 + method.
88 + """
89 raise NotImplementedError
90
91 @abstractmethod
vllm/model_executor/layers/quantization/squeezellm.py
+2 -3
@@ -52,11 +52,10 @@ class SqueezeLLMConfig(QuantizationConfig):
52 return cls(weight_bits)
53
54 def get_quant_method(
55 - self,
56 - layer: torch.nn.Module) -> Optional["SqueezeLLMLinearMethod"]:
55 + self, layer: torch.nn.Module) -> Optional[QuantizeMethodBase]:
56 if isinstance(layer, LinearBase):
57 return SqueezeLLMLinearMethod(self)
59 - return
58 + return None
59
60 def get_scaled_act_names(self) -> List[str]:
61 return []
vllm/model_executor/layers/rotary_embedding.py
+2 -2
@@ -431,8 +431,8 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
431 torch.full_like(positions, k)).long()
432 idx = (torch.add(positions, long_prompt_offset)
433 if long_prompt_offset is not None else positions)
434 - self.long_short_cos_sin_cache = self.long_short_cos_sin_cache.to(
435 - idx.device)
434 + self.long_short_cos_sin_cache: torch.Tensor = (
435 + self.long_short_cos_sin_cache.to(idx.device))
436 idx = torch.add(idx, offsets) if offsets is not None else idx
437 cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
438
vllm/model_executor/layers/sampler.py
+27 -20
@@ -13,6 +13,9 @@ from vllm.sampling_params import SamplingType
13 from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
14 SamplerOutput, SequenceGroupOutput, SequenceOutput)
15
16 +# (num_token_ids, num_parent_ids) per sequence group.
17 +SampleResultType = List[Tuple[List[int], List[int]]]
18 +
19
20 class Sampler(nn.Module):
21 """Samples the next tokens from the model's outputs.
@@ -155,7 +158,7 @@ def _apply_min_tokens_penalty(
158 have not been generated yet
159 """
160 # list of indices in logits that will be set to -inf
158 - logits_to_penalize = []
161 + logits_to_penalize: List[Tuple[int, int]] = []
162 logits_applied = 0
163 for seq_group in sampling_metadata.seq_groups:
164 seq_ids = seq_group.seq_ids
@@ -269,7 +272,7 @@ def _apply_min_p(
272 def _greedy_sample(
273 selected_seq_groups: List[SequenceGroupToSample],
274 samples: torch.Tensor,
272 -) -> List[Tuple[List[int], List[int]]]:
275 +) -> SampleResultType:
276 """Run greedy sampling on a given samples.
277
278 Args:
@@ -284,7 +287,7 @@ def _greedy_sample(
287 """
288 samples = samples.tolist()
289 sample_idx = 0
287 - results = []
290 + results: SampleResultType = []
291 for seq_group in selected_seq_groups:
292 if not seq_group.do_sample:
293 results.append(([], []))
@@ -304,7 +307,7 @@ def _greedy_sample(
307 def _random_sample(
308 selected_seq_groups: List[SequenceGroupToSample],
309 random_samples: torch.Tensor,
307 -) -> List[Tuple[List[int], List[int]]]:
310 +) -> SampleResultType:
311 """Run random sampling on a given samples.
312
313 Args:
@@ -320,7 +323,7 @@ def _random_sample(
323 # Find the maximum best_of value of the prompt phase requests.
324 random_samples = random_samples.cpu()
325 sample_idx = 0
323 - results = []
326 + results: SampleResultType = []
327 for seq_group in selected_seq_groups:
328 if not seq_group.do_sample:
329 results.append(([], []))
@@ -348,7 +351,7 @@ def _random_sample(
351 def _beam_search_sample(
352 selected_seq_groups: List[SequenceGroupToSample],
353 logprobs: torch.Tensor,
351 -) -> List[Tuple[List[int], List[int]]]:
354 +) -> SampleResultType:
355 """Run beam sampling on a given samples.
356
357 Args:
@@ -370,7 +373,7 @@ def _beam_search_sample(
373 # NOTE: Beam search is not vectorized, so its speed can be slower than
374 # other sampling methods.
375 sample_idx = 0
373 - results = []
376 + results: SampleResultType = []
377 for seq_group in selected_seq_groups:
378 if not seq_group.do_sample:
379 results.append(([], []))
@@ -391,16 +394,16 @@ def _beam_search_sample(
394 next_token_ids = next_token_ids.tolist()
395 else:
396 # Generation phase.
394 - cumulative_logprobs = [
397 + cumulative_logprobs: List[int] = [
398 seq_group.seq_data[seq_id].cumulative_logprob
399 for seq_id in seq_ids
400 ]
398 - cumulative_logprobs = torch.tensor(
401 + cumulative_logprobs_tensor = torch.tensor(
402 cumulative_logprobs,
403 dtype=torch.float,
404 device=seq_group_logprobs.device)
405 seq_group_logprobs = (seq_group_logprobs +
403 - cumulative_logprobs.unsqueeze(dim=1))
406 + cumulative_logprobs_tensor.unsqueeze(dim=1))
407 _, topk_ids = torch.topk(seq_group_logprobs.flatten(),
408 2 * beam_width)
409 topk_ids = topk_ids.tolist()
@@ -452,8 +455,10 @@ def _sample_with_torch(
455 sampling_metadata: SamplingMetadata,
456 include_gpu_probs_tensor: bool,
457 modify_greedy_probs: bool,
455 -) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
456 - categorized_seq_group_ids = {t: [] for t in SamplingType}
458 +) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
459 + categorized_seq_group_ids: Dict[SamplingType,
460 + List[int]] = {t: []
461 + for t in SamplingType}
462 categorized_sample_indices = sampling_metadata.categorized_sample_indices
463 for i, seq_group in enumerate(sampling_metadata.seq_groups):
464 sampling_params = seq_group.sampling_params
@@ -555,8 +560,10 @@ def _sample_with_triton_kernel(
560 logprobs: torch.Tensor,
561 sampling_metadata: SamplingMetadata,
562 sampling_tensors: SamplingTensors,
558 -) -> List[Tuple[List[int], List[int]]]:
559 - categorized_seq_group_ids = {t: [] for t in SamplingType}
563 +) -> SampleResultType:
564 + categorized_seq_group_ids: Dict[SamplingType,
565 + List[int]] = {t: []
566 + for t in SamplingType}
567 categorized_sample_indices = sampling_metadata.categorized_sample_indices
568 for i, seq_group in enumerate(sampling_metadata.seq_groups):
569 sampling_params = seq_group.sampling_params
@@ -632,7 +639,7 @@ def _sample(
639 probs: torch.Tensor, logprobs: torch.Tensor,
640 sampling_metadata: SamplingMetadata, sampling_tensors: SamplingTensors,
641 include_gpu_probs_tensor: bool, modify_greedy_probs: bool
635 -) -> Tuple[List[Tuple[List[int], List[int]]], Optional[torch.Tensor]]:
642 +) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
643 """
644 Args:
645 probs: (num_query_tokens_in_batch, num_vocab)
@@ -680,7 +687,7 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
687 def _get_logprobs(
688 logprobs: torch.Tensor,
689 sampling_metadata: SamplingMetadata,
683 - sample_results: List[Tuple[List[int], List[int]]],
690 + sample_results: SampleResultType,
691 ) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
692 """Return sample lobprobs and prompt logprobs.
693
@@ -751,8 +758,8 @@ def _get_logprobs(
758 assert len(next_token_ids) == len(query_indices)
759
760 if len(query_indices) == 0:
754 - empty_sampled_logprob = []
755 - empty_prompt_logprob = None
761 + empty_sampled_logprob: SampleLogprobs = []
762 + empty_prompt_logprob: Optional[PromptLogprobs] = None
763 return [empty_prompt_logprob], [empty_sampled_logprob]
764
765 query_indices_gpu = torch.tensor(query_indices, device=logprobs.device)
@@ -965,7 +972,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
972
973
974 def _build_sampler_output(
968 - sample_results: List[Tuple[List[int], List[int]]],
975 + sample_results: SampleResultType,
976 sampling_metadata: SamplingMetadata,
977 prompt_logprobs: List[Optional[PromptLogprobs]],
978 sample_logprobs: List[SampleLogprobs],
@@ -1009,7 +1016,7 @@ def _build_sampler_output(
1016 )
1017
1018
1012 -def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[str]:
1019 +def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
1020 """Get a list of next prompt tokens to compute logprob from a
1021 given sequence group.
1022
vllm/model_executor/model_loader/tensorizer.py
+3 -1
@@ -64,7 +64,7 @@ class TensorizerConfig:
64 "s3_secret_access_key": self.s3_secret_access_key,
65 "s3_endpoint": self.s3_endpoint,
66 }
67 - return TensorizerArgs(**tensorizer_args)
67 + return TensorizerArgs(**tensorizer_args) # type: ignore
68
69 def verify_with_parallel_config(
70 self,
@@ -270,8 +270,10 @@ class TensorizerAgent:
270 self.model = self._init_model()
271
272 def _init_model(self):
273 + assert self.tensorizer_config.hf_config is not None
274 model_args = self.tensorizer_config.hf_config
275 model_args.torch_dtype = self.tensorizer_config.dtype
276 + assert self.tensorizer_config.model_class is not None
277 with no_init_or_tensor():
278 return self.tensorizer_config.model_class(
279 config=model_args,