@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 533d2a1f39
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Typing] Mypy typing part 2 (#4043)

Co-authored-by: SangBin Cho <sangcho@sangcho-LT93GQWG9C.local>

SangBin Cho committed Apr 18, 2024 at 09:28 UTC 533d2a1f3962c45ddebbdeb0ff1cb7cd54d5e771
20 files changed +180 -126
.github/workflows/mypy.yaml
+4 -4
@@ -41,10 +41,10 @@ jobs:
41 mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
42 mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
43
44 + mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
45 + mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
46 + mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
47 + mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
48 # TODO(sang): Follow up
45 - # mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
46 - # mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
47 - # mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
48 - # mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
49 # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
50
format.sh
+4 -4
@@ -104,10 +104,10 @@ mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
104 mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
105
106 # TODO(sang): Follow up
107 -# mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108 -# mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109 -# mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
110 -# mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
107 +mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108 +mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109 +mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
110 +mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
111 # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
112
113
vllm/engine/async_llm_engine.py
+25 -19
@@ -2,8 +2,8 @@ import asyncio
2 import os
3 import time
4 from functools import partial
5 -from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
6 - Set, Tuple, Type, Union)
5 +from typing import (Any, AsyncIterator, Callable, Dict, Iterable, List,
6 + Optional, Set, Tuple, Type, Union)
7
8 from transformers import PreTrainedTokenizer
9
@@ -52,7 +52,7 @@ class AsyncStream:
52
53 def __init__(self, request_id: str) -> None:
54 self.request_id = request_id
55 - self._queue = asyncio.Queue()
55 + self._queue: asyncio.Queue = asyncio.Queue()
56 self._finished = False
57
58 def put(self, item: Union[RequestOutput, Exception]) -> None:
@@ -312,15 +312,17 @@ class AsyncLLMEngine:
312 self.max_log_len = max_log_len
313 self.engine = self._init_engine(*args, **kwargs)
314
315 - self.background_loop = None
315 + self.background_loop: Optional[asyncio.Future] = None
316 # We need to keep a reference to unshielded
317 # task as well to prevent it from being garbage
318 # collected
319 - self._background_loop_unshielded = None
319 + self._background_loop_unshielded: Optional[asyncio.Task[Any]] = None
320 self.start_engine_loop = start_engine_loop
321 - self._request_tracker: Optional[RequestTracker] = None
321 self._errored_with: Optional[BaseException] = None
322
323 + # Lazy initialized fields
324 + self._request_tracker: RequestTracker
325 +
326 @classmethod
327 def from_engine_args(
328 cls,
@@ -361,11 +363,13 @@ class AsyncLLMEngine:
363 @property
364 def is_running(self) -> bool:
365 return (self.background_loop is not None
366 + and self._background_loop_unshielded is not None
367 and not self._background_loop_unshielded.done())
368
369 @property
370 def is_stopped(self) -> bool:
368 - return self.errored or (self.background_loop is not None
371 + return self.errored or (self.background_loop is not None and
372 + self._background_loop_unshielded is not None
373 and self._background_loop_unshielded.done())
374
375 @property
@@ -381,7 +385,7 @@ class AsyncLLMEngine:
385
386 async def get_tokenizer(self) -> "PreTrainedTokenizer":
387 if self.engine_use_ray:
384 - return await self.engine.get_tokenizer.remote()
388 + return await self.engine.get_tokenizer.remote() # type: ignore
389 else:
390 return self.engine.get_tokenizer()
391
@@ -434,7 +438,8 @@ class AsyncLLMEngine:
438 # TODO: Maybe add add_request_batch to reduce Ray overhead
439 try:
440 if self.engine_use_ray:
437 - await self.engine.add_request.remote(**new_request)
441 + await self.engine.add_request.remote( # type: ignore
442 + **new_request)
443 else:
444 await self.engine.add_request_async(**new_request)
445 except ValueError as e:
@@ -449,7 +454,7 @@ class AsyncLLMEngine:
454 await self._engine_abort(finished_requests)
455
456 if self.engine_use_ray:
452 - request_outputs = await self.engine.step.remote()
457 + request_outputs = await self.engine.step.remote() # type: ignore
458 else:
459 request_outputs = await self.engine.step_async()
460
@@ -462,7 +467,7 @@ class AsyncLLMEngine:
467
468 async def _engine_abort(self, request_ids: Iterable[str]):
469 if self.engine_use_ray:
465 - await self.engine.abort_request.remote(request_ids)
470 + await self.engine.abort_request.remote(request_ids) # type: ignore
471 else:
472 self.engine.abort_request(request_ids)
473
@@ -525,11 +530,12 @@ class AsyncLLMEngine:
530 arrival_time = time.time()
531
532 if self.engine_use_ray:
528 - prompt_token_ids = await self.engine.encode_request_async.remote(
529 - request_id=request_id,
530 - prompt=prompt,
531 - prompt_token_ids=prompt_token_ids,
532 - lora_request=lora_request)
533 + prompt_token_ids = await (
534 + self.engine.encode_request_async.remote( # type: ignore
535 + request_id=request_id,
536 + prompt=prompt,
537 + prompt_token_ids=prompt_token_ids,
538 + lora_request=lora_request))
539 else:
540 prompt_token_ids = await self.engine.encode_request_async(
541 request_id=request_id,
@@ -676,13 +682,13 @@ class AsyncLLMEngine:
682 async def get_model_config(self) -> ModelConfig:
683 """Get the model configuration of the vLLM engine."""
684 if self.engine_use_ray:
679 - return await self.engine.get_model_config.remote()
685 + return await self.engine.get_model_config.remote() # type: ignore
686 else:
687 return self.engine.get_model_config()
688
689 async def do_log_stats(self) -> None:
690 if self.engine_use_ray:
685 - await self.engine.do_log_stats.remote()
691 + await self.engine.do_log_stats.remote() # type: ignore
692 else:
693 self.engine.do_log_stats()
694
@@ -695,7 +701,7 @@ class AsyncLLMEngine:
701
702 if self.engine_use_ray:
703 try:
698 - await self.engine.check_health.remote()
704 + await self.engine.check_health.remote() # type: ignore
705 except ray.exceptions.RayActorError as e:
706 raise RuntimeError("Engine is dead.") from e
707 else:
vllm/lora/worker_manager.py
+2 -2
@@ -107,12 +107,12 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
107 self._lora_manager: LoRAModelManager = lora_manager
108 return lora_manager.model
109
110 - def set_active_loras(self, lora_requests: List[LoRARequest],
110 + def set_active_loras(self, lora_requests: Set[LoRARequest],
111 lora_mapping: LoRAMapping) -> None:
112 self._apply_loras(lora_requests)
113 self._lora_manager.set_lora_mapping(lora_mapping)
114
115 - def _apply_loras(self, lora_requests: List[LoRARequest]) -> None:
115 + def _apply_loras(self, lora_requests: Set[LoRARequest]) -> None:
116 loras_that_exist = self.list_loras()
117 loras_map = {
118 lora_request.lora_int_id: lora_request
vllm/model_executor/guided_decoding/outlines_decoding.py
+2 -2
@@ -55,7 +55,7 @@ global_thread_pool = None # used for generating logits processor fsm
55
56 async def get_outlines_guided_decoding_logits_processor(
57 request: Union[CompletionRequest, ChatCompletionRequest],
58 - tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor]:
58 + tokenizer) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, None]:
59 """
60 Given an OpenAI-compatible request, check for guided decoding parameters
61 and get the necessary logits processor for the given guide.
@@ -84,7 +84,7 @@ async def get_outlines_guided_decoding_logits_processor(
84
85 def _get_guide_and_mode(
86 request: Union[CompletionRequest, ChatCompletionRequest]
87 -) -> Tuple[str, GuidedDecodingMode]:
87 +) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
88
89 if request.guided_json:
90 json = request.guided_json
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+5 -1
@@ -21,7 +21,7 @@ from functools import lru_cache
21 from typing import Callable, DefaultDict, Dict, List, Optional, Union
22
23 import torch
24 -from outlines.fsm.fsm import CFGFSM, RegexFSM
24 +from outlines.fsm.fsm import CFGFSM, FSM, RegexFSM
25 from outlines.fsm.json_schema import build_regex_from_schema
26 from pydantic import BaseModel
27 from transformers import PreTrainedTokenizerBase
@@ -29,6 +29,10 @@ from transformers import PreTrainedTokenizerBase
29
30 class BaseLogitsProcessor:
31
32 + def __init__(self):
33 + # Child class should use initialize in their init.
34 + self.fsm: FSM
35 +
36 def init_state(self):
37 """Initialize the FSM states."""
38 self.fsm_state: DefaultDict[int, int] = defaultdict(int)
vllm/model_executor/model_loader/neuron.py
+9 -7
@@ -1,7 +1,7 @@
1 """Utilities for selecting and loading neuron models."""
2 import importlib
3 import os
4 -from typing import Optional, Type
4 +from typing import Dict, Optional, Tuple
5
6 import torch
7 import torch.nn as nn
@@ -27,7 +27,7 @@ TORCH_DTYPE_TO_NEURON_AMP = {
27 }
28
29 # Models supported by Neuron.
30 -_NEURON_SUPPORTED_MODELS = {
30 +_NEURON_SUPPORTED_MODELS: Dict[str, Tuple[str, str, str]] = {
31 "LlamaForCausalLM": ("transformers_neuronx.llama.model",
32 "LlamaForSampling", "LlamaForCausalLM"),
33 "MistralForCausalLM": ("transformers_neuronx.mistral.model",
@@ -43,11 +43,13 @@ class NeuronCasualLM(nn.Module):
43 ) -> None:
44 super().__init__()
45 self.config = config
46 - self.model = None
46 self.logits_processor = LogitsProcessor(config.vocab_size,
47 logits_as_input=True)
48 self.sampler = Sampler()
49
50 + # Lazy initialized
51 + self.model: nn.Module
52 +
53 def forward(
54 self,
55 input_ids: torch.Tensor,
@@ -74,17 +76,17 @@ class NeuronCasualLM(nn.Module):
76
77 def load_weights(self, model_name_or_path: str, **kwargs):
78 arch = _get_model_architecture(self.config)
77 - neuronx_module_path, neuronx_model_cls, hf_model_cls = (
79 + neuronx_module_path, neuronx_model_cls_name, hf_model_cls_name = (
80 _NEURON_SUPPORTED_MODELS[arch])
81 neuronx_module = importlib.import_module(neuronx_module_path)
80 - neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls)
82 + neuronx_model_cls = getattr(neuronx_module, neuronx_model_cls_name)
83
84 split_model_dir = f"{model_name_or_path}-split"
85 if os.path.isdir(os.path.join(model_name_or_path,
86 "pytorch_model.bin")):
87 split_model_dir = model_name_or_path
88 elif not os.path.exists(f"{model_name_or_path}-split"):
87 - hf_model_cls = getattr(transformers, hf_model_cls)
89 + hf_model_cls = getattr(transformers, hf_model_cls_name)
90 from transformers_neuronx.module import save_pretrained_split
91
92 hf_model = hf_model_cls.from_pretrained(model_name_or_path,
@@ -96,7 +98,7 @@ class NeuronCasualLM(nn.Module):
98 self.model.to_neuron()
99
100
99 -def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
101 +def _get_model_architecture(config: PretrainedConfig) -> str:
102 architectures = getattr(config, "architectures", [])
103 for arch in architectures:
104 if arch in _NEURON_SUPPORTED_MODELS:
vllm/model_executor/model_loader/tensorizer.py
+1
@@ -167,6 +167,7 @@ class TensorizerArgs:
167 decryption_params = DecryptionParams.from_key(key)
168 self.deserializer_params['encryption'] = decryption_params
169
170 + @staticmethod
171 def add_cli_args(
172 parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
173 """Tensorizer CLI arguments"""
vllm/model_executor/sampling_metadata.py
+4
@@ -113,6 +113,8 @@ class SamplingTensors:
113 get_num_triton_sampler_splits(vocab_size))
114
115 sample_indices_start_idx = 0
116 + assert sampling_metadata.seq_groups is not None
117 + assert sampling_metadata.seq_data is not None
118 for i, seq_group in enumerate(sampling_metadata.seq_groups):
119 seq_ids, sampling_params = seq_group
120 temperature = sampling_params.temperature
@@ -147,6 +149,7 @@ class SamplingTensors:
149 and sampling_params.prompt_logprobs is not None):
150 # For tokens in the prompt that we only need to get
151 # their logprobs
152 + assert sampling_metadata.prompt_lens is not None
153 prompt_len = sampling_metadata.prompt_lens[i]
154 temperatures += [temperature] * (prompt_len - 1)
155 top_ps += [top_p] * (prompt_len - 1)
@@ -172,6 +175,7 @@ class SamplingTensors:
175 is_prompt = i < sampling_metadata.num_prompts
176 if is_prompt:
177 prompt_best_of.append(sampling_params.best_of)
178 + assert sampling_metadata.prompt_lens is not None
179 prompt_len = sampling_metadata.prompt_lens[i]
180
181 if sampling_params.prompt_logprobs is not None:
vllm/spec_decode/batch_expansion.py
+3 -3
@@ -106,7 +106,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
106 def _expand_batch(
107 self,
108 seq_group_metadata_list: List[SequenceGroupMetadata],
109 - proposal_token_ids_list: List[TokenId],
109 + proposal_token_ids_list: List[List[TokenId]],
110 proposal_lens_list: List[int],
111 ) -> Tuple[List[int], List[int], List[SequenceGroupMetadata], int]:
112 """Given the input sequences and potentially multiple corresponding
@@ -218,7 +218,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
218 def _create_target_seq_group_metadata(
219 self,
220 input_seq_group_metadata: SequenceGroupMetadata,
221 - proposal_token_ids: List[TokenId], # shape: [batch_size, k]
221 + proposal_token_ids: List[List[TokenId]], # shape: [batch_size, k]
222 batch_index: int,
223 target_seq_ids_iter: Iterator[TargetSeqId],
224 ) -> List[SequenceGroupMetadata]:
@@ -360,7 +360,7 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
360 [0, 1, 2]
361 [0, 1, 2, 3]
362 """
363 - empty_token_ids = []
363 + empty_token_ids: List[TokenId] = []
364
365 token_ids_to_score = [empty_token_ids]
366 token_ids_to_score.extend([
vllm/spec_decode/interfaces.py
+2 -2
@@ -1,6 +1,6 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 -from typing import Dict, List, Optional, Tuple
3 +from typing import Dict, List, Optional
4
5 import torch
6
@@ -73,5 +73,5 @@ class SpeculativeScorer(ABC):
73 blocks_to_copy: Optional[Dict[int, List[int]]],
74 k: int,
75 proposals: SpeculativeProposals,
76 - ) -> Tuple[torch.Tensor, torch.Tensor]:
76 + ) -> SpeculativeScores:
77 raise NotImplementedError
vllm/spec_decode/metrics.py
+1
@@ -112,6 +112,7 @@ class AsyncMetricsCollector:
112
113 Returns a CUDA event recording when the copy is complete.
114 """
115 + assert self._copy_stream is not None
116 self._copy_stream.wait_stream(torch.cuda.current_stream())
117
118 with torch.cuda.stream(self._copy_stream):
vllm/spec_decode/multi_step_worker.py
+11 -10
@@ -26,7 +26,8 @@ class MultiStepWorker(Worker):
26 def __init__(self, *args, **kwargs):
27 super().__init__(*args, **kwargs)
28
29 - self._proposer: Optional[DraftModelTop1Proposer] = None
29 + # Lazy initialization list.
30 + self._proposer: DraftModelTop1Proposer
31
32 def init_device(self):
33 super().init_device()
@@ -338,10 +339,10 @@ class DraftModelTop1Proposer(SpeculativeProposer):
339 self._vocab_size,
340 dtype=torch.float32,
341 device=self._device)
341 - proposal_lens = torch.zeros(len(proposal_lens),
342 - dtype=torch.long,
343 - device=self._device)
344 - return proposal_tokens, proposal_probs, proposal_lens
342 + proposal_lens_tensor = torch.zeros(len(proposal_lens),
343 + dtype=torch.long,
344 + device=self._device)
345 + return proposal_tokens, proposal_probs, proposal_lens_tensor
346
347 sampler_output = maybe_sampler_output
348
@@ -376,9 +377,9 @@ class DraftModelTop1Proposer(SpeculativeProposer):
377 proposal_tokens, proposal_probs = (entire_proposal_tokens,
378 entire_proposal_probs)
379
379 - proposal_lens = torch.zeros(batch_size,
380 - dtype=torch.long,
381 - device=self._device)
382 - proposal_lens[nonzero_proposal_len_indices] = max_proposal_len
380 + proposal_lens_tensor = torch.zeros(batch_size,
381 + dtype=torch.long,
382 + device=self._device)
383 + proposal_lens_tensor[nonzero_proposal_len_indices] = max_proposal_len
384
384 - return proposal_tokens, proposal_probs, proposal_lens
385 + return proposal_tokens, proposal_probs, proposal_lens_tensor
vllm/spec_decode/spec_decode_worker.py
+5 -1
@@ -89,7 +89,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
89 self.probs_dtype = self.rejection_sampler.probs_dtype
90 self.token_id_dtype = self.rejection_sampler.token_id_dtype
91
92 - self.scorer: SpeculativeScorer = None
92 + # Lazy initiazliation.
93 + self.scorer: SpeculativeScorer
94
95 def init_device(self) -> None:
96 """Initialize both scorer and proposer models.
@@ -233,6 +234,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
234
235 logger.info("get spec proposals")
236 # Generate proposals using draft worker.
237 + assert blocks_to_swap_in is not None
238 + assert blocks_to_swap_out is not None
239 + assert blocks_to_copy is not None
240 proposals = self.proposer_worker.get_spec_proposals(
241 seq_group_metadata_list, blocks_to_swap_in, blocks_to_swap_out,
242 blocks_to_copy, k)
vllm/worker/cpu_model_runner.py
+15 -10
@@ -1,6 +1,7 @@
1 from typing import Dict, List, Optional, Tuple
2
3 import torch
4 +from torch import nn
5
6 from vllm.attention import AttentionMetadata, get_attn_backend
7 from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -48,14 +49,15 @@ class CPUModelRunner:
49 if device_config is not None else DeviceConfig())
50 self.device = self.device_config.device
51
51 - self.model = None
52 - self.block_size = None # Set after initial profiling.
53 -
52 self.kv_cache_dtype = kv_cache_dtype
53
54 self.attn_backend = get_attn_backend(
55 self.model_config.dtype if model_config is not None else None)
56
57 + # Lazy initialization.
58 + self.model: nn.Module # Set after init_Model
59 + self.block_size: int # Set after initial profiling.
60 +
61 def load_model(self) -> None:
62 self.model = get_model(model_config=self.model_config,
63 load_config=self.load_config,
@@ -245,7 +247,11 @@ class CPUModelRunner:
247 selected_token_indices: List[int] = []
248 generators: List[torch.Generator] = []
249 selected_token_start_idx = 0
248 - categorized_sample_indices = {t: [] for t in SamplingType}
250 + categorized_sample_indices: Dict[SamplingType,
251 + List[Tuple[int, int]]] = {
252 + t: []
253 + for t in SamplingType
254 + }
255 categorized_sample_indices_start_idx = 0
256 categorized_sampled_token_indices_start_idx = 0
257
@@ -262,10 +268,9 @@ class CPUModelRunner:
268 categorized_sample_indices_start_idx += subquery_len - 1
269
270 categorized_sample_indices[
265 - sampling_params.sampling_type].append([
266 - categorized_sample_indices_start_idx,
267 - categorized_sampled_token_indices_start_idx
268 - ])
271 + sampling_params.sampling_type].append(
272 + (categorized_sample_indices_start_idx,
273 + categorized_sampled_token_indices_start_idx))
274 categorized_sample_indices_start_idx += 1
275 categorized_sampled_token_indices_start_idx += 1
276
@@ -328,7 +333,7 @@ class CPUModelRunner:
333
334 def prepare_input_tensors(
335 self,
331 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
336 + seq_group_metadata_list: List[SequenceGroupMetadata],
337 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata,
338 SamplingMetadata]:
339 if self.is_driver_worker:
@@ -381,7 +386,7 @@ class CPUModelRunner:
386 @torch.inference_mode()
387 def execute_model(
388 self,
384 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
389 + seq_group_metadata_list: List[SequenceGroupMetadata],
390 kv_caches: List[torch.Tensor],
391 ) -> Optional[SamplerOutput]:
392 (input_tokens, input_positions, attn_metadata, sampling_metadata
vllm/worker/cpu_worker.py
+6 -5
@@ -1,5 +1,5 @@
1 """A CPU worker class."""
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5 import torch.distributed
@@ -152,8 +152,8 @@ class CPUWorker(LoraNotSupportedWorkerBase):
152 is_driver_worker=is_driver_worker)
153 # Uninitialized cache engine. Will be initialized by
154 # initialize_cache.
155 - self.cache_engine = None
156 - self.cpu_cache = None
155 + self.cache_engine: CPUCacheEngine
156 + self.cpu_cache: List[torch.Tensor]
157
158 def init_device(self) -> None:
159 self.init_distributed_environment()
@@ -257,13 +257,13 @@ class CPUWorker(LoraNotSupportedWorkerBase):
257 ) -> List[SamplerOutput]:
258 if self.is_driver_worker:
259 assert seq_group_metadata_list is not None
260 - num_seq_groups = len(seq_group_metadata_list)
260 + num_seq_groups: int = len(seq_group_metadata_list)
261 assert blocks_to_swap_in is not None
262 assert blocks_to_swap_out is not None
263 assert blocks_to_copy is not None
264 assert len(blocks_to_swap_in) == 0
265 assert len(blocks_to_swap_out) == 0
266 - data = {
266 + data: Dict[str, Any] = {
267 "num_seq_groups": num_seq_groups,
268 "blocks_to_copy": blocks_to_copy,
269 }
@@ -273,6 +273,7 @@ class CPUWorker(LoraNotSupportedWorkerBase):
273 num_seq_groups = data["num_seq_groups"]
274 blocks_to_copy = data["blocks_to_copy"]
275
276 + assert blocks_to_copy is not None
277 self.cache_copy(blocks_to_copy)
278
279 # If there is no input, we don't need to execute the model.
vllm/worker/model_runner.py
+56 -40
@@ -128,23 +128,17 @@ class ModelRunner:
128 if device_config is not None else DeviceConfig())
129 self.device = self.device_config.device
130
131 - self.model = None
132 - self.block_size = None # Set after initial profiling.
133 - self.lora_manager = None
131 + # Set after load_model.
132 + self.lora_manager: LRUCacheWorkerLoRAManager = None
133
134 self.graph_runners: Dict[int, CUDAGraphRunner] = {}
136 - self.graph_memory_pool = None # Set during graph capture.
135 + self.graph_memory_pool: Optional[Tuple[
136 + int, int]] = None # Set during graph capture.
137
138 self.max_context_len_to_capture = (
139 self.model_config.max_context_len_to_capture
140 if self.model_config is not None else 0)
141 - # When using CUDA graph, the input block tables must be padded to
142 - # max_context_len_to_capture. However, creating the block table in
143 - # Python can be expensive. To optimize this, we cache the block table
144 - # in numpy and only copy the actual input content at every iteration.
145 - # The shape of the cached block table will be
146 - # (max batch size to capture, max context len to capture / block size).
147 - self.graph_block_tables = None # Set after initial profiling.
141 +
142 self.pin_memory = is_pin_memory_available()
143 self.kv_cache_dtype = kv_cache_dtype
144 self.vision_language_config = vision_language_config
@@ -152,6 +146,17 @@ class ModelRunner:
146 self.attn_backend = get_attn_backend(
147 self.model_config.dtype if model_config is not None else None)
148
149 + # Lazy initialization
150 + self.model: torch.nn.Module # Set after load_model
151 + self.block_size: int # Set after initial profiling.
152 + # When using CUDA graph, the input block tables must be padded to
153 + # max_context_len_to_capture. However, creating the block table in
154 + # Python can be expensive. To optimize this, we cache the block table
155 + # in numpy and only copy the actual input content at every iteration.
156 + # The shape of the cached block table will be
157 + # (max batch size to capture, max context len to capture / block size).
158 + self.graph_block_tables: torch.Tensor # Set after initial profiling.
159 +
160 def load_model(self) -> None:
161 with CudaMemoryProfiler() as m:
162 self.model = get_model(
@@ -489,16 +494,16 @@ class ModelRunner:
494 lora_index_mapping.append(0)
495 batch_size = graph_batch_size
496
492 - context_lens = torch.tensor(context_lens,
493 - dtype=torch.int,
494 - device=self.device)
497 + context_lens_tensor = torch.tensor(context_lens,
498 + dtype=torch.int,
499 + device=self.device)
500
501 if use_captured_graph:
502 # When using cuda-graph all these tensors should be
503 # padded.
499 - assert context_lens.shape[0] == len(input_tokens)
500 - assert context_lens.shape[0] == len(input_positions)
501 - assert context_lens.shape[0] == len(slot_mapping)
504 + assert context_lens_tensor.shape[0] == len(input_tokens)
505 + assert context_lens_tensor.shape[0] == len(input_positions)
506 + assert context_lens_tensor.shape[0] == len(slot_mapping)
507
508 # The shape of graph_block_tables is
509 # [max batch size, max context len // block size].
@@ -527,7 +532,7 @@ class ModelRunner:
532 max_prompt_len=None,
533 subquery_start_loc=None,
534 seq_start_loc=None,
530 - context_lens=context_lens,
535 + context_lens=context_lens_tensor,
536 block_tables=block_tables,
537 use_cuda_graph=use_captured_graph,
538 )
@@ -551,7 +556,11 @@ class ModelRunner:
556 selected_token_indices: List[int] = []
557 generators: List[torch.Generator] = []
558 selected_token_start_idx = 0
554 - categorized_sample_indices = {t: [] for t in SamplingType}
559 + categorized_sample_indices: Dict[SamplingType,
560 + List[Tuple[int, int]]] = {
561 + t: []
562 + for t in SamplingType
563 + }
564 categorized_sample_indices_start_idx = 0
565 categorized_sampled_token_indices_start_idx = 0
566
@@ -569,10 +578,9 @@ class ModelRunner:
578 categorized_sample_indices_start_idx += subquery_len - 1
579
580 categorized_sample_indices[
572 - sampling_params.sampling_type].append([
573 - categorized_sample_indices_start_idx,
574 - categorized_sampled_token_indices_start_idx
575 - ])
581 + sampling_params.sampling_type].append(
582 + (categorized_sample_indices_start_idx,
583 + categorized_sampled_token_indices_start_idx))
584 categorized_sample_indices_start_idx += 1
585 categorized_sampled_token_indices_start_idx += 1
586
@@ -596,15 +604,16 @@ class ModelRunner:
604
605 categorized_sample_indices[
606 sampling_params.sampling_type].extend(
599 - zip(
600 - range(
601 - categorized_sample_indices_start_idx,
602 - categorized_sample_indices_start_idx +
603 - num_seqs),
604 - range(
605 - categorized_sampled_token_indices_start_idx,
606 - categorized_sampled_token_indices_start_idx +
607 - num_seqs)))
607 + list(
608 + zip(
609 + range(
610 + categorized_sample_indices_start_idx,
611 + categorized_sample_indices_start_idx +
612 + num_seqs),
613 + range(
614 + categorized_sampled_token_indices_start_idx,
615 + categorized_sampled_token_indices_start_idx
616 + + num_seqs))))
617 categorized_sample_indices_start_idx += num_seqs
618 categorized_sampled_token_indices_start_idx += num_seqs
619
@@ -641,9 +650,9 @@ class ModelRunner:
650
651 def prepare_input_tensors(
652 self,
644 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
653 + seq_group_metadata_list: List[SequenceGroupMetadata],
654 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
646 - Set[int], LoRAMapping, torch.Tensor]:
655 + Set[LoRARequest], LoRAMapping, torch.Tensor]:
656 if self.is_driver_worker:
657 prefill_reqs = []
658 decode_reqs = []
@@ -741,6 +750,7 @@ class ModelRunner:
750 if prefill_attn_metadata is not None:
751 metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
752 else:
753 + assert decode_attn_metadata is not None
754 metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
755 broadcast_tensor_dict(metadata_dict, src=0)
756
@@ -809,7 +819,7 @@ class ModelRunner:
819 @torch.inference_mode()
820 def execute_model(
821 self,
812 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
822 + seq_group_metadata_list: List[SequenceGroupMetadata],
823 kv_caches: List[torch.Tensor],
824 ) -> Optional[SamplerOutput]:
825 (input_tokens, input_positions, attn_metadata, sampling_metadata,
@@ -923,7 +933,7 @@ class ModelRunner:
933 raise RuntimeError("LoRA is not enabled.")
934 return self.lora_manager.remove_all_loras()
935
926 - def set_active_loras(self, lora_requests: List[LoRARequest],
936 + def set_active_loras(self, lora_requests: Set[LoRARequest],
937 lora_mapping: LoRAMapping) -> None:
938 if not self.lora_manager:
939 raise RuntimeError("LoRA is not enabled.")
@@ -1065,10 +1075,16 @@ class CUDAGraphRunner:
1075
1076 def __init__(self, model: nn.Module):
1077 self.model = model
1068 - self.graph = None
1078 self.input_buffers: Dict[str, torch.Tensor] = {}
1079 self.output_buffers: Dict[str, torch.Tensor] = {}
1080
1081 + self._graph: Optional[torch.cuda.CUDAGraph] = None
1082 +
1083 + @property
1084 + def graph(self):
1085 + assert self._graph is not None
1086 + return self._graph
1087 +
1088 def capture(
1089 self,
1090 input_ids: torch.Tensor,
@@ -1078,7 +1094,7 @@ class CUDAGraphRunner:
1094 memory_pool,
1095 **kwargs,
1096 ) -> None:
1081 - assert self.graph is None
1097 + assert self._graph is None
1098 # Run the model once without capturing the graph.
1099 # This is to make sure that the captured graph does not include the
1100 # kernel launches for initial benchmarking (e.g., Triton autotune).
@@ -1095,8 +1111,8 @@ class CUDAGraphRunner:
1111 # Capture the graph.
1112 # NOTE(woosuk): Python 3.8 does not support multi-line with statements.
1113 # https://stackoverflow.com/questions/31039022/python-multi-line-with-statement
1098 - self.graph = torch.cuda.CUDAGraph()
1099 - with torch.cuda.graph(self.graph, pool=memory_pool): # noqa: SIM117
1114 + self._graph = torch.cuda.CUDAGraph()
1115 + with torch.cuda.graph(self._graph, pool=memory_pool): # noqa: SIM117
1116 with _maybe_pynccl():
1117 hidden_states = self.model(
1118 input_ids,
vllm/worker/neuron_model_runner.py
+14 -8
@@ -1,6 +1,7 @@
1 from typing import Dict, List, Optional, Tuple
2
3 import torch
4 +from torch import nn
5
6 from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
7 SchedulerConfig)
@@ -34,9 +35,11 @@ class NeuronModelRunner:
35 self.device_config = (device_config
36 if device_config is not None else DeviceConfig())
37 self.device = self.device_config.device
37 - self.model = None
38 self.pin_memory = is_pin_memory_available()
39
40 + # Lazy initialization.
41 + self.model: nn.Module # initialize after load_model.
42 +
43 def load_model(self) -> None:
44 self.model = get_neuron_model(self.model_config,
45 parallel_config=self.parallel_config,
@@ -147,7 +150,11 @@ class NeuronModelRunner:
150 selected_token_indices: List[int] = []
151 generators: List[torch.Generator] = []
152 selected_token_start_idx = 0
150 - categorized_sample_indices = {t: [] for t in SamplingType}
153 + categorized_sample_indices: Dict[SamplingType,
154 + List[Tuple[int, int]]] = {
155 + t: []
156 + for t in SamplingType
157 + }
158 categorized_sample_indices_start_idx = 0
159 categorized_sampled_token_indices_start_idx = 0
160
@@ -165,10 +172,9 @@ class NeuronModelRunner:
172 categorized_sample_indices_start_idx += prompt_len - 1
173
174 categorized_sample_indices[
168 - sampling_params.sampling_type].append([
169 - categorized_sample_indices_start_idx,
170 - categorized_sampled_token_indices_start_idx
171 - ])
175 + sampling_params.sampling_type].append(
176 + (categorized_sample_indices_start_idx,
177 + categorized_sampled_token_indices_start_idx))
178 categorized_sample_indices_start_idx += 1
179 categorized_sampled_token_indices_start_idx += 1
180
@@ -237,7 +243,7 @@ class NeuronModelRunner:
243
244 def prepare_input_tensors(
245 self,
240 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
246 + seq_group_metadata_list: List[SequenceGroupMetadata],
247 ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, SamplingMetadata]:
248 # NOTE: We assume that all sequences in the group are all prompts or
249 # all decodes.
@@ -259,7 +265,7 @@ class NeuronModelRunner:
265 @torch.inference_mode()
266 def execute_model(
267 self,
262 - seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
268 + seq_group_metadata_list: List[SequenceGroupMetadata],
269 ) -> Optional[SamplerOutput]:
270 (input_tokens, input_positions, input_block_ids, sampling_metadata
271 ) = self.prepare_input_tensors(seq_group_metadata_list)
vllm/worker/worker.py
+7 -4
@@ -1,7 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import Dict, List, Optional, Set, Tuple
4 +from typing import Any, Dict, List, Optional, Set, Tuple
5
6 import torch
7 import torch.distributed
@@ -82,8 +82,8 @@ class Worker(WorkerBase):
82 )
83 # Uninitialized cache engine. Will be initialized by
84 # initialize_cache.
85 - self.cache_engine = None
86 - self.gpu_cache = None
85 + self.cache_engine: CacheEngine
86 + self.gpu_cache: List[torch.Tensor]
87
88 def init_device(self) -> None:
89 if self.device_config.device.type == "cuda":
@@ -223,7 +223,7 @@ class Worker(WorkerBase):
223 assert blocks_to_swap_in is not None
224 assert blocks_to_swap_out is not None
225 assert blocks_to_copy is not None
226 - data = {
226 + data: Dict[str, Any] = {
227 "num_seq_groups": num_seq_groups,
228 "blocks_to_swap_in": blocks_to_swap_in,
229 "blocks_to_swap_out": blocks_to_swap_out,
@@ -237,6 +237,9 @@ class Worker(WorkerBase):
237 blocks_to_swap_out = data["blocks_to_swap_out"]
238 blocks_to_copy = data["blocks_to_copy"]
239
240 + assert blocks_to_swap_in is not None
241 + assert blocks_to_swap_out is not None
242 + assert blocks_to_copy is not None
243 self.cache_swap(blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy)
244
245 # If there is no input, we don't need to execute the model.
vllm/worker/worker_base.py
+4 -4
@@ -1,7 +1,7 @@
1 import importlib
2 import os
3 from abc import ABC, abstractmethod
4 -from typing import Dict, List, Tuple
4 +from typing import Dict, List, Set, Tuple
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
@@ -56,7 +56,7 @@ class WorkerBase(ABC):
56 raise NotImplementedError
57
58 @abstractmethod
59 - def get_cache_block_size_bytes() -> int:
59 + def get_cache_block_size_bytes(self) -> int:
60 """Return the size of a single cache block, in bytes. Used in
61 speculative decoding.
62 """
@@ -71,7 +71,7 @@ class WorkerBase(ABC):
71 raise NotImplementedError
72
73 @abstractmethod
74 - def list_loras(self) -> List[int]:
74 + def list_loras(self) -> Set[int]:
75 raise NotImplementedError
76
77
@@ -86,7 +86,7 @@ class LoraNotSupportedWorkerBase(WorkerBase):
86 def remove_lora(self, lora_id: int) -> bool:
87 raise ValueError(f"{type(self)} does not support LoRA")
88
89 - def list_loras(self) -> List[int]:
89 + def list_loras(self) -> Set[int]:
90 raise ValueError(f"{type(self)} does not support LoRA")
91
92