@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfbe60dc62
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Simplify code and fix type annotations in `conftest.py` (#5118)

Cyrus Leung committed Jun 3, 2024 at 07:05 UTC dfbe60dc62409f03aa9eebc70ab2582ae64f0e1f
1 file changed +42 -50
tests/conftest.py
+42 -50
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple
5
6 import pytest
7 import torch
8 +import torch.nn.functional as F
9 from PIL import Image
10 from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
11 LlavaConfig, LlavaForConditionalGeneration)
@@ -12,9 +13,9 @@ from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
13 from vllm import LLM, SamplingParams
14 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
15 from vllm.distributed import destroy_model_parallel
15 -from vllm.inputs import PromptInputs
16 +from vllm.inputs import TextPrompt
17 from vllm.logger import init_logger
17 -from vllm.sequence import MultiModalData
18 +from vllm.sequence import MultiModalData, SampleLogprobs
19
20 logger = init_logger(__name__)
21
@@ -188,10 +189,11 @@ class HfRunner:
189 prompts: List[str],
190 images: Optional[List[Image.Image]] = None,
191 **kwargs,
191 - ) -> List[Tuple[List[int], str]]:
192 - outputs: List[Tuple[List[int], str]] = []
192 + ) -> List[Tuple[List[List[int]], List[str]]]:
193 if images:
194 assert len(prompts) == len(images)
195 +
196 + outputs: List[Tuple[List[List[int]], List[str]]] = []
197 for i, prompt in enumerate(prompts):
198 processor_kwargs: Dict[str, Any] = {
199 "text": prompt,
@@ -201,17 +203,13 @@ class HfRunner:
203 processor_kwargs["images"] = images[i]
204
205 inputs = self.processor(**processor_kwargs)
204 - inputs = {
205 - key: value.cuda() if value is not None else None
206 - for key, value in inputs.items()
207 - }
206
207 output_ids = self.model.generate(
210 - **inputs,
208 + **inputs.to("cuda"),
209 use_cache=True,
210 **kwargs,
211 )
214 - output_str = self.tokenizer.batch_decode(
212 + output_str = self.processor.batch_decode(
213 output_ids,
214 skip_special_tokens=True,
215 clean_up_tokenization_spaces=False,
@@ -224,23 +222,22 @@ class HfRunner:
222 self,
223 prompts: List[str],
224 max_tokens: int,
227 - images: Optional["torch.Tensor"] = None,
225 + images: Optional[List[Image.Image]] = None,
226 ) -> List[Tuple[List[int], str]]:
227 outputs = self.generate(prompts,
228 do_sample=False,
229 max_new_tokens=max_tokens,
230 images=images)
233 - for i in range(len(outputs)):
234 - output_ids, output_str = outputs[i]
235 - outputs[i] = (output_ids[0], output_str[0])
236 - return outputs
231 +
232 + return [(output_ids[0], output_str[0])
233 + for output_ids, output_str in outputs]
234
235 def generate_beam_search(
236 self,
237 prompts: List[str],
238 beam_width: int,
239 max_tokens: int,
243 - ) -> List[Tuple[List[int], str]]:
240 + ) -> List[Tuple[List[List[int]], List[str]]]:
241 outputs = self.generate(prompts,
242 do_sample=False,
243 max_new_tokens=max_tokens,
@@ -282,9 +279,7 @@ class HfRunner:
279 if self.model.get_output_embeddings().bias is not None:
280 logits += self.model.get_output_embeddings(
281 ).bias.unsqueeze(0)
285 - logprobs = torch.nn.functional.log_softmax(logits,
286 - dim=-1,
287 - dtype=torch.float32)
282 + logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
283 seq_logprobs.append(logprobs)
284 all_logprobs.append(seq_logprobs)
285 return all_logprobs
@@ -294,10 +289,10 @@ class HfRunner:
289 prompts: List[str],
290 max_tokens: int,
291 num_logprobs: int,
297 - ) -> List[Tuple[List[int], str]]:
298 - all_logprobs = []
299 - all_output_ids = []
300 - all_output_strs = []
292 + ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
293 + all_logprobs: List[List[Dict[int, float]]] = []
294 + all_output_ids: List[List[int]] = []
295 + all_output_strs: List[str] = []
296
297 for prompt in prompts:
298 input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
@@ -310,7 +305,7 @@ class HfRunner:
305 return_dict_in_generate=True,
306 )
307
313 - seq_logprobs = []
308 + seq_logprobs: List[torch.Tensor] = []
309 for _, hidden_states in enumerate(output.hidden_states):
310 last_hidden_states = hidden_states[-1][0]
311 logits = torch.matmul(
@@ -321,13 +316,11 @@ class HfRunner:
316 None) is not None:
317 logits += self.model.get_output_embeddings(
318 ).bias.unsqueeze(0)
324 - logprobs = torch.nn.functional.log_softmax(logits,
325 - dim=-1,
326 - dtype=torch.float32)
319 + logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
320 seq_logprobs.append(logprobs)
321
322 # convert to dict
330 - seq_logprobs_lst = []
323 + seq_logprobs_lst: List[Dict[int, float]] = []
324 for tok_idx, tok_logprobs in enumerate(seq_logprobs):
325 # drop prompt logprobs
326 if tok_idx == 0:
@@ -372,13 +365,13 @@ class VllmRunner:
365 tokenizer_name: Optional[str] = None,
366 # Use smaller max model length, otherwise bigger model cannot run due
367 # to kv cache size limit.
375 - max_model_len=1024,
368 + max_model_len: int = 1024,
369 dtype: str = "half",
370 disable_log_stats: bool = True,
371 tensor_parallel_size: int = 1,
372 block_size: int = 16,
373 enable_chunked_prefill: bool = False,
381 - swap_space=4,
374 + swap_space: int = 4,
375 **kwargs,
376 ) -> None:
377 self.model = LLM(
@@ -399,32 +392,31 @@ class VllmRunner:
392 self,
393 prompts: List[str],
394 sampling_params: SamplingParams,
402 - images: Optional["torch.Tensor"] = None,
403 - ) -> List[Tuple[List[int], str]]:
395 + images: Optional[torch.Tensor] = None,
396 + ) -> List[Tuple[List[List[int]], List[str]]]:
397 if images is not None:
405 - assert len(prompts) == images.shape[0]
398 + assert len(prompts) == len(images)
399
407 - prompt_inputs: List[PromptInputs] = []
400 + prompt_inputs: List[TextPrompt] = []
401 for i, prompt in enumerate(prompts):
409 - image = None if images is None else images[i:i + 1]
410 - mm_data = None if image is None else MultiModalData(
411 - type=MultiModalData.Type.IMAGE,
412 - data=image,
413 - )
402 + prompt = TextPrompt(prompt=prompt)
403 + if images is not None:
404 + prompt["multi_modal_data"] = MultiModalData(
405 + type=MultiModalData.Type.IMAGE,
406 + data=images[i:i + 1],
407 + )
408
415 - prompt_inputs.append({
416 - "prompt": prompt,
417 - "multi_modal_data": mm_data,
418 - })
409 + prompt_inputs.append(prompt)
410
411 req_outputs = self.model.generate(prompt_inputs,
412 sampling_params=sampling_params)
422 - outputs = []
413 +
414 + outputs: List[Tuple[List[List[int]], List[str]]] = []
415 for req_output in req_outputs:
416 prompt_str = req_output.prompt
417 prompt_ids = req_output.prompt_token_ids
426 - req_sample_output_ids = []
427 - req_sample_output_strs = []
418 + req_sample_output_ids: List[List[int]] = []
419 + req_sample_output_strs: List[str] = []
420 for sample in req_output.outputs:
421 output_str = sample.text
422 output_ids = sample.token_ids
@@ -437,12 +429,12 @@ class VllmRunner:
429 self,
430 prompts: List[str],
431 sampling_params: SamplingParams,
440 - ) -> List[Tuple[List[int], str]]:
432 + ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
433 assert sampling_params.logprobs is not None
434
435 req_outputs = self.model.generate(prompts,
436 sampling_params=sampling_params)
445 - outputs = []
437 + outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
438 for req_output in req_outputs:
439 for sample in req_output.outputs:
440 output_str = sample.text
@@ -467,7 +459,7 @@ class VllmRunner:
459 prompts: List[str],
460 max_tokens: int,
461 num_logprobs: int,
470 - ) -> List[Tuple[List[int], str]]:
462 + ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
463 greedy_logprobs_params = SamplingParams(temperature=0.0,
464 max_tokens=max_tokens,
465 logprobs=num_logprobs)
@@ -481,7 +473,7 @@ class VllmRunner:
473 prompts: List[str],
474 beam_width: int,
475 max_tokens: int,
484 - ) -> List[Tuple[List[int], str]]:
476 + ) -> List[Tuple[List[List[int]], List[str]]]:
477 beam_search_params = SamplingParams(n=beam_width,
478 use_beam_search=True,
479 temperature=0.0,