A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Simplify code and fix type annotations in `conftest.py` (#5118)
Cyrus Leung committed
Jun 3, 2024 at 07:05 UTC
dfbe60dc62409f03aa9eebc70ab2582ae64f0e1f
1 file changed
+42
-50
tests/conftest.py
+42
-50
@@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple
5
6
import pytest
7
import torch
8
+import torch.nn.functional as F
9
from PIL import Image
10
from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
11
LlavaConfig, LlavaForConditionalGeneration)
@@ -12,9 +13,9 @@ from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
13
from vllm import LLM, SamplingParams
14
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
15
from vllm.distributed import destroy_model_parallel
15
-from vllm.inputs import PromptInputs
16
+from vllm.inputs import TextPrompt
17
from vllm.logger import init_logger
17
-from vllm.sequence import MultiModalData
18
+from vllm.sequence import MultiModalData, SampleLogprobs
19
20
logger = init_logger(__name__)
21
@@ -188,10 +189,11 @@ class HfRunner:
189
prompts: List[str],
190
images: Optional[List[Image.Image]] = None,
191
**kwargs,
191
- ) -> List[Tuple[List[int], str]]:
192
- outputs: List[Tuple[List[int], str]] = []
192
+ ) -> List[Tuple[List[List[int]], List[str]]]:
193
if images:
194
assert len(prompts) == len(images)
195
+
196
+ outputs: List[Tuple[List[List[int]], List[str]]] = []
197
for i, prompt in enumerate(prompts):
198
processor_kwargs: Dict[str, Any] = {
199
"text": prompt,
@@ -201,17 +203,13 @@ class HfRunner:
203
processor_kwargs["images"] = images[i]
204
205
inputs = self.processor(**processor_kwargs)
204
- inputs = {
205
- key: value.cuda() if value is not None else None
206
- for key, value in inputs.items()
207
- }
206
207
output_ids = self.model.generate(
210
- **inputs,
208
+ **inputs.to("cuda"),
209
use_cache=True,
210
**kwargs,
211
)
214
- output_str = self.tokenizer.batch_decode(
212
+ output_str = self.processor.batch_decode(
213
output_ids,
214
skip_special_tokens=True,
215
clean_up_tokenization_spaces=False,
@@ -224,23 +222,22 @@ class HfRunner:
222
self,
223
prompts: List[str],
224
max_tokens: int,
227
- images: Optional["torch.Tensor"] = None,
225
+ images: Optional[List[Image.Image]] = None,
226
) -> List[Tuple[List[int], str]]:
227
outputs = self.generate(prompts,
228
do_sample=False,
229
max_new_tokens=max_tokens,
230
images=images)
233
- for i in range(len(outputs)):
234
- output_ids, output_str = outputs[i]
235
- outputs[i] = (output_ids[0], output_str[0])
236
- return outputs
231
+
232
+ return [(output_ids[0], output_str[0])
233
+ for output_ids, output_str in outputs]
234
235
def generate_beam_search(
236
self,
237
prompts: List[str],
238
beam_width: int,
239
max_tokens: int,
243
- ) -> List[Tuple[List[int], str]]:
240
+ ) -> List[Tuple[List[List[int]], List[str]]]:
241
outputs = self.generate(prompts,
242
do_sample=False,
243
max_new_tokens=max_tokens,
@@ -282,9 +279,7 @@ class HfRunner:
279
if self.model.get_output_embeddings().bias is not None:
280
logits += self.model.get_output_embeddings(
281
).bias.unsqueeze(0)
285
- logprobs = torch.nn.functional.log_softmax(logits,
286
- dim=-1,
287
- dtype=torch.float32)
282
+ logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
283
seq_logprobs.append(logprobs)
284
all_logprobs.append(seq_logprobs)
285
return all_logprobs
@@ -294,10 +289,10 @@ class HfRunner:
289
prompts: List[str],
290
max_tokens: int,
291
num_logprobs: int,
297
- ) -> List[Tuple[List[int], str]]:
298
- all_logprobs = []
299
- all_output_ids = []
300
- all_output_strs = []
292
+ ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
293
+ all_logprobs: List[List[Dict[int, float]]] = []
294
+ all_output_ids: List[List[int]] = []
295
+ all_output_strs: List[str] = []
296
297
for prompt in prompts:
298
input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
@@ -310,7 +305,7 @@ class HfRunner:
305
return_dict_in_generate=True,
306
)
307
313
- seq_logprobs = []
308
+ seq_logprobs: List[torch.Tensor] = []
309
for _, hidden_states in enumerate(output.hidden_states):
310
last_hidden_states = hidden_states[-1][0]
311
logits = torch.matmul(
@@ -321,13 +316,11 @@ class HfRunner:
316
None) is not None:
317
logits += self.model.get_output_embeddings(
318
).bias.unsqueeze(0)
324
- logprobs = torch.nn.functional.log_softmax(logits,
325
- dim=-1,
326
- dtype=torch.float32)
319
+ logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
320
seq_logprobs.append(logprobs)
321
322
# convert to dict
330
- seq_logprobs_lst = []
323
+ seq_logprobs_lst: List[Dict[int, float]] = []
324
for tok_idx, tok_logprobs in enumerate(seq_logprobs):
325
# drop prompt logprobs
326
if tok_idx == 0:
@@ -372,13 +365,13 @@ class VllmRunner:
365
tokenizer_name: Optional[str] = None,
366
# Use smaller max model length, otherwise bigger model cannot run due
367
# to kv cache size limit.
375
- max_model_len=1024,
368
+ max_model_len: int = 1024,
369
dtype: str = "half",
370
disable_log_stats: bool = True,
371
tensor_parallel_size: int = 1,
372
block_size: int = 16,
373
enable_chunked_prefill: bool = False,
381
- swap_space=4,
374
+ swap_space: int = 4,
375
**kwargs,
376
) -> None:
377
self.model = LLM(
@@ -399,32 +392,31 @@ class VllmRunner:
392
self,
393
prompts: List[str],
394
sampling_params: SamplingParams,
402
- images: Optional["torch.Tensor"] = None,
403
- ) -> List[Tuple[List[int], str]]:
395
+ images: Optional[torch.Tensor] = None,
396
+ ) -> List[Tuple[List[List[int]], List[str]]]:
397
if images is not None:
405
- assert len(prompts) == images.shape[0]
398
+ assert len(prompts) == len(images)
399
407
- prompt_inputs: List[PromptInputs] = []
400
+ prompt_inputs: List[TextPrompt] = []
401
for i, prompt in enumerate(prompts):
409
- image = None if images is None else images[i:i + 1]
410
- mm_data = None if image is None else MultiModalData(
411
- type=MultiModalData.Type.IMAGE,
412
- data=image,
413
- )
402
+ prompt = TextPrompt(prompt=prompt)
403
+ if images is not None:
404
+ prompt["multi_modal_data"] = MultiModalData(
405
+ type=MultiModalData.Type.IMAGE,
406
+ data=images[i:i + 1],
407
+ )
408
415
- prompt_inputs.append({
416
- "prompt": prompt,
417
- "multi_modal_data": mm_data,
418
- })
409
+ prompt_inputs.append(prompt)
410
411
req_outputs = self.model.generate(prompt_inputs,
412
sampling_params=sampling_params)
422
- outputs = []
413
+
414
+ outputs: List[Tuple[List[List[int]], List[str]]] = []
415
for req_output in req_outputs:
416
prompt_str = req_output.prompt
417
prompt_ids = req_output.prompt_token_ids
426
- req_sample_output_ids = []
427
- req_sample_output_strs = []
418
+ req_sample_output_ids: List[List[int]] = []
419
+ req_sample_output_strs: List[str] = []
420
for sample in req_output.outputs:
421
output_str = sample.text
422
output_ids = sample.token_ids
@@ -437,12 +429,12 @@ class VllmRunner:
429
self,
430
prompts: List[str],
431
sampling_params: SamplingParams,
440
- ) -> List[Tuple[List[int], str]]:
432
+ ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
433
assert sampling_params.logprobs is not None
434
435
req_outputs = self.model.generate(prompts,
436
sampling_params=sampling_params)
445
- outputs = []
437
+ outputs: List[Tuple[List[int], str, Optional[SampleLogprobs]]] = []
438
for req_output in req_outputs:
439
for sample in req_output.outputs:
440
output_str = sample.text
@@ -467,7 +459,7 @@ class VllmRunner:
459
prompts: List[str],
460
max_tokens: int,
461
num_logprobs: int,
470
- ) -> List[Tuple[List[int], str]]:
462
+ ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
463
greedy_logprobs_params = SamplingParams(temperature=0.0,
464
max_tokens=max_tokens,
465
logprobs=num_logprobs)
@@ -481,7 +473,7 @@ class VllmRunner:
473
prompts: List[str],
474
beam_width: int,
475
max_tokens: int,
484
- ) -> List[Tuple[List[int], str]]:
476
+ ) -> List[Tuple[List[List[int]], List[str]]]:
477
beam_search_params = SamplingParams(n=beam_width,
478
use_beam_search=True,
479
temperature=0.0,