A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Avoid copying prompt/output tokens if no penalties are used (#5289)
Antoni Baum committed
Jun 6, 2024 at 18:12 UTC
a31cab7556f540b558b0b454b4a4b9b438542566
1 file changed
+50
-30
vllm/model_executor/sampling_metadata.py
+50
-30
@@ -386,16 +386,18 @@ class SamplingTensors:
386
presence_penalties += [0] * prefill_len
387
frequency_penalties += [0] * prefill_len
388
repetition_penalties += [1] * prefill_len
389
- prompt_tokens.extend([] for _ in range(prefill_len))
390
- output_tokens.extend([] for _ in range(prefill_len))
389
+ if do_penalties:
390
+ prompt_tokens.extend([] for _ in range(prefill_len))
391
+ output_tokens.extend([] for _ in range(prefill_len))
392
393
if seq_group.do_sample:
394
sample_lens = len(seq_group.sample_indices)
395
assert sample_lens == len(seq_ids)
396
for seq_id in seq_ids:
397
seq_data = seq_group.seq_data[seq_id]
397
- prompt_tokens.append(seq_data.prompt_token_ids)
398
- output_tokens.append(seq_data.output_token_ids)
398
+ if do_penalties:
399
+ prompt_tokens.append(seq_data.prompt_token_ids)
400
+ output_tokens.append(seq_data.output_token_ids)
401
temperatures += [temperature] * len(seq_ids)
402
top_ps += [top_p] * len(seq_ids)
403
top_ks += [top_k] * len(seq_ids)
@@ -443,18 +445,22 @@ class SamplingTensors:
445
# Note that the performance will be very bad without
446
# pinned memory.
447
pin_memory = is_pin_memory_available()
446
- prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
447
- default=0)
448
- prompt_padded_tokens = [
449
- tokens + [vocab_size] * (prompt_max_len - len(tokens))
450
- for tokens in prompt_tokens
451
- ]
452
- output_max_len = max([len(tokens) for tokens in output_tokens],
453
- default=0)
454
- output_padded_tokens = [
455
- tokens + [vocab_size] * (output_max_len - len(tokens))
456
- for tokens in output_tokens
457
- ]
448
+
449
+ do_penalties = prompt_tokens or output_tokens
450
+
451
+ if do_penalties:
452
+ prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
453
+ default=0)
454
+ prompt_padded_tokens = [
455
+ tokens + [vocab_size] * (prompt_max_len - len(tokens))
456
+ for tokens in prompt_tokens
457
+ ]
458
+ output_max_len = max([len(tokens) for tokens in output_tokens],
459
+ default=0)
460
+ output_padded_tokens = [
461
+ tokens + [vocab_size] * (output_max_len - len(tokens))
462
+ for tokens in output_tokens
463
+ ]
464
465
temperatures_t = torch.tensor(
466
temperatures,
@@ -504,18 +510,22 @@ class SamplingTensors:
510
dtype=torch.long,
511
pin_memory=pin_memory,
512
)
507
- prompt_tensor = torch.tensor(
508
- prompt_padded_tokens,
509
- device="cpu",
510
- dtype=torch.long,
511
- pin_memory=pin_memory,
512
- )
513
- output_tensor = torch.tensor(
514
- output_padded_tokens,
515
- device="cpu",
516
- dtype=torch.long,
517
- pin_memory=pin_memory,
518
- )
513
+ if do_penalties:
514
+ prompt_tensor = torch.tensor(
515
+ prompt_padded_tokens,
516
+ device="cpu",
517
+ dtype=torch.long,
518
+ pin_memory=pin_memory,
519
+ )
520
+ output_tensor = torch.tensor(
521
+ output_padded_tokens,
522
+ device="cpu",
523
+ dtype=torch.long,
524
+ pin_memory=pin_memory,
525
+ )
526
+ else:
527
+ prompt_tensor = None
528
+ output_tensor = None
529
# need to transpose and make contiguous to
530
# copy the tensor correctly.
531
# [batch_size, n_seeds] -> [n_seeds, batch_size]
@@ -538,6 +548,16 @@ class SamplingTensors:
548
extra_seeds_gpu = None
549
sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
550
551
+ if do_penalties:
552
+ prompt_tokens_gpu = prompt_tensor.to(device=device,
553
+ non_blocking=True)
554
+ output_tokens_gpu = output_tensor.to(device=device,
555
+ non_blocking=True)
556
+ else:
557
+ empty_tensor = torch.empty(0, device=device, dtype=torch.long)
558
+ prompt_tokens_gpu = empty_tensor
559
+ output_tokens_gpu = empty_tensor
560
+
561
return cls(
562
temperatures=temperatures_t.to(device=device, non_blocking=True),
563
top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -549,8 +569,8 @@ class SamplingTensors:
569
non_blocking=True),
570
repetition_penalties=repetition_penalties_t.to(device=device,
571
non_blocking=True),
552
- prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
553
- output_tokens=output_tensor.to(device=device, non_blocking=True),
572
+ prompt_tokens=prompt_tokens_gpu,
573
+ output_tokens=output_tokens_gpu,
574
sampling_seeds=sampling_seeds_gpu,
575
sample_indices=sample_indices_t.to(device=device,
576
non_blocking=True),