@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a31cab7556
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Avoid copying prompt/output tokens if no penalties are used (#5289)

Antoni Baum committed Jun 6, 2024 at 18:12 UTC a31cab7556f540b558b0b454b4a4b9b438542566
1 file changed +50 -30
vllm/model_executor/sampling_metadata.py
+50 -30
@@ -386,16 +386,18 @@ class SamplingTensors:
386 presence_penalties += [0] * prefill_len
387 frequency_penalties += [0] * prefill_len
388 repetition_penalties += [1] * prefill_len
389 - prompt_tokens.extend([] for _ in range(prefill_len))
390 - output_tokens.extend([] for _ in range(prefill_len))
389 + if do_penalties:
390 + prompt_tokens.extend([] for _ in range(prefill_len))
391 + output_tokens.extend([] for _ in range(prefill_len))
392
393 if seq_group.do_sample:
394 sample_lens = len(seq_group.sample_indices)
395 assert sample_lens == len(seq_ids)
396 for seq_id in seq_ids:
397 seq_data = seq_group.seq_data[seq_id]
397 - prompt_tokens.append(seq_data.prompt_token_ids)
398 - output_tokens.append(seq_data.output_token_ids)
398 + if do_penalties:
399 + prompt_tokens.append(seq_data.prompt_token_ids)
400 + output_tokens.append(seq_data.output_token_ids)
401 temperatures += [temperature] * len(seq_ids)
402 top_ps += [top_p] * len(seq_ids)
403 top_ks += [top_k] * len(seq_ids)
@@ -443,18 +445,22 @@ class SamplingTensors:
445 # Note that the performance will be very bad without
446 # pinned memory.
447 pin_memory = is_pin_memory_available()
446 - prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
447 - default=0)
448 - prompt_padded_tokens = [
449 - tokens + [vocab_size] * (prompt_max_len - len(tokens))
450 - for tokens in prompt_tokens
451 - ]
452 - output_max_len = max([len(tokens) for tokens in output_tokens],
453 - default=0)
454 - output_padded_tokens = [
455 - tokens + [vocab_size] * (output_max_len - len(tokens))
456 - for tokens in output_tokens
457 - ]
448 +
449 + do_penalties = prompt_tokens or output_tokens
450 +
451 + if do_penalties:
452 + prompt_max_len = max([len(tokens) for tokens in prompt_tokens],
453 + default=0)
454 + prompt_padded_tokens = [
455 + tokens + [vocab_size] * (prompt_max_len - len(tokens))
456 + for tokens in prompt_tokens
457 + ]
458 + output_max_len = max([len(tokens) for tokens in output_tokens],
459 + default=0)
460 + output_padded_tokens = [
461 + tokens + [vocab_size] * (output_max_len - len(tokens))
462 + for tokens in output_tokens
463 + ]
464
465 temperatures_t = torch.tensor(
466 temperatures,
@@ -504,18 +510,22 @@ class SamplingTensors:
510 dtype=torch.long,
511 pin_memory=pin_memory,
512 )
507 - prompt_tensor = torch.tensor(
508 - prompt_padded_tokens,
509 - device="cpu",
510 - dtype=torch.long,
511 - pin_memory=pin_memory,
512 - )
513 - output_tensor = torch.tensor(
514 - output_padded_tokens,
515 - device="cpu",
516 - dtype=torch.long,
517 - pin_memory=pin_memory,
518 - )
513 + if do_penalties:
514 + prompt_tensor = torch.tensor(
515 + prompt_padded_tokens,
516 + device="cpu",
517 + dtype=torch.long,
518 + pin_memory=pin_memory,
519 + )
520 + output_tensor = torch.tensor(
521 + output_padded_tokens,
522 + device="cpu",
523 + dtype=torch.long,
524 + pin_memory=pin_memory,
525 + )
526 + else:
527 + prompt_tensor = None
528 + output_tensor = None
529 # need to transpose and make contiguous to
530 # copy the tensor correctly.
531 # [batch_size, n_seeds] -> [n_seeds, batch_size]
@@ -538,6 +548,16 @@ class SamplingTensors:
548 extra_seeds_gpu = None
549 sampling_seeds_gpu = sampling_seeds_gpu[:num_base_seeds]
550
551 + if do_penalties:
552 + prompt_tokens_gpu = prompt_tensor.to(device=device,
553 + non_blocking=True)
554 + output_tokens_gpu = output_tensor.to(device=device,
555 + non_blocking=True)
556 + else:
557 + empty_tensor = torch.empty(0, device=device, dtype=torch.long)
558 + prompt_tokens_gpu = empty_tensor
559 + output_tokens_gpu = empty_tensor
560 +
561 return cls(
562 temperatures=temperatures_t.to(device=device, non_blocking=True),
563 top_ps=top_ps_t.to(device=device, non_blocking=True),
@@ -549,8 +569,8 @@ class SamplingTensors:
569 non_blocking=True),
570 repetition_penalties=repetition_penalties_t.to(device=device,
571 non_blocking=True),
552 - prompt_tokens=prompt_tensor.to(device=device, non_blocking=True),
553 - output_tokens=output_tensor.to(device=device, non_blocking=True),
572 + prompt_tokens=prompt_tokens_gpu,
573 + output_tokens=output_tokens_gpu,
574 sampling_seeds=sampling_seeds_gpu,
575 sample_indices=sample_indices_t.to(device=device,
576 non_blocking=True),