A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[FIX] Fix class naming (#1803)
Zhuohan Li committed
Nov 28, 2023 at 14:08 UTC
708e6c18b0eff37643458424391bcc68ef7f3467
3 files changed
+17
-17
vllm/engine/llm_engine.py
+4
-4
@@ -12,8 +12,8 @@ from vllm.logger import init_logger
12
from vllm.outputs import RequestOutput
13
from vllm.sampling_params import SamplingParams
14
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15
- SequenceGroupMetadata, SequenceGroupOutputs,
16
- SequenceOutputs, SequenceStatus)
15
+ SequenceGroupMetadata, SequenceGroupOutput,
16
+ SequenceOutput, SequenceStatus)
17
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18
get_tokenizer)
19
from vllm.utils import Counter
@@ -363,7 +363,7 @@ class LLMEngine:
363
return current_worst_score >= highest_attainable_score
364
365
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
366
- outputs: SequenceGroupOutputs) -> None:
366
+ outputs: SequenceGroupOutput) -> None:
367
# Process prompt logprobs
368
prompt_logprobs = outputs.prompt_logprobs
369
if prompt_logprobs is not None:
@@ -384,7 +384,7 @@ class LLMEngine:
384
385
# Process the child samples for each parent sequence
386
for parent in parent_seqs:
387
- child_samples: List[SequenceOutputs] = parent_child_dict[
387
+ child_samples: List[SequenceOutput] = parent_child_dict[
388
parent.seq_id]
389
if len(child_samples) == 0:
390
# This parent sequence has no children samples. Remove
vllm/model_executor/layers/sampler.py
+3
-3
@@ -9,7 +9,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
9
tensor_model_parallel_all_gather)
10
from vllm.sampling_params import SamplingParams, SamplingType
11
from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
- SequenceData, SequenceGroupOutputs, SequenceOutputs)
12
+ SequenceData, SequenceGroupOutput, SequenceOutput)
13
14
_SAMPLING_EPS = 1e-5
15
@@ -641,7 +641,7 @@ def _build_sampler_output(
641
next_token_ids,
642
group_sample_logprobs):
643
seq_outputs.append(
644
- SequenceOutputs(seq_ids[parent_id], next_token_id, logprobs))
644
+ SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
645
sampler_output.append(
646
- SequenceGroupOutputs(seq_outputs, group_prompt_logprobs))
646
+ SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
647
return sampler_output
vllm/sequence.py
+10
-10
@@ -352,7 +352,7 @@ class SequenceGroupMetadata:
352
self.block_tables = block_tables
353
354
355
-class SequenceOutputs:
355
+class SequenceOutput:
356
"""The model output associated with a sequence.
357
358
Args:
@@ -374,40 +374,40 @@ class SequenceOutputs:
374
self.logprobs = logprobs
375
376
def __repr__(self) -> str:
377
- return (f"SequenceOutputs(parent_seq_id={self.parent_seq_id}, "
377
+ return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "
378
f"output_token={self.output_token}, "
379
f"logprobs={self.logprobs})")
380
381
def __eq__(self, other: object) -> bool:
382
- if not isinstance(other, SequenceOutputs):
382
+ if not isinstance(other, SequenceOutput):
383
raise NotImplementedError()
384
return (self.parent_seq_id == other.parent_seq_id
385
and self.output_token == other.output_token
386
and self.logprobs == other.logprobs)
387
388
389
-class SequenceGroupOutputs:
390
- """The model outputs associated with a sequence group."""
389
+class SequenceGroupOutput:
390
+ """The model output associated with a sequence group."""
391
392
def __init__(
393
self,
394
- samples: List[SequenceOutputs],
394
+ samples: List[SequenceOutput],
395
prompt_logprobs: Optional[PromptLogprobs],
396
) -> None:
397
self.samples = samples
398
self.prompt_logprobs = prompt_logprobs
399
400
def __repr__(self) -> str:
401
- return (f"SequenceGroupOutputs(samples={self.samples}, "
401
+ return (f"SequenceGroupOutput(samples={self.samples}, "
402
f"prompt_logprobs={self.prompt_logprobs})")
403
404
def __eq__(self, other: object) -> bool:
405
- if not isinstance(other, SequenceGroupOutputs):
405
+ if not isinstance(other, SequenceGroupOutput):
406
raise NotImplementedError()
407
return (self.samples == other.samples
408
and self.prompt_logprobs == other.prompt_logprobs)
409
410
411
-# For each sequence group, we generate a list of SequenceOutputs object,
411
+# For each sequence group, we generate a list of SequenceOutput object,
412
# each of which contains one possible candidate for the next token.
413
-SamplerOutput = List[SequenceGroupOutputs]
413
+SamplerOutput = List[SequenceGroupOutput]