@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 708e6c18b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[FIX] Fix class naming (#1803)

Zhuohan Li committed Nov 28, 2023 at 14:08 UTC 708e6c18b0eff37643458424391bcc68ef7f3467
3 files changed +17 -17
vllm/engine/llm_engine.py
+4 -4
@@ -12,8 +12,8 @@ from vllm.logger import init_logger
12 from vllm.outputs import RequestOutput
13 from vllm.sampling_params import SamplingParams
14 from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
15 - SequenceGroupMetadata, SequenceGroupOutputs,
16 - SequenceOutputs, SequenceStatus)
15 + SequenceGroupMetadata, SequenceGroupOutput,
16 + SequenceOutput, SequenceStatus)
17 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
18 get_tokenizer)
19 from vllm.utils import Counter
@@ -363,7 +363,7 @@ class LLMEngine:
363 return current_worst_score >= highest_attainable_score
364
365 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
366 - outputs: SequenceGroupOutputs) -> None:
366 + outputs: SequenceGroupOutput) -> None:
367 # Process prompt logprobs
368 prompt_logprobs = outputs.prompt_logprobs
369 if prompt_logprobs is not None:
@@ -384,7 +384,7 @@ class LLMEngine:
384
385 # Process the child samples for each parent sequence
386 for parent in parent_seqs:
387 - child_samples: List[SequenceOutputs] = parent_child_dict[
387 + child_samples: List[SequenceOutput] = parent_child_dict[
388 parent.seq_id]
389 if len(child_samples) == 0:
390 # This parent sequence has no children samples. Remove
vllm/model_executor/layers/sampler.py
+3 -3
@@ -9,7 +9,7 @@ from vllm.model_executor.parallel_utils.communication_op import (
9 tensor_model_parallel_all_gather)
10 from vllm.sampling_params import SamplingParams, SamplingType
11 from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 - SequenceData, SequenceGroupOutputs, SequenceOutputs)
12 + SequenceData, SequenceGroupOutput, SequenceOutput)
13
14 _SAMPLING_EPS = 1e-5
15
@@ -641,7 +641,7 @@ def _build_sampler_output(
641 next_token_ids,
642 group_sample_logprobs):
643 seq_outputs.append(
644 - SequenceOutputs(seq_ids[parent_id], next_token_id, logprobs))
644 + SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
645 sampler_output.append(
646 - SequenceGroupOutputs(seq_outputs, group_prompt_logprobs))
646 + SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
647 return sampler_output
vllm/sequence.py
+10 -10
@@ -352,7 +352,7 @@ class SequenceGroupMetadata:
352 self.block_tables = block_tables
353
354
355 -class SequenceOutputs:
355 +class SequenceOutput:
356 """The model output associated with a sequence.
357
358 Args:
@@ -374,40 +374,40 @@ class SequenceOutputs:
374 self.logprobs = logprobs
375
376 def __repr__(self) -> str:
377 - return (f"SequenceOutputs(parent_seq_id={self.parent_seq_id}, "
377 + return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "
378 f"output_token={self.output_token}, "
379 f"logprobs={self.logprobs})")
380
381 def __eq__(self, other: object) -> bool:
382 - if not isinstance(other, SequenceOutputs):
382 + if not isinstance(other, SequenceOutput):
383 raise NotImplementedError()
384 return (self.parent_seq_id == other.parent_seq_id
385 and self.output_token == other.output_token
386 and self.logprobs == other.logprobs)
387
388
389 -class SequenceGroupOutputs:
390 - """The model outputs associated with a sequence group."""
389 +class SequenceGroupOutput:
390 + """The model output associated with a sequence group."""
391
392 def __init__(
393 self,
394 - samples: List[SequenceOutputs],
394 + samples: List[SequenceOutput],
395 prompt_logprobs: Optional[PromptLogprobs],
396 ) -> None:
397 self.samples = samples
398 self.prompt_logprobs = prompt_logprobs
399
400 def __repr__(self) -> str:
401 - return (f"SequenceGroupOutputs(samples={self.samples}, "
401 + return (f"SequenceGroupOutput(samples={self.samples}, "
402 f"prompt_logprobs={self.prompt_logprobs})")
403
404 def __eq__(self, other: object) -> bool:
405 - if not isinstance(other, SequenceGroupOutputs):
405 + if not isinstance(other, SequenceGroupOutput):
406 raise NotImplementedError()
407 return (self.samples == other.samples
408 and self.prompt_logprobs == other.prompt_logprobs)
409
410
411 -# For each sequence group, we generate a list of SequenceOutputs object,
411 +# For each sequence group, we generate a list of SequenceOutput object,
412 # each of which contains one possible candidate for the next token.
413 -SamplerOutput = List[SequenceGroupOutputs]
413 +SamplerOutput = List[SequenceGroupOutput]