@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a134ef6f5e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

Support eos_token_id from generation_config.json (#4182)

Simon Mo committed Apr 18, 2024 at 21:13 UTC a134ef6f5e6c24d3cd459c63557e5db276db25b2
2 files changed +30 -3
vllm/engine/llm_engine.py
+17 -2
@@ -1,7 +1,7 @@
1 import time
2 from typing import Iterable, List, Optional, Type, Union
3
4 -from transformers import PreTrainedTokenizer
4 +from transformers import GenerationConfig, PreTrainedTokenizer
5
6 import vllm
7 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
@@ -34,6 +34,17 @@ logger = init_logger(__name__)
34 _LOCAL_LOGGING_INTERVAL_SEC = 5
35
36
37 +def _load_generation_config_dict(model_config: ModelConfig):
38 + try:
39 + return GenerationConfig.from_pretrained(
40 + model_config.model,
41 + revision=model_config.revision,
42 + ).to_diff_dict()
43 + except OSError:
44 + # Not found.
45 + return {}
46 +
47 +
48 class LLMEngine:
49 """An LLM engine that receives requests and generates texts.
50
@@ -124,6 +135,8 @@ class LLMEngine:
135 self._init_tokenizer()
136 self.detokenizer = Detokenizer(self.tokenizer)
137 self.seq_counter = Counter()
138 + self.generation_config_fields = _load_generation_config_dict(
139 + model_config)
140
141 self.model_executor = executor_class(
142 model_config=model_config,
@@ -391,6 +404,8 @@ class LLMEngine:
404 # inject the eos token id into the sampling_params to support min_tokens
405 # processing
406 sampling_params.eos_token_id = seq.eos_token_id
407 + sampling_params.update_from_generation_config(
408 + self.generation_config_fields)
409
410 # Create the sequence group.
411 seq_group = SequenceGroup(request_id, [seq], sampling_params,
@@ -435,7 +450,7 @@ class LLMEngine:
450 scheduled_seq_groups: List[SequenceGroup],
451 ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
452 """Apply the model output to the sequences in the scheduled seq groups.
438 -
453 +
454 Returns RequestOutputs that can be returned to the client.
455 """
456
vllm/sampling_params.py
+13 -1
@@ -2,7 +2,7 @@
2 import copy
3 from enum import IntEnum
4 from functools import cached_property
5 -from typing import Callable, List, Optional, Union
5 +from typing import Any, Callable, Dict, List, Optional, Union
6
7 import torch
8 from pydantic import Field
@@ -271,6 +271,18 @@ class SamplingParams:
271 raise ValueError("best_of must be 1 when using greedy sampling."
272 f"Got {self.best_of}.")
273
274 + def update_from_generation_config(
275 + self, generation_config: Dict[str, Any]) -> None:
276 + """Update if there are non-default values from generation_config"""
277 + # Update eos_token_id for generation
278 + if eos_ids := generation_config.get("eos_token_id"):
279 + # it can be either int or list of int
280 + if isinstance(eos_ids, int):
281 + eos_ids = [eos_ids]
282 + original_stop_token_ids = set(self.stop_token_ids)
283 + original_stop_token_ids.update(eos_ids)
284 + self.stop_token_ids = list(original_stop_token_ids)
285 +
286 @cached_property
287 def sampling_type(self) -> SamplingType:
288 if self.use_beam_search: