A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
Support eos_token_id from generation_config.json (#4182)
Simon Mo committed
Apr 18, 2024 at 21:13 UTC
a134ef6f5e6c24d3cd459c63557e5db276db25b2
2 files changed
+30
-3
vllm/engine/llm_engine.py
+17
-2
@@ -1,7 +1,7 @@
1
import time
2
from typing import Iterable, List, Optional, Type, Union
3
4
-from transformers import PreTrainedTokenizer
4
+from transformers import GenerationConfig, PreTrainedTokenizer
5
6
import vllm
7
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
@@ -34,6 +34,17 @@ logger = init_logger(__name__)
34
_LOCAL_LOGGING_INTERVAL_SEC = 5
35
36
37
+def _load_generation_config_dict(model_config: ModelConfig):
38
+ try:
39
+ return GenerationConfig.from_pretrained(
40
+ model_config.model,
41
+ revision=model_config.revision,
42
+ ).to_diff_dict()
43
+ except OSError:
44
+ # Not found.
45
+ return {}
46
+
47
+
48
class LLMEngine:
49
"""An LLM engine that receives requests and generates texts.
50
@@ -124,6 +135,8 @@ class LLMEngine:
135
self._init_tokenizer()
136
self.detokenizer = Detokenizer(self.tokenizer)
137
self.seq_counter = Counter()
138
+ self.generation_config_fields = _load_generation_config_dict(
139
+ model_config)
140
141
self.model_executor = executor_class(
142
model_config=model_config,
@@ -391,6 +404,8 @@ class LLMEngine:
404
# inject the eos token id into the sampling_params to support min_tokens
405
# processing
406
sampling_params.eos_token_id = seq.eos_token_id
407
+ sampling_params.update_from_generation_config(
408
+ self.generation_config_fields)
409
410
# Create the sequence group.
411
seq_group = SequenceGroup(request_id, [seq], sampling_params,
@@ -435,7 +450,7 @@ class LLMEngine:
450
scheduled_seq_groups: List[SequenceGroup],
451
ignored_seq_groups: List[SequenceGroup]) -> List[RequestOutput]:
452
"""Apply the model output to the sequences in the scheduled seq groups.
438
-
453
+
454
Returns RequestOutputs that can be returned to the client.
455
"""
456
vllm/sampling_params.py
+13
-1
@@ -2,7 +2,7 @@
2
import copy
3
from enum import IntEnum
4
from functools import cached_property
5
-from typing import Callable, List, Optional, Union
5
+from typing import Any, Callable, Dict, List, Optional, Union
6
7
import torch
8
from pydantic import Field
@@ -271,6 +271,18 @@ class SamplingParams:
271
raise ValueError("best_of must be 1 when using greedy sampling."
272
f"Got {self.best_of}.")
273
274
+ def update_from_generation_config(
275
+ self, generation_config: Dict[str, Any]) -> None:
276
+ """Update if there are non-default values from generation_config"""
277
+ # Update eos_token_id for generation
278
+ if eos_ids := generation_config.get("eos_token_id"):
279
+ # it can be either int or list of int
280
+ if isinstance(eos_ids, int):
281
+ eos_ids = [eos_ids]
282
+ original_stop_token_ids = set(self.stop_token_ids)
283
+ original_stop_token_ids.update(eos_ids)
284
+ self.stop_token_ids = list(original_stop_token_ids)
285
+
286
@cached_property
287
def sampling_type(self) -> SamplingType:
288
if self.use_beam_search: