@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51e971d39e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Support `eos_token_id` from `config.json` (#5954)

Cyrus Leung committed Jun 29, 2024 at 19:19 UTC 51e971d39e1272f1c5b070a5da6b38ccfa92fc14
3 files changed +67 -11
tests/tokenization/test_get_eos.py new
+31
@@ -0,0 +1,31 @@
1 +"""
2 +This test file includes some cases where it is inappropriate to
3 +only get the `eos_token_id` from the tokenizer as defined by
4 +:meth:`vllm.LLMEngine._get_eos_token_id`.
5 +"""
6 +from vllm.transformers_utils.config import try_get_generation_config
7 +from vllm.transformers_utils.tokenizer import get_tokenizer
8 +
9 +
10 +def test_get_llama3_eos_token():
11 + model_name = "meta-llama/Meta-Llama-3-8B-Instruct"
12 +
13 + tokenizer = get_tokenizer(model_name)
14 + assert tokenizer.eos_token_id == 128009
15 +
16 + generation_config = try_get_generation_config(model_name,
17 + trust_remote_code=False)
18 + assert generation_config is not None
19 + assert generation_config.eos_token_id == [128001, 128009]
20 +
21 +
22 +def test_get_blip2_eos_token():
23 + model_name = "Salesforce/blip2-opt-2.7b"
24 +
25 + tokenizer = get_tokenizer(model_name)
26 + assert tokenizer.eos_token_id == 2
27 +
28 + generation_config = try_get_generation_config(model_name,
29 + trust_remote_code=False)
30 + assert generation_config is not None
31 + assert generation_config.eos_token_id == 50118
vllm/engine/llm_engine.py
+13 -10
@@ -1,10 +1,10 @@
1 import time
2 from contextlib import contextmanager
3 -from typing import TYPE_CHECKING, ClassVar, Dict, Iterable, List, Optional
3 +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Set, Type, TypeVar, Union
6
7 -from transformers import GenerationConfig, PreTrainedTokenizer
7 +from transformers import PreTrainedTokenizer
8
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10 LoRAConfig, ModelConfig, ObservabilityConfig,
@@ -34,6 +34,7 @@ from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
34 SequenceStatus)
35 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
36 init_tracer)
37 +from vllm.transformers_utils.config import try_get_generation_config
38 from vllm.transformers_utils.detokenizer import Detokenizer
39 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
40 get_tokenizer_group)
@@ -46,16 +47,18 @@ logger = init_logger(__name__)
47 _LOCAL_LOGGING_INTERVAL_SEC = 5
48
49
49 -def _load_generation_config_dict(model_config: ModelConfig):
50 - try:
51 - return GenerationConfig.from_pretrained(
52 - model_config.model,
53 - revision=model_config.revision,
54 - ).to_diff_dict()
55 - except OSError:
56 - # Not found.
50 +def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
51 + config = try_get_generation_config(
52 + model_config.model,
53 + trust_remote_code=model_config.trust_remote_code,
54 + revision=model_config.revision,
55 + )
56 +
57 + if config is None:
58 return {}
59
60 + return config.to_diff_dict()
61 +
62
63 _O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
64
vllm/transformers_utils/config.py
+23 -1
@@ -1,7 +1,7 @@
1 import contextlib
2 from typing import Dict, Optional, Type
3
4 -from transformers import PretrainedConfig
4 +from transformers import GenerationConfig, PretrainedConfig
5
6 from vllm.envs import VLLM_USE_MODELSCOPE
7 from vllm.logger import init_logger
@@ -80,3 +80,25 @@ def get_hf_text_config(config: PretrainedConfig):
80 return config.text_config
81 else:
82 return config
83 +
84 +
85 +def try_get_generation_config(
86 + model: str,
87 + trust_remote_code: bool,
88 + revision: Optional[str] = None,
89 +) -> Optional[GenerationConfig]:
90 + try:
91 + return GenerationConfig.from_pretrained(
92 + model,
93 + revision=revision,
94 + )
95 + except OSError: # Not found
96 + try:
97 + config = get_config(
98 + model,
99 + trust_remote_code=trust_remote_code,
100 + revision=revision,
101 + )
102 + return GenerationConfig.from_model_config(config)
103 + except OSError: # Not found
104 + return None