@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da224daaa9

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

[Bugfix] add hf_token to EngineArgs (#16093)

Signed-off-by: paolovic <paul-philipp.luley@uzh.ch> Co-authored-by: paolovic <paul-philipp.luley@uzh.ch>

paolovic committed Apr 6, 2025 at 16:47 UTC da224daaa9b4864e4ce08ca0fce1bc089e1baa98

4 files changed +27 -2

vllm/config.py

+5 -1

@@ -173,6 +173,9 @@ class ModelConfig:

173 Defaults to True.

174 config_format: The config format which shall be loaded.

175 Defaults to 'auto' which defaults to 'hf'.

176 + hf_token: The token to use as HTTP bearer authorization for remote files

177 + . If `True`, will use the token generated when running

178 + `huggingface-cli login` (stored in `~/.huggingface`).

179 hf_overrides: If a dictionary, contains arguments to be forwarded to the

180 HuggingFace config. If a callable, it is called to update the

181 HuggingFace config.

@@ -256,6 +259,7 @@ class ModelConfig:

259 limit_mm_per_prompt: Optional[Mapping[str, int]] = None,

260 use_async_output_proc: bool = True,

261 config_format: ConfigFormat = ConfigFormat.AUTO,

262 + hf_token: Optional[Union[bool, str]] = None,

263 hf_overrides: Optional[HfOverrides] = None,

264 mm_processor_kwargs: Optional[dict[str, Any]] = None,

265 disable_mm_preprocessor_cache: bool = False,

@@ -356,7 +360,7 @@ class ModelConfig:

360 self.hf_text_config = get_hf_text_config(self.hf_config)

361 self.encoder_config = self._get_encoder_config()

362 self.hf_image_processor_config = get_hf_image_processor_config(

359 - self.model, revision)

363 + self.model, hf_token=hf_token, revision=revision)

364 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)

365 self.use_async_output_proc = use_async_output_proc

366 self.mm_processor_kwargs = mm_processor_kwargs

vllm/engine/arg_utils.py

+12

@@ -138,6 +138,7 @@ class EngineArgs:

138 code_revision: Optional[str] = None

139 rope_scaling: Optional[Dict[str, Any]] = None

140 rope_theta: Optional[float] = None

141 + hf_token: Optional[Union[bool, str]] = None

142 hf_overrides: Optional[HfOverrides] = None

143 tokenizer_revision: Optional[str] = None

144 quantization: Optional[str] = None

@@ -602,6 +603,16 @@ class EngineArgs:

603 help='RoPE theta. Use with `rope_scaling`. In '

604 'some cases, changing the RoPE theta improves the '

605 'performance of the scaled model.')

606 + parser.add_argument(

607 + '--hf-token',

608 + type=str,

609 + nargs='?',

610 + const=True,

611 + default=None,

612 + help='The token to use as HTTP bearer authorization'

613 + ' for remote files. If `True`, will use the token '

614 + 'generated when running `huggingface-cli login` '

615 + '(stored in `~/.huggingface`).')

616 parser.add_argument('--hf-overrides',

617 type=json.loads,

618 default=EngineArgs.hf_overrides,

@@ -1038,6 +1049,7 @@ class EngineArgs:

1049 code_revision=self.code_revision,

1050 rope_scaling=self.rope_scaling,

1051 rope_theta=self.rope_theta,

1052 + hf_token=self.hf_token,

1053 hf_overrides=self.hf_overrides,

1054 tokenizer_revision=self.tokenizer_revision,

1055 max_model_len=self.max_model_len,

vllm/entrypoints/llm.py

@@ -117,6 +117,9 @@ class LLM:

117 disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`

118 disable_async_output_proc: Disable async output processing.

119 This may result in lower performance.

120 + hf_token: The token to use as HTTP bearer authorization for remote files

121 + . If `True`, will use the token generated when running

122 + `huggingface-cli login` (stored in `~/.huggingface`).

123 hf_overrides: If a dictionary, contains arguments to be forwarded to the

124 HuggingFace config. If a callable, it is called to update the

125 HuggingFace config.

@@ -177,6 +180,7 @@ class LLM:

180 max_seq_len_to_capture: int = 8192,

181 disable_custom_all_reduce: bool = False,

182 disable_async_output_proc: bool = False,

183 + hf_token: Optional[Union[bool, str]] = None,

184 hf_overrides: Optional[HfOverrides] = None,

185 mm_processor_kwargs: Optional[dict[str, Any]] = None,

186 # After positional args are removed, move this right below `model`

@@ -232,6 +236,7 @@ class LLM:

236 max_seq_len_to_capture=max_seq_len_to_capture,

237 disable_custom_all_reduce=disable_custom_all_reduce,

238 disable_async_output_proc=disable_async_output_proc,

239 + hf_token=hf_token,

240 hf_overrides=hf_overrides,

241 mm_processor_kwargs=mm_processor_kwargs,

242 override_pooler_config=override_pooler_config,

vllm/transformers_utils/config.py

+5 -1

@@ -712,6 +712,7 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],

712

713 def get_hf_image_processor_config(

714 model: Union[str, Path],

715 + hf_token: Optional[Union[bool, str]] = None,

716 revision: Optional[str] = None,

717 **kwargs,

718 ) -> Dict[str, Any]:

@@ -721,7 +722,10 @@ def get_hf_image_processor_config(

722 # Separate model folder from file path for GGUF models

723 if check_gguf_file(model):

724 model = Path(model).parent

724 - return get_image_processor_config(model, revision=revision, **kwargs)

725 + return get_image_processor_config(model,

726 + token=hf_token,

727 + revision=revision,

728 + **kwargs)

729

730

731 def get_hf_text_config(config: PretrainedConfig):