@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4302987069
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Fix inappropriate content of model_name tag in Prometheus metrics (#3937)

DearPlanet committed May 5, 2024 at 06:39 UTC 43029870694de0789a10ab49f181f1cba6ec741a
5 files changed +76 -14
tests/metrics/test_metrics.py
+30
@@ -1,3 +1,5 @@
1 +from typing import List
2 +
3 import pytest
4 from prometheus_client import REGISTRY
5
@@ -76,6 +78,34 @@ def test_metric_counter_generation_tokens(
78 f"metric: {metric_count!r}")
79
80
81 +@pytest.mark.parametrize("model", MODELS)
82 +@pytest.mark.parametrize("dtype", ["float"])
83 +@pytest.mark.parametrize(
84 + "served_model_name",
85 + [None, [], ["ModelName0"], ["ModelName0", "ModelName1", "ModelName2"]])
86 +def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
87 + served_model_name: List[str]) -> None:
88 + vllm_model = vllm_runner(model,
89 + dtype=dtype,
90 + disable_log_stats=False,
91 + gpu_memory_utilization=0.3,
92 + served_model_name=served_model_name)
93 + stat_logger = vllm_model.model.llm_engine.stat_logger
94 + metrics_tag_content = stat_logger.labels["model_name"]
95 +
96 + del vllm_model
97 +
98 + if served_model_name is None or served_model_name == []:
99 + assert metrics_tag_content == model, (
100 + f"Metrics tag model_name is wrong! expect: {model!r}\n"
101 + f"actual: {metrics_tag_content!r}")
102 + else:
103 + assert metrics_tag_content == served_model_name[0], (
104 + f"Metrics tag model_name is wrong! expect: "
105 + f"{served_model_name[0]!r}\n"
106 + f"actual: {metrics_tag_content!r}")
107 +
108 +
109 @pytest.mark.parametrize("model", MODELS)
110 @pytest.mark.parametrize("dtype", ["half"])
111 @pytest.mark.parametrize("max_tokens", [4])
vllm/config.py
+25
@@ -31,6 +31,8 @@ class ModelConfig:
31
32 Args:
33 model: Name or path of the huggingface model to use.
34 + It is also used as the content for `model_name` tag in metrics
35 + output when `served_model_name` is not specified.
36 tokenizer: Name or path of the huggingface tokenizer to use.
37 tokenizer_mode: Tokenizer mode. "auto" will use the fast tokenizer if
38 available, and "slow" will always use the slow tokenizer.
@@ -69,6 +71,10 @@ class ModelConfig:
71 to eager mode
72 skip_tokenizer_init: If true, skip initialization of tokenizer and
73 detokenizer.
74 + served_model_name: The model name used in metrics tag `model_name`,
75 + matches the model name exposed via the APIs. If multiple model
76 + names provided, the first name will be used. If not specified,
77 + the model name will be the same as `model`.
78 """
79
80 def __init__(
@@ -90,6 +96,7 @@ class ModelConfig:
96 max_seq_len_to_capture: Optional[int] = None,
97 max_logprobs: int = 5,
98 skip_tokenizer_init: bool = False,
99 + served_model_name: Optional[Union[str, List[str]]] = None,
100 ) -> None:
101 self.model = model
102 self.tokenizer = tokenizer
@@ -117,6 +124,8 @@ class ModelConfig:
124 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
125 self.max_model_len = _get_and_verify_max_len(self.hf_text_config,
126 max_model_len)
127 + self.served_model_name = get_served_model_name(model,
128 + served_model_name)
129 if not self.skip_tokenizer_init:
130 self._verify_tokenizer_mode()
131 self._verify_quantization()
@@ -1150,6 +1159,22 @@ def _get_and_verify_max_len(
1159 return int(max_model_len)
1160
1161
1162 +def get_served_model_name(model: str,
1163 + served_model_name: Optional[Union[str, List[str]]]):
1164 + """
1165 + If the input is a non-empty list, the first model_name in
1166 + `served_model_name` is taken.
1167 + If the input is a non-empty string, it is used directly.
1168 + For cases where the input is either an empty string or an
1169 + empty list, the fallback is to use `self.model`.
1170 + """
1171 + if not served_model_name:
1172 + return model
1173 + if isinstance(served_model_name, list):
1174 + return served_model_name[0]
1175 + return served_model_name
1176 +
1177 +
1178 @dataclass
1179 class DecodingConfig:
1180 """Dataclass which contains the decoding strategy of the engine"""
vllm/engine/arg_utils.py
+18 -2
@@ -1,7 +1,7 @@
1 import argparse
2 import dataclasses
3 from dataclasses import dataclass
4 -from typing import Optional
4 +from typing import List, Optional, Union
5
6 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
7 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -21,6 +21,7 @@ def nullable_str(val: str):
21 class EngineArgs:
22 """Arguments for vLLM engine."""
23 model: str
24 + served_model_name: Optional[Union[List[str]]] = None
25 tokenizer: Optional[str] = None
26 skip_tokenizer_init: bool = False
27 tokenizer_mode: str = 'auto'
@@ -489,6 +490,21 @@ class EngineArgs:
490 'This should be a JSON string that will be '
491 'parsed into a dictionary.')
492
493 + parser.add_argument(
494 + "--served-model-name",
495 + nargs="+",
496 + type=str,
497 + default=None,
498 + help="The model name(s) used in the API. If multiple "
499 + "names are provided, the server will respond to any "
500 + "of the provided names. The model name in the model "
501 + "field of a response will be the first name in this "
502 + "list. If not specified, the model name will be the "
503 + "same as the `--model` argument. Noted that this name(s)"
504 + "will also be used in `model_name` tag content of "
505 + "prometheus metrics, if multiple names provided, metrics"
506 + "tag will take the first one.")
507 +
508 return parser
509
510 @classmethod
@@ -508,7 +524,7 @@ class EngineArgs:
524 self.quantization, self.quantization_param_path,
525 self.enforce_eager, self.max_context_len_to_capture,
526 self.max_seq_len_to_capture, self.max_logprobs,
511 - self.skip_tokenizer_init)
527 + self.skip_tokenizer_init, self.served_model_name)
528 cache_config = CacheConfig(self.block_size,
529 self.gpu_memory_utilization,
530 self.swap_space, self.kv_cache_dtype,
vllm/engine/llm_engine.py
+3 -2
@@ -106,7 +106,7 @@ class LLMEngine:
106 "tensor_parallel_size=%d, disable_custom_all_reduce=%s, "
107 "quantization=%s, enforce_eager=%s, kv_cache_dtype=%s, "
108 "quantization_param_path=%s, device_config=%s, "
109 - "decoding_config=%r, seed=%d)",
109 + "decoding_config=%r, seed=%d, served_model_name=%s)",
110 vllm.__version__,
111 model_config.model,
112 speculative_config,
@@ -129,6 +129,7 @@ class LLMEngine:
129 device_config.device,
130 decoding_config,
131 model_config.seed,
132 + model_config.served_model_name,
133 )
134 # TODO(woosuk): Print more configs in debug mode.
135
@@ -219,7 +220,7 @@ class LLMEngine:
220 if self.log_stats:
221 self.stat_logger = StatLogger(
222 local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
222 - labels=dict(model_name=model_config.model),
223 + labels=dict(model_name=model_config.served_model_name),
224 max_model_len=self.model_config.max_model_len)
225 self.stat_logger.info("cache_config", self.cache_config)
226
vllm/entrypoints/openai/cli_args.py
-10
@@ -56,16 +56,6 @@ def make_arg_parser():
56 default=None,
57 help="If provided, the server will require this key "
58 "to be presented in the header.")
59 - parser.add_argument("--served-model-name",
60 - nargs="+",
61 - type=nullable_str,
62 - default=None,
63 - help="The model name(s) used in the API. If multiple "
64 - "names are provided, the server will respond to any "
65 - "of the provided names. The model name in the model "
66 - "field of a response will be the first name in this "
67 - "list. If not specified, the model name will be the "
68 - "same as the `--model` argument.")
59 parser.add_argument(
60 "--lora-modules",
61 type=nullable_str,