d97841078b · tduncan/Nvidia-Optimization-Stack-VLLM

docs/configuration/model_resolution.md

+1 -1

@@ -14,7 +14,7 @@ For example:

14 ```python

15 from vllm import LLM

16

17 -model = LLM(

17 +llm = LLM(

18 model="cerebras/Cerebras-GPT-1.3B",

19 hf_overrides={"architectures": ["GPT2LMHeadModel"]}, # GPT-2

20 )

docs/features/lora.md

+2 -2

@@ -302,7 +302,7 @@ To this end, we allow registration of default multimodal LoRAs to handle this au

302 return tokenizer.apply_chat_template(chat, tokenize=False)

303

304

305 - model = LLM(

305 + llm = LLM(

306 model=model_id,

307 enable_lora=True,

308 max_lora_rank=64,

@@ -329,7 +329,7 @@ To this end, we allow registration of default multimodal LoRAs to handle this au

329 }

330

331

332 - outputs = model.generate(

332 + outputs = llm.generate(

333 inputs,

334 sampling_params=SamplingParams(

335 temperature=0.2,

docs/features/quantization/fp8.md

+6 -4

@@ -86,8 +86,9 @@ Load and run the model in `vllm`:

86

87 ```python

88 from vllm import LLM

89 -model = LLM("./Meta-Llama-3-8B-Instruct-FP8-Dynamic")

90 -result = model.generate("Hello my name is")

89 +

90 +llm = LLM("./Meta-Llama-3-8B-Instruct-FP8-Dynamic")

91 +result = llm.generate("Hello my name is")

92 print(result[0].outputs[0].text)

93 ```

94

@@ -125,9 +126,10 @@ In this mode, all Linear modules (except for the final `lm_head`) have their wei

126

127 ```python

128 from vllm import LLM

128 -model = LLM("facebook/opt-125m", quantization="fp8")

129 +

130 +llm = LLM("facebook/opt-125m", quantization="fp8")

131 # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB

130 -result = model.generate("Hello, my name is")

132 +result = llm.generate("Hello, my name is")

133 print(result[0].outputs[0].text)

134 ```

135

docs/features/quantization/int4.md

+2 -1

@@ -108,7 +108,8 @@ After quantization, you can load and run the model in vLLM:

108

109 ```python

110 from vllm import LLM

111 -model = LLM("./Meta-Llama-3-8B-Instruct-W4A16-G128")

111 +

112 +llm = LLM("./Meta-Llama-3-8B-Instruct-W4A16-G128")

113 ```

114

115 To evaluate accuracy, you can use `lm_eval`:

docs/features/quantization/int8.md

+2 -1

@@ -114,7 +114,8 @@ After quantization, you can load and run the model in vLLM:

114

115 ```python

116 from vllm import LLM

117 -model = LLM("./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token")

117 +

118 +llm = LLM("./Meta-Llama-3-8B-Instruct-W8A8-Dynamic-Per-Token")

119 ```

120

121 To evaluate accuracy, you can use `lm_eval`:

docs/models/pooling_models.md

+5 -5

@@ -174,11 +174,11 @@ You can change the output dimensions of embedding models that support Matryoshka

174 ```python

175 from vllm import LLM, PoolingParams

176

177 -model = LLM(model="jinaai/jina-embeddings-v3",

178 - task="embed",

179 - trust_remote_code=True)

180 -outputs = model.embed(["Follow the white rabbit."],

181 - pooling_params=PoolingParams(dimensions=32))

177 +llm = LLM(model="jinaai/jina-embeddings-v3",

178 + task="embed",

179 + trust_remote_code=True)

180 +outputs = llm.embed(["Follow the white rabbit."],

181 + pooling_params=PoolingParams(dimensions=32))

182 print(outputs[0].outputs)

183 ```

184

examples/offline_inference/basic/classify.py

+2 -2

@@ -28,10 +28,10 @@ def main(args: Namespace):

28

29 # Create an LLM.

30 # You should pass task="classify" for classification models

31 - model = LLM(**vars(args))

31 + llm = LLM(**vars(args))

32

33 # Generate logits. The output is a list of ClassificationRequestOutputs.

34 - outputs = model.classify(prompts)

34 + outputs = llm.classify(prompts)

35

36 # Print the outputs.

37 print("\nGenerated Outputs:\n" + "-" * 60)

examples/offline_inference/basic/embed.py

+2 -2

@@ -31,10 +31,10 @@ def main(args: Namespace):

31

32 # Create an LLM.

33 # You should pass task="embed" for embedding models

34 - model = LLM(**vars(args))

34 + llm = LLM(**vars(args))

35

36 # Generate embedding. The output is a list of EmbeddingRequestOutputs.

37 - outputs = model.embed(prompts)

37 + outputs = llm.embed(prompts)

38

39 # Print the outputs.

40 print("\nGenerated Outputs:\n" + "-" * 60)

examples/offline_inference/basic/score.py

+2 -2

@@ -27,10 +27,10 @@ def main(args: Namespace):

27

28 # Create an LLM.

29 # You should pass task="score" for cross-encoder models

30 - model = LLM(**vars(args))

30 + llm = LLM(**vars(args))

31

32 # Generate scores. The output is a list of ScoringRequestOutputs.

33 - outputs = model.score(text_1, texts_2)

33 + outputs = llm.score(text_1, texts_2)

34

35 # Print the outputs.

36 print("\nGenerated Outputs:\n" + "-" * 60)

examples/offline_inference/embed_jina_embeddings_v3.py

+2 -2

@@ -30,11 +30,11 @@ def main(args: Namespace):

30

31 # Create an LLM.

32 # You should pass task="embed" for embedding models

33 - model = LLM(**vars(args))

33 + llm = LLM(**vars(args))

34

35 # Generate embedding. The output is a list of EmbeddingRequestOutputs.

36 # Only text matching task is supported for now. See #16120

37 - outputs = model.embed(prompts)

37 + outputs = llm.embed(prompts)

38

39 # Print the outputs.

40 print("\nGenerated Outputs:")

examples/offline_inference/embed_matryoshka_fy.py

+2 -2

@@ -30,10 +30,10 @@ def main(args: Namespace):

30

31 # Create an LLM.

32 # You should pass task="embed" for embedding models

33 - model = LLM(**vars(args))

33 + llm = LLM(**vars(args))

34

35 # Generate embedding. The output is a list of EmbeddingRequestOutputs.

36 - outputs = model.embed(prompts, pooling_params=PoolingParams(dimensions=32))

36 + outputs = llm.embed(prompts, pooling_params=PoolingParams(dimensions=32))

37

38 # Print the outputs.

39 print("\nGenerated Outputs:")

examples/offline_inference/neuron_speculation.py

+6 -6

@@ -25,7 +25,7 @@ def config_buckets():

25 os.environ["NEURON_TOKEN_GEN_BUCKETS"] = "128,512,1024,2048"

26

27

28 -def initialize_model():

28 +def initialize_llm():

29 """Create an LLM with speculative decoding."""

30 return LLM(

31 model="openlm-research/open_llama_7b",

@@ -43,9 +43,9 @@ def initialize_model():

43 )

44

45

46 -def process_requests(model: LLM, sampling_params: SamplingParams):

46 +def process_requests(llm: LLM, sampling_params: SamplingParams):

47 """Generate texts from prompts and print them."""

48 - outputs = model.generate(prompts, sampling_params)

48 + outputs = llm.generate(prompts, sampling_params)

49 for output in outputs:

50 prompt = output.prompt

51 generated_text = output.outputs[0].text

@@ -53,12 +53,12 @@ def process_requests(model: LLM, sampling_params: SamplingParams):

53

54

55 def main():

56 - """Main function that sets up the model and processes prompts."""

56 + """Main function that sets up the llm and processes prompts."""

57 config_buckets()

58 - model = initialize_model()

58 + llm = initialize_llm()

59 # Create a sampling params object.

60 sampling_params = SamplingParams(max_tokens=100, top_k=1)

61 - process_requests(model, sampling_params)

61 + process_requests(llm, sampling_params)

62

63

64 if __name__ == "__main__":

examples/offline_inference/prithvi_geospatial_mae.py

+2 -2

@@ -140,7 +140,7 @@ datamodule_config = {

140 class PrithviMAE:

141 def __init__(self):

142 print("Initializing PrithviMAE model")

143 - self.model = LLM(

143 + self.llm = LLM(

144 model=os.path.join(os.path.dirname(__file__), "./model"),

145 skip_tokenizer_init=True,

146 dtype="float32",

@@ -158,7 +158,7 @@ class PrithviMAE:

158

159 prompt = {"prompt_token_ids": [1], "multi_modal_data": mm_data}

160

161 - outputs = self.model.encode(prompt, use_tqdm=False)

161 + outputs = self.llm.encode(prompt, use_tqdm=False)

162 print("################ Inference done (it took seconds) ##############")

163

164 return outputs[0].outputs.data

examples/offline_inference/qwen3_reranker.py

+4 -4

@@ -17,13 +17,13 @@ model_name = "Qwen/Qwen3-Reranker-0.6B"

17 # Models converted offline using this method can not only be more efficient

18 # and support the vllm score API, but also make the init parameters more

19 # concise, for example.

20 -# model = LLM(model="tomaarsen/Qwen3-Reranker-0.6B-seq-cls", task="score")

20 +# llm = LLM(model="tomaarsen/Qwen3-Reranker-0.6B-seq-cls", task="score")

21

22 # If you want to load the official original version, the init parameters are

23 # as follows.

24

25

26 -def get_model() -> LLM:

26 +def get_llm() -> LLM:

27 """Initializes and returns the LLM model for Qwen3-Reranker."""

28 return LLM(

29 model=model_name,

@@ -77,8 +77,8 @@ def main() -> None:

77 ]

78 documents = [document_template.format(doc=doc, suffix=suffix) for doc in documents]

79

80 - model = get_model()

81 - outputs = model.score(queries, documents)

80 + llm = get_llm()

81 + outputs = llm.score(queries, documents)

82

83 print("-" * 30)

84 print([output.outputs.score for output in outputs])

tests/basic_correctness/test_basic_correctness.py

+2 -2

@@ -236,13 +236,13 @@ def test_failed_model_execution(vllm_runner, monkeypatch) -> None:

236 monkeypatch.setenv('VLLM_ENABLE_V1_MULTIPROCESSING', '0')

237

238 with vllm_runner('facebook/opt-125m', enforce_eager=True) as vllm_model:

239 - if isinstance(vllm_model.model.llm_engine, LLMEngineV1):

239 + if isinstance(vllm_model.llm.llm_engine, LLMEngineV1):

240 v1_test_failed_model_execution(vllm_model)

241

242

243 def v1_test_failed_model_execution(vllm_model):

244

245 - engine = vllm_model.model.llm_engine

245 + engine = vllm_model.llm.llm_engine

246 mocked_execute_model = Mock(

247 side_effect=RuntimeError("Mocked Critical Error"))

248 engine.engine_core.engine_core.model_executor.execute_model =\

tests/basic_correctness/test_preemption.py

+5 -5

@@ -81,7 +81,7 @@ def test_chunked_prefill_recompute(

81 disable_log_stats=False,

82 ) as vllm_model:

83 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)

84 - assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt

84 + assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt

85 < ARTIFICIAL_PREEMPTION_MAX_CNT)

86

87 for i in range(len(example_prompts)):

@@ -118,10 +118,10 @@ def test_preemption(

118 distributed_executor_backend=distributed_executor_backend,

119 ) as vllm_model:

120 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)

121 - assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt

121 + assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt

122 < ARTIFICIAL_PREEMPTION_MAX_CNT)

123 total_preemption = (

124 - vllm_model.model.llm_engine.scheduler[0].num_cumulative_preemption)

124 + vllm_model.llm.llm_engine.scheduler[0].num_cumulative_preemption)

125

126 check_outputs_equal(

127 outputs_0_lst=hf_outputs,

@@ -174,12 +174,12 @@ def test_preemption_infeasible(

174 ) as vllm_model:

175 sampling_params = SamplingParams(max_tokens=max_tokens,

176 ignore_eos=True)

177 - req_outputs = vllm_model.model.generate(

177 + req_outputs = vllm_model.llm.generate(

178 example_prompts,

179 sampling_params=sampling_params,

180 )

181

182 - assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt

182 + assert (vllm_model.llm.llm_engine.scheduler[0].artificial_preempt_cnt

183 < ARTIFICIAL_PREEMPTION_MAX_CNT)

184

185 # Verify the request is ignored and not hang.

tests/conftest.py

+16 -16

@@ -784,7 +784,7 @@ class VllmRunner:

784 enforce_eager: Optional[bool] = False,

785 **kwargs,

786 ) -> None:

787 - self.model = LLM(

787 + self.llm = LLM(

788 model=model_name,

789 task=task,

790 tokenizer=tokenizer_name,

@@ -854,9 +854,9 @@ class VllmRunner:

854 videos=videos,

855 audios=audios)

856

857 - req_outputs = self.model.generate(inputs,

858 - sampling_params=sampling_params,

859 - **kwargs)

857 + req_outputs = self.llm.generate(inputs,

858 + sampling_params=sampling_params,

859 + **kwargs)

860

861 outputs: list[tuple[list[list[int]], list[str]]] = []

862 for req_output in req_outputs:

@@ -902,9 +902,9 @@ class VllmRunner:

902 videos=videos,

903 audios=audios)

904

905 - req_outputs = self.model.generate(inputs,

906 - sampling_params=sampling_params,

907 - **kwargs)

905 + req_outputs = self.llm.generate(inputs,

906 + sampling_params=sampling_params,

907 + **kwargs)

908

909 toks_str_logsprobs_prompt_logprobs = (

910 self._final_steps_generate_w_logprobs(req_outputs))

@@ -924,8 +924,8 @@ class VllmRunner:

924 '''

925

926 assert sampling_params.logprobs is not None

927 - req_outputs = self.model.generate(encoder_decoder_prompts,

928 - sampling_params=sampling_params)

927 + req_outputs = self.llm.generate(encoder_decoder_prompts,

928 + sampling_params=sampling_params)

929 toks_str_logsprobs_prompt_logprobs = (

930 self._final_steps_generate_w_logprobs(req_outputs))

931 # Omit prompt logprobs if not required by sampling params

@@ -1018,7 +1018,7 @@ class VllmRunner:

1018 videos=videos,

1019 audios=audios)

1020

1021 - outputs = self.model.beam_search(

1021 + outputs = self.llm.beam_search(

1022 inputs,

1023 BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens))

1024 returned_outputs = []

@@ -1029,7 +1029,7 @@ class VllmRunner:

1029 return returned_outputs

1030

1031 def classify(self, prompts: list[str]) -> list[list[float]]:

1032 - req_outputs = self.model.classify(prompts)

1032 + req_outputs = self.llm.classify(prompts)

1033 return [req_output.outputs.probs for req_output in req_outputs]

1034

1035 def embed(self,

@@ -1044,11 +1044,11 @@ class VllmRunner:

1044 videos=videos,

1045 audios=audios)

1046

1047 - req_outputs = self.model.embed(inputs, *args, **kwargs)

1047 + req_outputs = self.llm.embed(inputs, *args, **kwargs)

1048 return [req_output.outputs.embedding for req_output in req_outputs]

1049

1050 def encode(self, prompts: list[str]) -> list[list[float]]:

1051 - req_outputs = self.model.encode(prompts)

1051 + req_outputs = self.llm.encode(prompts)

1052 return [req_output.outputs.data for req_output in req_outputs]

1053

1054 def score(

@@ -1058,18 +1058,18 @@ class VllmRunner:

1058 *args,

1059 **kwargs,

1060 ) -> list[float]:

1061 - req_outputs = self.model.score(text_1, text_2, *args, **kwargs)

1061 + req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)

1062 return [req_output.outputs.score for req_output in req_outputs]

1063

1064 def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:

1065 - executor = self.model.llm_engine.model_executor

1065 + executor = self.llm.llm_engine.model_executor

1066 return executor.apply_model(func)

1067

1068 def __enter__(self):

1069 return self

1070

1071 def __exit__(self, exc_type, exc_value, traceback):

1072 - del self.model

1072 + del self.llm

1073 cleanup_dist_env_and_memory()

1074

1075

tests/core/test_num_computed_tokens_update.py

+1 -1

@@ -37,7 +37,7 @@ def test_num_computed_tokens_update(num_scheduler_steps: int,

37 num_scheduler_steps=num_scheduler_steps,

38 enable_chunked_prefill=enable_chunked_prefill,

39 enforce_eager=enforce_eager)

40 - engine: LLMEngine = runner.model.llm_engine

40 + engine: LLMEngine = runner.llm.llm_engine

41

42 # In multi-step + chunked-prefill there is no separate single prompt step.

43 # What is scheduled will run for num_scheduler_steps always.

tests/detokenizer/test_stop_reason.py

+1 -1

@@ -28,7 +28,7 @@ def vllm_model(vllm_runner):

28 def test_stop_reason(vllm_model, example_prompts):

29 tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)

30 stop_token_id = tokenizer.convert_tokens_to_ids(STOP_STR)

31 - llm = vllm_model.model

31 + llm = vllm_model.llm

32

33 # test stop token

34 outputs = llm.generate(example_prompts,

tests/detokenizer/test_stop_strings.py

+21 -21

@@ -101,42 +101,42 @@ def _stop_token_id(llm):

101 def test_stop_strings():

102 # If V0, must set enforce_eager=False since we use

103 # async output processing below.

104 - vllm_model = LLM(MODEL, enforce_eager=envs.VLLM_USE_V1)

104 + llm = LLM(MODEL, enforce_eager=envs.VLLM_USE_V1)

105

106 if envs.VLLM_USE_V1:

107 - _stop_basic(vllm_model)

107 + _stop_basic(llm)

108 else:

109 - _set_async_mode(vllm_model, True)

110 - _stop_basic(vllm_model)

109 + _set_async_mode(llm, True)

110 + _stop_basic(llm)

111

112 - _set_async_mode(vllm_model, False)

113 - _stop_basic(vllm_model)

112 + _set_async_mode(llm, False)

113 + _stop_basic(llm)

114

115 if envs.VLLM_USE_V1:

116 - _stop_multi_tokens(vllm_model)

116 + _stop_multi_tokens(llm)

117 else:

118 - _set_async_mode(vllm_model, True)

119 - _stop_multi_tokens(vllm_model)

118 + _set_async_mode(llm, True)

119 + _stop_multi_tokens(llm)

120

121 - _set_async_mode(vllm_model, False)

122 - _stop_multi_tokens(vllm_model)

121 + _set_async_mode(llm, False)

122 + _stop_multi_tokens(llm)

123

124 if envs.VLLM_USE_V1:

125 - _stop_partial_token(vllm_model)

125 + _stop_partial_token(llm)

126 else:

127 - _set_async_mode(vllm_model, True)

128 - _stop_partial_token(vllm_model)

127 + _set_async_mode(llm, True)

128 + _stop_partial_token(llm)

129

130 - _set_async_mode(vllm_model, False)

131 - _stop_partial_token(vllm_model)

130 + _set_async_mode(llm, False)

131 + _stop_partial_token(llm)

132

133 if envs.VLLM_USE_V1:

134 # FIXME: this does not respect include_in_output=False

135 - # _stop_token_id(vllm_model)

135 + # _stop_token_id(llm)

136 pass

137 else:

138 - _set_async_mode(vllm_model, True)

139 - _stop_token_id(vllm_model)

138 + _set_async_mode(llm, True)

139 + _stop_token_id(llm)

140

141 - _set_async_mode(vllm_model, False)

142 - _stop_token_id(vllm_model)

141 + _set_async_mode(llm, False)

142 + _stop_token_id(llm)

tests/lora/test_llama_tp.py

+10 -10

@@ -186,25 +186,25 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,

186 model_uri = tmp_path / "vllm" / model_ref / suffix / model_name

187 tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))

188

189 - loaded_vllm_model = LLM(model=model_ref,

190 - load_format="tensorizer",

191 - enable_lora=True,

192 - enforce_eager=True,

193 - model_loader_extra_config=tensorizer_config,

194 - max_num_seqs=13,

195 - tensor_parallel_size=2,

196 - max_loras=2)

189 + loaded_llm = LLM(model=model_ref,

190 + load_format="tensorizer",

191 + enable_lora=True,

192 + enforce_eager=True,

193 + model_loader_extra_config=tensorizer_config,

194 + max_num_seqs=13,

195 + tensor_parallel_size=2,

196 + max_loras=2)

197

198 tc_as_dict = tensorizer_config.to_serializable()

199

200 print("lora adapter created")

201 - assert do_sample(loaded_vllm_model,

201 + assert do_sample(loaded_llm,

202 sql_lora_files,

203 tensorizer_config_dict=tc_as_dict,

204 lora_id=0) == EXPECTED_NO_LORA_OUTPUT

205

206 print("lora 1")

207 - assert do_sample(loaded_vllm_model,

207 + assert do_sample(loaded_llm,

208 sql_lora_files,

209 tensorizer_config_dict=tc_as_dict,

210 lora_id=1) == EXPECTED_LORA_OUTPUT

tests/metrics/test_metrics.py

+7 -7

@@ -41,7 +41,7 @@ def test_metric_counter_prompt_tokens(

41 dtype=dtype,

42 disable_log_stats=False,

43 gpu_memory_utilization=0.4) as vllm_model:

44 - tokenizer = vllm_model.model.get_tokenizer()

44 + tokenizer = vllm_model.llm.get_tokenizer()

45 prompt_token_counts = [

46 len(tokenizer.encode(p)) for p in example_prompts

47 ]

@@ -53,7 +53,7 @@ def test_metric_counter_prompt_tokens(

53 vllm_prompt_token_count = sum(prompt_token_counts)

54

55 _ = vllm_model.generate_greedy(example_prompts, max_tokens)

56 - stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']

56 + stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']

57 metric_count = stat_logger.metrics.counter_prompt_tokens.labels(

58 **stat_logger.labels)._value.get()

59

@@ -77,8 +77,8 @@ def test_metric_counter_generation_tokens(

77 disable_log_stats=False,

78 gpu_memory_utilization=0.4) as vllm_model:

79 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)

80 - tokenizer = vllm_model.model.get_tokenizer()

81 - stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']

80 + tokenizer = vllm_model.llm.get_tokenizer()

81 + stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']

82 metric_count = stat_logger.metrics.counter_generation_tokens.labels(

83 **stat_logger.labels)._value.get()

84 vllm_generation_count = 0

@@ -113,8 +113,8 @@ def test_metric_counter_generation_tokens_multi_step(

113 disable_async_output_proc=disable_async_output_proc,

114 ) as vllm_model:

115 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)

116 - tokenizer = vllm_model.model.get_tokenizer()

117 - stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']

116 + tokenizer = vllm_model.llm.get_tokenizer()

117 + stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']

118 metric_count = stat_logger.metrics.counter_generation_tokens.labels(

119 **stat_logger.labels)._value.get()

120 vllm_generation_count = 0

@@ -145,7 +145,7 @@ def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,

145 disable_log_stats=False,

146 gpu_memory_utilization=0.3,

147 served_model_name=served_model_name) as vllm_model:

148 - stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']

148 + stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']

149 metrics_tag_content = stat_logger.labels["model_name"]

150

151 if envs.VLLM_CI_USE_S3:

tests/model_executor/test_model_load_with_params.py

+5 -5

@@ -32,8 +32,8 @@ def test_model_loading_with_params(vllm_runner):

32 output = vllm_model.embed("Write a short story about a robot that"

33 " dreams for the first time.\n")

34

35 - model_config = vllm_model.model.llm_engine.model_config

36 - model_tokenizer = vllm_model.model.llm_engine.tokenizer

35 + model_config = vllm_model.llm.llm_engine.model_config

36 + model_tokenizer = vllm_model.llm.llm_engine.tokenizer

37

38 # asserts on the bert model config file

39 assert model_config.encoder_config["max_seq_length"] == 512

@@ -70,8 +70,8 @@ def test_roberta_model_loading_with_params(vllm_runner):

70 output = vllm_model.embed("Write a short story about a robot that"

71 " dreams for the first time.\n")

72

73 - model_config = vllm_model.model.llm_engine.model_config

74 - model_tokenizer = vllm_model.model.llm_engine.tokenizer

73 + model_config = vllm_model.llm.llm_engine.model_config

74 + model_tokenizer = vllm_model.llm.llm_engine.tokenizer

75

76 # asserts on the bert model config file

77 assert model_config.encoder_config["max_seq_length"] == 512

@@ -108,7 +108,7 @@ def test_facebook_roberta_model_loading_with_params(vllm_runner):

108 output = vllm_model.embed("Write a short story about a robot that"

109 " dreams for the first time.\n")

110

111 - model_tokenizer = vllm_model.model.llm_engine.tokenizer

111 + model_tokenizer = vllm_model.llm.llm_engine.tokenizer

112 assert model_tokenizer.tokenizer_id == model_name

113

114 def check_model(model):

tests/models/language/generation/test_hybrid.py

+1 -1

@@ -274,7 +274,7 @@ def test_models_preemption_recompute(

274 Tests that outputs are identical with and w/o preemptions (recompute).

275 """

276 with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:

277 - scheduler = vllm_model.model.llm_engine.scheduler[0]

277 + scheduler = vllm_model.llm.llm_engine.scheduler[0]

278 scheduler.ENABLE_ARTIFICIAL_PREEMPT = True

279 preempt_vllm_outputs = vllm_model.generate_greedy(

280 example_prompts, max_tokens)

tests/models/language/generation/test_mistral.py

+7 -7

@@ -238,8 +238,8 @@ def test_mistral_symbolic_languages(vllm_runner, model: str,

238 load_format="mistral") as vllm_model:

239 for prompt in SYMBOLIC_LANG_PROMPTS:

240 msg = {"role": "user", "content": prompt}

241 - outputs = vllm_model.model.chat([msg],

242 - sampling_params=SAMPLING_PARAMS)

241 + outputs = vllm_model.llm.chat([msg],

242 + sampling_params=SAMPLING_PARAMS)

243 assert "�" not in outputs[0].outputs[0].text.strip()

244

245

@@ -253,11 +253,11 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:

253 load_format="mistral") as vllm_model:

254

255 msgs = copy.deepcopy(MSGS)

256 - outputs = vllm_model.model.chat(msgs,

257 - tools=TOOLS,

258 - sampling_params=SAMPLING_PARAMS)

256 + outputs = vllm_model.llm.chat(msgs,

257 + tools=TOOLS,

258 + sampling_params=SAMPLING_PARAMS)

259

260 - tokenizer = vllm_model.model.get_tokenizer()

260 + tokenizer = vllm_model.llm.get_tokenizer()

261 tool_parser = MistralToolParser(tokenizer)

262

263 model_output = outputs[0].outputs[0].text.strip()

@@ -308,7 +308,7 @@ def test_mistral_guided_decoding(

308 f"Give an example JSON for an employee profile that "

309 f"fits this schema: {SAMPLE_JSON_SCHEMA}"

310 }]

311 - outputs = vllm_model.model.chat(messages, sampling_params=params)

311 + outputs = vllm_model.llm.chat(messages, sampling_params=params)

312

313 generated_text = outputs[0].outputs[0].text

314 json_response = json.loads(generated_text)

tests/models/language/pooling/mteb_utils.py

+9 -9

@@ -30,7 +30,7 @@ class VllmMtebEncoder(mteb.Encoder):

30

31 def __init__(self, vllm_model):

32 super().__init__()

33 - self.model = vllm_model

33 + self.llm = vllm_model

34 self.rng = np.random.default_rng(seed=42)

35

36 def encode(

@@ -43,7 +43,7 @@ class VllmMtebEncoder(mteb.Encoder):

43 # issues by randomizing the order.

44 r = self.rng.permutation(len(sentences))

45 sentences = [sentences[i] for i in r]

46 - outputs = self.model.embed(sentences, use_tqdm=False)

46 + outputs = self.llm.embed(sentences, use_tqdm=False)

47 embeds = np.array(outputs)

48 embeds = embeds[np.argsort(r)]

49 return embeds

@@ -61,10 +61,10 @@ class VllmMtebEncoder(mteb.Encoder):

61 queries = [s[0] for s in sentences]

62 corpus = [s[1] for s in sentences]

63

64 - outputs = self.model.score(queries,

65 - corpus,

66 - truncate_prompt_tokens=-1,

67 - use_tqdm=False)

64 + outputs = self.llm.score(queries,

65 + corpus,

66 + truncate_prompt_tokens=-1,

67 + use_tqdm=False)

68 scores = np.array(outputs)

69 scores = scores[np.argsort(r)]

70 return scores

@@ -178,11 +178,11 @@ def mteb_test_embed_models(hf_runner,

178

179 if model_info.architecture:

180 assert (model_info.architecture

181 - in vllm_model.model.llm_engine.model_config.architectures)

181 + in vllm_model.llm.llm_engine.model_config.architectures)

182

183 vllm_main_score = run_mteb_embed_task(VllmMtebEncoder(vllm_model),

184 MTEB_EMBED_TASKS)

185 - vllm_dtype = vllm_model.model.llm_engine.model_config.dtype

185 + vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype

186

187 with hf_runner(model_info.name,

188 is_sentence_transformer=True,

@@ -284,7 +284,7 @@ def mteb_test_rerank_models(hf_runner,

284 max_num_seqs=8,

285 **vllm_extra_kwargs) as vllm_model:

286

287 - model_config = vllm_model.model.llm_engine.model_config

287 + model_config = vllm_model.llm.llm_engine.model_config

288

289 if model_info.architecture:

290 assert (model_info.architecture in model_config.architectures)

tests/models/language/pooling/test_gritlm.py

+2 -2

@@ -120,7 +120,7 @@ def test_gritlm_offline_embedding(vllm_runner):

120 task="embed",

121 max_model_len=MAX_MODEL_LEN,

122 ) as vllm_model:

123 - llm = vllm_model.model

123 + llm = vllm_model.llm

124

125 d_rep = run_llm_encode(

126 llm,

@@ -167,7 +167,7 @@ def test_gritlm_offline_generate(monkeypatch: pytest.MonkeyPatch, vllm_runner):

167 task="generate",

168 max_model_len=MAX_MODEL_LEN,

169 ) as vllm_model:

170 - llm = vllm_model.model

170 + llm = vllm_model.llm

171

172 sampling_params = SamplingParams(temperature=0.0, max_tokens=256)

173 outputs = llm.generate(input, sampling_params=sampling_params)

tests/models/language/pooling/test_jina.py

+2 -2

@@ -87,10 +87,10 @@ def test_matryoshka(

87 task="embed",

88 dtype=dtype,

89 max_model_len=None) as vllm_model:

90 - assert vllm_model.model.llm_engine.model_config.is_matryoshka

90 + assert vllm_model.llm.llm_engine.model_config.is_matryoshka

91

92 matryoshka_dimensions = (

93 - vllm_model.model.llm_engine.model_config.matryoshka_dimensions)

93 + vllm_model.llm.llm_engine.model_config.matryoshka_dimensions)

94 assert matryoshka_dimensions is not None

95

96 if dimensions not in matryoshka_dimensions:

tests/models/language/pooling/test_nomic_max_model_len.py

+3 -3

@@ -23,7 +23,7 @@ max_model_len = int(original_max_position_embeddings * factor)

23 def test_default(model_info, vllm_runner):

24 with vllm_runner(model_info.name, task="embed",

25 max_model_len=None) as vllm_model:

26 - model_config = vllm_model.model.llm_engine.model_config

26 + model_config = vllm_model.llm.llm_engine.model_config

27 if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":

28 # For nomic-embed-text-v2-moe the length is set to 512

29 # by sentence_bert_config.json.

@@ -38,7 +38,7 @@ def test_set_max_model_len_legal(model_info, vllm_runner):

38 # set max_model_len <= 512

39 with vllm_runner(model_info.name, task="embed",

40 max_model_len=256) as vllm_model:

41 - model_config = vllm_model.model.llm_engine.model_config

41 + model_config = vllm_model.llm.llm_engine.model_config

42 assert model_config.max_model_len == 256

43

44 # set 512 < max_model_len <= 2048

@@ -52,7 +52,7 @@ def test_set_max_model_len_legal(model_info, vllm_runner):

52 else:

53 with vllm_runner(model_info.name, task="embed",

54 max_model_len=1024) as vllm_model:

55 - model_config = vllm_model.model.llm_engine.model_config

55 + model_config = vllm_model.llm.llm_engine.model_config

56 assert model_config.max_model_len == 1024

57

58

tests/models/language/pooling/test_truncation_control.py

+3 -3

@@ -28,7 +28,7 @@ def test_smaller_truncation_size(vllm_runner,

28

29 with vllm_runner(model_name, task="embed",

30 max_model_len=max_model_len) as vllm_model:

31 - vllm_output = vllm_model.model.encode(

31 + vllm_output = vllm_model.llm.encode(

32 input_str, truncate_prompt_tokens=truncate_prompt_tokens)

33

34 prompt_tokens = vllm_output[0].prompt_token_ids

@@ -43,7 +43,7 @@ def test_max_truncation_size(vllm_runner,

43

44 with vllm_runner(model_name, task="embed",

45 max_model_len=max_model_len) as vllm_model:

46 - vllm_output = vllm_model.model.encode(

46 + vllm_output = vllm_model.llm.encode(

47 input_str, truncate_prompt_tokens=truncate_prompt_tokens)

48

49 prompt_tokens = vllm_output[0].prompt_token_ids

@@ -61,7 +61,7 @@ def test_bigger_truncation_size(vllm_runner,

61 model_name, task="embed",

62 max_model_len=max_model_len) as vllm_model:

63

64 - llm_output = vllm_model.model.encode(

64 + llm_output = vllm_model.llm.encode(

65 input_str, truncate_prompt_tokens=truncate_prompt_tokens)

66

67 assert llm_output == f"""truncate_prompt_tokens value

tests/models/multimodal/generation/test_pixtral.py

+2 -3

@@ -180,8 +180,7 @@ def test_chat(

180 ) as vllm_model:

181 outputs = []

182 for msg in MSGS:

183 - output = vllm_model.model.chat(msg,

184 - sampling_params=SAMPLING_PARAMS)

183 + output = vllm_model.llm.chat(msg, sampling_params=SAMPLING_PARAMS)

184

185 outputs.extend(output)

186

@@ -217,7 +216,7 @@ def test_multi_modal_placeholders(vllm_runner, prompt,

216 max_model_len=8192,

217 limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,

218 ) as vllm_model:

220 - outputs = vllm_model.model.generate(prompt)

219 + outputs = vllm_model.llm.generate(prompt)

220

221 assert len(outputs) == 1, f"{len(outputs)=}"

222 output: RequestOutput = outputs[0]

tests/models/multimodal/generation/test_whisper.py

+1 -1

@@ -106,7 +106,7 @@ def run_test(

106 tensor_parallel_size=tensor_parallel_size,

107 distributed_executor_backend=distributed_executor_backend,

108 ) as vllm_model:

109 - llm = vllm_model.model

109 + llm = vllm_model.llm

110

111 sampling_params = SamplingParams(

112 temperature=0,

tests/models/multimodal/generation/vlm_utils/core.py

+1 -1

@@ -85,7 +85,7 @@ def run_test(

85 enforce_eager=enforce_eager,

86 task=task,

87 **vllm_runner_kwargs_) as vllm_model:

88 - tokenizer = vllm_model.model.get_tokenizer()

88 + tokenizer = vllm_model.llm.get_tokenizer()

89

90 vllm_kwargs: dict[str, Any] = {}

91 if get_stop_token_ids is not None:

tests/models/multimodal/pooling/test_dse_qwen2_vl.py

+1 -1

@@ -96,7 +96,7 @@ def _run_test(

96 dtype=dtype,

97 enforce_eager=True,

98 max_model_len=8192) as vllm_model:

99 - tokenizer = vllm_model.model.get_tokenizer()

99 + tokenizer = vllm_model.llm.get_tokenizer()

100 texts = [

101 # this is necessary because vllm_model.embed will not apply any

102 # templating to the prompt, and therefore lacks an image_pad

tests/models/multimodal/pooling/test_jinavl_reranker.py

+1 -1

@@ -56,7 +56,7 @@ def vllm_reranker(

56 mm_processor_kwargs=mm_processor_kwargs,

57 limit_mm_per_prompt=limit_mm_per_prompt,

58 ) as vllm_model:

59 - outputs = vllm_model.model.score(query, documents)

59 + outputs = vllm_model.llm.score(query, documents)

60

61 return [output.outputs.score for output in outputs]

62

tests/models/quantization/test_modelopt.py

+3 -3

@@ -45,7 +45,7 @@ EXPECTED_STRS_MAP = {

45 reason="fp8 is not supported on this GPU type.")

46 @pytest.mark.parametrize("model_name", MODELS)

47 def test_models(example_prompts, model_name) -> None:

48 - model = LLM(

48 + llm = LLM(

49 model=model_name,

50 max_model_len=MAX_MODEL_LEN,

51 trust_remote_code=True,

@@ -68,9 +68,9 @@ def test_models(example_prompts, model_name) -> None:

68 # Note: these need to be run 1 at a time due to numerical precision,

69 # since the expected strs were generated this way.

70 for prompt in formatted_prompts:

71 - outputs = model.generate(prompt, params)

71 + outputs = llm.generate(prompt, params)

72 generations.append(outputs[0].outputs[0].text)

73 - del model

73 + del llm

74

75 print(model_name, generations)

76 expected_strs = EXPECTED_STRS_MAP[model_name]

tests/models/quantization/test_nvfp4.py

+3 -3

@@ -46,7 +46,7 @@ EXPECTED_STRS_MAP = {

46 reason="modelopt_fp4 is not supported on this GPU type.")

47 @pytest.mark.parametrize("model_name", MODELS)

48 def test_models(example_prompts, model_name) -> None:

49 - model = LLM(

49 + llm = LLM(

50 model=model_name,

51 max_model_len=MAX_MODEL_LEN,

52 trust_remote_code=True,

@@ -69,9 +69,9 @@ def test_models(example_prompts, model_name) -> None:

69 # Note: these need to be run 1 at a time due to numerical precision,

70 # since the expected strs were generated this way.

71 for prompt in formatted_prompts:

72 - outputs = model.generate(prompt, params)

72 + outputs = llm.generate(prompt, params)

73 generations.append(outputs[0].outputs[0].text)

74 - del model

74 + del llm

75

76 print(model_name, generations)

77 expected_strs = EXPECTED_STRS_MAP[model_name]

tests/prefix_caching/test_disable_sliding_window.py

+11 -11

@@ -25,25 +25,25 @@ MODEL_LEN_LEN = [

25 @pytest.mark.parametrize("model_len_len", MODEL_LEN_LEN)

26 def test_disable_sliding_window(model_len_len, ):

27 model, sliding_len, full_len = model_len_len

28 - vllm_disabled_model = LLM(model, disable_sliding_window=True)

29 - vllm_disabled_model.generate("Hi my name is")

30 - model_config = vllm_disabled_model.llm_engine.model_config

28 + disabled_llm = LLM(model, disable_sliding_window=True)

29 + disabled_llm.generate("Hi my name is")

30 + model_config = disabled_llm.llm_engine.model_config

31 assert model_config.max_model_len == sliding_len, (

32 "Max len expected to equal sliding_len of %s, but got %s", sliding_len,

33 model_config.max_model_len)

34

35 - del vllm_disabled_model

35 + del disabled_llm

36 cleanup_dist_env_and_memory()

37

38 - vllm_enabled_model = LLM(model,

39 - enforce_eager=True,

40 - disable_sliding_window=False,

41 - enable_prefix_caching=False)

42 - vllm_enabled_model.generate("Hi my name is")

43 - model_config = vllm_enabled_model.llm_engine.model_config

38 + enabled_llm = LLM(model,

39 + enforce_eager=True,

40 + disable_sliding_window=False,

41 + enable_prefix_caching=False)

42 + enabled_llm.generate("Hi my name is")

43 + model_config = enabled_llm.llm_engine.model_config

44 assert model_config.max_model_len == full_len, (

45 "Max len expected to equal full_len of %s, but got %s", full_len,

46 model_config.max_model_len)

47

48 - del vllm_enabled_model

48 + del enabled_llm

49 cleanup_dist_env_and_memory()

tests/prefix_caching/test_prefix_caching.py

+3 -3

@@ -93,8 +93,8 @@ def test_mixed_requests(

93 # Run all the promopts

94 greedy_params = SamplingParams(temperature=0.0,

95 max_tokens=max_tokens)

96 - req_outputs = vllm_model.model.generate(example_prompts,

97 - greedy_params)

96 + req_outputs = vllm_model.llm.generate(example_prompts,

97 + greedy_params)

98

99 # Verify number of cached tokens

100 for i in range(len(req_outputs)):

@@ -161,7 +161,7 @@ def test_fully_cached_prefill_needs_uncached_token(model):

161 max_num_batched_tokens=max_num_batched_tokens,

162 max_num_seqs=max_num_batched_tokens,

163 )

164 - engine: LLMEngine = runner.model.llm_engine

164 + engine: LLMEngine = runner.llm.llm_engine

165

166 scheduler: Scheduler = SchedulerProxy(engine.scheduler[0]) # type: ignore

167 engine.scheduler[0] = scheduler

tests/quantization/test_gptq_dynamic.py

+1 -1

@@ -39,7 +39,7 @@ def test_gptq_with_dynamic(vllm_runner, model_id: str, use_marlin_kernel: bool,

39 linear_method_cls = GPTQMarlinLinearMethod if use_marlin_kernel else (

40 GPTQLinearMethod)

41

42 - for name, submodule in (vllm_model.model.llm_engine.model_executor.

42 + for name, submodule in (vllm_model.llm.llm_engine.model_executor.

43 driver_worker.model_runner.model.named_modules()):

44 if name == "lm_head":

45 assert isinstance(submodule.quant_method, linear_method_cls)

tests/quantization/test_quark.py

+2 -2

@@ -107,11 +107,11 @@ def test_quark_fp8_parity(vllm_runner):

107 }

108 with (vllm_runner(quark_model_id, **llm_kwargs) as

109 quark_handle, vllm_runner(fp8_model_id, **llm_kwargs) as fp8_handle):

110 - quark_model = (quark_handle.model.llm_engine.model_executor.

110 + quark_model = (quark_handle.llm.llm_engine.model_executor.

111 driver_worker.model_runner.model)

112 quark_state_dict = quark_model.state_dict()

113

114 - fp8_model = (fp8_handle.model.llm_engine.model_executor.driver_worker.

114 + fp8_model = (fp8_handle.llm.llm_engine.model_executor.driver_worker.

115 model_runner.model)

116 fp8_state_dict = fp8_model.state_dict()

117

tests/quantization/test_register_quantization_config.py

+1 -1

@@ -111,7 +111,7 @@ def test_custom_quant(vllm_runner, model, monkeypatch):

111 quantization="custom_quant",

112 enforce_eager=True) as llm:

113

114 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501

114 + model = llm.llm.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501

115 layer = model.model.layers[0]

116 qkv_proj = layer.self_attn.qkv_proj

117

tests/samplers/test_ignore_eos.py

+1 -1

@@ -36,7 +36,7 @@ def test_ignore_eos(

36 ignore_eos=True)

37

38 for prompt in example_prompts:

39 - ignore_eos_output = vllm_model.model.generate(

39 + ignore_eos_output = vllm_model.llm.generate(

40 prompt, sampling_params=sampling_params)

41 output_length = len(ignore_eos_output[0].outputs[0].token_ids)

42 assert output_length == max_tokens

tests/samplers/test_logits_processor.py

+5 -5

@@ -26,7 +26,7 @@ def test_logits_processor_force_generate(

26 dtype: str,

27 ) -> None:

28 with vllm_runner(model, dtype=dtype) as vllm_model:

29 - tokenizer = vllm_model.model.get_tokenizer()

29 + tokenizer = vllm_model.llm.get_tokenizer()

30 repeat_times = 2

31 enforced_answers = " vLLM"

32 vllm_token_ids = tokenizer.encode(enforced_answers,

@@ -45,13 +45,13 @@ def test_logits_processor_force_generate(

45 )

46

47 # test logits_processors when prompt_logprobs is not None

48 - vllm_model.model._add_request(

48 + vllm_model.llm._add_request(

49 example_prompts[0],

50 params=params_with_logprobs,

51 )

52

53 # test prompt_logprobs is not None

54 - vllm_model.model._add_request(

54 + vllm_model.llm._add_request(

55 example_prompts[1],

56 params=SamplingParams(

57 prompt_logprobs=3,

@@ -60,11 +60,11 @@ def test_logits_processor_force_generate(

60 )

61

62 # test grouped requests

63 - vllm_model.model._add_request(

63 + vllm_model.llm._add_request(

64 example_prompts[2],

65 params=SamplingParams(max_tokens=max_tokens),

66 )

67

68 - outputs = vllm_model.model._run_engine(use_tqdm=False)

68 + outputs = vllm_model.llm._run_engine(use_tqdm=False)

69

70 assert outputs[0].outputs[0].text == enforced_answers * repeat_times

tests/samplers/test_logprobs.py

+2 -2

@@ -64,7 +64,7 @@ def test_get_prompt_logprobs(

64 prompt_logprobs=num_top_logprobs,

65 temperature=0.0,

66 detokenize=detokenize)

67 - vllm_results = vllm_model.model.generate(

67 + vllm_results = vllm_model.llm.generate(

68 example_prompts, sampling_params=vllm_sampling_params)

69

70 # Test whether logprobs are included in the results.

@@ -174,7 +174,7 @@ def test_none_logprobs(vllm_runner, model, chunked_prefill_token_size: int,

174 logprobs=None,

175 temperature=0.0,

176 detokenize=detokenize)

177 - results_logprobs_none = vllm_model.model.generate(

177 + results_logprobs_none = vllm_model.llm.generate(

178 example_prompts, sampling_params=sampling_params_logprobs_none)

179

180 for i in range(len(results_logprobs_none)):

tests/samplers/test_no_bad_words.py

+6 -6

@@ -20,7 +20,7 @@ def v1(run_with_both_engines):

20

21

22 def _generate(

23 - model: LLM,

23 + llm: LLM,

24 prompt: str,

25 num_prompt_tokens: int,

26 temperature: float = 0,

@@ -32,7 +32,7 @@ def _generate(

32 )

33

34 # [([output_token_ids, ], [output_text, ]), ]

35 - output = model.generate([prompt], sampling_params=sampling_params)

35 + output = llm.generate([prompt], sampling_params=sampling_params)

36

37 output_token_ids = output[0][0][0][num_prompt_tokens:]

38 # [0] first (and only) request output

@@ -66,10 +66,10 @@ class TestOneTokenBadWord:

66 assert self.target_token_id not in output_token_ids

67

68 def _generate(self,

69 - model: LLM,

69 + llm: LLM,

70 bad_words: Optional[list[str]] = None) -> list[int]:

71 return _generate(

72 - model=model,

72 + llm=llm,

73 prompt=self.PROMPT,

74 num_prompt_tokens=self.num_prompt_tokens,

75 bad_words=bad_words,

@@ -156,10 +156,10 @@ class TestTwoTokenBadWord:

156 or (self.neighbour_token_id2 in output_token_ids))

157

158 def _generate(self,

159 - model: LLM,

159 + llm: LLM,

160 bad_words: Optional[list[str]] = None) -> list[int]:

161 return _generate(

162 - model=model,

162 + llm=llm,

163 prompt=self.PROMPT,

164 num_prompt_tokens=self.num_prompt_tokens,

165 bad_words=bad_words,

tests/samplers/test_seeded_generate.py

+1 -1

@@ -49,7 +49,7 @@ def test_random_sample_with_seed(

49 sampling_params_seed_2 = copy.deepcopy(sampling_params)

50 sampling_params_seed_2.seed = 200

51

52 - llm = vllm_model.model

52 + llm = vllm_model.llm

53

54 for prompt in example_prompts:

55 for params in (

tests/tokenization/test_detokenize.py

+1 -1

@@ -393,7 +393,7 @@ def test_decode_prompt_logprobs_chunked_prefill(

393 logprobs=5,

394 prompt_logprobs=5,

395 temperature=0.0)

396 - vllm_results = vllm_model.model.generate(

396 + vllm_results = vllm_model.llm.generate(

397 example_prompts, sampling_params=vllm_sampling_params)

398

399 for idx, result in enumerate(vllm_results):

tests/v1/core/test_scheduler_e2e.py

+6 -6

@@ -14,7 +14,7 @@ PROMPT = "Hello my name is Robert and I"

14

15

16 @pytest.fixture(scope="module")

17 -def model() -> LLM:

17 +def llm() -> LLM:

18 return LLM(MODEL,

19 enforce_eager=True,

20 enable_prefix_caching=True,

@@ -24,16 +24,16 @@ def model() -> LLM:

24 block_size=16)

25

26

27 -def test_concurrent_partial_prefill(model):

28 - outputs = model.generate([PROMPT] * 3)

27 +def test_concurrent_partial_prefill(llm):

28 + outputs = llm.generate([PROMPT] * 3)

29 assert len(outputs) == 3

30 for output in outputs:

31 assert len(output.outputs) == 1

32

33

34 -def test_prefix_cache_stats_is_recorded(model):

34 +def test_prefix_cache_stats_is_recorded(llm):

35 # 17 tokens will make sure first 16 tokens are cached in a block

36 input_tokens = {"prompt_token_ids": [101] * 17}

37 - _ = model.generate([input_tokens])

38 - outputs = model.generate([input_tokens])

37 + _ = llm.generate([input_tokens])

38 + outputs = llm.generate([input_tokens])

39 assert outputs[0].num_cached_tokens == 16

tests/v1/engine/test_llm_engine.py

+7 -7

@@ -112,9 +112,9 @@ def test_compatibility_with_skip_tokenizer_init(

112 example_prompts,

113 structured_outputs=True,

114 )

115 - model: LLM = vllm_model_skip_tokenizer_init.model

115 + llm: LLM = vllm_model_skip_tokenizer_init.llm

116 with pytest.raises(ValueError):

117 - _ = model.generate(example_prompts, sampling_params_list)

117 + _ = llm.generate(example_prompts, sampling_params_list)

118

119

120 def test_parallel_sampling(vllm_model, example_prompts) -> None:

@@ -125,8 +125,8 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:

125 example_prompt: test fixture providing prompts for testing.

126 """

127 sampling_params_list, n_list = _get_test_sampling_params(example_prompts)

128 - model: LLM = vllm_model.model

129 - outputs = model.generate(example_prompts, sampling_params_list)

128 + llm: LLM = vllm_model.llm

129 + outputs = llm.generate(example_prompts, sampling_params_list)

130

131 # Validate each request response

132 for out, n in zip(outputs, n_list):

@@ -166,10 +166,10 @@ def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):

166 speculative_config=speculative_config,

167 disable_log_stats=False,

168 ) as vllm_model:

169 - model: LLM = vllm_model.model

169 + llm: LLM = vllm_model.llm

170 sampling_params = SamplingParams(temperature=0.0,

171 max_tokens=max_tokens)

172 - outputs = model.generate(example_prompts, sampling_params)

172 + outputs = llm.generate(example_prompts, sampling_params)

173

174 n_prompts = len(example_prompts)

175 assert len(outputs) == n_prompts

@@ -180,7 +180,7 @@ def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):

180 total_tokens += len(out.outputs[0].token_ids)

181 assert total_tokens == max_tokens * n_prompts

182

183 - metrics = model.get_metrics()

183 + metrics = llm.get_metrics()

184

185 def find_metric(name) -> list[Metric]:

186 found = []

tests/v1/sample/test_logprobs.py

+4 -4

@@ -112,7 +112,7 @@ def _run_and_validate(

112 max_tokens: int,

113 do_apc: bool,

114 ) -> None:

115 - vllm_results = vllm_model.model.generate(

115 + vllm_results = vllm_model.llm.generate(

116 test_prompts, sampling_params=vllm_sampling_params)

117

118 for vllm_result, hf_logprob, hf_output, logprob_prompt_logprob in zip(

@@ -288,7 +288,7 @@ def test_get_logprobs_and_prompt_logprobs(

288 """

289 with monkeypatch.context() as m:

290 m.setenv("VLLM_USE_V1", "1")

291 - do_apc = vllm_model.model.llm_engine.cache_config.enable_prefix_caching

291 + do_apc = vllm_model.llm.llm_engine.cache_config.enable_prefix_caching

292 if do_apc and (temperature < 2.0

293 or batch_logprobs_composition != SAMPLE_PROMPT):

294 # Skip some test-cases to save time.

@@ -378,7 +378,7 @@ def test_none_logprobs(vllm_model, example_prompts,

378 prompt_logprobs=None,

379 temperature=0.0,

380 )

381 - results_logprobs_none = vllm_model.model.generate(

381 + results_logprobs_none = vllm_model.llm.generate(

382 example_prompts,

383 sampling_params=sampling_params_logprobs_none,

384 )

@@ -408,7 +408,7 @@ def test_zero_logprobs(vllm_model, example_prompts,

408 logprobs=0,

409 prompt_logprobs=0,

410 temperature=0.0)

411 - results_logprobs_zero = vllm_model.model.generate(

411 + results_logprobs_zero = vllm_model.llm.generate(

412 example_prompts, sampling_params=sampling_params_logprobs_zero)

413

414 for i in range(len(results_logprobs_zero)):

tests/v1/sample/test_sampling_params_e2e.py

+36 -38

@@ -14,30 +14,30 @@ PROMPT = "Hello my name is Robert and I"

14

15

16 @pytest.fixture(scope="module")

17 -def model() -> LLM:

17 +def llm() -> LLM:

18 # Disable prefix caching so that we can test prompt logprobs.

19 # TODO remove this after https://github.com/vllm-project/vllm/pull/13949

20 # is merged

21 return LLM(MODEL, enforce_eager=True, enable_prefix_caching=False)

22

23

24 -def test_n_gt_1(model):

24 +def test_n_gt_1(llm):

25 """ParallelSampling is supported."""

26

27 params = SamplingParams(n=3)

28 - outputs = model.generate(PROMPT, params)

28 + outputs = llm.generate(PROMPT, params)

29 assert len(outputs[0].outputs) == 3

30

31

32 -def test_best_of(model):

32 +def test_best_of(llm):

33 """Raise a ValueError since best_of is deprecated."""

34

35 params = SamplingParams(n=2, best_of=3)

36 with pytest.raises(ValueError):

37 - _ = model.generate(PROMPT, params)

37 + _ = llm.generate(PROMPT, params)

38

39

40 -def test_penalties(model):

40 +def test_penalties(llm):

41 """Check that we do not get errors if applied."""

42

43 params = SamplingParams(

@@ -49,18 +49,18 @@ def test_penalties(model):

49 top_p=0.5,

50 top_k=3,

51 )

52 - _ = model.generate(PROMPT, params)

52 + _ = llm.generate(PROMPT, params)

53

54

55 -def test_stop(model):

55 +def test_stop(llm):

56 """Check that we respect the stop words."""

57

58 - output = model.generate(PROMPT, SamplingParams(temperature=0))

58 + output = llm.generate(PROMPT, SamplingParams(temperature=0))

59 split_text = output[0].outputs[0].text.split()

60

61 STOP_IDX = 5

62 params = SamplingParams(temperature=0, stop=split_text[STOP_IDX])

63 - output = model.generate(PROMPT, params)

63 + output = llm.generate(PROMPT, params)

64 new_split_text = output[0].outputs[0].text.split()

65

66 # Output should not contain the stop word.

@@ -69,40 +69,40 @@ def test_stop(model):

69 params = SamplingParams(temperature=0,

70 stop=split_text[STOP_IDX],

71 include_stop_str_in_output=True)

72 - output = model.generate(PROMPT, params)

72 + output = llm.generate(PROMPT, params)

73 new_split_text = output[0].outputs[0].text.split()

74

75 # Output should contain the stop word.

76 assert len(new_split_text) == STOP_IDX + 1

77

78

79 -def test_stop_token_ids(model):

79 +def test_stop_token_ids(llm):

80 """Check that we respect the stop token ids."""

81

82 - output = model.generate(PROMPT, SamplingParams(temperature=0))

82 + output = llm.generate(PROMPT, SamplingParams(temperature=0))

83

84 stop_token_id_0 = output[0].outputs[0].token_ids[5]

85 stop_token_id_1 = output[0].outputs[0].token_ids[6]

86

87 stop_token_ids = [stop_token_id_1, stop_token_id_0]

88 params = SamplingParams(temperature=0, stop_token_ids=stop_token_ids)

89 - output = model.generate(PROMPT, params)

89 + output = llm.generate(PROMPT, params)

90 assert output[0].outputs[0].token_ids[-1] == stop_token_id_0

91

92 stop_token_ids = [stop_token_id_0, stop_token_id_1]

93 params = SamplingParams(temperature=0, stop_token_ids=stop_token_ids)

94 - output = model.generate(PROMPT, params)

94 + output = llm.generate(PROMPT, params)

95 assert output[0].outputs[0].token_ids[-1] == stop_token_id_0

96

97

98 -def test_detokenize_false(model):

98 +def test_detokenize_false(llm):

99 """Check that detokenize=False option works."""

100

101 - output = model.generate(PROMPT, SamplingParams(detokenize=False))

101 + output = llm.generate(PROMPT, SamplingParams(detokenize=False))

102 assert len(output[0].outputs[0].token_ids) > 0

103 assert len(output[0].outputs[0].text) == 0

104

105 - output = model.generate(

105 + output = llm.generate(

106 PROMPT, SamplingParams(detokenize=False, logprobs=3,

107 prompt_logprobs=3))

108 assert len(output[0].outputs[0].token_ids) > 0

@@ -118,28 +118,28 @@ def test_detokenize_false(model):

118 assert all(lp.decoded_token is None for lp in logprobs.values())

119

120

121 -def test_bad_words(model):

121 +def test_bad_words(llm):

122 """Check that we respect bad words."""

123

124 - output = model.generate(PROMPT, SamplingParams(temperature=0))

124 + output = llm.generate(PROMPT, SamplingParams(temperature=0))

125 split_text = output[0].outputs[0].text.split()

126

127 bad_words_1 = " ".join(split_text[:2])

128 params = SamplingParams(temperature=0, bad_words=[bad_words_1])

129 - output = model.generate(PROMPT, params)

129 + output = llm.generate(PROMPT, params)

130 new_text = output[0].outputs[0].text

131 assert bad_words_1 not in new_text

132

133 bad_words_2 = new_text.split()[-1]

134 params = SamplingParams(temperature=0,

135 bad_words=[bad_words_1, bad_words_2])

136 - output = model.generate(PROMPT, params)

136 + output = llm.generate(PROMPT, params)

137 new_text = output[0].outputs[0].text

138 assert bad_words_1 not in new_text

139 assert bad_words_2 not in new_text

140

141

142 -def test_logits_processor(model):

142 +def test_logits_processor(llm):

143 """Check that we reject logits processor."""

144

145 # This sample logits processor gives infinite score to the i-th token,

@@ -150,47 +150,45 @@ def test_logits_processor(model):

150 return logits

151

152 with pytest.raises(ValueError):

153 - _ = model.generate(PROMPT,

154 - SamplingParams(logits_processors=[pick_ith]))

153 + _ = llm.generate(PROMPT, SamplingParams(logits_processors=[pick_ith]))

154

155

157 -def test_allowed_token_ids(model):

156 +def test_allowed_token_ids(llm):

157 """Check that we can use allowed_token_ids."""

158

159 TOKEN_ID = 10

160 allowed_token_ids = [TOKEN_ID]

162 - output = model.generate(

163 - PROMPT, SamplingParams(allowed_token_ids=allowed_token_ids))

161 + output = llm.generate(PROMPT,

162 + SamplingParams(allowed_token_ids=allowed_token_ids))

163 assert output[0].outputs[0].token_ids[-1] == TOKEN_ID

164

165 # Reject empty allowed_token_ids.

166 with pytest.raises(ValueError):

168 - _ = model.generate(PROMPT, SamplingParams(allowed_token_ids=[]))

167 + _ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[]))

168

169 # Reject negative token id.

170 with pytest.raises(ValueError):

172 - _ = model.generate(PROMPT, SamplingParams(allowed_token_ids=[-1]))

171 + _ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[-1]))

172

173 # Reject out of vocabulary.

174 with pytest.raises(ValueError):

176 - _ = model.generate(PROMPT,

177 - SamplingParams(allowed_token_ids=[10000000]))

175 + _ = llm.generate(PROMPT, SamplingParams(allowed_token_ids=[10000000]))

176

177

180 -def test_priority(model):

178 +def test_priority(llm):

179 """Check that we reject requests with priority."""

180

181 # Reject all allowed token ids

182 with pytest.raises(ValueError):

185 - _ = model.generate(PROMPT, priority=[1])

183 + _ = llm.generate(PROMPT, priority=[1])

184

185

188 -def test_seed(model):

186 +def test_seed(llm):

187 """Check that seed impacts randomness."""

188

191 - out_1 = model.generate(PROMPT, SamplingParams(seed=42))

192 - out_2 = model.generate(PROMPT, SamplingParams(seed=42))

193 - out_3 = model.generate(PROMPT, SamplingParams(seed=43))

189 + out_1 = llm.generate(PROMPT, SamplingParams(seed=42))

190 + out_2 = llm.generate(PROMPT, SamplingParams(seed=42))

191 + out_3 = llm.generate(PROMPT, SamplingParams(seed=43))

192

193 assert out_1[0].outputs[0].text == out_2[0].outputs[0].text

194 assert out_1[0].outputs[0].text != out_3[0].outputs[0].text

tests/v1/test_oracle.py

+3 -3

@@ -106,9 +106,9 @@ def test_v1_llm_by_default(monkeypatch):

106 m.delenv("VLLM_USE_V1")

107

108 # Should default to V1 for supported config.

109 - model = LLM(MODEL, enforce_eager=True, enable_lora=True)

110 - print(model.generate("Hello my name is"))

111 - assert hasattr(model.llm_engine, "engine_core")

109 + llm = LLM(MODEL, enforce_eager=True, enable_lora=True)

110 + print(llm.generate("Hello my name is"))

111 + assert hasattr(llm.llm_engine, "engine_core")

112 m.delenv("VLLM_USE_V1")

113

114

[Misc] unify variable for LLM instance (#20996)