@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f6a593093a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI] Make mistral tests pass (#4596)

SangBin Cho committed May 9, 2024 at 00:44 UTC f6a593093ac201c286e99a849091801a88d83622
5 files changed +85 -19
.buildkite/test-pipeline.yaml
+1 -1
@@ -76,7 +76,7 @@ steps:
76 #mirror_hardwares: [amd]
77 commands:
78 - bash ../.buildkite/download-images.sh
79 - - pytest -v -s models --ignore=models/test_llava.py --ignore=models/test_mistral.py
79 + - pytest -v -s models --ignore=models/test_llava.py
80
81 - label: Llava Test
82 #mirror_hardwares: [amd]
tests/conftest.py
+62
@@ -272,6 +272,68 @@ class HfRunner:
272 all_logprobs.append(seq_logprobs)
273 return all_logprobs
274
275 + def generate_greedy_logprobs_limit(
276 + self,
277 + prompts: List[str],
278 + max_tokens: int,
279 + num_logprobs: int,
280 + ) -> List[Tuple[List[int], str]]:
281 + all_logprobs = []
282 + all_output_ids = []
283 + all_output_strs = []
284 +
285 + for prompt in prompts:
286 + input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
287 + output = self.model.generate(
288 + input_ids.cuda(),
289 + use_cache=True,
290 + do_sample=False,
291 + max_new_tokens=max_tokens,
292 + output_hidden_states=True,
293 + return_dict_in_generate=True,
294 + )
295 +
296 + seq_logprobs = []
297 + for _, hidden_states in enumerate(output.hidden_states):
298 + last_hidden_states = hidden_states[-1][0]
299 + logits = torch.matmul(
300 + last_hidden_states,
301 + self.model.get_output_embeddings().weight.t(),
302 + )
303 + if getattr(self.model.get_output_embeddings(), "bias",
304 + None) is not None:
305 + logits += self.model.get_output_embeddings(
306 + ).bias.unsqueeze(0)
307 + logprobs = torch.nn.functional.log_softmax(logits,
308 + dim=-1,
309 + dtype=torch.float32)
310 + seq_logprobs.append(logprobs)
311 +
312 + # convert to dict
313 + seq_logprobs_lst = []
314 + for tok_idx, tok_logprobs in enumerate(seq_logprobs):
315 + # drop prompt logprobs
316 + if tok_idx == 0:
317 + tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
318 + topk = tok_logprobs.topk(num_logprobs)
319 +
320 + tok_logprobs_dct = {}
321 + for token_id, logprob in zip(topk.indices[0], topk.values[0]):
322 + tok_logprobs_dct[token_id.item()] = logprob.item()
323 +
324 + seq_logprobs_lst.append(tok_logprobs_dct)
325 +
326 + all_logprobs.append(seq_logprobs_lst)
327 + seq_ids = output.sequences[0]
328 + output_len = seq_ids.shape[0] - input_ids.shape[1]
329 + output_ids = seq_ids[-output_len:]
330 + all_output_ids.append(output_ids.tolist())
331 + all_output_strs.append(self.tokenizer.decode(output_ids))
332 +
333 + outputs = zip(all_output_ids, all_output_strs, all_logprobs)
334 + return [(output_ids, output_str, output_logprobs)
335 + for output_ids, output_str, output_logprobs in outputs]
336 +
337 def __del__(self):
338 del self.model
339 cleanup()
tests/models/test_big_models.py
+1 -1
@@ -8,7 +8,7 @@ import pytest
8
9 MODELS = [
10 "meta-llama/Llama-2-7b-hf",
11 - # "mistralai/Mistral-7B-v0.1", # Broken
11 + # "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
12 # "Deci/DeciLM-7b", # Broken
13 # "tiiuae/falcon-7b", # Broken
14 "EleutherAI/gpt-j-6b",
tests/models/test_mistral.py
+18 -15
@@ -4,6 +4,8 @@ Run `pytest tests/models/test_mistral.py`.
4 """
5 import pytest
6
7 +from tests.models.utils import check_logprobs_close
8 +
9 MODELS = [
10 "mistralai/Mistral-7B-Instruct-v0.1",
11 ]
@@ -11,30 +13,31 @@ MODELS = [
13
14 @pytest.mark.parametrize("model", MODELS)
15 @pytest.mark.parametrize("dtype", ["bfloat16"])
14 -@pytest.mark.parametrize("max_tokens", [128])
15 -@pytest.mark.skip(
16 - "Two problems: 1. Failing correctness tests. 2. RuntimeError: expected "
17 - "scalar type BFloat16 but found Half (only in CI).")
16 +@pytest.mark.parametrize("max_tokens", [64])
17 +@pytest.mark.parametrize("num_logprobs", [5])
18 def test_models(
19 hf_runner,
20 vllm_runner,
21 - example_long_prompts,
21 + example_prompts,
22 model: str,
23 dtype: str,
24 max_tokens: int,
25 + num_logprobs: int,
26 ) -> None:
27 + # TODO(sang): Sliding window should be tested separately.
28 hf_model = hf_runner(model, dtype=dtype)
27 - hf_outputs = hf_model.generate_greedy(example_long_prompts, max_tokens)
29 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
30 + example_prompts, max_tokens, num_logprobs)
31 del hf_model
32
33 vllm_model = vllm_runner(model, dtype=dtype)
31 - vllm_outputs = vllm_model.generate_greedy(example_long_prompts, max_tokens)
34 + vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
35 + max_tokens,
36 + num_logprobs)
37 del vllm_model
33 -
34 - for i in range(len(example_long_prompts)):
35 - hf_output_ids, hf_output_str = hf_outputs[i]
36 - vllm_output_ids, vllm_output_str = vllm_outputs[i]
37 - assert hf_output_str == vllm_output_str, (
38 - f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
39 - assert hf_output_ids == vllm_output_ids, (
40 - f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
38 + check_logprobs_close(
39 + outputs_0_lst=hf_outputs,
40 + outputs_1_lst=vllm_outputs,
41 + name_0="hf",
42 + name_1="vllm",
43 + )
vllm/model_executor/layers/rotary_embedding.py
+3 -2
@@ -109,7 +109,7 @@ class RotaryEmbedding(nn.Module):
109 key_pass = key[..., self.rotary_dim:]
110
111 self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
112 - positions.device)
112 + positions.device, dtype=query.dtype)
113 cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
114 if offsets is not None else positions]
115 cos, sin = cos_sin.chunk(2, dim=-1)
@@ -143,7 +143,8 @@ class RotaryEmbedding(nn.Module):
143 key: torch.Tensor,
144 offsets: Optional[torch.Tensor] = None,
145 ) -> Tuple[torch.Tensor, torch.Tensor]:
146 - self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
146 + self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
147 + dtype=query.dtype)
148 # ops.rotary_embedding()/batched_rotary_embedding()
149 # are in-place operations that update the query and key tensors.
150 if offsets is not None: