A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI] Make mistral tests pass (#4596)
SangBin Cho committed
May 9, 2024 at 00:44 UTC
f6a593093ac201c286e99a849091801a88d83622
5 files changed
+85
-19
.buildkite/test-pipeline.yaml
+1
-1
@@ -76,7 +76,7 @@ steps:
76
#mirror_hardwares: [amd]
77
commands:
78
- bash ../.buildkite/download-images.sh
79
- - pytest -v -s models --ignore=models/test_llava.py --ignore=models/test_mistral.py
79
+ - pytest -v -s models --ignore=models/test_llava.py
80
81
- label: Llava Test
82
#mirror_hardwares: [amd]
tests/conftest.py
+62
@@ -272,6 +272,68 @@ class HfRunner:
272
all_logprobs.append(seq_logprobs)
273
return all_logprobs
274
275
+ def generate_greedy_logprobs_limit(
276
+ self,
277
+ prompts: List[str],
278
+ max_tokens: int,
279
+ num_logprobs: int,
280
+ ) -> List[Tuple[List[int], str]]:
281
+ all_logprobs = []
282
+ all_output_ids = []
283
+ all_output_strs = []
284
+
285
+ for prompt in prompts:
286
+ input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
287
+ output = self.model.generate(
288
+ input_ids.cuda(),
289
+ use_cache=True,
290
+ do_sample=False,
291
+ max_new_tokens=max_tokens,
292
+ output_hidden_states=True,
293
+ return_dict_in_generate=True,
294
+ )
295
+
296
+ seq_logprobs = []
297
+ for _, hidden_states in enumerate(output.hidden_states):
298
+ last_hidden_states = hidden_states[-1][0]
299
+ logits = torch.matmul(
300
+ last_hidden_states,
301
+ self.model.get_output_embeddings().weight.t(),
302
+ )
303
+ if getattr(self.model.get_output_embeddings(), "bias",
304
+ None) is not None:
305
+ logits += self.model.get_output_embeddings(
306
+ ).bias.unsqueeze(0)
307
+ logprobs = torch.nn.functional.log_softmax(logits,
308
+ dim=-1,
309
+ dtype=torch.float32)
310
+ seq_logprobs.append(logprobs)
311
+
312
+ # convert to dict
313
+ seq_logprobs_lst = []
314
+ for tok_idx, tok_logprobs in enumerate(seq_logprobs):
315
+ # drop prompt logprobs
316
+ if tok_idx == 0:
317
+ tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
318
+ topk = tok_logprobs.topk(num_logprobs)
319
+
320
+ tok_logprobs_dct = {}
321
+ for token_id, logprob in zip(topk.indices[0], topk.values[0]):
322
+ tok_logprobs_dct[token_id.item()] = logprob.item()
323
+
324
+ seq_logprobs_lst.append(tok_logprobs_dct)
325
+
326
+ all_logprobs.append(seq_logprobs_lst)
327
+ seq_ids = output.sequences[0]
328
+ output_len = seq_ids.shape[0] - input_ids.shape[1]
329
+ output_ids = seq_ids[-output_len:]
330
+ all_output_ids.append(output_ids.tolist())
331
+ all_output_strs.append(self.tokenizer.decode(output_ids))
332
+
333
+ outputs = zip(all_output_ids, all_output_strs, all_logprobs)
334
+ return [(output_ids, output_str, output_logprobs)
335
+ for output_ids, output_str, output_logprobs in outputs]
336
+
337
def __del__(self):
338
del self.model
339
cleanup()
tests/models/test_big_models.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
9
MODELS = [
10
"meta-llama/Llama-2-7b-hf",
11
- # "mistralai/Mistral-7B-v0.1", # Broken
11
+ # "mistralai/Mistral-7B-v0.1", # Tested by test_mistral.py
12
# "Deci/DeciLM-7b", # Broken
13
# "tiiuae/falcon-7b", # Broken
14
"EleutherAI/gpt-j-6b",
tests/models/test_mistral.py
+18
-15
@@ -4,6 +4,8 @@ Run `pytest tests/models/test_mistral.py`.
4
"""
5
import pytest
6
7
+from tests.models.utils import check_logprobs_close
8
+
9
MODELS = [
10
"mistralai/Mistral-7B-Instruct-v0.1",
11
]
@@ -11,30 +13,31 @@ MODELS = [
13
14
@pytest.mark.parametrize("model", MODELS)
15
@pytest.mark.parametrize("dtype", ["bfloat16"])
14
-@pytest.mark.parametrize("max_tokens", [128])
15
-@pytest.mark.skip(
16
- "Two problems: 1. Failing correctness tests. 2. RuntimeError: expected "
17
- "scalar type BFloat16 but found Half (only in CI).")
16
+@pytest.mark.parametrize("max_tokens", [64])
17
+@pytest.mark.parametrize("num_logprobs", [5])
18
def test_models(
19
hf_runner,
20
vllm_runner,
21
- example_long_prompts,
21
+ example_prompts,
22
model: str,
23
dtype: str,
24
max_tokens: int,
25
+ num_logprobs: int,
26
) -> None:
27
+ # TODO(sang): Sliding window should be tested separately.
28
hf_model = hf_runner(model, dtype=dtype)
27
- hf_outputs = hf_model.generate_greedy(example_long_prompts, max_tokens)
29
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
30
+ example_prompts, max_tokens, num_logprobs)
31
del hf_model
32
33
vllm_model = vllm_runner(model, dtype=dtype)
31
- vllm_outputs = vllm_model.generate_greedy(example_long_prompts, max_tokens)
34
+ vllm_outputs = vllm_model.generate_greedy_logprobs(example_prompts,
35
+ max_tokens,
36
+ num_logprobs)
37
del vllm_model
33
-
34
- for i in range(len(example_long_prompts)):
35
- hf_output_ids, hf_output_str = hf_outputs[i]
36
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
37
- assert hf_output_str == vllm_output_str, (
38
- f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
39
- assert hf_output_ids == vllm_output_ids, (
40
- f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
38
+ check_logprobs_close(
39
+ outputs_0_lst=hf_outputs,
40
+ outputs_1_lst=vllm_outputs,
41
+ name_0="hf",
42
+ name_1="vllm",
43
+ )
vllm/model_executor/layers/rotary_embedding.py
+3
-2
@@ -109,7 +109,7 @@ class RotaryEmbedding(nn.Module):
109
key_pass = key[..., self.rotary_dim:]
110
111
self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
112
- positions.device)
112
+ positions.device, dtype=query.dtype)
113
cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
114
if offsets is not None else positions]
115
cos, sin = cos_sin.chunk(2, dim=-1)
@@ -143,7 +143,8 @@ class RotaryEmbedding(nn.Module):
143
key: torch.Tensor,
144
offsets: Optional[torch.Tensor] = None,
145
) -> Tuple[torch.Tensor, torch.Tensor]:
146
- self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
146
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
147
+ dtype=query.dtype)
148
# ops.rotary_embedding()/batched_rotary_embedding()
149
# are in-place operations that update the query and key tensors.
150
if offsets is not None: