@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed812a73fa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[ Frontend ] Multiprocessing for OpenAI Server with `zeromq` (#6883)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <joe@joerun.de> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Simon Mo <simon.mo@hey.com>

Robert Shaw committed Aug 2, 2024 at 21:27 UTC ed812a73fae77bb520b739cfeaad36dbd61e2b03
20 files changed +1567 -101
tests/entrypoints/openai/test_disable_mp.py new
+715
@@ -0,0 +1,715 @@
1 +"""
2 +Repeat of tests in test_completion.py with the non-mp backend.
3 +"""
4 +
5 +# imports for guided decoding tests
6 +import json
7 +import re
8 +import shutil
9 +from tempfile import TemporaryDirectory
10 +from typing import List
11 +
12 +import jsonschema
13 +import openai # use the official client for correctness check
14 +import pytest
15 +# downloading lora to test lora requests
16 +from huggingface_hub import snapshot_download
17 +from openai import BadRequestError
18 +from transformers import AutoTokenizer
19 +
20 +from vllm.transformers_utils.tokenizer import get_tokenizer
21 +
22 +from ...utils import RemoteOpenAIServer
23 +
24 +# any model with a chat template should work here
25 +MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26 +# technically these adapters use a different base model,
27 +# but we're not testing generation quality here
28 +LORA_NAME = "typeof/zephyr-7b-beta-lora"
29 +PA_NAME = "swapnilbp/llama_tweet_ptune"
30 +# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
31 +# need to change to match the prompt adapter
32 +PA_NUM_VIRTUAL_TOKENS = 8
33 +
34 +
35 +@pytest.fixture(scope="module")
36 +def zephyr_lora_files():
37 + return snapshot_download(repo_id=LORA_NAME)
38 +
39 +
40 +@pytest.fixture(scope="module")
41 +def zephyr_lora_added_tokens_files(zephyr_lora_files):
42 + tmp_dir = TemporaryDirectory()
43 + tmp_model_dir = f"{tmp_dir.name}/zephyr"
44 + shutil.copytree(zephyr_lora_files, tmp_model_dir)
45 + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
46 + # Copy tokenizer to adapter and add some unique tokens
47 + # 32000, 32001, 32002
48 + added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
49 + special_tokens=True)
50 + assert added == 3
51 + tokenizer.save_pretrained(tmp_model_dir)
52 + yield tmp_model_dir
53 + tmp_dir.cleanup()
54 +
55 +
56 +@pytest.fixture(scope="module")
57 +def zephyr_pa_files():
58 + return snapshot_download(repo_id=PA_NAME)
59 +
60 +
61 +@pytest.fixture(scope="module")
62 +def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
63 + zephyr_pa_files):
64 + return [
65 + # use half precision for speed and memory savings in CI environment
66 + "--dtype",
67 + "bfloat16",
68 + "--max-model-len",
69 + "8192",
70 + "--max-num-seqs",
71 + "128",
72 + "--enforce-eager",
73 + # lora config
74 + "--enable-lora",
75 + "--lora-modules",
76 + f"zephyr-lora={zephyr_lora_files}",
77 + f"zephyr-lora2={zephyr_lora_added_tokens_files}",
78 + "--max-lora-rank",
79 + "64",
80 + "--max-cpu-loras",
81 + "2",
82 + # pa config
83 + "--enable-prompt-adapter",
84 + "--prompt-adapters",
85 + f"zephyr-pa={zephyr_pa_files}",
86 + f"zephyr-pa2={zephyr_pa_files}",
87 + "--max-prompt-adapters",
88 + "2",
89 + "--max-prompt-adapter-token",
90 + "128",
91 + "--disable-frontend-multiprocessing"
92 + ]
93 +
94 +
95 +@pytest.fixture(scope="module")
96 +def server(default_server_args):
97 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
98 + yield remote_server
99 +
100 +
101 +@pytest.fixture(scope="module")
102 +def client(server):
103 + return server.get_async_client()
104 +
105 +
106 +@pytest.mark.asyncio
107 +@pytest.mark.parametrize(
108 + # first test base model, then test loras, then test prompt adapters
109 + "model_name,num_virtual_tokens",
110 + [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
111 + ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
112 + ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
113 +)
114 +async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
115 + num_virtual_tokens: int):
116 + completion = await client.completions.create(model=model_name,
117 + prompt="Hello, my name is",
118 + max_tokens=5,
119 + temperature=0.0)
120 +
121 + assert completion.id is not None
122 + assert completion.choices is not None and len(completion.choices) == 1
123 +
124 + choice = completion.choices[0]
125 + assert len(choice.text) >= 5
126 + assert choice.finish_reason == "length"
127 + assert completion.usage == openai.types.CompletionUsage(
128 + completion_tokens=5,
129 + prompt_tokens=6 + num_virtual_tokens,
130 + total_tokens=11 + num_virtual_tokens)
131 +
132 + # test using token IDs
133 + completion = await client.completions.create(
134 + model=model_name,
135 + prompt=[0, 0, 0, 0, 0],
136 + max_tokens=5,
137 + temperature=0.0,
138 + )
139 + assert len(completion.choices[0].text) >= 1
140 +
141 +
142 +@pytest.mark.asyncio
143 +async def test_added_lora_tokens(client: openai.AsyncOpenAI):
144 + # test using token IDs
145 + completion = await client.completions.create(
146 + model="zephyr-lora2",
147 + prompt=[0, 0, 32000, 32001, 32002],
148 + echo=True,
149 + max_tokens=5,
150 + temperature=0.0,
151 + )
152 + # Added tokens should appear in tokenized prompt
153 + assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
154 +
155 +
156 +@pytest.mark.asyncio
157 +async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
158 + # test using token IDs
159 + completion = await client.completions.create(
160 + model=MODEL_NAME,
161 + prompt=[0, 0, 32000, 32001, 32002],
162 + echo=True,
163 + max_tokens=5,
164 + temperature=0.0,
165 + )
166 + # Added tokens should not appear in tokenized prompt
167 + assert "vllm" not in completion.choices[0].text
168 +
169 +
170 +@pytest.mark.asyncio
171 +@pytest.mark.parametrize(
172 + # first test base model, then test loras, then test prompt adapters
173 + "model_name",
174 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
175 +)
176 +async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
177 + # test using token IDs
178 + completion = await client.completions.create(
179 + model=model_name,
180 + prompt=[0, 0, 0, 0, 0],
181 + max_tokens=5,
182 + temperature=0.0,
183 + logprobs=None,
184 + )
185 + choice = completion.choices[0]
186 + assert choice.logprobs is None
187 +
188 +
189 +@pytest.mark.asyncio
190 +@pytest.mark.parametrize(
191 + # just test 1 lora and 1 pa hereafter
192 + "model_name",
193 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
194 +)
195 +async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
196 + # test using token IDs
197 + completion = await client.completions.create(
198 + model=model_name,
199 + prompt=[0, 0, 0, 0, 0],
200 + max_tokens=5,
201 + temperature=0.0,
202 + logprobs=0,
203 + )
204 + choice = completion.choices[0]
205 + assert choice.logprobs is not None
206 + assert choice.logprobs.token_logprobs is not None
207 + assert choice.logprobs.top_logprobs is not None
208 + assert len(choice.logprobs.top_logprobs[0]) == 1
209 +
210 +
211 +@pytest.mark.asyncio
212 +@pytest.mark.parametrize(
213 + "model_name",
214 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
215 +)
216 +async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
217 + # test using token IDs
218 + completion = await client.completions.create(
219 + model=model_name,
220 + prompt=[0, 0, 0, 0, 0],
221 + max_tokens=5,
222 + temperature=0.0,
223 + logprobs=5,
224 + )
225 + choice = completion.choices[0]
226 + assert choice.logprobs is not None
227 + assert choice.logprobs.token_logprobs is not None
228 + assert choice.logprobs.top_logprobs is not None
229 + assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
230 +
231 +
232 +@pytest.mark.asyncio
233 +@pytest.mark.parametrize(
234 + "model_name",
235 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
236 +)
237 +async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
238 + model_name: str):
239 +
240 + with pytest.raises(
241 + (openai.BadRequestError, openai.APIError)): # test using token IDs
242 + await client.completions.create(
243 + model=model_name,
244 + prompt=[0, 0, 0, 0, 0],
245 + max_tokens=5,
246 + temperature=0.0,
247 + # vLLM has higher default max_logprobs (20 instead of 5) to support
248 + # both Completion API and Chat Completion API
249 + logprobs=21,
250 + )
251 + ...
252 + with pytest.raises(
253 + (openai.BadRequestError, openai.APIError)): # test using token IDs
254 + stream = await client.completions.create(
255 + model=model_name,
256 + prompt=[0, 0, 0, 0, 0],
257 + max_tokens=5,
258 + temperature=0.0,
259 + # vLLM has higher default max_logprobs (20 instead of 5) to support
260 + # both Completion API and Chat Completion API
261 + logprobs=30,
262 + stream=True,
263 + )
264 + async for chunk in stream:
265 + ...
266 +
267 + # the server should still work afterwards
268 + completion = await client.completions.create(
269 + model=model_name,
270 + prompt=[0, 0, 0, 0, 0],
271 + max_tokens=5,
272 + temperature=0.0,
273 + )
274 + assert len(completion.choices[0].text) >= 0
275 +
276 +
277 +@pytest.mark.asyncio
278 +@pytest.mark.parametrize(
279 + "model_name",
280 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
281 +)
282 +async def test_completion_streaming(client: openai.AsyncOpenAI,
283 + model_name: str):
284 + prompt = "What is an LLM?"
285 +
286 + single_completion = await client.completions.create(
287 + model=model_name,
288 + prompt=prompt,
289 + max_tokens=5,
290 + temperature=0.0,
291 + )
292 + single_output = single_completion.choices[0].text
293 + stream = await client.completions.create(model=model_name,
294 + prompt=prompt,
295 + max_tokens=5,
296 + temperature=0.0,
297 + stream=True)
298 + chunks: List[str] = []
299 + finish_reason_count = 0
300 + async for chunk in stream:
301 + chunks.append(chunk.choices[0].text)
302 + if chunk.choices[0].finish_reason is not None:
303 + finish_reason_count += 1
304 + # finish reason should only return in last block
305 + assert finish_reason_count == 1
306 + assert chunk.choices[0].finish_reason == "length"
307 + assert chunk.choices[0].text
308 + assert "".join(chunks) == single_output
309 +
310 +
311 +@pytest.mark.asyncio
312 +@pytest.mark.parametrize(
313 + "model_name",
314 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
315 +)
316 +async def test_completion_stream_options(client: openai.AsyncOpenAI,
317 + model_name: str):
318 + prompt = "What is the capital of France?"
319 +
320 + # Test stream=True, stream_options=
321 + # {"include_usage": False, "continuous_usage_stats": False}
322 + stream = await client.completions.create(model=model_name,
323 + prompt=prompt,
324 + max_tokens=5,
325 + temperature=0.0,
326 + stream=True,
327 + stream_options={
328 + "include_usage": False,
329 + "continuous_usage_stats":
330 + False,
331 + })
332 +
333 + async for chunk in stream:
334 + assert chunk.usage is None
335 +
336 + # Test stream=True, stream_options=
337 + # {"include_usage": False, "continuous_usage_stats": True}
338 + stream = await client.completions.create(model=model_name,
339 + prompt=prompt,
340 + max_tokens=5,
341 + temperature=0.0,
342 + stream=True,
343 + stream_options={
344 + "include_usage": False,
345 + "continuous_usage_stats":
346 + True,
347 + })
348 + async for chunk in stream:
349 + assert chunk.usage is None
350 +
351 + # Test stream=True, stream_options=
352 + # {"include_usage": True, "continuous_usage_stats": False}
353 + stream = await client.completions.create(model=model_name,
354 + prompt=prompt,
355 + max_tokens=5,
356 + temperature=0.0,
357 + stream=True,
358 + stream_options={
359 + "include_usage": True,
360 + "continuous_usage_stats":
361 + False,
362 + })
363 + async for chunk in stream:
364 + if chunk.choices[0].finish_reason is None:
365 + assert chunk.usage is None
366 + else:
367 + assert chunk.usage is None
368 + final_chunk = await stream.__anext__()
369 + assert final_chunk.usage is not None
370 + assert final_chunk.usage.prompt_tokens > 0
371 + assert final_chunk.usage.completion_tokens > 0
372 + assert final_chunk.usage.total_tokens == (
373 + final_chunk.usage.prompt_tokens +
374 + final_chunk.usage.completion_tokens)
375 + assert final_chunk.choices == []
376 +
377 + # Test stream=True, stream_options=
378 + # {"include_usage": True, "continuous_usage_stats": True}
379 + stream = await client.completions.create(model=model_name,
380 + prompt=prompt,
381 + max_tokens=5,
382 + temperature=0.0,
383 + stream=True,
384 + stream_options={
385 + "include_usage": True,
386 + "continuous_usage_stats":
387 + True,
388 + })
389 + async for chunk in stream:
390 + assert chunk.usage is not None
391 + assert chunk.usage.prompt_tokens > 0
392 + assert chunk.usage.completion_tokens > 0
393 + assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
394 + chunk.usage.completion_tokens)
395 + if chunk.choices[0].finish_reason is not None:
396 + final_chunk = await stream.__anext__()
397 + assert final_chunk.usage is not None
398 + assert final_chunk.usage.prompt_tokens > 0
399 + assert final_chunk.usage.completion_tokens > 0
400 + assert final_chunk.usage.total_tokens == (
401 + final_chunk.usage.prompt_tokens +
402 + final_chunk.usage.completion_tokens)
403 + assert final_chunk.choices == []
404 +
405 + # Test stream=False, stream_options=
406 + # {"include_usage": None}
407 + with pytest.raises(BadRequestError):
408 + await client.completions.create(model=model_name,
409 + prompt=prompt,
410 + max_tokens=5,
411 + temperature=0.0,
412 + stream=False,
413 + stream_options={"include_usage": None})
414 +
415 + # Test stream=False, stream_options=
416 + # {"include_usage": True}
417 + with pytest.raises(BadRequestError):
418 + await client.completions.create(model=model_name,
419 + prompt=prompt,
420 + max_tokens=5,
421 + temperature=0.0,
422 + stream=False,
423 + stream_options={"include_usage": True})
424 +
425 + # Test stream=False, stream_options=
426 + # {"continuous_usage_stats": None}
427 + with pytest.raises(BadRequestError):
428 + await client.completions.create(
429 + model=model_name,
430 + prompt=prompt,
431 + max_tokens=5,
432 + temperature=0.0,
433 + stream=False,
434 + stream_options={"continuous_usage_stats": None})
435 +
436 + # Test stream=False, stream_options=
437 + # {"continuous_usage_stats": True}
438 + with pytest.raises(BadRequestError):
439 + await client.completions.create(
440 + model=model_name,
441 + prompt=prompt,
442 + max_tokens=5,
443 + temperature=0.0,
444 + stream=False,
445 + stream_options={"continuous_usage_stats": True})
446 +
447 +
448 +@pytest.mark.asyncio
449 +@pytest.mark.parametrize(
450 + "model_name",
451 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
452 +)
453 +async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
454 + # test both text and token IDs
455 + for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
456 + # test simple list
457 + batch = await client.completions.create(
458 + model=model_name,
459 + prompt=prompts,
460 + max_tokens=5,
461 + temperature=0.0,
462 + )
463 + assert len(batch.choices) == 2
464 + assert batch.choices[0].text == batch.choices[1].text
465 +
466 + # test n = 2
467 + batch = await client.completions.create(
468 + model=model_name,
469 + prompt=prompts,
470 + n=2,
471 + max_tokens=5,
472 + temperature=0.0,
473 + extra_body=dict(
474 + # NOTE: this has to be true for n > 1 in vLLM, but not necessary
475 + # for official client.
476 + use_beam_search=True),
477 + )
478 + assert len(batch.choices) == 4
479 + assert batch.choices[0].text != batch.choices[
480 + 1].text, "beam search should be different"
481 + assert batch.choices[0].text == batch.choices[
482 + 2].text, "two copies of the same prompt should be the same"
483 + assert batch.choices[1].text == batch.choices[
484 + 3].text, "two copies of the same prompt should be the same"
485 +
486 + # test streaming
487 + batch = await client.completions.create(
488 + model=model_name,
489 + prompt=prompts,
490 + max_tokens=5,
491 + temperature=0.0,
492 + stream=True,
493 + )
494 + texts = [""] * 2
495 + async for chunk in batch:
496 + assert len(chunk.choices) == 1
497 + choice = chunk.choices[0]
498 + texts[choice.index] += choice.text
499 + assert texts[0] == texts[1]
500 +
501 +
502 +@pytest.mark.asyncio
503 +async def test_logits_bias(client: openai.AsyncOpenAI):
504 + prompt = "Hello, my name is"
505 + max_tokens = 5
506 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
507 +
508 + # Test exclusive selection
509 + token_id = 1000
510 + completion = await client.completions.create(
511 + model=MODEL_NAME,
512 + prompt=prompt,
513 + max_tokens=max_tokens,
514 + temperature=0.0,
515 + logit_bias={str(token_id): 100},
516 + seed=42,
517 + )
518 + assert len(completion.choices[0].text) >= 5
519 + response_tokens = tokenizer(completion.choices[0].text,
520 + add_special_tokens=False)["input_ids"]
521 + expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
522 + add_special_tokens=False)["input_ids"]
523 + assert all([
524 + response == expected
525 + for response, expected in zip(response_tokens, expected_tokens)
526 + ])
527 +
528 + # Test ban
529 + completion = await client.completions.create(
530 + model=MODEL_NAME,
531 + prompt=prompt,
532 + max_tokens=max_tokens,
533 + temperature=0.0,
534 + )
535 + response_tokens = tokenizer(completion.choices[0].text,
536 + add_special_tokens=False)["input_ids"]
537 + first_response = completion.choices[0].text
538 + completion = await client.completions.create(
539 + model=MODEL_NAME,
540 + prompt=prompt,
541 + max_tokens=max_tokens,
542 + temperature=0.0,
543 + logit_bias={str(token): -100
544 + for token in response_tokens},
545 + )
546 + assert first_response != completion.choices[0].text
547 +
548 +
549 +@pytest.mark.asyncio
550 +async def test_allowed_token_ids(client: openai.AsyncOpenAI):
551 + prompt = "Hello, my name is"
552 + max_tokens = 1
553 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
554 +
555 + # Test exclusive selection
556 + allowed_ids = [21555, 21557, 21558]
557 + completion = await client.completions.create(
558 + model=MODEL_NAME,
559 + prompt=prompt,
560 + max_tokens=max_tokens,
561 + temperature=0.0,
562 + seed=42,
563 + extra_body=dict(allowed_token_ids=allowed_ids),
564 + logprobs=1,
565 + )
566 + response_tokens = completion.choices[0].logprobs.tokens
567 + assert len(response_tokens) == 1
568 + assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
569 +
570 +
571 +@pytest.mark.asyncio
572 +@pytest.mark.parametrize("guided_decoding_backend",
573 + ["outlines", "lm-format-enforcer"])
574 +async def test_guided_json_completion(client: openai.AsyncOpenAI,
575 + guided_decoding_backend: str,
576 + sample_json_schema):
577 + completion = await client.completions.create(
578 + model=MODEL_NAME,
579 + prompt=f"Give an example JSON for an employee profile "
580 + f"that fits this schema: {sample_json_schema}",
581 + n=3,
582 + temperature=1.0,
583 + max_tokens=500,
584 + extra_body=dict(guided_json=sample_json_schema,
585 + guided_decoding_backend=guided_decoding_backend))
586 +
587 + assert completion.id is not None
588 + assert len(completion.choices) == 3
589 + for i in range(3):
590 + output_json = json.loads(completion.choices[i].text)
591 + jsonschema.validate(instance=output_json, schema=sample_json_schema)
592 +
593 +
594 +@pytest.mark.asyncio
595 +@pytest.mark.parametrize("guided_decoding_backend",
596 + ["outlines", "lm-format-enforcer"])
597 +async def test_guided_regex_completion(client: openai.AsyncOpenAI,
598 + guided_decoding_backend: str,
599 + sample_regex):
600 + completion = await client.completions.create(
601 + model=MODEL_NAME,
602 + prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
603 + n=3,
604 + temperature=1.0,
605 + max_tokens=20,
606 + extra_body=dict(guided_regex=sample_regex,
607 + guided_decoding_backend=guided_decoding_backend))
608 +
609 + assert completion.id is not None
610 + assert len(completion.choices) == 3
611 + for i in range(3):
612 + assert re.fullmatch(sample_regex,
613 + completion.choices[i].text) is not None
614 +
615 +
616 +@pytest.mark.asyncio
617 +@pytest.mark.parametrize("guided_decoding_backend",
618 + ["outlines", "lm-format-enforcer"])
619 +async def test_guided_choice_completion(client: openai.AsyncOpenAI,
620 + guided_decoding_backend: str,
621 + sample_guided_choice):
622 + completion = await client.completions.create(
623 + model=MODEL_NAME,
624 + prompt="The best language for type-safe systems programming is ",
625 + n=2,
626 + temperature=1.0,
627 + max_tokens=10,
628 + extra_body=dict(guided_choice=sample_guided_choice,
629 + guided_decoding_backend=guided_decoding_backend))
630 +
631 + assert completion.id is not None
632 + assert len(completion.choices) == 2
633 + for i in range(2):
634 + assert completion.choices[i].text in sample_guided_choice
635 +
636 +
637 +@pytest.mark.asyncio
638 +async def test_guided_grammar(client: openai.AsyncOpenAI,
639 + sample_sql_statements):
640 +
641 + completion = await client.completions.create(
642 + model=MODEL_NAME,
643 + prompt=("Generate a sql state that select col_1 from "
644 + "table_1 where it is equals to 1"),
645 + temperature=1.0,
646 + max_tokens=500,
647 + extra_body=dict(guided_grammar=sample_sql_statements))
648 +
649 + content = completion.choices[0].text
650 +
651 + # use Lark to parse the output, and make sure it's a valid parse tree
652 + from lark import Lark
653 + parser = Lark(sample_sql_statements)
654 + parser.parse(content)
655 +
656 + # remove spaces for comparison b/c we removed them in the grammar
657 + ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
658 +
659 + assert content.strip() == ground_truth
660 +
661 +
662 +@pytest.mark.asyncio
663 +@pytest.mark.parametrize(
664 + # first test base model, then test loras
665 + "model_name",
666 + [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
667 +)
668 +@pytest.mark.parametrize("logprobs_arg", [1, 0])
669 +async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
670 + model_name: str, logprobs_arg: int):
671 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
672 + # test using text and token IDs
673 + for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
674 + completion = await client.completions.create(model=model_name,
675 + prompt=prompt,
676 + max_tokens=5,
677 + temperature=0.0,
678 + echo=True,
679 + logprobs=logprobs_arg)
680 +
681 + prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
682 + list) else prompt
683 + assert re.search(r"^" + prompt_text, completion.choices[0].text)
684 + logprobs = completion.choices[0].logprobs
685 + assert logprobs is not None
686 + assert len(logprobs.text_offset) > 5
687 + assert (len(logprobs.token_logprobs) > 5
688 + and logprobs.token_logprobs[0] is None)
689 + assert (len(logprobs.top_logprobs) > 5
690 + and logprobs.top_logprobs[0] is None)
691 + for top_logprobs in logprobs.top_logprobs[1:]:
692 + assert max(logprobs_arg,
693 + 1) <= len(top_logprobs) <= logprobs_arg + 1
694 + assert len(logprobs.tokens) > 5
695 +
696 +
697 +@pytest.mark.asyncio
698 +@pytest.mark.parametrize("guided_decoding_backend",
699 + ["outlines", "lm-format-enforcer"])
700 +async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
701 + guided_decoding_backend: str,
702 + sample_json_schema, sample_regex):
703 + with pytest.raises(openai.BadRequestError):
704 + _ = await client.completions.create(
705 + model=MODEL_NAME,
706 + prompt="Give an example JSON that fits this schema: 42",
707 + extra_body=dict(guided_json=42,
708 + guided_decoding_backend=guided_decoding_backend))
709 +
710 + with pytest.raises(openai.BadRequestError):
711 + _ = await client.completions.create(
712 + model=MODEL_NAME,
713 + prompt="Give an example string that fits this regex",
714 + extra_body=dict(guided_regex=sample_regex,
715 + guided_json=sample_json_schema))
vllm/engine/async_llm_engine.py
+26 -1
@@ -7,7 +7,8 @@ from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
7 from transformers import PreTrainedTokenizer
8
9 import vllm.envs as envs
10 -from vllm.config import DecodingConfig, EngineConfig, ModelConfig
10 +from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11 + ParallelConfig, SchedulerConfig)
12 from vllm.core.scheduler import SchedulerOutputs
13 from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_timeout import asyncio_timeout
@@ -928,6 +929,14 @@ class AsyncLLMEngine:
929 else:
930 return self.engine.get_model_config()
931
932 + async def get_parallel_config(self) -> ParallelConfig:
933 + """Get the parallel configuration of the vLLM engine."""
934 + if self.engine_use_ray:
935 + return await self.engine.get_parallel_config.remote( # type: ignore
936 + )
937 + else:
938 + return self.engine.get_parallel_config()
939 +
940 async def get_decoding_config(self) -> DecodingConfig:
941 """Get the decoding configuration of the vLLM engine."""
942 if self.engine_use_ray:
@@ -936,6 +945,22 @@ class AsyncLLMEngine:
945 else:
946 return self.engine.get_decoding_config()
947
948 + async def get_scheduler_config(self) -> SchedulerConfig:
949 + """Get the scheduling configuration of the vLLM engine."""
950 + if self.engine_use_ray:
951 + return await self.engine.get_scheduler_config.remote( # type: ignore
952 + )
953 + else:
954 + return self.engine.get_scheduler_config()
955 +
956 + async def get_lora_config(self) -> LoRAConfig:
957 + """Get the lora configuration of the vLLM engine."""
958 + if self.engine_use_ray:
959 + return await self.engine.get_lora_config.remote( # type: ignore
960 + )
961 + else:
962 + return self.engine.get_lora_config()
963 +
964 async def do_log_stats(
965 self,
966 scheduler_outputs: Optional[SchedulerOutputs] = None,
vllm/engine/llm_engine.py
+20 -16
@@ -38,9 +38,8 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
38 init_tracer)
39 from vllm.transformers_utils.config import try_get_generation_config
40 from vllm.transformers_utils.detokenizer import Detokenizer
41 -from vllm.transformers_utils.tokenizer_group import (AnyTokenizer,
42 - BaseTokenizerGroup,
43 - get_tokenizer_group)
41 +from vllm.transformers_utils.tokenizer_group import (
42 + AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
43 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
44 usage_message)
45 from vllm.utils import Counter
@@ -485,19 +484,12 @@ class LLMEngine:
484 return self.get_tokenizer_group().get_lora_tokenizer(
485 sequence.lora_request)
486
488 - def _init_tokenizer(self, **tokenizer_init_kwargs) -> BaseTokenizerGroup:
489 - init_kwargs = dict(
490 - tokenizer_id=self.model_config.tokenizer,
491 - enable_lora=bool(self.lora_config),
492 - max_num_seqs=self.scheduler_config.max_num_seqs,
493 - max_input_length=None,
494 - tokenizer_mode=self.model_config.tokenizer_mode,
495 - trust_remote_code=self.model_config.trust_remote_code,
496 - revision=self.model_config.tokenizer_revision)
497 - init_kwargs.update(tokenizer_init_kwargs)
498 -
499 - return get_tokenizer_group(self.parallel_config.tokenizer_pool_config,
500 - **init_kwargs)
487 + def _init_tokenizer(self) -> BaseTokenizerGroup:
488 + return init_tokenizer_from_configs(
489 + model_config=self.model_config,
490 + scheduler_config=self.scheduler_config,
491 + parallel_config=self.parallel_config,
492 + enable_lora=bool(self.lora_config))
493
494 def _verify_args(self) -> None:
495 self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -759,10 +751,22 @@ class LLMEngine:
751 """Gets the model configuration."""
752 return self.model_config
753
754 + def get_parallel_config(self) -> ParallelConfig:
755 + """Gets the parallel configuration."""
756 + return self.parallel_config
757 +
758 def get_decoding_config(self) -> DecodingConfig:
759 """Gets the decoding configuration."""
760 return self.decoding_config
761
762 + def get_scheduler_config(self) -> SchedulerConfig:
763 + """Gets the scheduler configuration."""
764 + return self.scheduler_config
765 +
766 + def get_lora_config(self) -> LoRAConfig:
767 + """Gets the LoRA configuration."""
768 + return self.lora_config
769 +
770 def get_num_unfinished_requests(self) -> int:
771 """Gets the number of unfinished requests."""
772 return sum(scheduler.get_num_unfinished_seq_groups()
vllm/engine/protocol.py new
+84
@@ -0,0 +1,84 @@
1 +from typing import (AsyncIterator, List, Mapping, Optional, Protocol,
2 + runtime_checkable)
3 +
4 +from transformers import PreTrainedTokenizer
5 +
6 +from vllm.config import DecodingConfig, ModelConfig
7 +from vllm.core.scheduler import SchedulerOutputs
8 +from vllm.inputs.data import PromptInputs
9 +from vllm.lora.request import LoRARequest
10 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
11 +from vllm.pooling_params import PoolingParams
12 +from vllm.prompt_adapter.request import PromptAdapterRequest
13 +from vllm.sampling_params import SamplingParams
14 +from vllm.sequence import SamplerOutput
15 +
16 +
17 +@runtime_checkable
18 +class AsyncEngineClient(Protocol):
19 + """Protocol class for Clients to AsyncLLMEngine"""
20 +
21 + @property
22 + def is_running(self) -> bool:
23 + ...
24 +
25 + @property
26 + def is_stopped(self) -> bool:
27 + ...
28 +
29 + @property
30 + def errored(self) -> bool:
31 + ...
32 +
33 + async def generate(
34 + self,
35 + inputs: PromptInputs,
36 + sampling_params: SamplingParams,
37 + request_id: str,
38 + lora_request: Optional[LoRARequest] = None,
39 + trace_headers: Optional[Mapping[str, str]] = None,
40 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
41 + ) -> AsyncIterator[RequestOutput]:
42 + """Generates outputs for a request"""
43 +
44 + async def encode(
45 + self,
46 + inputs: PromptInputs,
47 + pooling_params: PoolingParams,
48 + request_id: str,
49 + lora_request: Optional[LoRARequest] = None,
50 + trace_headers: Optional[Mapping[str, str]] = None,
51 + ) -> AsyncIterator[EmbeddingRequestOutput]:
52 + """Generate outputs for a request from an embedding model."""
53 +
54 + async def abort(self, request_id: str) -> None:
55 + """Abort a request.
56 +
57 + Args:
58 + request_id: The unique id of the request.
59 + """
60 +
61 + async def get_model_config(self) -> ModelConfig:
62 + """Get the model configuration of the vLLM engine."""
63 +
64 + async def get_decoding_config(self) -> DecodingConfig:
65 + """Get the decoding configuration of the vLLM engine."""
66 +
67 + async def get_tokenizer(
68 + self,
69 + lora_request: Optional[LoRARequest] = None,
70 + ) -> PreTrainedTokenizer:
71 + """Get the appropriate Tokenizer for the request"""
72 +
73 + async def is_tracing_enabled(self) -> bool:
74 + pass
75 +
76 + async def do_log_stats(
77 + self,
78 + scheduler_outputs: Optional[SchedulerOutputs] = None,
79 + model_output: Optional[List[SamplerOutput]] = None,
80 + ) -> None:
81 + pass
82 +
83 + async def check_health(self) -> None:
84 + """Raise if unhealthy"""
vllm/entrypoints/openai/api_server.py
+96 -36
@@ -5,7 +5,8 @@ import re
5 import signal
6 from contextlib import asynccontextmanager
7 from http import HTTPStatus
8 -from typing import Optional, Set
8 +from multiprocessing import Process
9 +from typing import AsyncIterator, Set
10
11 import fastapi
12 import uvicorn
@@ -17,8 +18,10 @@ from prometheus_client import make_asgi_app
18 from starlette.routing import Mount
19
20 import vllm.envs as envs
21 +from vllm.config import ModelConfig
22 from vllm.engine.arg_utils import AsyncEngineArgs
23 from vllm.engine.async_llm_engine import AsyncLLMEngine
24 +from vllm.engine.protocol import AsyncEngineClient
25 from vllm.entrypoints.logger import RequestLogger
26 from vllm.entrypoints.openai.cli_args import make_arg_parser
27 # yapf conflicts with isort for this block
@@ -31,6 +34,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
34 EmbeddingRequest, ErrorResponse,
35 TokenizeRequest,
36 TokenizeResponse)
37 +from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
38 +from vllm.entrypoints.openai.rpc.server import run_rpc_server
39 # yapf: enable
40 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
41 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -39,12 +44,12 @@ from vllm.entrypoints.openai.serving_tokenization import (
44 OpenAIServingTokenization)
45 from vllm.logger import init_logger
46 from vllm.usage.usage_lib import UsageContext
42 -from vllm.utils import FlexibleArgumentParser
47 +from vllm.utils import FlexibleArgumentParser, get_open_port
48 from vllm.version import __version__ as VLLM_VERSION
49
50 TIMEOUT_KEEP_ALIVE = 5 # seconds
51
47 -engine: AsyncLLMEngine
52 +async_engine_client: AsyncEngineClient
53 engine_args: AsyncEngineArgs
54 openai_serving_chat: OpenAIServingChat
55 openai_serving_completion: OpenAIServingCompletion
@@ -56,13 +61,22 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
61 _running_tasks: Set[asyncio.Task] = set()
62
63
64 +def model_is_embedding(model_name: str) -> bool:
65 + return ModelConfig(model=model_name,
66 + tokenizer=model_name,
67 + tokenizer_mode="auto",
68 + trust_remote_code=False,
69 + seed=0,
70 + dtype="float16").embedding_mode
71 +
72 +
73 @asynccontextmanager
74 async def lifespan(app: fastapi.FastAPI):
75
76 async def _force_log():
77 while True:
78 await asyncio.sleep(10)
65 - await engine.do_log_stats()
79 + await async_engine_client.do_log_stats()
80
81 if not engine_args.disable_log_stats:
82 task = asyncio.create_task(_force_log())
@@ -72,6 +86,52 @@ async def lifespan(app: fastapi.FastAPI):
86 yield
87
88
89 +@asynccontextmanager
90 +async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
91 + # Context manager to handle async_engine_client lifecycle
92 + # Ensures everything is shutdown and cleaned up on error/exit
93 + global engine_args
94 + engine_args = AsyncEngineArgs.from_cli_args(args)
95 +
96 + # Backend itself still global for the silly lil' health handler
97 + global async_engine_client
98 +
99 + # If manually triggered or embedding model, use AsyncLLMEngine in process.
100 + # TODO: support embedding model via RPC.
101 + if (model_is_embedding(args.model)
102 + or args.disable_frontend_multiprocessing):
103 + async_engine_client = AsyncLLMEngine.from_engine_args(
104 + engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
105 + yield async_engine_client
106 + return
107 +
108 + # Otherwise, use the multiprocessing AsyncLLMEngine.
109 + else:
110 + # Start RPCServer in separate process (holds the AsyncLLMEngine).
111 + port = get_open_port(envs.VLLM_RPC_PORT)
112 + rpc_server_process = Process(target=run_rpc_server,
113 + args=(engine_args,
114 + UsageContext.OPENAI_API_SERVER,
115 + port))
116 + rpc_server_process.start()
117 +
118 + # Build RPCClient, which conforms to AsyncEngineClient Protocol.
119 + async_engine_client = AsyncEngineRPCClient(port)
120 + await async_engine_client.setup()
121 +
122 + try:
123 + yield async_engine_client
124 + finally:
125 + # Ensure rpc server process was terminated
126 + rpc_server_process.terminate()
127 +
128 + # Close all open connections to the backend
129 + async_engine_client.close()
130 +
131 + # Wait for server process to join
132 + rpc_server_process.join()
133 +
134 +
135 router = APIRouter()
136
137
@@ -86,7 +146,7 @@ def mount_metrics(app: fastapi.FastAPI):
146 @router.get("/health")
147 async def health() -> Response:
148 """Health check."""
89 - await openai_serving_chat.engine.check_health()
149 + await async_engine_client.check_health()
150 return Response(status_code=200)
151
152
@@ -215,8 +275,8 @@ def build_app(args):
275
276
277 async def build_server(
278 + async_engine_client: AsyncEngineClient,
279 args,
219 - llm_engine: Optional[AsyncLLMEngine] = None,
280 **uvicorn_kwargs,
281 ) -> uvicorn.Server:
282 app = build_app(args)
@@ -226,14 +286,7 @@ async def build_server(
286 else:
287 served_model_names = [args.model]
288
229 - global engine, engine_args
230 -
231 - engine_args = AsyncEngineArgs.from_cli_args(args)
232 - engine = (llm_engine
233 - if llm_engine is not None else AsyncLLMEngine.from_engine_args(
234 - engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
235 -
236 - model_config = await engine.get_model_config()
289 + model_config = await async_engine_client.get_model_config()
290
291 if args.disable_log_requests:
292 request_logger = None
@@ -246,7 +299,7 @@ async def build_server(
299 global openai_serving_tokenization
300
301 openai_serving_chat = OpenAIServingChat(
249 - engine,
302 + async_engine_client,
303 model_config,
304 served_model_names,
305 args.response_role,
@@ -257,7 +310,7 @@ async def build_server(
310 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
311 )
312 openai_serving_completion = OpenAIServingCompletion(
260 - engine,
313 + async_engine_client,
314 model_config,
315 served_model_names,
316 lora_modules=args.lora_modules,
@@ -266,13 +319,13 @@ async def build_server(
319 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
320 )
321 openai_serving_embedding = OpenAIServingEmbedding(
269 - engine,
322 + async_engine_client,
323 model_config,
324 served_model_names,
325 request_logger=request_logger,
326 )
327 openai_serving_tokenization = OpenAIServingTokenization(
275 - engine,
328 + async_engine_client,
329 model_config,
330 served_model_names,
331 lora_modules=args.lora_modules,
@@ -304,32 +357,39 @@ async def build_server(
357 return uvicorn.Server(config)
358
359
307 -async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
360 +async def run_server(args, **uvicorn_kwargs) -> None:
361 logger.info("vLLM API server version %s", VLLM_VERSION)
362 logger.info("args: %s", args)
363
311 - server = await build_server(
312 - args,
313 - llm_engine,
314 - **uvicorn_kwargs,
315 - )
364 + shutdown_task = None
365 + async with build_async_engine_client(args) as async_engine_client:
366 +
367 + server = await build_server(
368 + async_engine_client,
369 + args,
370 + **uvicorn_kwargs,
371 + )
372 +
373 + loop = asyncio.get_running_loop()
374
317 - loop = asyncio.get_running_loop()
375 + server_task = loop.create_task(server.serve())
376
319 - server_task = loop.create_task(server.serve())
377 + def signal_handler() -> None:
378 + # prevents the uvicorn signal handler to exit early
379 + server_task.cancel()
380
321 - def signal_handler() -> None:
322 - # prevents the uvicorn signal handler to exit early
323 - server_task.cancel()
381 + loop.add_signal_handler(signal.SIGINT, signal_handler)
382 + loop.add_signal_handler(signal.SIGTERM, signal_handler)
383
325 - loop.add_signal_handler(signal.SIGINT, signal_handler)
326 - loop.add_signal_handler(signal.SIGTERM, signal_handler)
384 + try:
385 + await server_task
386 + except asyncio.CancelledError:
387 + logger.info("Gracefully stopping http server")
388 + shutdown_task = server.shutdown()
389
328 - try:
329 - await server_task
330 - except asyncio.CancelledError:
331 - print("Gracefully stopping http server")
332 - await server.shutdown()
390 + if shutdown_task:
391 + # NB: Await server shutdown only after the backend context is exited
392 + await shutdown_task
393
394
395 if __name__ == "__main__":
vllm/entrypoints/openai/cli_args.py
+7 -2
@@ -131,9 +131,14 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
131 parser.add_argument(
132 "--return-tokens-as-token-ids",
133 action="store_true",
134 - help="When --max-logprobs is specified, represents single tokens as"
135 - "strings of the form 'token_id:{token_id}' so that tokens that"
134 + help="When --max-logprobs is specified, represents single tokens as "
135 + "strings of the form 'token_id:{token_id}' so that tokens that "
136 "are not JSON-encodable can be identified.")
137 + parser.add_argument(
138 + "--disable-frontend-multiprocessing",
139 + action="store_true",
140 + help="If specified, will run the OpenAI frontend server in the same "
141 + "process as the model serving engine.")
142
143 parser = AsyncEngineArgs.add_cli_args(parser)
144
vllm/entrypoints/openai/logits_processors.py
+11 -8
@@ -1,4 +1,4 @@
1 -from functools import lru_cache
1 +from functools import lru_cache, partial
2 from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3
4 import torch
@@ -40,6 +40,14 @@ def _get_allowed_token_ids_logits_processor(
40 return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41
42
43 +def logit_bias_logits_processor(logit_bias: Dict[str,
44 + float], token_ids: List[int],
45 + logits: torch.Tensor) -> torch.Tensor:
46 + for token_id, bias in logit_bias.items():
47 + logits[token_id] += bias
48 + return logits
49 +
50 +
51 def get_logits_processors(
52 logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
53 allowed_token_ids: Optional[List[int]],
@@ -64,13 +72,8 @@ def get_logits_processors(
72 raise ValueError("token_id in logit_bias contains "
73 "out-of-vocab token id")
74
67 - def logit_bias_logits_processor(token_ids: List[int],
68 - logits: torch.Tensor) -> torch.Tensor:
69 - for token_id, bias in clamped_logit_bias.items():
70 - logits[token_id] += bias
71 - return logits
72 -
73 - logits_processors.append(logit_bias_logits_processor)
75 + logits_processors.append(
76 + partial(logit_bias_logits_processor, clamped_logit_bias))
77
78 if allowed_token_ids is not None:
79 logits_processors.append(
vllm/entrypoints/openai/rpc/__init__.py new
+42
@@ -0,0 +1,42 @@
1 +from dataclasses import dataclass
2 +from enum import Enum
3 +from typing import Mapping, Optional, Union
4 +
5 +from vllm.inputs import PromptInputs
6 +from vllm.lora.request import LoRARequest
7 +from vllm.prompt_adapter.request import PromptAdapterRequest
8 +from vllm.sampling_params import SamplingParams
9 +
10 +VLLM_RPC_SUCCESS_STR = "SUCCESS"
11 +VLLM_RPC_HEALTHY_STR = "HEALTHY"
12 +
13 +
14 +@dataclass
15 +class RPCGenerateRequest:
16 + inputs: PromptInputs
17 + sampling_params: SamplingParams
18 + request_id: str
19 + lora_request: Optional[LoRARequest] = None
20 + trace_headers: Optional[Mapping[str, str]] = None
21 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
22 +
23 +
24 +@dataclass
25 +class RPCAbortRequest:
26 + request_id: str
27 +
28 +
29 +class RPCUtilityRequest(Enum):
30 + IS_SERVER_READY = 1
31 + GET_MODEL_CONFIG = 2
32 + GET_DECODING_CONFIG = 3
33 + GET_PARALLEL_CONFIG = 4
34 + GET_SCHEDULER_CONFIG = 5
35 + GET_LORA_CONFIG = 6
36 + DO_LOG_STATS = 7
37 + CHECK_HEALTH = 8
38 + IS_TRACING_ENABLED = 9
39 +
40 +
41 +RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
42 + RPCUtilityRequest]
vllm/entrypoints/openai/rpc/client.py new
+248
@@ -0,0 +1,248 @@
1 +from contextlib import contextmanager
2 +from typing import Any, AsyncIterator, Mapping, Optional
3 +
4 +import cloudpickle
5 +import zmq
6 +import zmq.asyncio
7 +
8 +from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
9 + ParallelConfig, SchedulerConfig)
10 +from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
11 + VLLM_RPC_HEALTHY_STR,
12 + VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
13 + RPCGenerateRequest, RPCUtilityRequest)
14 +from vllm.inputs import PromptInputs
15 +from vllm.lora.request import LoRARequest
16 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
17 +from vllm.prompt_adapter.request import PromptAdapterRequest
18 +from vllm.sampling_params import SamplingParams
19 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
20 +
21 +
22 +class AsyncEngineRPCClient:
23 +
24 + def __init__(self, port: int):
25 + self.context = zmq.asyncio.Context()
26 + self.path = f"tcp://localhost:{port}"
27 +
28 + async def setup(self):
29 + """Setup the client before it starts sending server requests."""
30 +
31 + # Wait until server is ready.
32 + await self.wait_for_server()
33 +
34 + # Get the configs.
35 + self.model_config = await self._get_model_config_rpc()
36 + self.decoding_config = await self._get_decoding_config_rpc()
37 + self.tracing_flag = await self._is_tracing_enabled_rpc()
38 +
39 + # Create the tokenizer group.
40 + # TODO: refactor OAI server to avoid needing this info.
41 + self.tokenizer = init_tokenizer_from_configs(
42 + model_config=self.model_config,
43 + scheduler_config=(await self._get_scheduler_config_rpc()),
44 + parallel_config=(await self._get_parallel_config_rpc()),
45 + enable_lora=bool(await self._get_lora_config_rpc()),
46 + )
47 +
48 + def close(self):
49 + """Destroy the ZeroMQ Context."""
50 + self.context.destroy()
51 +
52 + @contextmanager
53 + def socket(self):
54 + # Ensure client sockets are always closed after use
55 +
56 + # Connect to RPC socket for Request-Reply pattern,
57 + # Note that we use DEALER to enable asynchronous communication
58 + # to enable streaming.
59 + socket = self.context.socket(zmq.constants.DEALER)
60 + try:
61 + socket.connect(self.path)
62 + yield socket
63 + finally:
64 + socket.close()
65 +
66 + async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
67 + expected_type: Any,
68 + error_message: str) -> Any:
69 + """Send an RPC request that is expecting data back."""
70 +
71 + with self.socket() as socket:
72 +
73 + # Ping RPCServer with a request.
74 + await socket.send(cloudpickle.dumps(request))
75 +
76 + # Await the data from the Server.
77 + data = cloudpickle.loads(await socket.recv())
78 +
79 + if not isinstance(data, expected_type):
80 + # LoRAConfig can be None.
81 + if expected_type == LoRAConfig and data is None:
82 + pass
83 + else:
84 + raise ValueError(error_message)
85 +
86 + return data
87 +
88 + async def _send_one_way_rpc_request(self, request: RPC_REQUEST_TYPE,
89 + error_message: str):
90 + """Send one-way RPC request to trigger an action."""
91 + with self.socket() as socket:
92 + # Ping RPC Server with request.
93 + await socket.send(cloudpickle.dumps(request))
94 +
95 + # Await acknowledgement from RPCServer.
96 + response = cloudpickle.loads(await socket.recv())
97 +
98 + if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
99 + raise ValueError(error_message)
100 +
101 + return response
102 +
103 + async def get_tokenizer(self, lora_request: LoRARequest):
104 + return await self.tokenizer.get_lora_tokenizer_async(lora_request)
105 +
106 + async def get_decoding_config(self) -> DecodingConfig:
107 + return self.decoding_config
108 +
109 + async def get_model_config(self) -> ModelConfig:
110 + return self.model_config
111 +
112 + async def is_tracing_enabled(self) -> bool:
113 + return self.tracing_flag
114 +
115 + async def wait_for_server(self):
116 + """Wait for the RPCServer to start up."""
117 +
118 + await self._send_one_way_rpc_request(
119 + request=RPCUtilityRequest.IS_SERVER_READY,
120 + error_message="Unable to start RPC Server.")
121 +
122 + async def _get_model_config_rpc(self) -> ModelConfig:
123 + """Get the ModelConfig object from the RPC Server"""
124 +
125 + return await self._send_get_data_rpc_request(
126 + RPCUtilityRequest.GET_MODEL_CONFIG,
127 + expected_type=ModelConfig,
128 + error_message="Could not get ModelConfig from RPC Server")
129 +
130 + async def _get_decoding_config_rpc(self) -> DecodingConfig:
131 + """Get DecodingConfig from the RPCServer"""
132 +
133 + return await self._send_get_data_rpc_request(
134 + RPCUtilityRequest.GET_DECODING_CONFIG,
135 + expected_type=DecodingConfig,
136 + error_message="Could not get DecodingConfig from RPC Server")
137 +
138 + async def _get_parallel_config_rpc(self) -> ParallelConfig:
139 + """Get ParallelConfig from the RPCServer"""
140 +
141 + return await self._send_get_data_rpc_request(
142 + RPCUtilityRequest.GET_PARALLEL_CONFIG,
143 + expected_type=ParallelConfig,
144 + error_message="Could not get ParallelConfig from RPC Server")
145 +
146 + async def _get_scheduler_config_rpc(self) -> SchedulerConfig:
147 + """Get SchedulerConfig from the RPCServer"""
148 +
149 + return await self._send_get_data_rpc_request(
150 + RPCUtilityRequest.GET_SCHEDULER_CONFIG,
151 + expected_type=SchedulerConfig,
152 + error_message="Could not get SchedulerConfig from RPC Server")
153 +
154 + async def _get_lora_config_rpc(self):
155 + """Get LoRAConfig from the RPCServer"""
156 +
157 + return await self._send_get_data_rpc_request(
158 + RPCUtilityRequest.GET_LORA_CONFIG,
159 + expected_type=LoRAConfig,
160 + error_message="Could not get LoRAConfig from RPC Server")
161 +
162 + async def _is_tracing_enabled_rpc(self) -> ParallelConfig:
163 + """Get is_tracing_enabled flag from the RPCServer"""
164 +
165 + return await self._send_get_data_rpc_request(
166 + RPCUtilityRequest.IS_TRACING_ENABLED,
167 + expected_type=bool,
168 + error_message="Could not get is_tracing_enabled flag from RPC "
169 + "Server")
170 +
171 + async def abort(self, request_id: str):
172 + """Send an ABORT_REQUEST signal to the RPC Server"""
173 +
174 + await self._send_one_way_rpc_request(
175 + request=RPCAbortRequest(request_id),
176 + error_message=f"RPCAbortRequest {request_id} failed")
177 +
178 + async def do_log_stats(self):
179 + """Send a DO_LOG_STATS signal to the RPC Server"""
180 +
181 + await self._send_one_way_rpc_request(
182 + request=RPCUtilityRequest.DO_LOG_STATS,
183 + error_message="RPCRequest DO_LOG_STATS failed.")
184 +
185 + async def generate(
186 + self,
187 + inputs: PromptInputs,
188 + sampling_params: SamplingParams,
189 + request_id: str,
190 + lora_request: Optional[LoRARequest] = None,
191 + trace_headers: Optional[Mapping[str, str]] = None,
192 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
193 + ) -> AsyncIterator[RequestOutput]:
194 + """Send an RPCGenerateRequest to the RPCServer and stream responses."""
195 +
196 + with self.socket() as socket:
197 +
198 + # Send RPCGenerateRequest to the RPCServer.
199 + await socket.send_multipart([
200 + cloudpickle.dumps(
201 + RPCGenerateRequest(
202 + inputs=inputs,
203 + sampling_params=sampling_params,
204 + request_id=request_id,
205 + lora_request=lora_request,
206 + trace_headers=trace_headers,
207 + prompt_adapter_request=prompt_adapter_request))
208 + ])
209 +
210 + # Stream back the results from the RPC Server.
211 + while True:
212 + message = await socket.recv()
213 + request_output = cloudpickle.loads(message)
214 +
215 + if isinstance(request_output, Exception):
216 + raise request_output
217 +
218 + if request_output.finished:
219 + break
220 + yield request_output
221 +
222 + yield request_output
223 +
224 + async def check_health(self) -> None:
225 + """Raise if unhealthy"""
226 +
227 + with self.socket() as socket:
228 +
229 + # Ping RPCServer with CHECK_HEALTH request.
230 + await socket.send(cloudpickle.dumps(RPCUtilityRequest.CHECK_HEALTH)
231 + )
232 +
233 + # Await the reply from the server.
234 + # TODO: do we need an internal timeout here?
235 + # Or do we expect the external probe to timeout and let this chill?
236 + health_message = cloudpickle.loads(await socket.recv())
237 +
238 + if isinstance(health_message, Exception):
239 + raise health_message
240 +
241 + if health_message != VLLM_RPC_HEALTHY_STR:
242 + raise ValueError("Expected healthy response from backend but got "
243 + "f{health_message}")
244 +
245 + async def encode(self, *args,
246 + **kwargs) -> AsyncIterator[EmbeddingRequestOutput]:
247 + raise NotImplementedError(
248 + "Embeddings not supported with multiprocessing backend")
vllm/entrypoints/openai/rpc/server.py new
+216
@@ -0,0 +1,216 @@
1 +import asyncio
2 +import signal
3 +from typing import Any, Coroutine
4 +
5 +import cloudpickle
6 +import zmq
7 +import zmq.asyncio
8 +from typing_extensions import Never
9 +
10 +from vllm import AsyncEngineArgs, AsyncLLMEngine
11 +from vllm.entrypoints.openai.rpc import (VLLM_RPC_HEALTHY_STR,
12 + VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
13 + RPCGenerateRequest, RPCUtilityRequest)
14 +from vllm.logger import init_logger
15 +from vllm.usage.usage_lib import UsageContext
16 +
17 +logger = init_logger(__name__)
18 +
19 +
20 +class AsyncEngineRPCServer:
21 +
22 + def __init__(self, async_engine_args: AsyncEngineArgs,
23 + usage_context: UsageContext, port: int):
24 + # Initialize engine first.
25 + self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
26 + usage_context)
27 +
28 + # Initialize context.
29 + self.context = zmq.asyncio.Context()
30 +
31 + # Init socket for readiness state.
32 + self.socket = self.context.socket(zmq.constants.ROUTER)
33 + self.socket.bind(f"tcp://localhost:{port}")
34 +
35 + def cleanup(self):
36 + """Cleanup all resources."""
37 + self.socket.close()
38 + self.context.destroy()
39 +
40 + async def get_model_config(self, identity):
41 + """Send the ModelConfig"""
42 + model_config = await self.engine.get_model_config()
43 +
44 + await self.socket.send_multipart(
45 + [identity, cloudpickle.dumps(model_config)])
46 +
47 + async def get_decoding_config(self, identity):
48 + """Send the DecodingConfig"""
49 + decoding_config = await self.engine.get_decoding_config()
50 +
51 + await self.socket.send_multipart(
52 + [identity, cloudpickle.dumps(decoding_config)])
53 +
54 + async def get_lora_config(self, identity):
55 + lora_config = await self.engine.get_lora_config()
56 +
57 + await self.socket.send_multipart(
58 + [identity, cloudpickle.dumps(lora_config)])
59 +
60 + async def get_scheduler_config(self, identity):
61 + """Send the SchedulerConfig"""
62 + parallel_config = await self.engine.get_scheduler_config()
63 +
64 + await self.socket.send_multipart(
65 + [identity, cloudpickle.dumps(parallel_config)])
66 +
67 + async def get_parallel_config(self, identity):
68 + """Send the ParallelConfig"""
69 + parallel_config = await self.engine.get_parallel_config()
70 +
71 + await self.socket.send_multipart(
72 + [identity, cloudpickle.dumps(parallel_config)])
73 +
74 + async def is_tracing_enabled(self, identity):
75 + """Send the is_tracing_enabled flag"""
76 + tracing_flag = await self.engine.is_tracing_enabled()
77 +
78 + await self.socket.send_multipart(
79 + [identity, cloudpickle.dumps(tracing_flag)])
80 +
81 + async def do_log_stats(self, identity):
82 + """Log stats and confirm success."""
83 + await self.engine.do_log_stats()
84 +
85 + await self.socket.send_multipart([
86 + identity,
87 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
88 + ])
89 +
90 + async def is_server_ready(self, identity):
91 + """Notify the client that we are ready."""
92 + await self.socket.send_multipart([
93 + identity,
94 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
95 + ])
96 +
97 + async def abort(self, identity, request: RPCAbortRequest):
98 + """Abort request and notify the client of success."""
99 + # Abort the request in the llm engine.
100 + await self.engine.abort(request.request_id)
101 +
102 + # Send confirmation to the client.
103 + await self.socket.send_multipart([
104 + identity,
105 + cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
106 + ])
107 +
108 + async def generate(self, identity, generate_request: RPCGenerateRequest):
109 + try:
110 + results_generator = self.engine.generate(
111 + generate_request.inputs,
112 + sampling_params=generate_request.sampling_params,
113 + request_id=generate_request.request_id,
114 + lora_request=generate_request.lora_request,
115 + trace_headers=generate_request.trace_headers,
116 + prompt_adapter_request=generate_request.prompt_adapter_request)
117 +
118 + async for request_output in results_generator:
119 + await self.socket.send_multipart(
120 + [identity, cloudpickle.dumps(request_output)])
121 +
122 + except Exception as e:
123 + ### Notify client of all failures
124 + await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
125 +
126 + async def check_health(self, identity):
127 + try:
128 + await self.engine.check_health()
129 + await self.socket.send_multipart(
130 + [identity, cloudpickle.dumps(VLLM_RPC_HEALTHY_STR)])
131 + except Exception as e:
132 + await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
133 +
134 + def _make_handler_coro(self, identity,
135 + message) -> Coroutine[Any, Any, Never]:
136 + """Route the zmq message to the handler coroutine."""
137 +
138 + request = cloudpickle.loads(message)
139 +
140 + if isinstance(request, RPCGenerateRequest):
141 + return self.generate(identity, request)
142 +
143 + elif isinstance(request, RPCAbortRequest):
144 + return self.abort(identity, request)
145 +
146 + elif isinstance(request, RPCUtilityRequest):
147 + if request == RPCUtilityRequest.GET_MODEL_CONFIG:
148 + return self.get_model_config(identity)
149 + elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
150 + return self.get_parallel_config(identity)
151 + elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
152 + return self.get_decoding_config(identity)
153 + elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
154 + return self.get_scheduler_config(identity)
155 + elif request == RPCUtilityRequest.GET_LORA_CONFIG:
156 + return self.get_lora_config(identity)
157 + elif request == RPCUtilityRequest.DO_LOG_STATS:
158 + return self.do_log_stats(identity)
159 + elif request == RPCUtilityRequest.IS_SERVER_READY:
160 + return self.is_server_ready(identity)
161 + elif request == RPCUtilityRequest.CHECK_HEALTH:
162 + return self.check_health(identity)
163 + elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
164 + return self.is_tracing_enabled(identity)
165 + else:
166 + raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
167 +
168 + else:
169 + raise ValueError(f"Unknown RPCRequest type: {request}")
170 +
171 + async def run_server_loop(self):
172 + """Inner RPC Server Loop"""
173 +
174 + running_tasks = set()
175 + while True:
176 + # Wait for a request.
177 + identity, message = await self.socket.recv_multipart()
178 +
179 + # Process the request async.
180 + task = asyncio.create_task(
181 + self._make_handler_coro(identity, message))
182 +
183 + # We need to keep around a strong reference to the task,
184 + # to avoid the task disappearing mid-execution as running tasks
185 + # can be GC'ed. Below is a common "fire-and-forget" tasks
186 + # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
187 + running_tasks.add(task)
188 + task.add_done_callback(running_tasks.discard)
189 +
190 +
191 +async def run_server(server: AsyncEngineRPCServer):
192 + # Put the server task into the asyncio loop.
193 + loop = asyncio.get_running_loop()
194 + server_task = loop.create_task(server.run_server_loop())
195 +
196 + # Interruption handling.
197 + def signal_handler() -> None:
198 + # Kill the server on interrupt / terminate
199 + server_task.cancel()
200 +
201 + loop.add_signal_handler(signal.SIGINT, signal_handler)
202 + loop.add_signal_handler(signal.SIGTERM, signal_handler)
203 +
204 + try:
205 + await server_task
206 + except asyncio.CancelledError:
207 + logger.info("vLLM ZMQ RPC Server was interrupted.")
208 + finally:
209 + # Clean up all resources.
210 + server.cleanup()
211 +
212 +
213 +def run_rpc_server(async_engine_args: AsyncEngineArgs,
214 + usage_context: UsageContext, port: int):
215 + server = AsyncEngineRPCServer(async_engine_args, usage_context, port)
216 + asyncio.run(run_server(server))
vllm/entrypoints/openai/serving_chat.py
+9 -7
@@ -8,7 +8,7 @@ from fastapi import Request
8 from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.engine.protocol import AsyncEngineClient
12 from vllm.entrypoints.chat_utils import (ConversationMessage,
13 load_chat_template,
14 parse_chat_message_content)
@@ -39,7 +39,7 @@ class OpenAIServingChat(OpenAIServing):
39
40 def __init__(
41 self,
42 - engine: AsyncLLMEngine,
42 + async_engine_client: AsyncEngineClient,
43 model_config: ModelConfig,
44 served_model_names: List[str],
45 response_role: str,
@@ -50,7 +50,7 @@ class OpenAIServingChat(OpenAIServing):
50 chat_template: Optional[str],
51 return_tokens_as_token_ids: bool = False,
52 ):
53 - super().__init__(engine=engine,
53 + super().__init__(async_engine_client=async_engine_client,
54 model_config=model_config,
55 served_model_names=served_model_names,
56 lora_modules=lora_modules,
@@ -89,7 +89,8 @@ class OpenAIServingChat(OpenAIServing):
89 ) = self._maybe_get_adapters(request)
90
91 model_config = self.model_config
92 - tokenizer = await self.engine.get_tokenizer(lora_request)
92 + tokenizer = await self.async_engine_client.get_tokenizer(
93 + lora_request)
94
95 conversation: List[ConversationMessage] = []
96 mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -161,7 +162,8 @@ class OpenAIServingChat(OpenAIServing):
162 if mm_data is not None:
163 engine_inputs["multi_modal_data"] = mm_data
164
164 - is_tracing_enabled = await self.engine.is_tracing_enabled()
165 + is_tracing_enabled = (
166 + await self.async_engine_client.is_tracing_enabled())
167 trace_headers = None
168 if is_tracing_enabled and raw_request:
169 trace_headers = extract_trace_headers(raw_request.headers)
@@ -169,7 +171,7 @@ class OpenAIServingChat(OpenAIServing):
171 and contains_trace_headers(raw_request.headers)):
172 log_tracing_disabled_warning()
173
172 - result_generator = self.engine.generate(
174 + result_generator = self.async_engine_client.generate(
175 engine_inputs,
176 sampling_params,
177 request_id,
@@ -441,7 +443,7 @@ class OpenAIServingChat(OpenAIServing):
443 async for res in result_generator:
444 if raw_request is not None and await raw_request.is_disconnected():
445 # Abort the request if the client disconnects.
444 - await self.engine.abort(request_id)
446 + await self.async_engine_client.abort(request_id)
447 return self.create_error_response("Client disconnected")
448 final_res = res
449 assert final_res is not None
vllm/entrypoints/openai/serving_completion.py
+11 -8
@@ -8,7 +8,7 @@ from fastapi import Request
8 from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.engine.protocol import AsyncEngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
14 # yapf: disable
@@ -42,7 +42,7 @@ class OpenAIServingCompletion(OpenAIServing):
42
43 def __init__(
44 self,
45 - engine: AsyncLLMEngine,
45 + async_engine_client: AsyncEngineClient,
46 model_config: ModelConfig,
47 served_model_names: List[str],
48 *,
@@ -51,7 +51,7 @@ class OpenAIServingCompletion(OpenAIServing):
51 request_logger: Optional[RequestLogger],
52 return_tokens_as_token_ids: bool = False,
53 ):
54 - super().__init__(engine=engine,
54 + super().__init__(async_engine_client=async_engine_client,
55 model_config=model_config,
56 served_model_names=served_model_names,
57 lora_modules=lora_modules,
@@ -91,7 +91,8 @@ class OpenAIServingCompletion(OpenAIServing):
91 prompt_adapter_request,
92 ) = self._maybe_get_adapters(request)
93
94 - tokenizer = await self.engine.get_tokenizer(lora_request)
94 + tokenizer = await self.async_engine_client.get_tokenizer(
95 + lora_request)
96
97 guided_decode_logits_processor = (
98 await self._guided_decode_logits_processor(request, tokenizer))
@@ -119,7 +120,8 @@ class OpenAIServingCompletion(OpenAIServing):
120 lora_request=lora_request,
121 prompt_adapter_request=prompt_adapter_request)
122
122 - is_tracing_enabled = await self.engine.is_tracing_enabled()
123 + is_tracing_enabled = (
124 + await self.async_engine_client.is_tracing_enabled())
125 trace_headers = None
126 if is_tracing_enabled:
127 trace_headers = extract_trace_headers(raw_request.headers)
@@ -127,7 +129,7 @@ class OpenAIServingCompletion(OpenAIServing):
129 raw_request.headers):
130 log_tracing_disabled_warning()
131
130 - generator = self.engine.generate(
132 + generator = self.async_engine_client.generate(
133 {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
134 sampling_params,
135 request_id_item,
@@ -168,7 +170,7 @@ class OpenAIServingCompletion(OpenAIServing):
170 async for i, res in result_generator:
171 if await raw_request.is_disconnected():
172 # Abort the request if the client disconnects.
171 - await self.engine.abort(f"{request_id}-{i}")
173 + await self.async_engine_client.abort(f"{request_id}-{i}")
174 return self.create_error_response("Client disconnected")
175 final_res_batch[i] = res
176
@@ -230,7 +232,8 @@ class OpenAIServingCompletion(OpenAIServing):
232
233 # Abort the request if the client disconnects.
234 if await raw_request.is_disconnected():
233 - await self.engine.abort(f"{request_id}-{prompt_idx}")
235 + await self.async_engine_client.abort(
236 + f"{request_id}-{prompt_idx}")
237 raise StopAsyncIteration()
238
239 for output in res.outputs:
vllm/entrypoints/openai/serving_embedding.py
+7 -6
@@ -6,7 +6,7 @@ import numpy as np
6 from fastapi import Request
7
8 from vllm.config import ModelConfig
9 -from vllm.engine.async_llm_engine import AsyncLLMEngine
9 +from vllm.engine.protocol import AsyncEngineClient
10 from vllm.entrypoints.logger import RequestLogger
11 from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
12 EmbeddingResponse,
@@ -56,13 +56,13 @@ class OpenAIServingEmbedding(OpenAIServing):
56
57 def __init__(
58 self,
59 - engine: AsyncLLMEngine,
59 + async_engine_client: AsyncEngineClient,
60 model_config: ModelConfig,
61 served_model_names: List[str],
62 *,
63 request_logger: Optional[RequestLogger],
64 ):
65 - super().__init__(engine=engine,
65 + super().__init__(async_engine_client=async_engine_client,
66 model_config=model_config,
67 served_model_names=served_model_names,
68 lora_modules=None,
@@ -99,7 +99,8 @@ class OpenAIServingEmbedding(OpenAIServing):
99 prompt_adapter_request,
100 ) = self._maybe_get_adapters(request)
101
102 - tokenizer = await self.engine.get_tokenizer(lora_request)
102 + tokenizer = await self.async_engine_client.get_tokenizer(
103 + lora_request)
104
105 pooling_params = request.to_pooling_params()
106
@@ -124,7 +125,7 @@ class OpenAIServingEmbedding(OpenAIServing):
125 "Prompt adapter is not supported "
126 "for embedding models")
127
127 - generator = self.engine.encode(
128 + generator = self.async_engine_client.encode(
129 {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
130 pooling_params,
131 request_id_item,
@@ -146,7 +147,7 @@ class OpenAIServingEmbedding(OpenAIServing):
147 async for i, res in result_generator:
148 if await raw_request.is_disconnected():
149 # Abort the request if the client disconnects.
149 - await self.engine.abort(f"{request_id}-{i}")
150 + await self.async_engine_client.abort(f"{request_id}-{i}")
151 return self.create_error_response("Client disconnected")
152 final_res_batch[i] = res
153
vllm/entrypoints/openai/serving_engine.py
+4 -4
@@ -8,7 +8,7 @@ from pydantic import Field
8 from typing_extensions import Annotated
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.async_llm_engine import AsyncLLMEngine
11 +from vllm.engine.protocol import AsyncEngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
14 # yapf: disable
@@ -61,7 +61,7 @@ class OpenAIServing:
61
62 def __init__(
63 self,
64 - engine: AsyncLLMEngine,
64 + async_engine_client: AsyncEngineClient,
65 model_config: ModelConfig,
66 served_model_names: List[str],
67 *,
@@ -72,7 +72,7 @@ class OpenAIServing:
72 ):
73 super().__init__()
74
75 - self.engine = engine
75 + self.async_engine_client = async_engine_client
76 self.model_config = model_config
77 self.max_model_len = model_config.max_model_len
78
@@ -155,7 +155,7 @@ class OpenAIServing:
155 async def _guided_decode_logits_processor(
156 self, request: Union[ChatCompletionRequest, CompletionRequest],
157 tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
158 - decoding_config = await self.engine.get_decoding_config()
158 + decoding_config = await self.async_engine_client.get_decoding_config()
159 guided_decoding_backend = request.guided_decoding_backend \
160 or decoding_config.guided_decoding_backend
161 return await get_guided_decoding_logits_processor(
vllm/entrypoints/openai/serving_tokenization.py
+5 -5
@@ -1,9 +1,9 @@
1 from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 -from vllm.engine.async_llm_engine import AsyncLLMEngine
4 # yapf conflicts with isort for this block
5 # yapf: disable
6 +from vllm.engine.protocol import AsyncEngineClient
7 from vllm.entrypoints.chat_utils import (ConversationMessage,
8 load_chat_template,
9 parse_chat_message_content)
@@ -24,7 +24,7 @@ class OpenAIServingTokenization(OpenAIServing):
24
25 def __init__(
26 self,
27 - engine: AsyncLLMEngine,
27 + async_engine_client: AsyncEngineClient,
28 model_config: ModelConfig,
29 served_model_names: List[str],
30 *,
@@ -32,7 +32,7 @@ class OpenAIServingTokenization(OpenAIServing):
32 request_logger: Optional[RequestLogger],
33 chat_template: Optional[str],
34 ):
35 - super().__init__(engine=engine,
35 + super().__init__(async_engine_client=async_engine_client,
36 model_config=model_config,
37 served_model_names=served_model_names,
38 lora_modules=lora_modules,
@@ -57,7 +57,7 @@ class OpenAIServingTokenization(OpenAIServing):
57 prompt_adapter_request,
58 ) = self._maybe_get_adapters(request)
59
60 - tokenizer = await self.engine.get_tokenizer(lora_request)
60 + tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
61
62 if isinstance(request, TokenizeChatRequest):
63 model_config = self.model_config
@@ -113,7 +113,7 @@ class OpenAIServingTokenization(OpenAIServing):
113 prompt_adapter_request,
114 ) = self._maybe_get_adapters(request)
115
116 - tokenizer = await self.engine.get_tokenizer(lora_request)
116 + tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
117
118 self._log_inputs(request_id,
119 request.tokens,
vllm/envs.py
+6
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
4 if TYPE_CHECKING:
5 VLLM_HOST_IP: str = ""
6 VLLM_PORT: Optional[int] = None
7 + VLLM_RPC_PORT: int = 5570
8 VLLM_USE_MODELSCOPE: bool = False
9 VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
10 VLLM_INSTANCE_ID: Optional[str] = None
@@ -140,6 +141,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
141 lambda: int(os.getenv('VLLM_PORT', '0'))
142 if 'VLLM_PORT' in os.environ else None,
143
144 + # used when the frontend api server is running in multi-processing mode,
145 + # to communicate with the backend engine process over ZMQ.
146 + 'VLLM_RPC_PORT':
147 + lambda: int(os.getenv('VLLM_PORT', '5570')),
148 +
149 # If true, will load models from ModelScope instead of Hugging Face Hub.
150 # note that the value is true or false, not numbers
151 "VLLM_USE_MODELSCOPE":
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+19
@@ -21,6 +21,8 @@ from functools import lru_cache
21 from typing import Callable, DefaultDict, Dict, List, Union
22
23 import torch
24 +from lark import Lark
25 +from outlines import grammars
26 from outlines.caching import cache
27 from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
28 from outlines.fsm.json_schema import build_regex_from_schema
@@ -44,6 +46,23 @@ class BaseLogitsProcessor:
46 last_seq_id = hash(tuple(input_ids[:-1]))
47 self._fsm_state[seq_id] = self._guide.get_next_state(
48 state=self._fsm_state[last_seq_id], token_id=last_token)
49 + else:
50 + # Note: this is a hack.
51 + # Lark pickling does not work properly (silent failure),
52 + # which breaks the RPC (which uses python pickleing).
53 + # We need to find a better solution.
54 + # On the first time this is called, we simply re-create
55 + # the Lark object.
56 + if isinstance(self._guide, CFGGuide):
57 + self._guide.parser = Lark(
58 + self._guide.cfg_string,
59 + parser="lalr",
60 + lexer="contextual",
61 + propagate_positions=False,
62 + maybe_placeholders=False,
63 + regex=True,
64 + import_paths=[grammars.GRAMMAR_PATH],
65 + )
66
67 instruction = self._guide.get_next_instruction(
68 state=self._fsm_state[seq_id])
vllm/tracing.py
+1 -1
@@ -60,7 +60,7 @@ def get_span_exporter(endpoint):
60 OTLPSpanExporter)
61 elif protocol == "http/protobuf":
62 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
63 - OTLPSpanExporter)
63 + OTLPSpanExporter) # type: ignore
64 else:
65 raise ValueError(
66 f"Unsupported OTLP protocol '{protocol}' is configured")
vllm/transformers_utils/tokenizer_group/__init__.py
+18 -1
@@ -1,6 +1,7 @@
1 from typing import Optional, Type
2
3 -from vllm.config import TokenizerPoolConfig
3 +from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
4 + TokenizerPoolConfig)
5 from vllm.executor.ray_utils import ray
6
7 from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
@@ -13,6 +14,22 @@ else:
14 RayTokenizerGroupPool = None # type: ignore
15
16
17 +def init_tokenizer_from_configs(model_config: ModelConfig,
18 + scheduler_config: SchedulerConfig,
19 + parallel_config: ParallelConfig,
20 + enable_lora: bool):
21 + init_kwargs = dict(tokenizer_id=model_config.tokenizer,
22 + enable_lora=enable_lora,
23 + max_num_seqs=scheduler_config.max_num_seqs,
24 + max_input_length=None,
25 + tokenizer_mode=model_config.tokenizer_mode,
26 + trust_remote_code=model_config.trust_remote_code,
27 + revision=model_config.tokenizer_revision)
28 +
29 + return get_tokenizer_group(parallel_config.tokenizer_pool_config,
30 + **init_kwargs)
31 +
32 +
33 def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
34 **init_kwargs) -> BaseTokenizerGroup:
35 tokenizer_cls: Type[BaseTokenizerGroup]
vllm/utils.py
+22 -6
@@ -290,6 +290,10 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
290 return _async_wrapper
291
292
293 +class ProducerFinished:
294 + pass
295 +
296 +
297 def merge_async_iterators(
298 *iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
299 """Merge multiple asynchronous iterators into a single iterator.
@@ -298,9 +302,10 @@ def merge_async_iterators(
302 When it yields, it yields a tuple (i, item) where i is the index of the
303 iterator that yields the item.
304 """
301 - queue: asyncio.Queue[Union[Tuple[int, T], Exception]] = asyncio.Queue()
305 + queue: asyncio.Queue[Union[Tuple[int, T], ProducerFinished,
306 + Exception]] = asyncio.Queue()
307
303 - finished = [False] * len(iterators)
308 + producers = len(iterators)
309
310 async def producer(i: int, iterator: AsyncIterator[T]):
311 try:
@@ -308,7 +313,8 @@ def merge_async_iterators(
313 await queue.put((i, item))
314 except Exception as e:
315 await queue.put(e)
311 - finished[i] = True
316 + # Signal to the consumer that we've finished
317 + await queue.put(ProducerFinished())
318
319 _tasks = [
320 asyncio.create_task(producer(i, iterator))
@@ -316,9 +322,17 @@ def merge_async_iterators(
322 ]
323
324 async def consumer():
325 + remaining = producers
326 try:
320 - while not all(finished) or not queue.empty():
327 + while remaining or not queue.empty():
328 + # we think there is a race condition here
329 item = await queue.get()
330 +
331 + if isinstance(item, ProducerFinished):
332 + # Signal that a producer finished- not a real item
333 + remaining -= 1
334 + continue
335 +
336 if isinstance(item, Exception):
337 raise item
338 yield item
@@ -374,8 +388,10 @@ def get_distributed_init_method(ip: str, port: int) -> str:
388 return f"tcp://[{ip}]:{port}" if ":" in ip else f"tcp://{ip}:{port}"
389
390
377 -def get_open_port() -> int:
378 - port = envs.VLLM_PORT
391 +def get_open_port(port: Optional[int] = None) -> int:
392 + if port is None:
393 + # Default behavior here is to return a port for multi-gpu communication
394 + port = envs.VLLM_PORT
395 if port is not None:
396 while True:
397 try: