A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[ Frontend ] Multiprocessing for OpenAI Server with `zeromq` (#6883)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Joe Runde <joe@joerun.de> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
Robert Shaw committed
Aug 2, 2024 at 21:27 UTC
ed812a73fae77bb520b739cfeaad36dbd61e2b03
20 files changed
+1567
-101
tests/entrypoints/openai/test_disable_mp.py
new
+715
@@ -0,0 +1,715 @@
1
+"""
2
+Repeat of tests in test_completion.py with the non-mp backend.
3
+"""
4
+
5
+# imports for guided decoding tests
6
+import json
7
+import re
8
+import shutil
9
+from tempfile import TemporaryDirectory
10
+from typing import List
11
+
12
+import jsonschema
13
+import openai # use the official client for correctness check
14
+import pytest
15
+# downloading lora to test lora requests
16
+from huggingface_hub import snapshot_download
17
+from openai import BadRequestError
18
+from transformers import AutoTokenizer
19
+
20
+from vllm.transformers_utils.tokenizer import get_tokenizer
21
+
22
+from ...utils import RemoteOpenAIServer
23
+
24
+# any model with a chat template should work here
25
+MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26
+# technically these adapters use a different base model,
27
+# but we're not testing generation quality here
28
+LORA_NAME = "typeof/zephyr-7b-beta-lora"
29
+PA_NAME = "swapnilbp/llama_tweet_ptune"
30
+# if PA_NAME changes, PA_NUM_VIRTUAL_TOKENS might also
31
+# need to change to match the prompt adapter
32
+PA_NUM_VIRTUAL_TOKENS = 8
33
+
34
+
35
+@pytest.fixture(scope="module")
36
+def zephyr_lora_files():
37
+ return snapshot_download(repo_id=LORA_NAME)
38
+
39
+
40
+@pytest.fixture(scope="module")
41
+def zephyr_lora_added_tokens_files(zephyr_lora_files):
42
+ tmp_dir = TemporaryDirectory()
43
+ tmp_model_dir = f"{tmp_dir.name}/zephyr"
44
+ shutil.copytree(zephyr_lora_files, tmp_model_dir)
45
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
46
+ # Copy tokenizer to adapter and add some unique tokens
47
+ # 32000, 32001, 32002
48
+ added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],
49
+ special_tokens=True)
50
+ assert added == 3
51
+ tokenizer.save_pretrained(tmp_model_dir)
52
+ yield tmp_model_dir
53
+ tmp_dir.cleanup()
54
+
55
+
56
+@pytest.fixture(scope="module")
57
+def zephyr_pa_files():
58
+ return snapshot_download(repo_id=PA_NAME)
59
+
60
+
61
+@pytest.fixture(scope="module")
62
+def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files,
63
+ zephyr_pa_files):
64
+ return [
65
+ # use half precision for speed and memory savings in CI environment
66
+ "--dtype",
67
+ "bfloat16",
68
+ "--max-model-len",
69
+ "8192",
70
+ "--max-num-seqs",
71
+ "128",
72
+ "--enforce-eager",
73
+ # lora config
74
+ "--enable-lora",
75
+ "--lora-modules",
76
+ f"zephyr-lora={zephyr_lora_files}",
77
+ f"zephyr-lora2={zephyr_lora_added_tokens_files}",
78
+ "--max-lora-rank",
79
+ "64",
80
+ "--max-cpu-loras",
81
+ "2",
82
+ # pa config
83
+ "--enable-prompt-adapter",
84
+ "--prompt-adapters",
85
+ f"zephyr-pa={zephyr_pa_files}",
86
+ f"zephyr-pa2={zephyr_pa_files}",
87
+ "--max-prompt-adapters",
88
+ "2",
89
+ "--max-prompt-adapter-token",
90
+ "128",
91
+ "--disable-frontend-multiprocessing"
92
+ ]
93
+
94
+
95
+@pytest.fixture(scope="module")
96
+def server(default_server_args):
97
+ with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
98
+ yield remote_server
99
+
100
+
101
+@pytest.fixture(scope="module")
102
+def client(server):
103
+ return server.get_async_client()
104
+
105
+
106
+@pytest.mark.asyncio
107
+@pytest.mark.parametrize(
108
+ # first test base model, then test loras, then test prompt adapters
109
+ "model_name,num_virtual_tokens",
110
+ [(MODEL_NAME, 0), ("zephyr-lora", 0), ("zephyr-lora2", 0),
111
+ ("zephyr-pa", PA_NUM_VIRTUAL_TOKENS),
112
+ ("zephyr-pa2", PA_NUM_VIRTUAL_TOKENS)],
113
+)
114
+async def test_single_completion(client: openai.AsyncOpenAI, model_name: str,
115
+ num_virtual_tokens: int):
116
+ completion = await client.completions.create(model=model_name,
117
+ prompt="Hello, my name is",
118
+ max_tokens=5,
119
+ temperature=0.0)
120
+
121
+ assert completion.id is not None
122
+ assert completion.choices is not None and len(completion.choices) == 1
123
+
124
+ choice = completion.choices[0]
125
+ assert len(choice.text) >= 5
126
+ assert choice.finish_reason == "length"
127
+ assert completion.usage == openai.types.CompletionUsage(
128
+ completion_tokens=5,
129
+ prompt_tokens=6 + num_virtual_tokens,
130
+ total_tokens=11 + num_virtual_tokens)
131
+
132
+ # test using token IDs
133
+ completion = await client.completions.create(
134
+ model=model_name,
135
+ prompt=[0, 0, 0, 0, 0],
136
+ max_tokens=5,
137
+ temperature=0.0,
138
+ )
139
+ assert len(completion.choices[0].text) >= 1
140
+
141
+
142
+@pytest.mark.asyncio
143
+async def test_added_lora_tokens(client: openai.AsyncOpenAI):
144
+ # test using token IDs
145
+ completion = await client.completions.create(
146
+ model="zephyr-lora2",
147
+ prompt=[0, 0, 32000, 32001, 32002],
148
+ echo=True,
149
+ max_tokens=5,
150
+ temperature=0.0,
151
+ )
152
+ # Added tokens should appear in tokenized prompt
153
+ assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")
154
+
155
+
156
+@pytest.mark.asyncio
157
+async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):
158
+ # test using token IDs
159
+ completion = await client.completions.create(
160
+ model=MODEL_NAME,
161
+ prompt=[0, 0, 32000, 32001, 32002],
162
+ echo=True,
163
+ max_tokens=5,
164
+ temperature=0.0,
165
+ )
166
+ # Added tokens should not appear in tokenized prompt
167
+ assert "vllm" not in completion.choices[0].text
168
+
169
+
170
+@pytest.mark.asyncio
171
+@pytest.mark.parametrize(
172
+ # first test base model, then test loras, then test prompt adapters
173
+ "model_name",
174
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2", "zephyr-pa", "zephyr-pa2"],
175
+)
176
+async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
177
+ # test using token IDs
178
+ completion = await client.completions.create(
179
+ model=model_name,
180
+ prompt=[0, 0, 0, 0, 0],
181
+ max_tokens=5,
182
+ temperature=0.0,
183
+ logprobs=None,
184
+ )
185
+ choice = completion.choices[0]
186
+ assert choice.logprobs is None
187
+
188
+
189
+@pytest.mark.asyncio
190
+@pytest.mark.parametrize(
191
+ # just test 1 lora and 1 pa hereafter
192
+ "model_name",
193
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
194
+)
195
+async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
196
+ # test using token IDs
197
+ completion = await client.completions.create(
198
+ model=model_name,
199
+ prompt=[0, 0, 0, 0, 0],
200
+ max_tokens=5,
201
+ temperature=0.0,
202
+ logprobs=0,
203
+ )
204
+ choice = completion.choices[0]
205
+ assert choice.logprobs is not None
206
+ assert choice.logprobs.token_logprobs is not None
207
+ assert choice.logprobs.top_logprobs is not None
208
+ assert len(choice.logprobs.top_logprobs[0]) == 1
209
+
210
+
211
+@pytest.mark.asyncio
212
+@pytest.mark.parametrize(
213
+ "model_name",
214
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
215
+)
216
+async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
217
+ # test using token IDs
218
+ completion = await client.completions.create(
219
+ model=model_name,
220
+ prompt=[0, 0, 0, 0, 0],
221
+ max_tokens=5,
222
+ temperature=0.0,
223
+ logprobs=5,
224
+ )
225
+ choice = completion.choices[0]
226
+ assert choice.logprobs is not None
227
+ assert choice.logprobs.token_logprobs is not None
228
+ assert choice.logprobs.top_logprobs is not None
229
+ assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
230
+
231
+
232
+@pytest.mark.asyncio
233
+@pytest.mark.parametrize(
234
+ "model_name",
235
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
236
+)
237
+async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
238
+ model_name: str):
239
+
240
+ with pytest.raises(
241
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
242
+ await client.completions.create(
243
+ model=model_name,
244
+ prompt=[0, 0, 0, 0, 0],
245
+ max_tokens=5,
246
+ temperature=0.0,
247
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
248
+ # both Completion API and Chat Completion API
249
+ logprobs=21,
250
+ )
251
+ ...
252
+ with pytest.raises(
253
+ (openai.BadRequestError, openai.APIError)): # test using token IDs
254
+ stream = await client.completions.create(
255
+ model=model_name,
256
+ prompt=[0, 0, 0, 0, 0],
257
+ max_tokens=5,
258
+ temperature=0.0,
259
+ # vLLM has higher default max_logprobs (20 instead of 5) to support
260
+ # both Completion API and Chat Completion API
261
+ logprobs=30,
262
+ stream=True,
263
+ )
264
+ async for chunk in stream:
265
+ ...
266
+
267
+ # the server should still work afterwards
268
+ completion = await client.completions.create(
269
+ model=model_name,
270
+ prompt=[0, 0, 0, 0, 0],
271
+ max_tokens=5,
272
+ temperature=0.0,
273
+ )
274
+ assert len(completion.choices[0].text) >= 0
275
+
276
+
277
+@pytest.mark.asyncio
278
+@pytest.mark.parametrize(
279
+ "model_name",
280
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
281
+)
282
+async def test_completion_streaming(client: openai.AsyncOpenAI,
283
+ model_name: str):
284
+ prompt = "What is an LLM?"
285
+
286
+ single_completion = await client.completions.create(
287
+ model=model_name,
288
+ prompt=prompt,
289
+ max_tokens=5,
290
+ temperature=0.0,
291
+ )
292
+ single_output = single_completion.choices[0].text
293
+ stream = await client.completions.create(model=model_name,
294
+ prompt=prompt,
295
+ max_tokens=5,
296
+ temperature=0.0,
297
+ stream=True)
298
+ chunks: List[str] = []
299
+ finish_reason_count = 0
300
+ async for chunk in stream:
301
+ chunks.append(chunk.choices[0].text)
302
+ if chunk.choices[0].finish_reason is not None:
303
+ finish_reason_count += 1
304
+ # finish reason should only return in last block
305
+ assert finish_reason_count == 1
306
+ assert chunk.choices[0].finish_reason == "length"
307
+ assert chunk.choices[0].text
308
+ assert "".join(chunks) == single_output
309
+
310
+
311
+@pytest.mark.asyncio
312
+@pytest.mark.parametrize(
313
+ "model_name",
314
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
315
+)
316
+async def test_completion_stream_options(client: openai.AsyncOpenAI,
317
+ model_name: str):
318
+ prompt = "What is the capital of France?"
319
+
320
+ # Test stream=True, stream_options=
321
+ # {"include_usage": False, "continuous_usage_stats": False}
322
+ stream = await client.completions.create(model=model_name,
323
+ prompt=prompt,
324
+ max_tokens=5,
325
+ temperature=0.0,
326
+ stream=True,
327
+ stream_options={
328
+ "include_usage": False,
329
+ "continuous_usage_stats":
330
+ False,
331
+ })
332
+
333
+ async for chunk in stream:
334
+ assert chunk.usage is None
335
+
336
+ # Test stream=True, stream_options=
337
+ # {"include_usage": False, "continuous_usage_stats": True}
338
+ stream = await client.completions.create(model=model_name,
339
+ prompt=prompt,
340
+ max_tokens=5,
341
+ temperature=0.0,
342
+ stream=True,
343
+ stream_options={
344
+ "include_usage": False,
345
+ "continuous_usage_stats":
346
+ True,
347
+ })
348
+ async for chunk in stream:
349
+ assert chunk.usage is None
350
+
351
+ # Test stream=True, stream_options=
352
+ # {"include_usage": True, "continuous_usage_stats": False}
353
+ stream = await client.completions.create(model=model_name,
354
+ prompt=prompt,
355
+ max_tokens=5,
356
+ temperature=0.0,
357
+ stream=True,
358
+ stream_options={
359
+ "include_usage": True,
360
+ "continuous_usage_stats":
361
+ False,
362
+ })
363
+ async for chunk in stream:
364
+ if chunk.choices[0].finish_reason is None:
365
+ assert chunk.usage is None
366
+ else:
367
+ assert chunk.usage is None
368
+ final_chunk = await stream.__anext__()
369
+ assert final_chunk.usage is not None
370
+ assert final_chunk.usage.prompt_tokens > 0
371
+ assert final_chunk.usage.completion_tokens > 0
372
+ assert final_chunk.usage.total_tokens == (
373
+ final_chunk.usage.prompt_tokens +
374
+ final_chunk.usage.completion_tokens)
375
+ assert final_chunk.choices == []
376
+
377
+ # Test stream=True, stream_options=
378
+ # {"include_usage": True, "continuous_usage_stats": True}
379
+ stream = await client.completions.create(model=model_name,
380
+ prompt=prompt,
381
+ max_tokens=5,
382
+ temperature=0.0,
383
+ stream=True,
384
+ stream_options={
385
+ "include_usage": True,
386
+ "continuous_usage_stats":
387
+ True,
388
+ })
389
+ async for chunk in stream:
390
+ assert chunk.usage is not None
391
+ assert chunk.usage.prompt_tokens > 0
392
+ assert chunk.usage.completion_tokens > 0
393
+ assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
394
+ chunk.usage.completion_tokens)
395
+ if chunk.choices[0].finish_reason is not None:
396
+ final_chunk = await stream.__anext__()
397
+ assert final_chunk.usage is not None
398
+ assert final_chunk.usage.prompt_tokens > 0
399
+ assert final_chunk.usage.completion_tokens > 0
400
+ assert final_chunk.usage.total_tokens == (
401
+ final_chunk.usage.prompt_tokens +
402
+ final_chunk.usage.completion_tokens)
403
+ assert final_chunk.choices == []
404
+
405
+ # Test stream=False, stream_options=
406
+ # {"include_usage": None}
407
+ with pytest.raises(BadRequestError):
408
+ await client.completions.create(model=model_name,
409
+ prompt=prompt,
410
+ max_tokens=5,
411
+ temperature=0.0,
412
+ stream=False,
413
+ stream_options={"include_usage": None})
414
+
415
+ # Test stream=False, stream_options=
416
+ # {"include_usage": True}
417
+ with pytest.raises(BadRequestError):
418
+ await client.completions.create(model=model_name,
419
+ prompt=prompt,
420
+ max_tokens=5,
421
+ temperature=0.0,
422
+ stream=False,
423
+ stream_options={"include_usage": True})
424
+
425
+ # Test stream=False, stream_options=
426
+ # {"continuous_usage_stats": None}
427
+ with pytest.raises(BadRequestError):
428
+ await client.completions.create(
429
+ model=model_name,
430
+ prompt=prompt,
431
+ max_tokens=5,
432
+ temperature=0.0,
433
+ stream=False,
434
+ stream_options={"continuous_usage_stats": None})
435
+
436
+ # Test stream=False, stream_options=
437
+ # {"continuous_usage_stats": True}
438
+ with pytest.raises(BadRequestError):
439
+ await client.completions.create(
440
+ model=model_name,
441
+ prompt=prompt,
442
+ max_tokens=5,
443
+ temperature=0.0,
444
+ stream=False,
445
+ stream_options={"continuous_usage_stats": True})
446
+
447
+
448
+@pytest.mark.asyncio
449
+@pytest.mark.parametrize(
450
+ "model_name",
451
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
452
+)
453
+async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
454
+ # test both text and token IDs
455
+ for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
456
+ # test simple list
457
+ batch = await client.completions.create(
458
+ model=model_name,
459
+ prompt=prompts,
460
+ max_tokens=5,
461
+ temperature=0.0,
462
+ )
463
+ assert len(batch.choices) == 2
464
+ assert batch.choices[0].text == batch.choices[1].text
465
+
466
+ # test n = 2
467
+ batch = await client.completions.create(
468
+ model=model_name,
469
+ prompt=prompts,
470
+ n=2,
471
+ max_tokens=5,
472
+ temperature=0.0,
473
+ extra_body=dict(
474
+ # NOTE: this has to be true for n > 1 in vLLM, but not necessary
475
+ # for official client.
476
+ use_beam_search=True),
477
+ )
478
+ assert len(batch.choices) == 4
479
+ assert batch.choices[0].text != batch.choices[
480
+ 1].text, "beam search should be different"
481
+ assert batch.choices[0].text == batch.choices[
482
+ 2].text, "two copies of the same prompt should be the same"
483
+ assert batch.choices[1].text == batch.choices[
484
+ 3].text, "two copies of the same prompt should be the same"
485
+
486
+ # test streaming
487
+ batch = await client.completions.create(
488
+ model=model_name,
489
+ prompt=prompts,
490
+ max_tokens=5,
491
+ temperature=0.0,
492
+ stream=True,
493
+ )
494
+ texts = [""] * 2
495
+ async for chunk in batch:
496
+ assert len(chunk.choices) == 1
497
+ choice = chunk.choices[0]
498
+ texts[choice.index] += choice.text
499
+ assert texts[0] == texts[1]
500
+
501
+
502
+@pytest.mark.asyncio
503
+async def test_logits_bias(client: openai.AsyncOpenAI):
504
+ prompt = "Hello, my name is"
505
+ max_tokens = 5
506
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
507
+
508
+ # Test exclusive selection
509
+ token_id = 1000
510
+ completion = await client.completions.create(
511
+ model=MODEL_NAME,
512
+ prompt=prompt,
513
+ max_tokens=max_tokens,
514
+ temperature=0.0,
515
+ logit_bias={str(token_id): 100},
516
+ seed=42,
517
+ )
518
+ assert len(completion.choices[0].text) >= 5
519
+ response_tokens = tokenizer(completion.choices[0].text,
520
+ add_special_tokens=False)["input_ids"]
521
+ expected_tokens = tokenizer(tokenizer.decode([token_id] * 5),
522
+ add_special_tokens=False)["input_ids"]
523
+ assert all([
524
+ response == expected
525
+ for response, expected in zip(response_tokens, expected_tokens)
526
+ ])
527
+
528
+ # Test ban
529
+ completion = await client.completions.create(
530
+ model=MODEL_NAME,
531
+ prompt=prompt,
532
+ max_tokens=max_tokens,
533
+ temperature=0.0,
534
+ )
535
+ response_tokens = tokenizer(completion.choices[0].text,
536
+ add_special_tokens=False)["input_ids"]
537
+ first_response = completion.choices[0].text
538
+ completion = await client.completions.create(
539
+ model=MODEL_NAME,
540
+ prompt=prompt,
541
+ max_tokens=max_tokens,
542
+ temperature=0.0,
543
+ logit_bias={str(token): -100
544
+ for token in response_tokens},
545
+ )
546
+ assert first_response != completion.choices[0].text
547
+
548
+
549
+@pytest.mark.asyncio
550
+async def test_allowed_token_ids(client: openai.AsyncOpenAI):
551
+ prompt = "Hello, my name is"
552
+ max_tokens = 1
553
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
554
+
555
+ # Test exclusive selection
556
+ allowed_ids = [21555, 21557, 21558]
557
+ completion = await client.completions.create(
558
+ model=MODEL_NAME,
559
+ prompt=prompt,
560
+ max_tokens=max_tokens,
561
+ temperature=0.0,
562
+ seed=42,
563
+ extra_body=dict(allowed_token_ids=allowed_ids),
564
+ logprobs=1,
565
+ )
566
+ response_tokens = completion.choices[0].logprobs.tokens
567
+ assert len(response_tokens) == 1
568
+ assert tokenizer.convert_tokens_to_ids(response_tokens)[0] in allowed_ids
569
+
570
+
571
+@pytest.mark.asyncio
572
+@pytest.mark.parametrize("guided_decoding_backend",
573
+ ["outlines", "lm-format-enforcer"])
574
+async def test_guided_json_completion(client: openai.AsyncOpenAI,
575
+ guided_decoding_backend: str,
576
+ sample_json_schema):
577
+ completion = await client.completions.create(
578
+ model=MODEL_NAME,
579
+ prompt=f"Give an example JSON for an employee profile "
580
+ f"that fits this schema: {sample_json_schema}",
581
+ n=3,
582
+ temperature=1.0,
583
+ max_tokens=500,
584
+ extra_body=dict(guided_json=sample_json_schema,
585
+ guided_decoding_backend=guided_decoding_backend))
586
+
587
+ assert completion.id is not None
588
+ assert len(completion.choices) == 3
589
+ for i in range(3):
590
+ output_json = json.loads(completion.choices[i].text)
591
+ jsonschema.validate(instance=output_json, schema=sample_json_schema)
592
+
593
+
594
+@pytest.mark.asyncio
595
+@pytest.mark.parametrize("guided_decoding_backend",
596
+ ["outlines", "lm-format-enforcer"])
597
+async def test_guided_regex_completion(client: openai.AsyncOpenAI,
598
+ guided_decoding_backend: str,
599
+ sample_regex):
600
+ completion = await client.completions.create(
601
+ model=MODEL_NAME,
602
+ prompt=f"Give an example IPv4 address with this regex: {sample_regex}",
603
+ n=3,
604
+ temperature=1.0,
605
+ max_tokens=20,
606
+ extra_body=dict(guided_regex=sample_regex,
607
+ guided_decoding_backend=guided_decoding_backend))
608
+
609
+ assert completion.id is not None
610
+ assert len(completion.choices) == 3
611
+ for i in range(3):
612
+ assert re.fullmatch(sample_regex,
613
+ completion.choices[i].text) is not None
614
+
615
+
616
+@pytest.mark.asyncio
617
+@pytest.mark.parametrize("guided_decoding_backend",
618
+ ["outlines", "lm-format-enforcer"])
619
+async def test_guided_choice_completion(client: openai.AsyncOpenAI,
620
+ guided_decoding_backend: str,
621
+ sample_guided_choice):
622
+ completion = await client.completions.create(
623
+ model=MODEL_NAME,
624
+ prompt="The best language for type-safe systems programming is ",
625
+ n=2,
626
+ temperature=1.0,
627
+ max_tokens=10,
628
+ extra_body=dict(guided_choice=sample_guided_choice,
629
+ guided_decoding_backend=guided_decoding_backend))
630
+
631
+ assert completion.id is not None
632
+ assert len(completion.choices) == 2
633
+ for i in range(2):
634
+ assert completion.choices[i].text in sample_guided_choice
635
+
636
+
637
+@pytest.mark.asyncio
638
+async def test_guided_grammar(client: openai.AsyncOpenAI,
639
+ sample_sql_statements):
640
+
641
+ completion = await client.completions.create(
642
+ model=MODEL_NAME,
643
+ prompt=("Generate a sql state that select col_1 from "
644
+ "table_1 where it is equals to 1"),
645
+ temperature=1.0,
646
+ max_tokens=500,
647
+ extra_body=dict(guided_grammar=sample_sql_statements))
648
+
649
+ content = completion.choices[0].text
650
+
651
+ # use Lark to parse the output, and make sure it's a valid parse tree
652
+ from lark import Lark
653
+ parser = Lark(sample_sql_statements)
654
+ parser.parse(content)
655
+
656
+ # remove spaces for comparison b/c we removed them in the grammar
657
+ ground_truth = "SELECT col_1 from table_1 where col_1 = 1".replace(" ", "")
658
+
659
+ assert content.strip() == ground_truth
660
+
661
+
662
+@pytest.mark.asyncio
663
+@pytest.mark.parametrize(
664
+ # first test base model, then test loras
665
+ "model_name",
666
+ [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],
667
+)
668
+@pytest.mark.parametrize("logprobs_arg", [1, 0])
669
+async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
670
+ model_name: str, logprobs_arg: int):
671
+ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
672
+ # test using text and token IDs
673
+ for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
674
+ completion = await client.completions.create(model=model_name,
675
+ prompt=prompt,
676
+ max_tokens=5,
677
+ temperature=0.0,
678
+ echo=True,
679
+ logprobs=logprobs_arg)
680
+
681
+ prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
682
+ list) else prompt
683
+ assert re.search(r"^" + prompt_text, completion.choices[0].text)
684
+ logprobs = completion.choices[0].logprobs
685
+ assert logprobs is not None
686
+ assert len(logprobs.text_offset) > 5
687
+ assert (len(logprobs.token_logprobs) > 5
688
+ and logprobs.token_logprobs[0] is None)
689
+ assert (len(logprobs.top_logprobs) > 5
690
+ and logprobs.top_logprobs[0] is None)
691
+ for top_logprobs in logprobs.top_logprobs[1:]:
692
+ assert max(logprobs_arg,
693
+ 1) <= len(top_logprobs) <= logprobs_arg + 1
694
+ assert len(logprobs.tokens) > 5
695
+
696
+
697
+@pytest.mark.asyncio
698
+@pytest.mark.parametrize("guided_decoding_backend",
699
+ ["outlines", "lm-format-enforcer"])
700
+async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
701
+ guided_decoding_backend: str,
702
+ sample_json_schema, sample_regex):
703
+ with pytest.raises(openai.BadRequestError):
704
+ _ = await client.completions.create(
705
+ model=MODEL_NAME,
706
+ prompt="Give an example JSON that fits this schema: 42",
707
+ extra_body=dict(guided_json=42,
708
+ guided_decoding_backend=guided_decoding_backend))
709
+
710
+ with pytest.raises(openai.BadRequestError):
711
+ _ = await client.completions.create(
712
+ model=MODEL_NAME,
713
+ prompt="Give an example string that fits this regex",
714
+ extra_body=dict(guided_regex=sample_regex,
715
+ guided_json=sample_json_schema))
vllm/engine/async_llm_engine.py
+26
-1
@@ -7,7 +7,8 @@ from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
7
from transformers import PreTrainedTokenizer
8
9
import vllm.envs as envs
10
-from vllm.config import DecodingConfig, EngineConfig, ModelConfig
10
+from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11
+ ParallelConfig, SchedulerConfig)
12
from vllm.core.scheduler import SchedulerOutputs
13
from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_timeout import asyncio_timeout
@@ -928,6 +929,14 @@ class AsyncLLMEngine:
929
else:
930
return self.engine.get_model_config()
931
932
+ async def get_parallel_config(self) -> ParallelConfig:
933
+ """Get the parallel configuration of the vLLM engine."""
934
+ if self.engine_use_ray:
935
+ return await self.engine.get_parallel_config.remote( # type: ignore
936
+ )
937
+ else:
938
+ return self.engine.get_parallel_config()
939
+
940
async def get_decoding_config(self) -> DecodingConfig:
941
"""Get the decoding configuration of the vLLM engine."""
942
if self.engine_use_ray:
@@ -936,6 +945,22 @@ class AsyncLLMEngine:
945
else:
946
return self.engine.get_decoding_config()
947
948
+ async def get_scheduler_config(self) -> SchedulerConfig:
949
+ """Get the scheduling configuration of the vLLM engine."""
950
+ if self.engine_use_ray:
951
+ return await self.engine.get_scheduler_config.remote( # type: ignore
952
+ )
953
+ else:
954
+ return self.engine.get_scheduler_config()
955
+
956
+ async def get_lora_config(self) -> LoRAConfig:
957
+ """Get the lora configuration of the vLLM engine."""
958
+ if self.engine_use_ray:
959
+ return await self.engine.get_lora_config.remote( # type: ignore
960
+ )
961
+ else:
962
+ return self.engine.get_lora_config()
963
+
964
async def do_log_stats(
965
self,
966
scheduler_outputs: Optional[SchedulerOutputs] = None,
vllm/engine/llm_engine.py
+20
-16
@@ -38,9 +38,8 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
38
init_tracer)
39
from vllm.transformers_utils.config import try_get_generation_config
40
from vllm.transformers_utils.detokenizer import Detokenizer
41
-from vllm.transformers_utils.tokenizer_group import (AnyTokenizer,
42
- BaseTokenizerGroup,
43
- get_tokenizer_group)
41
+from vllm.transformers_utils.tokenizer_group import (
42
+ AnyTokenizer, BaseTokenizerGroup, init_tokenizer_from_configs)
43
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
44
usage_message)
45
from vllm.utils import Counter
@@ -485,19 +484,12 @@ class LLMEngine:
484
return self.get_tokenizer_group().get_lora_tokenizer(
485
sequence.lora_request)
486
488
- def _init_tokenizer(self, **tokenizer_init_kwargs) -> BaseTokenizerGroup:
489
- init_kwargs = dict(
490
- tokenizer_id=self.model_config.tokenizer,
491
- enable_lora=bool(self.lora_config),
492
- max_num_seqs=self.scheduler_config.max_num_seqs,
493
- max_input_length=None,
494
- tokenizer_mode=self.model_config.tokenizer_mode,
495
- trust_remote_code=self.model_config.trust_remote_code,
496
- revision=self.model_config.tokenizer_revision)
497
- init_kwargs.update(tokenizer_init_kwargs)
498
-
499
- return get_tokenizer_group(self.parallel_config.tokenizer_pool_config,
500
- **init_kwargs)
487
+ def _init_tokenizer(self) -> BaseTokenizerGroup:
488
+ return init_tokenizer_from_configs(
489
+ model_config=self.model_config,
490
+ scheduler_config=self.scheduler_config,
491
+ parallel_config=self.parallel_config,
492
+ enable_lora=bool(self.lora_config))
493
494
def _verify_args(self) -> None:
495
self.model_config.verify_with_parallel_config(self.parallel_config)
@@ -759,10 +751,22 @@ class LLMEngine:
751
"""Gets the model configuration."""
752
return self.model_config
753
754
+ def get_parallel_config(self) -> ParallelConfig:
755
+ """Gets the parallel configuration."""
756
+ return self.parallel_config
757
+
758
def get_decoding_config(self) -> DecodingConfig:
759
"""Gets the decoding configuration."""
760
return self.decoding_config
761
762
+ def get_scheduler_config(self) -> SchedulerConfig:
763
+ """Gets the scheduler configuration."""
764
+ return self.scheduler_config
765
+
766
+ def get_lora_config(self) -> LoRAConfig:
767
+ """Gets the LoRA configuration."""
768
+ return self.lora_config
769
+
770
def get_num_unfinished_requests(self) -> int:
771
"""Gets the number of unfinished requests."""
772
return sum(scheduler.get_num_unfinished_seq_groups()
vllm/engine/protocol.py
new
+84
@@ -0,0 +1,84 @@
1
+from typing import (AsyncIterator, List, Mapping, Optional, Protocol,
2
+ runtime_checkable)
3
+
4
+from transformers import PreTrainedTokenizer
5
+
6
+from vllm.config import DecodingConfig, ModelConfig
7
+from vllm.core.scheduler import SchedulerOutputs
8
+from vllm.inputs.data import PromptInputs
9
+from vllm.lora.request import LoRARequest
10
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
11
+from vllm.pooling_params import PoolingParams
12
+from vllm.prompt_adapter.request import PromptAdapterRequest
13
+from vllm.sampling_params import SamplingParams
14
+from vllm.sequence import SamplerOutput
15
+
16
+
17
+@runtime_checkable
18
+class AsyncEngineClient(Protocol):
19
+ """Protocol class for Clients to AsyncLLMEngine"""
20
+
21
+ @property
22
+ def is_running(self) -> bool:
23
+ ...
24
+
25
+ @property
26
+ def is_stopped(self) -> bool:
27
+ ...
28
+
29
+ @property
30
+ def errored(self) -> bool:
31
+ ...
32
+
33
+ async def generate(
34
+ self,
35
+ inputs: PromptInputs,
36
+ sampling_params: SamplingParams,
37
+ request_id: str,
38
+ lora_request: Optional[LoRARequest] = None,
39
+ trace_headers: Optional[Mapping[str, str]] = None,
40
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
41
+ ) -> AsyncIterator[RequestOutput]:
42
+ """Generates outputs for a request"""
43
+
44
+ async def encode(
45
+ self,
46
+ inputs: PromptInputs,
47
+ pooling_params: PoolingParams,
48
+ request_id: str,
49
+ lora_request: Optional[LoRARequest] = None,
50
+ trace_headers: Optional[Mapping[str, str]] = None,
51
+ ) -> AsyncIterator[EmbeddingRequestOutput]:
52
+ """Generate outputs for a request from an embedding model."""
53
+
54
+ async def abort(self, request_id: str) -> None:
55
+ """Abort a request.
56
+
57
+ Args:
58
+ request_id: The unique id of the request.
59
+ """
60
+
61
+ async def get_model_config(self) -> ModelConfig:
62
+ """Get the model configuration of the vLLM engine."""
63
+
64
+ async def get_decoding_config(self) -> DecodingConfig:
65
+ """Get the decoding configuration of the vLLM engine."""
66
+
67
+ async def get_tokenizer(
68
+ self,
69
+ lora_request: Optional[LoRARequest] = None,
70
+ ) -> PreTrainedTokenizer:
71
+ """Get the appropriate Tokenizer for the request"""
72
+
73
+ async def is_tracing_enabled(self) -> bool:
74
+ pass
75
+
76
+ async def do_log_stats(
77
+ self,
78
+ scheduler_outputs: Optional[SchedulerOutputs] = None,
79
+ model_output: Optional[List[SamplerOutput]] = None,
80
+ ) -> None:
81
+ pass
82
+
83
+ async def check_health(self) -> None:
84
+ """Raise if unhealthy"""
vllm/entrypoints/openai/api_server.py
+96
-36
@@ -5,7 +5,8 @@ import re
5
import signal
6
from contextlib import asynccontextmanager
7
from http import HTTPStatus
8
-from typing import Optional, Set
8
+from multiprocessing import Process
9
+from typing import AsyncIterator, Set
10
11
import fastapi
12
import uvicorn
@@ -17,8 +18,10 @@ from prometheus_client import make_asgi_app
18
from starlette.routing import Mount
19
20
import vllm.envs as envs
21
+from vllm.config import ModelConfig
22
from vllm.engine.arg_utils import AsyncEngineArgs
23
from vllm.engine.async_llm_engine import AsyncLLMEngine
24
+from vllm.engine.protocol import AsyncEngineClient
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.cli_args import make_arg_parser
27
# yapf conflicts with isort for this block
@@ -31,6 +34,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
34
EmbeddingRequest, ErrorResponse,
35
TokenizeRequest,
36
TokenizeResponse)
37
+from vllm.entrypoints.openai.rpc.client import AsyncEngineRPCClient
38
+from vllm.entrypoints.openai.rpc.server import run_rpc_server
39
# yapf: enable
40
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
41
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -39,12 +44,12 @@ from vllm.entrypoints.openai.serving_tokenization import (
44
OpenAIServingTokenization)
45
from vllm.logger import init_logger
46
from vllm.usage.usage_lib import UsageContext
42
-from vllm.utils import FlexibleArgumentParser
47
+from vllm.utils import FlexibleArgumentParser, get_open_port
48
from vllm.version import __version__ as VLLM_VERSION
49
50
TIMEOUT_KEEP_ALIVE = 5 # seconds
51
47
-engine: AsyncLLMEngine
52
+async_engine_client: AsyncEngineClient
53
engine_args: AsyncEngineArgs
54
openai_serving_chat: OpenAIServingChat
55
openai_serving_completion: OpenAIServingCompletion
@@ -56,13 +61,22 @@ logger = init_logger('vllm.entrypoints.openai.api_server')
61
_running_tasks: Set[asyncio.Task] = set()
62
63
64
+def model_is_embedding(model_name: str) -> bool:
65
+ return ModelConfig(model=model_name,
66
+ tokenizer=model_name,
67
+ tokenizer_mode="auto",
68
+ trust_remote_code=False,
69
+ seed=0,
70
+ dtype="float16").embedding_mode
71
+
72
+
73
@asynccontextmanager
74
async def lifespan(app: fastapi.FastAPI):
75
76
async def _force_log():
77
while True:
78
await asyncio.sleep(10)
65
- await engine.do_log_stats()
79
+ await async_engine_client.do_log_stats()
80
81
if not engine_args.disable_log_stats:
82
task = asyncio.create_task(_force_log())
@@ -72,6 +86,52 @@ async def lifespan(app: fastapi.FastAPI):
86
yield
87
88
89
+@asynccontextmanager
90
+async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]:
91
+ # Context manager to handle async_engine_client lifecycle
92
+ # Ensures everything is shutdown and cleaned up on error/exit
93
+ global engine_args
94
+ engine_args = AsyncEngineArgs.from_cli_args(args)
95
+
96
+ # Backend itself still global for the silly lil' health handler
97
+ global async_engine_client
98
+
99
+ # If manually triggered or embedding model, use AsyncLLMEngine in process.
100
+ # TODO: support embedding model via RPC.
101
+ if (model_is_embedding(args.model)
102
+ or args.disable_frontend_multiprocessing):
103
+ async_engine_client = AsyncLLMEngine.from_engine_args(
104
+ engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
105
+ yield async_engine_client
106
+ return
107
+
108
+ # Otherwise, use the multiprocessing AsyncLLMEngine.
109
+ else:
110
+ # Start RPCServer in separate process (holds the AsyncLLMEngine).
111
+ port = get_open_port(envs.VLLM_RPC_PORT)
112
+ rpc_server_process = Process(target=run_rpc_server,
113
+ args=(engine_args,
114
+ UsageContext.OPENAI_API_SERVER,
115
+ port))
116
+ rpc_server_process.start()
117
+
118
+ # Build RPCClient, which conforms to AsyncEngineClient Protocol.
119
+ async_engine_client = AsyncEngineRPCClient(port)
120
+ await async_engine_client.setup()
121
+
122
+ try:
123
+ yield async_engine_client
124
+ finally:
125
+ # Ensure rpc server process was terminated
126
+ rpc_server_process.terminate()
127
+
128
+ # Close all open connections to the backend
129
+ async_engine_client.close()
130
+
131
+ # Wait for server process to join
132
+ rpc_server_process.join()
133
+
134
+
135
router = APIRouter()
136
137
@@ -86,7 +146,7 @@ def mount_metrics(app: fastapi.FastAPI):
146
@router.get("/health")
147
async def health() -> Response:
148
"""Health check."""
89
- await openai_serving_chat.engine.check_health()
149
+ await async_engine_client.check_health()
150
return Response(status_code=200)
151
152
@@ -215,8 +275,8 @@ def build_app(args):
275
276
277
async def build_server(
278
+ async_engine_client: AsyncEngineClient,
279
args,
219
- llm_engine: Optional[AsyncLLMEngine] = None,
280
**uvicorn_kwargs,
281
) -> uvicorn.Server:
282
app = build_app(args)
@@ -226,14 +286,7 @@ async def build_server(
286
else:
287
served_model_names = [args.model]
288
229
- global engine, engine_args
230
-
231
- engine_args = AsyncEngineArgs.from_cli_args(args)
232
- engine = (llm_engine
233
- if llm_engine is not None else AsyncLLMEngine.from_engine_args(
234
- engine_args, usage_context=UsageContext.OPENAI_API_SERVER))
235
-
236
- model_config = await engine.get_model_config()
289
+ model_config = await async_engine_client.get_model_config()
290
291
if args.disable_log_requests:
292
request_logger = None
@@ -246,7 +299,7 @@ async def build_server(
299
global openai_serving_tokenization
300
301
openai_serving_chat = OpenAIServingChat(
249
- engine,
302
+ async_engine_client,
303
model_config,
304
served_model_names,
305
args.response_role,
@@ -257,7 +310,7 @@ async def build_server(
310
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
311
)
312
openai_serving_completion = OpenAIServingCompletion(
260
- engine,
313
+ async_engine_client,
314
model_config,
315
served_model_names,
316
lora_modules=args.lora_modules,
@@ -266,13 +319,13 @@ async def build_server(
319
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
320
)
321
openai_serving_embedding = OpenAIServingEmbedding(
269
- engine,
322
+ async_engine_client,
323
model_config,
324
served_model_names,
325
request_logger=request_logger,
326
)
327
openai_serving_tokenization = OpenAIServingTokenization(
275
- engine,
328
+ async_engine_client,
329
model_config,
330
served_model_names,
331
lora_modules=args.lora_modules,
@@ -304,32 +357,39 @@ async def build_server(
357
return uvicorn.Server(config)
358
359
307
-async def run_server(args, llm_engine=None, **uvicorn_kwargs) -> None:
360
+async def run_server(args, **uvicorn_kwargs) -> None:
361
logger.info("vLLM API server version %s", VLLM_VERSION)
362
logger.info("args: %s", args)
363
311
- server = await build_server(
312
- args,
313
- llm_engine,
314
- **uvicorn_kwargs,
315
- )
364
+ shutdown_task = None
365
+ async with build_async_engine_client(args) as async_engine_client:
366
+
367
+ server = await build_server(
368
+ async_engine_client,
369
+ args,
370
+ **uvicorn_kwargs,
371
+ )
372
+
373
+ loop = asyncio.get_running_loop()
374
317
- loop = asyncio.get_running_loop()
375
+ server_task = loop.create_task(server.serve())
376
319
- server_task = loop.create_task(server.serve())
377
+ def signal_handler() -> None:
378
+ # prevents the uvicorn signal handler to exit early
379
+ server_task.cancel()
380
321
- def signal_handler() -> None:
322
- # prevents the uvicorn signal handler to exit early
323
- server_task.cancel()
381
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
382
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
383
325
- loop.add_signal_handler(signal.SIGINT, signal_handler)
326
- loop.add_signal_handler(signal.SIGTERM, signal_handler)
384
+ try:
385
+ await server_task
386
+ except asyncio.CancelledError:
387
+ logger.info("Gracefully stopping http server")
388
+ shutdown_task = server.shutdown()
389
328
- try:
329
- await server_task
330
- except asyncio.CancelledError:
331
- print("Gracefully stopping http server")
332
- await server.shutdown()
390
+ if shutdown_task:
391
+ # NB: Await server shutdown only after the backend context is exited
392
+ await shutdown_task
393
394
395
if __name__ == "__main__":
vllm/entrypoints/openai/cli_args.py
+7
-2
@@ -131,9 +131,14 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
131
parser.add_argument(
132
"--return-tokens-as-token-ids",
133
action="store_true",
134
- help="When --max-logprobs is specified, represents single tokens as"
135
- "strings of the form 'token_id:{token_id}' so that tokens that"
134
+ help="When --max-logprobs is specified, represents single tokens as "
135
+ "strings of the form 'token_id:{token_id}' so that tokens that "
136
"are not JSON-encodable can be identified.")
137
+ parser.add_argument(
138
+ "--disable-frontend-multiprocessing",
139
+ action="store_true",
140
+ help="If specified, will run the OpenAI frontend server in the same "
141
+ "process as the model serving engine.")
142
143
parser = AsyncEngineArgs.add_cli_args(parser)
144
vllm/entrypoints/openai/logits_processors.py
+11
-8
@@ -1,4 +1,4 @@
1
-from functools import lru_cache
1
+from functools import lru_cache, partial
2
from typing import Dict, FrozenSet, Iterable, List, Optional, Union
3
4
import torch
@@ -40,6 +40,14 @@ def _get_allowed_token_ids_logits_processor(
40
return AllowedTokenIdsLogitsProcessor(allowed_token_ids)
41
42
43
+def logit_bias_logits_processor(logit_bias: Dict[str,
44
+ float], token_ids: List[int],
45
+ logits: torch.Tensor) -> torch.Tensor:
46
+ for token_id, bias in logit_bias.items():
47
+ logits[token_id] += bias
48
+ return logits
49
+
50
+
51
def get_logits_processors(
52
logit_bias: Optional[Union[Dict[int, float], Dict[str, float]]],
53
allowed_token_ids: Optional[List[int]],
@@ -64,13 +72,8 @@ def get_logits_processors(
72
raise ValueError("token_id in logit_bias contains "
73
"out-of-vocab token id")
74
67
- def logit_bias_logits_processor(token_ids: List[int],
68
- logits: torch.Tensor) -> torch.Tensor:
69
- for token_id, bias in clamped_logit_bias.items():
70
- logits[token_id] += bias
71
- return logits
72
-
73
- logits_processors.append(logit_bias_logits_processor)
75
+ logits_processors.append(
76
+ partial(logit_bias_logits_processor, clamped_logit_bias))
77
78
if allowed_token_ids is not None:
79
logits_processors.append(
vllm/entrypoints/openai/rpc/__init__.py
new
+42
@@ -0,0 +1,42 @@
1
+from dataclasses import dataclass
2
+from enum import Enum
3
+from typing import Mapping, Optional, Union
4
+
5
+from vllm.inputs import PromptInputs
6
+from vllm.lora.request import LoRARequest
7
+from vllm.prompt_adapter.request import PromptAdapterRequest
8
+from vllm.sampling_params import SamplingParams
9
+
10
+VLLM_RPC_SUCCESS_STR = "SUCCESS"
11
+VLLM_RPC_HEALTHY_STR = "HEALTHY"
12
+
13
+
14
+@dataclass
15
+class RPCGenerateRequest:
16
+ inputs: PromptInputs
17
+ sampling_params: SamplingParams
18
+ request_id: str
19
+ lora_request: Optional[LoRARequest] = None
20
+ trace_headers: Optional[Mapping[str, str]] = None
21
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
22
+
23
+
24
+@dataclass
25
+class RPCAbortRequest:
26
+ request_id: str
27
+
28
+
29
+class RPCUtilityRequest(Enum):
30
+ IS_SERVER_READY = 1
31
+ GET_MODEL_CONFIG = 2
32
+ GET_DECODING_CONFIG = 3
33
+ GET_PARALLEL_CONFIG = 4
34
+ GET_SCHEDULER_CONFIG = 5
35
+ GET_LORA_CONFIG = 6
36
+ DO_LOG_STATS = 7
37
+ CHECK_HEALTH = 8
38
+ IS_TRACING_ENABLED = 9
39
+
40
+
41
+RPC_REQUEST_TYPE = Union[RPCGenerateRequest, RPCAbortRequest,
42
+ RPCUtilityRequest]
vllm/entrypoints/openai/rpc/client.py
new
+248
@@ -0,0 +1,248 @@
1
+from contextlib import contextmanager
2
+from typing import Any, AsyncIterator, Mapping, Optional
3
+
4
+import cloudpickle
5
+import zmq
6
+import zmq.asyncio
7
+
8
+from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
9
+ ParallelConfig, SchedulerConfig)
10
+from vllm.entrypoints.openai.rpc import (RPC_REQUEST_TYPE,
11
+ VLLM_RPC_HEALTHY_STR,
12
+ VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
13
+ RPCGenerateRequest, RPCUtilityRequest)
14
+from vllm.inputs import PromptInputs
15
+from vllm.lora.request import LoRARequest
16
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
17
+from vllm.prompt_adapter.request import PromptAdapterRequest
18
+from vllm.sampling_params import SamplingParams
19
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
20
+
21
+
22
+class AsyncEngineRPCClient:
23
+
24
+ def __init__(self, port: int):
25
+ self.context = zmq.asyncio.Context()
26
+ self.path = f"tcp://localhost:{port}"
27
+
28
+ async def setup(self):
29
+ """Setup the client before it starts sending server requests."""
30
+
31
+ # Wait until server is ready.
32
+ await self.wait_for_server()
33
+
34
+ # Get the configs.
35
+ self.model_config = await self._get_model_config_rpc()
36
+ self.decoding_config = await self._get_decoding_config_rpc()
37
+ self.tracing_flag = await self._is_tracing_enabled_rpc()
38
+
39
+ # Create the tokenizer group.
40
+ # TODO: refactor OAI server to avoid needing this info.
41
+ self.tokenizer = init_tokenizer_from_configs(
42
+ model_config=self.model_config,
43
+ scheduler_config=(await self._get_scheduler_config_rpc()),
44
+ parallel_config=(await self._get_parallel_config_rpc()),
45
+ enable_lora=bool(await self._get_lora_config_rpc()),
46
+ )
47
+
48
+ def close(self):
49
+ """Destroy the ZeroMQ Context."""
50
+ self.context.destroy()
51
+
52
+ @contextmanager
53
+ def socket(self):
54
+ # Ensure client sockets are always closed after use
55
+
56
+ # Connect to RPC socket for Request-Reply pattern,
57
+ # Note that we use DEALER to enable asynchronous communication
58
+ # to enable streaming.
59
+ socket = self.context.socket(zmq.constants.DEALER)
60
+ try:
61
+ socket.connect(self.path)
62
+ yield socket
63
+ finally:
64
+ socket.close()
65
+
66
+ async def _send_get_data_rpc_request(self, request: RPCUtilityRequest,
67
+ expected_type: Any,
68
+ error_message: str) -> Any:
69
+ """Send an RPC request that is expecting data back."""
70
+
71
+ with self.socket() as socket:
72
+
73
+ # Ping RPCServer with a request.
74
+ await socket.send(cloudpickle.dumps(request))
75
+
76
+ # Await the data from the Server.
77
+ data = cloudpickle.loads(await socket.recv())
78
+
79
+ if not isinstance(data, expected_type):
80
+ # LoRAConfig can be None.
81
+ if expected_type == LoRAConfig and data is None:
82
+ pass
83
+ else:
84
+ raise ValueError(error_message)
85
+
86
+ return data
87
+
88
+ async def _send_one_way_rpc_request(self, request: RPC_REQUEST_TYPE,
89
+ error_message: str):
90
+ """Send one-way RPC request to trigger an action."""
91
+ with self.socket() as socket:
92
+ # Ping RPC Server with request.
93
+ await socket.send(cloudpickle.dumps(request))
94
+
95
+ # Await acknowledgement from RPCServer.
96
+ response = cloudpickle.loads(await socket.recv())
97
+
98
+ if not isinstance(response, str) or response != VLLM_RPC_SUCCESS_STR:
99
+ raise ValueError(error_message)
100
+
101
+ return response
102
+
103
+ async def get_tokenizer(self, lora_request: LoRARequest):
104
+ return await self.tokenizer.get_lora_tokenizer_async(lora_request)
105
+
106
+ async def get_decoding_config(self) -> DecodingConfig:
107
+ return self.decoding_config
108
+
109
+ async def get_model_config(self) -> ModelConfig:
110
+ return self.model_config
111
+
112
+ async def is_tracing_enabled(self) -> bool:
113
+ return self.tracing_flag
114
+
115
+ async def wait_for_server(self):
116
+ """Wait for the RPCServer to start up."""
117
+
118
+ await self._send_one_way_rpc_request(
119
+ request=RPCUtilityRequest.IS_SERVER_READY,
120
+ error_message="Unable to start RPC Server.")
121
+
122
+ async def _get_model_config_rpc(self) -> ModelConfig:
123
+ """Get the ModelConfig object from the RPC Server"""
124
+
125
+ return await self._send_get_data_rpc_request(
126
+ RPCUtilityRequest.GET_MODEL_CONFIG,
127
+ expected_type=ModelConfig,
128
+ error_message="Could not get ModelConfig from RPC Server")
129
+
130
+ async def _get_decoding_config_rpc(self) -> DecodingConfig:
131
+ """Get DecodingConfig from the RPCServer"""
132
+
133
+ return await self._send_get_data_rpc_request(
134
+ RPCUtilityRequest.GET_DECODING_CONFIG,
135
+ expected_type=DecodingConfig,
136
+ error_message="Could not get DecodingConfig from RPC Server")
137
+
138
+ async def _get_parallel_config_rpc(self) -> ParallelConfig:
139
+ """Get ParallelConfig from the RPCServer"""
140
+
141
+ return await self._send_get_data_rpc_request(
142
+ RPCUtilityRequest.GET_PARALLEL_CONFIG,
143
+ expected_type=ParallelConfig,
144
+ error_message="Could not get ParallelConfig from RPC Server")
145
+
146
+ async def _get_scheduler_config_rpc(self) -> SchedulerConfig:
147
+ """Get SchedulerConfig from the RPCServer"""
148
+
149
+ return await self._send_get_data_rpc_request(
150
+ RPCUtilityRequest.GET_SCHEDULER_CONFIG,
151
+ expected_type=SchedulerConfig,
152
+ error_message="Could not get SchedulerConfig from RPC Server")
153
+
154
+ async def _get_lora_config_rpc(self):
155
+ """Get LoRAConfig from the RPCServer"""
156
+
157
+ return await self._send_get_data_rpc_request(
158
+ RPCUtilityRequest.GET_LORA_CONFIG,
159
+ expected_type=LoRAConfig,
160
+ error_message="Could not get LoRAConfig from RPC Server")
161
+
162
+ async def _is_tracing_enabled_rpc(self) -> ParallelConfig:
163
+ """Get is_tracing_enabled flag from the RPCServer"""
164
+
165
+ return await self._send_get_data_rpc_request(
166
+ RPCUtilityRequest.IS_TRACING_ENABLED,
167
+ expected_type=bool,
168
+ error_message="Could not get is_tracing_enabled flag from RPC "
169
+ "Server")
170
+
171
+ async def abort(self, request_id: str):
172
+ """Send an ABORT_REQUEST signal to the RPC Server"""
173
+
174
+ await self._send_one_way_rpc_request(
175
+ request=RPCAbortRequest(request_id),
176
+ error_message=f"RPCAbortRequest {request_id} failed")
177
+
178
+ async def do_log_stats(self):
179
+ """Send a DO_LOG_STATS signal to the RPC Server"""
180
+
181
+ await self._send_one_way_rpc_request(
182
+ request=RPCUtilityRequest.DO_LOG_STATS,
183
+ error_message="RPCRequest DO_LOG_STATS failed.")
184
+
185
+ async def generate(
186
+ self,
187
+ inputs: PromptInputs,
188
+ sampling_params: SamplingParams,
189
+ request_id: str,
190
+ lora_request: Optional[LoRARequest] = None,
191
+ trace_headers: Optional[Mapping[str, str]] = None,
192
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
193
+ ) -> AsyncIterator[RequestOutput]:
194
+ """Send an RPCGenerateRequest to the RPCServer and stream responses."""
195
+
196
+ with self.socket() as socket:
197
+
198
+ # Send RPCGenerateRequest to the RPCServer.
199
+ await socket.send_multipart([
200
+ cloudpickle.dumps(
201
+ RPCGenerateRequest(
202
+ inputs=inputs,
203
+ sampling_params=sampling_params,
204
+ request_id=request_id,
205
+ lora_request=lora_request,
206
+ trace_headers=trace_headers,
207
+ prompt_adapter_request=prompt_adapter_request))
208
+ ])
209
+
210
+ # Stream back the results from the RPC Server.
211
+ while True:
212
+ message = await socket.recv()
213
+ request_output = cloudpickle.loads(message)
214
+
215
+ if isinstance(request_output, Exception):
216
+ raise request_output
217
+
218
+ if request_output.finished:
219
+ break
220
+ yield request_output
221
+
222
+ yield request_output
223
+
224
+ async def check_health(self) -> None:
225
+ """Raise if unhealthy"""
226
+
227
+ with self.socket() as socket:
228
+
229
+ # Ping RPCServer with CHECK_HEALTH request.
230
+ await socket.send(cloudpickle.dumps(RPCUtilityRequest.CHECK_HEALTH)
231
+ )
232
+
233
+ # Await the reply from the server.
234
+ # TODO: do we need an internal timeout here?
235
+ # Or do we expect the external probe to timeout and let this chill?
236
+ health_message = cloudpickle.loads(await socket.recv())
237
+
238
+ if isinstance(health_message, Exception):
239
+ raise health_message
240
+
241
+ if health_message != VLLM_RPC_HEALTHY_STR:
242
+ raise ValueError("Expected healthy response from backend but got "
243
+ "f{health_message}")
244
+
245
+ async def encode(self, *args,
246
+ **kwargs) -> AsyncIterator[EmbeddingRequestOutput]:
247
+ raise NotImplementedError(
248
+ "Embeddings not supported with multiprocessing backend")
vllm/entrypoints/openai/rpc/server.py
new
+216
@@ -0,0 +1,216 @@
1
+import asyncio
2
+import signal
3
+from typing import Any, Coroutine
4
+
5
+import cloudpickle
6
+import zmq
7
+import zmq.asyncio
8
+from typing_extensions import Never
9
+
10
+from vllm import AsyncEngineArgs, AsyncLLMEngine
11
+from vllm.entrypoints.openai.rpc import (VLLM_RPC_HEALTHY_STR,
12
+ VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
13
+ RPCGenerateRequest, RPCUtilityRequest)
14
+from vllm.logger import init_logger
15
+from vllm.usage.usage_lib import UsageContext
16
+
17
+logger = init_logger(__name__)
18
+
19
+
20
+class AsyncEngineRPCServer:
21
+
22
+ def __init__(self, async_engine_args: AsyncEngineArgs,
23
+ usage_context: UsageContext, port: int):
24
+ # Initialize engine first.
25
+ self.engine = AsyncLLMEngine.from_engine_args(async_engine_args,
26
+ usage_context)
27
+
28
+ # Initialize context.
29
+ self.context = zmq.asyncio.Context()
30
+
31
+ # Init socket for readiness state.
32
+ self.socket = self.context.socket(zmq.constants.ROUTER)
33
+ self.socket.bind(f"tcp://localhost:{port}")
34
+
35
+ def cleanup(self):
36
+ """Cleanup all resources."""
37
+ self.socket.close()
38
+ self.context.destroy()
39
+
40
+ async def get_model_config(self, identity):
41
+ """Send the ModelConfig"""
42
+ model_config = await self.engine.get_model_config()
43
+
44
+ await self.socket.send_multipart(
45
+ [identity, cloudpickle.dumps(model_config)])
46
+
47
+ async def get_decoding_config(self, identity):
48
+ """Send the DecodingConfig"""
49
+ decoding_config = await self.engine.get_decoding_config()
50
+
51
+ await self.socket.send_multipart(
52
+ [identity, cloudpickle.dumps(decoding_config)])
53
+
54
+ async def get_lora_config(self, identity):
55
+ lora_config = await self.engine.get_lora_config()
56
+
57
+ await self.socket.send_multipart(
58
+ [identity, cloudpickle.dumps(lora_config)])
59
+
60
+ async def get_scheduler_config(self, identity):
61
+ """Send the SchedulerConfig"""
62
+ parallel_config = await self.engine.get_scheduler_config()
63
+
64
+ await self.socket.send_multipart(
65
+ [identity, cloudpickle.dumps(parallel_config)])
66
+
67
+ async def get_parallel_config(self, identity):
68
+ """Send the ParallelConfig"""
69
+ parallel_config = await self.engine.get_parallel_config()
70
+
71
+ await self.socket.send_multipart(
72
+ [identity, cloudpickle.dumps(parallel_config)])
73
+
74
+ async def is_tracing_enabled(self, identity):
75
+ """Send the is_tracing_enabled flag"""
76
+ tracing_flag = await self.engine.is_tracing_enabled()
77
+
78
+ await self.socket.send_multipart(
79
+ [identity, cloudpickle.dumps(tracing_flag)])
80
+
81
+ async def do_log_stats(self, identity):
82
+ """Log stats and confirm success."""
83
+ await self.engine.do_log_stats()
84
+
85
+ await self.socket.send_multipart([
86
+ identity,
87
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
88
+ ])
89
+
90
+ async def is_server_ready(self, identity):
91
+ """Notify the client that we are ready."""
92
+ await self.socket.send_multipart([
93
+ identity,
94
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
95
+ ])
96
+
97
+ async def abort(self, identity, request: RPCAbortRequest):
98
+ """Abort request and notify the client of success."""
99
+ # Abort the request in the llm engine.
100
+ await self.engine.abort(request.request_id)
101
+
102
+ # Send confirmation to the client.
103
+ await self.socket.send_multipart([
104
+ identity,
105
+ cloudpickle.dumps(VLLM_RPC_SUCCESS_STR),
106
+ ])
107
+
108
+ async def generate(self, identity, generate_request: RPCGenerateRequest):
109
+ try:
110
+ results_generator = self.engine.generate(
111
+ generate_request.inputs,
112
+ sampling_params=generate_request.sampling_params,
113
+ request_id=generate_request.request_id,
114
+ lora_request=generate_request.lora_request,
115
+ trace_headers=generate_request.trace_headers,
116
+ prompt_adapter_request=generate_request.prompt_adapter_request)
117
+
118
+ async for request_output in results_generator:
119
+ await self.socket.send_multipart(
120
+ [identity, cloudpickle.dumps(request_output)])
121
+
122
+ except Exception as e:
123
+ ### Notify client of all failures
124
+ await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
125
+
126
+ async def check_health(self, identity):
127
+ try:
128
+ await self.engine.check_health()
129
+ await self.socket.send_multipart(
130
+ [identity, cloudpickle.dumps(VLLM_RPC_HEALTHY_STR)])
131
+ except Exception as e:
132
+ await self.socket.send_multipart([identity, cloudpickle.dumps(e)])
133
+
134
+ def _make_handler_coro(self, identity,
135
+ message) -> Coroutine[Any, Any, Never]:
136
+ """Route the zmq message to the handler coroutine."""
137
+
138
+ request = cloudpickle.loads(message)
139
+
140
+ if isinstance(request, RPCGenerateRequest):
141
+ return self.generate(identity, request)
142
+
143
+ elif isinstance(request, RPCAbortRequest):
144
+ return self.abort(identity, request)
145
+
146
+ elif isinstance(request, RPCUtilityRequest):
147
+ if request == RPCUtilityRequest.GET_MODEL_CONFIG:
148
+ return self.get_model_config(identity)
149
+ elif request == RPCUtilityRequest.GET_PARALLEL_CONFIG:
150
+ return self.get_parallel_config(identity)
151
+ elif request == RPCUtilityRequest.GET_DECODING_CONFIG:
152
+ return self.get_decoding_config(identity)
153
+ elif request == RPCUtilityRequest.GET_SCHEDULER_CONFIG:
154
+ return self.get_scheduler_config(identity)
155
+ elif request == RPCUtilityRequest.GET_LORA_CONFIG:
156
+ return self.get_lora_config(identity)
157
+ elif request == RPCUtilityRequest.DO_LOG_STATS:
158
+ return self.do_log_stats(identity)
159
+ elif request == RPCUtilityRequest.IS_SERVER_READY:
160
+ return self.is_server_ready(identity)
161
+ elif request == RPCUtilityRequest.CHECK_HEALTH:
162
+ return self.check_health(identity)
163
+ elif request == RPCUtilityRequest.IS_TRACING_ENABLED:
164
+ return self.is_tracing_enabled(identity)
165
+ else:
166
+ raise ValueError(f"Unknown RPCUtilityRequest type: {request}")
167
+
168
+ else:
169
+ raise ValueError(f"Unknown RPCRequest type: {request}")
170
+
171
+ async def run_server_loop(self):
172
+ """Inner RPC Server Loop"""
173
+
174
+ running_tasks = set()
175
+ while True:
176
+ # Wait for a request.
177
+ identity, message = await self.socket.recv_multipart()
178
+
179
+ # Process the request async.
180
+ task = asyncio.create_task(
181
+ self._make_handler_coro(identity, message))
182
+
183
+ # We need to keep around a strong reference to the task,
184
+ # to avoid the task disappearing mid-execution as running tasks
185
+ # can be GC'ed. Below is a common "fire-and-forget" tasks
186
+ # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
187
+ running_tasks.add(task)
188
+ task.add_done_callback(running_tasks.discard)
189
+
190
+
191
+async def run_server(server: AsyncEngineRPCServer):
192
+ # Put the server task into the asyncio loop.
193
+ loop = asyncio.get_running_loop()
194
+ server_task = loop.create_task(server.run_server_loop())
195
+
196
+ # Interruption handling.
197
+ def signal_handler() -> None:
198
+ # Kill the server on interrupt / terminate
199
+ server_task.cancel()
200
+
201
+ loop.add_signal_handler(signal.SIGINT, signal_handler)
202
+ loop.add_signal_handler(signal.SIGTERM, signal_handler)
203
+
204
+ try:
205
+ await server_task
206
+ except asyncio.CancelledError:
207
+ logger.info("vLLM ZMQ RPC Server was interrupted.")
208
+ finally:
209
+ # Clean up all resources.
210
+ server.cleanup()
211
+
212
+
213
+def run_rpc_server(async_engine_args: AsyncEngineArgs,
214
+ usage_context: UsageContext, port: int):
215
+ server = AsyncEngineRPCServer(async_engine_args, usage_context, port)
216
+ asyncio.run(run_server(server))
vllm/entrypoints/openai/serving_chat.py
+9
-7
@@ -8,7 +8,7 @@ from fastapi import Request
8
from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.engine.protocol import AsyncEngineClient
12
from vllm.entrypoints.chat_utils import (ConversationMessage,
13
load_chat_template,
14
parse_chat_message_content)
@@ -39,7 +39,7 @@ class OpenAIServingChat(OpenAIServing):
39
40
def __init__(
41
self,
42
- engine: AsyncLLMEngine,
42
+ async_engine_client: AsyncEngineClient,
43
model_config: ModelConfig,
44
served_model_names: List[str],
45
response_role: str,
@@ -50,7 +50,7 @@ class OpenAIServingChat(OpenAIServing):
50
chat_template: Optional[str],
51
return_tokens_as_token_ids: bool = False,
52
):
53
- super().__init__(engine=engine,
53
+ super().__init__(async_engine_client=async_engine_client,
54
model_config=model_config,
55
served_model_names=served_model_names,
56
lora_modules=lora_modules,
@@ -89,7 +89,8 @@ class OpenAIServingChat(OpenAIServing):
89
) = self._maybe_get_adapters(request)
90
91
model_config = self.model_config
92
- tokenizer = await self.engine.get_tokenizer(lora_request)
92
+ tokenizer = await self.async_engine_client.get_tokenizer(
93
+ lora_request)
94
95
conversation: List[ConversationMessage] = []
96
mm_futures: List[Awaitable[MultiModalDataDict]] = []
@@ -161,7 +162,8 @@ class OpenAIServingChat(OpenAIServing):
162
if mm_data is not None:
163
engine_inputs["multi_modal_data"] = mm_data
164
164
- is_tracing_enabled = await self.engine.is_tracing_enabled()
165
+ is_tracing_enabled = (
166
+ await self.async_engine_client.is_tracing_enabled())
167
trace_headers = None
168
if is_tracing_enabled and raw_request:
169
trace_headers = extract_trace_headers(raw_request.headers)
@@ -169,7 +171,7 @@ class OpenAIServingChat(OpenAIServing):
171
and contains_trace_headers(raw_request.headers)):
172
log_tracing_disabled_warning()
173
172
- result_generator = self.engine.generate(
174
+ result_generator = self.async_engine_client.generate(
175
engine_inputs,
176
sampling_params,
177
request_id,
@@ -441,7 +443,7 @@ class OpenAIServingChat(OpenAIServing):
443
async for res in result_generator:
444
if raw_request is not None and await raw_request.is_disconnected():
445
# Abort the request if the client disconnects.
444
- await self.engine.abort(request_id)
446
+ await self.async_engine_client.abort(request_id)
447
return self.create_error_response("Client disconnected")
448
final_res = res
449
assert final_res is not None
vllm/entrypoints/openai/serving_completion.py
+11
-8
@@ -8,7 +8,7 @@ from fastapi import Request
8
from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.engine.protocol import AsyncEngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
14
# yapf: disable
@@ -42,7 +42,7 @@ class OpenAIServingCompletion(OpenAIServing):
42
43
def __init__(
44
self,
45
- engine: AsyncLLMEngine,
45
+ async_engine_client: AsyncEngineClient,
46
model_config: ModelConfig,
47
served_model_names: List[str],
48
*,
@@ -51,7 +51,7 @@ class OpenAIServingCompletion(OpenAIServing):
51
request_logger: Optional[RequestLogger],
52
return_tokens_as_token_ids: bool = False,
53
):
54
- super().__init__(engine=engine,
54
+ super().__init__(async_engine_client=async_engine_client,
55
model_config=model_config,
56
served_model_names=served_model_names,
57
lora_modules=lora_modules,
@@ -91,7 +91,8 @@ class OpenAIServingCompletion(OpenAIServing):
91
prompt_adapter_request,
92
) = self._maybe_get_adapters(request)
93
94
- tokenizer = await self.engine.get_tokenizer(lora_request)
94
+ tokenizer = await self.async_engine_client.get_tokenizer(
95
+ lora_request)
96
97
guided_decode_logits_processor = (
98
await self._guided_decode_logits_processor(request, tokenizer))
@@ -119,7 +120,8 @@ class OpenAIServingCompletion(OpenAIServing):
120
lora_request=lora_request,
121
prompt_adapter_request=prompt_adapter_request)
122
122
- is_tracing_enabled = await self.engine.is_tracing_enabled()
123
+ is_tracing_enabled = (
124
+ await self.async_engine_client.is_tracing_enabled())
125
trace_headers = None
126
if is_tracing_enabled:
127
trace_headers = extract_trace_headers(raw_request.headers)
@@ -127,7 +129,7 @@ class OpenAIServingCompletion(OpenAIServing):
129
raw_request.headers):
130
log_tracing_disabled_warning()
131
130
- generator = self.engine.generate(
132
+ generator = self.async_engine_client.generate(
133
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
134
sampling_params,
135
request_id_item,
@@ -168,7 +170,7 @@ class OpenAIServingCompletion(OpenAIServing):
170
async for i, res in result_generator:
171
if await raw_request.is_disconnected():
172
# Abort the request if the client disconnects.
171
- await self.engine.abort(f"{request_id}-{i}")
173
+ await self.async_engine_client.abort(f"{request_id}-{i}")
174
return self.create_error_response("Client disconnected")
175
final_res_batch[i] = res
176
@@ -230,7 +232,8 @@ class OpenAIServingCompletion(OpenAIServing):
232
233
# Abort the request if the client disconnects.
234
if await raw_request.is_disconnected():
233
- await self.engine.abort(f"{request_id}-{prompt_idx}")
235
+ await self.async_engine_client.abort(
236
+ f"{request_id}-{prompt_idx}")
237
raise StopAsyncIteration()
238
239
for output in res.outputs:
vllm/entrypoints/openai/serving_embedding.py
+7
-6
@@ -6,7 +6,7 @@ import numpy as np
6
from fastapi import Request
7
8
from vllm.config import ModelConfig
9
-from vllm.engine.async_llm_engine import AsyncLLMEngine
9
+from vllm.engine.protocol import AsyncEngineClient
10
from vllm.entrypoints.logger import RequestLogger
11
from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
12
EmbeddingResponse,
@@ -56,13 +56,13 @@ class OpenAIServingEmbedding(OpenAIServing):
56
57
def __init__(
58
self,
59
- engine: AsyncLLMEngine,
59
+ async_engine_client: AsyncEngineClient,
60
model_config: ModelConfig,
61
served_model_names: List[str],
62
*,
63
request_logger: Optional[RequestLogger],
64
):
65
- super().__init__(engine=engine,
65
+ super().__init__(async_engine_client=async_engine_client,
66
model_config=model_config,
67
served_model_names=served_model_names,
68
lora_modules=None,
@@ -99,7 +99,8 @@ class OpenAIServingEmbedding(OpenAIServing):
99
prompt_adapter_request,
100
) = self._maybe_get_adapters(request)
101
102
- tokenizer = await self.engine.get_tokenizer(lora_request)
102
+ tokenizer = await self.async_engine_client.get_tokenizer(
103
+ lora_request)
104
105
pooling_params = request.to_pooling_params()
106
@@ -124,7 +125,7 @@ class OpenAIServingEmbedding(OpenAIServing):
125
"Prompt adapter is not supported "
126
"for embedding models")
127
127
- generator = self.engine.encode(
128
+ generator = self.async_engine_client.encode(
129
{"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
130
pooling_params,
131
request_id_item,
@@ -146,7 +147,7 @@ class OpenAIServingEmbedding(OpenAIServing):
147
async for i, res in result_generator:
148
if await raw_request.is_disconnected():
149
# Abort the request if the client disconnects.
149
- await self.engine.abort(f"{request_id}-{i}")
150
+ await self.async_engine_client.abort(f"{request_id}-{i}")
151
return self.create_error_response("Client disconnected")
152
final_res_batch[i] = res
153
vllm/entrypoints/openai/serving_engine.py
+4
-4
@@ -8,7 +8,7 @@ from pydantic import Field
8
from typing_extensions import Annotated
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.async_llm_engine import AsyncLLMEngine
11
+from vllm.engine.protocol import AsyncEngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
14
# yapf: disable
@@ -61,7 +61,7 @@ class OpenAIServing:
61
62
def __init__(
63
self,
64
- engine: AsyncLLMEngine,
64
+ async_engine_client: AsyncEngineClient,
65
model_config: ModelConfig,
66
served_model_names: List[str],
67
*,
@@ -72,7 +72,7 @@ class OpenAIServing:
72
):
73
super().__init__()
74
75
- self.engine = engine
75
+ self.async_engine_client = async_engine_client
76
self.model_config = model_config
77
self.max_model_len = model_config.max_model_len
78
@@ -155,7 +155,7 @@ class OpenAIServing:
155
async def _guided_decode_logits_processor(
156
self, request: Union[ChatCompletionRequest, CompletionRequest],
157
tokenizer: AnyTokenizer) -> Optional[LogitsProcessor]:
158
- decoding_config = await self.engine.get_decoding_config()
158
+ decoding_config = await self.async_engine_client.get_decoding_config()
159
guided_decoding_backend = request.guided_decoding_backend \
160
or decoding_config.guided_decoding_backend
161
return await get_guided_decoding_logits_processor(
vllm/entrypoints/openai/serving_tokenization.py
+5
-5
@@ -1,9 +1,9 @@
1
from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
-from vllm.engine.async_llm_engine import AsyncLLMEngine
4
# yapf conflicts with isort for this block
5
# yapf: disable
6
+from vllm.engine.protocol import AsyncEngineClient
7
from vllm.entrypoints.chat_utils import (ConversationMessage,
8
load_chat_template,
9
parse_chat_message_content)
@@ -24,7 +24,7 @@ class OpenAIServingTokenization(OpenAIServing):
24
25
def __init__(
26
self,
27
- engine: AsyncLLMEngine,
27
+ async_engine_client: AsyncEngineClient,
28
model_config: ModelConfig,
29
served_model_names: List[str],
30
*,
@@ -32,7 +32,7 @@ class OpenAIServingTokenization(OpenAIServing):
32
request_logger: Optional[RequestLogger],
33
chat_template: Optional[str],
34
):
35
- super().__init__(engine=engine,
35
+ super().__init__(async_engine_client=async_engine_client,
36
model_config=model_config,
37
served_model_names=served_model_names,
38
lora_modules=lora_modules,
@@ -57,7 +57,7 @@ class OpenAIServingTokenization(OpenAIServing):
57
prompt_adapter_request,
58
) = self._maybe_get_adapters(request)
59
60
- tokenizer = await self.engine.get_tokenizer(lora_request)
60
+ tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
61
62
if isinstance(request, TokenizeChatRequest):
63
model_config = self.model_config
@@ -113,7 +113,7 @@ class OpenAIServingTokenization(OpenAIServing):
113
prompt_adapter_request,
114
) = self._maybe_get_adapters(request)
115
116
- tokenizer = await self.engine.get_tokenizer(lora_request)
116
+ tokenizer = await self.async_engine_client.get_tokenizer(lora_request)
117
118
self._log_inputs(request_id,
119
request.tokens,
vllm/envs.py
+6
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
4
if TYPE_CHECKING:
5
VLLM_HOST_IP: str = ""
6
VLLM_PORT: Optional[int] = None
7
+ VLLM_RPC_PORT: int = 5570
8
VLLM_USE_MODELSCOPE: bool = False
9
VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
10
VLLM_INSTANCE_ID: Optional[str] = None
@@ -140,6 +141,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
141
lambda: int(os.getenv('VLLM_PORT', '0'))
142
if 'VLLM_PORT' in os.environ else None,
143
144
+ # used when the frontend api server is running in multi-processing mode,
145
+ # to communicate with the backend engine process over ZMQ.
146
+ 'VLLM_RPC_PORT':
147
+ lambda: int(os.getenv('VLLM_PORT', '5570')),
148
+
149
# If true, will load models from ModelScope instead of Hugging Face Hub.
150
# note that the value is true or false, not numbers
151
"VLLM_USE_MODELSCOPE":
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+19
@@ -21,6 +21,8 @@ from functools import lru_cache
21
from typing import Callable, DefaultDict, Dict, List, Union
22
23
import torch
24
+from lark import Lark
25
+from outlines import grammars
26
from outlines.caching import cache
27
from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
28
from outlines.fsm.json_schema import build_regex_from_schema
@@ -44,6 +46,23 @@ class BaseLogitsProcessor:
46
last_seq_id = hash(tuple(input_ids[:-1]))
47
self._fsm_state[seq_id] = self._guide.get_next_state(
48
state=self._fsm_state[last_seq_id], token_id=last_token)
49
+ else:
50
+ # Note: this is a hack.
51
+ # Lark pickling does not work properly (silent failure),
52
+ # which breaks the RPC (which uses python pickleing).
53
+ # We need to find a better solution.
54
+ # On the first time this is called, we simply re-create
55
+ # the Lark object.
56
+ if isinstance(self._guide, CFGGuide):
57
+ self._guide.parser = Lark(
58
+ self._guide.cfg_string,
59
+ parser="lalr",
60
+ lexer="contextual",
61
+ propagate_positions=False,
62
+ maybe_placeholders=False,
63
+ regex=True,
64
+ import_paths=[grammars.GRAMMAR_PATH],
65
+ )
66
67
instruction = self._guide.get_next_instruction(
68
state=self._fsm_state[seq_id])
vllm/tracing.py
+1
-1
@@ -60,7 +60,7 @@ def get_span_exporter(endpoint):
60
OTLPSpanExporter)
61
elif protocol == "http/protobuf":
62
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
63
- OTLPSpanExporter)
63
+ OTLPSpanExporter) # type: ignore
64
else:
65
raise ValueError(
66
f"Unsupported OTLP protocol '{protocol}' is configured")
vllm/transformers_utils/tokenizer_group/__init__.py
+18
-1
@@ -1,6 +1,7 @@
1
from typing import Optional, Type
2
3
-from vllm.config import TokenizerPoolConfig
3
+from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
4
+ TokenizerPoolConfig)
5
from vllm.executor.ray_utils import ray
6
7
from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
@@ -13,6 +14,22 @@ else:
14
RayTokenizerGroupPool = None # type: ignore
15
16
17
+def init_tokenizer_from_configs(model_config: ModelConfig,
18
+ scheduler_config: SchedulerConfig,
19
+ parallel_config: ParallelConfig,
20
+ enable_lora: bool):
21
+ init_kwargs = dict(tokenizer_id=model_config.tokenizer,
22
+ enable_lora=enable_lora,
23
+ max_num_seqs=scheduler_config.max_num_seqs,
24
+ max_input_length=None,
25
+ tokenizer_mode=model_config.tokenizer_mode,
26
+ trust_remote_code=model_config.trust_remote_code,
27
+ revision=model_config.tokenizer_revision)
28
+
29
+ return get_tokenizer_group(parallel_config.tokenizer_pool_config,
30
+ **init_kwargs)
31
+
32
+
33
def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
34
**init_kwargs) -> BaseTokenizerGroup:
35
tokenizer_cls: Type[BaseTokenizerGroup]
vllm/utils.py
+22
-6
@@ -290,6 +290,10 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
290
return _async_wrapper
291
292
293
+class ProducerFinished:
294
+ pass
295
+
296
+
297
def merge_async_iterators(
298
*iterators: AsyncIterator[T]) -> AsyncIterator[Tuple[int, T]]:
299
"""Merge multiple asynchronous iterators into a single iterator.
@@ -298,9 +302,10 @@ def merge_async_iterators(
302
When it yields, it yields a tuple (i, item) where i is the index of the
303
iterator that yields the item.
304
"""
301
- queue: asyncio.Queue[Union[Tuple[int, T], Exception]] = asyncio.Queue()
305
+ queue: asyncio.Queue[Union[Tuple[int, T], ProducerFinished,
306
+ Exception]] = asyncio.Queue()
307
303
- finished = [False] * len(iterators)
308
+ producers = len(iterators)
309
310
async def producer(i: int, iterator: AsyncIterator[T]):
311
try:
@@ -308,7 +313,8 @@ def merge_async_iterators(
313
await queue.put((i, item))
314
except Exception as e:
315
await queue.put(e)
311
- finished[i] = True
316
+ # Signal to the consumer that we've finished
317
+ await queue.put(ProducerFinished())
318
319
_tasks = [
320
asyncio.create_task(producer(i, iterator))
@@ -316,9 +322,17 @@ def merge_async_iterators(
322
]
323
324
async def consumer():
325
+ remaining = producers
326
try:
320
- while not all(finished) or not queue.empty():
327
+ while remaining or not queue.empty():
328
+ # we think there is a race condition here
329
item = await queue.get()
330
+
331
+ if isinstance(item, ProducerFinished):
332
+ # Signal that a producer finished- not a real item
333
+ remaining -= 1
334
+ continue
335
+
336
if isinstance(item, Exception):
337
raise item
338
yield item
@@ -374,8 +388,10 @@ def get_distributed_init_method(ip: str, port: int) -> str:
388
return f"tcp://[{ip}]:{port}" if ":" in ip else f"tcp://{ip}:{port}"
389
390
377
-def get_open_port() -> int:
378
- port = envs.VLLM_PORT
391
+def get_open_port(port: Optional[int] = None) -> int:
392
+ if port is None:
393
+ # Default behavior here is to return a port for multi-gpu communication
394
+ port = envs.VLLM_PORT
395
if port is not None:
396
while True:
397
try: