A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc][core] lazy import outlines (#7831)
youkaichao committed
Aug 24, 2024 at 00:51 UTC
7d9ffa2ae102cbfae65035c511f8d3c8e5fab986
4 files changed
+64
-7
.buildkite/test-pipeline.yaml
+2
-1
@@ -87,7 +87,8 @@ steps:
87
commands:
88
- pip install -e ./plugins/vllm_add_dummy_model
89
- pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@a4987bba6e9e9b3f22bd3a6c1ecf0abd04fd5622#egg=lm_eval[api]
90
- - pytest -v -s entrypoints/llm
90
+ - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py
91
+ - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
92
- pytest -v -s entrypoints/openai
93
94
- label: Distributed Tests (4 GPUs) # 10min
tests/entrypoints/llm/test_lazy_outlines.py
new
+48
@@ -0,0 +1,48 @@
1
+import sys
2
+
3
+from vllm import LLM, SamplingParams
4
+
5
+
6
+def test_lazy_outlines(sample_regex):
7
+ """If users don't use guided decoding, outlines should not be imported.
8
+ """
9
+ prompts = [
10
+ "Hello, my name is",
11
+ "The president of the United States is",
12
+ "The capital of France is",
13
+ "The future of AI is",
14
+ ]
15
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
16
+
17
+ llm = LLM(model="facebook/opt-125m",
18
+ enforce_eager=True,
19
+ gpu_memory_utilization=0.3)
20
+ outputs = llm.generate(prompts, sampling_params)
21
+ for output in outputs:
22
+ prompt = output.prompt
23
+ generated_text = output.outputs[0].text
24
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
25
+
26
+ # make sure outlines is not imported
27
+ assert 'outlines' not in sys.modules
28
+
29
+ llm = LLM(model="facebook/opt-125m",
30
+ enforce_eager=True,
31
+ guided_decoding_backend="lm-format-enforcer",
32
+ gpu_memory_utilization=0.3)
33
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
34
+ outputs = llm.generate(
35
+ prompts=[
36
+ f"Give an example IPv4 address with this regex: {sample_regex}"
37
+ ] * 2,
38
+ sampling_params=sampling_params,
39
+ use_tqdm=True,
40
+ guided_options_request=dict(guided_regex=sample_regex))
41
+
42
+ for output in outputs:
43
+ prompt = output.prompt
44
+ generated_text = output.outputs[0].text
45
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
46
+
47
+ # make sure outlines is not imported
48
+ assert 'outlines' not in sys.modules
vllm/model_executor/guided_decoding/__init__.py
+6
-3
@@ -5,9 +5,6 @@ from vllm.entrypoints.openai.protocol import (
5
CompletionRequest)
6
from vllm.model_executor.guided_decoding.guided_fields import (
7
GuidedDecodingRequest)
8
-from vllm.model_executor.guided_decoding.outlines_decoding import (
9
- get_local_outlines_guided_decoding_logits_processor,
10
- get_outlines_guided_decoding_logits_processor)
8
from vllm.sampling_params import LogitsProcessor
9
10
@@ -18,6 +15,9 @@ async def get_guided_decoding_logits_processor(
15
request = _adapt_request_for_tool_use(request)
16
17
if guided_decoding_backend == 'outlines':
18
+ # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
19
+ from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
20
+ get_outlines_guided_decoding_logits_processor)
21
return await get_outlines_guided_decoding_logits_processor(
22
request, tokenizer)
23
if guided_decoding_backend == 'lm-format-enforcer':
@@ -37,6 +37,9 @@ def get_local_guided_decoding_logits_processor(
37
# request = _adapt_request_for_tool_use(request)
38
39
if guided_decoding_backend == 'outlines':
40
+ # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
41
+ from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
42
+ get_local_outlines_guided_decoding_logits_processor)
43
return get_local_outlines_guided_decoding_logits_processor(
44
guided_options, tokenizer)
45
if guided_decoding_backend == 'lm-format-enforcer':
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+8
-3
@@ -14,9 +14,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14
CompletionRequest)
15
from vllm.model_executor.guided_decoding.guided_fields import (
16
GuidedDecodingRequest)
17
-from vllm.model_executor.guided_decoding.outlines_decoding import (
18
- get_local_outlines_guided_decoding_logits_processor,
19
- get_outlines_guided_decoding_logits_processor)
17
from vllm.sampling_params import LogitsProcessor
18
19
@@ -43,6 +40,10 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
40
character_level_parser = RegexParser(request.guided_regex)
41
elif request.guided_grammar:
42
# CFG grammar not supported by LMFE, revert to outlines
43
+
44
+ # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
45
+ from vllm.model_executor.guided_decoding.outlines_decoding import (
46
+ get_outlines_guided_decoding_logits_processor)
47
return await get_outlines_guided_decoding_logits_processor(
48
request, tokenizer)
49
elif (request.response_format is not None
@@ -87,6 +88,10 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
88
character_level_parser = RegexParser(guided_options.guided_regex)
89
elif guided_options.guided_grammar:
90
# CFG grammar not supported by LMFE, revert to outlines
91
+
92
+ # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
93
+ from vllm.model_executor.guided_decoding.outlines_decoding import (
94
+ get_local_outlines_guided_decoding_logits_processor)
95
return get_local_outlines_guided_decoding_logits_processor(
96
guided_options, tokenizer)
97
elif guided_options.guided_json_object: