@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7d9ffa2ae1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc][core] lazy import outlines (#7831)

youkaichao committed Aug 24, 2024 at 00:51 UTC 7d9ffa2ae102cbfae65035c511f8d3c8e5fab986
4 files changed +64 -7
.buildkite/test-pipeline.yaml
+2 -1
@@ -87,7 +87,8 @@ steps:
87 commands:
88 - pip install -e ./plugins/vllm_add_dummy_model
89 - pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@a4987bba6e9e9b3f22bd3a6c1ecf0abd04fd5622#egg=lm_eval[api]
90 - - pytest -v -s entrypoints/llm
90 + - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_lazy_outlines.py
91 + - pytest -v -s entrypoints/llm/test_lazy_outlines.py # it needs a clean process
92 - pytest -v -s entrypoints/openai
93
94 - label: Distributed Tests (4 GPUs) # 10min
tests/entrypoints/llm/test_lazy_outlines.py new
+48
@@ -0,0 +1,48 @@
1 +import sys
2 +
3 +from vllm import LLM, SamplingParams
4 +
5 +
6 +def test_lazy_outlines(sample_regex):
7 + """If users don't use guided decoding, outlines should not be imported.
8 + """
9 + prompts = [
10 + "Hello, my name is",
11 + "The president of the United States is",
12 + "The capital of France is",
13 + "The future of AI is",
14 + ]
15 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
16 +
17 + llm = LLM(model="facebook/opt-125m",
18 + enforce_eager=True,
19 + gpu_memory_utilization=0.3)
20 + outputs = llm.generate(prompts, sampling_params)
21 + for output in outputs:
22 + prompt = output.prompt
23 + generated_text = output.outputs[0].text
24 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
25 +
26 + # make sure outlines is not imported
27 + assert 'outlines' not in sys.modules
28 +
29 + llm = LLM(model="facebook/opt-125m",
30 + enforce_eager=True,
31 + guided_decoding_backend="lm-format-enforcer",
32 + gpu_memory_utilization=0.3)
33 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
34 + outputs = llm.generate(
35 + prompts=[
36 + f"Give an example IPv4 address with this regex: {sample_regex}"
37 + ] * 2,
38 + sampling_params=sampling_params,
39 + use_tqdm=True,
40 + guided_options_request=dict(guided_regex=sample_regex))
41 +
42 + for output in outputs:
43 + prompt = output.prompt
44 + generated_text = output.outputs[0].text
45 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
46 +
47 + # make sure outlines is not imported
48 + assert 'outlines' not in sys.modules
vllm/model_executor/guided_decoding/__init__.py
+6 -3
@@ -5,9 +5,6 @@ from vllm.entrypoints.openai.protocol import (
5 CompletionRequest)
6 from vllm.model_executor.guided_decoding.guided_fields import (
7 GuidedDecodingRequest)
8 -from vllm.model_executor.guided_decoding.outlines_decoding import (
9 - get_local_outlines_guided_decoding_logits_processor,
10 - get_outlines_guided_decoding_logits_processor)
8 from vllm.sampling_params import LogitsProcessor
9
10
@@ -18,6 +15,9 @@ async def get_guided_decoding_logits_processor(
15 request = _adapt_request_for_tool_use(request)
16
17 if guided_decoding_backend == 'outlines':
18 + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
19 + from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
20 + get_outlines_guided_decoding_logits_processor)
21 return await get_outlines_guided_decoding_logits_processor(
22 request, tokenizer)
23 if guided_decoding_backend == 'lm-format-enforcer':
@@ -37,6 +37,9 @@ def get_local_guided_decoding_logits_processor(
37 # request = _adapt_request_for_tool_use(request)
38
39 if guided_decoding_backend == 'outlines':
40 + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
41 + from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
42 + get_local_outlines_guided_decoding_logits_processor)
43 return get_local_outlines_guided_decoding_logits_processor(
44 guided_options, tokenizer)
45 if guided_decoding_backend == 'lm-format-enforcer':
vllm/model_executor/guided_decoding/lm_format_enforcer_decoding.py
+8 -3
@@ -14,9 +14,6 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
14 CompletionRequest)
15 from vllm.model_executor.guided_decoding.guided_fields import (
16 GuidedDecodingRequest)
17 -from vllm.model_executor.guided_decoding.outlines_decoding import (
18 - get_local_outlines_guided_decoding_logits_processor,
19 - get_outlines_guided_decoding_logits_processor)
17 from vllm.sampling_params import LogitsProcessor
18
19
@@ -43,6 +40,10 @@ async def get_lm_format_enforcer_guided_decoding_logits_processor(
40 character_level_parser = RegexParser(request.guided_regex)
41 elif request.guided_grammar:
42 # CFG grammar not supported by LMFE, revert to outlines
43 +
44 + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
45 + from vllm.model_executor.guided_decoding.outlines_decoding import (
46 + get_outlines_guided_decoding_logits_processor)
47 return await get_outlines_guided_decoding_logits_processor(
48 request, tokenizer)
49 elif (request.response_format is not None
@@ -87,6 +88,10 @@ def get_local_lm_format_enforcer_guided_decoding_logits_processor(
88 character_level_parser = RegexParser(guided_options.guided_regex)
89 elif guided_options.guided_grammar:
90 # CFG grammar not supported by LMFE, revert to outlines
91 +
92 + # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
93 + from vllm.model_executor.guided_decoding.outlines_decoding import (
94 + get_local_outlines_guided_decoding_logits_processor)
95 return get_local_outlines_guided_decoding_logits_processor(
96 guided_options, tokenizer)
97 elif guided_options.guided_json_object: