A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] use diskcache in outlines _get_guide #5436 (#6203)
Eric committed
Jul 9, 2024 at 02:23 UTC
185ad31f37541ac205b55f446bfd71542f83075a
1 file changed
+3
-2
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+3
-2
@@ -21,6 +21,7 @@ from functools import lru_cache
21
from typing import Callable, DefaultDict, Dict, List, Union
22
23
import torch
24
+from outlines.caching import cache
25
from outlines.fsm.guide import CFGGuide, Generate, Guide, RegexGuide, Write
26
from outlines.fsm.json_schema import build_regex_from_schema
27
from pydantic import BaseModel
@@ -67,7 +68,7 @@ class BaseLogitsProcessor:
68
class RegexLogitsProcessor(BaseLogitsProcessor):
69
70
@classmethod
70
- @lru_cache(maxsize=32)
71
+ @cache()
72
def _get_guide(cls, regex_string: str,
73
tokenizer: PreTrainedTokenizerBase) -> Guide:
74
tokenizer = _adapt_tokenizer(tokenizer)
@@ -126,7 +127,7 @@ class JSONLogitsProcessor(RegexLogitsProcessor):
127
class CFGLogitsProcessor(BaseLogitsProcessor):
128
129
@classmethod
129
- @lru_cache(maxsize=32)
130
+ @cache()
131
def _get_guide(cls, cfg: str, tokenizer: PreTrainedTokenizerBase) -> Guide:
132
tokenizer = _adapt_tokenizer(tokenizer)
133
return CFGGuide(cfg, tokenizer)