@samitouri / QOSami_Agents_Zero-1 / commits / bc2d447d

fix(document-query): bound long PDF processing

Disable LiteParse OCR automatically for PDFs at or above the configured page threshold, independent of sampled text density. Add adaptive index chunk sizing for large extracted documents so first-run document_query calls avoid spending the full batch timeout embedding thousands of small chunks. Update the settings UI, README, dependency hook, and regression tests for the new behavior.

Alessandro committed Jun 2, 2026 at 17:52 UTC bc2d447dddf45e7b4007a1c728537caa6f4e3283

7 files changed +133 -27

plugins/_document_query/README.md

+4 -2

@@ -8,7 +8,8 @@ timeouts and thread-safe parsers.

8 - **Strategy-pattern parsers** - MIME-type routing to dedicated parser classes

9 - **Centralized fetching** - local and HTTP(S) resources are fetched once, size-checked, then passed to parsers

10 - **LiteParse first path** - fast local parsing for PDFs and supported document/image formats, with legacy fallbacks

11 -- **Adaptive OCR** - large text-rich PDFs skip OCR automatically to avoid pathological parse times

11 +- **Adaptive OCR** - long PDFs skip OCR automatically to avoid pathological parse times

12 +- **Adaptive indexing** - very large extracted documents increase chunk size to keep embedding work bounded

13 - **Bounded parser execution** - sync parsers are offloaded to asyncio.to_thread and globally capped across chats

14 - **Configurable timeouts** - per-document and gather-level timeouts

15 - **Expanded format support** - PDF, HTML, text, YAML, XML, TOML, JS, TS, images, and catch-all Unstructured

@@ -28,10 +29,11 @@ See default_config.yaml for all options. Key settings:

29 | context_intro_chunks | 2 | Leading chunks included per document for title/abstract grounding |

30 | chunk_size | 1000 | Text splitter chunk size |

31 | chunk_overlap | 100 | Text splitter overlap |

32 +| max_index_chunks | 1200 | Maximum indexed chunks before adaptive chunk sizing, or 0 for no cap |

33 | search_threshold | 0.5 | Similarity search threshold |

34 | liteparse_enabled | true | Prefer LiteParse before legacy parser fallbacks |

35 | liteparse_num_workers | 2 | Max LiteParse OCR workers per parser job |

34 -| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for text-rich PDFs at or above this effective page count |

36 +| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for PDFs at or above this effective page count |

37 | thread_offload | true | Offload sync parsers to thread pool |

39 LiteParse is installed into the Agent Zero framework runtime from hooks.py during

plugins/_document_query/default_config.yaml

+2 -2

@@ -13,6 +13,7 @@ parser_concurrency: 1 # max parser jobs running across all chats in this

13 context_intro_chunks: 2 # always include leading chunks per document for title/abstract grounding

14 chunk_size: 1000

15 chunk_overlap: 100

16 +max_index_chunks: 1200 # adapt chunk size above this many indexed chunks

17 search_threshold: 0.5

18 search_limit: 100

19 max_remote_bytes: 52428800 # 50 MB

@@ -29,9 +30,8 @@ liteparse_dpi: 150

30 liteparse_preserve_very_small_text: false

31 liteparse_output_format: text

32 liteparse_num_workers: 2 # balanced default for OCR speed without overloading shared Web UI runtime

32 -liteparse_ocr_auto_disable: true # disable OCR automatically for large text-rich PDFs

33 +liteparse_ocr_auto_disable: true # disable OCR automatically for long PDFs

34 liteparse_ocr_auto_disable_pages: 30 # OCR-on runtime climbs sharply around this page count

34 -liteparse_ocr_auto_min_chars_per_page: 80

35 liteparse_ocr_auto_sample_pages: 5

36 pdf_ocr_fallback: true # enable legacy Tesseract fallback after PyMuPDF

37 thread_offload: true # offload sync parsers to thread pool

plugins/_document_query/helpers/document_query.py

+55 -6

@@ -40,6 +40,14 @@ def _positive_int(value: Any, default: int) -> int:

40 return parsed if parsed > 0 else default

43 +def _nonnegative_int(value: Any, default: int) -> int:

44 + try:

45 + parsed = int(value)

46 + except (TypeError, ValueError):

47 + return default

48 + return parsed if parsed >= 0 else default

49 +

50 +

51 def _parser_semaphore(config: dict) -> asyncio.Semaphore:

52 concurrency = _positive_int(

53 config.get("parser_concurrency"),

@@ -66,6 +74,7 @@ class DocumentQueryStore:

75 DEFAULT_CHUNK_SIZE = 1000

76 DEFAULT_CHUNK_OVERLAP = 100

77 + DEFAULT_MAX_INDEX_CHUNKS = 1200

79 @staticmethod

80 def get(agent: Agent):

@@ -103,12 +112,7 @@ class DocumentQueryStore:

112 doc_metadata = metadata or {}

113 doc_metadata["document_uri"] = document_uri

114 doc_metadata["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

106 - chunk_size = self.config.get("chunk_size", self.DEFAULT_CHUNK_SIZE)

107 - chunk_overlap = self.config.get("chunk_overlap", self.DEFAULT_CHUNK_OVERLAP)

108 - text_splitter = RecursiveCharacterTextSplitter(

109 - chunk_size=chunk_size, chunk_overlap=chunk_overlap

110 - )

111 - chunks = text_splitter.split_text(text)

115 + chunks = self._split_text_for_index(text)

116 docs = []

117 for i, chunk in enumerate(chunks):

118 chunk_metadata = doc_metadata.copy()

@@ -129,6 +133,51 @@ class DocumentQueryStore:

133 PrintStyle.error(f"Error adding document '{document_uri}': {err_text}")

134 return False, []

135

136 + def _split_text_for_index(self, text: str) -> list[str]:

137 + chunk_size = _positive_int(

138 + self.config.get("chunk_size"),

139 + self.DEFAULT_CHUNK_SIZE,

140 + )

141 + chunk_overlap = min(

142 + _nonnegative_int(

143 + self.config.get("chunk_overlap"),

144 + self.DEFAULT_CHUNK_OVERLAP,

145 + ),

146 + max(0, chunk_size - 1),

147 + )

148 + chunks = self._split_text(text, chunk_size, chunk_overlap)

149 +

150 + max_chunks = _nonnegative_int(

151 + self.config.get("max_index_chunks"),

152 + self.DEFAULT_MAX_INDEX_CHUNKS,

153 + )

154 + if not max_chunks or len(chunks) <= max_chunks:

155 + return chunks

156 +

157 + overlap_ratio = chunk_overlap / chunk_size if chunk_size else 0

158 + overlap_ratio = max(0, min(overlap_ratio, 0.5))

159 + target_size = max(

160 + chunk_size + 1,

161 + int(len(text) / max(1, max_chunks * (1 - overlap_ratio))) + 1,

162 + )

163 +

164 + for _ in range(8):

165 + target_overlap = min(int(target_size * overlap_ratio), target_size - 1)

166 + chunks = self._split_text(text, target_size, target_overlap)

167 + if len(chunks) <= max_chunks or target_size >= len(text):

168 + return chunks

169 + target_size = min(len(text), int(target_size * 1.25) + 1)

170 +

171 + return chunks

172 +

173 + @staticmethod

174 + def _split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:

175 + text_splitter = RecursiveCharacterTextSplitter(

176 + chunk_size=chunk_size,

177 + chunk_overlap=chunk_overlap,

178 + )

179 + return text_splitter.split_text(text)

180 +

181 async def get_document(self, document_uri: str) -> Optional[Document]:

182 if not self.vector_db:

183 return None

plugins/_document_query/helpers/parsers/liteparse.py

+1 -6

@@ -14,7 +14,6 @@ from .base import BaseParser

16 DEFAULT_OCR_AUTO_DISABLE_PAGES = 30

17 -DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE = 80

17 DEFAULT_OCR_AUTO_SAMPLE_PAGES = 5

@@ -182,11 +181,7 @@ class LiteParseParser(BaseParser):

181 if effective_pages < auto_disable_pages:

182 return False

183

185 - min_chars_per_page = _positive_int(

186 - config.get("liteparse_ocr_auto_min_chars_per_page"),

187 - DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE,

188 - )

189 - return (profile.text_chars / profile.sampled_pages) >= min_chars_per_page

184 + return True

185

186

187 def _pdf_text_profile(file_path: str, config: dict) -> _PdfTextProfile | None:

plugins/_document_query/hooks.py

+15 -5

@@ -15,7 +15,7 @@ from helpers.print_style import PrintStyle

15 _LOCK = threading.Lock()

16 _CHECKED = False

17 _PLUGIN_DIR = Path(__file__).resolve().parent

18 -_REQUIREMENTS_FILE = _PLUGIN_DIR / "requirements.txt"

18 +_ROOT_REQUIREMENTS_FILE = _PLUGIN_DIR.parents[1] / "requirements.txt"

21 def has_liteparse() -> bool:

@@ -66,9 +66,10 @@ def _install_requirements() -> None:

66 raise RuntimeError(

67 "Document Query plugin requires 'uv' to install liteparse automatically"

68 )

69 - if not _REQUIREMENTS_FILE.is_file():

69 + requirement = _liteparse_requirement()

70 + if not requirement:

71 raise RuntimeError(

71 - f"Document Query requirements file not found: {_REQUIREMENTS_FILE}"

72 + f"Document Query LiteParse requirement not found in {_ROOT_REQUIREMENTS_FILE}"

73 )

75 cmd = [

@@ -77,9 +78,18 @@ def _install_requirements() -> None:

78 "install",

79 "--python",

80 sys.executable,

80 - "-r",

81 - str(_REQUIREMENTS_FILE),

81 + requirement,

82 ]

84 PrintStyle.info("Document Query: liteparse not found, installing plugin dependency")

85 subprocess.check_call(cmd, cwd=str(_PLUGIN_DIR))

86 +

87 +

88 +def _liteparse_requirement() -> str:

89 + if not _ROOT_REQUIREMENTS_FILE.is_file():

90 + return ""

91 + for line in _ROOT_REQUIREMENTS_FILE.read_text(encoding="utf-8").splitlines():

92 + requirement = line.strip()

93 + if requirement.startswith("liteparse"):

94 + return requirement

95 + return ""

plugins/_document_query/webui/config.html

+16

@@ -15,6 +15,7 @@

15 context_intro_chunks: 2,

16 chunk_size: 1000,

17 chunk_overlap: 100,

18 + max_index_chunks: 1200,

19 search_threshold: 0.5,

20 search_limit: 100,

21 max_remote_bytes: 52428800,

@@ -40,6 +41,7 @@

41 this.ensureInt('context_intro_chunks', 2, 0);

42 this.ensureInt('chunk_size', 1000, 100);

43 this.ensureInt('chunk_overlap', 100, 0);

44 + this.ensureInt('max_index_chunks', 1200, 0);

45 this.ensureInt('search_limit', 100, 1);

46 this.ensureInt('max_remote_bytes', 52428800, 1);

47 this.ensureNumber('search_threshold', 0.5, 0, 1);

@@ -172,6 +174,20 @@

174 </div>

175 </div>

176

177 + <div class="field">

178 + <div class="field-label">

179 + <div class="field-title">Max index chunks</div>

180 + <div class="field-description">

181 + Adapt chunk size when a parsed document would exceed this many indexed chunks. Use 0 for no cap.

182 + </div>

183 + </div>

184 + <div class="field-control">

185 + <input type="number" min="0" step="50"

186 + @change="ensureInt('max_index_chunks', 1200, 0)"

187 + x-model.number="config.max_index_chunks" />

188 + </div>

189 + </div>

190 +

191 <div class="field">

192 <div class="field-label">

193 <div class="field-title">Search limit</div>

tests/test_document_query_plugin.py

+40 -6

@@ -6,8 +6,12 @@ from pathlib import Path

6 import pytest

7 from PIL import Image

9 +from plugins._document_query import hooks as document_query_hooks

10 from plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_resource

10 -from plugins._document_query.helpers.document_query import DocumentQueryHelper

11 +from plugins._document_query.helpers.document_query import (

12 + DocumentQueryHelper,

13 + DocumentQueryStore,

14 +)

15 from plugins._document_query.helpers.parsers.base import BaseParser

16 from plugins._document_query.helpers.parsers import get_parsers_for_mimetype

17 from plugins._document_query.helpers.parsers import liteparse as liteparse_module

@@ -105,12 +109,14 @@ def test_compatibility_imports_point_to_plugin_classes():

109

110 def test_liteparse_is_installed_by_docker_and_plugin_hook_requirements():

111 root_requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8")

108 - plugin_requirements = (

109 - ROOT / "plugins" / "_document_query" / "requirements.txt"

112 + hooks_source = (

113 + ROOT / "plugins" / "_document_query" / "hooks.py"

114 ).read_text(encoding="utf-8")

115

116 assert "liteparse==2.0.3" in root_requirements

113 - assert plugin_requirements.strip().splitlines() == ["liteparse==2.0.3"]

117 + assert "_ROOT_REQUIREMENTS_FILE" in hooks_source

118 + assert document_query_hooks._liteparse_requirement() == "liteparse==2.0.3"

119 + assert not (ROOT / "plugins" / "_document_query" / "requirements.txt").exists()

120

121

122 def test_default_config_bounds_liteparse_runtime_concurrency():

@@ -120,6 +126,7 @@ def test_default_config_bounds_liteparse_runtime_concurrency():

126

127 assert "parser_concurrency: 1" in default_config

128 assert "context_intro_chunks: 2" in default_config

129 + assert "max_index_chunks: 1200" in default_config

130 assert "liteparse_num_workers: 2" in default_config

131 assert "liteparse_ocr_auto_disable_pages: 30" in default_config

132 assert "liteparse_subprocess" not in default_config

@@ -137,6 +144,7 @@ def test_config_panel_exposes_document_query_settings():

144 "gather_timeout",

145 "chunk_size",

146 "chunk_overlap",

147 + "max_index_chunks",

148 "search_threshold",

149 "search_limit",

150 "context_intro_chunks",

@@ -162,6 +170,32 @@ def test_config_panel_exposes_document_query_settings():

170 assert "liteparse_subprocess" not in config_html

171

172

173 +def test_document_query_adapts_chunk_size_for_large_documents():

174 + store = object.__new__(DocumentQueryStore)

175 + store.config = {

176 + "chunk_size": 100,

177 + "chunk_overlap": 10,

178 + "max_index_chunks": 10,

179 + }

180 +

181 + chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())

182 +

183 + assert 1 < len(chunks) <= 10

184 +

185 +

186 +def test_document_query_allows_uncapped_index_chunks():

187 + store = object.__new__(DocumentQueryStore)

188 + store.config = {

189 + "chunk_size": 100,

190 + "chunk_overlap": 10,

191 + "max_index_chunks": 0,

192 + }

193 +

194 + chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())

195 +

196 + assert len(chunks) > 10

197 +

198 +

199 def test_document_query_thumbnail_matches_plugin_hub_limits():

200 thumbnail = ROOT / "plugins" / "_document_query" / "webui" / "thumbnail.jpg"

201

@@ -251,7 +285,7 @@ def test_liteparse_keeps_ocr_for_small_pdf(monkeypatch):

285 assert kwargs["ocr_enabled"] is True

286

287

254 -def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):

288 +def test_liteparse_disables_ocr_for_large_text_sparse_pdf(monkeypatch):

289 parser = LiteParseParser()

290 fetched = FetchedDocument(

291 uri="/tmp/scan.pdf",

@@ -273,7 +307,7 @@ def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):

307

308 kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/scan.pdf")

309

276 - assert kwargs["ocr_enabled"] is True

310 + assert kwargs["ocr_enabled"] is False

311

312

313 def test_liteparse_respects_explicit_ocr_disabled(monkeypatch):