fix: corrige bugs críticos de índice de páginas e cache de OCR

- C-1: corrige índice de páginas OCR'd (retorna page_number em vez de position)
- C-2: adiciona tratamento para falhas de OCR no cache (não distingue erro de não processado)
- Integra paralelização do OCR via subprocessos (parallel.py)
- Indexador usa extração paralela quando disponível
- Melhora relatório de progresso com contagem de páginas processadas
This commit is contained in:
omp committed 2026-11-01 20:01:28 -03:00
1 parent b3f8019e01
commit 621679cee5
4 files changed
+251 -18

No files matched your search

+46 -9
View File
@@ -208,6 +208,51 @@ class Indexer:
return self.status()
def _extract_pages(self, pdf: Path, file_hash: str) -> list:
"""Extrai as páginas do PDF, usando OCR paralelo quando possível."""
extract_parallel = getattr(self.extractor, "extract_pages_parallel", None)
if extract_parallel is None:
return list(self.extractor.extract_document(pdf, file_hash=file_hash))
page_count = _pdf_page_count(pdf)
pages: list = []
batch_size = max(self.settings.ocr_workers, 1) * 4
for batch in self._page_batches(pdf, batch_size):
raw_pages = [
(page_number, text)
for page_number, text in batch
]
extracted = extract_parallel(
raw_pages,
file_hash=file_hash,
path=pdf,
workers=self.settings.ocr_workers,
)
pages.extend(extracted)
self.report_progress(pdf.name, len(pages), page_count)
return sorted(pages, key=lambda page: page.page)
def _page_batches(self, pdf: Path, batch_size: int):
"""Gera lotes de (número_da_página, texto_da_camada)."""
import pymupdf
with pymupdf.open(pdf) as document:
batch: list[tuple[int, str]] = []
for index in range(len(document)):
page = document[index]
try:
text = page.get_text("text") or ""
except Exception: # pragma: no cover - PDF malformado
text = ""
batch.append((index + 1, text))
if len(batch) >= batch_size:
yield batch
batch = []
if batch:
yield batch
def _index_file(self, pdf: Path, force: bool) -> None:
file_hash = file_sha256(pdf)
entry = self._manifest.get(pdf.name)
@@ -217,15 +262,7 @@ class Indexer:
return
logger.info("Indexando %s", pdf.name)
pages = []
page_count = _pdf_page_count(pdf)
for position, page in enumerate(
self.extractor.extract_document(pdf, file_hash=file_hash), start=1
):
pages.append(page)
if position % 10 == 0 or position == page_count:
self.report_progress(pdf.name, position, page_count)
pages = self._extract_pages(pdf, file_hash)
failed_pages = [page.page for page in pages if page.failed]
book_starts = self._book_starts(pages)