feat: report indexing progress with page counts in status panel

This commit is contained in:
omp committed 2026-11-01 19:32:33 -03:00
1 parent 8f79993646
commit b3f8019e01
7 files changed
+156 -13

No files matched your search

+68 -13
View File
@@ -38,6 +38,9 @@ class IndexStatus:
model: str = ""
last_indexed_at: str | None = None
errors: list[str] = field(default_factory=list)
pages_processed: int = 0
pages_total: int = 0
current_file: str | None = None
@dataclass
@@ -78,6 +81,10 @@ class Indexer:
self._lock = threading.Lock()
self._last_indexed_at: str | None = None
self._errors: list[str] = []
self._indexing = False
self._pages_processed = 0
self._pages_total = 0
self._current_file: str | None = None
self._load_manifest()
# ------------------------------------------------------------------ #
@@ -154,6 +161,21 @@ class Indexer:
# Indexação
# ------------------------------------------------------------------ #
def mark_indexing(self) -> None:
"""Sinaliza que uma indexação está em andamento."""
self._indexing = True
self._pages_processed = 0
self._pages_total = 0
self._current_file = None
def report_progress(
self, pdf_name: str, pages_done: int, pages_total: int
) -> None:
"""Registra o progresso da extração de um arquivo."""
self._current_file = pdf_name
self._pages_processed = pages_done
self._pages_total = pages_total
def discover_pdfs(self) -> list[Path]:
"""Lista os PDFs do diretório configurado, ordenados por nome."""
directory = self.settings.pdf_dir
@@ -165,20 +187,25 @@ class Indexer:
"""Indexa os PDFs novos ou alterados."""
with self._lock:
self._errors = []
self.mark_indexing()
pdfs = self.discover_pdfs()
for pdf in pdfs:
try:
self._index_file(pdf, force=force)
except Exception as exc: # pragma: no cover - falha inesperada
logger.exception("Falha ao indexar %s", pdf.name)
self._manifest[pdf.name] = _ManifestEntry(
sha256="", state=STATE_ERROR, indexed_at=_now()
)
self._errors.append(f"{pdf.name}: {exc}")
try:
for pdf in pdfs:
try:
self._index_file(pdf, force=force)
except Exception as exc: # pragma: no cover - falha inesperada
logger.exception("Falha ao indexar %s", pdf.name)
self._manifest[pdf.name] = _ManifestEntry(
sha256="", state=STATE_ERROR, indexed_at=_now()
)
self._errors.append(f"{pdf.name}: {exc}")
self._save_manifest()
self._last_indexed_at = _now()
finally:
self._indexing = False
self._save_manifest()
self._last_indexed_at = _now()
return self.status()
def _index_file(self, pdf: Path, force: bool) -> None:
@@ -190,7 +217,15 @@ class Indexer:
return
logger.info("Indexando %s", pdf.name)
pages = list(self.extractor.extract_document(pdf, file_hash=file_hash))
pages = []
page_count = _pdf_page_count(pdf)
for position, page in enumerate(
self.extractor.extract_document(pdf, file_hash=file_hash), start=1
):
pages.append(page)
if position % 10 == 0 or position == page_count:
self.report_progress(pdf.name, position, page_count)
failed_pages = [page.page for page in pages if page.failed]
book_starts = self._book_starts(pages)
@@ -229,16 +264,36 @@ class Indexer:
for entry in self._manifest.values()
if entry.state in (STATE_COMPLETE, STATE_PARTIAL)
)
if self._indexing:
state = "indexing"
elif self._errors and indexed == 0:
state = "error"
else:
state = "ready"
return IndexStatus(
state="error" if self._errors and indexed == 0 else "ready",
state=state,
pdfs_total=len(self.discover_pdfs()),
pdfs_indexed=indexed,
chunks=self.store.count(),
model=self.settings.embedding_model,
last_indexed_at=self._last_indexed_at,
errors=list(self._errors),
pages_processed=self._pages_processed,
pages_total=self._pages_total,
current_file=self._current_file,
)
def _pdf_page_count(pdf: Path) -> int:
"""Número de páginas do PDF, ou 0 se não for possível determinar."""
try:
import pymupdf
with pymupdf.open(pdf) as document:
return len(document)
except Exception: # pragma: no cover - PDF malformado
return 0
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
+3
View File
@@ -182,6 +182,9 @@ def create_app(
model=current.model,
last_indexed_at=current.last_indexed_at,
errors=current.errors,
pages_processed=current.pages_processed,
pages_total=current.pages_total,
current_file=current.current_file,
)
@app.post("/api/reindex", response_model=ReindexResponse, status_code=202)
+3
View File
@@ -38,6 +38,9 @@ class StatusResponse(BaseModel):
model: str = ""
last_indexed_at: str | None = None
errors: list[str] = Field(default_factory=list)
pages_processed: int = 0
pages_total: int = 0
current_file: str | None = None
class HealthResponse(BaseModel):