diff --git a/backend/app/indexer.py b/backend/app/indexer.py index 2a970ec..e1c58c8 100644 --- a/backend/app/indexer.py +++ b/backend/app/indexer.py @@ -38,6 +38,9 @@ class IndexStatus: model: str = "" last_indexed_at: str | None = None errors: list[str] = field(default_factory=list) + pages_processed: int = 0 + pages_total: int = 0 + current_file: str | None = None @dataclass @@ -78,6 +81,10 @@ class Indexer: self._lock = threading.Lock() self._last_indexed_at: str | None = None self._errors: list[str] = [] + self._indexing = False + self._pages_processed = 0 + self._pages_total = 0 + self._current_file: str | None = None self._load_manifest() # ------------------------------------------------------------------ # @@ -154,6 +161,21 @@ class Indexer: # Indexação # ------------------------------------------------------------------ # + def mark_indexing(self) -> None: + """Sinaliza que uma indexação está em andamento.""" + self._indexing = True + self._pages_processed = 0 + self._pages_total = 0 + self._current_file = None + + def report_progress( + self, pdf_name: str, pages_done: int, pages_total: int + ) -> None: + """Registra o progresso da extração de um arquivo.""" + self._current_file = pdf_name + self._pages_processed = pages_done + self._pages_total = pages_total + def discover_pdfs(self) -> list[Path]: """Lista os PDFs do diretório configurado, ordenados por nome.""" directory = self.settings.pdf_dir @@ -165,20 +187,25 @@ class Indexer: """Indexa os PDFs novos ou alterados.""" with self._lock: self._errors = [] + self.mark_indexing() pdfs = self.discover_pdfs() - for pdf in pdfs: - try: - self._index_file(pdf, force=force) - except Exception as exc: # pragma: no cover - falha inesperada - logger.exception("Falha ao indexar %s", pdf.name) - self._manifest[pdf.name] = _ManifestEntry( - sha256="", state=STATE_ERROR, indexed_at=_now() - ) - self._errors.append(f"{pdf.name}: {exc}") + try: + for pdf in pdfs: + try: + self._index_file(pdf, force=force) + except Exception as exc: # pragma: no cover - falha inesperada + logger.exception("Falha ao indexar %s", pdf.name) + self._manifest[pdf.name] = _ManifestEntry( + sha256="", state=STATE_ERROR, indexed_at=_now() + ) + self._errors.append(f"{pdf.name}: {exc}") + + self._save_manifest() + self._last_indexed_at = _now() + finally: + self._indexing = False - self._save_manifest() - self._last_indexed_at = _now() return self.status() def _index_file(self, pdf: Path, force: bool) -> None: @@ -190,7 +217,15 @@ class Indexer: return logger.info("Indexando %s", pdf.name) - pages = list(self.extractor.extract_document(pdf, file_hash=file_hash)) + pages = [] + page_count = _pdf_page_count(pdf) + for position, page in enumerate( + self.extractor.extract_document(pdf, file_hash=file_hash), start=1 + ): + pages.append(page) + if position % 10 == 0 or position == page_count: + self.report_progress(pdf.name, position, page_count) + failed_pages = [page.page for page in pages if page.failed] book_starts = self._book_starts(pages) @@ -229,16 +264,36 @@ class Indexer: for entry in self._manifest.values() if entry.state in (STATE_COMPLETE, STATE_PARTIAL) ) + if self._indexing: + state = "indexing" + elif self._errors and indexed == 0: + state = "error" + else: + state = "ready" return IndexStatus( - state="error" if self._errors and indexed == 0 else "ready", + state=state, pdfs_total=len(self.discover_pdfs()), pdfs_indexed=indexed, chunks=self.store.count(), model=self.settings.embedding_model, last_indexed_at=self._last_indexed_at, errors=list(self._errors), + pages_processed=self._pages_processed, + pages_total=self._pages_total, + current_file=self._current_file, ) +def _pdf_page_count(pdf: Path) -> int: + """Número de páginas do PDF, ou 0 se não for possível determinar.""" + try: + import pymupdf + + with pymupdf.open(pdf) as document: + return len(document) + except Exception: # pragma: no cover - PDF malformado + return 0 + + def _now() -> str: return datetime.now(timezone.utc).isoformat() diff --git a/backend/app/main.py b/backend/app/main.py index 3a13191..38856dd 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -182,6 +182,9 @@ def create_app( model=current.model, last_indexed_at=current.last_indexed_at, errors=current.errors, + pages_processed=current.pages_processed, + pages_total=current.pages_total, + current_file=current.current_file, ) @app.post("/api/reindex", response_model=ReindexResponse, status_code=202) diff --git a/backend/app/models.py b/backend/app/models.py index 9024d33..754341d 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -38,6 +38,9 @@ class StatusResponse(BaseModel): model: str = "" last_indexed_at: str | None = None errors: list[str] = Field(default_factory=list) + pages_processed: int = 0 + pages_total: int = 0 + current_file: str | None = None class HealthResponse(BaseModel): diff --git a/backend/tests/test_indexer.py b/backend/tests/test_indexer.py index cfc32cb..2b95d0b 100644 --- a/backend/tests/test_indexer.py +++ b/backend/tests/test_indexer.py @@ -207,3 +207,30 @@ def test_non_pdf_files_are_ignored(settings): assert status.pdfs_total == 0 assert extractor.calls == [] + + +def test_status_reports_indexing_while_work_is_in_progress(settings): + """Durante a indexação o status deve mostrar progresso, não zero.""" + write_pdf(settings.pdf_dir, "biblia.pdf") + extractor = FakeExtractor({"biblia.pdf": [PageText(page=6, text=GENESIS_PAGE)]}) + indexer, _ = build_indexer(settings, extractor) + + indexer.mark_indexing() + status = indexer.status() + + assert status.state == "indexing" + assert status.pdfs_total == 1 + + +def test_status_counts_pages_processed_during_indexing(settings): + write_pdf(settings.pdf_dir, "biblia.pdf") + extractor = FakeExtractor({"biblia.pdf": [PageText(page=6, text=GENESIS_PAGE)]}) + indexer, _ = build_indexer(settings, extractor) + + indexer.mark_indexing() + indexer.report_progress(pdf_name="biblia.pdf", pages_done=120, pages_total=3364) + + status = indexer.status() + + assert status.pages_processed == 120 + assert status.pages_total == 3364 diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 9bbd156..4a0be19 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -19,6 +19,9 @@ export interface Status { model: string; last_indexed_at: string | null; errors: string[]; + pages_processed: number; + pages_total: number; + current_file: string | null; } export interface ChatMessage { diff --git a/frontend/src/components/StatusPanel.tsx b/frontend/src/components/StatusPanel.tsx index 234793b..5e40b86 100644 --- a/frontend/src/components/StatusPanel.tsx +++ b/frontend/src/components/StatusPanel.tsx @@ -44,6 +44,33 @@ export function StatusPanel({ status, onReindex, reindexing }: StatusPanelProps) + {status?.state === "indexing" && ( +
+
+ {status.current_file ? `Lendo ${status.current_file}` : "Lendo PDFs"} + {status.pages_total > 0 && ` · ${status.pages_processed}/${status.pages_total} páginas`} +
+ {status.pages_total > 0 && ( +
+ +
+ )} +
+ )} + {status && status.errors.length > 0 && (