"""Indexador: orquestra leitura do corpus, embeddings e persistência vetorial. O manifesto (`manifest.json`) registra, por arquivo, o SHA-256 do conteúdo, o estado (`complete` ou `error`) e a contagem de chunks. Arquivos cujo hash não mudou não são reprocessados, então reiniciar o container não repete o trabalho de embedding. """ from __future__ import annotations import hashlib import json import logging import threading from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from .config import Settings from .corpus import CorpusParseError, VerseChunk, parse_usfx from .embedding import Embedder from .vector_store import VectorStore logger = logging.getLogger(__name__) STATE_COMPLETE = "complete" STATE_ERROR = "error" CORPUS_SUFFIXES = (".xml",) def file_sha256(path: Path, chunk_size: int = 1024 * 1024) -> str: """SHA-256 do conteúdo do arquivo, lido em blocos.""" digest = hashlib.sha256() with Path(path).open("rb") as handle: while chunk := handle.read(chunk_size): digest.update(chunk) return digest.hexdigest() @dataclass class IndexStatus: """Estado público da indexação.""" state: str = "ready" sources_total: int = 0 sources_indexed: int = 0 chunks: int = 0 model: str = "" last_indexed_at: str | None = None errors: list[str] = field(default_factory=list) verses_indexed: int = 0 verses_total: int = 0 current_file: str | None = None @dataclass class _ManifestEntry: sha256: str state: str chunks: int = 0 indexed_at: str = "" def to_dict(self) -> dict: return { "sha256": self.sha256, "state": self.state, "chunks": self.chunks, "indexed_at": self.indexed_at, } class Indexer: """Coordena a indexação dos corpora presentes em `settings.corpus_dir`.""" def __init__( self, settings: Settings, embedder: Embedder, store: VectorStore, ): self.settings = settings self.embedder = embedder self.store = store self._manifest: dict[str, _ManifestEntry] = {} self._lock = threading.Lock() self._last_indexed_at: str | None = None self._errors: list[str] = [] self._indexing = False self._verses_indexed = 0 self._verses_total = 0 self._current_file: str | None = None self._load_manifest() @property def reindex_lock(self) -> threading.Lock: """Lock que serializa reindexações concorrentes.""" return self._lock # ------------------------------------------------------------------ # # Manifesto # ------------------------------------------------------------------ # def _load_manifest(self) -> None: path = self.settings.manifest_path if not path.exists(): return try: raw = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: logger.warning("Manifesto ilegível em %s: %s", path, exc) return for name, entry in raw.items(): self._manifest[name] = _ManifestEntry( sha256=entry.get("sha256", ""), state=entry.get("state", STATE_ERROR), chunks=int(entry.get("chunks", 0)), indexed_at=entry.get("indexed_at", ""), ) def _save_manifest(self) -> None: path = self.settings.manifest_path path.parent.mkdir(parents=True, exist_ok=True) payload = {name: entry.to_dict() for name, entry in self._manifest.items()} temporary = path.with_suffix(".json.tmp") temporary.write_text( json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" ) temporary.replace(path) # ------------------------------------------------------------------ # # Progresso # ------------------------------------------------------------------ # def mark_indexing(self) -> None: """Sinaliza que uma indexação está em andamento, zerando contadores.""" self._indexing = True self._verses_indexed = 0 self._verses_total = 0 self._current_file = None def report_progress(self, file_name: str, verses_done: int, verses_total: int) -> None: """Registra o progresso da indexação de um arquivo.""" self._current_file = file_name self._verses_indexed = verses_done self._verses_total = verses_total # ------------------------------------------------------------------ # # Indexação # ------------------------------------------------------------------ # def discover_sources(self) -> list[Path]: """Lista os arquivos de corpus do diretório configurado, por nome.""" directory = self.settings.corpus_dir if not directory.exists(): return [] return sorted( path for suffix in CORPUS_SUFFIXES for path in directory.glob(f"*{suffix}") if path.is_file() ) def reindex(self, force: bool = False) -> IndexStatus: """Indexa os corpora novos ou alterados.""" with self._lock: self._errors = [] self.mark_indexing() sources = self.discover_sources() try: for source in sources: try: self._index_file(source, force=force) except Exception as exc: # pragma: no cover - falha inesperada logger.exception("Falha ao indexar %s", source.name) self._manifest[source.name] = _ManifestEntry( sha256="", state=STATE_ERROR, indexed_at=_now() ) self._errors.append(f"{source.name}: {exc}") self._save_manifest() self._last_indexed_at = _now() finally: self._indexing = False return self.status() def _index_file(self, source: Path, force: bool) -> None: file_hash = file_sha256(source) entry = self._manifest.get(source.name) if entry is not None and entry.sha256 == file_hash and entry.state == STATE_COMPLETE and not force: logger.info("%s já está indexado; pulando", source.name) return logger.info("Indexando %s", source.name) self.report_progress(source.name, 0, 0) try: chunks = list(parse_usfx(source, file_name=source.name)) except CorpusParseError as exc: self._manifest[source.name] = _ManifestEntry( sha256=file_hash, state=STATE_ERROR, indexed_at=_now() ) self._errors.append(str(exc)) return if not chunks: self._manifest[source.name] = _ManifestEntry( sha256=file_hash, state=STATE_ERROR, indexed_at=_now() ) self._errors.append(f"{source.name}: nenhum versículo pôde ser lido") return # Remove os chunks anteriores deste arquivo antes de gravar os novos. self.store.delete_file(source.name) total = len(chunks) # O total é publicado antes do primeiro lote: a barra de progresso do # frontend precisa do denominador assim que a leitura termina, e não só # depois do primeiro embedding (que é a etapa mais lenta). self.report_progress(source.name, 0, total) batch_size = max(1, self.settings.index_batch_size) for start in range(0, total, batch_size): batch = chunks[start : start + batch_size] vectors = self.embedder.encode([chunk.text for chunk in batch]) self.store.upsert(batch, vectors) self.report_progress(source.name, min(start + batch_size, total), total) self._manifest[source.name] = _ManifestEntry( sha256=file_hash, state=STATE_COMPLETE, chunks=total, indexed_at=_now(), ) def status(self) -> IndexStatus: """Estado atual da indexação.""" indexed = sum( 1 for entry in self._manifest.values() if entry.state == STATE_COMPLETE ) if self._indexing: state = "indexing" elif self._errors and indexed == 0: state = "error" else: state = "ready" return IndexStatus( state=state, sources_total=len(self.discover_sources()), sources_indexed=indexed, chunks=self.store.count(), # Reporta o embedder *em uso*, não o configurado: se o modelo falhou # ao carregar e houve fallback, o operador precisa ver isso aqui. model=getattr(self.embedder, "name", self.settings.embedding_model), last_indexed_at=self._last_indexed_at, errors=list(self._errors), verses_indexed=self._verses_indexed, verses_total=self._verses_total, current_file=self._current_file, ) def _now() -> str: return datetime.now(timezone.utc).isoformat()