pastor-online/backend/tests/test_indexer.py

"""Testes do indexador: descoberta de corpus, manifesto incremental e falhas.

O indexador lê arquivos USFX de `settings.corpus_dir`. O que precisa valer:
incremento por SHA-256 (não reprocessar o que não mudou), substituição dos
chunks do mesmo arquivo em vez de duplicação, falha de um arquivo registrada sem
derrubar os demais, e progresso em versículos durante a indexação.
"""

from __future__ import annotations

import json
import sys
import threading
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.config import Settings  # noqa: E402
from app.corpus import CorpusParseError  # noqa: E402
from app.embedding import HashEmbedder  # noqa: E402
from app.indexer import Indexer, IndexStatus  # noqa: E402
from app.vector_store import InMemoryVectorStore  # noqa: E402

USFX_HEADER = (
    "<?xml version='1.0' encoding='UTF-8'?>\n"
    "<usfx xmlns:xsi='http://eBible.org/usfx.xsd'>\n"
)


def usfx(*books: str) -> str:
    return USFX_HEADER + "".join(books) + "</usfx>\n"


def book(book_id: str, name: str, *chapters: str) -> str:
    return (
        f"<book id='{book_id}'><h>{name}</h>"
        + "".join(chapters)
        + "</book>"
    )


def chapter(number: int, verses: dict[int, str]) -> str:
    parts = [f"<c id='{number}'/>"]
    for verse, text in verses.items():
        parts.append(f"<v id='{verse}'/>{text}   <ve/>")
    return "".join(parts)


GENESIS = book("GEN", "Gênesis", chapter(1, {1: "No princípio criou Deus.", 2: "E a terra era vazia."}))
JOHN = book("JHN", "João", chapter(3, {16: "Porque Deus amou o mundo."}))


@pytest.fixture
def settings(tmp_path) -> Settings:
    return Settings(
        corpus_dir=tmp_path / "corpus",
        vector_db_dir=tmp_path / "vector_db",
        embedding_model="hash-test",
        embedding_batch_size=2,
        index_batch_size=3,
        auto_index=False,
    )


def build_indexer(settings, embedder=None, store=None):
    embedder = embedder or HashEmbedder(dimensions=32)
    store = store if store is not None else InMemoryVectorStore()
    indexer = Indexer(settings=settings, embedder=embedder, store=store)
    return indexer, store


def write_corpus(directory: Path, name: str, content: str | None = None) -> Path:
    directory.mkdir(parents=True, exist_ok=True)
    path = directory / name
    path.write_text(content if content is not None else usfx(GENESIS), encoding="utf-8")
    return path


# --------------------------------------------------------------------------
# Descoberta
# --------------------------------------------------------------------------


def test_discover_sources_lists_xml_files_sorted(settings):
    write_corpus(settings.corpus_dir, "b-almeida.usfx.xml")
    write_corpus(settings.corpus_dir, "a-almeida.usfx.xml")

    indexer, _ = build_indexer(settings)

    assert [path.name for path in indexer.discover_sources()] == [
        "a-almeida.usfx.xml",
        "b-almeida.usfx.xml",
    ]


def test_non_corpus_files_are_ignored(settings):
    write_corpus(settings.corpus_dir, "biblia.usfx.xml")
    write_corpus(settings.corpus_dir, "notas.txt", "texto qualquer")
    write_corpus(settings.corpus_dir, "biblia.pdf", "PDF antigo")

    indexer, _ = build_indexer(settings)

    assert [path.name for path in indexer.discover_sources()] == ["biblia.usfx.xml"]


def test_missing_corpus_directory_yields_no_sources(settings):
    indexer, _ = build_indexer(settings)

    assert indexer.discover_sources() == []


def test_empty_corpus_directory_is_ready_with_zero_chunks(settings):
    settings.corpus_dir.mkdir(parents=True, exist_ok=True)
    indexer, _ = build_indexer(settings)

    status = indexer.reindex()

    assert status.state == "ready"
    assert status.chunks == 0
    assert status.sources_total == 0


# --------------------------------------------------------------------------
# Indexação
# --------------------------------------------------------------------------


def test_reindex_indexes_verses_and_reports_counts(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS, JOHN))
    indexer, store = build_indexer(settings)

    status = indexer.reindex()

    assert status.state == "ready"
    assert status.sources_total == 1
    assert status.sources_indexed == 1
    assert status.chunks == 3
    assert store.count() == 3


def test_chunks_carry_book_chapter_and_verse(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(JOHN))
    indexer, store = build_indexer(settings)
    indexer.reindex()

    results = store.query([0.1] * 32, n_results=5, min_similarity=-1.0)

    assert results[0].book == "João"
    assert results[0].chapter == 3
    assert results[0].verse == 16
    assert results[0].reference == "João 3:16"
    assert results[0].file == "almeida.usfx.xml"


def test_placeholder_verses_do_not_become_chunks(settings):
    """Versículo só com travessão é ruído e não pode ocupar slot no índice."""
    content = usfx(
        book("MAT", "Mateus", chapter(17, {20: "Versículo real.", 21: "—", 22: "Outro."}))
    )
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", content)
    indexer, store = build_indexer(settings)

    indexer.reindex()

    assert store.count() == 2


def test_reindex_is_incremental_when_hash_is_unchanged(settings, monkeypatch):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    indexer.reindex()

    calls: list[str] = []
    import app.indexer as indexer_module

    real_parse = indexer_module.parse_usfx

    def tracked_parse(path, file_name=None):
        calls.append(Path(path).name)
        return real_parse(path, file_name)

    monkeypatch.setattr(indexer_module, "parse_usfx", tracked_parse)
    indexer.reindex()

    assert calls == [], "corpus inalterado não deveria ser reprocessado"


def test_reindex_reprocesses_when_content_changes(settings, monkeypatch):
    path = write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, store = build_indexer(settings)
    indexer.reindex()

    calls: list[str] = []
    import app.indexer as indexer_module

    real_parse = indexer_module.parse_usfx

    def tracked_parse(p, file_name=None):
        calls.append(Path(p).name)
        return real_parse(p, file_name)

    monkeypatch.setattr(indexer_module, "parse_usfx", tracked_parse)

    path.write_text(usfx(GENESIS, JOHN), encoding="utf-8")
    indexer.reindex()

    assert calls == ["almeida.usfx.xml"]
    assert store.count() == 3, "chunks antigos devem ser substituídos, não somados"


def test_force_reprocesses_everything(settings, monkeypatch):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    indexer.reindex()

    calls: list[str] = []
    import app.indexer as indexer_module

    real_parse = indexer_module.parse_usfx

    def tracked_parse(path, file_name=None):
        calls.append(Path(path).name)
        return real_parse(path, file_name)

    monkeypatch.setattr(indexer_module, "parse_usfx", tracked_parse)
    indexer.reindex(force=True)

    assert calls == ["almeida.usfx.xml"]


def test_reindex_replaces_old_chunks_of_the_same_file(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS, JOHN))
    indexer, store = build_indexer(settings)
    indexer.reindex()
    indexer.reindex(force=True)

    assert store.count() == 3, "não deve haver duplicação após reindexação"


def test_two_corpus_files_coexist_without_id_collision(settings):
    write_corpus(settings.corpus_dir, "a.usfx.xml", usfx(JOHN))
    write_corpus(settings.corpus_dir, "b.usfx.xml", usfx(JOHN))
    indexer, store = build_indexer(settings)

    status = indexer.reindex()

    assert status.sources_indexed == 2
    assert store.count() == 2, "mesmo versículo em arquivos distintos não colide"


# --------------------------------------------------------------------------
# Manifesto
# --------------------------------------------------------------------------


def test_manifest_is_persisted(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    indexer.reindex()

    manifest = json.loads(settings.manifest_path.read_text(encoding="utf-8"))

    assert manifest["almeida.usfx.xml"]["state"] == "complete"
    assert manifest["almeida.usfx.xml"]["chunks"] == 2
    assert len(manifest["almeida.usfx.xml"]["sha256"]) == 64


def test_manifest_survives_a_new_indexer_instance(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    indexer.reindex()

    restarted, _ = build_indexer(settings)

    assert restarted.status().sources_indexed == 1


# --------------------------------------------------------------------------
# Falhas
# --------------------------------------------------------------------------


def test_malformed_corpus_is_recorded_as_error(settings):
    settings.corpus_dir.mkdir(parents=True, exist_ok=True)
    (settings.corpus_dir / "quebrado.usfx.xml").write_text(
        "<usfx><book id='GEN'><h>Gênesis</h>", encoding="utf-8"
    )
    indexer, store = build_indexer(settings)

    status = indexer.reindex()

    assert status.sources_indexed == 0
    assert store.count() == 0
    assert status.errors
    assert "quebrado.usfx.xml" in status.errors[0]


def test_corpus_without_verses_is_recorded_as_error(settings):
    write_corpus(settings.corpus_dir, "vazio.usfx.xml", usfx())
    indexer, store = build_indexer(settings)

    status = indexer.reindex()

    assert status.sources_indexed == 0
    assert store.count() == 0
    assert status.errors


def test_one_bad_file_does_not_block_the_others(settings):
    settings.corpus_dir.mkdir(parents=True, exist_ok=True)
    (settings.corpus_dir / "a-bom.usfx.xml").write_text(usfx(GENESIS), encoding="utf-8")
    (settings.corpus_dir / "b-ruim.usfx.xml").write_text("<usfx><book", encoding="utf-8")
    indexer, store = build_indexer(settings)

    status = indexer.reindex()

    assert status.sources_indexed == 1
    assert store.count() == 2
    assert len(status.errors) == 1


def test_failed_file_is_retried_on_the_next_reindex(settings):
    path = write_corpus(settings.corpus_dir, "almeida.usfx.xml", "<usfx><book")
    indexer, store = build_indexer(settings)
    assert indexer.reindex().sources_indexed == 0

    path.write_text(usfx(GENESIS), encoding="utf-8")
    status = indexer.reindex()

    assert status.sources_indexed == 1
    assert store.count() == 2


def test_error_state_is_reported_when_nothing_was_indexed(settings):
    write_corpus(settings.corpus_dir, "vazio.usfx.xml", usfx())
    indexer, _ = build_indexer(settings)

    assert indexer.reindex().state == "error"


# --------------------------------------------------------------------------
# Progresso
# --------------------------------------------------------------------------


def test_status_reports_indexing_while_work_is_in_progress(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))

    class BlockingEmbedder(HashEmbedder):
        def encode(self, texts):
            captured["status"] = indexer.status()
            return super().encode(texts)

    captured: dict[str, IndexStatus] = {}
    indexer, _ = build_indexer(settings, embedder=BlockingEmbedder(dimensions=32))
    indexer.reindex()

    assert captured["status"].state == "indexing"
    assert captured["status"].sources_total == 1
    assert captured["status"].current_file == "almeida.usfx.xml"


def test_status_counts_verses_indexed_during_indexing(settings):
    content = usfx(
        book("GEN", "Gênesis", chapter(1, {i: f"Versículo {i}." for i in range(1, 11)}))
    )
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", content)

    seen: list[tuple[int, int]] = []

    class TrackingEmbedder(HashEmbedder):
        def encode(self, texts):
            result = super().encode(texts)
            seen.append(
                (indexer.status().verses_indexed, indexer.status().verses_total)
            )
            return result

    indexer, _ = build_indexer(settings, embedder=TrackingEmbedder(dimensions=32))
    status = indexer.reindex()

    assert status.verses_total == 10
    assert status.verses_indexed == 10
    assert seen[0][1] == 10, "o total precisa estar disponível desde o primeiro lote"
    assert len(seen) > 1, "o progresso precisa avançar em mais de um lote"
    assert [done for done, _ in seen] == sorted(done for done, _ in seen)
    assert seen[-1][0] > seen[0][0], "o contador deve avançar durante a indexação"


def test_verses_total_is_reset_between_runs(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    indexer.reindex()

    assert indexer.status().verses_indexed == 2

    indexer.mark_indexing()

    assert indexer.status().verses_indexed == 0
    assert indexer.status().verses_total == 0


def test_reindex_lock_blocks_concurrent_runs(settings):
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    indexer, _ = build_indexer(settings)
    started = threading.Event()
    release = threading.Event()

    class SlowEmbedder(HashEmbedder):
        def encode(self, texts):
            started.set()
            release.wait(timeout=5)
            return super().encode(texts)

    indexer.embedder = SlowEmbedder(dimensions=32)
    worker = threading.Thread(target=indexer.reindex, daemon=True)
    worker.start()
    assert started.wait(timeout=5)

    assert indexer.reindex_lock.locked()

    release.set()
    worker.join(timeout=5)
    assert not indexer.reindex_lock.locked()


def test_status_reports_the_embedder_actually_in_use(settings):
    """O nome reportado é o do embedder em uso, não o configurado no .env.

    Se o modelo configurado falhar ao carregar e houver fallback, anunciar o nome
    do .env esconderia que a busca está rodando sem semântica nenhuma.
    """
    write_corpus(settings.corpus_dir, "almeida.usfx.xml", usfx(GENESIS))
    settings.embedding_model = "modelo-que-nao-existe"
    indexer, _ = build_indexer(settings, embedder=HashEmbedder(dimensions=32))

    assert indexer.reindex().model == HashEmbedder.name