pastor-online/backend/tests/test_chat.py

"""Testes do serviço de chat: contexto, fidelidade e validação de citações."""

from __future__ import annotations

import asyncio
import sys
from pathlib import Path

import pytest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.corpus import VerseChunk  # noqa: E402
from app.chat import (  # noqa: E402
    NO_EVIDENCE_MESSAGE,
    ChatService,
    extract_reference,
    validate_citations,
)
from app.config import Settings  # noqa: E402
from app.embedding import HashEmbedder  # noqa: E402
from app.vector_store import InMemoryVectorStore  # noqa: E402


class RecordingProvider:
    """Provedor que registra as chamadas e devolve texto configurável."""

    def __init__(self, response: str = "Resposta com (Gênesis 1:1)."):
        self.response = response
        self.calls: list[list[dict]] = []
        self.systems: list[str] = []

    async def stream(self, messages, system):
        self.calls.append(list(messages))
        self.systems.append(system)
        for word in self.response.split(" "):
            yield word + " "

    async def aclose(self):
        return None


class ExplodingProvider:
    """Provedor que falha se for chamado."""

    async def stream(self, messages, system):
        raise AssertionError("o provedor não deveria ser chamado")
        yield ""  # pragma: no cover

    async def aclose(self):
        return None


def make_chunk(book="Gênesis", chapter=1, verse=1, text="No princípio criou Deus os céus e a terra."):
    return VerseChunk(
        book=book,
        chapter=chapter,
        verse=verse,
        file="biblia.usfx.xml",
        text=text,
    )


@pytest.fixture
def settings():
    return Settings(
        corpus_dir=Path("/tmp/corpus-test"),
        vector_db_dir=Path("/tmp/vector-test"),
        embedding_model="hash-test",
        top_k=6,
        min_similarity=0.0,
    )


@pytest.fixture
def store():
    return InMemoryVectorStore(embedder=HashEmbedder(dimensions=64))


def build_service(settings, store, provider):
    embedder = HashEmbedder(dimensions=64)
    return ChatService(
        settings=settings,
        store=store,
        embedder=embedder,
        provider=provider,
    )


def run(coro):
    return asyncio.run(coro)


# --------------------------------------------------------------------------- #
# Extração de referência
# --------------------------------------------------------------------------- #

def test_extract_reference_finds_book_chapter_verse():
    assert extract_reference("o que diz João 3:16?") == ("João", 3, 16)


def test_extract_reference_handles_ranges():
    assert extract_reference("me mostre Salmos 23:1-3") == ("Salmos", 23, 1)


def test_extract_reference_returns_none_without_reference():
    assert extract_reference("estou triste hoje") is None


def test_extract_reference_is_case_insensitive():
    assert extract_reference("GÊNESIS 1:1") == ("Gênesis", 1, 1)


# --------------------------------------------------------------------------- #
# Validação de citações
# --------------------------------------------------------------------------- #

def test_supported_citation_is_preserved():
    text = "Deus criou o mundo (Gênesis 1:1)."
    cleaned, citations = validate_citations(text, {"Gênesis 1:1"})

    assert cleaned == text
    assert citations == ["Gênesis 1:1"]


def test_unsupported_citation_is_removed():
    text = "Isso está escrito em João 3:16 e em Gênesis 1:1."
    cleaned, citations = validate_citations(text, {"Gênesis 1:1"})

    assert "João 3:16" not in cleaned
    assert "Gênesis 1:1" in cleaned
    assert citations == ["Gênesis 1:1"]


def test_citation_outside_the_source_list_is_removed():
    text = "Veja Apocalipse 22:21."
    cleaned, citations = validate_citations(text, {"Gênesis 1:1"})

    assert "Apocalipse 22:21" not in cleaned
    assert citations == []


def test_validation_reports_no_citation_when_there_is_none():
    cleaned, citations = validate_citations("Uma reflexão sem citação.", {"Gênesis 1:1"})

    assert cleaned == "Uma reflexão sem citação."
    assert citations == []


# --------------------------------------------------------------------------- #
# Resposta sem evidência
# --------------------------------------------------------------------------- #

def test_without_evidence_returns_fixed_message_without_calling_llm(settings, store):
    provider = ExplodingProvider()
    service = build_service(settings, store, provider)

    result = run(service.answer("Qual a melhor investment?", []))

    assert result.text == NO_EVIDENCE_MESSAGE
    assert result.sources == []
    assert result.citations == []


# --------------------------------------------------------------------------- #
# Resposta com evidência
# --------------------------------------------------------------------------- #

def test_answer_uses_the_provider_when_evidence_exists(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider("A Palavra nos diz (Gênesis 1:1).")
    service = build_service(settings, store, provider)

    result = run(service.answer("Quem criou os céus e a terra?", []))

    assert "Gênesis 1:1" in result.text
    assert len(provider.calls) == 1
    assert result.sources
    assert result.sources[0].reference == "Gênesis 1:1"


def test_context_contains_the_retrieved_versicles(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider()
    service = build_service(settings, store, provider)

    run(service.answer("Quem criou?", []))

    sent = provider.calls[0]
    assert sent[0]["role"] == "user"
    assert "No princípio criou Deus os céus e a terra" in sent[0]["content"]
    assert "Gênesis 1:1" in sent[0]["content"]


def test_context_marks_sources_as_data_not_instructions(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider()
    service = build_service(settings, store, provider)

    run(service.answer("Quem criou?", []))

    content = provider.calls[0][0]["content"]
    assert "[FONTE 1]" in content
    assert "evidência" in content.lower() or "dado" in content.lower()


def test_system_prompt_is_passed_to_the_provider(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider()
    service = build_service(settings, store, provider)

    run(service.answer("Quem criou?", []))

    system = provider.systems[0]
    assert "Pastor Online" in system
    assert "única fonte de autoridade" in system


def test_unsupported_citation_from_the_model_is_removed(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider("Veja também Efésios 2:8 e Gênesis 1:1.")
    service = build_service(settings, store, provider)

    result = run(service.answer("Fé e obras?", []))

    assert "Efésios 2:8" not in result.text
    assert "Gênesis 1:1" in result.text
    assert result.citations == ["Gênesis 1:1"]


def test_explicit_reference_question_uses_the_exact_verse(settings, store):
    """`João 3:16` deve buscar o versículo pedido, não o mais similar."""
    genesis = make_chunk(text="No princípio criou Deus os céus e a terra.")
    john = make_chunk(
        book="João", chapter=3, verse=16, text="Porque Deus amou o mundo de tal maneira..."
    )
    embedder = HashEmbedder(dimensions=64)
    store.upsert([genesis, john], embedder.encode([genesis.text, john.text]))

    provider = RecordingProvider("Está escrito em João 3:16.")
    service = build_service(settings, store, provider)

    result = run(service.answer("o que diz João 3:16?", []))

    assert [source.reference for source in result.sources] == ["João 3:16"]
    assert "Porque Deus amou o mundo" in provider.calls[0][0]["content"]


def test_history_is_included_in_the_prompt(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider()
    service = build_service(settings, store, provider)

    history = [
        {"role": "user", "content": "Quem criou o mundo?"},
        {"role": "assistant", "content": "No princípio (Gênesis 1:1)."},
    ]
    run(service.answer("E o que isso significa?", history))

    sent = provider.calls[0]
    assert len(sent) >= 3
    assert any("Quem criou o mundo?" in message["content"] for message in sent)


def test_history_is_limited(settings, store):
    chunk = make_chunk()
    embedder = HashEmbedder(dimensions=64)
    store.upsert([chunk], embedder.encode([chunk.text]))

    provider = RecordingProvider()
    service = build_service(settings, store, provider)

    history = [
        {"role": "user", "content": f"mensagem {index}"} for index in range(30)
    ]
    run(service.answer("pergunta", history))

    sent = provider.calls[0]
    assert len(sent) <= settings.max_history_turns + 1