feat(backend): add embeddings and ChromaDB vector store

backend/app/embedding.py

@@ -0,0 +1,87 @@
+"""Geração de embeddings de texto.
+
+Em produção usa `sentence-transformers` com um modelo multilíngue executado em
+CPU. `HashEmbedder` é um embedder determinístico e sem dependências, usado nos
+testes e como fallback explícito.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+from typing import Protocol, Sequence
+
+logger = logging.getLogger(__name__)
+
+
+def normalize(vector: Sequence[float]) -> list[float]:
+    """Normaliza um vetor para comprimento 1. Vetor nulo vira vetor nulo."""
+    magnitude = math.sqrt(sum(component * component for component in vector))
+    if magnitude == 0:
+        return [0.0] * len(vector)
+    return [component / magnitude for component in vector]
+
+
+class Embedder(Protocol):
+    """Interface de geração de embeddings."""
+
+    dimensions: int
+
+    def encode(self, texts: Sequence[str]) -> list[list[float]]:
+        """Devolve um vetor normalizado por texto."""
+        ...
+
+
+class HashEmbedder:
+    """Embedder determinístico baseado em hash.
+
+    Não captura semântica, mas serve para testes e para verificações de
+    integridade do pipeline (ids, contagens, filtros de metadados).
+    """
+
+    def __init__(self, dimensions: int = 384):
+        self.dimensions = dimensions
+
+    def encode(self, texts: Sequence[str]) -> list[list[float]]:
+        vectors: list[list[float]] = []
+        for text in texts:
+            vector = [0.0] * self.dimensions
+            tokens = _tokenize(text)
+            for token in tokens:
+                digest = hashlib.sha256(token.encode("utf-8")).digest()
+                index = int.from_bytes(digest[:4], "big") % self.dimensions
+                sign = 1.0 if digest[4] % 2 == 0 else -1.0
+                vector[index] += sign
+            vectors.append(normalize(vector))
+        return vectors
+
+
+class SentenceTransformerEmbedder:
+    """Embedder de produção, baseado em sentence-transformers."""
+
+    def __init__(self, model_name: str, batch_size: int = 32, device: str = "cpu"):
+        from sentence_transformers import SentenceTransformer
+
+        self.model_name = model_name
+        self.batch_size = batch_size
+        self._model = SentenceTransformer(model_name, device=device)
+        self.dimensions = int(self._model.get_sentence_embedding_dimension())
+
+    def encode(self, texts: Sequence[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        vectors = self._model.encode(
+            list(texts),
+            batch_size=self.batch_size,
+            convert_to_numpy=True,
+            normalize_embeddings=True,
+            show_progress_bar=False,
+        )
+        return [vector.tolist() for vector in vectors]
+
+
+def _tokenize(text: str) -> list[str]:
+    """Separa o texto em tokens simples, sem dependências externas."""
+    cleaned = "".join(ch.lower() if ch.isalnum() else " " for ch in text)
+    return [token for token in cleaned.split() if token]

backend/app/vector_store.py

@@ -0,0 +1,245 @@
+"""Banco vetorial persistente (ChromaDB) e implementação em memória para testes."""
+
+from __future__ import annotations
+
+import logging
+import math
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable, Sequence
+
+from .bible_parser import VerseChunk
+from .embedding import Embedder, HashEmbedder
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    """Trecho recuperado da busca vetorial."""
+
+    book: str
+    chapter: int
+    verse: int
+    page: int
+    file: str
+    text: str
+    distance: float = 0.0
+
+    @property
+    def reference(self) -> str:
+        return f"{self.book} {self.chapter}:{self.verse}"
+
+    @property
+    def score(self) -> float:
+        """Similaridade em escala 0..1 (1 = idêntico)."""
+        return max(0.0, min(1.0, 1.0 - self.distance))
+
+
+def cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float:
+    """Similaridade de cosseno entre dois vetores."""
+    if not left or not right:
+        return 0.0
+    size = min(len(left), len(right))
+    dot = sum(left[i] * right[i] for i in range(size))
+    left_norm = math.sqrt(sum(value * value for value in left[:size]))
+    right_norm = math.sqrt(sum(value * value for value in right[:size]))
+    if left_norm == 0 or right_norm == 0:
+        return 0.0
+    return dot / (left_norm * right_norm)
+
+
+class VectorStore:
+    """Fachada sobre o ChromaDB persistente."""
+
+    def __init__(
+        self,
+        persist_dir: Path | str,
+        collection_name: str = "biblia",
+        embedder: Embedder | None = None,
+    ):
+        import chromadb
+        from chromadb.config import Settings as ChromaSettings
+
+        self.persist_dir = Path(persist_dir)
+        self.persist_dir.mkdir(parents=True, exist_ok=True)
+        self.collection_name = collection_name
+        self.embedder = embedder or HashEmbedder()
+        self._client = chromadb.PersistentClient(
+            path=str(self.persist_dir),
+            settings=ChromaSettings(anonymized_telemetry=False, allow_reset=True),
+        )
+        self._collection = self._client.get_or_create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+
+    def upsert(self, chunks: Iterable[VerseChunk], vectors: Sequence[Sequence[float]]) -> None:
+        """Grava chunks e vetores, substituindo os de mesmo id."""
+        chunk_list = list(chunks)
+        if not chunk_list:
+            return
+        if len(vectors) != len(chunk_list):
+            raise ValueError("vectors e chunks devem ter o mesmo tamanho")
+
+        self._collection.upsert(
+            ids=[chunk.chunk_id for chunk in chunk_list],
+            embeddings=[list(vector) for vector in vectors],
+            documents=[chunk.text for chunk in chunk_list],
+            metadatas=[
+                {
+                    "book": chunk.book,
+                    "chapter": chunk.chapter,
+                    "verse": chunk.verse,
+                    "page": chunk.page,
+                    "file": chunk.file,
+                    "reference": f"{chunk.book} {chunk.chapter}:{chunk.verse}",
+                }
+                for chunk in chunk_list
+            ],
+        )
+
+    def delete_file(self, file_name: str) -> None:
+        """Remove todos os chunks de um arquivo."""
+        existing = self._collection.get(where={"file": file_name})
+        ids = existing.get("ids") or []
+        if ids:
+            self._collection.delete(ids=ids)
+
+    def query(
+        self, vector: Sequence[float], n_results: int = 6, min_similarity: float = 0.35
+    ) -> list[SearchResult]:
+        """Busca os chunks mais próximos, descartando os abaixo do limiar."""
+        total = self._collection.count()
+        if total == 0:
+            return []
+
+        raw = self._collection.query(
+            query_embeddings=[list(vector)],
+            n_results=min(n_results, total),
+            include=["documents", "metadatas", "distances"],
+        )
+
+        documents = (raw.get("documents") or [[]])[0]
+        metadatas = (raw.get("metadatas") or [[]])[0]
+        distances = (raw.get("distances") or [[]])[0]
+
+        results: list[SearchResult] = []
+        for document, metadata, distance in zip(documents, metadatas, distances):
+            similarity = 1.0 - float(distance)
+            if similarity < min_similarity:
+                continue
+            results.append(
+                SearchResult(
+                    book=str(metadata.get("book", "")),
+                    chapter=int(metadata.get("chapter", 0)),
+                    verse=int(metadata.get("verse", 0)),
+                    page=int(metadata.get("page", 0)),
+                    file=str(metadata.get("file", "")),
+                    text=document or "",
+                    distance=float(distance),
+                )
+            )
+        return results
+
+    def get_by_reference(
+        self, book: str, chapter: int, verse: int
+    ) -> list[SearchResult]:
+        """Busca determinística por livro, capítulo e versículo."""
+        raw = self._collection.get(
+            where={
+                "$and": [
+                    {"book": book},
+                    {"chapter": int(chapter)},
+                    {"verse": int(verse)},
+                ]
+            },
+            include=["documents", "metadatas"],
+        )
+        documents = raw.get("documents") or []
+        metadatas = raw.get("metadatas") or []
+        return [
+            SearchResult(
+                book=str(metadata.get("book", book)),
+                chapter=int(metadata.get("chapter", chapter)),
+                verse=int(metadata.get("verse", verse)),
+                page=int(metadata.get("page", 0)),
+                file=str(metadata.get("file", "")),
+                text=document or "",
+                distance=0.0,
+            )
+            for document, metadata in zip(documents, metadatas)
+        ]
+
+    def count(self) -> int:
+        """Total de chunks indexados."""
+        return int(self._collection.count())
+
+
+class InMemoryVectorStore:
+    """Banco vetorial em memória, usado nos testes.
+
+    Implementa a mesma interface de `VectorStore`, com busca por similaridade
+    de cosseno e filtros por metadados.
+    """
+
+    def __init__(self, embedder: Embedder | None = None):
+        self.embedder = embedder or HashEmbedder()
+        self._vectors: dict[str, list[float]] = {}
+        self._chunks: dict[str, VerseChunk] = {}
+
+    def upsert(self, chunks: Iterable[VerseChunk], vectors: Sequence[Sequence[float]]) -> None:
+        chunk_list = list(chunks)
+        if len(vectors) != len(chunk_list):
+            raise ValueError("vectors e chunks devem ter o mesmo tamanho")
+        for chunk, vector in zip(chunk_list, vectors):
+            self._vectors[chunk.chunk_id] = list(vector)
+            self._chunks[chunk.chunk_id] = chunk
+
+    def delete_file(self, file_name: str) -> None:
+        for chunk_id in [cid for cid, chunk in self._chunks.items() if chunk.file == file_name]:
+            self._vectors.pop(chunk_id, None)
+            self._chunks.pop(chunk_id, None)
+
+    def query(
+        self, vector: Sequence[float], n_results: int = 6, min_similarity: float = 0.35
+    ) -> list[SearchResult]:
+        scored: list[SearchResult] = []
+        for chunk_id, stored in self._vectors.items():
+            similarity = cosine_similarity(vector, stored)
+            if similarity < min_similarity:
+                continue
+            chunk = self._chunks[chunk_id]
+            scored.append(
+                SearchResult(
+                    book=chunk.book,
+                    chapter=chunk.chapter,
+                    verse=chunk.verse,
+                    page=chunk.page,
+                    file=chunk.file,
+                    text=chunk.text,
+                    distance=1.0 - similarity,
+                )
+            )
+        scored.sort(key=lambda result: result.distance)
+        return scored[:n_results]
+
+    def get_by_reference(
+        self, book: str, chapter: int, verse: int
+    ) -> list[SearchResult]:
+        return [
+            SearchResult(
+                book=chunk.book,
+                chapter=chunk.chapter,
+                verse=chunk.verse,
+                page=chunk.page,
+                file=chunk.file,
+                text=chunk.text,
+                distance=0.0,
+            )
+            for chunk in self._chunks.values()
+            if chunk.book == book and chunk.chapter == chapter and chunk.verse == verse
+        ]
+
+    def count(self) -> int:
+        return len(self._chunks)

backend/tests/test_vector_store.py

@@ -0,0 +1,157 @@
+"""Testes do banco vetorial e dos embeddings."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from app.bible_parser import VerseChunk  # noqa: E402
+from app.embedding import HashEmbedder, normalize  # noqa: E402
+from app.vector_store import InMemoryVectorStore, VectorStore  # noqa: E402
+
+
+def make_chunk(verse: int, book: str = "Gênesis", chapter: int = 1, file: str = "biblia.pdf"):
+    return VerseChunk(
+        book=book,
+        chapter=chapter,
+        verse=verse,
+        page=6,
+        file=file,
+        text=f"Versículo {verse} de {book} {chapter}",
+    )
+
+
+@pytest.fixture
+def embedder():
+    return HashEmbedder(dimensions=32)
+
+
+@pytest.fixture
+def store(embedder):
+    return InMemoryVectorStore(embedder=embedder)
+
+
+def test_normalize_produces_unit_vector():
+    vector = normalize([3.0, 4.0])
+    assert vector == [0.6, 0.8]
+
+
+def test_normalize_of_zero_vector_is_safe():
+    assert normalize([0.0, 0.0]) == [0.0, 0.0]
+
+
+def test_hash_embedder_is_deterministic():
+    embedder = HashEmbedder(dimensions=16)
+    assert embedder.encode(["olá mundo"]) == embedder.encode(["olá mundo"])
+
+
+def test_hash_embedder_produces_requested_dimensions():
+    embedder = HashEmbedder(dimensions=7)
+    vector = embedder.encode(["texto"])[0]
+    assert len(vector) == 7
+
+
+def test_upsert_then_query_finds_the_matching_chunk(store, embedder):
+    chunks = [make_chunk(1), make_chunk(2), make_chunk(3)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+
+    store.upsert(chunks, vectors)
+
+    query_vector = embedder.encode(["Versículo 2 de Gênesis 1"])[0]
+    results = store.query(query_vector, n_results=3, min_similarity=0.0)
+
+    assert results
+    assert results[0].verse == 2
+    assert results[0].book == "Gênesis"
+
+
+def test_query_respects_min_similarity(store, embedder):
+    chunks = [make_chunk(1)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+    store.upsert(chunks, vectors)
+
+    query_vector = embedder.encode(["palavra completamente diferente xyz"])[0]
+    results = store.query(query_vector, n_results=5, min_similarity=0.99)
+
+    assert results == [], "nada deveria passar com limiar 0.99"
+
+
+def test_query_respects_n_results(store, embedder):
+    chunks = [make_chunk(i) for i in range(1, 11)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+    store.upsert(chunks, vectors)
+
+    query_vector = embedder.encode(["Versículo"])[0]
+    results = store.query(query_vector, n_results=3, min_similarity=0.0)
+
+    assert len(results) == 3
+
+
+def test_delete_file_removes_only_that_file(store, embedder):
+    chunks = [make_chunk(1, file="a.pdf"), make_chunk(2, file="b.pdf")]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+    store.upsert(chunks, vectors)
+
+    store.delete_file("a.pdf")
+
+    assert store.count() == 1
+    query_vector = embedder.encode(["Versículo"])[0]
+    results = store.query(query_vector, n_results=10, min_similarity=0.0)
+    assert {result.file for result in results} == {"b.pdf"}
+
+
+def test_get_by_reference_returns_exact_verse(store, embedder):
+    chunks = [make_chunk(1), make_chunk(2, chapter=2)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+    store.upsert(chunks, vectors)
+
+    results = store.get_by_reference("Gênesis", 2, 2)
+
+    assert len(results) == 1
+    assert results[0].chapter == 2
+    assert results[0].verse == 2
+
+
+def test_get_by_reference_returns_empty_when_missing(store, embedder):
+    chunks = [make_chunk(1)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+    store.upsert(chunks, vectors)
+
+    assert store.get_by_reference("João", 3, 16) == []
+
+
+def test_count_reflects_upserts(store, embedder):
+    chunks = [make_chunk(1), make_chunk(2)]
+    vectors = embedder.encode([chunk.text for chunk in chunks])
+
+    store.upsert(chunks, vectors)
+
+    assert store.count() == 2
+
+
+def test_upsert_is_idempotent(store, embedder):
+    chunk = make_chunk(1)
+    vector = embedder.encode([chunk.text])[0]
+
+    store.upsert([chunk], [vector])
+    store.upsert([chunk], [vector])
+
+    assert store.count() == 1
+
+
+def test_chroma_store_roundtrip(tmp_path, embedder):
+    """Garante que a implementação Chroma real tem a mesma interface."""
+    store = VectorStore(persist_dir=tmp_path, embedder=embedder)
+    chunk = make_chunk(1)
+    vector = embedder.encode([chunk.text])[0]
+
+    store.upsert([chunk], [vector])
+
+    assert store.count() == 1
+    results = store.get_by_reference("Gênesis", 1, 1)
+    assert len(results) == 1
+    assert results[0].text == chunk.text