diff --git a/backend/app/embedding.py b/backend/app/embedding.py new file mode 100644 index 0000000..c303501 --- /dev/null +++ b/backend/app/embedding.py @@ -0,0 +1,87 @@ +"""Geração de embeddings de texto. + +Em produção usa `sentence-transformers` com um modelo multilíngue executado em +CPU. `HashEmbedder` é um embedder determinístico e sem dependências, usado nos +testes e como fallback explícito. +""" + +from __future__ import annotations + +import hashlib +import logging +import math +from typing import Protocol, Sequence + +logger = logging.getLogger(__name__) + + +def normalize(vector: Sequence[float]) -> list[float]: + """Normaliza um vetor para comprimento 1. Vetor nulo vira vetor nulo.""" + magnitude = math.sqrt(sum(component * component for component in vector)) + if magnitude == 0: + return [0.0] * len(vector) + return [component / magnitude for component in vector] + + +class Embedder(Protocol): + """Interface de geração de embeddings.""" + + dimensions: int + + def encode(self, texts: Sequence[str]) -> list[list[float]]: + """Devolve um vetor normalizado por texto.""" + ... + + +class HashEmbedder: + """Embedder determinístico baseado em hash. + + Não captura semântica, mas serve para testes e para verificações de + integridade do pipeline (ids, contagens, filtros de metadados). + """ + + def __init__(self, dimensions: int = 384): + self.dimensions = dimensions + + def encode(self, texts: Sequence[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + vector = [0.0] * self.dimensions + tokens = _tokenize(text) + for token in tokens: + digest = hashlib.sha256(token.encode("utf-8")).digest() + index = int.from_bytes(digest[:4], "big") % self.dimensions + sign = 1.0 if digest[4] % 2 == 0 else -1.0 + vector[index] += sign + vectors.append(normalize(vector)) + return vectors + + +class SentenceTransformerEmbedder: + """Embedder de produção, baseado em sentence-transformers.""" + + def __init__(self, model_name: str, batch_size: int = 32, device: str = "cpu"): + from sentence_transformers import SentenceTransformer + + self.model_name = model_name + self.batch_size = batch_size + self._model = SentenceTransformer(model_name, device=device) + self.dimensions = int(self._model.get_sentence_embedding_dimension()) + + def encode(self, texts: Sequence[str]) -> list[list[float]]: + if not texts: + return [] + vectors = self._model.encode( + list(texts), + batch_size=self.batch_size, + convert_to_numpy=True, + normalize_embeddings=True, + show_progress_bar=False, + ) + return [vector.tolist() for vector in vectors] + + +def _tokenize(text: str) -> list[str]: + """Separa o texto em tokens simples, sem dependências externas.""" + cleaned = "".join(ch.lower() if ch.isalnum() else " " for ch in text) + return [token for token in cleaned.split() if token] diff --git a/backend/app/vector_store.py b/backend/app/vector_store.py new file mode 100644 index 0000000..65c080f --- /dev/null +++ b/backend/app/vector_store.py @@ -0,0 +1,245 @@ +"""Banco vetorial persistente (ChromaDB) e implementação em memória para testes.""" + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Sequence + +from .bible_parser import VerseChunk +from .embedding import Embedder, HashEmbedder + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class SearchResult: + """Trecho recuperado da busca vetorial.""" + + book: str + chapter: int + verse: int + page: int + file: str + text: str + distance: float = 0.0 + + @property + def reference(self) -> str: + return f"{self.book} {self.chapter}:{self.verse}" + + @property + def score(self) -> float: + """Similaridade em escala 0..1 (1 = idêntico).""" + return max(0.0, min(1.0, 1.0 - self.distance)) + + +def cosine_similarity(left: Sequence[float], right: Sequence[float]) -> float: + """Similaridade de cosseno entre dois vetores.""" + if not left or not right: + return 0.0 + size = min(len(left), len(right)) + dot = sum(left[i] * right[i] for i in range(size)) + left_norm = math.sqrt(sum(value * value for value in left[:size])) + right_norm = math.sqrt(sum(value * value for value in right[:size])) + if left_norm == 0 or right_norm == 0: + return 0.0 + return dot / (left_norm * right_norm) + + +class VectorStore: + """Fachada sobre o ChromaDB persistente.""" + + def __init__( + self, + persist_dir: Path | str, + collection_name: str = "biblia", + embedder: Embedder | None = None, + ): + import chromadb + from chromadb.config import Settings as ChromaSettings + + self.persist_dir = Path(persist_dir) + self.persist_dir.mkdir(parents=True, exist_ok=True) + self.collection_name = collection_name + self.embedder = embedder or HashEmbedder() + self._client = chromadb.PersistentClient( + path=str(self.persist_dir), + settings=ChromaSettings(anonymized_telemetry=False, allow_reset=True), + ) + self._collection = self._client.get_or_create_collection( + name=collection_name, + metadata={"hnsw:space": "cosine"}, + ) + + def upsert(self, chunks: Iterable[VerseChunk], vectors: Sequence[Sequence[float]]) -> None: + """Grava chunks e vetores, substituindo os de mesmo id.""" + chunk_list = list(chunks) + if not chunk_list: + return + if len(vectors) != len(chunk_list): + raise ValueError("vectors e chunks devem ter o mesmo tamanho") + + self._collection.upsert( + ids=[chunk.chunk_id for chunk in chunk_list], + embeddings=[list(vector) for vector in vectors], + documents=[chunk.text for chunk in chunk_list], + metadatas=[ + { + "book": chunk.book, + "chapter": chunk.chapter, + "verse": chunk.verse, + "page": chunk.page, + "file": chunk.file, + "reference": f"{chunk.book} {chunk.chapter}:{chunk.verse}", + } + for chunk in chunk_list + ], + ) + + def delete_file(self, file_name: str) -> None: + """Remove todos os chunks de um arquivo.""" + existing = self._collection.get(where={"file": file_name}) + ids = existing.get("ids") or [] + if ids: + self._collection.delete(ids=ids) + + def query( + self, vector: Sequence[float], n_results: int = 6, min_similarity: float = 0.35 + ) -> list[SearchResult]: + """Busca os chunks mais próximos, descartando os abaixo do limiar.""" + total = self._collection.count() + if total == 0: + return [] + + raw = self._collection.query( + query_embeddings=[list(vector)], + n_results=min(n_results, total), + include=["documents", "metadatas", "distances"], + ) + + documents = (raw.get("documents") or [[]])[0] + metadatas = (raw.get("metadatas") or [[]])[0] + distances = (raw.get("distances") or [[]])[0] + + results: list[SearchResult] = [] + for document, metadata, distance in zip(documents, metadatas, distances): + similarity = 1.0 - float(distance) + if similarity < min_similarity: + continue + results.append( + SearchResult( + book=str(metadata.get("book", "")), + chapter=int(metadata.get("chapter", 0)), + verse=int(metadata.get("verse", 0)), + page=int(metadata.get("page", 0)), + file=str(metadata.get("file", "")), + text=document or "", + distance=float(distance), + ) + ) + return results + + def get_by_reference( + self, book: str, chapter: int, verse: int + ) -> list[SearchResult]: + """Busca determinística por livro, capítulo e versículo.""" + raw = self._collection.get( + where={ + "$and": [ + {"book": book}, + {"chapter": int(chapter)}, + {"verse": int(verse)}, + ] + }, + include=["documents", "metadatas"], + ) + documents = raw.get("documents") or [] + metadatas = raw.get("metadatas") or [] + return [ + SearchResult( + book=str(metadata.get("book", book)), + chapter=int(metadata.get("chapter", chapter)), + verse=int(metadata.get("verse", verse)), + page=int(metadata.get("page", 0)), + file=str(metadata.get("file", "")), + text=document or "", + distance=0.0, + ) + for document, metadata in zip(documents, metadatas) + ] + + def count(self) -> int: + """Total de chunks indexados.""" + return int(self._collection.count()) + + +class InMemoryVectorStore: + """Banco vetorial em memória, usado nos testes. + + Implementa a mesma interface de `VectorStore`, com busca por similaridade + de cosseno e filtros por metadados. + """ + + def __init__(self, embedder: Embedder | None = None): + self.embedder = embedder or HashEmbedder() + self._vectors: dict[str, list[float]] = {} + self._chunks: dict[str, VerseChunk] = {} + + def upsert(self, chunks: Iterable[VerseChunk], vectors: Sequence[Sequence[float]]) -> None: + chunk_list = list(chunks) + if len(vectors) != len(chunk_list): + raise ValueError("vectors e chunks devem ter o mesmo tamanho") + for chunk, vector in zip(chunk_list, vectors): + self._vectors[chunk.chunk_id] = list(vector) + self._chunks[chunk.chunk_id] = chunk + + def delete_file(self, file_name: str) -> None: + for chunk_id in [cid for cid, chunk in self._chunks.items() if chunk.file == file_name]: + self._vectors.pop(chunk_id, None) + self._chunks.pop(chunk_id, None) + + def query( + self, vector: Sequence[float], n_results: int = 6, min_similarity: float = 0.35 + ) -> list[SearchResult]: + scored: list[SearchResult] = [] + for chunk_id, stored in self._vectors.items(): + similarity = cosine_similarity(vector, stored) + if similarity < min_similarity: + continue + chunk = self._chunks[chunk_id] + scored.append( + SearchResult( + book=chunk.book, + chapter=chunk.chapter, + verse=chunk.verse, + page=chunk.page, + file=chunk.file, + text=chunk.text, + distance=1.0 - similarity, + ) + ) + scored.sort(key=lambda result: result.distance) + return scored[:n_results] + + def get_by_reference( + self, book: str, chapter: int, verse: int + ) -> list[SearchResult]: + return [ + SearchResult( + book=chunk.book, + chapter=chunk.chapter, + verse=chunk.verse, + page=chunk.page, + file=chunk.file, + text=chunk.text, + distance=0.0, + ) + for chunk in self._chunks.values() + if chunk.book == book and chunk.chapter == chapter and chunk.verse == verse + ] + + def count(self) -> int: + return len(self._chunks) diff --git a/backend/tests/test_vector_store.py b/backend/tests/test_vector_store.py new file mode 100644 index 0000000..f92d5c6 --- /dev/null +++ b/backend/tests/test_vector_store.py @@ -0,0 +1,157 @@ +"""Testes do banco vetorial e dos embeddings.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.bible_parser import VerseChunk # noqa: E402 +from app.embedding import HashEmbedder, normalize # noqa: E402 +from app.vector_store import InMemoryVectorStore, VectorStore # noqa: E402 + + +def make_chunk(verse: int, book: str = "Gênesis", chapter: int = 1, file: str = "biblia.pdf"): + return VerseChunk( + book=book, + chapter=chapter, + verse=verse, + page=6, + file=file, + text=f"Versículo {verse} de {book} {chapter}", + ) + + +@pytest.fixture +def embedder(): + return HashEmbedder(dimensions=32) + + +@pytest.fixture +def store(embedder): + return InMemoryVectorStore(embedder=embedder) + + +def test_normalize_produces_unit_vector(): + vector = normalize([3.0, 4.0]) + assert vector == [0.6, 0.8] + + +def test_normalize_of_zero_vector_is_safe(): + assert normalize([0.0, 0.0]) == [0.0, 0.0] + + +def test_hash_embedder_is_deterministic(): + embedder = HashEmbedder(dimensions=16) + assert embedder.encode(["olá mundo"]) == embedder.encode(["olá mundo"]) + + +def test_hash_embedder_produces_requested_dimensions(): + embedder = HashEmbedder(dimensions=7) + vector = embedder.encode(["texto"])[0] + assert len(vector) == 7 + + +def test_upsert_then_query_finds_the_matching_chunk(store, embedder): + chunks = [make_chunk(1), make_chunk(2), make_chunk(3)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + + store.upsert(chunks, vectors) + + query_vector = embedder.encode(["Versículo 2 de Gênesis 1"])[0] + results = store.query(query_vector, n_results=3, min_similarity=0.0) + + assert results + assert results[0].verse == 2 + assert results[0].book == "Gênesis" + + +def test_query_respects_min_similarity(store, embedder): + chunks = [make_chunk(1)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + store.upsert(chunks, vectors) + + query_vector = embedder.encode(["palavra completamente diferente xyz"])[0] + results = store.query(query_vector, n_results=5, min_similarity=0.99) + + assert results == [], "nada deveria passar com limiar 0.99" + + +def test_query_respects_n_results(store, embedder): + chunks = [make_chunk(i) for i in range(1, 11)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + store.upsert(chunks, vectors) + + query_vector = embedder.encode(["Versículo"])[0] + results = store.query(query_vector, n_results=3, min_similarity=0.0) + + assert len(results) == 3 + + +def test_delete_file_removes_only_that_file(store, embedder): + chunks = [make_chunk(1, file="a.pdf"), make_chunk(2, file="b.pdf")] + vectors = embedder.encode([chunk.text for chunk in chunks]) + store.upsert(chunks, vectors) + + store.delete_file("a.pdf") + + assert store.count() == 1 + query_vector = embedder.encode(["Versículo"])[0] + results = store.query(query_vector, n_results=10, min_similarity=0.0) + assert {result.file for result in results} == {"b.pdf"} + + +def test_get_by_reference_returns_exact_verse(store, embedder): + chunks = [make_chunk(1), make_chunk(2, chapter=2)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + store.upsert(chunks, vectors) + + results = store.get_by_reference("Gênesis", 2, 2) + + assert len(results) == 1 + assert results[0].chapter == 2 + assert results[0].verse == 2 + + +def test_get_by_reference_returns_empty_when_missing(store, embedder): + chunks = [make_chunk(1)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + store.upsert(chunks, vectors) + + assert store.get_by_reference("João", 3, 16) == [] + + +def test_count_reflects_upserts(store, embedder): + chunks = [make_chunk(1), make_chunk(2)] + vectors = embedder.encode([chunk.text for chunk in chunks]) + + store.upsert(chunks, vectors) + + assert store.count() == 2 + + +def test_upsert_is_idempotent(store, embedder): + chunk = make_chunk(1) + vector = embedder.encode([chunk.text])[0] + + store.upsert([chunk], [vector]) + store.upsert([chunk], [vector]) + + assert store.count() == 1 + + +def test_chroma_store_roundtrip(tmp_path, embedder): + """Garante que a implementação Chroma real tem a mesma interface.""" + store = VectorStore(persist_dir=tmp_path, embedder=embedder) + chunk = make_chunk(1) + vector = embedder.encode([chunk.text])[0] + + store.upsert([chunk], [vector]) + + assert store.count() == 1 + results = store.get_by_reference("Gênesis", 1, 1) + assert len(results) == 1 + assert results[0].text == chunk.text