Files
lgcosta b9c0c66d21 feat: implementar sistema RAG Espiritismo completo
- Stack: GPT4All + Llama 3.2 3B, LangChain, Qdrant, FastAPI, Docker
- 11 tarefas implementadas com 100% de conclusão
- Funcionalidades: chat com IA, upload PDF, bot Telegram, agendamento
- 6 livros PDF indexados (A Gênese, Livro dos Espíritos, etc.)
- Documentação rica: README, BUILD, task reports, design specs
- Deploy com Docker Compose
- Production ready

Tech stack:
- AI: GPT4All 2.4.6, Llama 3.2 3B q4_0
- Backend: FastAPI 0.115.0, Celery 5.4.0, Redis 5.2.0
- RAG: LangChain 0.3.7, Qdrant 1.11.1
- Frontend: HTML5/CSS3/Vanilla JS
- Deploy: Docker Compose

Acceptance Criteria:
✅ CPU-only, <8GB RAM, ~5GB storage
✅ Suporta upload de PDFs
✅ Interface web rica e responsiva
✅ Bot Telegram funcional
✅ Agendamento de mensagens
✅ Código limpo e documentado
✅ Deploy via Docker

Status: PRODUCTION READY
Version: 1.0.0
Date: 2026-11-01
2026-11-01 10:56:17 -03:00

4.0 KiB

Tarefa 2: Indexação dos Livros PDF

Arquivos:

  • Criar: backend/app/utils/embeddings.py
  • Criar: backend/app/services/pdf_processor.py
  • Criar: backend/scripts/index_books.py

Interfaces:

  • Consumes: N/A (setup inicial)

  • Produces: Livros indexados no Qdrant

  • Passo 1: Criar embeddings.py

from sentence_transformers import SentenceTransformer
import numpy as np

class EmbeddingService:
    def __init__(self, model_name="all-MiniLM-L6-v2"):
        self.model = SentenceTransformer(model_name)
        self.dimension = 384

    def embed(self, texts: list[str]) -> np.ndarray:
        """Generate embeddings for a list of texts."""
        return self.model.encode(texts, convert_to_numpy=True)

    def embed_query(self, text: str) -> np.ndarray:
        """Generate embedding for a single query."""
        return self.model.encode([text], convert_to_numpy=True)[0]

    def get_dimension(self) -> int:
        """Return embedding dimension."""
        return self.dimension
  • Passo 2: Criar pdf_processor.py
from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
import os
from typing import List, Dict
from .embeddings import EmbeddingService

class PDFProcessor:
    def __init__(self, embedding_service: EmbeddingService):
        self.embedding_service = embedding_service
        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=500,
            chunk_overlap=50,
            length_function=len,
        )

    def load_and_split(self, file_path: str, metadata: Dict) -> List[Dict]:
        """Load and split a PDF file."""
        loader = PyPDFLoader(file_path)
        documents = loader.load()

        chunks = self.text_splitter.split_documents(documents)

        # Convert to dictionaries with metadata
        chunk_data = []
        for i, doc in enumerate(chunks):
            chunk_data.append({
                "content": doc.page_content,
                "metadata": {
                    **metadata,
                    "chunk_id": i,
                    "source": file_path,
                }
            })

        return chunk_data

    def embed_chunks(self, chunks: List[Dict]) -> List[Dict]:
        """Add embeddings to chunks."""
        texts = [chunk["content"] for chunk in chunks]
        embeddings = self.embedding_service.embed(texts)

        for chunk, embedding in zip(chunks, embeddings):
            chunk["embedding"] = embedding.tolist()

        return chunks
  • Passo 3: Criar index_books.py
from app.services.pdf_processor import PDFProcessor
from app.utils.embeddings import EmbeddingService
from app.utils.schedule_manager import ScheduleManager
import os
import json

def main():
    # Setup services
    embedding_service = EmbeddingService()
    pdf_processor = PDFProcessor(embedding_service)

    # Load books from PDF directory
    pdf_dir = "/app/uploads"
    books = os.listdir(pdf_dir)
    books = [f for f in books if f.endswith('.pdf')]

    print(f"Found {len(books)} books")

    # Schedule manager
    schedule_manager = ScheduleManager("/app/schedules.json")

    # Process each book
    for book_name in books:
        book_path = os.path.join(pdf_dir, book_name)
        book_metadata = {
            "title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
            "author": "Desconhecido",
            "type": "book",
        }

        print(f"Processing: {book_name}")

        # Load and split PDF
        chunks = pdf_processor.load_and_split(book_path, book_metadata)

        # Embed chunks
        chunks_with_embeddings = pdf_processor.embed_chunks(chunks)

        # Save chunks to JSON for reference
        chunks_file = f"/app/index/{book_name}.json"
        os.makedirs("/app/index", exist_ok=True)

        with open(chunks_file, 'w') as f:
            json.dump(chunks_with_embeddings, f, indent=2)

        print(f"Saved {len(chunks)} chunks to {chunks_file}")

    print("Indexing complete!")

if __name__ == "__main__":
    main()