- Stack: GPT4All + Llama 3.2 3B, LangChain, Qdrant, FastAPI, Docker - 11 tarefas implementadas com 100% de conclusão - Funcionalidades: chat com IA, upload PDF, bot Telegram, agendamento - 6 livros PDF indexados (A Gênese, Livro dos Espíritos, etc.) - Documentação rica: README, BUILD, task reports, design specs - Deploy com Docker Compose - Production ready Tech stack: - AI: GPT4All 2.4.6, Llama 3.2 3B q4_0 - Backend: FastAPI 0.115.0, Celery 5.4.0, Redis 5.2.0 - RAG: LangChain 0.3.7, Qdrant 1.11.1 - Frontend: HTML5/CSS3/Vanilla JS - Deploy: Docker Compose Acceptance Criteria: ✅ CPU-only, <8GB RAM, ~5GB storage ✅ Suporta upload de PDFs ✅ Interface web rica e responsiva ✅ Bot Telegram funcional ✅ Agendamento de mensagens ✅ Código limpo e documentado ✅ Deploy via Docker Status: PRODUCTION READY Version: 1.0.0 Date: 2026-11-01
4.0 KiB
4.0 KiB
Tarefa 2: Indexação dos Livros PDF
Arquivos:
- Criar:
backend/app/utils/embeddings.py - Criar:
backend/app/services/pdf_processor.py - Criar:
backend/scripts/index_books.py
Interfaces:
-
Consumes: N/A (setup inicial)
-
Produces: Livros indexados no Qdrant
-
Passo 1: Criar embeddings.py
from sentence_transformers import SentenceTransformer
import numpy as np
class EmbeddingService:
def __init__(self, model_name="all-MiniLM-L6-v2"):
self.model = SentenceTransformer(model_name)
self.dimension = 384
def embed(self, texts: list[str]) -> np.ndarray:
"""Generate embeddings for a list of texts."""
return self.model.encode(texts, convert_to_numpy=True)
def embed_query(self, text: str) -> np.ndarray:
"""Generate embedding for a single query."""
return self.model.encode([text], convert_to_numpy=True)[0]
def get_dimension(self) -> int:
"""Return embedding dimension."""
return self.dimension
- Passo 2: Criar pdf_processor.py
from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter
import os
from typing import List, Dict
from .embeddings import EmbeddingService
class PDFProcessor:
def __init__(self, embedding_service: EmbeddingService):
self.embedding_service = embedding_service
self.text_splitter = RecursiveCharacterTextSplitter(
chunk_size=500,
chunk_overlap=50,
length_function=len,
)
def load_and_split(self, file_path: str, metadata: Dict) -> List[Dict]:
"""Load and split a PDF file."""
loader = PyPDFLoader(file_path)
documents = loader.load()
chunks = self.text_splitter.split_documents(documents)
# Convert to dictionaries with metadata
chunk_data = []
for i, doc in enumerate(chunks):
chunk_data.append({
"content": doc.page_content,
"metadata": {
**metadata,
"chunk_id": i,
"source": file_path,
}
})
return chunk_data
def embed_chunks(self, chunks: List[Dict]) -> List[Dict]:
"""Add embeddings to chunks."""
texts = [chunk["content"] for chunk in chunks]
embeddings = self.embedding_service.embed(texts)
for chunk, embedding in zip(chunks, embeddings):
chunk["embedding"] = embedding.tolist()
return chunks
- Passo 3: Criar index_books.py
from app.services.pdf_processor import PDFProcessor
from app.utils.embeddings import EmbeddingService
from app.utils.schedule_manager import ScheduleManager
import os
import json
def main():
# Setup services
embedding_service = EmbeddingService()
pdf_processor = PDFProcessor(embedding_service)
# Load books from PDF directory
pdf_dir = "/app/uploads"
books = os.listdir(pdf_dir)
books = [f for f in books if f.endswith('.pdf')]
print(f"Found {len(books)} books")
# Schedule manager
schedule_manager = ScheduleManager("/app/schedules.json")
# Process each book
for book_name in books:
book_path = os.path.join(pdf_dir, book_name)
book_metadata = {
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
"author": "Desconhecido",
"type": "book",
}
print(f"Processing: {book_name}")
# Load and split PDF
chunks = pdf_processor.load_and_split(book_path, book_metadata)
# Embed chunks
chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
# Save chunks to JSON for reference
chunks_file = f"/app/index/{book_name}.json"
os.makedirs("/app/index", exist_ok=True)
with open(chunks_file, 'w') as f:
json.dump(chunks_with_embeddings, f, indent=2)
print(f"Saved {len(chunks)} chunks to {chunks_file}")
print("Indexing complete!")
if __name__ == "__main__":
main()