lgcosta
b9c0c66d21
- Stack: GPT4All + Llama 3.2 3B, LangChain, Qdrant, FastAPI, Docker - 11 tarefas implementadas com 100% de conclusão - Funcionalidades: chat com IA, upload PDF, bot Telegram, agendamento - 6 livros PDF indexados (A Gênese, Livro dos Espíritos, etc.) - Documentação rica: README, BUILD, task reports, design specs - Deploy com Docker Compose - Production ready Tech stack: - AI: GPT4All 2.4.6, Llama 3.2 3B q4_0 - Backend: FastAPI 0.115.0, Celery 5.4.0, Redis 5.2.0 - RAG: LangChain 0.3.7, Qdrant 1.11.1 - Frontend: HTML5/CSS3/Vanilla JS - Deploy: Docker Compose Acceptance Criteria: ✅ CPU-only, <8GB RAM, ~5GB storage ✅ Suporta upload de PDFs ✅ Interface web rica e responsiva ✅ Bot Telegram funcional ✅ Agendamento de mensagens ✅ Código limpo e documentado ✅ Deploy via Docker Status: PRODUCTION READY Version: 1.0.0 Date: 2026-11-01
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
from langchain_community.document_loaders import PyPDFLoader
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
import os
|
|
from typing import List, Dict
|
|
from .embeddings import EmbeddingService
|
|
|
|
class PDFProcessor:
|
|
def __init__(self, embedding_service: EmbeddingService):
|
|
self.embedding_service = embedding_service
|
|
self.text_splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=50,
|
|
length_function=len,
|
|
)
|
|
|
|
def load_and_split(self, file_path: str, metadata: Dict) -> List[Dict]:
|
|
"""Load and split a PDF file."""
|
|
loader = PyPDFLoader(file_path)
|
|
documents = loader.load()
|
|
|
|
chunks = self.text_splitter.split_documents(documents)
|
|
|
|
# Convert to dictionaries with metadata
|
|
chunk_data = []
|
|
for i, doc in enumerate(chunks):
|
|
chunk_data.append({
|
|
"content": doc.page_content,
|
|
"metadata": {
|
|
**metadata,
|
|
"chunk_id": i,
|
|
"source": file_path,
|
|
}
|
|
})
|
|
|
|
return chunk_data
|
|
|
|
def embed_chunks(self, chunks: List[Dict]) -> List[Dict]:
|
|
"""Add embeddings to chunks."""
|
|
texts = [chunk["content"] for chunk in chunks]
|
|
embeddings = self.embedding_service.embed(texts)
|
|
|
|
for chunk, embedding in zip(chunks, embeddings):
|
|
chunk["embedding"] = embedding.tolist()
|
|
|
|
return chunks
|