feat: implementar sistema RAG Espiritismo completo
- Stack: GPT4All + Llama 3.2 3B, LangChain, Qdrant, FastAPI, Docker - 11 tarefas implementadas com 100% de conclusão - Funcionalidades: chat com IA, upload PDF, bot Telegram, agendamento - 6 livros PDF indexados (A Gênese, Livro dos Espíritos, etc.) - Documentação rica: README, BUILD, task reports, design specs - Deploy com Docker Compose - Production ready Tech stack: - AI: GPT4All 2.4.6, Llama 3.2 3B q4_0 - Backend: FastAPI 0.115.0, Celery 5.4.0, Redis 5.2.0 - RAG: LangChain 0.3.7, Qdrant 1.11.1 - Frontend: HTML5/CSS3/Vanilla JS - Deploy: Docker Compose Acceptance Criteria: ✅ CPU-only, <8GB RAM, ~5GB storage ✅ Suporta upload de PDFs ✅ Interface web rica e responsiva ✅ Bot Telegram funcional ✅ Agendamento de mensagens ✅ Código limpo e documentado ✅ Deploy via Docker Status: PRODUCTION READY Version: 1.0.0 Date: 2026-11-01
This commit is contained in:
commit
b9c0c66d21
56 files changed
+5793
No files matched your search
@@ -0,0 +1,51 @@
|
||||
from app.services.pdf_processor import PDFProcessor
|
||||
from app.utils.embeddings import EmbeddingService
|
||||
from app.utils.schedule_manager import ScheduleManager
|
||||
import os
|
||||
import json
|
||||
|
||||
def main():
|
||||
# Setup services
|
||||
embedding_service = EmbeddingService()
|
||||
pdf_processor = PDFProcessor(embedding_service)
|
||||
|
||||
# Load books from PDF directory
|
||||
pdf_dir = "/app/uploads"
|
||||
books = os.listdir(pdf_dir)
|
||||
books = [f for f in books if f.endswith('.pdf')]
|
||||
|
||||
print(f"Found {len(books)} books")
|
||||
|
||||
# Schedule manager
|
||||
schedule_manager = ScheduleManager("/app/schedules.json")
|
||||
|
||||
# Process each book
|
||||
for book_name in books:
|
||||
book_path = os.path.join(pdf_dir, book_name)
|
||||
book_metadata = {
|
||||
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
|
||||
"author": "Desconhecido",
|
||||
"type": "book",
|
||||
}
|
||||
|
||||
print(f"Processing: {book_name}")
|
||||
|
||||
# Load and split PDF
|
||||
chunks = pdf_processor.load_and_split(book_path, book_metadata)
|
||||
|
||||
# Embed chunks
|
||||
chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
|
||||
|
||||
# Save chunks to JSON for reference
|
||||
chunks_file = f"/app/index/{book_name}.json"
|
||||
os.makedirs("/app/index", exist_ok=True)
|
||||
|
||||
with open(chunks_file, 'w') as f:
|
||||
json.dump(chunks_with_embeddings, f, indent=2)
|
||||
|
||||
print(f"Saved {len(chunks)} chunks to {chunks_file}")
|
||||
|
||||
print("Indexing complete!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user