lgcosta
b9c0c66d21
- Stack: GPT4All + Llama 3.2 3B, LangChain, Qdrant, FastAPI, Docker - 11 tarefas implementadas com 100% de conclusão - Funcionalidades: chat com IA, upload PDF, bot Telegram, agendamento - 6 livros PDF indexados (A Gênese, Livro dos Espíritos, etc.) - Documentação rica: README, BUILD, task reports, design specs - Deploy com Docker Compose - Production ready Tech stack: - AI: GPT4All 2.4.6, Llama 3.2 3B q4_0 - Backend: FastAPI 0.115.0, Celery 5.4.0, Redis 5.2.0 - RAG: LangChain 0.3.7, Qdrant 1.11.1 - Frontend: HTML5/CSS3/Vanilla JS - Deploy: Docker Compose Acceptance Criteria: ✅ CPU-only, <8GB RAM, ~5GB storage ✅ Suporta upload de PDFs ✅ Interface web rica e responsiva ✅ Bot Telegram funcional ✅ Agendamento de mensagens ✅ Código limpo e documentado ✅ Deploy via Docker Status: PRODUCTION READY Version: 1.0.0 Date: 2026-11-01
52 lines
1.5 KiB
Python
52 lines
1.5 KiB
Python
from app.services.pdf_processor import PDFProcessor
|
|
from app.utils.embeddings import EmbeddingService
|
|
from app.utils.schedule_manager import ScheduleManager
|
|
import os
|
|
import json
|
|
|
|
def main():
|
|
# Setup services
|
|
embedding_service = EmbeddingService()
|
|
pdf_processor = PDFProcessor(embedding_service)
|
|
|
|
# Load books from PDF directory
|
|
pdf_dir = "/app/uploads"
|
|
books = os.listdir(pdf_dir)
|
|
books = [f for f in books if f.endswith('.pdf')]
|
|
|
|
print(f"Found {len(books)} books")
|
|
|
|
# Schedule manager
|
|
schedule_manager = ScheduleManager("/app/schedules.json")
|
|
|
|
# Process each book
|
|
for book_name in books:
|
|
book_path = os.path.join(pdf_dir, book_name)
|
|
book_metadata = {
|
|
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
|
|
"author": "Desconhecido",
|
|
"type": "book",
|
|
}
|
|
|
|
print(f"Processing: {book_name}")
|
|
|
|
# Load and split PDF
|
|
chunks = pdf_processor.load_and_split(book_path, book_metadata)
|
|
|
|
# Embed chunks
|
|
chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
|
|
|
|
# Save chunks to JSON for reference
|
|
chunks_file = f"/app/index/{book_name}.json"
|
|
os.makedirs("/app/index", exist_ok=True)
|
|
|
|
with open(chunks_file, 'w') as f:
|
|
json.dump(chunks_with_embeddings, f, indent=2)
|
|
|
|
print(f"Saved {len(chunks)} chunks to {chunks_file}")
|
|
|
|
print("Indexing complete!")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|