fix: make docker compose build and start the API

The image context was backend/, so COPY frontend/ failed. Build from the
repo root, place Python modules on the import path, and pin packages that
exist on PyPI. The web process no longer loads the LLM at startup.
This commit is contained in:
lgcosta committed 2026-11-01 12:00:08 -03:00
1 parent 67739dcf7e
commit 7af7ba530d
19 files changed
+342 -250

No files matched your search

+8
View File
@@ -0,0 +1,8 @@
.git
.superpowers
PDF
**/*.gguf
**/__pycache__
**/.venv
**/.env
models
+5 -5
View File
@@ -174,7 +174,7 @@ docker-compose up -d
After starting the services, index the 6 existing books: After starting the services, index the 6 existing books:
```bash ```bash
docker-compose run --rm web python backend/scripts/index_books.py docker compose run --rm web python scripts/index_books.py
``` ```
This will: This will:
@@ -443,10 +443,10 @@ ls -lh PDF/
```bash ```bash
# Método 1: Via container # Método 1: Via container
docker-compose run --rm web python backend/scripts/index_books.py docker compose run --rm web python scripts/index_books.py
# Método 2: Com limite de logs # Método 2: Com limite de logs
docker-compose run --rm web python backend/scripts/index_books.py 2>&1 | tee index.log docker compose run --rm web python scripts/index_books.py 2>&1 | tee index.log
``` ```
### 3. Monitorar Progresso ### 3. Monitorar Progresso
@@ -485,7 +485,7 @@ rm -rf backend/index/*
rm backend/schedules.json rm backend/schedules.json
# Reindexar # Reindexar
docker-compose run --rm web python backend/scripts/index_books.py docker compose run --rm web python scripts/index_books.py
``` ```
## 🎮 Uso da Aplicação ## 🎮 Uso da Aplicação
@@ -750,7 +750,7 @@ ls -lh PDF/
chmod -R 755 PDF/ chmod -R 755 PDF/
# Reindexar # Reindexar
docker-compose run --rm web python backend/scripts/index_books.py 2>&1 | tee reindex.log docker compose run --rm web python scripts/index_books.py 2>&1 | tee reindex.log
``` ```
### Comandos de Diagnóstico ### Comandos de Diagnóstico
+1 -1
View File
@@ -671,7 +671,7 @@ docker-compose build
docker-compose up -d docker-compose up -d
# Indexar livros # Indexar livros
docker-compose run --rm web python backend/scripts/index_books.py docker compose run --rm web python scripts/index_books.py
``` ```
### Produção - Single Server ### Produção - Single Server
+1 -1
View File
@@ -185,7 +185,7 @@ docker-compose ps
### 5. Indexar os Livros ### 5. Indexar os Livros
```bash ```bash
docker-compose run --rm web python backend/scripts/index_books.py docker compose run --rm web python scripts/index_books.py
``` ```
Isso irá: Isso irá:
+10 -15
View File
@@ -2,32 +2,27 @@ FROM python:3.11-slim
WORKDIR /app WORKDIR /app
# Install system dependencies RUN apt-get update && apt-get install -y --no-install-recommends \
RUN apt-get update && apt-get install -y \
curl \ curl \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Create directories RUN mkdir -p /app/uploads /app/index /app/models /app/books
RUN mkdir -p /app/uploads /app/index /app/models
# Copy requirements COPY backend/requirements.txt .
COPY requirements.txt .
# Install Python dependencies
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
# Set environment variables ENV PYTHONPATH=/app
ENV GPT4ALL_MODEL=llama-3.2-3b-instruct-q4_0.gguf ENV GPT4ALL_MODEL=/app/models/llama-3.2-3b-instruct-q4_0.gguf
ENV UPLOAD_DIR=/app/uploads ENV UPLOAD_DIR=/app/uploads
ENV BOOKS_DIR=/app/books
ENV INDEX_DIR=/app/index ENV INDEX_DIR=/app/index
ENV SCHEDULES_PATH=/app/schedules.json ENV SCHEDULES_PATH=/app/schedules.json
# Copy application COPY backend/app ./app
COPY app/ ./app/ COPY backend/templates ./templates
COPY frontend/ ./frontend/ COPY backend/scripts ./scripts
COPY frontend ./frontend
# Expose ports
EXPOSE 8000 EXPOSE 8000
# Run the application
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+111 -101
View File
@@ -1,24 +1,37 @@
from fastapi import FastAPI, File, UploadFile, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from fastapi.middleware.cors import CORSMiddleware
from app.services.rag_service import RAGService
from app.services.bot_service import BotService
from app.utils.schedule_manager import ScheduleManager
from app.models import ChatRequest, ChatResponse, BookInfo, UploadResponse, ScheduleInfo, ScheduleRequest
import os
import json import json
import os
from typing import List from typing import List
# Initialize services from fastapi import FastAPI, File, HTTPException, UploadFile
qdrant_service = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) from fastapi.middleware.cors import CORSMiddleware
rag_service = RAGService(qdrant_service, model_name=os.getenv("GPT4ALL_MODEL", "llama-3.2-3b-instruct-q4_0.gguf")) from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from app.models import (
BookInfo,
ChatRequest,
ChatResponse,
ScheduleInfo,
ScheduleRequest,
UploadResponse,
)
from app.services.bot_service import BotService
from app.services.rag_service import QdrantService, RAGService
from app.utils.schedule_manager import ScheduleManager
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
SCHEDULES_PATH = os.getenv("SCHEDULES_PATH", "/app/schedules.json")
UPLOAD_DIR = os.getenv("UPLOAD_DIR", "/app/uploads")
QDRANT_URL = os.getenv("QDRANT_URL", "http://qdrant:6333")
MODEL_PATH = os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf")
qdrant_service: QdrantService | None = None
rag_service: RAGService | None = None
bot_service = BotService(os.getenv("TELEGRAM_BOT_TOKEN", ""), os.getenv("TELEGRAM_CHAT_ID", "")) bot_service = BotService(os.getenv("TELEGRAM_BOT_TOKEN", ""), os.getenv("TELEGRAM_CHAT_ID", ""))
schedule_manager = ScheduleManager(os.getenv("SCHEDULES_PATH", "/app/schedules.json")) schedule_manager = ScheduleManager(SCHEDULES_PATH)
app = FastAPI(title="Espiritismo RAG API") app = FastAPI(title="Espiritismo RAG API")
# CORS middleware
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origins=["*"], allow_origins=["*"],
@@ -27,128 +40,125 @@ app.add_middleware(
allow_headers=["*"], allow_headers=["*"],
) )
# Mount static files app.mount(
app.mount("/static", StaticFiles(directory="../frontend/static"), name="static") "/static",
StaticFiles(directory=os.path.join(BASE_DIR, "frontend", "static")),
name="static",
)
def get_qdrant() -> QdrantService:
global qdrant_service
if qdrant_service is None:
qdrant_service = QdrantService(QDRANT_URL)
return qdrant_service
def get_rag() -> RAGService:
global rag_service
if rag_service is None:
rag_service = RAGService(get_qdrant(), model_name=MODEL_PATH)
return rag_service
# Load existing schedules from file
def load_schedules(): def load_schedules():
if os.path.exists("/app/schedules.json"): if os.path.isfile(SCHEDULES_PATH):
with open("/app/schedules.json", "r") as f: with open(SCHEDULES_PATH, "r", encoding="utf-8") as handle:
return json.load(f) data = json.load(handle)
return data if isinstance(data, list) else []
return [] return []
def save_schedules(schedules: list):
parent = os.path.dirname(SCHEDULES_PATH)
if parent:
os.makedirs(parent, exist_ok=True)
with open(SCHEDULES_PATH, "w", encoding="utf-8") as handle:
json.dump(schedules, handle, indent=2)
@app.get("/health")
async def health():
return {"status": "ok"}
@app.get("/", response_class=HTMLResponse) @app.get("/", response_class=HTMLResponse)
async def index(): async def index():
"""Serve the index page.""" template = os.path.join(BASE_DIR, "templates", "index.html")
with open("../backend/templates/index.html", "r") as f: with open(template, "r", encoding="utf-8") as handle:
return f.read() return handle.read()
@app.get("/api/books", response_model=List[BookInfo]) @app.get("/api/books", response_model=List[BookInfo])
async def get_books(): async def get_books():
"""Get list of books in the collection.""" books = get_qdrant().get_books()
books = qdrant_service.get_books()
return [{"title": book, "source": book} for book in books] return [{"title": book, "source": book} for book in books]
@app.post("/api/chat", response_model=ChatResponse) @app.post("/api/chat", response_model=ChatResponse)
async def chat(request: ChatRequest): async def chat(request: ChatRequest):
"""Query the RAG system."""
try: try:
result = rag_service.query(request.message, request.top_k) return get_rag().query(request.message, request.top_k)
return result except Exception as exc:
except Exception as e: raise HTTPException(status_code=500, detail=str(exc)) from exc
raise HTTPException(status_code=500, detail=str(e))
@app.post("/api/upload", response_model=UploadResponse) @app.post("/api/upload", response_model=UploadResponse)
async def upload_pdf(file: UploadFile = File(...)): async def upload_pdf(file: UploadFile = File(...)):
"""Upload a new PDF file.""" if not file.filename or not file.filename.endswith(".pdf"):
upload_dir = "/app/uploads"
if not file.filename.endswith('.pdf'):
raise HTTPException(status_code=400, detail="Only PDF files are allowed") raise HTTPException(status_code=400, detail="Only PDF files are allowed")
file_path = os.path.join(upload_dir, file.filename) os.makedirs(UPLOAD_DIR, exist_ok=True)
file_path = os.path.join(UPLOAD_DIR, os.path.basename(file.filename))
try: try:
with open(file_path, "wb") as f: content = await file.read()
content = await file.read() with open(file_path, "wb") as handle:
f.write(content) handle.write(content)
return UploadResponse(message="File uploaded successfully", filename=file.filename) return UploadResponse(message="File uploaded successfully", filename=file.filename)
except Exception as e: except Exception as exc:
raise HTTPException(status_code=500, detail=str(e)) raise HTTPException(status_code=500, detail=str(exc)) from exc
@app.get("/api/schedules", response_model=List[ScheduleInfo]) @app.get("/api/schedules", response_model=List[ScheduleInfo])
async def get_schedules(): async def get_schedules():
"""Get all schedules.""" return [ScheduleInfo(**item) for item in load_schedules()]
schedules = load_schedules()
return [ScheduleInfo(**s) for s in schedules]
@app.post("/api/schedules", response_model=ScheduleInfo) @app.post("/api/schedules", response_model=ScheduleInfo)
async def create_schedule(request: ScheduleRequest): async def create_schedule(request: ScheduleRequest):
"""Create a new schedule."""
schedules = load_schedules() schedules = load_schedules()
new_schedule = { new_schedule = schedule_manager.add_schedule(request.model_dump())
"id": len(schedules) + 1, if not any(item.get("id") == new_schedule["id"] for item in schedules):
"time": request.time, schedules.append(new_schedule)
"book": request.book, save_schedules(schedules)
"enabled": request.enabled,
}
schedules.append(new_schedule)
with open("/app/schedules.json", "w") as f:
json.dump(schedules, f, indent=2)
# Start celery beat if needed
try: try:
bot_service.add_schedule(new_schedule) bot_service.add_schedule(new_schedule)
except Exception as e: except Exception as exc:
print(f"Warning: Could not add schedule to bot: {e}") print(f"Warning: Could not add schedule to bot: {exc}")
return ScheduleInfo(**new_schedule) return ScheduleInfo(**new_schedule)
@app.put("/api/schedules/{schedule_id}", response_model=ScheduleInfo) @app.put("/api/schedules/{schedule_id}", response_model=ScheduleInfo)
async def update_schedule(schedule_id: int, request: ScheduleRequest): async def update_schedule(schedule_id: int, request: ScheduleRequest):
"""Update an existing schedule.""" updated = schedule_manager.update_schedule(schedule_id, request.model_dump())
schedules = load_schedules() if updated is None:
raise HTTPException(status_code=404, detail="Schedule not found")
save_schedules(schedule_manager.list_schedules())
try:
bot_service.update_schedule(updated)
except Exception as exc:
print(f"Warning: Could not update schedule in bot: {exc}")
return ScheduleInfo(**updated)
for i, s in enumerate(schedules):
if s["id"] == schedule_id:
schedules[i] = {
"id": schedule_id,
"time": request.time,
"book": request.book,
"enabled": request.enabled,
}
with open("/app/schedules.json", "w") as f:
json.dump(schedules, f, indent=2)
try:
bot_service.update_schedule(schedules[i])
except Exception as e:
print(f"Warning: Could not update schedule in bot: {e}")
return ScheduleInfo(**schedules[i])
raise HTTPException(status_code=404, detail="Schedule not found")
@app.delete("/api/schedules/{schedule_id}") @app.delete("/api/schedules/{schedule_id}")
async def delete_schedule(schedule_id: int): async def delete_schedule(schedule_id: int):
"""Delete a schedule.""" current = schedule_manager.get_schedule(schedule_id)
schedules = load_schedules() if current is None or not schedule_manager.delete_schedule(schedule_id):
raise HTTPException(status_code=404, detail="Schedule not found")
for i, s in enumerate(schedules): save_schedules(schedule_manager.list_schedules())
if s["id"] == schedule_id: try:
deleted = schedules.pop(i) bot_service.remove_schedule(current)
except Exception as exc:
with open("/app/schedules.json", "w") as f: print(f"Warning: Could not remove schedule from bot: {exc}")
json.dump(schedules, f, indent=2) return {"message": "Schedule deleted"}
try:
bot_service.remove_schedule(deleted)
except Exception as e:
print(f"Warning: Could not remove schedule from bot: {e}")
return {"message": "Schedule deleted"}
raise HTTPException(status_code=404, detail="Schedule not found")
-31
View File
@@ -1,31 +0,0 @@
from pydantic import BaseModel
from typing import Optional, List, Dict
class ChatRequest(BaseModel):
message: str
top_k: int = 3
class ChatResponse(BaseModel):
query: str
response: str
sources: List[Dict]
class BookInfo(BaseModel):
title: str
source: str
class UploadResponse(BaseModel):
message: str
filename: str
class ScheduleInfo(BaseModel):
id: int
time: str
book: Optional[str]
enabled: bool
next_run: Optional[str] = None
class ScheduleRequest(BaseModel):
time: str
book: Optional[str] = None
enabled: bool = True
+38 -1
View File
@@ -1 +1,38 @@
# Pydantic models package from typing import Dict, List, Optional
from pydantic import BaseModel
class ChatRequest(BaseModel):
message: str
top_k: int = 3
class ChatResponse(BaseModel):
query: str
response: str
sources: List[Dict]
class BookInfo(BaseModel):
title: str
source: str
class UploadResponse(BaseModel):
message: str
filename: str
class ScheduleInfo(BaseModel):
id: int
time: str
book: Optional[str] = None
enabled: bool
next_run: Optional[str] = None
class ScheduleRequest(BaseModel):
time: str
book: Optional[str] = None
enabled: bool = True
@@ -1,25 +1,16 @@
import telebot
from telebot import types
import os
import json import json
from datetime import datetime, timedelta import os
from typing import List, Dict, Optional
from celery import Celery
import random import random
from typing import Dict, List
celery = Celery( import telebot
'tasks',
broker=os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"),
backend=os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0"),
)
class BotService: class BotService:
def __init__(self, token: str, chat_id: str): def __init__(self, token: str, chat_id: str):
self.bot = telebot.TeleBot(token)
self.chat_id = chat_id self.chat_id = chat_id
self.bot = telebot.TeleBot(token) if token else None
# Register handlers if self.bot is not None:
self._register_handlers() self._register_handlers()
# Store pending schedules # Store pending schedules
self.pending_schedules: List[Dict] = [] self.pending_schedules: List[Dict] = []
@@ -81,11 +72,13 @@ Direcione uma pergunta sobre espiritismo para eu responder!
def _query_rag_service(self, question: str, chat_id: int): def _query_rag_service(self, question: str, chat_id: int):
"""Query RAG service and send response.""" """Query RAG service and send response."""
try: try:
from app.services.rag_service import RAGService from app.services.rag_service import QdrantService, RAGService
from app.services.qdrant_service import QdrantService
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
rag = RAGService(qdrant, model_name=os.getenv("GPT4ALL_MODEL", "llama-3.2-3b-instruct-q4_0.gguf")) rag = RAGService(
qdrant,
model_name=os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf"),
)
result = rag.query(question, top_k=3) result = rag.query(question, top_k=3)
@@ -160,14 +153,23 @@ Direcione uma pergunta sobre espiritismo para eu responder!
def start_polling(self): def start_polling(self):
"""Start the bot polling.""" """Start the bot polling."""
if self.bot is None:
print("TELEGRAM_BOT_TOKEN ausente; polling não iniciado")
return
print("Starting Telegram bot polling...") print("Starting Telegram bot polling...")
self.bot.infinity_polling(timeout=10, long_polling_timeout=5) self.bot.infinity_polling(timeout=10, long_polling_timeout=5)
@celery.task
def send_scheduled_message(self, schedule: Dict): def send_scheduled_message(self, schedule: Dict):
"""Send scheduled message to Telegram.""" """Send scheduled message to Telegram."""
try: try:
response = rag.query(schedule.get('book', ''), top_k=3) from app.services.rag_service import QdrantService, RAGService
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
rag = RAGService(
qdrant,
model_name=os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf"),
)
response = rag.query(schedule.get('book') or 'espiritismo', top_k=3)
message = f""" message = f"""
⏰ Mensagem Agendada - {schedule.get('book', 'Todos os livros')} ⏰ Mensagem Agendada - {schedule.get('book', 'Todos os livros')}
@@ -176,11 +178,13 @@ Direcione uma pergunta sobre espiritismo para eu responder!
📚 Fontes: 📚 Fontes:
{self._format_sources(response['sources'])} {self._format_sources(response['sources'])}
""" """
if self.bot is None:
print("TELEGRAM_BOT_TOKEN ausente; mensagem agendada não enviada")
return
self.bot.send_message(self.chat_id, message) self.bot.send_message(self.chat_id, message)
except Exception as e: except Exception as e:
print(f"Error sending scheduled message: {e}") print(f"Error sending scheduled message: {e}")
@celery.task
def generate_daily_content(self): def generate_daily_content(self):
"""Generate daily content for all books.""" """Generate daily content for all books."""
books = [ books = [
@@ -204,4 +208,7 @@ Direcione uma pergunta sobre espiritismo para eu responder!
Este conteúdo foi gerado automaticamente pelo Sistema RAG Espiritismo. Este conteúdo foi gerado automaticamente pelo Sistema RAG Espiritismo.
""" """
if self.bot is None:
print("TELEGRAM_BOT_TOKEN ausente; conteúdo diário não enviado")
return
self.bot.send_message(self.chat_id, message) self.bot.send_message(self.chat_id, message)
@@ -2,7 +2,7 @@ from langchain_community.document_loaders import PyPDFLoader
from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_text_splitters import RecursiveCharacterTextSplitter
import os import os
from typing import List, Dict from typing import List, Dict
from .embeddings import EmbeddingService from app.utils.embeddings import EmbeddingService
class PDFProcessor: class PDFProcessor:
def __init__(self, embedding_service: EmbeddingService): def __init__(self, embedding_service: EmbeddingService):
@@ -4,6 +4,7 @@ import numpy as np
from typing import List, Dict, Optional from typing import List, Dict, Optional
import os import os
import json import json
import time
class QdrantService: class QdrantService:
def __init__(self, url: str, collection_name: str = "espiritismo_books"): def __init__(self, url: str, collection_name: str = "espiritismo_books"):
@@ -15,7 +16,17 @@ class QdrantService:
def _ensure_collection(self): def _ensure_collection(self):
"""Create collection if it doesn't exist.""" """Create collection if it doesn't exist."""
collections = self.client.get_collections().collections last_error = None
for attempt in range(15):
try:
collections = self.client.get_collections().collections
break
except Exception as exc:
last_error = exc
print(f"Qdrant indisponível ({attempt + 1}/15): {exc}")
time.sleep(2)
else:
raise RuntimeError(f"Não foi possível conectar ao Qdrant: {last_error}")
collection_exists = any( collection_exists = any(
c.name == self.collection_name for c in collections c.name == self.collection_name for c in collections
@@ -39,7 +50,7 @@ class QdrantService:
for i, chunk in enumerate(chunks): for i, chunk in enumerate(chunks):
point = PointStruct( point = PointStruct(
id=i, id=chunk.get("id") or i,
vector=chunk["embedding"], vector=chunk["embedding"],
payload={ payload={
"content": chunk["content"], "content": chunk["content"],
@@ -55,8 +66,10 @@ class QdrantService:
) )
print(f"Added {len(points)} chunks to Qdrant") print(f"Added {len(points)} chunks to Qdrant")
def search(self, query_embedding: List[float], top_k: int = 3) -> List[Dict]: def search(self, query_embedding, top_k: int = 3) -> List[Dict]:
"""Search for similar chunks.""" """Search for similar chunks."""
if hasattr(query_embedding, "tolist"):
query_embedding = query_embedding.tolist()
results = self.client.search( results = self.client.search(
collection_name=self.collection_name, collection_name=self.collection_name,
query_vector=query_embedding, query_vector=query_embedding,
@@ -100,11 +113,26 @@ class RAGService:
def __init__(self, qdrant_service: QdrantService, model_name: str = "llama-3.2-3b-instruct-q4_0.gguf"): def __init__(self, qdrant_service: QdrantService, model_name: str = "llama-3.2-3b-instruct-q4_0.gguf"):
self.qdrant = qdrant_service self.qdrant = qdrant_service
self.model_name = model_name self.model_name = model_name
self.llm = None
# Setup GPT4All def _get_llm(self):
from gpt4all import GPT4All if self.llm is None:
if not os.path.isfile(self.model_name):
raise FileNotFoundError(
f"Modelo não encontrado em {self.model_name}. "
"Baixe llama-3.2-3b-instruct-q4_0.gguf para backend/models/."
)
from gpt4all import GPT4All
self.llm = GPT4All(model_name=model_name, device="cpu") model_dir = os.path.dirname(self.model_name) or "."
model_file = os.path.basename(self.model_name)
self.llm = GPT4All(
model_name=model_file,
model_path=model_dir,
device="cpu",
allow_download=False,
)
return self.llm
def generate_response(self, query: str, context: List[str], top_k: int = 3) -> str: def generate_response(self, query: str, context: List[str], top_k: int = 3) -> str:
"""Generate a response based on the query and context.""" """Generate a response based on the query and context."""
@@ -121,11 +149,8 @@ Instruções:
- Quando citar passagens, mencione o livro e capítulo (se disponível) - Quando citar passagens, mencione o livro e capítulo (se disponível)
- Se não souber a resposta, diga que não encontrou na base de dados""" - Se não souber a resposta, diga que não encontrou na base de dados"""
response = self.llm.generate( prompt = f"{system_prompt}\n\nPergunta: {query}\nResposta:"
system_prompt, response = self._get_llm().generate(prompt, max_tokens=500)
query,
max_tokens=500,
)
return response return response
@@ -149,5 +174,8 @@ Instruções:
"query": query, "query": query,
"response": response, "response": response,
"context": context, "context": context,
"sources": [chunk["metadata"] for chunk in chunks], "sources": [
{"metadata": chunk["metadata"], "score": chunk["score"]}
for chunk in chunks
],
} }
@@ -31,17 +31,19 @@ def save_schedules(schedules: List[Dict]):
json.dump(schedules, f, indent=2) json.dump(schedules, f, indent=2)
@celery.task @celery.task
def send_scheduled_message(schedule_id: int): def send_scheduled_message(schedule_id: int = None):
"""Send scheduled message to Telegram.""" """Send one schedule, or every enabled schedule when id is omitted."""
schedules = load_schedules() schedules = load_schedules()
schedule = next((s for s in schedules if s['id'] == schedule_id), None) if schedule_id is None:
selected = [item for item in schedules if item.get("enabled", True)]
else:
selected = [item for item in schedules if item.get("id") == schedule_id and item.get("enabled", True)]
if not schedule: if not selected:
print(f"Schedule {schedule_id} not found") print(f"Schedule {schedule_id} not found")
return return
if schedule.get('enabled', True): for schedule in selected:
# Send to bot service
try: try:
bot_service.send_scheduled_message(schedule) bot_service.send_scheduled_message(schedule)
print(f"Sent scheduled message: {schedule['time']}") print(f"Sent scheduled message: {schedule['time']}")
File renamed without changes.
File renamed without changes.
View File
Whitespace-only changes.
+3 -2
View File
@@ -2,10 +2,9 @@ fastapi==0.115.0
uvicorn[standard]==0.32.0 uvicorn[standard]==0.32.0
pydantic==2.9.2 pydantic==2.9.2
python-multipart==0.0.9 python-multipart==0.0.9
gpt4all==2.4.6 gpt4all==2.8.2
langchain==0.3.7 langchain==0.3.7
langchain-community==0.3.5 langchain-community==0.3.5
langchain-experimental==0.0.73
sentence-transformers==2.7.0 sentence-transformers==2.7.0
pytelegrambotapi==4.26.0 pytelegrambotapi==4.26.0
celery==5.4.0 celery==5.4.0
@@ -14,3 +13,5 @@ python-dotenv==1.0.1
python-dateutil==2.9.0.post0 python-dateutil==2.9.0.post0
aiofiles==24.1.0 aiofiles==24.1.0
qdrant-client==1.11.1 qdrant-client==1.11.1
pypdf==5.1.0
langchain-text-splitters==0.3.2
+43 -27
View File
@@ -1,51 +1,67 @@
from app.services.pdf_processor import PDFProcessor
from app.utils.embeddings import EmbeddingService
from app.utils.schedule_manager import ScheduleManager
import os
import json import json
import os
import uuid
from app.services.pdf_processor import PDFProcessor
from app.services.rag_service import QdrantService
from app.utils.embeddings import EmbeddingService
def book_dirs() -> list[str]:
paths = [
os.getenv("BOOKS_DIR", "/app/books"),
os.getenv("UPLOAD_DIR", "/app/uploads"),
]
seen = []
for path in paths:
if path and path not in seen and os.path.isdir(path):
seen.append(path)
return seen
def main(): def main():
# Setup services
embedding_service = EmbeddingService() embedding_service = EmbeddingService()
pdf_processor = PDFProcessor(embedding_service) pdf_processor = PDFProcessor(embedding_service)
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
# Load books from PDF directory books = []
pdf_dir = "/app/uploads" for directory in book_dirs():
books = os.listdir(pdf_dir) for name in sorted(os.listdir(directory)):
books = [f for f in books if f.endswith('.pdf')] if name.lower().endswith(".pdf"):
books.append(os.path.join(directory, name))
print(f"Found {len(books)} books") print(f"Found {len(books)} books")
os.makedirs(os.getenv("INDEX_DIR", "/app/index"), exist_ok=True)
# Schedule manager for book_path in books:
schedule_manager = ScheduleManager("/app/schedules.json") book_name = os.path.basename(book_path)
# Process each book
for book_name in books:
book_path = os.path.join(pdf_dir, book_name)
book_metadata = { book_metadata = {
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "), "title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
"author": "Desconhecido", "author": "Desconhecido",
"type": "book", "type": "book",
} }
print(f"Processing: {book_name}") print(f"Processing: {book_name}")
# Load and split PDF
chunks = pdf_processor.load_and_split(book_path, book_metadata) chunks = pdf_processor.load_and_split(book_path, book_metadata)
# Embed chunks
chunks_with_embeddings = pdf_processor.embed_chunks(chunks) chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
for chunk in chunks_with_embeddings:
chunk["id"] = str(uuid.uuid4())
# Save chunks to JSON for reference qdrant.add_chunks(chunks_with_embeddings)
chunks_file = f"/app/index/{book_name}.json"
os.makedirs("/app/index", exist_ok=True)
with open(chunks_file, 'w') as f: chunks_file = os.path.join(os.getenv("INDEX_DIR", "/app/index"), f"{book_name}.json")
json.dump(chunks_with_embeddings, f, indent=2) with open(chunks_file, "w", encoding="utf-8") as handle:
json.dump(
print(f"Saved {len(chunks)} chunks to {chunks_file}") [
{"content": chunk["content"], "metadata": chunk["metadata"]}
for chunk in chunks_with_embeddings
],
handle,
ensure_ascii=False,
)
print(f"Indexed {len(chunks_with_embeddings)} chunks from {book_name}")
print("Indexing complete!") print("Indexing complete!")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
+1
View File
@@ -0,0 +1 @@
[]
+44 -26
View File
@@ -1,8 +1,6 @@
version: '3.8'
services: services:
qdrant: qdrant:
image: qdrant/qdrant:latest image: docker.io/qdrant/qdrant:latest
container_name: espiritismo-qdrant container_name: espiritismo-qdrant
ports: ports:
- "6333:6333" - "6333:6333"
@@ -11,7 +9,7 @@ services:
restart: unless-stopped restart: unless-stopped
redis: redis:
image: redis:7-alpine image: docker.io/library/redis:7-alpine
container_name: espiritismo-redis container_name: espiritismo-redis
ports: ports:
- "6379:6379" - "6379:6379"
@@ -20,52 +18,71 @@ services:
restart: unless-stopped restart: unless-stopped
web: web:
build: ./backend build:
context: .
dockerfile: backend/Dockerfile
container_name: espiritismo-web container_name: espiritismo-web
ports: ports:
- "8000:8000" - "8000:8000"
volumes: volumes:
- ./PDF:/app/uploads:ro - ./PDF:/app/books:ro
- ./backend/schedules.json:/app/schedules.json - uploads_data:/app/uploads
- ./data/schedules.json:/app/schedules.json
- ./backend/models:/app/models
environment: environment:
- QDRANT_URL=http://qdrant:6333 QDRANT_URL: http://qdrant:6333
- CELERY_BROKER_URL=redis://redis:6379/0 CELERY_BROKER_URL: redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0
- TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
GPT4ALL_MODEL: /app/models/llama-3.2-3b-instruct-q4_0.gguf
BOOKS_DIR: /app/books
UPLOAD_DIR: /app/uploads
SCHEDULES_PATH: /app/schedules.json
depends_on: depends_on:
- qdrant - qdrant
- redis - redis
restart: unless-stopped restart: unless-stopped
celery: celery:
build: ./backend build:
context: .
dockerfile: backend/Dockerfile
container_name: espiritismo-celery container_name: espiritismo-celery
command: celery -A app.tasks.celery_tasks worker --loglevel=info command: celery -A app.tasks.celery_tasks worker --loglevel=info
volumes: volumes:
- ./PDF:/app/uploads:ro - ./PDF:/app/books:ro
- ./backend/schedules.json:/app/schedules.json - uploads_data:/app/uploads
- ./data/schedules.json:/app/schedules.json
- ./backend/models:/app/models
environment: environment:
- QDRANT_URL=http://qdrant:6333 QDRANT_URL: http://qdrant:6333
- CELERY_BROKER_URL=redis://redis:6379/0 CELERY_BROKER_URL: redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0
- TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
GPT4ALL_MODEL: /app/models/llama-3.2-3b-instruct-q4_0.gguf
BOOKS_DIR: /app/books
UPLOAD_DIR: /app/uploads
SCHEDULES_PATH: /app/schedules.json
depends_on: depends_on:
- qdrant - qdrant
- redis - redis
restart: unless-stopped restart: unless-stopped
celery-beat: celery-beat:
build: ./backend build:
context: .
dockerfile: backend/Dockerfile
command: celery -A app.tasks.celery_tasks beat --loglevel=info command: celery -A app.tasks.celery_tasks beat --loglevel=info
volumes: volumes:
- ./backend/schedules.json:/app/schedules.json - ./data/schedules.json:/app/schedules.json
environment: environment:
- CELERY_BROKER_URL=redis://redis:6379/0 CELERY_BROKER_URL: redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0
- TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
- TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
SCHEDULES_PATH: /app/schedules.json
depends_on: depends_on:
- redis - redis
restart: unless-stopped restart: unless-stopped
@@ -73,3 +90,4 @@ services:
volumes: volumes:
qdrant_data: qdrant_data:
redis_data: redis_data:
uploads_data: