diff --git a/backend/app/main.py b/backend/app/main.py
index 2ee7833..55575f7 100644
--- a/backend/app/main.py
+++ b/backend/app/main.py
@@ -1,5 +1,7 @@
+import asyncio
import json
import os
+import uuid
from typing import List
from fastapi import FastAPI, File, HTTPException, UploadFile
@@ -69,12 +71,30 @@ def load_schedules():
return []
+def index_pdf(file_path: str, filename: str):
+ from app.services.pdf_processor import PDFProcessor
+ from app.utils.embeddings import EmbeddingService
+
+ title = os.path.splitext(os.path.basename(filename))[0].replace("-", " ").replace("_", " ")
+ processor = PDFProcessor(EmbeddingService())
+ chunks = processor.embed_chunks(
+ processor.load_and_split(
+ file_path,
+ {"title": title, "author": "Desconhecido", "type": "book"},
+ )
+ )
+ for chunk in chunks:
+ chunk["id"] = str(uuid.uuid4())
+ get_qdrant().add_chunks(chunks)
+
+
def save_schedules(schedules: list):
parent = os.path.dirname(SCHEDULES_PATH)
if parent:
os.makedirs(parent, exist_ok=True)
with open(SCHEDULES_PATH, "w", encoding="utf-8") as handle:
json.dump(schedules, handle, indent=2)
+ handle.write("\n")
@app.get("/health")
@@ -96,7 +116,7 @@ async def get_books():
@app.post("/api/chat", response_model=ChatResponse)
-async def chat(request: ChatRequest):
+def chat(request: ChatRequest):
try:
return get_rag().query(request.message, request.top_k)
except Exception as exc:
@@ -114,7 +134,8 @@ async def upload_pdf(file: UploadFile = File(...)):
content = await file.read()
with open(file_path, "wb") as handle:
handle.write(content)
- return UploadResponse(message="File uploaded successfully", filename=file.filename)
+ await asyncio.to_thread(index_pdf, file_path, file.filename)
+ return UploadResponse(message="File uploaded and indexed", filename=file.filename)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
@@ -126,15 +147,7 @@ async def get_schedules():
@app.post("/api/schedules", response_model=ScheduleInfo)
async def create_schedule(request: ScheduleRequest):
- schedules = load_schedules()
new_schedule = schedule_manager.add_schedule(request.model_dump())
- if not any(item.get("id") == new_schedule["id"] for item in schedules):
- schedules.append(new_schedule)
- save_schedules(schedules)
- try:
- bot_service.add_schedule(new_schedule)
- except Exception as exc:
- print(f"Warning: Could not add schedule to bot: {exc}")
return ScheduleInfo(**new_schedule)
diff --git a/backend/app/services/pdf_processor.py b/backend/app/services/pdf_processor.py
index db88520..79b1396 100644
--- a/backend/app/services/pdf_processor.py
+++ b/backend/app/services/pdf_processor.py
@@ -22,12 +22,15 @@ class PDFProcessor:
# Convert to dictionaries with metadata
chunk_data = []
- for i, doc in enumerate(chunks):
+ for doc in chunks:
+ content = " ".join(doc.page_content.split())
+ if len(content) < 80:
+ continue
chunk_data.append({
- "content": doc.page_content,
+ "content": content,
"metadata": {
**metadata,
- "chunk_id": i,
+ "chunk_id": len(chunk_data),
"source": file_path,
}
})
diff --git a/backend/app/services/rag_service.py b/backend/app/services/rag_service.py
index 2e70522..e47ce44 100644
--- a/backend/app/services/rag_service.py
+++ b/backend/app/services/rag_service.py
@@ -2,9 +2,10 @@ from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue
import numpy as np
from typing import List, Dict, Optional
-import os
import json
+import os
import time
+import uuid
class QdrantService:
def __init__(self, url: str, collection_name: str = "espiritismo_books"):
@@ -46,24 +47,22 @@ class QdrantService:
def add_chunks(self, chunks: List[Dict]):
"""Add chunks to the collection."""
- points = []
-
- for i, chunk in enumerate(chunks):
- point = PointStruct(
- id=chunk.get("id") or i,
+ points = [
+ PointStruct(
+ id=str(chunk.get("id") or uuid.uuid4()),
vector=chunk["embedding"],
payload={
"content": chunk["content"],
"metadata": chunk["metadata"],
- }
+ },
)
- points.append(point)
+ for chunk in chunks
+ ]
+ for start in range(0, len(points), 128):
+ batch = points[start:start + 128]
+ self.client.upsert(collection_name=self.collection_name, points=batch)
if points:
- self.client.upsert(
- collection_name=self.collection_name,
- points=points,
- )
print(f"Added {len(points)} chunks to Qdrant")
def search(self, query_embedding, top_k: int = 3) -> List[Dict]:
@@ -87,17 +86,22 @@ class QdrantService:
def get_books(self) -> List[str]:
"""Get list of books in the collection."""
- points = self.client.scroll(
- collection_name=self.collection_name,
- limit=10000,
- )[0]
-
books = set()
- for point in points:
- metadata = point.payload["metadata"]
- books.add(metadata.get("title", "Unknown"))
+ offset = None
+ while True:
+ points, offset = self.client.scroll(
+ collection_name=self.collection_name,
+ limit=256,
+ offset=offset,
+ with_vectors=False,
+ )
+ for point in points:
+ metadata = (point.payload or {}).get("metadata") or {}
+ books.add(metadata.get("title", "Unknown"))
+ if offset is None:
+ break
- return sorted(list(books))
+ return sorted(books)
def get_collection_info(self) -> Dict:
"""Get information about the collection."""
@@ -138,21 +142,19 @@ class RAGService:
"""Generate a response based on the query and context."""
context_text = "\n\n".join(context)
- system_prompt = f"""Você é um assistente especializado em espiritismo. Use as informações abaixo para responder à pergunta do usuário.
+ system_prompt = f"""Você responde perguntas sobre espiritismo usando apenas o contexto.
+Escreva uma ou duas frases completas, em português. Não repita a pergunta nem as instruções.
+Se o contexto não bastar, responda apenas: Não encontrei essa informação nos livros indexados.
-Informações:
-{context_text}
-
-Instruções:
-- Responda de forma clara e concisa
-- Use o contexto fornecido
-- Quando citar passagens, mencione o livro e capítulo (se disponível)
-- Se não souber a resposta, diga que não encontrou na base de dados"""
-
- prompt = f"{system_prompt}\n\nPergunta: {query}\nResposta:"
- response = self._get_llm().generate(prompt, max_tokens=500)
-
- return response
+Contexto:
+{context_text}"""
+ template = (
+ "<|start_header_id|>user<|end_header_id|>\n\n"
+ "{0}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+ )
+ with self._get_llm().chat_session(system_prompt=system_prompt, prompt_template=template):
+ response = self.llm.generate(query, max_tokens=160, temp=0.1)
+ return response.strip()
def query(self, query: str, top_k: int = 3) -> Dict:
"""Query the RAG system."""
diff --git a/backend/app/tasks/celery_tasks.py b/backend/app/tasks/celery_tasks.py
index 69e7afb..cd9e750 100644
--- a/backend/app/tasks/celery_tasks.py
+++ b/backend/app/tasks/celery_tasks.py
@@ -30,25 +30,47 @@ def save_schedules(schedules: List[Dict]):
with open("/app/schedules.json", "w") as f:
json.dump(schedules, f, indent=2)
+def schedule_is_due(schedule: Dict) -> bool:
+ raw = str(schedule.get("time", "")).replace("Z", "+00:00")
+ when = datetime.fromisoformat(raw)
+ if when.tzinfo is not None:
+ when = when.astimezone().replace(tzinfo=None)
+ return datetime.now() >= when
+
+
@celery.task
def send_scheduled_message(schedule_id: int = None):
- """Send one schedule, or every enabled schedule when id is omitted."""
+ """Send enabled schedules whose time has arrived and that were not sent yet."""
schedules = load_schedules()
- if schedule_id is None:
- selected = [item for item in schedules if item.get("enabled", True)]
- else:
- selected = [item for item in schedules if item.get("id") == schedule_id and item.get("enabled", True)]
+ due = []
+ for item in schedules:
+ if schedule_id is not None and item.get("id") != schedule_id:
+ continue
+ if not item.get("enabled", True) or item.get("sent"):
+ continue
+ try:
+ ready = schedule_is_due(item)
+ except ValueError:
+ print(f"Invalid schedule time: {item.get('time')}")
+ continue
+ if ready:
+ due.append(item)
- if not selected:
- print(f"Schedule {schedule_id} not found")
+ if not due:
+ print("No schedules due")
return
- for schedule in selected:
+ changed = False
+ for schedule in due:
try:
bot_service.send_scheduled_message(schedule)
+ schedule["sent"] = True
+ changed = True
print(f"Sent scheduled message: {schedule['time']}")
except Exception as e:
print(f"Error sending scheduled message: {e}")
+ if changed:
+ save_schedules(schedules)
@celery.task
def generate_daily_content():
diff --git a/backend/app/utils/embeddings.py b/backend/app/utils/embeddings.py
index e6b4367..0bf6547 100644
--- a/backend/app/utils/embeddings.py
+++ b/backend/app/utils/embeddings.py
@@ -1,9 +1,13 @@
from sentence_transformers import SentenceTransformer
import numpy as np
+_MODELS = {}
+
class EmbeddingService:
- def __init__(self, model_name="all-MiniLM-L6-v2"):
- self.model = SentenceTransformer(model_name)
+ def __init__(self, model_name="paraphrase-multilingual-MiniLM-L12-v2"):
+ if model_name not in _MODELS:
+ _MODELS[model_name] = SentenceTransformer(model_name)
+ self.model = _MODELS[model_name]
self.dimension = 384
def embed(self, texts: list[str]) -> np.ndarray:
diff --git a/backend/app/utils/schedule_manager.py b/backend/app/utils/schedule_manager.py
index 1c0f19e..f8856e5 100644
--- a/backend/app/utils/schedule_manager.py
+++ b/backend/app/utils/schedule_manager.py
@@ -23,6 +23,7 @@ class ScheduleManager:
os.makedirs(os.path.dirname(self.schedules_path), exist_ok=True)
with open(self.schedules_path, 'w') as f:
json.dump(self.schedules, f, indent=2)
+ f.write("\n")
def add_schedule(self, schedule: Dict) -> Dict:
"""Add a new schedule."""
diff --git a/backend/scripts/index_books.py b/backend/scripts/index_books.py
index cf0b4d6..2708e7c 100644
--- a/backend/scripts/index_books.py
+++ b/backend/scripts/index_books.py
@@ -23,6 +23,8 @@ def main():
embedding_service = EmbeddingService()
pdf_processor = PDFProcessor(embedding_service)
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
+ qdrant.client.delete_collection(qdrant.collection_name)
+ qdrant._ensure_collection()
books = []
for directory in book_dirs():
diff --git a/backend/templates/index.html b/backend/templates/index.html
index 12e4ef1..f9bbfc8 100644
--- a/backend/templates/index.html
+++ b/backend/templates/index.html
@@ -4,6 +4,7 @@
Sistema RAG Espiritismo
+
diff --git a/docker-compose.yml b/docker-compose.yml
index d2def15..09cecb5 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,6 +39,7 @@ services:
BOOKS_DIR: /app/books
UPLOAD_DIR: /app/uploads
SCHEDULES_PATH: /app/schedules.json
+ TZ: America/Sao_Paulo
depends_on:
- qdrant
- redis
@@ -65,6 +66,7 @@ services:
BOOKS_DIR: /app/books
UPLOAD_DIR: /app/uploads
SCHEDULES_PATH: /app/schedules.json
+ TZ: America/Sao_Paulo
depends_on:
- qdrant
- redis
@@ -83,6 +85,7 @@ services:
TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
SCHEDULES_PATH: /app/schedules.json
+ TZ: America/Sao_Paulo
depends_on:
- redis
restart: unless-stopped
diff --git a/frontend/static/css/style.css b/frontend/static/css/style.css
index d2c123e..89dfee6 100644
--- a/frontend/static/css/style.css
+++ b/frontend/static/css/style.css
@@ -85,6 +85,7 @@ header p {
.message-content {
line-height: 1.6;
+ white-space: pre-wrap;
}
.message-source {
@@ -170,3 +171,36 @@ th {
border-color: #667eea;
background: #f9f9f9;
}
+
+.book-item {
+ display: flex;
+ gap: 8px;
+ align-items: center;
+ padding: 8px 0;
+ border-bottom: 1px solid #eee;
+}
+
+.schedule-item {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ gap: 12px;
+ padding: 8px 0;
+ border-bottom: 1px solid #eee;
+}
+
+.schedule-info {
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
+}
+
+button.schedule-delete {
+ width: auto;
+ padding: 8px 12px;
+}
+
+#upload-status {
+ margin-top: 10px;
+ min-height: 1.2em;
+}
diff --git a/frontend/static/js/script.js b/frontend/static/js/script.js
index 61c4df2..f48a37b 100644
--- a/frontend/static/js/script.js
+++ b/frontend/static/js/script.js
@@ -17,9 +17,11 @@ document.addEventListener('DOMContentLoaded', () => {
async function sendChatMessage() {
const input = document.getElementById('chat-input');
+ const button = document.getElementById('send-btn');
const message = input.value.trim();
- if (!message) return;
+ if (!message || button.disabled) return;
+ button.disabled = true;
// Add user message to chat
addMessage(message, 'user');
@@ -35,39 +37,39 @@ async function sendChatMessage() {
body: JSON.stringify({ message, top_k: 3 }),
});
+ const data = await response.json().catch(() => ({}));
if (!response.ok) {
- throw new Error('Erro ao enviar mensagem');
+ const detail = typeof data.detail === 'string' ? data.detail : 'Erro ao enviar mensagem';
+ throw new Error(detail);
}
- const data = await response.json();
-
- // Remove loading
hideLoading();
-
- // Add bot response
addMessage(data.response, 'bot');
- // Add sources
if (data.sources && data.sources.length > 0) {
- const sourceText = data.sources.map(s => `📖 ${s.metadata.title} (chunk ${s.metadata.chunk_id})`).join('\n');
- addMessage(`Fontes: ${sourceText}`, 'bot', true);
+ const sourceText = data.sources.map((source) => {
+ const meta = source.metadata || source;
+ return `📖 ${meta.title || 'Desconhecido'} (trecho ${meta.chunk_id ?? '?'})`;
+ }).join('\n');
+ addMessage(`Fontes:\n${sourceText}`, 'bot');
}
} catch (error) {
hideLoading();
- addMessage(`❌ Erro: ${error.message}`, 'bot');
+ addMessage(`Erro: ${error.message}`, 'bot');
+ } finally {
+ button.disabled = false;
+ input.focus();
}
}
-function addMessage(content, type, isSource = false) {
+function addMessage(content, type) {
const container = document.getElementById('chat-messages');
const messageDiv = document.createElement('div');
messageDiv.className = `message ${type}`;
-
- if (isSource) {
- messageDiv.style.fontSize = '0.85rem';
- }
-
- messageDiv.innerHTML = `${content}
`;
+ const body = document.createElement('div');
+ body.className = 'message-content';
+ body.textContent = content;
+ messageDiv.appendChild(body);
container.appendChild(messageDiv);
container.scrollTop = container.scrollHeight;
}
@@ -103,10 +105,13 @@ async function loadBooks() {
books.forEach(book => {
const bookDiv = document.createElement('div');
bookDiv.className = 'book-item';
- bookDiv.innerHTML = `
- 📖
- ${book.title}
- `;
+ const icon = document.createElement('div');
+ icon.className = 'book-icon';
+ icon.textContent = '📖';
+ const name = document.createElement('div');
+ name.className = 'book-name';
+ name.textContent = book.title;
+ bookDiv.append(icon, name);
booksList.appendChild(bookDiv);
});
@@ -189,17 +194,22 @@ function renderSchedules(schedules) {
schedules.forEach(schedule => {
const div = document.createElement('div');
div.className = 'schedule-item';
- div.innerHTML = `
-
- 🕐 ${formatDateTime(schedule.time)}
- ${schedule.book || 'Todos os livros'}
-
-
- `;
+ const info = document.createElement('div');
+ info.className = 'schedule-info';
+ const time = document.createElement('span');
+ time.className = 'schedule-time';
+ time.textContent = formatDateTime(schedule.time);
+ const book = document.createElement('span');
+ book.className = 'schedule-book';
+ book.textContent = schedule.book || 'Todos os livros';
+ info.append(time, book);
+ const remove = document.createElement('button');
+ remove.className = 'schedule-delete';
+ remove.type = 'button';
+ remove.textContent = 'Excluir';
+ remove.addEventListener('click', () => deleteSchedule(schedule.id));
+ div.append(info, remove);
container.appendChild(div);
-
- // Add delete handler
- div.querySelector('.schedule-delete').addEventListener('click', () => deleteSchedule(schedule.id));
});
}