fix: responder a partir dos livros com embeddings em português

Usa o modelo multilingual, ignora trechos curtos e o template do Llama 3.2
para a resposta sair em uma frase completa. O chat deixa a API livre durante
a geração, o upload indexa o PDF e os agendamentos não são gravados duas vezes.

backend/app/main.py

@@ -1,5 +1,7 @@
+import asyncio
 import json
 import os
+import uuid
 from typing import List
 
 from fastapi import FastAPI, File, HTTPException, UploadFile
@@ -69,12 +71,30 @@ def load_schedules():
     return []
 
 
+def index_pdf(file_path: str, filename: str):
+    from app.services.pdf_processor import PDFProcessor
+    from app.utils.embeddings import EmbeddingService
+
+    title = os.path.splitext(os.path.basename(filename))[0].replace("-", " ").replace("_", " ")
+    processor = PDFProcessor(EmbeddingService())
+    chunks = processor.embed_chunks(
+        processor.load_and_split(
+            file_path,
+            {"title": title, "author": "Desconhecido", "type": "book"},
+        )
+    )
+    for chunk in chunks:
+        chunk["id"] = str(uuid.uuid4())
+    get_qdrant().add_chunks(chunks)
+
+
 def save_schedules(schedules: list):
     parent = os.path.dirname(SCHEDULES_PATH)
     if parent:
         os.makedirs(parent, exist_ok=True)
     with open(SCHEDULES_PATH, "w", encoding="utf-8") as handle:
         json.dump(schedules, handle, indent=2)
+        handle.write("\n")
 
 
 @app.get("/health")
@@ -96,7 +116,7 @@ async def get_books():
 
 
 @app.post("/api/chat", response_model=ChatResponse)
-async def chat(request: ChatRequest):
+def chat(request: ChatRequest):
     try:
         return get_rag().query(request.message, request.top_k)
     except Exception as exc:
@@ -114,7 +134,8 @@ async def upload_pdf(file: UploadFile = File(...)):
         content = await file.read()
         with open(file_path, "wb") as handle:
             handle.write(content)
-        return UploadResponse(message="File uploaded successfully", filename=file.filename)
+        await asyncio.to_thread(index_pdf, file_path, file.filename)
+        return UploadResponse(message="File uploaded and indexed", filename=file.filename)
     except Exception as exc:
         raise HTTPException(status_code=500, detail=str(exc)) from exc
 
@@ -126,15 +147,7 @@ async def get_schedules():
 
 @app.post("/api/schedules", response_model=ScheduleInfo)
 async def create_schedule(request: ScheduleRequest):
-    schedules = load_schedules()
     new_schedule = schedule_manager.add_schedule(request.model_dump())
-    if not any(item.get("id") == new_schedule["id"] for item in schedules):
-        schedules.append(new_schedule)
-        save_schedules(schedules)
-    try:
-        bot_service.add_schedule(new_schedule)
-    except Exception as exc:
-        print(f"Warning: Could not add schedule to bot: {exc}")
     return ScheduleInfo(**new_schedule)
 
 

backend/app/services/pdf_processor.py

@@ -22,12 +22,15 @@ class PDFProcessor:
 
         # Convert to dictionaries with metadata
         chunk_data = []
-        for i, doc in enumerate(chunks):
+        for doc in chunks:
+            content = " ".join(doc.page_content.split())
+            if len(content) < 80:
+                continue
             chunk_data.append({
-                "content": doc.page_content,
+                "content": content,
                 "metadata": {
                     **metadata,
-                    "chunk_id": i,
+                    "chunk_id": len(chunk_data),
                     "source": file_path,
                 }
             })

backend/app/services/rag_service.py

@@ -2,9 +2,10 @@ from qdrant_client import QdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue
 import numpy as np
 from typing import List, Dict, Optional
-import os
 import json
+import os
 import time
+import uuid
 
 class QdrantService:
     def __init__(self, url: str, collection_name: str = "espiritismo_books"):
@@ -46,24 +47,22 @@ class QdrantService:
 
     def add_chunks(self, chunks: List[Dict]):
         """Add chunks to the collection."""
-        points = []
-
-        for i, chunk in enumerate(chunks):
-            point = PointStruct(
-                id=chunk.get("id") or i,
+        points = [
+            PointStruct(
+                id=str(chunk.get("id") or uuid.uuid4()),
                 vector=chunk["embedding"],
                 payload={
                     "content": chunk["content"],
                     "metadata": chunk["metadata"],
-                }
+                },
             )
-            points.append(point)
+            for chunk in chunks
+        ]
 
+        for start in range(0, len(points), 128):
+            batch = points[start:start + 128]
+            self.client.upsert(collection_name=self.collection_name, points=batch)
         if points:
-            self.client.upsert(
-                collection_name=self.collection_name,
-                points=points,
-            )
             print(f"Added {len(points)} chunks to Qdrant")
 
     def search(self, query_embedding, top_k: int = 3) -> List[Dict]:
@@ -87,17 +86,22 @@ class QdrantService:
 
     def get_books(self) -> List[str]:
         """Get list of books in the collection."""
-        points = self.client.scroll(
-            collection_name=self.collection_name,
-            limit=10000,
-        )[0]
-
         books = set()
-        for point in points:
-            metadata = point.payload["metadata"]
-            books.add(metadata.get("title", "Unknown"))
+        offset = None
+        while True:
+            points, offset = self.client.scroll(
+                collection_name=self.collection_name,
+                limit=256,
+                offset=offset,
+                with_vectors=False,
+            )
+            for point in points:
+                metadata = (point.payload or {}).get("metadata") or {}
+                books.add(metadata.get("title", "Unknown"))
+            if offset is None:
+                break
 
-        return sorted(list(books))
+        return sorted(books)
 
     def get_collection_info(self) -> Dict:
         """Get information about the collection."""
@@ -138,21 +142,19 @@ class RAGService:
         """Generate a response based on the query and context."""
         context_text = "\n\n".join(context)
 
-        system_prompt = f"""Você é um assistente especializado em espiritismo. Use as informações abaixo para responder à pergunta do usuário.
+        system_prompt = f"""Você responde perguntas sobre espiritismo usando apenas o contexto.
+Escreva uma ou duas frases completas, em português. Não repita a pergunta nem as instruções.
+Se o contexto não bastar, responda apenas: Não encontrei essa informação nos livros indexados.
 
-Informações:
-{context_text}
-
-Instruções:
-- Responda de forma clara e concisa
-- Use o contexto fornecido
-- Quando citar passagens, mencione o livro e capítulo (se disponível)
-- Se não souber a resposta, diga que não encontrou na base de dados"""
-
-        prompt = f"{system_prompt}\n\nPergunta: {query}\nResposta:"
-        response = self._get_llm().generate(prompt, max_tokens=500)
-
-        return response
+Contexto:
+{context_text}"""
+        template = (
+            "<|start_header_id|>user<|end_header_id|>\n\n"
+            "{0}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+        )
+        with self._get_llm().chat_session(system_prompt=system_prompt, prompt_template=template):
+            response = self.llm.generate(query, max_tokens=160, temp=0.1)
+        return response.strip()
 
     def query(self, query: str, top_k: int = 3) -> Dict:
         """Query the RAG system."""

backend/app/tasks/celery_tasks.py

@@ -30,25 +30,47 @@ def save_schedules(schedules: List[Dict]):
     with open("/app/schedules.json", "w") as f:
         json.dump(schedules, f, indent=2)
 
+def schedule_is_due(schedule: Dict) -> bool:
+    raw = str(schedule.get("time", "")).replace("Z", "+00:00")
+    when = datetime.fromisoformat(raw)
+    if when.tzinfo is not None:
+        when = when.astimezone().replace(tzinfo=None)
+    return datetime.now() >= when
+
+
 @celery.task
 def send_scheduled_message(schedule_id: int = None):
-    """Send one schedule, or every enabled schedule when id is omitted."""
+    """Send enabled schedules whose time has arrived and that were not sent yet."""
     schedules = load_schedules()
-    if schedule_id is None:
-        selected = [item for item in schedules if item.get("enabled", True)]
-    else:
-        selected = [item for item in schedules if item.get("id") == schedule_id and item.get("enabled", True)]
+    due = []
+    for item in schedules:
+        if schedule_id is not None and item.get("id") != schedule_id:
+            continue
+        if not item.get("enabled", True) or item.get("sent"):
+            continue
+        try:
+            ready = schedule_is_due(item)
+        except ValueError:
+            print(f"Invalid schedule time: {item.get('time')}")
+            continue
+        if ready:
+            due.append(item)
 
-    if not selected:
-        print(f"Schedule {schedule_id} not found")
+    if not due:
+        print("No schedules due")
         return
 
-    for schedule in selected:
+    changed = False
+    for schedule in due:
         try:
             bot_service.send_scheduled_message(schedule)
+            schedule["sent"] = True
+            changed = True
             print(f"Sent scheduled message: {schedule['time']}")
         except Exception as e:
             print(f"Error sending scheduled message: {e}")
+    if changed:
+        save_schedules(schedules)
 
 @celery.task
 def generate_daily_content():

backend/app/utils/embeddings.py

@@ -1,9 +1,13 @@
 from sentence_transformers import SentenceTransformer
 import numpy as np
 
+_MODELS = {}
+
 class EmbeddingService:
-    def __init__(self, model_name="all-MiniLM-L6-v2"):
-        self.model = SentenceTransformer(model_name)
+    def __init__(self, model_name="paraphrase-multilingual-MiniLM-L12-v2"):
+        if model_name not in _MODELS:
+            _MODELS[model_name] = SentenceTransformer(model_name)
+        self.model = _MODELS[model_name]
         self.dimension = 384
 
     def embed(self, texts: list[str]) -> np.ndarray:

backend/app/utils/schedule_manager.py

@@ -23,6 +23,7 @@ class ScheduleManager:
         os.makedirs(os.path.dirname(self.schedules_path), exist_ok=True)
         with open(self.schedules_path, 'w') as f:
             json.dump(self.schedules, f, indent=2)
+            f.write("\n")
 
     def add_schedule(self, schedule: Dict) -> Dict:
         """Add a new schedule."""

backend/scripts/index_books.py

@@ -23,6 +23,8 @@ def main():
     embedding_service = EmbeddingService()
     pdf_processor = PDFProcessor(embedding_service)
     qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
+    qdrant.client.delete_collection(qdrant.collection_name)
+    qdrant._ensure_collection()
 
     books = []
     for directory in book_dirs():

backend/templates/index.html

@@ -4,6 +4,7 @@
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Sistema RAG Espiritismo</title>
+    <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ctext y='.9em' font-size='90'%3E📚%3C/text%3E%3C/svg%3E">
     <link rel="stylesheet" href="/static/css/style.css">
 </head>
 <body>

docker-compose.yml

@@ -39,6 +39,7 @@ services:
       BOOKS_DIR: /app/books
       UPLOAD_DIR: /app/uploads
       SCHEDULES_PATH: /app/schedules.json
+      TZ: America/Sao_Paulo
     depends_on:
       - qdrant
       - redis
@@ -65,6 +66,7 @@ services:
       BOOKS_DIR: /app/books
       UPLOAD_DIR: /app/uploads
       SCHEDULES_PATH: /app/schedules.json
+      TZ: America/Sao_Paulo
     depends_on:
       - qdrant
       - redis
@@ -83,6 +85,7 @@ services:
       TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-}
       TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-}
       SCHEDULES_PATH: /app/schedules.json
+      TZ: America/Sao_Paulo
     depends_on:
       - redis
     restart: unless-stopped

frontend/static/css/style.css

@@ -85,6 +85,7 @@ header p {
 
 .message-content {
     line-height: 1.6;
+    white-space: pre-wrap;
 }
 
 .message-source {
@@ -170,3 +171,36 @@ th {
     border-color: #667eea;
     background: #f9f9f9;
 }
+
+.book-item {
+    display: flex;
+    gap: 8px;
+    align-items: center;
+    padding: 8px 0;
+    border-bottom: 1px solid #eee;
+}
+
+.schedule-item {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    gap: 12px;
+    padding: 8px 0;
+    border-bottom: 1px solid #eee;
+}
+
+.schedule-info {
+    display: flex;
+    flex-direction: column;
+    gap: 4px;
+}
+
+button.schedule-delete {
+    width: auto;
+    padding: 8px 12px;
+}
+
+#upload-status {
+    margin-top: 10px;
+    min-height: 1.2em;
+}

frontend/static/js/script.js

@@ -17,9 +17,11 @@ document.addEventListener('DOMContentLoaded', () => {
 
 async function sendChatMessage() {
     const input = document.getElementById('chat-input');
+    const button = document.getElementById('send-btn');
     const message = input.value.trim();
 
-    if (!message) return;
+    if (!message || button.disabled) return;
+    button.disabled = true;
 
     // Add user message to chat
     addMessage(message, 'user');
@@ -35,39 +37,39 @@ async function sendChatMessage() {
             body: JSON.stringify({ message, top_k: 3 }),
         });
 
+        const data = await response.json().catch(() => ({}));
         if (!response.ok) {
-            throw new Error('Erro ao enviar mensagem');
+            const detail = typeof data.detail === 'string' ? data.detail : 'Erro ao enviar mensagem';
+            throw new Error(detail);
         }
 
-        const data = await response.json();
-
-        // Remove loading
         hideLoading();
-
-        // Add bot response
         addMessage(data.response, 'bot');
 
-        // Add sources
         if (data.sources && data.sources.length > 0) {
-            const sourceText = data.sources.map(s => `📖 ${s.metadata.title} (chunk ${s.metadata.chunk_id})`).join('\n');
-            addMessage(`Fontes: ${sourceText}`, 'bot', true);
+            const sourceText = data.sources.map((source) => {
+                const meta = source.metadata || source;
+                return `📖 ${meta.title || 'Desconhecido'} (trecho ${meta.chunk_id ?? '?'})`;
+            }).join('\n');
+            addMessage(`Fontes:\n${sourceText}`, 'bot');
         }
     } catch (error) {
         hideLoading();
-        addMessage(`❌ Erro: ${error.message}`, 'bot');
+        addMessage(`Erro: ${error.message}`, 'bot');
+    } finally {
+        button.disabled = false;
+        input.focus();
     }
 }
 
-function addMessage(content, type, isSource = false) {
+function addMessage(content, type) {
     const container = document.getElementById('chat-messages');
     const messageDiv = document.createElement('div');
     messageDiv.className = `message ${type}`;
-
-    if (isSource) {
-        messageDiv.style.fontSize = '0.85rem';
-    }
-
-    messageDiv.innerHTML = `<div class="message-content">${content}</div>`;
+    const body = document.createElement('div');
+    body.className = 'message-content';
+    body.textContent = content;
+    messageDiv.appendChild(body);
     container.appendChild(messageDiv);
     container.scrollTop = container.scrollHeight;
 }
@@ -103,10 +105,13 @@ async function loadBooks() {
         books.forEach(book => {
             const bookDiv = document.createElement('div');
             bookDiv.className = 'book-item';
-            bookDiv.innerHTML = `
-                <div class="book-icon">📖</div>
-                <div class="book-name">${book.title}</div>
-            `;
+            const icon = document.createElement('div');
+            icon.className = 'book-icon';
+            icon.textContent = '📖';
+            const name = document.createElement('div');
+            name.className = 'book-name';
+            name.textContent = book.title;
+            bookDiv.append(icon, name);
             booksList.appendChild(bookDiv);
         });
 
@@ -189,17 +194,22 @@ function renderSchedules(schedules) {
     schedules.forEach(schedule => {
         const div = document.createElement('div');
         div.className = 'schedule-item';
-        div.innerHTML = `
-            <div class="schedule-info">
-                <span class="schedule-time">🕐 ${formatDateTime(schedule.time)}</span>
-                <span class="schedule-book">${schedule.book || 'Todos os livros'}</span>
-            </div>
-            <button class="schedule-delete" data-id="${schedule.id}">🗑️</button>
-        `;
+        const info = document.createElement('div');
+        info.className = 'schedule-info';
+        const time = document.createElement('span');
+        time.className = 'schedule-time';
+        time.textContent = formatDateTime(schedule.time);
+        const book = document.createElement('span');
+        book.className = 'schedule-book';
+        book.textContent = schedule.book || 'Todos os livros';
+        info.append(time, book);
+        const remove = document.createElement('button');
+        remove.className = 'schedule-delete';
+        remove.type = 'button';
+        remove.textContent = 'Excluir';
+        remove.addEventListener('click', () => deleteSchedule(schedule.id));
+        div.append(info, remove);
         container.appendChild(div);
-
-        // Add delete handler
-        div.querySelector('.schedule-delete').addEventListener('click', () => deleteSchedule(schedule.id));
     });
 }