fix: responder a partir dos livros com embeddings em português

Usa o modelo multilingual, ignora trechos curtos e o template do Llama 3.2
para a resposta sair em uma frase completa. O chat deixa a API livre durante
a geração, o upload indexa o PDF e os agendamentos não são gravados duas vezes.

backend/app/main.py

@@ -1,5 +1,7 @@
+import asyncio
 import json
 import os
+import uuid
 from typing import List
 
 from fastapi import FastAPI, File, HTTPException, UploadFile
@@ -69,12 +71,30 @@ def load_schedules():
     return []
 
 
+def index_pdf(file_path: str, filename: str):
+    from app.services.pdf_processor import PDFProcessor
+    from app.utils.embeddings import EmbeddingService
+
+    title = os.path.splitext(os.path.basename(filename))[0].replace("-", " ").replace("_", " ")
+    processor = PDFProcessor(EmbeddingService())
+    chunks = processor.embed_chunks(
+        processor.load_and_split(
+            file_path,
+            {"title": title, "author": "Desconhecido", "type": "book"},
+        )
+    )
+    for chunk in chunks:
+        chunk["id"] = str(uuid.uuid4())
+    get_qdrant().add_chunks(chunks)
+
+
 def save_schedules(schedules: list):
     parent = os.path.dirname(SCHEDULES_PATH)
     if parent:
         os.makedirs(parent, exist_ok=True)
     with open(SCHEDULES_PATH, "w", encoding="utf-8") as handle:
         json.dump(schedules, handle, indent=2)
+        handle.write("\n")
 
 
 @app.get("/health")
@@ -96,7 +116,7 @@ async def get_books():
 
 
 @app.post("/api/chat", response_model=ChatResponse)
-async def chat(request: ChatRequest):
+def chat(request: ChatRequest):
     try:
         return get_rag().query(request.message, request.top_k)
     except Exception as exc:
@@ -114,7 +134,8 @@ async def upload_pdf(file: UploadFile = File(...)):
         content = await file.read()
         with open(file_path, "wb") as handle:
             handle.write(content)
-        return UploadResponse(message="File uploaded successfully", filename=file.filename)
+        await asyncio.to_thread(index_pdf, file_path, file.filename)
+        return UploadResponse(message="File uploaded and indexed", filename=file.filename)
     except Exception as exc:
         raise HTTPException(status_code=500, detail=str(exc)) from exc
 
@@ -126,15 +147,7 @@ async def get_schedules():
 
 @app.post("/api/schedules", response_model=ScheduleInfo)
 async def create_schedule(request: ScheduleRequest):
-    schedules = load_schedules()
     new_schedule = schedule_manager.add_schedule(request.model_dump())
-    if not any(item.get("id") == new_schedule["id"] for item in schedules):
-        schedules.append(new_schedule)
-        save_schedules(schedules)
-    try:
-        bot_service.add_schedule(new_schedule)
-    except Exception as exc:
-        print(f"Warning: Could not add schedule to bot: {exc}")
     return ScheduleInfo(**new_schedule)
 
 

backend/app/services/pdf_processor.py

@@ -22,12 +22,15 @@ class PDFProcessor:
 
         # Convert to dictionaries with metadata
         chunk_data = []
-        for i, doc in enumerate(chunks):
+        for doc in chunks:
+            content = " ".join(doc.page_content.split())
+            if len(content) < 80:
+                continue
             chunk_data.append({
-                "content": doc.page_content,
+                "content": content,
                 "metadata": {
                     **metadata,
-                    "chunk_id": i,
+                    "chunk_id": len(chunk_data),
                     "source": file_path,
                 }
             })

backend/app/services/rag_service.py

@@ -2,9 +2,10 @@ from qdrant_client import QdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct, Filter, FieldCondition, MatchValue
 import numpy as np
 from typing import List, Dict, Optional
-import os
 import json
+import os
 import time
+import uuid
 
 class QdrantService:
     def __init__(self, url: str, collection_name: str = "espiritismo_books"):
@@ -46,24 +47,22 @@ class QdrantService:
 
     def add_chunks(self, chunks: List[Dict]):
         """Add chunks to the collection."""
-        points = []
-
-        for i, chunk in enumerate(chunks):
-            point = PointStruct(
-                id=chunk.get("id") or i,
+        points = [
+            PointStruct(
+                id=str(chunk.get("id") or uuid.uuid4()),
                 vector=chunk["embedding"],
                 payload={
                     "content": chunk["content"],
                     "metadata": chunk["metadata"],
-                }
+                },
             )
-            points.append(point)
+            for chunk in chunks
+        ]
 
+        for start in range(0, len(points), 128):
+            batch = points[start:start + 128]
+            self.client.upsert(collection_name=self.collection_name, points=batch)
         if points:
-            self.client.upsert(
-                collection_name=self.collection_name,
-                points=points,
-            )
             print(f"Added {len(points)} chunks to Qdrant")
 
     def search(self, query_embedding, top_k: int = 3) -> List[Dict]:
@@ -87,17 +86,22 @@ class QdrantService:
 
     def get_books(self) -> List[str]:
         """Get list of books in the collection."""
-        points = self.client.scroll(
-            collection_name=self.collection_name,
-            limit=10000,
-        )[0]
-
         books = set()
-        for point in points:
-            metadata = point.payload["metadata"]
-            books.add(metadata.get("title", "Unknown"))
+        offset = None
+        while True:
+            points, offset = self.client.scroll(
+                collection_name=self.collection_name,
+                limit=256,
+                offset=offset,
+                with_vectors=False,
+            )
+            for point in points:
+                metadata = (point.payload or {}).get("metadata") or {}
+                books.add(metadata.get("title", "Unknown"))
+            if offset is None:
+                break
 
-        return sorted(list(books))
+        return sorted(books)
 
     def get_collection_info(self) -> Dict:
         """Get information about the collection."""
@@ -138,21 +142,19 @@ class RAGService:
         """Generate a response based on the query and context."""
         context_text = "\n\n".join(context)
 
-        system_prompt = f"""Você é um assistente especializado em espiritismo. Use as informações abaixo para responder à pergunta do usuário.
+        system_prompt = f"""Você responde perguntas sobre espiritismo usando apenas o contexto.
+Escreva uma ou duas frases completas, em português. Não repita a pergunta nem as instruções.
+Se o contexto não bastar, responda apenas: Não encontrei essa informação nos livros indexados.
 
-Informações:
-{context_text}
-
-Instruções:
-- Responda de forma clara e concisa
-- Use o contexto fornecido
-- Quando citar passagens, mencione o livro e capítulo (se disponível)
-- Se não souber a resposta, diga que não encontrou na base de dados"""
-
-        prompt = f"{system_prompt}\n\nPergunta: {query}\nResposta:"
-        response = self._get_llm().generate(prompt, max_tokens=500)
-
-        return response
+Contexto:
+{context_text}"""
+        template = (
+            "<|start_header_id|>user<|end_header_id|>\n\n"
+            "{0}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+        )
+        with self._get_llm().chat_session(system_prompt=system_prompt, prompt_template=template):
+            response = self.llm.generate(query, max_tokens=160, temp=0.1)
+        return response.strip()
 
     def query(self, query: str, top_k: int = 3) -> Dict:
         """Query the RAG system."""

backend/app/tasks/celery_tasks.py

@@ -30,25 +30,47 @@ def save_schedules(schedules: List[Dict]):
     with open("/app/schedules.json", "w") as f:
         json.dump(schedules, f, indent=2)
 
+def schedule_is_due(schedule: Dict) -> bool:
+    raw = str(schedule.get("time", "")).replace("Z", "+00:00")
+    when = datetime.fromisoformat(raw)
+    if when.tzinfo is not None:
+        when = when.astimezone().replace(tzinfo=None)
+    return datetime.now() >= when
+
+
 @celery.task
 def send_scheduled_message(schedule_id: int = None):
-    """Send one schedule, or every enabled schedule when id is omitted."""
+    """Send enabled schedules whose time has arrived and that were not sent yet."""
     schedules = load_schedules()
-    if schedule_id is None:
-        selected = [item for item in schedules if item.get("enabled", True)]
-    else:
-        selected = [item for item in schedules if item.get("id") == schedule_id and item.get("enabled", True)]
+    due = []
+    for item in schedules:
+        if schedule_id is not None and item.get("id") != schedule_id:
+            continue
+        if not item.get("enabled", True) or item.get("sent"):
+            continue
+        try:
+            ready = schedule_is_due(item)
+        except ValueError:
+            print(f"Invalid schedule time: {item.get('time')}")
+            continue
+        if ready:
+            due.append(item)
 
-    if not selected:
-        print(f"Schedule {schedule_id} not found")
+    if not due:
+        print("No schedules due")
         return
 
-    for schedule in selected:
+    changed = False
+    for schedule in due:
         try:
             bot_service.send_scheduled_message(schedule)
+            schedule["sent"] = True
+            changed = True
             print(f"Sent scheduled message: {schedule['time']}")
         except Exception as e:
             print(f"Error sending scheduled message: {e}")
+    if changed:
+        save_schedules(schedules)
 
 @celery.task
 def generate_daily_content():

backend/app/utils/embeddings.py

@@ -1,9 +1,13 @@
 from sentence_transformers import SentenceTransformer
 import numpy as np
 
+_MODELS = {}
+
 class EmbeddingService:
-    def __init__(self, model_name="all-MiniLM-L6-v2"):
-        self.model = SentenceTransformer(model_name)
+    def __init__(self, model_name="paraphrase-multilingual-MiniLM-L12-v2"):
+        if model_name not in _MODELS:
+            _MODELS[model_name] = SentenceTransformer(model_name)
+        self.model = _MODELS[model_name]
         self.dimension = 384
 
     def embed(self, texts: list[str]) -> np.ndarray:

backend/app/utils/schedule_manager.py

@@ -23,6 +23,7 @@ class ScheduleManager:
         os.makedirs(os.path.dirname(self.schedules_path), exist_ok=True)
         with open(self.schedules_path, 'w') as f:
             json.dump(self.schedules, f, indent=2)
+            f.write("\n")
 
     def add_schedule(self, schedule: Dict) -> Dict:
         """Add a new schedule."""