diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..42f5bf2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.superpowers +PDF +**/*.gguf +**/__pycache__ +**/.venv +**/.env +models diff --git a/BUILD.md b/BUILD.md index 46c52cd..42951f6 100644 --- a/BUILD.md +++ b/BUILD.md @@ -174,7 +174,7 @@ docker-compose up -d After starting the services, index the 6 existing books: ```bash -docker-compose run --rm web python backend/scripts/index_books.py +docker compose run --rm web python scripts/index_books.py ``` This will: @@ -443,10 +443,10 @@ ls -lh PDF/ ```bash # Método 1: Via container -docker-compose run --rm web python backend/scripts/index_books.py +docker compose run --rm web python scripts/index_books.py # Método 2: Com limite de logs -docker-compose run --rm web python backend/scripts/index_books.py 2>&1 | tee index.log +docker compose run --rm web python scripts/index_books.py 2>&1 | tee index.log ``` ### 3. Monitorar Progresso @@ -485,7 +485,7 @@ rm -rf backend/index/* rm backend/schedules.json # Reindexar -docker-compose run --rm web python backend/scripts/index_books.py +docker compose run --rm web python scripts/index_books.py ``` ## 🎮 Uso da Aplicação @@ -750,7 +750,7 @@ ls -lh PDF/ chmod -R 755 PDF/ # Reindexar -docker-compose run --rm web python backend/scripts/index_books.py 2>&1 | tee reindex.log +docker compose run --rm web python scripts/index_books.py 2>&1 | tee reindex.log ``` ### Comandos de Diagnóstico diff --git a/IMPLEMENTATION_COMPLETE.md b/IMPLEMENTATION_COMPLETE.md index eae91ef..700194f 100644 --- a/IMPLEMENTATION_COMPLETE.md +++ b/IMPLEMENTATION_COMPLETE.md @@ -671,7 +671,7 @@ docker-compose build docker-compose up -d # Indexar livros -docker-compose run --rm web python backend/scripts/index_books.py +docker compose run --rm web python scripts/index_books.py ``` ### Produção - Single Server diff --git a/README.md b/README.md index 4db312c..220c364 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ docker-compose ps ### 5. Indexar os Livros ```bash -docker-compose run --rm web python backend/scripts/index_books.py +docker compose run --rm web python scripts/index_books.py ``` Isso irá: diff --git a/backend/Dockerfile b/backend/Dockerfile index 6e1d885..923e636 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,32 +2,27 @@ FROM python:3.11-slim WORKDIR /app -# Install system dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ && rm -rf /var/lib/apt/lists/* -# Create directories -RUN mkdir -p /app/uploads /app/index /app/models +RUN mkdir -p /app/uploads /app/index /app/models /app/books -# Copy requirements -COPY requirements.txt . - -# Install Python dependencies +COPY backend/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# Set environment variables -ENV GPT4ALL_MODEL=llama-3.2-3b-instruct-q4_0.gguf +ENV PYTHONPATH=/app +ENV GPT4ALL_MODEL=/app/models/llama-3.2-3b-instruct-q4_0.gguf ENV UPLOAD_DIR=/app/uploads +ENV BOOKS_DIR=/app/books ENV INDEX_DIR=/app/index ENV SCHEDULES_PATH=/app/schedules.json -# Copy application -COPY app/ ./app/ -COPY frontend/ ./frontend/ +COPY backend/app ./app +COPY backend/templates ./templates +COPY backend/scripts ./scripts +COPY frontend ./frontend -# Expose ports EXPOSE 8000 -# Run the application CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/app/main.py b/backend/app/main.py index b987c5b..2ee7833 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -1,24 +1,37 @@ -from fastapi import FastAPI, File, UploadFile, HTTPException -from fastapi.responses import HTMLResponse -from fastapi.staticfiles import StaticFiles -from fastapi.middleware.cors import CORSMiddleware -from app.services.rag_service import RAGService -from app.services.bot_service import BotService -from app.utils.schedule_manager import ScheduleManager -from app.models import ChatRequest, ChatResponse, BookInfo, UploadResponse, ScheduleInfo, ScheduleRequest -import os import json +import os from typing import List -# Initialize services -qdrant_service = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) -rag_service = RAGService(qdrant_service, model_name=os.getenv("GPT4ALL_MODEL", "llama-3.2-3b-instruct-q4_0.gguf")) +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import HTMLResponse +from fastapi.staticfiles import StaticFiles + +from app.models import ( + BookInfo, + ChatRequest, + ChatResponse, + ScheduleInfo, + ScheduleRequest, + UploadResponse, +) +from app.services.bot_service import BotService +from app.services.rag_service import QdrantService, RAGService +from app.utils.schedule_manager import ScheduleManager + +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +SCHEDULES_PATH = os.getenv("SCHEDULES_PATH", "/app/schedules.json") +UPLOAD_DIR = os.getenv("UPLOAD_DIR", "/app/uploads") +QDRANT_URL = os.getenv("QDRANT_URL", "http://qdrant:6333") +MODEL_PATH = os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf") + +qdrant_service: QdrantService | None = None +rag_service: RAGService | None = None bot_service = BotService(os.getenv("TELEGRAM_BOT_TOKEN", ""), os.getenv("TELEGRAM_CHAT_ID", "")) -schedule_manager = ScheduleManager(os.getenv("SCHEDULES_PATH", "/app/schedules.json")) +schedule_manager = ScheduleManager(SCHEDULES_PATH) app = FastAPI(title="Espiritismo RAG API") -# CORS middleware app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -27,128 +40,125 @@ app.add_middleware( allow_headers=["*"], ) -# Mount static files -app.mount("/static", StaticFiles(directory="../frontend/static"), name="static") +app.mount( + "/static", + StaticFiles(directory=os.path.join(BASE_DIR, "frontend", "static")), + name="static", +) + + +def get_qdrant() -> QdrantService: + global qdrant_service + if qdrant_service is None: + qdrant_service = QdrantService(QDRANT_URL) + return qdrant_service + + +def get_rag() -> RAGService: + global rag_service + if rag_service is None: + rag_service = RAGService(get_qdrant(), model_name=MODEL_PATH) + return rag_service + -# Load existing schedules from file def load_schedules(): - if os.path.exists("/app/schedules.json"): - with open("/app/schedules.json", "r") as f: - return json.load(f) + if os.path.isfile(SCHEDULES_PATH): + with open(SCHEDULES_PATH, "r", encoding="utf-8") as handle: + data = json.load(handle) + return data if isinstance(data, list) else [] return [] + +def save_schedules(schedules: list): + parent = os.path.dirname(SCHEDULES_PATH) + if parent: + os.makedirs(parent, exist_ok=True) + with open(SCHEDULES_PATH, "w", encoding="utf-8") as handle: + json.dump(schedules, handle, indent=2) + + +@app.get("/health") +async def health(): + return {"status": "ok"} + + @app.get("/", response_class=HTMLResponse) async def index(): - """Serve the index page.""" - with open("../backend/templates/index.html", "r") as f: - return f.read() + template = os.path.join(BASE_DIR, "templates", "index.html") + with open(template, "r", encoding="utf-8") as handle: + return handle.read() + @app.get("/api/books", response_model=List[BookInfo]) async def get_books(): - """Get list of books in the collection.""" - books = qdrant_service.get_books() + books = get_qdrant().get_books() return [{"title": book, "source": book} for book in books] + @app.post("/api/chat", response_model=ChatResponse) async def chat(request: ChatRequest): - """Query the RAG system.""" try: - result = rag_service.query(request.message, request.top_k) - return result - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return get_rag().query(request.message, request.top_k) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + @app.post("/api/upload", response_model=UploadResponse) async def upload_pdf(file: UploadFile = File(...)): - """Upload a new PDF file.""" - upload_dir = "/app/uploads" - - if not file.filename.endswith('.pdf'): + if not file.filename or not file.filename.endswith(".pdf"): raise HTTPException(status_code=400, detail="Only PDF files are allowed") - file_path = os.path.join(upload_dir, file.filename) - + os.makedirs(UPLOAD_DIR, exist_ok=True) + file_path = os.path.join(UPLOAD_DIR, os.path.basename(file.filename)) try: - with open(file_path, "wb") as f: - content = await file.read() - f.write(content) - + content = await file.read() + with open(file_path, "wb") as handle: + handle.write(content) return UploadResponse(message="File uploaded successfully", filename=file.filename) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + except Exception as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + @app.get("/api/schedules", response_model=List[ScheduleInfo]) async def get_schedules(): - """Get all schedules.""" - schedules = load_schedules() - return [ScheduleInfo(**s) for s in schedules] + return [ScheduleInfo(**item) for item in load_schedules()] + @app.post("/api/schedules", response_model=ScheduleInfo) async def create_schedule(request: ScheduleRequest): - """Create a new schedule.""" schedules = load_schedules() - new_schedule = { - "id": len(schedules) + 1, - "time": request.time, - "book": request.book, - "enabled": request.enabled, - } - schedules.append(new_schedule) - - with open("/app/schedules.json", "w") as f: - json.dump(schedules, f, indent=2) - - # Start celery beat if needed + new_schedule = schedule_manager.add_schedule(request.model_dump()) + if not any(item.get("id") == new_schedule["id"] for item in schedules): + schedules.append(new_schedule) + save_schedules(schedules) try: bot_service.add_schedule(new_schedule) - except Exception as e: - print(f"Warning: Could not add schedule to bot: {e}") - + except Exception as exc: + print(f"Warning: Could not add schedule to bot: {exc}") return ScheduleInfo(**new_schedule) + @app.put("/api/schedules/{schedule_id}", response_model=ScheduleInfo) async def update_schedule(schedule_id: int, request: ScheduleRequest): - """Update an existing schedule.""" - schedules = load_schedules() + updated = schedule_manager.update_schedule(schedule_id, request.model_dump()) + if updated is None: + raise HTTPException(status_code=404, detail="Schedule not found") + save_schedules(schedule_manager.list_schedules()) + try: + bot_service.update_schedule(updated) + except Exception as exc: + print(f"Warning: Could not update schedule in bot: {exc}") + return ScheduleInfo(**updated) - for i, s in enumerate(schedules): - if s["id"] == schedule_id: - schedules[i] = { - "id": schedule_id, - "time": request.time, - "book": request.book, - "enabled": request.enabled, - } - - with open("/app/schedules.json", "w") as f: - json.dump(schedules, f, indent=2) - - try: - bot_service.update_schedule(schedules[i]) - except Exception as e: - print(f"Warning: Could not update schedule in bot: {e}") - - return ScheduleInfo(**schedules[i]) - - raise HTTPException(status_code=404, detail="Schedule not found") @app.delete("/api/schedules/{schedule_id}") async def delete_schedule(schedule_id: int): - """Delete a schedule.""" - schedules = load_schedules() - - for i, s in enumerate(schedules): - if s["id"] == schedule_id: - deleted = schedules.pop(i) - - with open("/app/schedules.json", "w") as f: - json.dump(schedules, f, indent=2) - - try: - bot_service.remove_schedule(deleted) - except Exception as e: - print(f"Warning: Could not remove schedule from bot: {e}") - - return {"message": "Schedule deleted"} - - raise HTTPException(status_code=404, detail="Schedule not found") + current = schedule_manager.get_schedule(schedule_id) + if current is None or not schedule_manager.delete_schedule(schedule_id): + raise HTTPException(status_code=404, detail="Schedule not found") + save_schedules(schedule_manager.list_schedules()) + try: + bot_service.remove_schedule(current) + except Exception as exc: + print(f"Warning: Could not remove schedule from bot: {exc}") + return {"message": "Schedule deleted"} diff --git a/backend/app/models.py b/backend/app/models.py deleted file mode 100644 index d2b1e19..0000000 --- a/backend/app/models.py +++ /dev/null @@ -1,31 +0,0 @@ -from pydantic import BaseModel -from typing import Optional, List, Dict - -class ChatRequest(BaseModel): - message: str - top_k: int = 3 - -class ChatResponse(BaseModel): - query: str - response: str - sources: List[Dict] - -class BookInfo(BaseModel): - title: str - source: str - -class UploadResponse(BaseModel): - message: str - filename: str - -class ScheduleInfo(BaseModel): - id: int - time: str - book: Optional[str] - enabled: bool - next_run: Optional[str] = None - -class ScheduleRequest(BaseModel): - time: str - book: Optional[str] = None - enabled: bool = True diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 3d38a0a..88940e3 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1 +1,38 @@ -# Pydantic models package +from typing import Dict, List, Optional + +from pydantic import BaseModel + + +class ChatRequest(BaseModel): + message: str + top_k: int = 3 + + +class ChatResponse(BaseModel): + query: str + response: str + sources: List[Dict] + + +class BookInfo(BaseModel): + title: str + source: str + + +class UploadResponse(BaseModel): + message: str + filename: str + + +class ScheduleInfo(BaseModel): + id: int + time: str + book: Optional[str] = None + enabled: bool + next_run: Optional[str] = None + + +class ScheduleRequest(BaseModel): + time: str + book: Optional[str] = None + enabled: bool = True diff --git a/backend/services/bot_service.py b/backend/app/services/bot_service.py similarity index 83% rename from backend/services/bot_service.py rename to backend/app/services/bot_service.py index 4ea7320..d3d870d 100644 --- a/backend/services/bot_service.py +++ b/backend/app/services/bot_service.py @@ -1,25 +1,16 @@ -import telebot -from telebot import types -import os import json -from datetime import datetime, timedelta -from typing import List, Dict, Optional -from celery import Celery +import os import random +from typing import Dict, List -celery = Celery( - 'tasks', - broker=os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0"), - backend=os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0"), -) +import telebot class BotService: def __init__(self, token: str, chat_id: str): - self.bot = telebot.TeleBot(token) self.chat_id = chat_id - - # Register handlers - self._register_handlers() + self.bot = telebot.TeleBot(token) if token else None + if self.bot is not None: + self._register_handlers() # Store pending schedules self.pending_schedules: List[Dict] = [] @@ -81,11 +72,13 @@ Direcione uma pergunta sobre espiritismo para eu responder! def _query_rag_service(self, question: str, chat_id: int): """Query RAG service and send response.""" try: - from app.services.rag_service import RAGService - from app.services.qdrant_service import QdrantService + from app.services.rag_service import QdrantService, RAGService qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) - rag = RAGService(qdrant, model_name=os.getenv("GPT4ALL_MODEL", "llama-3.2-3b-instruct-q4_0.gguf")) + rag = RAGService( + qdrant, + model_name=os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf"), + ) result = rag.query(question, top_k=3) @@ -160,14 +153,23 @@ Direcione uma pergunta sobre espiritismo para eu responder! def start_polling(self): """Start the bot polling.""" + if self.bot is None: + print("TELEGRAM_BOT_TOKEN ausente; polling não iniciado") + return print("Starting Telegram bot polling...") self.bot.infinity_polling(timeout=10, long_polling_timeout=5) - @celery.task def send_scheduled_message(self, schedule: Dict): """Send scheduled message to Telegram.""" try: - response = rag.query(schedule.get('book', ''), top_k=3) + from app.services.rag_service import QdrantService, RAGService + + qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) + rag = RAGService( + qdrant, + model_name=os.getenv("GPT4ALL_MODEL", "/app/models/llama-3.2-3b-instruct-q4_0.gguf"), + ) + response = rag.query(schedule.get('book') or 'espiritismo', top_k=3) message = f""" ⏰ Mensagem Agendada - {schedule.get('book', 'Todos os livros')} @@ -176,11 +178,13 @@ Direcione uma pergunta sobre espiritismo para eu responder! 📚 Fontes: {self._format_sources(response['sources'])} """ + if self.bot is None: + print("TELEGRAM_BOT_TOKEN ausente; mensagem agendada não enviada") + return self.bot.send_message(self.chat_id, message) except Exception as e: print(f"Error sending scheduled message: {e}") - @celery.task def generate_daily_content(self): """Generate daily content for all books.""" books = [ @@ -204,4 +208,7 @@ Direcione uma pergunta sobre espiritismo para eu responder! Este conteúdo foi gerado automaticamente pelo Sistema RAG Espiritismo. """ + if self.bot is None: + print("TELEGRAM_BOT_TOKEN ausente; conteúdo diário não enviado") + return self.bot.send_message(self.chat_id, message) diff --git a/backend/services/pdf_processor.py b/backend/app/services/pdf_processor.py similarity index 96% rename from backend/services/pdf_processor.py rename to backend/app/services/pdf_processor.py index f6b2752..db88520 100644 --- a/backend/services/pdf_processor.py +++ b/backend/app/services/pdf_processor.py @@ -2,7 +2,7 @@ from langchain_community.document_loaders import PyPDFLoader from langchain_text_splitters import RecursiveCharacterTextSplitter import os from typing import List, Dict -from .embeddings import EmbeddingService +from app.utils.embeddings import EmbeddingService class PDFProcessor: def __init__(self, embedding_service: EmbeddingService): diff --git a/backend/services/rag_service.py b/backend/app/services/rag_service.py similarity index 72% rename from backend/services/rag_service.py rename to backend/app/services/rag_service.py index c0d8235..2e70522 100644 --- a/backend/services/rag_service.py +++ b/backend/app/services/rag_service.py @@ -4,6 +4,7 @@ import numpy as np from typing import List, Dict, Optional import os import json +import time class QdrantService: def __init__(self, url: str, collection_name: str = "espiritismo_books"): @@ -15,7 +16,17 @@ class QdrantService: def _ensure_collection(self): """Create collection if it doesn't exist.""" - collections = self.client.get_collections().collections + last_error = None + for attempt in range(15): + try: + collections = self.client.get_collections().collections + break + except Exception as exc: + last_error = exc + print(f"Qdrant indisponível ({attempt + 1}/15): {exc}") + time.sleep(2) + else: + raise RuntimeError(f"Não foi possível conectar ao Qdrant: {last_error}") collection_exists = any( c.name == self.collection_name for c in collections @@ -39,7 +50,7 @@ class QdrantService: for i, chunk in enumerate(chunks): point = PointStruct( - id=i, + id=chunk.get("id") or i, vector=chunk["embedding"], payload={ "content": chunk["content"], @@ -55,8 +66,10 @@ class QdrantService: ) print(f"Added {len(points)} chunks to Qdrant") - def search(self, query_embedding: List[float], top_k: int = 3) -> List[Dict]: + def search(self, query_embedding, top_k: int = 3) -> List[Dict]: """Search for similar chunks.""" + if hasattr(query_embedding, "tolist"): + query_embedding = query_embedding.tolist() results = self.client.search( collection_name=self.collection_name, query_vector=query_embedding, @@ -100,11 +113,26 @@ class RAGService: def __init__(self, qdrant_service: QdrantService, model_name: str = "llama-3.2-3b-instruct-q4_0.gguf"): self.qdrant = qdrant_service self.model_name = model_name + self.llm = None - # Setup GPT4All - from gpt4all import GPT4All + def _get_llm(self): + if self.llm is None: + if not os.path.isfile(self.model_name): + raise FileNotFoundError( + f"Modelo não encontrado em {self.model_name}. " + "Baixe llama-3.2-3b-instruct-q4_0.gguf para backend/models/." + ) + from gpt4all import GPT4All - self.llm = GPT4All(model_name=model_name, device="cpu") + model_dir = os.path.dirname(self.model_name) or "." + model_file = os.path.basename(self.model_name) + self.llm = GPT4All( + model_name=model_file, + model_path=model_dir, + device="cpu", + allow_download=False, + ) + return self.llm def generate_response(self, query: str, context: List[str], top_k: int = 3) -> str: """Generate a response based on the query and context.""" @@ -121,11 +149,8 @@ Instruções: - Quando citar passagens, mencione o livro e capítulo (se disponível) - Se não souber a resposta, diga que não encontrou na base de dados""" - response = self.llm.generate( - system_prompt, - query, - max_tokens=500, - ) + prompt = f"{system_prompt}\n\nPergunta: {query}\nResposta:" + response = self._get_llm().generate(prompt, max_tokens=500) return response @@ -149,5 +174,8 @@ Instruções: "query": query, "response": response, "context": context, - "sources": [chunk["metadata"] for chunk in chunks], + "sources": [ + {"metadata": chunk["metadata"], "score": chunk["score"]} + for chunk in chunks + ], } diff --git a/backend/tasks/celery_tasks.py b/backend/app/tasks/celery_tasks.py similarity index 82% rename from backend/tasks/celery_tasks.py rename to backend/app/tasks/celery_tasks.py index dba3edc..69e7afb 100644 --- a/backend/tasks/celery_tasks.py +++ b/backend/app/tasks/celery_tasks.py @@ -31,17 +31,19 @@ def save_schedules(schedules: List[Dict]): json.dump(schedules, f, indent=2) @celery.task -def send_scheduled_message(schedule_id: int): - """Send scheduled message to Telegram.""" +def send_scheduled_message(schedule_id: int = None): + """Send one schedule, or every enabled schedule when id is omitted.""" schedules = load_schedules() - schedule = next((s for s in schedules if s['id'] == schedule_id), None) + if schedule_id is None: + selected = [item for item in schedules if item.get("enabled", True)] + else: + selected = [item for item in schedules if item.get("id") == schedule_id and item.get("enabled", True)] - if not schedule: + if not selected: print(f"Schedule {schedule_id} not found") return - if schedule.get('enabled', True): - # Send to bot service + for schedule in selected: try: bot_service.send_scheduled_message(schedule) print(f"Sent scheduled message: {schedule['time']}") diff --git a/backend/utils/embeddings.py b/backend/app/utils/embeddings.py similarity index 100% rename from backend/utils/embeddings.py rename to backend/app/utils/embeddings.py diff --git a/backend/utils/schedule_manager.py b/backend/app/utils/schedule_manager.py similarity index 100% rename from backend/utils/schedule_manager.py rename to backend/app/utils/schedule_manager.py diff --git a/backend/models/.gitkeep b/backend/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/requirements.txt b/backend/requirements.txt index 39a5429..03141bc 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,10 +2,9 @@ fastapi==0.115.0 uvicorn[standard]==0.32.0 pydantic==2.9.2 python-multipart==0.0.9 -gpt4all==2.4.6 +gpt4all==2.8.2 langchain==0.3.7 langchain-community==0.3.5 -langchain-experimental==0.0.73 sentence-transformers==2.7.0 pytelegrambotapi==4.26.0 celery==5.4.0 @@ -14,3 +13,5 @@ python-dotenv==1.0.1 python-dateutil==2.9.0.post0 aiofiles==24.1.0 qdrant-client==1.11.1 +pypdf==5.1.0 +langchain-text-splitters==0.3.2 diff --git a/backend/scripts/index_books.py b/backend/scripts/index_books.py index f8d23bc..cf0b4d6 100644 --- a/backend/scripts/index_books.py +++ b/backend/scripts/index_books.py @@ -1,51 +1,67 @@ -from app.services.pdf_processor import PDFProcessor -from app.utils.embeddings import EmbeddingService -from app.utils.schedule_manager import ScheduleManager -import os import json +import os +import uuid + +from app.services.pdf_processor import PDFProcessor +from app.services.rag_service import QdrantService +from app.utils.embeddings import EmbeddingService + + +def book_dirs() -> list[str]: + paths = [ + os.getenv("BOOKS_DIR", "/app/books"), + os.getenv("UPLOAD_DIR", "/app/uploads"), + ] + seen = [] + for path in paths: + if path and path not in seen and os.path.isdir(path): + seen.append(path) + return seen + def main(): - # Setup services embedding_service = EmbeddingService() pdf_processor = PDFProcessor(embedding_service) + qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333")) - # Load books from PDF directory - pdf_dir = "/app/uploads" - books = os.listdir(pdf_dir) - books = [f for f in books if f.endswith('.pdf')] + books = [] + for directory in book_dirs(): + for name in sorted(os.listdir(directory)): + if name.lower().endswith(".pdf"): + books.append(os.path.join(directory, name)) print(f"Found {len(books)} books") + os.makedirs(os.getenv("INDEX_DIR", "/app/index"), exist_ok=True) - # Schedule manager - schedule_manager = ScheduleManager("/app/schedules.json") - - # Process each book - for book_name in books: - book_path = os.path.join(pdf_dir, book_name) + for book_path in books: + book_name = os.path.basename(book_path) book_metadata = { "title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "), "author": "Desconhecido", "type": "book", } - print(f"Processing: {book_name}") - - # Load and split PDF chunks = pdf_processor.load_and_split(book_path, book_metadata) - - # Embed chunks chunks_with_embeddings = pdf_processor.embed_chunks(chunks) + for chunk in chunks_with_embeddings: + chunk["id"] = str(uuid.uuid4()) - # Save chunks to JSON for reference - chunks_file = f"/app/index/{book_name}.json" - os.makedirs("/app/index", exist_ok=True) + qdrant.add_chunks(chunks_with_embeddings) - with open(chunks_file, 'w') as f: - json.dump(chunks_with_embeddings, f, indent=2) - - print(f"Saved {len(chunks)} chunks to {chunks_file}") + chunks_file = os.path.join(os.getenv("INDEX_DIR", "/app/index"), f"{book_name}.json") + with open(chunks_file, "w", encoding="utf-8") as handle: + json.dump( + [ + {"content": chunk["content"], "metadata": chunk["metadata"]} + for chunk in chunks_with_embeddings + ], + handle, + ensure_ascii=False, + ) + print(f"Indexed {len(chunks_with_embeddings)} chunks from {book_name}") print("Indexing complete!") + if __name__ == "__main__": main() diff --git a/data/schedules.json b/data/schedules.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/data/schedules.json @@ -0,0 +1 @@ +[] diff --git a/docker-compose.yml b/docker-compose.yml index 9e7cbe6..d2def15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,6 @@ -version: '3.8' - services: qdrant: - image: qdrant/qdrant:latest + image: docker.io/qdrant/qdrant:latest container_name: espiritismo-qdrant ports: - "6333:6333" @@ -11,7 +9,7 @@ services: restart: unless-stopped redis: - image: redis:7-alpine + image: docker.io/library/redis:7-alpine container_name: espiritismo-redis ports: - "6379:6379" @@ -20,52 +18,71 @@ services: restart: unless-stopped web: - build: ./backend + build: + context: . + dockerfile: backend/Dockerfile container_name: espiritismo-web ports: - "8000:8000" volumes: - - ./PDF:/app/uploads:ro - - ./backend/schedules.json:/app/schedules.json + - ./PDF:/app/books:ro + - uploads_data:/app/uploads + - ./data/schedules.json:/app/schedules.json + - ./backend/models:/app/models environment: - - QDRANT_URL=http://qdrant:6333 - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} - - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} + QDRANT_URL: http://qdrant:6333 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} + TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-} + GPT4ALL_MODEL: /app/models/llama-3.2-3b-instruct-q4_0.gguf + BOOKS_DIR: /app/books + UPLOAD_DIR: /app/uploads + SCHEDULES_PATH: /app/schedules.json depends_on: - qdrant - redis restart: unless-stopped celery: - build: ./backend + build: + context: . + dockerfile: backend/Dockerfile container_name: espiritismo-celery command: celery -A app.tasks.celery_tasks worker --loglevel=info volumes: - - ./PDF:/app/uploads:ro - - ./backend/schedules.json:/app/schedules.json + - ./PDF:/app/books:ro + - uploads_data:/app/uploads + - ./data/schedules.json:/app/schedules.json + - ./backend/models:/app/models environment: - - QDRANT_URL=http://qdrant:6333 - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} - - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} + QDRANT_URL: http://qdrant:6333 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} + TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-} + GPT4ALL_MODEL: /app/models/llama-3.2-3b-instruct-q4_0.gguf + BOOKS_DIR: /app/books + UPLOAD_DIR: /app/uploads + SCHEDULES_PATH: /app/schedules.json depends_on: - qdrant - redis restart: unless-stopped celery-beat: - build: ./backend + build: + context: . + dockerfile: backend/Dockerfile command: celery -A app.tasks.celery_tasks beat --loglevel=info volumes: - - ./backend/schedules.json:/app/schedules.json + - ./data/schedules.json:/app/schedules.json environment: - - CELERY_BROKER_URL=redis://redis:6379/0 - - CELERY_RESULT_BACKEND=redis://redis:6379/0 - - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN} - - TELEGRAM_CHAT_ID=${TELEGRAM_CHAT_ID} + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} + TELEGRAM_CHAT_ID: ${TELEGRAM_CHAT_ID:-} + SCHEDULES_PATH: /app/schedules.json depends_on: - redis restart: unless-stopped @@ -73,3 +90,4 @@ services: volumes: qdrant_data: redis_data: + uploads_data: