diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..254d45a --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + OMP_NUM_THREADS=2 \ + TOKENIZERS_PARALLELISM=false + +# tesseract-ocr + pacote de idioma português + libs de imagem do MuPDF +RUN apt-get update && apt-get install -y --no-install-recommends \ + tesseract-ocr \ + tesseract-ocr-por \ + tesseract-ocr-eng \ + curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt requirements-dev.txt ./ +RUN pip install --no-cache-dir -r requirements-dev.txt + +COPY app ./app +COPY tests ./tests + +RUN mkdir -p /data/vector_db /data/pdfs + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://localhost:8000/api/health || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e64e772 --- /dev/null +++ b/backend/app/__init__.py @@ -0,0 +1,3 @@ +"""Backend do Pastor Online.""" + +__version__ = "1.0.0" diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..718058e --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,83 @@ +"""Configuração do backend via variáveis de ambiente.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Configuração lida do ambiente ou do arquivo .env.""" + + model_config = SettingsConfigDict( + env_file=(".env", "../.env"), + env_file_encoding="utf-8", + extra="ignore", + ) + + # Caminhos + pdf_dir: Path = Path("/data/pdfs") + vector_db_dir: Path = Path("/data/vector_db") + + # Provedor de LLM: ollama | openai | groq | anthropic | stub + llm_provider: str = "ollama" + ollama_base_url: str = "http://ollama:11434" + ollama_model: str = "qwen2.5:1.5b" + openai_base_url: str = "https://api.openai.com/v1" + openai_model: str = "gpt-4o-mini" + openai_api_key: str = "" + groq_base_url: str = "https://api.groq.com/openai/v1" + groq_model: str = "llama-3.3-70b-versatile" + groq_api_key: str = "" + anthropic_base_url: str = "https://api.anthropic.com" + anthropic_model: str = "claude-sonnet-4-5" + anthropic_api_key: str = "" + + # Embeddings + embedding_model: str = "paraphrase-multilingual-MiniLM-L12-v2" + embedding_batch_size: int = 32 + + # Busca + top_k: int = 6 + min_similarity: float = 0.35 + collection_name: str = "biblia" + + # OCR + ocr_dpi: int = 200 + ocr_language: str = "por+eng" + ocr_timeout: int = 120 + ocr_workers: int = 4 + min_legible_ratio: float = 0.35 + + # API + cors_origins: str = "http://localhost:8080,http://127.0.0.1:8080" + reindex_token: str = "" + max_message_length: int = 4000 + max_history_turns: int = 10 + + # Limites de ingestão + max_pdf_bytes: int = 200 * 1024 * 1024 + + @property + def ocr_cache_dir(self) -> Path: + return self.vector_db_dir / "ocr_cache" + + @property + def manifest_path(self) -> Path: + return self.vector_db_dir / "manifest.json" + + @property + def book_index_path(self) -> Path: + return self.vector_db_dir / "book_index.json" + + @property + def cors_origin_list(self) -> list[str]: + return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()] + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + """Devolve as settings singleton do processo.""" + return Settings() diff --git a/backend/app/extraction.py b/backend/app/extraction.py new file mode 100644 index 0000000..b526506 --- /dev/null +++ b/backend/app/extraction.py @@ -0,0 +1,213 @@ +"""Extração de texto de PDFs com fallback para OCR. + +A camada textual de muitos PDFs de Bíblia usa fontes sem mapeamento Unicode: o +extrator devolve sequências de caracteres de controle em vez de letras. Detectamos +esse caso e rasterizamos a página para o Tesseract. O resultado do OCR é cacheado +por (sha256 do arquivo, página) para que reindexações não repitam horas de OCR. +""" + +from __future__ import annotations + +import hashlib +import logging +import shutil +import subprocess +import tempfile +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +logger = logging.getLogger(__name__) + +try: # pragma: no cover - dependência real do ambiente + import pymupdf +except ImportError: # pragma: no cover - permite rodar testes sem MuPDF + pymupdf = None # type: ignore[assignment] + + +class OcrError(RuntimeError): + """Falha ao executar o Tesseract.""" + + +@dataclass(frozen=True) +class PageText: + """Texto extraído de uma página do PDF.""" + + page: int + text: str + failed: bool = False + + +def legible_ratio(text: str) -> float: + """Proporção de caracteres imprimíveis legíveis no texto. + + Um texto normal de Bíblia fica acima de 0.9; a camada corrompida do PDF real + fica abaixo de 0.05. Espaços em branco não contam como denominador. + """ + if not text: + return 0.0 + meaningful = [ch for ch in text if not ch.isspace()] + if not meaningful: + return 0.0 + legible = sum(1 for ch in meaningful if _is_legible_char(ch)) + return legible / len(meaningful) + + +def _is_legible_char(ch: str) -> bool: + """Caractere que um humano consegue ler.""" + if unicodedata.category(ch).startswith("C"): + # C0/C1 de controle: sinal de fonte sem ToUnicode. + return False + return ch.isalnum() or unicodedata.category(ch).startswith(("L", "N", "P", "S", "Z")) + + +def _is_blank_pixmap(pixmap) -> bool: + """Diz se a página rasterizada está visualmente em branco. + + Usa as amostras do pixmap: uma página de texto tem variância alta entre + pixels; uma página em branco tem todos os pixels iguais. + """ + try: + samples = pixmap.samples + stride = max(1, pixmap.n if getattr(pixmap, "n", 1) else 1) + total = pixmap.width * pixmap.height + if total == 0: + return True + step = max(1, (total * stride) // 4000) + values = bytes(samples[0 : len(samples) : step]) + if not values: + return True + return len(set(values)) <= 1 + except Exception: # pragma: no cover - formatos de pixmap inesperados + return False + + +def file_sha256(path: Path, chunk_size: int = 1024 * 1024) -> str: + """SHA-256 do conteúdo do arquivo.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(chunk_size): + digest.update(chunk) + return digest.hexdigest() + + +class PdfExtractor: + """Extrai texto de PDFs página a página, com OCR como fallback.""" + + def __init__( + self, + cache_dir: Path, + dpi: int = 200, + ocr_lang: str = "por+eng", + ocr_timeout: int = 120, + min_legible_ratio: float = 0.35, + pymupdf_module=None, + ): + self.cache_dir = Path(cache_dir) + self.dpi = dpi + self.ocr_lang = ocr_lang + self.ocr_timeout = ocr_timeout + self.min_legible_ratio = min_legible_ratio + self._pymupdf = pymupdf_module if pymupdf_module is not None else pymupdf + + def extract_document( + self, path: Path, file_hash: str | None = None + ) -> Iterator[PageText]: + """Extrai o texto de todas as páginas do PDF.""" + if self._pymupdf is None: # pragma: no cover + raise OcrError("PyMuPDF não está disponível") + + path = Path(path) + file_hash = file_hash or file_sha256(path) + document = self._pymupdf.open(path) + try: + for index in range(len(document)): + page = document[index] + yield self.extract_page(page, file_hash, index + 1) + finally: + document.close() + + def extract_page(self, page, file_hash: str, page_number: int) -> PageText: + """Extrai o texto de uma página, usando cache quando disponível.""" + cache_file = self._cache_path(file_hash, page_number) + if cache_file.exists(): + return PageText(page=page_number, text=cache_file.read_text(encoding="utf-8")) + + try: + text = page.get_text("text") or "" + except Exception as exc: # pragma: no cover - PDF malformado + logger.warning("Falha ao extrair texto da página %s: %s", page_number, exc) + text = "" + + failed = False + if legible_ratio(text) < self.min_legible_ratio: + text, failed = self._ocr_page(page) + + self._write_cache(cache_file, text) + return PageText(page=page_number, text=text, failed=failed) + + def _ocr_page(self, page) -> tuple[str, bool]: + """Rasteriza a página e aplica OCR. Devolve (texto, falhou). + + Uma página visualmente em branco não é falha: o documento pode ter + separadores. Ela é detectada pela rasterização e não gasta OCR. + Falha é o OCR que não pôde ser executado. + """ + try: + pixmap = page.get_pixmap(dpi=self.dpi) + except Exception as exc: + logger.warning("Falha ao rasterizar página para OCR: %s", exc) + return "", True + + if _is_blank_pixmap(pixmap): + return "", False + + try: + text = self._run_tesseract(pixmap, self.ocr_lang, self.ocr_timeout) + except OcrError as exc: + logger.warning("OCR falhou: %s", exc) + return "", True + + return text, False + + def _run_tesseract(self, pixmap, language: str, timeout: int) -> str: + """Executa o Tesseract sobre o pixmap e devolve o texto.""" + if shutil.which("tesseract") is None: + raise OcrError("tesseract não está instalado no sistema") + + with tempfile.TemporaryDirectory() as tmp: + image_path = Path(tmp) / "page.png" + pixmap.save(str(image_path)) + try: + completed = subprocess.run( + [ + "tesseract", + str(image_path), + "stdout", + "-l", + language, + "--psm", + "6", + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise OcrError(f"tesseract excedeu o timeout de {timeout}s") from exc + + if completed.returncode != 0: + raise OcrError(f"tesseract falhou: {completed.stderr.strip()[:200]}") + return completed.stdout + + def _cache_path(self, file_hash: str, page_number: int) -> Path: + return self.cache_dir / file_hash / f"{page_number:05d}.txt" + + def _write_cache(self, cache_file: Path, text: str) -> None: + try: + cache_file.parent.mkdir(parents=True, exist_ok=True) + cache_file.write_text(text, encoding="utf-8") + except OSError as exc: # pragma: no cover - disco cheio / permissões + logger.warning("Não foi possível gravar cache de OCR em %s: %s", cache_file, exc) diff --git a/backend/app/models.py b/backend/app/models.py new file mode 100644 index 0000000..9024d33 --- /dev/null +++ b/backend/app/models.py @@ -0,0 +1,55 @@ +"""Schemas Pydantic de entrada e saída da API.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, Field + + +class ChatMessage(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(min_length=1, max_length=8000) + + +class ChatRequest(BaseModel): + message: str = Field(min_length=1, max_length=4000) + history: list[ChatMessage] = Field(default_factory=list) + + +class Source(BaseModel): + """Trecho bíblico recuperado, exibido no accordion de fontes.""" + + reference: str + text: str + book: str + chapter: int + verse: int + page: int + file: str + score: float = 0.0 + + +class StatusResponse(BaseModel): + state: Literal["ready", "indexing", "error"] + pdfs_total: int = 0 + pdfs_indexed: int = 0 + chunks: int = 0 + model: str = "" + last_indexed_at: str | None = None + errors: list[str] = Field(default_factory=list) + + +class HealthResponse(BaseModel): + status: Literal["ok"] = "ok" + llm_provider: str + indexing: Literal["ready", "indexing", "error"] + version: str = "1.0.0" + + +class ReindexResponse(BaseModel): + state: Literal["indexing"] = "indexing" + + +class SourcesResponse(BaseModel): + sources: list[Source] = Field(default_factory=list) diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 0000000..135ac86 --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==8.3.4 +pytest-asyncio==0.25.0 diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..2b10254 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +pydantic-settings==2.7.0 +httpx==0.28.1 +pymupdf==1.25.1 +chromadb==1.5.9 +sentence-transformers==5.2.0 +numpy==2.2.1 diff --git a/backend/tests/test_extraction.py b/backend/tests/test_extraction.py new file mode 100644 index 0000000..dc23449 --- /dev/null +++ b/backend/tests/test_extraction.py @@ -0,0 +1,249 @@ +"""Testes do pipeline de extração de texto de PDF. + +Cobre a detecção de camada textual corrompida (o PDF real do projeto), o fallback +para OCR, o cache por página e o isolamento de falhas de OCR. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.extraction import OcrError, PdfExtractor, legible_ratio # noqa: E402 + + +class FakePage: + """Página de PDF simulada.""" + + def __init__(self, text: str = "", pixmap=None): + self._text = text + self._pixmap = pixmap or FakePixmap() + + def get_text(self, kind: str) -> str: + assert kind == "text" + return self._text + + def get_pixmap(self, dpi: int = 200): + return self._pixmap + + +class FakeDocument: + def __init__(self, pages): + self._pages = pages + self.closed = False + + def __len__(self): + return len(self._pages) + + def __getitem__(self, index): + return self._pages[index] + + def __iter__(self): + return iter(self._pages) + + def close(self): + self.closed = True + + +class FakePixmap: + def __init__(self, width=10, height=10, blank=False): + self.width = width + self.height = height + self.n = 3 + self.blank = blank + self.saved_to: list[Path] = [] + + @property + def samples(self) -> bytes: + if self.blank: + return bytes([255]) * (self.width * self.height * self.n) + return bytes(range(self.width * self.height * self.n))[: self.width * self.height * self.n] + + def save(self, path): + self.saved_to.append(Path(path)) + Path(path).write_bytes(b"PNG") + + +class FakePyMuPDF: + """Dublê do módulo pymupdf para testes.""" + + def __init__(self, document): + self._document = document + self.opened: list[Path] = [] + + def open(self, path): + self.opened.append(Path(path)) + return self._document + + +@pytest.fixture +def tmp_cache(tmp_path): + return tmp_path / "ocr_cache" + + +def test_legible_ratio_is_high_for_normal_text(): + text = "No princípio criou Deus os céus e a terra. E a terra era sem forma e vazia." + assert legible_ratio(text) > 0.9 + + +def test_legible_ratio_is_low_for_control_characters(): + text = "".join(chr(i) for i in range(1, 32)) + assert legible_ratio(text) < 0.35 + + +def test_legible_ratio_of_empty_text_is_zero(): + assert legible_ratio("") == 0.0 + assert legible_ratio(" \n\t ") == 0.0 + + +def test_legible_text_does_not_trigger_ocr(tmp_cache, tmp_path, monkeypatch): + pdf = tmp_path / "biblia.pdf" + pdf.write_bytes(b"%PDF-1.3") + doc = FakeDocument([FakePage(text="1 No princípio criou Deus os céus e a terra.")]) + extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc)) + + def fail_ocr(*args, **kwargs): + raise AssertionError("OCR não deveria ser chamado para texto legível") + + monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr) + + pages = list(extractor.extract_document(pdf)) + + assert len(pages) == 1 + assert "No princípio" in pages[0].text + assert pages[0].page == 1 + + +def test_corrupted_text_layer_triggers_ocr(tmp_cache, tmp_path, monkeypatch): + corrupted = "".join(chr(i) for i in range(1, 32)) + page = FakePage(text=corrupted, pixmap=FakePixmap()) + doc = FakeDocument([page]) + extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc)) + + calls: list[int] = [] + + def fake_tesseract(pixmap, language, timeout): + calls.append(pixmap.width) + return "1 No princípio criou Deus os céus e a terra." + + monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) + + pdf = tmp_path / "biblia.pdf" + pdf.write_bytes(b"%PDF-1.3") + pages = list(extractor.extract_document(pdf)) + + assert calls == [10], "OCR deveria ter sido chamado uma vez" + assert pages[0].text.startswith("1 No princípio") + + +def test_ocr_result_is_cached_and_reused(tmp_cache, tmp_path, monkeypatch): + corrupted = "".join(chr(i) for i in range(1, 32)) + page = FakePage(text=corrupted, pixmap=FakePixmap()) + extractor = PdfExtractor( + cache_dir=tmp_cache, + pymupdf_module=FakePyMuPDF(FakeDocument([page])), + ) + + calls = 0 + + def fake_tesseract(pixmap, language, timeout): + nonlocal calls + calls += 1 + return "texto ocr" + + monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) + + pdf = tmp_path / "biblia.pdf" + pdf.write_bytes(b"%PDF-1.3") + first = list(extractor.extract_document(pdf, file_hash="abc123")) + second = list(extractor.extract_document(pdf, file_hash="abc123")) + + assert calls == 1, "segunda extração deveria usar o cache" + assert first[0].text == second[0].text == "texto ocr" + + +def test_cache_is_invalidated_by_different_file_hash(tmp_cache, tmp_path, monkeypatch): + corrupted = "".join(chr(i) for i in range(1, 32)) + page = FakePage(text=corrupted, pixmap=FakePixmap()) + extractor = PdfExtractor( + cache_dir=tmp_cache, + pymupdf_module=FakePyMuPDF(FakeDocument([page])), + ) + + calls = 0 + + def fake_tesseract(pixmap, language, timeout): + nonlocal calls + calls += 1 + return "texto ocr" + + monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) + + pdf = tmp_path / "biblia.pdf" + pdf.write_bytes(b"%PDF-1.3") + list(extractor.extract_document(pdf, file_hash="hash1")) + list(extractor.extract_document(pdf, file_hash="hash2")) + + assert calls == 2, "hash diferente deve invalidar o cache" + + +def test_ocr_failure_is_isolated(tmp_cache, tmp_path, monkeypatch): + corrupted = "".join(chr(i) for i in range(1, 32)) + page = FakePage(text=corrupted, pixmap=FakePixmap()) + extractor = PdfExtractor( + cache_dir=tmp_cache, + pymupdf_module=FakePyMuPDF(FakeDocument([page])), + ) + + def failing_tesseract(pixmap, language, timeout): + raise OcrError("tesseract falhou") + + monkeypatch.setattr(extractor, "_run_tesseract", failing_tesseract) + + pdf = tmp_path / "biblia.pdf" + pdf.write_bytes(b"%PDF-1.3") + pages = list(extractor.extract_document(pdf)) + + assert len(pages) == 1 + assert pages[0].text == "" + assert pages[0].failed is True + + +def test_extract_page_returns_empty_text_for_blank_page(tmp_cache, monkeypatch): + page = FakePage(text="", pixmap=FakePixmap(blank=True)) + extractor = PdfExtractor( + cache_dir=tmp_cache, + pymupdf_module=FakePyMuPDF(FakeDocument([page])), + ) + + def fail_ocr(*args, **kwargs): + raise AssertionError("OCR não deveria ser chamado em página em branco") + + monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr) + + result = extractor.extract_page(page, "abc", 1) + + assert result.text == "" + assert result.failed is False + + +def test_real_pdf_page_is_detected_as_corrupted(tmp_path): + """O PDF do projeto tem camada textual corrompida: precisa cair no OCR.""" + import pymupdf + + real_pdf = Path(__file__).resolve().parents[2] / "PDF" / "biblia.pdf" + if not real_pdf.exists(): + pytest.skip("PDF real não disponível") + + doc = pymupdf.open(real_pdf) + try: + page = doc[5] + text = page.get_text("text") + finally: + doc.close() + + assert legible_ratio(text) < 0.35, "camada textual do PDF real deveria ser ilegível"