feat(backend): add PDF extraction with OCR fallback and cache

backend/Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    OMP_NUM_THREADS=2 \
+    TOKENIZERS_PARALLELISM=false
+
+# tesseract-ocr + pacote de idioma português + libs de imagem do MuPDF
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        tesseract-ocr \
+        tesseract-ocr-por \
+        tesseract-ocr-eng \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt requirements-dev.txt ./
+RUN pip install --no-cache-dir -r requirements-dev.txt
+
+COPY app ./app
+COPY tests ./tests
+
+RUN mkdir -p /data/vector_db /data/pdfs
+
+EXPOSE 8000
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
+    CMD curl -fsS http://localhost:8000/api/health || exit 1
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

backend/app/__init__.py

@@ -0,0 +1,3 @@
+"""Backend do Pastor Online."""
+
+__version__ = "1.0.0"

backend/app/config.py

@@ -0,0 +1,83 @@
+"""Configuração do backend via variáveis de ambiente."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Configuração lida do ambiente ou do arquivo .env."""
+
+    model_config = SettingsConfigDict(
+        env_file=(".env", "../.env"),
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # Caminhos
+    pdf_dir: Path = Path("/data/pdfs")
+    vector_db_dir: Path = Path("/data/vector_db")
+
+    # Provedor de LLM: ollama | openai | groq | anthropic | stub
+    llm_provider: str = "ollama"
+    ollama_base_url: str = "http://ollama:11434"
+    ollama_model: str = "qwen2.5:1.5b"
+    openai_base_url: str = "https://api.openai.com/v1"
+    openai_model: str = "gpt-4o-mini"
+    openai_api_key: str = ""
+    groq_base_url: str = "https://api.groq.com/openai/v1"
+    groq_model: str = "llama-3.3-70b-versatile"
+    groq_api_key: str = ""
+    anthropic_base_url: str = "https://api.anthropic.com"
+    anthropic_model: str = "claude-sonnet-4-5"
+    anthropic_api_key: str = ""
+
+    # Embeddings
+    embedding_model: str = "paraphrase-multilingual-MiniLM-L12-v2"
+    embedding_batch_size: int = 32
+
+    # Busca
+    top_k: int = 6
+    min_similarity: float = 0.35
+    collection_name: str = "biblia"
+
+    # OCR
+    ocr_dpi: int = 200
+    ocr_language: str = "por+eng"
+    ocr_timeout: int = 120
+    ocr_workers: int = 4
+    min_legible_ratio: float = 0.35
+
+    # API
+    cors_origins: str = "http://localhost:8080,http://127.0.0.1:8080"
+    reindex_token: str = ""
+    max_message_length: int = 4000
+    max_history_turns: int = 10
+
+    # Limites de ingestão
+    max_pdf_bytes: int = 200 * 1024 * 1024
+
+    @property
+    def ocr_cache_dir(self) -> Path:
+        return self.vector_db_dir / "ocr_cache"
+
+    @property
+    def manifest_path(self) -> Path:
+        return self.vector_db_dir / "manifest.json"
+
+    @property
+    def book_index_path(self) -> Path:
+        return self.vector_db_dir / "book_index.json"
+
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()]
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Devolve as settings singleton do processo."""
+    return Settings()

backend/app/extraction.py

@@ -0,0 +1,213 @@
+"""Extração de texto de PDFs com fallback para OCR.
+
+A camada textual de muitos PDFs de Bíblia usa fontes sem mapeamento Unicode: o
+extrator devolve sequências de caracteres de controle em vez de letras. Detectamos
+esse caso e rasterizamos a página para o Tesseract. O resultado do OCR é cacheado
+por (sha256 do arquivo, página) para que reindexações não repitam horas de OCR.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import shutil
+import subprocess
+import tempfile
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator
+
+logger = logging.getLogger(__name__)
+
+try:  # pragma: no cover - dependência real do ambiente
+    import pymupdf
+except ImportError:  # pragma: no cover - permite rodar testes sem MuPDF
+    pymupdf = None  # type: ignore[assignment]
+
+
+class OcrError(RuntimeError):
+    """Falha ao executar o Tesseract."""
+
+
+@dataclass(frozen=True)
+class PageText:
+    """Texto extraído de uma página do PDF."""
+
+    page: int
+    text: str
+    failed: bool = False
+
+
+def legible_ratio(text: str) -> float:
+    """Proporção de caracteres imprimíveis legíveis no texto.
+
+    Um texto normal de Bíblia fica acima de 0.9; a camada corrompida do PDF real
+    fica abaixo de 0.05. Espaços em branco não contam como denominador.
+    """
+    if not text:
+        return 0.0
+    meaningful = [ch for ch in text if not ch.isspace()]
+    if not meaningful:
+        return 0.0
+    legible = sum(1 for ch in meaningful if _is_legible_char(ch))
+    return legible / len(meaningful)
+
+
+def _is_legible_char(ch: str) -> bool:
+    """Caractere que um humano consegue ler."""
+    if unicodedata.category(ch).startswith("C"):
+        # C0/C1 de controle: sinal de fonte sem ToUnicode.
+        return False
+    return ch.isalnum() or unicodedata.category(ch).startswith(("L", "N", "P", "S", "Z"))
+
+
+def _is_blank_pixmap(pixmap) -> bool:
+    """Diz se a página rasterizada está visualmente em branco.
+
+    Usa as amostras do pixmap: uma página de texto tem variância alta entre
+    pixels; uma página em branco tem todos os pixels iguais.
+    """
+    try:
+        samples = pixmap.samples
+        stride = max(1, pixmap.n if getattr(pixmap, "n", 1) else 1)
+        total = pixmap.width * pixmap.height
+        if total == 0:
+            return True
+        step = max(1, (total * stride) // 4000)
+        values = bytes(samples[0 : len(samples) : step])
+        if not values:
+            return True
+        return len(set(values)) <= 1
+    except Exception:  # pragma: no cover - formatos de pixmap inesperados
+        return False
+
+
+def file_sha256(path: Path, chunk_size: int = 1024 * 1024) -> str:
+    """SHA-256 do conteúdo do arquivo."""
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(chunk_size):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class PdfExtractor:
+    """Extrai texto de PDFs página a página, com OCR como fallback."""
+
+    def __init__(
+        self,
+        cache_dir: Path,
+        dpi: int = 200,
+        ocr_lang: str = "por+eng",
+        ocr_timeout: int = 120,
+        min_legible_ratio: float = 0.35,
+        pymupdf_module=None,
+    ):
+        self.cache_dir = Path(cache_dir)
+        self.dpi = dpi
+        self.ocr_lang = ocr_lang
+        self.ocr_timeout = ocr_timeout
+        self.min_legible_ratio = min_legible_ratio
+        self._pymupdf = pymupdf_module if pymupdf_module is not None else pymupdf
+
+    def extract_document(
+        self, path: Path, file_hash: str | None = None
+    ) -> Iterator[PageText]:
+        """Extrai o texto de todas as páginas do PDF."""
+        if self._pymupdf is None:  # pragma: no cover
+            raise OcrError("PyMuPDF não está disponível")
+
+        path = Path(path)
+        file_hash = file_hash or file_sha256(path)
+        document = self._pymupdf.open(path)
+        try:
+            for index in range(len(document)):
+                page = document[index]
+                yield self.extract_page(page, file_hash, index + 1)
+        finally:
+            document.close()
+
+    def extract_page(self, page, file_hash: str, page_number: int) -> PageText:
+        """Extrai o texto de uma página, usando cache quando disponível."""
+        cache_file = self._cache_path(file_hash, page_number)
+        if cache_file.exists():
+            return PageText(page=page_number, text=cache_file.read_text(encoding="utf-8"))
+
+        try:
+            text = page.get_text("text") or ""
+        except Exception as exc:  # pragma: no cover - PDF malformado
+            logger.warning("Falha ao extrair texto da página %s: %s", page_number, exc)
+            text = ""
+
+        failed = False
+        if legible_ratio(text) < self.min_legible_ratio:
+            text, failed = self._ocr_page(page)
+
+        self._write_cache(cache_file, text)
+        return PageText(page=page_number, text=text, failed=failed)
+
+    def _ocr_page(self, page) -> tuple[str, bool]:
+        """Rasteriza a página e aplica OCR. Devolve (texto, falhou).
+
+        Uma página visualmente em branco não é falha: o documento pode ter
+        separadores. Ela é detectada pela rasterização e não gasta OCR.
+        Falha é o OCR que não pôde ser executado.
+        """
+        try:
+            pixmap = page.get_pixmap(dpi=self.dpi)
+        except Exception as exc:
+            logger.warning("Falha ao rasterizar página para OCR: %s", exc)
+            return "", True
+
+        if _is_blank_pixmap(pixmap):
+            return "", False
+
+        try:
+            text = self._run_tesseract(pixmap, self.ocr_lang, self.ocr_timeout)
+        except OcrError as exc:
+            logger.warning("OCR falhou: %s", exc)
+            return "", True
+
+        return text, False
+
+    def _run_tesseract(self, pixmap, language: str, timeout: int) -> str:
+        """Executa o Tesseract sobre o pixmap e devolve o texto."""
+        if shutil.which("tesseract") is None:
+            raise OcrError("tesseract não está instalado no sistema")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            image_path = Path(tmp) / "page.png"
+            pixmap.save(str(image_path))
+            try:
+                completed = subprocess.run(
+                    [
+                        "tesseract",
+                        str(image_path),
+                        "stdout",
+                        "-l",
+                        language,
+                        "--psm",
+                        "6",
+                    ],
+                    capture_output=True,
+                    text=True,
+                    timeout=timeout,
+                    check=False,
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise OcrError(f"tesseract excedeu o timeout de {timeout}s") from exc
+
+        if completed.returncode != 0:
+            raise OcrError(f"tesseract falhou: {completed.stderr.strip()[:200]}")
+        return completed.stdout
+
+    def _cache_path(self, file_hash: str, page_number: int) -> Path:
+        return self.cache_dir / file_hash / f"{page_number:05d}.txt"
+
+    def _write_cache(self, cache_file: Path, text: str) -> None:
+        try:
+            cache_file.parent.mkdir(parents=True, exist_ok=True)
+            cache_file.write_text(text, encoding="utf-8")
+        except OSError as exc:  # pragma: no cover - disco cheio / permissões
+            logger.warning("Não foi possível gravar cache de OCR em %s: %s", cache_file, exc)

backend/app/models.py

@@ -0,0 +1,55 @@
+"""Schemas Pydantic de entrada e saída da API."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class ChatMessage(BaseModel):
+    role: Literal["user", "assistant"]
+    content: str = Field(min_length=1, max_length=8000)
+
+
+class ChatRequest(BaseModel):
+    message: str = Field(min_length=1, max_length=4000)
+    history: list[ChatMessage] = Field(default_factory=list)
+
+
+class Source(BaseModel):
+    """Trecho bíblico recuperado, exibido no accordion de fontes."""
+
+    reference: str
+    text: str
+    book: str
+    chapter: int
+    verse: int
+    page: int
+    file: str
+    score: float = 0.0
+
+
+class StatusResponse(BaseModel):
+    state: Literal["ready", "indexing", "error"]
+    pdfs_total: int = 0
+    pdfs_indexed: int = 0
+    chunks: int = 0
+    model: str = ""
+    last_indexed_at: str | None = None
+    errors: list[str] = Field(default_factory=list)
+
+
+class HealthResponse(BaseModel):
+    status: Literal["ok"] = "ok"
+    llm_provider: str
+    indexing: Literal["ready", "indexing", "error"]
+    version: str = "1.0.0"
+
+
+class ReindexResponse(BaseModel):
+    state: Literal["indexing"] = "indexing"
+
+
+class SourcesResponse(BaseModel):
+    sources: list[Source] = Field(default_factory=list)

backend/requirements-dev.txt

@@ -0,0 +1,3 @@
+-r requirements.txt
+pytest==8.3.4
+pytest-asyncio==0.25.0

backend/requirements.txt

@@ -0,0 +1,9 @@
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+pydantic==2.10.4
+pydantic-settings==2.7.0
+httpx==0.28.1
+pymupdf==1.25.1
+chromadb==1.5.9
+sentence-transformers==5.2.0
+numpy==2.2.1

backend/tests/test_extraction.py

@@ -0,0 +1,249 @@
+"""Testes do pipeline de extração de texto de PDF.
+
+Cobre a detecção de camada textual corrompida (o PDF real do projeto), o fallback
+para OCR, o cache por página e o isolamento de falhas de OCR.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from app.extraction import OcrError, PdfExtractor, legible_ratio  # noqa: E402
+
+
+class FakePage:
+    """Página de PDF simulada."""
+
+    def __init__(self, text: str = "", pixmap=None):
+        self._text = text
+        self._pixmap = pixmap or FakePixmap()
+
+    def get_text(self, kind: str) -> str:
+        assert kind == "text"
+        return self._text
+
+    def get_pixmap(self, dpi: int = 200):
+        return self._pixmap
+
+
+class FakeDocument:
+    def __init__(self, pages):
+        self._pages = pages
+        self.closed = False
+
+    def __len__(self):
+        return len(self._pages)
+
+    def __getitem__(self, index):
+        return self._pages[index]
+
+    def __iter__(self):
+        return iter(self._pages)
+
+    def close(self):
+        self.closed = True
+
+
+class FakePixmap:
+    def __init__(self, width=10, height=10, blank=False):
+        self.width = width
+        self.height = height
+        self.n = 3
+        self.blank = blank
+        self.saved_to: list[Path] = []
+
+    @property
+    def samples(self) -> bytes:
+        if self.blank:
+            return bytes([255]) * (self.width * self.height * self.n)
+        return bytes(range(self.width * self.height * self.n))[: self.width * self.height * self.n]
+
+    def save(self, path):
+        self.saved_to.append(Path(path))
+        Path(path).write_bytes(b"PNG")
+
+
+class FakePyMuPDF:
+    """Dublê do módulo pymupdf para testes."""
+
+    def __init__(self, document):
+        self._document = document
+        self.opened: list[Path] = []
+
+    def open(self, path):
+        self.opened.append(Path(path))
+        return self._document
+
+
+@pytest.fixture
+def tmp_cache(tmp_path):
+    return tmp_path / "ocr_cache"
+
+
+def test_legible_ratio_is_high_for_normal_text():
+    text = "No princípio criou Deus os céus e a terra. E a terra era sem forma e vazia."
+    assert legible_ratio(text) > 0.9
+
+
+def test_legible_ratio_is_low_for_control_characters():
+    text = "".join(chr(i) for i in range(1, 32))
+    assert legible_ratio(text) < 0.35
+
+
+def test_legible_ratio_of_empty_text_is_zero():
+    assert legible_ratio("") == 0.0
+    assert legible_ratio("   \n\t ") == 0.0
+
+
+def test_legible_text_does_not_trigger_ocr(tmp_cache, tmp_path, monkeypatch):
+    pdf = tmp_path / "biblia.pdf"
+    pdf.write_bytes(b"%PDF-1.3")
+    doc = FakeDocument([FakePage(text="1 No princípio criou Deus os céus e a terra.")])
+    extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc))
+
+    def fail_ocr(*args, **kwargs):
+        raise AssertionError("OCR não deveria ser chamado para texto legível")
+
+    monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr)
+
+    pages = list(extractor.extract_document(pdf))
+
+    assert len(pages) == 1
+    assert "No princípio" in pages[0].text
+    assert pages[0].page == 1
+
+
+def test_corrupted_text_layer_triggers_ocr(tmp_cache, tmp_path, monkeypatch):
+    corrupted = "".join(chr(i) for i in range(1, 32))
+    page = FakePage(text=corrupted, pixmap=FakePixmap())
+    doc = FakeDocument([page])
+    extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc))
+
+    calls: list[int] = []
+
+    def fake_tesseract(pixmap, language, timeout):
+        calls.append(pixmap.width)
+        return "1 No princípio criou Deus os céus e a terra."
+
+    monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract)
+
+    pdf = tmp_path / "biblia.pdf"
+    pdf.write_bytes(b"%PDF-1.3")
+    pages = list(extractor.extract_document(pdf))
+
+    assert calls == [10], "OCR deveria ter sido chamado uma vez"
+    assert pages[0].text.startswith("1 No princípio")
+
+
+def test_ocr_result_is_cached_and_reused(tmp_cache, tmp_path, monkeypatch):
+    corrupted = "".join(chr(i) for i in range(1, 32))
+    page = FakePage(text=corrupted, pixmap=FakePixmap())
+    extractor = PdfExtractor(
+        cache_dir=tmp_cache,
+        pymupdf_module=FakePyMuPDF(FakeDocument([page])),
+    )
+
+    calls = 0
+
+    def fake_tesseract(pixmap, language, timeout):
+        nonlocal calls
+        calls += 1
+        return "texto ocr"
+
+    monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract)
+
+    pdf = tmp_path / "biblia.pdf"
+    pdf.write_bytes(b"%PDF-1.3")
+    first = list(extractor.extract_document(pdf, file_hash="abc123"))
+    second = list(extractor.extract_document(pdf, file_hash="abc123"))
+
+    assert calls == 1, "segunda extração deveria usar o cache"
+    assert first[0].text == second[0].text == "texto ocr"
+
+
+def test_cache_is_invalidated_by_different_file_hash(tmp_cache, tmp_path, monkeypatch):
+    corrupted = "".join(chr(i) for i in range(1, 32))
+    page = FakePage(text=corrupted, pixmap=FakePixmap())
+    extractor = PdfExtractor(
+        cache_dir=tmp_cache,
+        pymupdf_module=FakePyMuPDF(FakeDocument([page])),
+    )
+
+    calls = 0
+
+    def fake_tesseract(pixmap, language, timeout):
+        nonlocal calls
+        calls += 1
+        return "texto ocr"
+
+    monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract)
+
+    pdf = tmp_path / "biblia.pdf"
+    pdf.write_bytes(b"%PDF-1.3")
+    list(extractor.extract_document(pdf, file_hash="hash1"))
+    list(extractor.extract_document(pdf, file_hash="hash2"))
+
+    assert calls == 2, "hash diferente deve invalidar o cache"
+
+
+def test_ocr_failure_is_isolated(tmp_cache, tmp_path, monkeypatch):
+    corrupted = "".join(chr(i) for i in range(1, 32))
+    page = FakePage(text=corrupted, pixmap=FakePixmap())
+    extractor = PdfExtractor(
+        cache_dir=tmp_cache,
+        pymupdf_module=FakePyMuPDF(FakeDocument([page])),
+    )
+
+    def failing_tesseract(pixmap, language, timeout):
+        raise OcrError("tesseract falhou")
+
+    monkeypatch.setattr(extractor, "_run_tesseract", failing_tesseract)
+
+    pdf = tmp_path / "biblia.pdf"
+    pdf.write_bytes(b"%PDF-1.3")
+    pages = list(extractor.extract_document(pdf))
+
+    assert len(pages) == 1
+    assert pages[0].text == ""
+    assert pages[0].failed is True
+
+
+def test_extract_page_returns_empty_text_for_blank_page(tmp_cache, monkeypatch):
+    page = FakePage(text="", pixmap=FakePixmap(blank=True))
+    extractor = PdfExtractor(
+        cache_dir=tmp_cache,
+        pymupdf_module=FakePyMuPDF(FakeDocument([page])),
+    )
+
+    def fail_ocr(*args, **kwargs):
+        raise AssertionError("OCR não deveria ser chamado em página em branco")
+
+    monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr)
+
+    result = extractor.extract_page(page, "abc", 1)
+
+    assert result.text == ""
+    assert result.failed is False
+
+
+def test_real_pdf_page_is_detected_as_corrupted(tmp_path):
+    """O PDF do projeto tem camada textual corrompida: precisa cair no OCR."""
+    import pymupdf
+
+    real_pdf = Path(__file__).resolve().parents[2] / "PDF" / "biblia.pdf"
+    if not real_pdf.exists():
+        pytest.skip("PDF real não disponível")
+
+    doc = pymupdf.open(real_pdf)
+    try:
+        page = doc[5]
+        text = page.get_text("text")
+    finally:
+        doc.close()
+
+    assert legible_ratio(text) < 0.35, "camada textual do PDF real deveria ser ilegível"