"""Testes do pipeline de extração de texto de PDF. Cobre a detecção de camada textual corrompida (o PDF real do projeto), o fallback para OCR, o cache por página e o isolamento de falhas de OCR. """ from __future__ import annotations import sys from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from app.extraction import ( # noqa: E402 OcrError, PdfExtractor, control_char_ratio, legible_ratio, ) class FakePage: """Página de PDF simulada.""" def __init__(self, text: str = "", pixmap=None): self._text = text self._pixmap = pixmap or FakePixmap() def get_text(self, kind: str) -> str: assert kind == "text" return self._text def get_pixmap(self, dpi: int = 200): return self._pixmap class FakeDocument: def __init__(self, pages): self._pages = pages self.closed = False def __len__(self): return len(self._pages) def __getitem__(self, index): return self._pages[index] def __iter__(self): return iter(self._pages) def close(self): self.closed = True class FakePixmap: def __init__(self, width=10, height=10, blank=False): self.width = width self.height = height self.n = 3 self.blank = blank self.saved_to: list[Path] = [] @property def samples(self) -> bytes: if self.blank: return bytes([255]) * (self.width * self.height * self.n) return bytes(range(self.width * self.height * self.n))[: self.width * self.height * self.n] def save(self, path): self.saved_to.append(Path(path)) Path(path).write_bytes(b"PNG") class FakePyMuPDF: """Dublê do módulo pymupdf para testes.""" def __init__(self, document): self._document = document self.opened: list[Path] = [] def open(self, path): self.opened.append(Path(path)) return self._document @pytest.fixture def tmp_cache(tmp_path): return tmp_path / "ocr_cache" def test_legible_ratio_is_high_for_normal_text(): text = "No princípio criou Deus os céus e a terra. E a terra era sem forma e vazia." assert legible_ratio(text) > 0.9 def test_legible_ratio_is_low_for_control_characters(): text = "".join(chr(i) for i in range(1, 32)) assert legible_ratio(text) < 0.35 def _corrupted_index_layer() -> str: """Reproduz a camada textual corrompida da página 3 do índice real. Medido no PDF: 442 caracteres não-branco, sendo 276 controles C0 (62%) e 166 legíveis (38%) — pontuação e letras soltas. A letra é o que empurra a razão de legibilidade para 0.376, acima do limiar de 0.35. """ controls = "".join(chr(index) for index in range(1, 32)) * 9 # 279 controles legible = "!\"#$%&'()*+,-./0123:;<=>?Gênesis:6" * 8 # letras e pontuação return f"ÍNDICE\n{controls}{legible}\n" def test_control_char_ratio_is_high_for_corrupted_layer(): """A camada corrompida do PDF real mistura letras avulsas e controles. A razão de caracteres legíveis não basta para detectar o problema: no índice do PDF, letras soltas entre os controles elevam o placar de legibilidade acima do limiar. """ corrupted = _corrupted_index_layer() assert legible_ratio(corrupted) > 0.35, "premissa do teste: legível alto" assert control_char_ratio(corrupted) > 0.01 def test_control_char_ratio_is_low_for_clean_text(): text = "GÊNESIS\n[1]\n1 No princípio criou Deus os céus e a terra." assert control_char_ratio(text) < 0.01 def test_control_char_ratio_of_empty_text_is_zero(): assert control_char_ratio("") == 0.0 assert control_char_ratio(" \n\t ") == 0.0 def test_legible_ratio_of_empty_text_is_zero(): assert legible_ratio("") == 0.0 assert legible_ratio(" \n\t ") == 0.0 def test_legible_text_does_not_trigger_ocr(tmp_cache, tmp_path, monkeypatch): pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") doc = FakeDocument([FakePage(text="1 No princípio criou Deus os céus e a terra.")]) extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc)) def fail_ocr(*args, **kwargs): raise AssertionError("OCR não deveria ser chamado para texto legível") monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr) pages = list(extractor.extract_document(pdf)) assert len(pages) == 1 assert "No princípio" in pages[0].text assert pages[0].page == 1 def test_corrupted_text_layer_triggers_ocr(tmp_cache, tmp_path, monkeypatch): corrupted = "".join(chr(i) for i in range(1, 32)) page = FakePage(text=corrupted, pixmap=FakePixmap()) doc = FakeDocument([page]) extractor = PdfExtractor(cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(doc)) calls: list[int] = [] def fake_tesseract(pixmap, language, timeout): calls.append(pixmap.width) return "1 No princípio criou Deus os céus e a terra." monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") pages = list(extractor.extract_document(pdf)) assert calls == [10], "OCR deveria ter sido chamado uma vez" assert pages[0].text.startswith("1 No princípio") def test_partially_legible_layer_with_controls_still_triggers_ocr( tmp_cache, tmp_path, monkeypatch ): """Regressão do índice do PDF real. A página do índice tem "ÍNDICE" legível e o resto corrompido. A razão de legibilidade fica acima de 0.35, mas os caracteres de controle revelam a corrupção — sem esta regra a página seria indexada com lixo. """ corrupted = _corrupted_index_layer() assert legible_ratio(corrupted) > 0.35 page = FakePage(text=corrupted, pixmap=FakePixmap()) extractor = PdfExtractor( cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(FakeDocument([page])), ) calls: list[int] = [] def fake_tesseract(pixmap, language, timeout): calls.append(pixmap.width) return "ÍNDICE\nGênesis: 6" monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") pages = list(extractor.extract_document(pdf)) assert calls == [10], "OCR deveria ter sido chamado apesar da legibilidade alta" assert "Gênesis" in pages[0].text def test_ocr_result_is_cached_and_reused(tmp_cache, tmp_path, monkeypatch): corrupted = "".join(chr(i) for i in range(1, 32)) page = FakePage(text=corrupted, pixmap=FakePixmap()) extractor = PdfExtractor( cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(FakeDocument([page])), ) calls = 0 def fake_tesseract(pixmap, language, timeout): nonlocal calls calls += 1 return "texto ocr" monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") first = list(extractor.extract_document(pdf, file_hash="abc123")) second = list(extractor.extract_document(pdf, file_hash="abc123")) assert calls == 1, "segunda extração deveria usar o cache" assert first[0].text == second[0].text == "texto ocr" def test_cache_is_invalidated_by_different_file_hash(tmp_cache, tmp_path, monkeypatch): corrupted = "".join(chr(i) for i in range(1, 32)) page = FakePage(text=corrupted, pixmap=FakePixmap()) extractor = PdfExtractor( cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(FakeDocument([page])), ) calls = 0 def fake_tesseract(pixmap, language, timeout): nonlocal calls calls += 1 return "texto ocr" monkeypatch.setattr(extractor, "_run_tesseract", fake_tesseract) pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") list(extractor.extract_document(pdf, file_hash="hash1")) list(extractor.extract_document(pdf, file_hash="hash2")) assert calls == 2, "hash diferente deve invalidar o cache" def test_ocr_failure_is_isolated(tmp_cache, tmp_path, monkeypatch): corrupted = "".join(chr(i) for i in range(1, 32)) page = FakePage(text=corrupted, pixmap=FakePixmap()) extractor = PdfExtractor( cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(FakeDocument([page])), ) def failing_tesseract(pixmap, language, timeout): raise OcrError("tesseract falhou") monkeypatch.setattr(extractor, "_run_tesseract", failing_tesseract) pdf = tmp_path / "biblia.pdf" pdf.write_bytes(b"%PDF-1.3") pages = list(extractor.extract_document(pdf)) assert len(pages) == 1 assert pages[0].text == "" assert pages[0].failed is True def test_extract_page_returns_empty_text_for_blank_page(tmp_cache, monkeypatch): page = FakePage(text="", pixmap=FakePixmap(blank=True)) extractor = PdfExtractor( cache_dir=tmp_cache, pymupdf_module=FakePyMuPDF(FakeDocument([page])), ) def fail_ocr(*args, **kwargs): raise AssertionError("OCR não deveria ser chamado em página em branco") monkeypatch.setattr(extractor, "_run_tesseract", fail_ocr) result = extractor.extract_page(page, "abc", 1) assert result.text == "" assert result.failed is False def test_real_pdf_page_is_detected_as_corrupted(tmp_path): """O PDF do projeto tem camada textual corrompida: precisa cair no OCR.""" import pymupdf real_pdf = Path(__file__).resolve().parents[2] / "PDF" / "biblia.pdf" if not real_pdf.exists(): pytest.skip("PDF real não disponível") doc = pymupdf.open(real_pdf) try: page = doc[5] text = page.get_text("text") finally: doc.close() assert legible_ratio(text) < 0.35, "camada textual do PDF real deveria ser ilegível"