lgcosta
8eeb6b89f4
Usa o modelo multilingual, ignora trechos curtos e o template do Llama 3.2 para a resposta sair em uma frase completa. O chat deixa a API livre durante a geração, o upload indexa o PDF e os agendamentos não são gravados duas vezes.
49 lines
1.6 KiB
Python
49 lines
1.6 KiB
Python
from langchain_community.document_loaders import PyPDFLoader
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
|
import os
|
|
from typing import List, Dict
|
|
from app.utils.embeddings import EmbeddingService
|
|
|
|
class PDFProcessor:
|
|
def __init__(self, embedding_service: EmbeddingService):
|
|
self.embedding_service = embedding_service
|
|
self.text_splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=500,
|
|
chunk_overlap=50,
|
|
length_function=len,
|
|
)
|
|
|
|
def load_and_split(self, file_path: str, metadata: Dict) -> List[Dict]:
|
|
"""Load and split a PDF file."""
|
|
loader = PyPDFLoader(file_path)
|
|
documents = loader.load()
|
|
|
|
chunks = self.text_splitter.split_documents(documents)
|
|
|
|
# Convert to dictionaries with metadata
|
|
chunk_data = []
|
|
for doc in chunks:
|
|
content = " ".join(doc.page_content.split())
|
|
if len(content) < 80:
|
|
continue
|
|
chunk_data.append({
|
|
"content": content,
|
|
"metadata": {
|
|
**metadata,
|
|
"chunk_id": len(chunk_data),
|
|
"source": file_path,
|
|
}
|
|
})
|
|
|
|
return chunk_data
|
|
|
|
def embed_chunks(self, chunks: List[Dict]) -> List[Dict]:
|
|
"""Add embeddings to chunks."""
|
|
texts = [chunk["content"] for chunk in chunks]
|
|
embeddings = self.embedding_service.embed(texts)
|
|
|
|
for chunk, embedding in zip(chunks, embeddings):
|
|
chunk["embedding"] = embedding.tolist()
|
|
|
|
return chunks
|