lgcosta
7af7ba530d
The image context was backend/, so COPY frontend/ failed. Build from the repo root, place Python modules on the import path, and pin packages that exist on PyPI. The web process no longer loads the LLM at startup.
68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
import json
|
|
import os
|
|
import uuid
|
|
|
|
from app.services.pdf_processor import PDFProcessor
|
|
from app.services.rag_service import QdrantService
|
|
from app.utils.embeddings import EmbeddingService
|
|
|
|
|
|
def book_dirs() -> list[str]:
|
|
paths = [
|
|
os.getenv("BOOKS_DIR", "/app/books"),
|
|
os.getenv("UPLOAD_DIR", "/app/uploads"),
|
|
]
|
|
seen = []
|
|
for path in paths:
|
|
if path and path not in seen and os.path.isdir(path):
|
|
seen.append(path)
|
|
return seen
|
|
|
|
|
|
def main():
|
|
embedding_service = EmbeddingService()
|
|
pdf_processor = PDFProcessor(embedding_service)
|
|
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
|
|
|
|
books = []
|
|
for directory in book_dirs():
|
|
for name in sorted(os.listdir(directory)):
|
|
if name.lower().endswith(".pdf"):
|
|
books.append(os.path.join(directory, name))
|
|
|
|
print(f"Found {len(books)} books")
|
|
os.makedirs(os.getenv("INDEX_DIR", "/app/index"), exist_ok=True)
|
|
|
|
for book_path in books:
|
|
book_name = os.path.basename(book_path)
|
|
book_metadata = {
|
|
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
|
|
"author": "Desconhecido",
|
|
"type": "book",
|
|
}
|
|
print(f"Processing: {book_name}")
|
|
chunks = pdf_processor.load_and_split(book_path, book_metadata)
|
|
chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
|
|
for chunk in chunks_with_embeddings:
|
|
chunk["id"] = str(uuid.uuid4())
|
|
|
|
qdrant.add_chunks(chunks_with_embeddings)
|
|
|
|
chunks_file = os.path.join(os.getenv("INDEX_DIR", "/app/index"), f"{book_name}.json")
|
|
with open(chunks_file, "w", encoding="utf-8") as handle:
|
|
json.dump(
|
|
[
|
|
{"content": chunk["content"], "metadata": chunk["metadata"]}
|
|
for chunk in chunks_with_embeddings
|
|
],
|
|
handle,
|
|
ensure_ascii=False,
|
|
)
|
|
print(f"Indexed {len(chunks_with_embeddings)} chunks from {book_name}")
|
|
|
|
print("Indexing complete!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|