fix: make docker compose build and start the API
The image context was backend/, so COPY frontend/ failed. Build from the repo root, place Python modules on the import path, and pin packages that exist on PyPI. The web process no longer loads the LLM at startup.
This commit is contained in:
1 parent
67739dcf7e
commit
7af7ba530d
19 files changed
+342
-250
No files matched your search
@@ -1,51 +1,67 @@
|
||||
from app.services.pdf_processor import PDFProcessor
|
||||
from app.utils.embeddings import EmbeddingService
|
||||
from app.utils.schedule_manager import ScheduleManager
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from app.services.pdf_processor import PDFProcessor
|
||||
from app.services.rag_service import QdrantService
|
||||
from app.utils.embeddings import EmbeddingService
|
||||
|
||||
|
||||
def book_dirs() -> list[str]:
|
||||
paths = [
|
||||
os.getenv("BOOKS_DIR", "/app/books"),
|
||||
os.getenv("UPLOAD_DIR", "/app/uploads"),
|
||||
]
|
||||
seen = []
|
||||
for path in paths:
|
||||
if path and path not in seen and os.path.isdir(path):
|
||||
seen.append(path)
|
||||
return seen
|
||||
|
||||
|
||||
def main():
|
||||
# Setup services
|
||||
embedding_service = EmbeddingService()
|
||||
pdf_processor = PDFProcessor(embedding_service)
|
||||
qdrant = QdrantService(os.getenv("QDRANT_URL", "http://qdrant:6333"))
|
||||
|
||||
# Load books from PDF directory
|
||||
pdf_dir = "/app/uploads"
|
||||
books = os.listdir(pdf_dir)
|
||||
books = [f for f in books if f.endswith('.pdf')]
|
||||
books = []
|
||||
for directory in book_dirs():
|
||||
for name in sorted(os.listdir(directory)):
|
||||
if name.lower().endswith(".pdf"):
|
||||
books.append(os.path.join(directory, name))
|
||||
|
||||
print(f"Found {len(books)} books")
|
||||
os.makedirs(os.getenv("INDEX_DIR", "/app/index"), exist_ok=True)
|
||||
|
||||
# Schedule manager
|
||||
schedule_manager = ScheduleManager("/app/schedules.json")
|
||||
|
||||
# Process each book
|
||||
for book_name in books:
|
||||
book_path = os.path.join(pdf_dir, book_name)
|
||||
for book_path in books:
|
||||
book_name = os.path.basename(book_path)
|
||||
book_metadata = {
|
||||
"title": book_name.replace(".pdf", "").replace("-", " ").replace("_", " "),
|
||||
"author": "Desconhecido",
|
||||
"type": "book",
|
||||
}
|
||||
|
||||
print(f"Processing: {book_name}")
|
||||
|
||||
# Load and split PDF
|
||||
chunks = pdf_processor.load_and_split(book_path, book_metadata)
|
||||
|
||||
# Embed chunks
|
||||
chunks_with_embeddings = pdf_processor.embed_chunks(chunks)
|
||||
for chunk in chunks_with_embeddings:
|
||||
chunk["id"] = str(uuid.uuid4())
|
||||
|
||||
# Save chunks to JSON for reference
|
||||
chunks_file = f"/app/index/{book_name}.json"
|
||||
os.makedirs("/app/index", exist_ok=True)
|
||||
qdrant.add_chunks(chunks_with_embeddings)
|
||||
|
||||
with open(chunks_file, 'w') as f:
|
||||
json.dump(chunks_with_embeddings, f, indent=2)
|
||||
|
||||
print(f"Saved {len(chunks)} chunks to {chunks_file}")
|
||||
chunks_file = os.path.join(os.getenv("INDEX_DIR", "/app/index"), f"{book_name}.json")
|
||||
with open(chunks_file, "w", encoding="utf-8") as handle:
|
||||
json.dump(
|
||||
[
|
||||
{"content": chunk["content"], "metadata": chunk["metadata"]}
|
||||
for chunk in chunks_with_embeddings
|
||||
],
|
||||
handle,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
print(f"Indexed {len(chunks_with_embeddings)} chunks from {book_name}")
|
||||
|
||||
print("Indexing complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user