# -*- coding: utf-8 -*- """Paperless → Qdrant indexing pipeline.""" import logging from typing import Optional LOGGER = logging.getLogger("loogle_mcp.indexer") from ..db import get_conn from . import embeddings, gitea_indexer, paperless, qdrant_store from .text_chunk import chunk_text as _chunk_text def index_document(doc_id: int, force: bool = False, paperless_user: Optional[str] = None) -> dict: conn = get_conn() existing = conn.execute( "SELECT doc_id FROM indexed_documents WHERE doc_id=?", (doc_id,) ).fetchone() if existing and not force: return {"doc_id": doc_id, "skipped": True} doc = paperless.get_document(doc_id, username=paperless_user) text = paperless.download_document_text(doc_id, username=paperless_user) owner = paperless.document_owner(doc) visibility = paperless.document_visibility(doc, owner) chunks = _chunk_text(text) if not chunks: return {"doc_id": doc_id, "chunks": 0} vectors = embeddings.embed_texts(chunks) collection = qdrant_store.SHARED_COLLECTION if visibility == "family" else qdrant_store.kb_collection(owner) qdrant_store.delete_by_doc(collection, doc_id) ids = [] payloads = [] for i, chunk in enumerate(chunks): point_id = f"doc-{doc_id}-chunk-{i}" ids.append(point_id) payloads.append( { "doc_id": doc_id, "chunk_index": i, "title": doc.get("title") or f"Documento {doc_id}", "text": chunk, "owner": owner, "visibility": visibility, } ) qdrant_store.upsert_chunks(collection, ids, vectors, payloads) conn.execute( "INSERT INTO indexed_documents(doc_id,title,owner,visibility,chunk_count,indexed_at)" " VALUES (?,?,?,?,?,datetime('now'))" " ON CONFLICT(doc_id) DO UPDATE SET" " title=excluded.title, owner=excluded.owner, visibility=excluded.visibility," " chunk_count=excluded.chunk_count, indexed_at=datetime('now')", (doc_id, doc.get("title"), owner, visibility, len(chunks)), ) conn.commit() return {"doc_id": doc_id, "chunks": len(chunks), "collection": collection} def index_all(max_pages: int = 20) -> dict: indexed = 0 errors = 0 seen: set[int] = set() users = paperless.list_configured_users() or ["daniele"] for paperless_user in users: page = 1 while page <= max_pages: try: data = paperless.list_documents(page=page, page_size=25, username=paperless_user) except Exception as exc: LOGGER.error("Paperless list failed for %s: %s", paperless_user, exc) break results = data.get("results") or [] if not results: break for doc in results: doc_id = doc["id"] if doc_id in seen: continue seen.add(doc_id) try: index_document(doc_id, paperless_user=paperless_user) indexed += 1 except Exception as exc: LOGGER.warning("Index doc %s failed (%s): %s", doc_id, paperless_user, exc) errors += 1 if not data.get("next"): break page += 1 return {"indexed": indexed, "errors": errors, "users": users} def index_context_snippet(username: str, project_id: str, text: str, title: str) -> None: chunks = _chunk_text(text) if not chunks: return vectors = embeddings.embed_texts(chunks) collection = qdrant_store.ctx_collection(username) doc_key = f"ctx-{username}-{project_id}" qdrant_store.delete_by_doc(collection, hash(doc_key) % (2**31)) ids = [] payloads = [] for i, chunk in enumerate(chunks): point_id = f"{doc_key}-chunk-{i}" ids.append(point_id) payloads.append( { "doc_id": hash(doc_key) % (2**31), "project_id": project_id, "chunk_index": i, "title": title, "text": chunk, "owner": username, "visibility": "personal", } ) qdrant_store.upsert_chunks(collection, ids, vectors, payloads) def search_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list: vectors = embeddings.embed_texts([query]) collections = [ qdrant_store.SHARED_COLLECTION, qdrant_store.kb_collection(username), qdrant_store.ctx_collection(username), ] collections.extend(gitea_indexer.gitea_collections_for_user(username, is_admin=is_admin)) collections.append(qdrant_store.APPS_SHARED_COLLECTION) if is_admin: for u in ("daniele", "lucia", "davide", "luca"): collections.append(qdrant_store.kb_collection(u)) collections = list(dict.fromkeys(collections)) hits = qdrant_store.search(collections, vectors[0], limit=limit) for hit in hits: if hit.get("repo") and not hit.get("source"): hit["source"] = "gitea" elif hit.get("source") in ("irrigazione", "turni"): hit["source_type"] = "apps" elif hit.get("doc_id") and not hit.get("source"): hit["source"] = "paperless" return hits def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list: return gitea_indexer.search_gitea_knowledge(username, query, limit=limit, is_admin=is_admin) def search_context(username: str, query: str, limit: int = 8) -> list: vectors = embeddings.embed_texts([query]) return qdrant_store.search([qdrant_store.ctx_collection(username)], vectors[0], limit=limit) def list_recent_documents(limit: int = 20) -> list: rows = get_conn().execute( "SELECT * FROM indexed_documents ORDER BY indexed_at DESC LIMIT ?", (limit,) ).fetchall() return [dict(r) for r in rows]