Backup automatico script del 2026-08-23 12:04

services/loogle-mcp/app/knowledge/indexer.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+"""Paperless → Qdrant indexing pipeline."""
+
+import logging
+from typing import Optional
+
+LOGGER = logging.getLogger("loogle_mcp.indexer")
+
+from ..db import get_conn
+from . import embeddings, gitea_indexer, paperless, qdrant_store
+from .text_chunk import chunk_text as _chunk_text
+
+
+def index_document(doc_id: int, force: bool = False, paperless_user: Optional[str] = None) -> dict:
+    conn = get_conn()
+    existing = conn.execute(
+        "SELECT doc_id FROM indexed_documents WHERE doc_id=?", (doc_id,)
+    ).fetchone()
+    if existing and not force:
+        return {"doc_id": doc_id, "skipped": True}
+    doc = paperless.get_document(doc_id, username=paperless_user)
+    text = paperless.download_document_text(doc_id, username=paperless_user)
+    owner = paperless.document_owner(doc)
+    visibility = paperless.document_visibility(doc, owner)
+    chunks = _chunk_text(text)
+    if not chunks:
+        return {"doc_id": doc_id, "chunks": 0}
+    vectors = embeddings.embed_texts(chunks)
+    collection = qdrant_store.SHARED_COLLECTION if visibility == "family" else qdrant_store.kb_collection(owner)
+    qdrant_store.delete_by_doc(collection, doc_id)
+    ids = []
+    payloads = []
+    for i, chunk in enumerate(chunks):
+        point_id = f"doc-{doc_id}-chunk-{i}"
+        ids.append(point_id)
+        payloads.append(
+            {
+                "doc_id": doc_id,
+                "chunk_index": i,
+                "title": doc.get("title") or f"Documento {doc_id}",
+                "text": chunk,
+                "owner": owner,
+                "visibility": visibility,
+            }
+        )
+    qdrant_store.upsert_chunks(collection, ids, vectors, payloads)
+    conn.execute(
+        "INSERT INTO indexed_documents(doc_id,title,owner,visibility,chunk_count,indexed_at)"
+        " VALUES (?,?,?,?,?,datetime('now'))"
+        " ON CONFLICT(doc_id) DO UPDATE SET"
+        " title=excluded.title, owner=excluded.owner, visibility=excluded.visibility,"
+        " chunk_count=excluded.chunk_count, indexed_at=datetime('now')",
+        (doc_id, doc.get("title"), owner, visibility, len(chunks)),
+    )
+    conn.commit()
+    return {"doc_id": doc_id, "chunks": len(chunks), "collection": collection}
+
+
+def index_all(max_pages: int = 20) -> dict:
+    indexed = 0
+    errors = 0
+    seen: set[int] = set()
+    users = paperless.list_configured_users() or ["daniele"]
+    for paperless_user in users:
+        page = 1
+        while page <= max_pages:
+            try:
+                data = paperless.list_documents(page=page, page_size=25, username=paperless_user)
+            except Exception as exc:
+                LOGGER.error("Paperless list failed for %s: %s", paperless_user, exc)
+                break
+            results = data.get("results") or []
+            if not results:
+                break
+            for doc in results:
+                doc_id = doc["id"]
+                if doc_id in seen:
+                    continue
+                seen.add(doc_id)
+                try:
+                    index_document(doc_id, paperless_user=paperless_user)
+                    indexed += 1
+                except Exception as exc:
+                    LOGGER.warning("Index doc %s failed (%s): %s", doc_id, paperless_user, exc)
+                    errors += 1
+            if not data.get("next"):
+                break
+            page += 1
+    return {"indexed": indexed, "errors": errors, "users": users}
+
+
+def index_context_snippet(username: str, project_id: str, text: str, title: str) -> None:
+    chunks = _chunk_text(text)
+    if not chunks:
+        return
+    vectors = embeddings.embed_texts(chunks)
+    collection = qdrant_store.ctx_collection(username)
+    doc_key = f"ctx-{username}-{project_id}"
+    qdrant_store.delete_by_doc(collection, hash(doc_key) % (2**31))
+    ids = []
+    payloads = []
+    for i, chunk in enumerate(chunks):
+        point_id = f"{doc_key}-chunk-{i}"
+        ids.append(point_id)
+        payloads.append(
+            {
+                "doc_id": hash(doc_key) % (2**31),
+                "project_id": project_id,
+                "chunk_index": i,
+                "title": title,
+                "text": chunk,
+                "owner": username,
+                "visibility": "personal",
+            }
+        )
+    qdrant_store.upsert_chunks(collection, ids, vectors, payloads)
+
+
+def search_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
+    vectors = embeddings.embed_texts([query])
+    collections = [
+        qdrant_store.SHARED_COLLECTION,
+        qdrant_store.kb_collection(username),
+        qdrant_store.ctx_collection(username),
+    ]
+    collections.extend(gitea_indexer.gitea_collections_for_user(username, is_admin=is_admin))
+    collections.append(qdrant_store.APPS_SHARED_COLLECTION)
+    if is_admin:
+        for u in ("daniele", "lucia", "davide", "luca"):
+            collections.append(qdrant_store.kb_collection(u))
+    collections = list(dict.fromkeys(collections))
+    hits = qdrant_store.search(collections, vectors[0], limit=limit)
+    for hit in hits:
+        if hit.get("repo") and not hit.get("source"):
+            hit["source"] = "gitea"
+        elif hit.get("source") in ("irrigazione", "turni"):
+            hit["source_type"] = "apps"
+        elif hit.get("doc_id") and not hit.get("source"):
+            hit["source"] = "paperless"
+    return hits
+
+
+def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
+    return gitea_indexer.search_gitea_knowledge(username, query, limit=limit, is_admin=is_admin)
+
+
+def search_context(username: str, query: str, limit: int = 8) -> list:
+    vectors = embeddings.embed_texts([query])
+    return qdrant_store.search([qdrant_store.ctx_collection(username)], vectors[0], limit=limit)
+
+
+def list_recent_documents(limit: int = 20) -> list:
+    rows = get_conn().execute(
+        "SELECT * FROM indexed_documents ORDER BY indexed_at DESC LIMIT ?", (limit,)
+    ).fetchall()
+    return [dict(r) for r in rows]