Backup automatico script del 2026-09-27 07:00
This commit is contained in:
1 parent
f347badf3b
commit
09dcbbdadb
24 files changed
+1164
-193
No files matched your search
@@ -267,7 +267,7 @@ def index_all(*, username: Optional[str] = None) -> dict:
|
||||
|
||||
|
||||
def search_apps_knowledge(query: str, limit: int = 8, source: Optional[str] = None) -> list:
|
||||
vectors = embeddings.embed_texts([query])
|
||||
vectors = [embeddings.embed_query(query)]
|
||||
flt = {"source": source} if source else None
|
||||
hits = qdrant_store.search(
|
||||
[qdrant_store.APPS_SHARED_COLLECTION],
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
@@ -13,39 +14,123 @@ from . import thermal
|
||||
LOGGER = logging.getLogger("loogle_mcp.embeddings")
|
||||
|
||||
|
||||
def embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
def embed_query(text: str) -> list[float]:
|
||||
"""Embedding interattivo per una query di ricerca.
|
||||
|
||||
Ordine: OLLAMA_QUERY_URL (istanza dedicata su pi1, ~0.1s, mai in coda dietro
|
||||
l'indicizzazione) → OLLAMA_URL (DS920, senza thermal gate) → OpenAI.
|
||||
Timeout corti così il client MCP riceve un errore invece di un hang.
|
||||
Cache LRU: gli agenti ripetono spesso le stesse query.
|
||||
"""
|
||||
return list(_embed_query_cached(text.strip(), os.environ.get("OLLAMA_EMBED_MODEL", "nomic-embed-text")))
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _embed_query_cached(text: str, _model: str) -> tuple[float, ...]:
|
||||
return tuple(_embed_query_uncached(text))
|
||||
|
||||
|
||||
def _embed_query_uncached(text: str) -> list[float]:
|
||||
query_url = os.environ.get("OLLAMA_QUERY_URL", "").strip()
|
||||
ollama_url = os.environ.get("OLLAMA_URL", "").strip()
|
||||
openai_key = os.environ.get("OPENAI_API_KEY", "").strip()
|
||||
last_exc: Optional[Exception] = None
|
||||
if query_url:
|
||||
try:
|
||||
# Istanza dedicata: options proprie, il modello resta caricato (keep_alive -1)
|
||||
return _embed_ollama_query(
|
||||
text, query_url, read_s=_float_env("OLLAMA_QUERY_URL_TIMEOUT_S", 10.0),
|
||||
keep_alive=-1, options=_ollama_options("OLLAMA_QUERY_NUM_THREAD"),
|
||||
)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("Ollama query (%s) failed, fallback: %s", query_url, exc)
|
||||
last_exc = exc
|
||||
if ollama_url:
|
||||
try:
|
||||
return _embed_ollama(texts, ollama_url)
|
||||
return _embed_ollama_query(
|
||||
text, ollama_url, read_s=_float_env("OLLAMA_QUERY_TIMEOUT_S", 20.0),
|
||||
keep_alive=int(_float_env("OLLAMA_KEEP_ALIVE_COOL", 300)), options=_ollama_options(),
|
||||
)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("Ollama embedding failed: %s", exc)
|
||||
openai_key = os.environ.get("OPENAI_API_KEY", "").strip()
|
||||
LOGGER.warning("Ollama query embedding failed: %s", exc)
|
||||
last_exc = exc
|
||||
if openai_key:
|
||||
return _embed_openai(texts, openai_key)
|
||||
return _embed_openai([text], openai_key, timeout=_float_env("OPENAI_QUERY_TIMEOUT_S", 20.0))[0]
|
||||
if last_exc is not None:
|
||||
raise RuntimeError(
|
||||
f"Servizio embedding (Ollama) non disponibile o lento: {last_exc}. Riprova tra poco."
|
||||
) from last_exc
|
||||
raise RuntimeError("Nessun provider embedding configurato (OLLAMA_URL o OPENAI_API_KEY)")
|
||||
|
||||
|
||||
def _embed_ollama(texts: list[str], base_url: str) -> list[list[float]]:
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _ollama_options(env_name: str = "OLLAMA_NUM_THREAD") -> Optional[dict]:
|
||||
# Sulla stessa istanza usare sempre le stesse options: se cambiano Ollama ricarica il modello
|
||||
num_thread = os.environ.get(env_name, "").strip()
|
||||
if num_thread:
|
||||
try:
|
||||
return {"num_thread": int(num_thread)}
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _embed_ollama_query(
|
||||
text: str, base_url: str, *, read_s: float, keep_alive: int, options: Optional[dict]
|
||||
) -> list[float]:
|
||||
model = os.environ.get("OLLAMA_EMBED_MODEL", "nomic-embed-text")
|
||||
timeout = httpx.Timeout(connect=5.0, read=read_s, write=10.0, pool=5.0)
|
||||
payload = {"model": model, "prompt": text, "keep_alive": keep_alive}
|
||||
if options:
|
||||
payload["options"] = options
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
resp = client.post(f"{base_url.rstrip('/')}/api/embeddings", json=payload)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["embedding"]
|
||||
|
||||
|
||||
def embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
# OLLAMA_INDEX_URL: istanza dedicata su pi1 (condivisa con le query). Nessun fallback
|
||||
# sul DS920 per l'indicizzazione: lì un chunk costa minuti, meglio riprovare al ciclo dopo
|
||||
index_url = os.environ.get("OLLAMA_INDEX_URL", "").strip()
|
||||
ollama_url = index_url or os.environ.get("OLLAMA_URL", "").strip()
|
||||
if ollama_url:
|
||||
try:
|
||||
return _embed_ollama(texts, ollama_url, dedicated=bool(index_url))
|
||||
except Exception as exc:
|
||||
LOGGER.warning("Ollama embedding failed (%s): %s", ollama_url, exc)
|
||||
openai_key = os.environ.get("OPENAI_API_KEY", "").strip()
|
||||
if openai_key:
|
||||
return _embed_openai(texts, openai_key)
|
||||
if ollama_url:
|
||||
raise RuntimeError(f"Ollama non disponibile per l'indicizzazione ({ollama_url})")
|
||||
raise RuntimeError("Nessun provider embedding configurato (OLLAMA_URL o OPENAI_API_KEY)")
|
||||
|
||||
|
||||
def _embed_ollama(texts: list[str], base_url: str, dedicated: bool = False) -> list[list[float]]:
|
||||
"""dedicated=True: istanza condivisa con le query → stesse options e modello sempre caricato."""
|
||||
model = os.environ.get("OLLAMA_EMBED_MODEL", "nomic-embed-text")
|
||||
vectors = []
|
||||
timeout = httpx.Timeout(connect=30.0, read=300.0, write=30.0, pool=30.0)
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
for i, text in enumerate(texts):
|
||||
status = thermal.wait_for_headroom(context=f"embed:{i+1}/{len(texts)}")
|
||||
keep_alive = thermal.suggested_keep_alive(status)
|
||||
keep_alive = -1 if dedicated else thermal.suggested_keep_alive(status)
|
||||
delay = thermal.suggested_delay_s(status)
|
||||
|
||||
payload = {"model": model, "prompt": text, "keep_alive": keep_alive}
|
||||
# options.num_thread limita i thread CPU lato Ollama (se supportato)
|
||||
num_thread = os.environ.get("OLLAMA_NUM_THREAD", "").strip()
|
||||
if num_thread:
|
||||
try:
|
||||
payload["options"] = {"num_thread": int(num_thread)}
|
||||
except ValueError:
|
||||
pass
|
||||
options = _ollama_options("OLLAMA_QUERY_NUM_THREAD" if dedicated else "OLLAMA_NUM_THREAD")
|
||||
if options:
|
||||
payload["options"] = options
|
||||
|
||||
resp = client.post(f"{base_url.rstrip('/')}/api/embeddings", json=payload)
|
||||
if resp.status_code >= 400:
|
||||
@@ -74,9 +159,9 @@ def _embed_ollama(texts: list[str], base_url: str) -> list[list[float]]:
|
||||
return vectors
|
||||
|
||||
|
||||
def _embed_openai(texts: list[str], api_key: str) -> list[list[float]]:
|
||||
def _embed_openai(texts: list[str], api_key: str, timeout: float = 120.0) -> list[list[float]]:
|
||||
model = os.environ.get("OPENAI_EMBED_MODEL", "text-embedding-3-small")
|
||||
with httpx.Client(timeout=120.0) as client:
|
||||
with httpx.Client(timeout=timeout) as client:
|
||||
resp = client.post(
|
||||
"https://api.openai.com/v1/embeddings",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
|
||||
@@ -111,7 +111,7 @@ def _request(
|
||||
params: Optional[dict] = None,
|
||||
json_body: Optional[dict] = None,
|
||||
) -> Any:
|
||||
with httpx.Client(timeout=60.0, verify=True) as client:
|
||||
with httpx.Client(timeout=httpx.Timeout(30.0, connect=5.0), verify=True) as client:
|
||||
resp = client.request(
|
||||
method,
|
||||
_api_url(path),
|
||||
|
||||
@@ -29,6 +29,33 @@ SKIP_PATH_PARTS = (
|
||||
)
|
||||
PRIORITY_PREFIXES = ("docs/", "ha/", "doc/", "README")
|
||||
|
||||
# File che esistono per contenere segreti: mai indicizzati (le collezioni RAG possono
|
||||
# essere condivise con la famiglia). .env.example resta indicizzabile.
|
||||
SECRET_FILE_RE = re.compile(
|
||||
r"(^|/)(\.env(\.(?!example$)[^/]*)?|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?|\.htpasswd|\.netrc|\.pgpass"
|
||||
r"|\.claude/settings\.local\.json"
|
||||
r"|[^/]*\.(pem|key|p12|pfx|jks|keystore))$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SECRET_KEY = r"[A-Za-z0-9_.-]*(?:pass(?:word|wd)?|secret|token|api[_-]?key|private[_-]?key)[A-Za-z0-9_.-]*"
|
||||
# chiave = valore / chiave: valore (valori letterali, non espressioni di codice)
|
||||
_SECRET_ASSIGN_RE = re.compile(
|
||||
rf"(?i)\b({_SECRET_KEY})(\s*[:=]\s*)([\"']?)([^\s\"'#(){{}}\[\]$<>,;]{{4,}})\3(?=[\s\"'#;,]|$)"
|
||||
)
|
||||
# stile config "chiave valore" a inizio riga (es. keepalived: auth_pass xxxx)
|
||||
_SECRET_CONF_RE = re.compile(rf"(?im)^(\s*{_SECRET_KEY})([ \t]+)(\S{{4,}})[ \t]*$")
|
||||
# password passate a comandi: sshpass -p X, --password X / --password=X
|
||||
_SECRET_CLI_RE = re.compile(r"(sshpass\s+-p\s*|--password[=\s]\s*)(\\?[\"']?)([^\s\"'\\]{3,})")
|
||||
_PRIVATE_KEY_RE = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----", re.S)
|
||||
|
||||
|
||||
def redact_secrets(text: str) -> str:
|
||||
"""Maschera i valori segreti prima di embedding e payload Qdrant."""
|
||||
text = _PRIVATE_KEY_RE.sub("<chiave privata rimossa>", text)
|
||||
text = _SECRET_CLI_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}<redacted>", text)
|
||||
text = _SECRET_ASSIGN_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}{m.group(3)}<redacted>{m.group(3)}", text)
|
||||
return _SECRET_CONF_RE.sub(lambda m: f"{m.group(1)}{m.group(2)}<redacted>", text)
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
flag = os.environ.get("GITEA_INDEX_ENABLED", "yes").strip().lower()
|
||||
@@ -71,7 +98,7 @@ def _repos_for_user(username: str) -> list[str]:
|
||||
|
||||
def _should_index_path(path: str) -> bool:
|
||||
lowered = path.lower()
|
||||
if any(part in lowered for part in SKIP_PATH_PARTS):
|
||||
if any(part in lowered for part in SKIP_PATH_PARTS) or SECRET_FILE_RE.search(path):
|
||||
return False
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
return bool(ext and ext in gitea.TEXT_EXTENSIONS)
|
||||
@@ -158,7 +185,7 @@ def index_file(
|
||||
if row and row["sha"] == sha and not force:
|
||||
return {"repo": repo, "path": path, "skipped": True, "sha": sha}
|
||||
|
||||
text = (file_data.get("content") or "").strip()
|
||||
text = redact_secrets((file_data.get("content") or "").strip())
|
||||
if not text:
|
||||
return {"repo": repo, "path": path, "chunks": 0, "sha": sha}
|
||||
|
||||
@@ -327,7 +354,7 @@ def gitea_collections_for_user(username: str, is_admin: bool = False) -> list[st
|
||||
|
||||
|
||||
def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
|
||||
vectors = embeddings.embed_texts([query])
|
||||
vectors = [embeddings.embed_query(query)]
|
||||
collections = gitea_collections_for_user(username, is_admin=is_admin)
|
||||
hits = qdrant_store.search(collections, vectors[0], limit=limit)
|
||||
for hit in hits:
|
||||
|
||||
@@ -2,10 +2,15 @@
|
||||
"""Paperless → Qdrant indexing pipeline."""
|
||||
|
||||
import logging
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Optional
|
||||
|
||||
LOGGER = logging.getLogger("loogle_mcp.indexer")
|
||||
|
||||
# Indicizzazioni avviate dai tool MCP: fuori dalla richiesta, una alla volta (thermal gate)
|
||||
_background = ThreadPoolExecutor(max_workers=1, thread_name_prefix="bg-index")
|
||||
|
||||
from ..db import get_conn
|
||||
from . import embeddings, gitea_indexer, paperless, qdrant_store
|
||||
from .text_chunk import chunk_text as _chunk_text
|
||||
@@ -104,7 +109,8 @@ def index_context_snippet(
|
||||
doc_key = f"ctx-{username}-{project_id}"
|
||||
if snippet_id:
|
||||
doc_key = f"{doc_key}-{snippet_id}"
|
||||
doc_id = hash(doc_key) % (2**31)
|
||||
# crc32 e non hash(): hash() cambia a ogni processo e il delete non trovava i vecchi chunk
|
||||
doc_id = zlib.crc32(doc_key.encode()) % (2**31)
|
||||
qdrant_store.delete_by_doc(collection, doc_id)
|
||||
ids = []
|
||||
payloads = []
|
||||
@@ -127,12 +133,82 @@ def index_context_snippet(
|
||||
qdrant_store.upsert_chunks(collection, ids, vectors, payloads)
|
||||
|
||||
|
||||
def index_project_context(username: str, project_id: str) -> dict:
|
||||
"""Indicizza il context.md completo del progetto sostituendo la versione precedente.
|
||||
|
||||
Personali → ctx_{utente}; condivisi → ctx_shared_family (visibili a tutta la famiglia).
|
||||
Così un contesto che cambia (es. inventario) non lascia in ricerca informazioni superate.
|
||||
"""
|
||||
from ..context import store as context_store
|
||||
|
||||
data = context_store.get_project_context(username, project_id, session_limit=0, include_gitea=False)
|
||||
meta = data["meta"]
|
||||
shared = bool(meta.get("shared"))
|
||||
collection = qdrant_store.CTX_SHARED_COLLECTION if shared else qdrant_store.ctx_collection(username)
|
||||
namespace = context_store.SHARED_NS if shared else username
|
||||
drop_project_context(collection, project_id)
|
||||
chunks = _chunk_text(data["context_md"])
|
||||
if not chunks:
|
||||
return {"project_id": project_id, "collection": collection, "chunks": 0}
|
||||
vectors = embeddings.embed_texts(chunks)
|
||||
doc_key = f"ctx-{namespace}-{project_id}"
|
||||
doc_id = zlib.crc32(doc_key.encode()) % (2**31)
|
||||
ids, payloads = [], []
|
||||
for i, chunk in enumerate(chunks):
|
||||
ids.append(f"{doc_key}-chunk-{i}")
|
||||
payloads.append(
|
||||
{
|
||||
"doc_id": doc_id,
|
||||
"project_id": project_id,
|
||||
"chunk_index": i,
|
||||
"title": meta.get("title") or project_id,
|
||||
"text": chunk,
|
||||
"owner": meta.get("owner") or username,
|
||||
"visibility": "family" if shared else "personal",
|
||||
"shared": shared,
|
||||
"updated_by": meta.get("updated_by"),
|
||||
"source": "context",
|
||||
"snippet_id": "full",
|
||||
}
|
||||
)
|
||||
qdrant_store.upsert_chunks(collection, ids, vectors, payloads)
|
||||
return {"project_id": project_id, "collection": collection, "chunks": len(chunks)}
|
||||
|
||||
|
||||
_DIGEST_SNIPPET_PREFIXES = ("daily-", "plan-")
|
||||
|
||||
|
||||
def drop_project_context(collection: str, project_id: str) -> int:
|
||||
"""Rimuove tutti i chunk di un progetto (anche quelli indicizzati a pezzi in passato),
|
||||
tranne gli snippet del digest giornaliero (snippet_id daily-*/plan-*)."""
|
||||
return qdrant_store.delete_matching(
|
||||
collection,
|
||||
"project_id",
|
||||
project_id,
|
||||
lambda payload: not str(payload.get("snippet_id") or "").startswith(_DIGEST_SNIPPET_PREFIXES),
|
||||
)
|
||||
|
||||
|
||||
def submit_background(label: str, fn, *args, **kwargs) -> None:
|
||||
"""Esegue fn in background; gli errori finiscono nel log invece che al client."""
|
||||
|
||||
def _run():
|
||||
try:
|
||||
fn(*args, **kwargs)
|
||||
LOGGER.info("Indicizzazione background completata: %s", label)
|
||||
except Exception:
|
||||
LOGGER.exception("Indicizzazione background fallita: %s", label)
|
||||
|
||||
_background.submit(_run)
|
||||
|
||||
|
||||
def search_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
|
||||
vectors = embeddings.embed_texts([query])
|
||||
vectors = [embeddings.embed_query(query)]
|
||||
collections = [
|
||||
qdrant_store.SHARED_COLLECTION,
|
||||
qdrant_store.kb_collection(username),
|
||||
qdrant_store.ctx_collection(username),
|
||||
qdrant_store.CTX_SHARED_COLLECTION,
|
||||
]
|
||||
collections.extend(gitea_indexer.gitea_collections_for_user(username, is_admin=is_admin))
|
||||
collections.append(qdrant_store.APPS_SHARED_COLLECTION)
|
||||
@@ -156,8 +232,9 @@ def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin:
|
||||
|
||||
|
||||
def search_context(username: str, query: str, limit: int = 8) -> list:
|
||||
vectors = embeddings.embed_texts([query])
|
||||
return qdrant_store.search([qdrant_store.ctx_collection(username)], vectors[0], limit=limit)
|
||||
vectors = [embeddings.embed_query(query)]
|
||||
collections = [qdrant_store.ctx_collection(username), qdrant_store.CTX_SHARED_COLLECTION]
|
||||
return qdrant_store.search(collections, vectors[0], limit=limit)
|
||||
|
||||
|
||||
def list_recent_documents(limit: int = 20) -> list:
|
||||
|
||||
@@ -75,7 +75,7 @@ def list_documents(
|
||||
ordering: str = "-modified",
|
||||
username: Optional[str] = None,
|
||||
) -> dict:
|
||||
with httpx.Client(timeout=60.0, verify=True) as client:
|
||||
with httpx.Client(timeout=httpx.Timeout(60.0, connect=5.0), verify=True) as client:
|
||||
resp = client.get(
|
||||
f"{PAPERLESS_URL}/api/documents/",
|
||||
headers=_headers(username),
|
||||
@@ -86,7 +86,7 @@ def list_documents(
|
||||
|
||||
|
||||
def get_document(doc_id: int, username: Optional[str] = None) -> dict:
|
||||
with httpx.Client(timeout=60.0, verify=True) as client:
|
||||
with httpx.Client(timeout=httpx.Timeout(60.0, connect=5.0), verify=True) as client:
|
||||
resp = client.get(
|
||||
f"{PAPERLESS_URL}/api/documents/{doc_id}/",
|
||||
headers=_headers(username),
|
||||
|
||||
@@ -6,6 +6,8 @@ import logging
|
||||
import math
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from typing import Optional
|
||||
|
||||
@@ -13,8 +15,8 @@ LOGGER = logging.getLogger("loogle_mcp.vector_store")
|
||||
VECTOR_SIZE = 768
|
||||
_local = sqlite3.connect(":memory:", check_same_thread=False) # placeholder
|
||||
_qdrant_client = None
|
||||
_qdrant_checked = False
|
||||
_use_fallback = False
|
||||
_next_qdrant_check = 0.0 # monotonic: prossimo tentativo di connessione mentre si usa il fallback
|
||||
_qdrant_lock = threading.Lock()
|
||||
|
||||
|
||||
def _fallback_path() -> str:
|
||||
@@ -41,28 +43,45 @@ def _fallback_conn() -> sqlite3.Connection:
|
||||
|
||||
|
||||
def _get_qdrant():
|
||||
global _qdrant_client, _qdrant_checked, _use_fallback
|
||||
if _qdrant_checked:
|
||||
return None if _use_fallback else _qdrant_client
|
||||
_qdrant_checked = True
|
||||
"""Client Qdrant, oppure None (fallback SQLite).
|
||||
|
||||
Se Qdrant non risponde si usa il fallback, ma si riprova ogni QDRANT_RETRY_S secondi
|
||||
invece di restare sul fallback fino al riavvio.
|
||||
"""
|
||||
global _next_qdrant_check
|
||||
if _qdrant_client is not None or time.monotonic() < _next_qdrant_check:
|
||||
return _qdrant_client
|
||||
with _qdrant_lock:
|
||||
if _qdrant_client is not None or time.monotonic() < _next_qdrant_check:
|
||||
return _qdrant_client
|
||||
client = _connect_qdrant()
|
||||
if client is None:
|
||||
_next_qdrant_check = time.monotonic() + float(os.environ.get("QDRANT_RETRY_S", "60"))
|
||||
return client
|
||||
|
||||
|
||||
def _connect_qdrant():
|
||||
global _qdrant_client
|
||||
url = os.environ.get("QDRANT_URL", "").strip()
|
||||
if not url:
|
||||
_use_fallback = True
|
||||
LOGGER.warning("QDRANT_URL non impostato — fallback SQLite")
|
||||
return None
|
||||
try:
|
||||
import httpx
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.http import models as qm
|
||||
|
||||
client = QdrantClient(url=url, timeout=60)
|
||||
# Sonda breve: un tentativo con Qdrant giù non deve bloccare la ricerca per 20s
|
||||
httpx.get(f"{url.rstrip('/')}/readyz", timeout=3.0).raise_for_status()
|
||||
client = QdrantClient(url=url, timeout=20)
|
||||
client.get_collections()
|
||||
_qdrant_client = client
|
||||
globals()["qm"] = qm
|
||||
LOGGER.info("Qdrant connesso: %s", url)
|
||||
return client
|
||||
except Exception as exc:
|
||||
_use_fallback = True
|
||||
LOGGER.warning("Qdrant non disponibile (%s) — fallback SQLite", exc)
|
||||
LOGGER.warning("Qdrant non disponibile (%s) — fallback SQLite, nuovo tentativo tra %ss",
|
||||
exc, os.environ.get("QDRANT_RETRY_S", "60"))
|
||||
return None
|
||||
|
||||
|
||||
@@ -92,6 +111,7 @@ def ctx_collection(username: str) -> str:
|
||||
SHARED_COLLECTION = "kb_shared_family"
|
||||
GITEA_SHARED_COLLECTION = "gitea_shared_family"
|
||||
APPS_SHARED_COLLECTION = "apps_shared_family"
|
||||
CTX_SHARED_COLLECTION = "ctx_shared_family"
|
||||
|
||||
|
||||
def gitea_collection(username: str) -> str:
|
||||
@@ -138,6 +158,11 @@ def upsert_chunks(
|
||||
|
||||
|
||||
def delete_by_doc(collection: str, doc_id: int) -> None:
|
||||
delete_where(collection, "doc_id", doc_id)
|
||||
|
||||
|
||||
def delete_where(collection: str, key: str, value) -> None:
|
||||
"""Elimina i punti il cui payload[key] == value."""
|
||||
client = _get_qdrant()
|
||||
if client is not None:
|
||||
from qdrant_client.http import models as qm
|
||||
@@ -147,7 +172,7 @@ def delete_by_doc(collection: str, doc_id: int) -> None:
|
||||
collection_name=collection,
|
||||
points_selector=qm.FilterSelector(
|
||||
filter=qm.Filter(
|
||||
must=[qm.FieldCondition(key="doc_id", match=qm.MatchValue(value=doc_id))]
|
||||
must=[qm.FieldCondition(key=key, match=qm.MatchValue(value=value))]
|
||||
)
|
||||
),
|
||||
)
|
||||
@@ -156,11 +181,42 @@ def delete_by_doc(collection: str, doc_id: int) -> None:
|
||||
rows = conn.execute("SELECT id,payload FROM vectors WHERE collection=?", (collection,)).fetchall()
|
||||
for row_id, payload_raw in rows:
|
||||
payload = json.loads(payload_raw)
|
||||
if payload.get("doc_id") == doc_id:
|
||||
if payload.get(key) == value:
|
||||
conn.execute("DELETE FROM vectors WHERE id=?", (row_id,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def delete_matching(collection: str, key: str, value, predicate) -> int:
|
||||
"""Elimina i punti con payload[key] == value per cui predicate(payload) è vero."""
|
||||
client = _get_qdrant()
|
||||
if client is not None:
|
||||
from qdrant_client.http import models as qm
|
||||
|
||||
ensure_collection(collection)
|
||||
flt = qm.Filter(must=[qm.FieldCondition(key=key, match=qm.MatchValue(value=value))])
|
||||
ids, offset = [], None
|
||||
while True:
|
||||
points, offset = client.scroll(
|
||||
collection_name=collection, scroll_filter=flt, limit=256, offset=offset, with_payload=True
|
||||
)
|
||||
ids.extend(p.id for p in points if predicate(p.payload or {}))
|
||||
if offset is None:
|
||||
break
|
||||
if ids:
|
||||
client.delete(collection_name=collection, points_selector=qm.PointIdsList(points=ids))
|
||||
return len(ids)
|
||||
conn = _fallback_conn()
|
||||
rows = conn.execute("SELECT id,payload FROM vectors WHERE collection=?", (collection,)).fetchall()
|
||||
removed = 0
|
||||
for row_id, payload_raw in rows:
|
||||
payload = json.loads(payload_raw)
|
||||
if payload.get(key) == value and predicate(payload):
|
||||
conn.execute("DELETE FROM vectors WHERE id=?", (row_id,))
|
||||
removed += 1
|
||||
conn.commit()
|
||||
return removed
|
||||
|
||||
|
||||
def count_by_doc(collection: str, doc_id: int) -> int:
|
||||
client = _get_qdrant()
|
||||
if client is not None:
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Thermal / load gate per proteggere il DS920 durante gli embedding Ollama.
|
||||
"""Thermal / load gate per proteggere l'host degli embedding Ollama.
|
||||
|
||||
Legge temperatura CPU e load average dal NAS e regola pause/keep_alive.
|
||||
Legge temperatura CPU e load average (dell'host locale, pi1, oppure del DS920 via
|
||||
sonda HTTP/SSH) e regola pause/keep_alive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -104,8 +105,29 @@ def _read_via_ssh() -> Optional[dict]:
|
||||
return None
|
||||
|
||||
|
||||
def _read_local() -> Optional[dict]:
|
||||
"""Temperatura e carico dell'host che esegue gli embedding (pi1): il container
|
||||
vede /sys/class/thermal e /proc/loadavg dell'host."""
|
||||
try:
|
||||
with open("/sys/class/thermal/thermal_zone0/temp", encoding="ascii") as f:
|
||||
temp = int(f.read().strip()) / 1000
|
||||
load1, load5, _ = os.getloadavg()
|
||||
return {"cpu_temp_c": temp, "load1": load1, "load5": load5, "nproc": os.cpu_count() or 4}
|
||||
except Exception as exc:
|
||||
LOGGER.debug("Thermal probe locale fallita: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def read_status() -> Optional[dict]:
|
||||
"""Ritorna {cpu_temp_c, load1, load5?, nproc} oppure None se non raggiungibile."""
|
||||
"""Ritorna {cpu_temp_c, load1, load5?, nproc} oppure None se non raggiungibile.
|
||||
|
||||
THERMAL_SOURCE=local legge l'host locale (embedding su pi1); default: DS920.
|
||||
"""
|
||||
if os.environ.get("THERMAL_SOURCE", "ds920").strip().lower() == "local":
|
||||
data = _read_local()
|
||||
if data:
|
||||
data["source"] = "local"
|
||||
return data
|
||||
data = _read_via_http()
|
||||
if data:
|
||||
data["source"] = "http"
|
||||
|
||||