Backup automatico script del 2026-08-23 12:04
This commit is contained in:
daniele committed
1 parent
11823447a2
commit
795a15a7b6
66 files changed
+8546
No files matched your search
@@ -0,0 +1,369 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Indicizzazione semantica (RAG) su file testo dei repository Gitea."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import zlib
|
||||
from typing import Optional
|
||||
|
||||
from ..db import get_conn
|
||||
from . import embeddings, gitea, qdrant_store
|
||||
from .text_chunk import CHUNK_OVERLAP, CHUNK_SIZE, chunk_text as _chunk_text
|
||||
|
||||
LOGGER = logging.getLogger("loogle_mcp.gitea_indexer")
|
||||
|
||||
SKIP_PATH_PARTS = (
|
||||
"node_modules/",
|
||||
"vendor/",
|
||||
".git/",
|
||||
"dist/",
|
||||
"build/",
|
||||
"__pycache__/",
|
||||
".venv/",
|
||||
"venv/",
|
||||
".tox/",
|
||||
"coverage/",
|
||||
)
|
||||
PRIORITY_PREFIXES = ("docs/", "ha/", "doc/", "README")
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
flag = os.environ.get("GITEA_INDEX_ENABLED", "yes").strip().lower()
|
||||
return flag not in ("0", "false", "no", "off")
|
||||
|
||||
|
||||
def _max_files_per_repo() -> int:
|
||||
return int(os.environ.get("GITEA_INDEX_MAX_FILES_PER_REPO", "150"))
|
||||
|
||||
|
||||
def _max_file_bytes() -> int:
|
||||
return int(os.environ.get("GITEA_INDEX_MAX_FILE_BYTES", "120000"))
|
||||
|
||||
|
||||
def _repos_for_user(username: str) -> list[str]:
|
||||
env_key = f"GITEA_INDEX_REPOS_{username.upper()}"
|
||||
raw = os.environ.get(env_key, "").strip()
|
||||
if raw:
|
||||
repos: list[str] = []
|
||||
for item in raw.split(","):
|
||||
item = item.strip()
|
||||
if not item:
|
||||
continue
|
||||
owner, name = gitea.parse_repo(item)
|
||||
repos.append(f"{owner}/{name}")
|
||||
return repos
|
||||
repos: list[str] = []
|
||||
page = 1
|
||||
while page <= 5:
|
||||
data = gitea.list_repos(username=username, page=page, limit=50)
|
||||
batch = [r["full_name"] for r in data.get("repos") or [] if r.get("full_name")]
|
||||
if not batch:
|
||||
break
|
||||
repos.extend(batch)
|
||||
if len(batch) < 50:
|
||||
break
|
||||
page += 1
|
||||
return repos
|
||||
|
||||
|
||||
def _should_index_path(path: str) -> bool:
|
||||
lowered = path.lower()
|
||||
if any(part in lowered for part in SKIP_PATH_PARTS):
|
||||
return False
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
return bool(ext and ext in gitea.TEXT_EXTENSIONS)
|
||||
|
||||
|
||||
def _path_priority(path: str) -> tuple[int, str]:
|
||||
lowered = path.lower()
|
||||
for idx, prefix in enumerate(PRIORITY_PREFIXES):
|
||||
if lowered.startswith(prefix.lower()) or os.path.basename(lowered).startswith(prefix.lower()):
|
||||
return (idx, path)
|
||||
return (len(PRIORITY_PREFIXES), path)
|
||||
|
||||
|
||||
def list_repo_text_files(repo: str, username: str) -> list[dict]:
|
||||
owner, name = gitea.parse_repo(repo)
|
||||
tree = gitea._request(
|
||||
"GET",
|
||||
f"/repos/{owner}/{name}/git/trees/HEAD",
|
||||
username=username,
|
||||
params={"recursive": "1"},
|
||||
)
|
||||
max_files = _max_files_per_repo()
|
||||
max_bytes = _max_file_bytes()
|
||||
files: list[dict] = []
|
||||
for node in tree.get("tree") or []:
|
||||
if node.get("type") != "blob":
|
||||
continue
|
||||
path = node.get("path") or ""
|
||||
if not _should_index_path(path):
|
||||
continue
|
||||
size = int(node.get("size") or 0)
|
||||
if size > max_bytes:
|
||||
continue
|
||||
files.append({"path": path, "sha": node.get("sha"), "size": size})
|
||||
files.sort(key=lambda item: _path_priority(item["path"]))
|
||||
return files[:max_files]
|
||||
|
||||
|
||||
def _repo_owner(repo: str) -> str:
|
||||
owner, _ = gitea.parse_repo(repo)
|
||||
return owner.lower()
|
||||
|
||||
|
||||
def _repo_visibility(private: bool, owner: str) -> str:
|
||||
if private:
|
||||
return "personal"
|
||||
if owner in gitea.MCP_USERS:
|
||||
return "family"
|
||||
return "family"
|
||||
|
||||
|
||||
def _collection_for_repo(private: bool, owner: str) -> str:
|
||||
visibility = _repo_visibility(private, owner)
|
||||
if visibility == "personal":
|
||||
return qdrant_store.gitea_collection(owner)
|
||||
return qdrant_store.GITEA_SHARED_COLLECTION
|
||||
|
||||
|
||||
def _file_doc_id(repo: str, path: str) -> int:
|
||||
return zlib.adler32(f"{repo}:{path}".encode("utf-8")) & 0x7FFFFFFF
|
||||
|
||||
|
||||
def _file_title(repo: str, path: str) -> str:
|
||||
return f"{repo}/{path}"
|
||||
|
||||
|
||||
def index_file(
|
||||
repo: str,
|
||||
path: str,
|
||||
*,
|
||||
username: str,
|
||||
private: bool,
|
||||
force: bool = False,
|
||||
) -> dict:
|
||||
owner = _repo_owner(repo)
|
||||
conn = get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT sha, chunk_count FROM indexed_gitea_files WHERE repo=? AND path=?",
|
||||
(repo, path),
|
||||
).fetchone()
|
||||
|
||||
file_data = gitea.get_file(repo, path, username=username)
|
||||
sha = file_data.get("sha") or ""
|
||||
if row and row["sha"] == sha and not force:
|
||||
return {"repo": repo, "path": path, "skipped": True, "sha": sha}
|
||||
|
||||
text = (file_data.get("content") or "").strip()
|
||||
if not text:
|
||||
return {"repo": repo, "path": path, "chunks": 0, "sha": sha}
|
||||
|
||||
header = f"# {_file_title(repo, path)}\n\nSource: gitea:{repo}:{path}\n\n"
|
||||
chunks = _chunk_text(header + text)
|
||||
if not chunks:
|
||||
return {"repo": repo, "path": path, "chunks": 0, "sha": sha}
|
||||
|
||||
visibility = _repo_visibility(private, owner)
|
||||
collection = _collection_for_repo(private, owner)
|
||||
doc_id = _file_doc_id(repo, path)
|
||||
qdrant_store.delete_by_doc(collection, doc_id)
|
||||
|
||||
vectors = embeddings.embed_texts(chunks)
|
||||
ids = []
|
||||
payloads = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
point_id = f"gitea-{repo}-{path}-chunk-{i}"
|
||||
ids.append(point_id)
|
||||
payloads.append(
|
||||
{
|
||||
"source": "gitea",
|
||||
"doc_id": doc_id,
|
||||
"repo": repo,
|
||||
"path": path,
|
||||
"chunk_index": i,
|
||||
"title": _file_title(repo, path),
|
||||
"text": chunk,
|
||||
"owner": owner,
|
||||
"visibility": visibility,
|
||||
"sha": sha,
|
||||
}
|
||||
)
|
||||
qdrant_store.upsert_chunks(collection, ids, vectors, payloads)
|
||||
|
||||
stored = qdrant_store.count_by_doc(collection, doc_id)
|
||||
if stored < len(chunks):
|
||||
raise RuntimeError(
|
||||
f"Qdrant upsert incompleto per {repo}/{path}: attesi {len(chunks)} chunk, trovati {stored}"
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO indexed_gitea_files(repo,path,sha,owner,visibility,chunk_count,indexed_at)"
|
||||
" VALUES (?,?,?,?,?,?,datetime('now'))"
|
||||
" ON CONFLICT(repo,path) DO UPDATE SET"
|
||||
" sha=excluded.sha, owner=excluded.owner, visibility=excluded.visibility,"
|
||||
" chunk_count=excluded.chunk_count, indexed_at=datetime('now')",
|
||||
(repo, path, sha, owner, visibility, len(chunks)),
|
||||
)
|
||||
conn.commit()
|
||||
return {
|
||||
"repo": repo,
|
||||
"path": path,
|
||||
"sha": sha,
|
||||
"chunks": len(chunks),
|
||||
"collection": collection,
|
||||
"visibility": visibility,
|
||||
}
|
||||
|
||||
|
||||
def index_repo(
|
||||
repo: str,
|
||||
*,
|
||||
username: Optional[str] = None,
|
||||
private: Optional[bool] = None,
|
||||
force: bool = False,
|
||||
max_files: Optional[int] = None,
|
||||
) -> dict:
|
||||
gitea_user = username or _repo_owner(repo)
|
||||
if not gitea.is_configured(gitea_user):
|
||||
raise RuntimeError(f"Gitea non configurato per {gitea_user}")
|
||||
|
||||
if private is None:
|
||||
owner, name = gitea.parse_repo(repo)
|
||||
meta = gitea._request("GET", f"/repos/{owner}/{name}", username=gitea_user)
|
||||
private = bool(meta.get("private"))
|
||||
|
||||
files = list_repo_text_files(repo, gitea_user)
|
||||
if max_files is not None:
|
||||
files = files[: max(1, max_files)]
|
||||
|
||||
indexed = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
chunks = 0
|
||||
for item in files:
|
||||
try:
|
||||
result = index_file(
|
||||
repo,
|
||||
item["path"],
|
||||
username=gitea_user,
|
||||
private=bool(private),
|
||||
force=force,
|
||||
)
|
||||
if result.get("skipped"):
|
||||
skipped += 1
|
||||
else:
|
||||
indexed += 1
|
||||
chunks += int(result.get("chunks") or 0)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("Index gitea %s/%s failed: %s", repo, item.get("path"), exc)
|
||||
errors += 1
|
||||
|
||||
return {
|
||||
"repo": repo,
|
||||
"files_seen": len(files),
|
||||
"files_indexed": indexed,
|
||||
"files_skipped": skipped,
|
||||
"errors": errors,
|
||||
"chunks": chunks,
|
||||
"private": bool(private),
|
||||
}
|
||||
|
||||
|
||||
def index_all(max_files_per_repo: Optional[int] = None) -> dict:
|
||||
if not _enabled():
|
||||
return {"enabled": False, "indexed_files": 0}
|
||||
|
||||
total_indexed = 0
|
||||
total_skipped = 0
|
||||
total_errors = 0
|
||||
total_chunks = 0
|
||||
repos_done: list[str] = []
|
||||
users = gitea.list_configured_users() or []
|
||||
|
||||
for username in users:
|
||||
try:
|
||||
repos = _repos_for_user(username)
|
||||
except Exception as exc:
|
||||
LOGGER.error("Lista repo Gitea fallita per %s: %s", username, exc)
|
||||
continue
|
||||
for repo in repos:
|
||||
try:
|
||||
result = index_repo(
|
||||
repo,
|
||||
username=username,
|
||||
force=False,
|
||||
max_files=max_files_per_repo,
|
||||
)
|
||||
repos_done.append(repo)
|
||||
total_indexed += int(result.get("files_indexed") or 0)
|
||||
total_skipped += int(result.get("files_skipped") or 0)
|
||||
total_errors += int(result.get("errors") or 0)
|
||||
total_chunks += int(result.get("chunks") or 0)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("Index repo %s failed (%s): %s", repo, username, exc)
|
||||
total_errors += 1
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"repos": repos_done,
|
||||
"files_indexed": total_indexed,
|
||||
"files_skipped": total_skipped,
|
||||
"errors": total_errors,
|
||||
"chunks": total_chunks,
|
||||
"users": users,
|
||||
}
|
||||
|
||||
|
||||
def gitea_collections_for_user(username: str, is_admin: bool = False) -> list[str]:
|
||||
cols = [qdrant_store.GITEA_SHARED_COLLECTION, qdrant_store.gitea_collection(username)]
|
||||
if is_admin:
|
||||
for user in gitea.MCP_USERS:
|
||||
cols.append(qdrant_store.gitea_collection(user))
|
||||
return list(dict.fromkeys(cols))
|
||||
|
||||
|
||||
def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
|
||||
vectors = embeddings.embed_texts([query])
|
||||
collections = gitea_collections_for_user(username, is_admin=is_admin)
|
||||
hits = qdrant_store.search(collections, vectors[0], limit=limit)
|
||||
for hit in hits:
|
||||
hit["source"] = hit.get("source") or "gitea"
|
||||
return hits
|
||||
|
||||
|
||||
def list_indexed_files(limit: int = 30, repo: Optional[str] = None) -> list:
|
||||
conn = get_conn()
|
||||
if repo:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM indexed_gitea_files WHERE repo=? ORDER BY indexed_at DESC LIMIT ?",
|
||||
(repo, limit),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM indexed_gitea_files ORDER BY indexed_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def index_stats() -> dict:
|
||||
conn = get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*), COALESCE(SUM(chunk_count), 0) FROM indexed_gitea_files"
|
||||
).fetchone()
|
||||
files_count = int(row[0] if row else 0)
|
||||
chunks_meta = int(row[1] if row else 0)
|
||||
collections = {qdrant_store.GITEA_SHARED_COLLECTION}
|
||||
for user in gitea.MCP_USERS:
|
||||
collections.add(qdrant_store.gitea_collection(user))
|
||||
qdrant_points = sum(qdrant_store.collection_point_count(c) for c in collections)
|
||||
return {
|
||||
"files_indexed": files_count,
|
||||
"chunks_in_metadata": chunks_meta,
|
||||
"qdrant_points": qdrant_points,
|
||||
"collections": {c: qdrant_store.collection_point_count(c) for c in sorted(collections)},
|
||||
}
|
||||
Reference in new issue
Block a user