# -*- coding: utf-8 -*- """Indicizzazione semantica (RAG) su file testo dei repository Gitea.""" from __future__ import annotations import logging import os import re import zlib from typing import Optional from ..db import get_conn from . import embeddings, gitea, qdrant_store from .text_chunk import CHUNK_OVERLAP, CHUNK_SIZE, chunk_text as _chunk_text LOGGER = logging.getLogger("loogle_mcp.gitea_indexer") SKIP_PATH_PARTS = ( "node_modules/", "vendor/", ".git/", "dist/", "build/", "__pycache__/", ".venv/", "venv/", ".tox/", "coverage/", ) PRIORITY_PREFIXES = ("docs/", "ha/", "doc/", "README") def _enabled() -> bool: flag = os.environ.get("GITEA_INDEX_ENABLED", "yes").strip().lower() return flag not in ("0", "false", "no", "off") def _max_files_per_repo() -> int: return int(os.environ.get("GITEA_INDEX_MAX_FILES_PER_REPO", "150")) def _max_file_bytes() -> int: return int(os.environ.get("GITEA_INDEX_MAX_FILE_BYTES", "120000")) def _repos_for_user(username: str) -> list[str]: env_key = f"GITEA_INDEX_REPOS_{username.upper()}" raw = os.environ.get(env_key, "").strip() if raw: repos: list[str] = [] for item in raw.split(","): item = item.strip() if not item: continue owner, name = gitea.parse_repo(item) repos.append(f"{owner}/{name}") return repos repos: list[str] = [] page = 1 while page <= 5: data = gitea.list_repos(username=username, page=page, limit=50) batch = [r["full_name"] for r in data.get("repos") or [] if r.get("full_name")] if not batch: break repos.extend(batch) if len(batch) < 50: break page += 1 return repos def _should_index_path(path: str) -> bool: lowered = path.lower() if any(part in lowered for part in SKIP_PATH_PARTS): return False ext = os.path.splitext(path)[1].lower() return bool(ext and ext in gitea.TEXT_EXTENSIONS) def _path_priority(path: str) -> tuple[int, str]: lowered = path.lower() for idx, prefix in enumerate(PRIORITY_PREFIXES): if lowered.startswith(prefix.lower()) or os.path.basename(lowered).startswith(prefix.lower()): return (idx, path) return (len(PRIORITY_PREFIXES), path) def list_repo_text_files(repo: str, username: str) -> list[dict]: owner, name = gitea.parse_repo(repo) tree = gitea._request( "GET", f"/repos/{owner}/{name}/git/trees/HEAD", username=username, params={"recursive": "1"}, ) max_files = _max_files_per_repo() max_bytes = _max_file_bytes() files: list[dict] = [] for node in tree.get("tree") or []: if node.get("type") != "blob": continue path = node.get("path") or "" if not _should_index_path(path): continue size = int(node.get("size") or 0) if size > max_bytes: continue files.append({"path": path, "sha": node.get("sha"), "size": size}) files.sort(key=lambda item: _path_priority(item["path"])) return files[:max_files] def _repo_owner(repo: str) -> str: owner, _ = gitea.parse_repo(repo) return owner.lower() def _repo_visibility(private: bool, owner: str) -> str: if private: return "personal" if owner in gitea.MCP_USERS: return "family" return "family" def _collection_for_repo(private: bool, owner: str) -> str: visibility = _repo_visibility(private, owner) if visibility == "personal": return qdrant_store.gitea_collection(owner) return qdrant_store.GITEA_SHARED_COLLECTION def _file_doc_id(repo: str, path: str) -> int: return zlib.adler32(f"{repo}:{path}".encode("utf-8")) & 0x7FFFFFFF def _file_title(repo: str, path: str) -> str: return f"{repo}/{path}" def index_file( repo: str, path: str, *, username: str, private: bool, force: bool = False, ) -> dict: owner = _repo_owner(repo) conn = get_conn() row = conn.execute( "SELECT sha, chunk_count FROM indexed_gitea_files WHERE repo=? AND path=?", (repo, path), ).fetchone() file_data = gitea.get_file(repo, path, username=username) sha = file_data.get("sha") or "" if row and row["sha"] == sha and not force: return {"repo": repo, "path": path, "skipped": True, "sha": sha} text = (file_data.get("content") or "").strip() if not text: return {"repo": repo, "path": path, "chunks": 0, "sha": sha} header = f"# {_file_title(repo, path)}\n\nSource: gitea:{repo}:{path}\n\n" chunks = _chunk_text(header + text) if not chunks: return {"repo": repo, "path": path, "chunks": 0, "sha": sha} visibility = _repo_visibility(private, owner) collection = _collection_for_repo(private, owner) doc_id = _file_doc_id(repo, path) qdrant_store.delete_by_doc(collection, doc_id) vectors = embeddings.embed_texts(chunks) ids = [] payloads = [] for i, chunk in enumerate(chunks): point_id = f"gitea-{repo}-{path}-chunk-{i}" ids.append(point_id) payloads.append( { "source": "gitea", "doc_id": doc_id, "repo": repo, "path": path, "chunk_index": i, "title": _file_title(repo, path), "text": chunk, "owner": owner, "visibility": visibility, "sha": sha, } ) qdrant_store.upsert_chunks(collection, ids, vectors, payloads) stored = qdrant_store.count_by_doc(collection, doc_id) if stored < len(chunks): raise RuntimeError( f"Qdrant upsert incompleto per {repo}/{path}: attesi {len(chunks)} chunk, trovati {stored}" ) conn.execute( "INSERT INTO indexed_gitea_files(repo,path,sha,owner,visibility,chunk_count,indexed_at)" " VALUES (?,?,?,?,?,?,datetime('now'))" " ON CONFLICT(repo,path) DO UPDATE SET" " sha=excluded.sha, owner=excluded.owner, visibility=excluded.visibility," " chunk_count=excluded.chunk_count, indexed_at=datetime('now')", (repo, path, sha, owner, visibility, len(chunks)), ) conn.commit() return { "repo": repo, "path": path, "sha": sha, "chunks": len(chunks), "collection": collection, "visibility": visibility, } def index_repo( repo: str, *, username: Optional[str] = None, private: Optional[bool] = None, force: bool = False, max_files: Optional[int] = None, ) -> dict: gitea_user = username or _repo_owner(repo) if not gitea.is_configured(gitea_user): raise RuntimeError(f"Gitea non configurato per {gitea_user}") if private is None: owner, name = gitea.parse_repo(repo) meta = gitea._request("GET", f"/repos/{owner}/{name}", username=gitea_user) private = bool(meta.get("private")) files = list_repo_text_files(repo, gitea_user) if max_files is not None: files = files[: max(1, max_files)] indexed = 0 skipped = 0 errors = 0 chunks = 0 for item in files: try: result = index_file( repo, item["path"], username=gitea_user, private=bool(private), force=force, ) if result.get("skipped"): skipped += 1 else: indexed += 1 chunks += int(result.get("chunks") or 0) except Exception as exc: LOGGER.warning("Index gitea %s/%s failed: %s", repo, item.get("path"), exc) errors += 1 return { "repo": repo, "files_seen": len(files), "files_indexed": indexed, "files_skipped": skipped, "errors": errors, "chunks": chunks, "private": bool(private), } def index_all(max_files_per_repo: Optional[int] = None) -> dict: if not _enabled(): return {"enabled": False, "indexed_files": 0} total_indexed = 0 total_skipped = 0 total_errors = 0 total_chunks = 0 repos_done: list[str] = [] users = gitea.list_configured_users() or [] for username in users: try: repos = _repos_for_user(username) except Exception as exc: LOGGER.error("Lista repo Gitea fallita per %s: %s", username, exc) continue for repo in repos: try: result = index_repo( repo, username=username, force=False, max_files=max_files_per_repo, ) repos_done.append(repo) total_indexed += int(result.get("files_indexed") or 0) total_skipped += int(result.get("files_skipped") or 0) total_errors += int(result.get("errors") or 0) total_chunks += int(result.get("chunks") or 0) except Exception as exc: LOGGER.warning("Index repo %s failed (%s): %s", repo, username, exc) total_errors += 1 return { "enabled": True, "repos": repos_done, "files_indexed": total_indexed, "files_skipped": total_skipped, "errors": total_errors, "chunks": total_chunks, "users": users, } def gitea_collections_for_user(username: str, is_admin: bool = False) -> list[str]: cols = [qdrant_store.GITEA_SHARED_COLLECTION, qdrant_store.gitea_collection(username)] if is_admin: for user in gitea.MCP_USERS: cols.append(qdrant_store.gitea_collection(user)) return list(dict.fromkeys(cols)) def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list: vectors = embeddings.embed_texts([query]) collections = gitea_collections_for_user(username, is_admin=is_admin) hits = qdrant_store.search(collections, vectors[0], limit=limit) for hit in hits: hit["source"] = hit.get("source") or "gitea" return hits def list_indexed_files(limit: int = 30, repo: Optional[str] = None) -> list: conn = get_conn() if repo: rows = conn.execute( "SELECT * FROM indexed_gitea_files WHERE repo=? ORDER BY indexed_at DESC LIMIT ?", (repo, limit), ).fetchall() else: rows = conn.execute( "SELECT * FROM indexed_gitea_files ORDER BY indexed_at DESC LIMIT ?", (limit,), ).fetchall() return [dict(r) for r in rows] def index_stats() -> dict: conn = get_conn() row = conn.execute( "SELECT COUNT(*), COALESCE(SUM(chunk_count), 0) FROM indexed_gitea_files" ).fetchone() files_count = int(row[0] if row else 0) chunks_meta = int(row[1] if row else 0) collections = {qdrant_store.GITEA_SHARED_COLLECTION} for user in gitea.MCP_USERS: collections.add(qdrant_store.gitea_collection(user)) qdrant_points = sum(qdrant_store.collection_point_count(c) for c in collections) return { "files_indexed": files_count, "chunks_in_metadata": chunks_meta, "qdrant_points": qdrant_points, "collections": {c: qdrant_store.collection_point_count(c) for c in sorted(collections)}, }