loogle-scripts/services/loogle-mcp/app/knowledge/gitea_indexer.py

# -*- coding: utf-8 -*-
"""Indicizzazione semantica (RAG) su file testo dei repository Gitea."""

from __future__ import annotations

import logging
import os
import re
import zlib
from typing import Optional

from ..db import get_conn
from . import embeddings, gitea, qdrant_store
from .text_chunk import CHUNK_OVERLAP, CHUNK_SIZE, chunk_text as _chunk_text

LOGGER = logging.getLogger("loogle_mcp.gitea_indexer")

SKIP_PATH_PARTS = (
    "node_modules/",
    "vendor/",
    ".git/",
    "dist/",
    "build/",
    "__pycache__/",
    ".venv/",
    "venv/",
    ".tox/",
    "coverage/",
)
PRIORITY_PREFIXES = ("docs/", "ha/", "doc/", "README")


def _enabled() -> bool:
    flag = os.environ.get("GITEA_INDEX_ENABLED", "yes").strip().lower()
    return flag not in ("0", "false", "no", "off")


def _max_files_per_repo() -> int:
    return int(os.environ.get("GITEA_INDEX_MAX_FILES_PER_REPO", "150"))


def _max_file_bytes() -> int:
    return int(os.environ.get("GITEA_INDEX_MAX_FILE_BYTES", "120000"))


def _repos_for_user(username: str) -> list[str]:
    env_key = f"GITEA_INDEX_REPOS_{username.upper()}"
    raw = os.environ.get(env_key, "").strip()
    if raw:
        repos: list[str] = []
        for item in raw.split(","):
            item = item.strip()
            if not item:
                continue
            owner, name = gitea.parse_repo(item)
            repos.append(f"{owner}/{name}")
        return repos
    repos: list[str] = []
    page = 1
    while page <= 5:
        data = gitea.list_repos(username=username, page=page, limit=50)
        batch = [r["full_name"] for r in data.get("repos") or [] if r.get("full_name")]
        if not batch:
            break
        repos.extend(batch)
        if len(batch) < 50:
            break
        page += 1
    return repos


def _should_index_path(path: str) -> bool:
    lowered = path.lower()
    if any(part in lowered for part in SKIP_PATH_PARTS):
        return False
    ext = os.path.splitext(path)[1].lower()
    return bool(ext and ext in gitea.TEXT_EXTENSIONS)


def _path_priority(path: str) -> tuple[int, str]:
    lowered = path.lower()
    for idx, prefix in enumerate(PRIORITY_PREFIXES):
        if lowered.startswith(prefix.lower()) or os.path.basename(lowered).startswith(prefix.lower()):
            return (idx, path)
    return (len(PRIORITY_PREFIXES), path)


def list_repo_text_files(repo: str, username: str) -> list[dict]:
    owner, name = gitea.parse_repo(repo)
    tree = gitea._request(
        "GET",
        f"/repos/{owner}/{name}/git/trees/HEAD",
        username=username,
        params={"recursive": "1"},
    )
    max_files = _max_files_per_repo()
    max_bytes = _max_file_bytes()
    files: list[dict] = []
    for node in tree.get("tree") or []:
        if node.get("type") != "blob":
            continue
        path = node.get("path") or ""
        if not _should_index_path(path):
            continue
        size = int(node.get("size") or 0)
        if size > max_bytes:
            continue
        files.append({"path": path, "sha": node.get("sha"), "size": size})
    files.sort(key=lambda item: _path_priority(item["path"]))
    return files[:max_files]


def _repo_owner(repo: str) -> str:
    owner, _ = gitea.parse_repo(repo)
    return owner.lower()


def _repo_visibility(private: bool, owner: str) -> str:
    if private:
        return "personal"
    if owner in gitea.MCP_USERS:
        return "family"
    return "family"


def _collection_for_repo(private: bool, owner: str) -> str:
    visibility = _repo_visibility(private, owner)
    if visibility == "personal":
        return qdrant_store.gitea_collection(owner)
    return qdrant_store.GITEA_SHARED_COLLECTION


def _file_doc_id(repo: str, path: str) -> int:
    return zlib.adler32(f"{repo}:{path}".encode("utf-8")) & 0x7FFFFFFF


def _file_title(repo: str, path: str) -> str:
    return f"{repo}/{path}"


def index_file(
    repo: str,
    path: str,
    *,
    username: str,
    private: bool,
    force: bool = False,
) -> dict:
    owner = _repo_owner(repo)
    conn = get_conn()
    row = conn.execute(
        "SELECT sha, chunk_count FROM indexed_gitea_files WHERE repo=? AND path=?",
        (repo, path),
    ).fetchone()

    file_data = gitea.get_file(repo, path, username=username)
    sha = file_data.get("sha") or ""
    if row and row["sha"] == sha and not force:
        return {"repo": repo, "path": path, "skipped": True, "sha": sha}

    text = (file_data.get("content") or "").strip()
    if not text:
        return {"repo": repo, "path": path, "chunks": 0, "sha": sha}

    header = f"# {_file_title(repo, path)}\n\nSource: gitea:{repo}:{path}\n\n"
    chunks = _chunk_text(header + text)
    if not chunks:
        return {"repo": repo, "path": path, "chunks": 0, "sha": sha}

    visibility = _repo_visibility(private, owner)
    collection = _collection_for_repo(private, owner)
    doc_id = _file_doc_id(repo, path)
    qdrant_store.delete_by_doc(collection, doc_id)

    vectors = embeddings.embed_texts(chunks)
    ids = []
    payloads = []
    for i, chunk in enumerate(chunks):
        point_id = f"gitea-{repo}-{path}-chunk-{i}"
        ids.append(point_id)
        payloads.append(
            {
                "source": "gitea",
                "doc_id": doc_id,
                "repo": repo,
                "path": path,
                "chunk_index": i,
                "title": _file_title(repo, path),
                "text": chunk,
                "owner": owner,
                "visibility": visibility,
                "sha": sha,
            }
        )
    qdrant_store.upsert_chunks(collection, ids, vectors, payloads)

    stored = qdrant_store.count_by_doc(collection, doc_id)
    if stored < len(chunks):
        raise RuntimeError(
            f"Qdrant upsert incompleto per {repo}/{path}: attesi {len(chunks)} chunk, trovati {stored}"
        )

    conn.execute(
        "INSERT INTO indexed_gitea_files(repo,path,sha,owner,visibility,chunk_count,indexed_at)"
        " VALUES (?,?,?,?,?,?,datetime('now'))"
        " ON CONFLICT(repo,path) DO UPDATE SET"
        " sha=excluded.sha, owner=excluded.owner, visibility=excluded.visibility,"
        " chunk_count=excluded.chunk_count, indexed_at=datetime('now')",
        (repo, path, sha, owner, visibility, len(chunks)),
    )
    conn.commit()
    return {
        "repo": repo,
        "path": path,
        "sha": sha,
        "chunks": len(chunks),
        "collection": collection,
        "visibility": visibility,
    }


def index_repo(
    repo: str,
    *,
    username: Optional[str] = None,
    private: Optional[bool] = None,
    force: bool = False,
    max_files: Optional[int] = None,
) -> dict:
    gitea_user = username or _repo_owner(repo)
    if not gitea.is_configured(gitea_user):
        raise RuntimeError(f"Gitea non configurato per {gitea_user}")

    if private is None:
        owner, name = gitea.parse_repo(repo)
        meta = gitea._request("GET", f"/repos/{owner}/{name}", username=gitea_user)
        private = bool(meta.get("private"))

    files = list_repo_text_files(repo, gitea_user)
    if max_files is not None:
        files = files[: max(1, max_files)]

    indexed = 0
    skipped = 0
    errors = 0
    chunks = 0
    for item in files:
        try:
            result = index_file(
                repo,
                item["path"],
                username=gitea_user,
                private=bool(private),
                force=force,
            )
            if result.get("skipped"):
                skipped += 1
            else:
                indexed += 1
                chunks += int(result.get("chunks") or 0)
        except Exception as exc:
            LOGGER.warning("Index gitea %s/%s failed: %s", repo, item.get("path"), exc)
            errors += 1

    return {
        "repo": repo,
        "files_seen": len(files),
        "files_indexed": indexed,
        "files_skipped": skipped,
        "errors": errors,
        "chunks": chunks,
        "private": bool(private),
    }


def index_all(max_files_per_repo: Optional[int] = None) -> dict:
    if not _enabled():
        return {"enabled": False, "indexed_files": 0}

    total_indexed = 0
    total_skipped = 0
    total_errors = 0
    total_chunks = 0
    repos_done: list[str] = []
    users = gitea.list_configured_users() or []

    for username in users:
        try:
            repos = _repos_for_user(username)
        except Exception as exc:
            LOGGER.error("Lista repo Gitea fallita per %s: %s", username, exc)
            continue
        for repo in repos:
            try:
                result = index_repo(
                    repo,
                    username=username,
                    force=False,
                    max_files=max_files_per_repo,
                )
                repos_done.append(repo)
                total_indexed += int(result.get("files_indexed") or 0)
                total_skipped += int(result.get("files_skipped") or 0)
                total_errors += int(result.get("errors") or 0)
                total_chunks += int(result.get("chunks") or 0)
            except Exception as exc:
                LOGGER.warning("Index repo %s failed (%s): %s", repo, username, exc)
                total_errors += 1

    return {
        "enabled": True,
        "repos": repos_done,
        "files_indexed": total_indexed,
        "files_skipped": total_skipped,
        "errors": total_errors,
        "chunks": total_chunks,
        "users": users,
    }


def gitea_collections_for_user(username: str, is_admin: bool = False) -> list[str]:
    cols = [qdrant_store.GITEA_SHARED_COLLECTION, qdrant_store.gitea_collection(username)]
    if is_admin:
        for user in gitea.MCP_USERS:
            cols.append(qdrant_store.gitea_collection(user))
    return list(dict.fromkeys(cols))


def search_gitea_knowledge(username: str, query: str, limit: int = 8, is_admin: bool = False) -> list:
    vectors = embeddings.embed_texts([query])
    collections = gitea_collections_for_user(username, is_admin=is_admin)
    hits = qdrant_store.search(collections, vectors[0], limit=limit)
    for hit in hits:
        hit["source"] = hit.get("source") or "gitea"
    return hits


def list_indexed_files(limit: int = 30, repo: Optional[str] = None) -> list:
    conn = get_conn()
    if repo:
        rows = conn.execute(
            "SELECT * FROM indexed_gitea_files WHERE repo=? ORDER BY indexed_at DESC LIMIT ?",
            (repo, limit),
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT * FROM indexed_gitea_files ORDER BY indexed_at DESC LIMIT ?",
            (limit,),
        ).fetchall()
    return [dict(r) for r in rows]


def index_stats() -> dict:
    conn = get_conn()
    row = conn.execute(
        "SELECT COUNT(*), COALESCE(SUM(chunk_count), 0) FROM indexed_gitea_files"
    ).fetchone()
    files_count = int(row[0] if row else 0)
    chunks_meta = int(row[1] if row else 0)
    collections = {qdrant_store.GITEA_SHARED_COLLECTION}
    for user in gitea.MCP_USERS:
        collections.add(qdrant_store.gitea_collection(user))
    qdrant_points = sum(qdrant_store.collection_point_count(c) for c in collections)
    return {
        "files_indexed": files_count,
        "chunks_in_metadata": chunks_meta,
        "qdrant_points": qdrant_points,
        "collections": {c: qdrant_store.collection_point_count(c) for c in sorted(collections)},
    }