Files
loogle-scripts/services/loogle-mcp/scripts/export_daily_agent_digest.py
T

722 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Esporta un blocco giornaliero ricco su progetto MCP (default: homelab-loogle).
Fonti (massimo contesto utile, non dump grezzo):
- transcript agenti Cursor (query utente + conclusioni assistente)
- audit_log tool MCP del giorno
- commit git recenti in repo tipici (rete, loogle-mcp)
- piani Cursor `~/.cursor/plans/*.plan.md` → artifacts/plans/ + indice in context.md
Idempotente: marker <!-- daily-digest:YYYY-MM-DD --> — con --force sostituisce il blocco del giorno.
Esempi:
python3 scripts/export_daily_agent_digest.py
python3 scripts/export_daily_agent_digest.py --date 2026-09-03 --force
python3 scripts/export_daily_agent_digest.py --dry-run
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shutil
import sqlite3
import subprocess
import sys
from collections import Counter
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any, Optional
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
MARKER_RE = re.compile(r"<!--\s*daily-digest:(\d{4}-\d{2}-\d{2})\s*-->")
USER_QUERY_RE = re.compile(
r"<timestamp>(.*?)</timestamp>\s*<user_query>\s*(.*?)\s*</user_query>",
re.DOTALL | re.IGNORECASE,
)
TS_PREFIX_RE = re.compile(r"^\[?\d{4}-\d{2}-\d{2}")
FRONTMATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", re.DOTALL)
DEFAULT_TRANSCRIPT_ROOTS = [
Path.home() / ".cursor/projects/home-daniely/agent-transcripts",
Path.home() / ".cursor/projects/home-daniely-docker-loogle-mcp/agent-transcripts",
]
DEFAULT_GIT_REPOS = [
Path.home() / "rete",
Path.home() / "docker/loogle-mcp",
]
DEFAULT_PLANS_DIR = Path.home() / ".cursor/plans"
def _load_dotenv() -> None:
env_path = ROOT / ".env"
if not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip().strip("'").strip('"')
def _configure_paths() -> None:
if Path("/.dockerenv").is_file() or (Path("/data").is_dir() and os.access("/data", os.W_OK)):
os.environ.setdefault("MCP_DB", "/data/loogle_mcp.db")
os.environ.setdefault("MCP_CONTEXT_ROOT", "/data/context")
os.environ.setdefault("MCP_VECTOR_FALLBACK", "/data/vector_fallback.db")
return
os.environ.setdefault("MCP_DB", str(ROOT / "data" / "loogle_mcp.db"))
os.environ.setdefault("MCP_CONTEXT_ROOT", "/mnt/ha-apps/mcp/context")
os.environ.setdefault("MCP_VECTOR_FALLBACK", str(ROOT / "data" / "vector_fallback.db"))
def _parse_day(s: Optional[str]) -> date:
if not s:
return date.today()
return date.fromisoformat(s)
def _day_bounds(day: date) -> tuple[datetime, datetime]:
start = datetime.combine(day, datetime.min.time())
end = start + timedelta(days=1)
return start, end
def _is_noise_query(q: str) -> bool:
q = (q or "").strip()
if len(q) < 12:
return True
low = q.lower()
if q.startswith("<") or q.startswith("[REDACTED]"):
return True
noise_prefixes = (
"you are ",
"you have access",
"start multitasking",
"briefly inform the user",
"perform any necessary follow-up",
"the following task has finished",
"<mcp_",
"<dynamic_tools>",
"<agent_transcripts>",
)
return any(low.startswith(p) or p in low[:80] for p in noise_prefixes)
def _truncate(text: str, limit: int) -> str:
text = re.sub(r"\s+", " ", (text or "").strip())
if len(text) <= limit:
return text
return text[: limit - 1].rstrip() + "…"
def _extract_text_blocks(message: Any) -> list[str]:
if not isinstance(message, dict):
return []
content = message.get("content")
if isinstance(content, str):
return [content]
out: list[str] = []
if isinstance(content, list):
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
t = part.get("text") or ""
if t and t != "[REDACTED]":
out.append(t)
return out
def _parse_event_time(raw: str, file_mtime: float) -> Optional[datetime]:
raw = (raw or "").strip()
cleaned = re.sub(r"\s*\([^)]*\)\s*$", "", raw).strip()
for fmt in (
"%A, %b %d, %Y, %I:%M %p",
"%A, %B %d, %Y, %I:%M %p",
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%d %H:%M:%S",
):
try:
return datetime.strptime(cleaned, fmt)
except ValueError:
continue
return datetime.fromtimestamp(file_mtime)
def collect_transcripts(day: date, roots: list[Path], max_chats: int = 25) -> list[dict]:
start, end = _day_bounds(day)
chats: list[dict] = []
files: list[Path] = []
for root in roots:
if not root.is_dir():
continue
for path in root.rglob("*.jsonl"):
if "subagents" in path.parts:
continue
files.append(path)
for path in sorted(files, key=lambda p: p.stat().st_mtime, reverse=True):
mtime = path.stat().st_mtime
# quick skip: file entirely older than day-1 or newer handled by content
if datetime.fromtimestamp(mtime) < start - timedelta(days=2):
continue
user_queries: list[str] = []
assistant_tails: list[str] = []
paths_touched: Counter[str] = Counter()
day_hit = False
try:
with open(path, encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
role = obj.get("role")
for text in _extract_text_blocks(obj.get("message") or {}):
if role == "user":
for ts_raw, query in USER_QUERY_RE.findall(text):
when = _parse_event_time(ts_raw, mtime)
if when and start <= when < end:
day_hit = True
q = _truncate(query, 280)
if not _is_noise_query(q):
user_queries.append(q)
# bare user text without wrapper — solo se breve messaggio umano
if (
"<user_query>" not in text
and start.timestamp() <= mtime < end.timestamp()
and len(text) > 40
and not _is_noise_query(text)
and "<" not in text[:20]
):
day_hit = True
user_queries.append(_truncate(text, 280))
elif role == "assistant":
if start.timestamp() <= mtime < end.timestamp() or day_hit:
if (
len(text) > 80
and not text.startswith("[REDACTED]")
and not text.startswith("<")
and "tool_use" not in text[:40]
):
# preferisci paragrafi conclusivi (markdown grassetto / verdetto)
assistant_tails.append(_truncate(text, 320))
# tool paths
msg = obj.get("message") or {}
content = msg.get("content") if isinstance(msg, dict) else None
if isinstance(content, list):
for part in content:
if not isinstance(part, dict) or part.get("type") != "tool_use":
continue
inp = part.get("input") or {}
for key in ("path", "target_notebook", "file_path"):
if key in inp and isinstance(inp[key], str):
paths_touched[inp[key]] += 1
except OSError:
continue
if not day_hit and not user_queries:
# include if file modified that day and has substance
if not (start.timestamp() <= mtime < end.timestamp()):
continue
if not assistant_tails:
continue
# dedupe queries
seen = set()
uniq_q = []
for q in user_queries:
if q not in seen:
seen.add(q)
uniq_q.append(q)
chats.append(
{
"id": path.parent.name if path.parent.name != "agent-transcripts" else path.stem,
"queries": uniq_q[:8],
"conclusions": assistant_tails[-3:],
"paths": [p for p, _ in paths_touched.most_common(8)],
}
)
if len(chats) >= max_chats:
break
return chats
def collect_audit(day: date, db_path: Path) -> dict:
if not db_path.is_file():
return {"tools": [], "total": 0}
start = day.isoformat()
end = (day + timedelta(days=1)).isoformat()
conn = sqlite3.connect(str(db_path))
conn.row_factory = sqlite3.Row
rows = conn.execute(
"SELECT tool_name, COUNT(*) AS n FROM audit_log"
" WHERE created_at >= ? AND created_at < ? AND username=?"
" GROUP BY tool_name ORDER BY n DESC",
(start, end, "daniele"),
).fetchall()
samples = conn.execute(
"SELECT created_at, tool_name, detail FROM audit_log"
" WHERE created_at >= ? AND created_at < ? AND username=?"
" ORDER BY id DESC LIMIT 15",
(start, end, "daniele"),
).fetchall()
conn.close()
return {
"tools": [(r["tool_name"], r["n"]) for r in rows],
"total": sum(r["n"] for r in rows),
"samples": [
{
"at": r["created_at"],
"tool": r["tool_name"],
"detail": _truncate(r["detail"] or "", 120),
}
for r in samples
],
}
def collect_git(day: date, repos: list[Path], limit: int = 12) -> list[str]:
since = day.isoformat()
until = (day + timedelta(days=1)).isoformat()
lines: list[str] = []
for repo in repos:
if not (repo / ".git").exists():
continue
try:
out = subprocess.check_output(
[
"git",
"-C",
str(repo),
"log",
f"--since={since}",
f"--until={until}",
"--pretty=format:%h %s",
f"-n{limit}",
],
stderr=subprocess.DEVNULL,
text=True,
timeout=15,
).strip()
except (subprocess.CalledProcessError, subprocess.TimeoutExpired):
continue
if out:
for line in out.splitlines():
lines.append(f"{repo.name}: {line}")
return lines[:limit]
def _parse_plan_file(path: Path) -> dict:
text = path.read_text(encoding="utf-8", errors="replace")
name = path.stem
overview = ""
todos_total = 0
todos_done = 0
m = FRONTMATTER_RE.match(text)
if m:
fm = m.group(1)
nm = re.search(r"^name:\s*(.+)$", fm, re.M)
if nm:
name = nm.group(1).strip().strip("\"'")
ov = re.search(r"^overview:\s*(.+)$", fm, re.M)
if ov:
overview = ov.group(1).strip().strip("\"'")
# overview può essere su più righe YAML quoted — fallback grezzo
if overview.startswith("|") or not overview:
ov2 = re.search(r"^overview:\s*[>|]?\s*\n((?:[ \t]+.+\n)+)", fm, re.M)
if ov2:
overview = " ".join(line.strip() for line in ov2.group(1).splitlines())
statuses = re.findall(r"^\s+status:\s*(\w+)", fm, re.M)
todos_total = len(statuses)
todos_done = sum(1 for s in statuses if s == "completed")
mtime = datetime.fromtimestamp(path.stat().st_mtime)
return {
"file": path.name,
"name": name,
"overview": _truncate(overview, 220),
"todos_total": todos_total,
"todos_done": todos_done,
"mtime": mtime,
"text": text,
"path": path,
}
def collect_and_sync_plans(
day: date,
plans_dir: Path,
dest_dir: Path,
*,
dry_run: bool = False,
) -> dict:
"""Copia tutti i .plan.md in artifacts/plans/; ritorna catalogo + aggiornati nel giorno."""
start, end = _day_bounds(day)
plans: list[dict] = []
if plans_dir.is_dir():
for path in sorted(plans_dir.glob("*.plan.md")):
try:
plans.append(_parse_plan_file(path))
except OSError:
continue
copied = 0
updated_today: list[dict] = []
if not dry_run:
dest_dir.mkdir(parents=True, exist_ok=True)
for plan in plans:
target = dest_dir / plan["file"]
shutil.copy2(plan["path"], target)
copied += 1
if start <= plan["mtime"] < end:
updated_today.append(plan)
# INDEX.md per lettura umana / futuri tool
idx_lines = [
"# Piani Cursor (sync)",
"",
f"_Aggiornato {datetime.now().astimezone().strftime('%Y-%m-%d %H:%M %Z')} "
f"da `~/.cursor/plans` → `artifacts/plans/`._",
"",
]
for plan in sorted(plans, key=lambda p: p["mtime"], reverse=True):
prog = (
f"{plan['todos_done']}/{plan['todos_total']}"
if plan["todos_total"]
else "?"
)
idx_lines.append(
f"- **{plan['name']}** (`{plan['file']}`, todos {prog}, "
f"mtime {plan['mtime'].date().isoformat()})"
)
if plan["overview"]:
idx_lines.append(f" - {plan['overview']}")
idx_lines.append("")
(dest_dir / "INDEX.md").write_text("\n".join(idx_lines), encoding="utf-8")
else:
updated_today = [p for p in plans if start <= p["mtime"] < end]
return {
"plans": plans,
"copied": copied,
"updated_today": updated_today,
"dest": str(dest_dir),
}
def upsert_plans_index_section(context_md: str, plans: list[dict]) -> str:
"""Sezione stabile in context.md (non nel digest giornaliero) con indice piani."""
begin = "<!-- BEGIN CURSOR PLANS -->"
end = "<!-- END CURSOR PLANS -->"
lines = [
begin,
"## Piani Cursor (indice sync)",
"",
"_File completi in `artifacts/plans/`. Qui solo indice per `get_project_context`._",
"",
]
if not plans:
lines.append("- Nessun piano in `~/.cursor/plans`.")
else:
for plan in sorted(plans, key=lambda p: p["mtime"], reverse=True):
prog = (
f"{plan['todos_done']}/{plan['todos_total']} done"
if plan["todos_total"]
else "n/d"
)
lines.append(
f"- **{plan['name']}** — {prog} — `{plan['file']}` "
f"({plan['mtime'].date().isoformat()})"
)
if plan["overview"]:
lines.append(f" - {plan['overview']}")
lines.extend(["", end, ""])
block = "\n".join(lines)
if begin in context_md and end in context_md:
pattern = re.compile(
re.escape(begin) + r".*?" + re.escape(end),
re.DOTALL,
)
return pattern.sub(block.strip(), context_md)
# inserisci prima dei daily digests se presenti
dig = "<!-- BEGIN DAILY DIGESTS -->"
if dig in context_md:
head, tail = context_md.split(dig, 1)
return head.rstrip() + "\n\n" + block + "\n" + dig + tail
return context_md.rstrip() + "\n\n" + block
def build_markdown(
day: date,
chats: list[dict],
audit: dict,
git_lines: list[str],
project_id: str,
plans_updated: Optional[list[dict]] = None,
) -> str:
lines: list[str] = [
f"<!-- daily-digest:{day.isoformat()} -->",
f"## Digest agenti {day.isoformat()} — `{project_id}`",
"",
f"_Generato automaticamente da `export_daily_agent_digest.py` "
f"({datetime.now().astimezone().strftime('%Y-%m-%d %H:%M %Z')})._",
"",
]
lines.append("### Chat / agenti Cursor")
if not chats:
lines.append("- Nessun transcript rilevante per questo giorno.")
else:
lines.append(f"- Sessioni considerate: **{len(chats)}**")
for i, chat in enumerate(chats, 1):
title = chat["queries"][0] if chat["queries"] else chat["id"]
lines.append(f"{i}. **{_truncate(title, 120)}** `[{chat['id'][:8]}]`")
for q in chat["queries"][1:4]:
lines.append(f" - Q: {_truncate(q, 160)}")
for c in chat["conclusions"][-2:]:
lines.append(f" - → {_truncate(c, 200)}")
if chat["paths"]:
short_paths = ", ".join(_truncate(p, 60) for p in chat["paths"][:5])
lines.append(f" - File: `{short_paths}`")
lines.append("")
lines.append("### Tool MCP usati")
if not audit.get("total"):
lines.append("- Nessuna chiamata tool in audit_log.")
else:
lines.append(f"- Totale chiamate: **{audit['total']}**")
top = ", ".join(f"`{name}`×{n}" for name, n in audit["tools"][:10])
lines.append(f"- Top: {top}")
for s in (audit.get("samples") or [])[:8]:
lines.append(f" - {s['at']} `{s['tool']}` {s['detail']}")
lines.append("")
lines.append("### Commit git (homelab)")
if not git_lines:
lines.append("- Nessun commit nel giorno.")
else:
for g in git_lines:
lines.append(f"- `{g}`")
lines.append("")
lines.append("### Piani Cursor aggiornati oggi")
plans_updated = plans_updated or []
if not plans_updated:
lines.append("- Nessun `.plan.md` modificato in questa data.")
else:
for plan in plans_updated:
prog = (
f"{plan['todos_done']}/{plan['todos_total']}"
if plan["todos_total"]
else "?"
)
lines.append(
f"- **{plan['name']}** (`artifacts/plans/{plan['file']}`, todos {prog})"
)
if plan["overview"]:
lines.append(f" - {plan['overview']}")
lines.append("")
lines.append("### Per Claude / prossimi agenti")
lines.append(
"- Usa questo blocco come memoria del giorno; per dettagli codice preferisci "
"`search_gitea_knowledge` / `get_file` sui path citati."
)
lines.append(
"- Piani completi: sezione **Piani Cursor** in context.md + file in `artifacts/plans/`."
)
lines.append(
"- Non ripetere setup già conclusi; riparti da decisioni e next step qui sopra."
)
lines.append("")
return "\n".join(lines)
def upsert_daily_block(context_md: str, day: date, block: str) -> str:
"""Rimuove eventuale blocco del giorno e inserisce il nuovo in cima alla sezione digest."""
pattern = re.compile(
rf"<!--\s*daily-digest:{day.isoformat()}\s*-->.*?"
rf"(?=<!--\s*daily-digest:\d{{4}}-\d{{2}}-\d{{2}}\s*-->|<!--\s*END DAILY DIGESTS\s*-->|\Z)",
re.DOTALL,
)
context_md = pattern.sub("", context_md)
begin = "<!-- BEGIN DAILY DIGESTS -->"
end = "<!-- END DAILY DIGESTS -->"
if begin not in context_md:
context_md = context_md.rstrip() + f"\n\n{begin}\n\n{end}\n"
# Assicura END
if end not in context_md:
context_md = context_md.rstrip() + f"\n\n{end}\n"
head, rest = context_md.split(begin, 1)
# rest inizia dopo BEGIN; togli END temporaneamente dalla porzione digest
if end in rest:
mid, tail = rest.split(end, 1)
else:
mid, tail = rest, ""
mid = mid.strip()
new_mid = block.strip() + ("\n\n" + mid if mid else "")
return head.rstrip() + f"\n\n{begin}\n\n{new_mid}\n\n{end}" + tail
def has_daily_block(context_md: str, day: date) -> bool:
return f"<!-- daily-digest:{day.isoformat()} -->" in context_md
def main() -> int:
parser = argparse.ArgumentParser(description="Digest giornaliero agenti → MCP context")
parser.add_argument("--date", help="YYYY-MM-DD (default: oggi)")
parser.add_argument("--user", default="daniele")
parser.add_argument("--project", default="homelab-loogle")
parser.add_argument("--force", action="store_true", help="Sostituisci blocco del giorno se già presente")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--no-index", action="store_true", help="Non aggiornare Qdrant ctx_*")
parser.add_argument("--max-chats", type=int, default=20)
parser.add_argument(
"--no-plans",
action="store_true",
help="Non sincronizzare ~/.cursor/plans",
)
args = parser.parse_args()
_load_dotenv()
_configure_paths()
day = _parse_day(args.date)
proj = Path(os.environ["MCP_CONTEXT_ROOT"]) / args.user / "projects" / args.project
plans_dest = proj / "artifacts" / "plans"
plans_info: dict = {"plans": [], "copied": 0, "updated_today": [], "dest": str(plans_dest)}
if not args.no_plans:
plans_info = collect_and_sync_plans(
day,
DEFAULT_PLANS_DIR,
plans_dest,
dry_run=args.dry_run,
)
chats = collect_transcripts(day, DEFAULT_TRANSCRIPT_ROOTS, max_chats=args.max_chats)
audit = collect_audit(day, Path(os.environ["MCP_DB"]))
git_lines = collect_git(day, DEFAULT_GIT_REPOS)
block = build_markdown(
day,
chats,
audit,
git_lines,
args.project,
plans_updated=plans_info.get("updated_today") or [],
)
if args.dry_run:
print(block)
if plans_info.get("plans"):
print("\n# plans index preview", file=sys.stderr)
for p in plans_info["plans"][:5]:
print(f"# - {p['name']} ({p['file']})", file=sys.stderr)
print(
f"\n# dry-run chats={len(chats)} audit={audit.get('total', 0)} "
f"git={len(git_lines)} plans={len(plans_info.get('plans') or [])} "
f"plans_today={len(plans_info.get('updated_today') or [])} chars={len(block)}",
file=sys.stderr,
)
return 0
from app.context import store as context_store
from app.db import init_db
from app.knowledge import indexer
init_db()
data = context_store.get_project_context(args.user, args.project, session_limit=1, include_gitea=False)
existing = data.get("context_md") or ""
# I piani si sincronizzano sempre; il digest può essere skippato se già presente
existing = upsert_plans_index_section(existing, plans_info.get("plans") or [])
digest_skipped = False
if has_daily_block(existing, day) and not args.force:
digest_skipped = True
new_md = existing
print(f"SKIP digest: {day.isoformat()} già presente (usa --force); plans sync ok")
else:
new_md = upsert_daily_block(existing, day, block)
ctx_path = proj / "context.md"
meta_path = proj / "meta.json"
sessions = proj / "sessions"
sessions.mkdir(parents=True, exist_ok=True)
ctx_path.write_text(new_md, encoding="utf-8")
meta = json.loads(meta_path.read_text(encoding="utf-8"))
meta["updated_at"] = datetime.now().astimezone().isoformat(timespec="seconds")
meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
snap = None
if not digest_skipped:
snap = sessions / f"{day.isoformat()}-daily.md"
snap.write_text(block, encoding="utf-8")
if not args.no_index:
try:
if not digest_skipped:
indexer.index_context_snippet(
args.user,
args.project,
block,
f"Digest {day.isoformat()}{meta.get('title', args.project)}",
snippet_id=f"daily-{day.isoformat()}",
)
# indicizza ogni piano (testo ridotto: name+overview+body troncato)
for plan in plans_info.get("plans") or []:
body = plan["text"]
if len(body) > 12000:
body = body[:12000] + "\n…[troncato]"
indexer.index_context_snippet(
args.user,
args.project,
f"# Piano Cursor: {plan['name']}\n\n{plan['overview']}\n\n{body}",
f"Plan: {plan['name']}",
snippet_id=f"plan-{plan['file']}",
)
print("indexed: ok")
except Exception as exc:
print(f"indexed: skip ({exc})")
print(
json.dumps(
{
"ok": True,
"day": day.isoformat(),
"project": args.project,
"digest_skipped": digest_skipped,
"chats": len(chats),
"audit_calls": audit.get("total", 0),
"git_commits": len(git_lines),
"plans_synced": plans_info.get("copied", 0),
"plans_updated_today": len(plans_info.get("updated_today") or []),
"chars": len(block),
"context_chars": len(new_md),
"session": str(snap) if snap else None,
"plans_dest": plans_info.get("dest"),
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())