Backup automatico script del 2026-08-23 12:04
This commit is contained in:
daniele committed
1 parent
11823447a2
commit
795a15a7b6
66 files changed
+8546
No files matched your search
@@ -0,0 +1,242 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Thermal / load gate per proteggere il DS920 durante gli embedding Ollama.
|
||||
|
||||
Legge temperatura CPU e load average dal NAS e regola pause/keep_alive.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
LOGGER = logging.getLogger("loogle_mcp.thermal")
|
||||
|
||||
# Soft: rallenta. Hard: pausa (raro se il profilo lento tiene).
|
||||
# Profilo "lento ma regolare": anticipare soft, cap load basso.
|
||||
DEFAULT_SOFT_C = 58.0
|
||||
DEFAULT_HARD_C = 70.0
|
||||
DEFAULT_CPU_TARGET_PCT = 40.0 # load1 <= nproc * 0.40
|
||||
|
||||
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def soft_temp_c() -> float:
|
||||
return _float_env("THERMAL_TEMP_SOFT_C", DEFAULT_SOFT_C)
|
||||
|
||||
|
||||
def hard_temp_c() -> float:
|
||||
return _float_env("THERMAL_TEMP_HARD_C", DEFAULT_HARD_C)
|
||||
|
||||
|
||||
def cpu_target_pct() -> float:
|
||||
return _float_env("THERMAL_CPU_TARGET_PCT", DEFAULT_CPU_TARGET_PCT)
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
flag = os.environ.get("THERMAL_GATE_ENABLED", "yes").strip().lower()
|
||||
return flag not in ("0", "false", "no", "off")
|
||||
|
||||
|
||||
def _read_via_http() -> Optional[dict]:
|
||||
url = os.environ.get("DS920_THERMAL_URL", "").strip()
|
||||
if not url:
|
||||
# default probe se non configurato
|
||||
url = os.environ.get(
|
||||
"DS920_THERMAL_URL_DEFAULT",
|
||||
"http://192.168.128.100:9191/thermal",
|
||||
).strip()
|
||||
try:
|
||||
with httpx.Client(timeout=3.0) as client:
|
||||
resp = client.get(url)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json()
|
||||
if isinstance(data, dict) and "cpu_temp_c" in data:
|
||||
return data
|
||||
except Exception as exc:
|
||||
LOGGER.debug("Thermal HTTP probe fallita: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _read_via_ssh() -> Optional[dict]:
|
||||
host = os.environ.get("DS920_SSH_HOST", "192.168.128.100").strip()
|
||||
user = os.environ.get("DS920_SSH_USER", "daniely").strip()
|
||||
key = os.environ.get("DS920_SSH_KEY", "").strip()
|
||||
if not host:
|
||||
return None
|
||||
remote = (
|
||||
"python3 -c \"import json,os;"
|
||||
"b='/sys/class/hwmon/hwmon0';"
|
||||
"t=[int(open(f'{b}/'+n).read())/1000 for n in sorted(os.listdir(b)) "
|
||||
"if n.startswith('temp') and n.endswith('_input')];"
|
||||
"l=os.getloadavg();"
|
||||
"print(json.dumps({'cpu_temp_c':max(t) if t else None,"
|
||||
"'load1':l[0],'load5':l[1],'nproc':os.cpu_count() or 4}))\""
|
||||
)
|
||||
cmd = [
|
||||
"ssh",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
]
|
||||
if key and os.path.isfile(key):
|
||||
cmd.extend(["-i", key])
|
||||
cmd.append(f"{user}@{host}")
|
||||
cmd.append(remote)
|
||||
try:
|
||||
out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL, timeout=12, text=True)
|
||||
data = json.loads(out.strip())
|
||||
if isinstance(data, dict) and data.get("cpu_temp_c") is not None:
|
||||
return data
|
||||
except Exception as exc:
|
||||
LOGGER.debug("Thermal SSH probe fallita: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def read_status() -> Optional[dict]:
|
||||
"""Ritorna {cpu_temp_c, load1, load5?, nproc} oppure None se non raggiungibile."""
|
||||
data = _read_via_http()
|
||||
if data:
|
||||
data["source"] = "http"
|
||||
return data
|
||||
data = _read_via_ssh()
|
||||
if data:
|
||||
data["source"] = "ssh"
|
||||
return data
|
||||
return None
|
||||
|
||||
|
||||
def load_over_target(status: dict) -> bool:
|
||||
load1 = float(status.get("load1") or 0)
|
||||
nproc = float(status.get("nproc") or 4)
|
||||
target = nproc * (cpu_target_pct() / 100.0)
|
||||
return load1 > target
|
||||
|
||||
|
||||
def suggested_keep_alive(status: Optional[dict]) -> int:
|
||||
"""Secondi keep_alive Ollama.
|
||||
|
||||
Il container resta sempre acceso: non si fa unload per throttling termico
|
||||
(evita cicli load/unload). Si scarica solo se OLLAMA_UNLOAD_ON_HARD=yes.
|
||||
"""
|
||||
cool = int(_float_env("OLLAMA_KEEP_ALIVE_COOL", 300))
|
||||
default = int(_float_env("OLLAMA_KEEP_ALIVE_DEFAULT", 120))
|
||||
if not status or status.get("cpu_temp_c") is None:
|
||||
return default
|
||||
temp = float(status["cpu_temp_c"])
|
||||
unload = os.environ.get("OLLAMA_UNLOAD_ON_HARD", "no").strip().lower()
|
||||
if temp >= hard_temp_c() and unload in ("1", "true", "yes", "on"):
|
||||
return 0
|
||||
return cool
|
||||
|
||||
|
||||
def suggested_delay_s(status: Optional[dict]) -> float:
|
||||
"""Duty-cycle lento: delay base sempre presente; cresce con temp/load."""
|
||||
base = _float_env("OLLAMA_EMBED_DELAY_S", 8.0)
|
||||
if not status or status.get("cpu_temp_c") is None:
|
||||
return max(base, 5.0)
|
||||
temp = float(status["cpu_temp_c"])
|
||||
hard = hard_temp_c()
|
||||
soft = soft_temp_c()
|
||||
if temp >= hard:
|
||||
return max(base, 45.0)
|
||||
if temp >= soft:
|
||||
# soft→hard: ~base*1.5 … ~35s (continuo, non on/off)
|
||||
ratio = (temp - soft) / max(hard - soft, 1.0)
|
||||
return max(base, base * 1.5 + ratio * 25.0)
|
||||
if load_over_target(status):
|
||||
return max(base, base * 2.0)
|
||||
if temp >= soft - 4:
|
||||
return max(base, base * 1.25)
|
||||
return base
|
||||
|
||||
|
||||
def wait_for_headroom(*, context: str = "embed") -> Optional[dict]:
|
||||
"""Attende headroom: HARD = pausa lunga; altrimenti delay proporzionale.
|
||||
|
||||
Obiettivo: ritmo lento e regolare, evitando oscillazioni start/stop.
|
||||
"""
|
||||
if not enabled():
|
||||
return None
|
||||
|
||||
poll = _float_env("THERMAL_POLL_S", 30.0)
|
||||
hard = hard_temp_c()
|
||||
soft = soft_temp_c()
|
||||
# Riprendi solo quando sotto soft - 2°C (isteresi anti-oscillazione)
|
||||
resume_below = soft - _float_env("THERMAL_RESUME_MARGIN_C", 2.0)
|
||||
|
||||
while True:
|
||||
status = read_status()
|
||||
if status is None:
|
||||
LOGGER.warning("Thermal gate: probe non disponibile — delay conservativo")
|
||||
time.sleep(max(suggested_delay_s(None), 5.0))
|
||||
return None
|
||||
|
||||
temp = float(status.get("cpu_temp_c") or 0)
|
||||
load1 = float(status.get("load1") or 0)
|
||||
|
||||
if temp >= hard:
|
||||
LOGGER.warning(
|
||||
"Thermal gate [%s]: PAUSA HARD temp=%.1f°C (tetto=%.0f°C resume<=%.0f°C "
|
||||
"load1=%.2f) — riprovo tra %.0fs",
|
||||
context,
|
||||
temp,
|
||||
hard,
|
||||
resume_below,
|
||||
load1,
|
||||
poll,
|
||||
)
|
||||
time.sleep(poll)
|
||||
# Isteresi: resta in pausa finché non scende sotto soft
|
||||
while True:
|
||||
cooled = read_status()
|
||||
if cooled is None:
|
||||
time.sleep(poll)
|
||||
continue
|
||||
t2 = float(cooled.get("cpu_temp_c") or 0)
|
||||
if t2 <= resume_below and not load_over_target(cooled):
|
||||
LOGGER.info(
|
||||
"Thermal gate [%s]: ripresa dopo HARD (temp=%.1f°C)",
|
||||
context,
|
||||
t2,
|
||||
)
|
||||
status = cooled
|
||||
break
|
||||
time.sleep(poll)
|
||||
# dopo ripresa applica comunque un delay soft prima dell'embed
|
||||
time.sleep(suggested_delay_s(status))
|
||||
return status
|
||||
|
||||
if temp >= soft or load_over_target(status):
|
||||
delay = suggested_delay_s(status)
|
||||
LOGGER.info(
|
||||
"Thermal gate [%s]: rallento temp=%.1f°C load1=%.2f — delay %.1fs (source=%s)",
|
||||
context,
|
||||
temp,
|
||||
load1,
|
||||
delay,
|
||||
status.get("source"),
|
||||
)
|
||||
time.sleep(delay)
|
||||
again = read_status()
|
||||
if again and float(again.get("cpu_temp_c") or 0) >= hard:
|
||||
status = again
|
||||
continue
|
||||
return again or status
|
||||
|
||||
# Zona fredda: piccolo delay fisso per duty-cycle regolare
|
||||
cool_delay = _float_env("OLLAMA_EMBED_COOL_DELAY_S", 0.0)
|
||||
if cool_delay > 0:
|
||||
time.sleep(cool_delay)
|
||||
return status
|
||||
Reference in new issue
Block a user