#!/usr/bin/env bash
# Runner des séries de calibration (10 parties, graines 0-9, protocole v2 avec préchauffage).
# Utilise le test ignoré `real_local_ollama_smoke` de la branche feature/banc-incertitude (non modifiée).
set -u
OUT=$HOME/banc-calibration-20261010
BIN=$HOME/local-cockpit/local-cockpit-app/src-tauri/target/debug/deps/outilsia_local_cockpit_lib-91ce6b119f65bbc6
WSMI=/mnt/c/Windows/System32/nvidia-smi.exe
WOLL=ollama.exe
LOG=$OUT/runner.log
log(){ echo "$(date -Iseconds) $*" | tee -a "$LOG"; }

competitors(){ ps -eo pid,args | grep -E "features cuda|arenascript|ollama_tune|llama-server|llama-cli" | grep -v grep; }
gpu_line(){ nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu,power.draw --format=csv,noheader; }

gate(){ # attend une carte au repos ; max 3 h
  local ok=0 tries=0
  while [ $ok -lt 3 ]; do
    tries=$((tries+1))
    local g u m c wps lps
    g=$(gpu_line); u=$(echo "$g" | cut -d, -f1 | tr -dc 0-9); m=$(echo "$g" | cut -d, -f2 | tr -dc 0-9)
    c=$(competitors | wc -l)
    lps=$(ollama ps | tail -n +2 | wc -l)
    wps=$(timeout 20 "$WOLL" ps 2>/dev/null | tail -n +2 | wc -l)
    local pw; pw=$(echo "$g" | cut -d, -f5 | tr -dc 0-9. | cut -d. -f1)
    # Garde-fou : carte au repos (util <= 10 %, < 60 W, VRAM < 3 Go) et aucun modèle chargé (Ollama WSL + Windows).
    # Les processus d'autres agents (tests CUDA endormis) sont consignés, pas bloquants tant que la carte est au repos.
    if [ "$u" -le 10 ] && [ "${pw:-999}" -lt 60 ] && [ "$m" -le 3000 ] && [ "$lps" -eq 0 ] && [ "$wps" -eq 0 ]; then ok=$((ok+1)); else ok=0; fi
    [ $((tries % 10)) -eq 1 ] && log "gate: gpu=[$g] concurrents=$c ollama_wsl_charges=$lps ollama_win_charges=$wps ok=$ok"
    [ $tries -gt 1080 ] && { log "gate: abandon (carte jamais au repos)"; return 1; }
    sleep 10
  done
  log "gate OK: gpu=[$(gpu_line)]"
  return 0
}

for MODEL in "$@"; do
  SAFE=$(echo "$MODEL" | tr ':/' '__')
  log "=== $MODEL ==="
  if ! gate; then log "SKIP $MODEL"; continue; fi
  {
    echo "# Etat avant serie $MODEL $(date -Iseconds)"
    nvidia-smi; echo; "$WSMI" ; echo; echo "ollama WSL ps:"; ollama ps; echo "ollama Windows ps:"; timeout 20 "$WOLL" ps
    echo; echo "concurrents:"; competitors; echo; ollama --version; echo; ollama show "$MODEL"; echo; ollama list | grep -F "$MODEL"
  } > "$OUT/$SAFE.preflight.txt" 2>&1
  # monitoring GPU toutes les 30 s
  ( while true; do echo "$(date -Iseconds),$(gpu_line),concurrents=$(competitors | wc -l)"; sleep 30; done ) > "$OUT/$SAFE.gpu-monitor.csv" 2>&1 &
  MON=$!
  T0=$(date +%s)
  DQ_REAL_MODEL="$MODEL" DQ_GAMES=10 "$BIN" real_local_ollama --ignored --nocapture --test-threads=1 \
     > "$OUT/$SAFE.stdout.txt" 2> "$OUT/$SAFE.progress.txt"
  RC=$?
  T1=$(date +%s)
  kill $MON 2>/dev/null
  log "$MODEL fini rc=$RC duree=$((T1-T0))s"
  echo "rc=$RC wall_seconds=$((T1-T0)) start=$T0 end=$T1" > "$OUT/$SAFE.meta.txt"
  ollama ps >> "$OUT/$SAFE.meta.txt"
  ollama stop "$MODEL" >/dev/null 2>&1
  sleep 5
done
log "TOUTES SERIES TERMINEES"