RAG_helper/routers/eval.py

"""Эндпоинты eval-наборов и прогонов регрессии (Спринты 7.5 + 8a).

- `/eval/router-cases` — список кейсов классификатора для UI Настроек (готовые
  фразы пациентов из реального корпуса).
- `/eval/runs` — прогоны регрессии роутера (Спринт 8a). POST запускает фоновый
  прогон, GET возвращает историю и детали.
"""
import json
import logging
from pathlib import Path

from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession

from db.models import AgentConfig
from db.session import get_session
from models.responses import (
    EvalRunCaseInfo,
    EvalRunDetailResponse,
    EvalRunDiffInfo,
    EvalRunInfo,
    EvalRunListResponse,
)
from services import eval_run_service

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/eval", tags=["eval"])

EVAL_DIR = Path(__file__).resolve().parent.parent / "eval"
ROUTER_CASE_FILES = [
    EVAL_DIR / "router_cases_booking.jsonl",
    EVAL_DIR / "router_cases_other.jsonl",
]


# Кэш загруженных кейсов: грузим один раз при первом запросе. Файлы JSONL не
# меняются на лету (это часть репо), поэтому горячая перезагрузка не нужна.
_router_cases_cache: list[dict] | None = None


def _load_router_cases() -> list[dict]:
    global _router_cases_cache
    if _router_cases_cache is not None:
        return _router_cases_cache

    all_cases: list[dict] = []
    for path in ROUTER_CASE_FILES:
        if not path.is_file():
            logger.warning("Router case file not found: %s", path)
            continue
        with path.open(encoding="utf-8") as f:
            for line_no, raw in enumerate(f, 1):
                line = raw.strip()
                if not line:
                    continue
                try:
                    data = json.loads(line)
                except json.JSONDecodeError as e:
                    logger.warning("%s:%d JSON decode error: %s", path.name, line_no, e)
                    continue
                all_cases.append(data)

    _router_cases_cache = all_cases
    logger.info("Loaded %d router eval cases from %d file(s)", len(all_cases), len(ROUTER_CASE_FILES))
    return all_cases


@router.get("/router-cases")
def list_router_cases(intent_code: str | None = None, limit: int = 500):
    """Список кейсов маршрутизатора, опционально с фильтром по `expected_intent`.

    Сортировка — по `count` desc (самые частотные фразы вверху). `limit` ограничивает
    объём, чтобы UI не давился на 800+ опциях datalist.
    """
    cases = _load_router_cases()

    if intent_code:
        filtered = [c for c in cases if c.get("expected_intent") == intent_code]
    else:
        filtered = list(cases)

    filtered.sort(key=lambda c: c.get("count", 0), reverse=True)
    filtered = filtered[:max(1, min(limit, 5000))]

    items = [
        {
            "text": c.get("text", ""),
            "expected_intent": c.get("expected_intent", ""),
            "count": int(c.get("count", 0) or 0),
            "note": c.get("note") or None,
        }
        for c in filtered
    ]

    return {
        "intent_code": intent_code,
        "total": len(items),
        "cases": items,
    }


# ---------- Прогоны регрессии (Спринт 8a) ----------

class StartRouterRunRequest(BaseModel):
    suite: str = Field("router", description="Сейчас поддерживается только 'router'")
    text_hashes: list[str] = Field(..., min_length=1, description="sha256(text) выбранных кейсов")


def _run_to_info(run, router_config_version: int | None) -> EvalRunInfo:
    return EvalRunInfo(
        id=run.id,
        suite=run.suite,
        router_config_id=run.router_config_id,
        router_config_version=router_config_version,
        min_count=run.min_count,
        status=run.status,
        total=run.total,
        passed=run.passed,
        failed=run.failed,
        cache_hits=run.cache_hits,
        error_text=run.error_text,
        started_at=run.started_at.isoformat(),
        finished_at=run.finished_at.isoformat() if run.finished_at else None,
    )


def _case_to_info(c) -> EvalRunCaseInfo:
    return EvalRunCaseInfo(
        text=c.text,
        expected_intent=c.expected_intent,
        predicted_intent=c.predicted_intent,
        count_weight=c.count_weight,
        is_pass=c.is_pass,
    )


async def _config_version(session: AsyncSession, config_id: int | None) -> int | None:
    if config_id is None:
        return None
    cfg = await session.get(AgentConfig, config_id)
    return cfg.version if cfg else None


@router.post("/runs", response_model=EvalRunInfo)
async def start_run(req: StartRouterRunRequest, session: AsyncSession = Depends(get_session)):
    if req.suite != "router":
        raise HTTPException(status_code=400, detail="Only suite='router' is supported in 8a")
    try:
        run = await eval_run_service.start_router_run(session, req.text_hashes)
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    version = await _config_version(session, run.router_config_id)
    return _run_to_info(run, version)


@router.get("/router-cases-with-status")
async def router_cases_with_status(session: AsyncSession = Depends(get_session)):
    """Все кейсы из JSONL, отсортированы по count desc, + кэш на активной версии роутера.

    Под каждым кейсом — последний предсказанный intent для этой версии (если был),
    и pass/fail сравнение с expected. UI строит таблицу выбора с массовыми операциями.
    """
    cases = eval_run_service.load_all_router_cases()
    router_config_id = await eval_run_service._resolve_active_router_config_id(session)
    version = await _config_version(session, router_config_id)
    cache = await eval_run_service.cached_predictions(session, router_config_id)

    items = []
    for idx, c in enumerate(cases, 1):
        th = eval_run_service._text_hash(c.text)
        cached_predicted = cache.get(th)
        cached_is_pass = (
            None if cached_predicted is None else cached_predicted == c.expected_intent
        )
        items.append({
            "idx": idx,
            "text": c.text,
            "text_hash": th,
            "expected_intent": c.expected_intent,
            "count": c.count,
            "cached_predicted": cached_predicted,
            "cached_is_pass": cached_is_pass,
        })
    return {
        "router_config_id": router_config_id,
        "router_config_version": version,
        "total": len(items),
        "cases": items,
    }


@router.get("/runs", response_model=EvalRunListResponse)
async def list_runs(session: AsyncSession = Depends(get_session)):
    runs = await eval_run_service.list_runs(session, limit=50)
    items = []
    for r in runs:
        version = await _config_version(session, r.router_config_id)
        items.append(_run_to_info(r, version))
    return EvalRunListResponse(runs=items, total=len(items))


@router.get("/runs/{run_id}", response_model=EvalRunDetailResponse)
async def get_run(run_id: int, session: AsyncSession = Depends(get_session)):
    run = await eval_run_service.get_run(session, run_id)
    if run is None:
        raise HTTPException(status_code=404, detail="Run not found")
    version = await _config_version(session, run.router_config_id)
    cases = await eval_run_service.list_run_cases(session, run_id)
    diff = await eval_run_service.compute_diff_vs_previous(session, run)
    return EvalRunDetailResponse(
        run=_run_to_info(run, version),
        cases=[_case_to_info(c) for c in cases],
        diff=EvalRunDiffInfo(
            prev_run_id=diff.prev_run_id,
            new_fails=[_case_to_info(c) for c in diff.new_fails],
            new_passes=[_case_to_info(c) for c in diff.new_passes],
        ),
    )