RAG_helper/routers/eval.py

"""Эндпоинты eval-наборов и прогонов регрессии (Спринты 7.5 + 8a).

- `/eval/router-cases` — список кейсов классификатора для UI Настроек (готовые
  фразы пациентов из реального корпуса).
- `/eval/runs` — прогоны регрессии роутера (Спринт 8a). POST запускает фоновый
  прогон, GET возвращает историю и детали.
"""
import json
import logging
from pathlib import Path

from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession

from db.models import AgentConfig
from db.session import get_session
from models.responses import (
    EvalBranchRunCaseInfo,
    EvalBranchRunDetailResponse,
    EvalBranchRunInfo,
    EvalBranchRunListResponse,
    EvalRunCaseInfo,
    EvalRunDetailResponse,
    EvalRunDiffInfo,
    EvalRunInfo,
    EvalRunListResponse,
)
from services import eval_branch_run_service, eval_run_service

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/eval", tags=["eval"])

EVAL_DIR = Path(__file__).resolve().parent.parent / "eval"
ROUTER_CASE_FILES = [
    EVAL_DIR / "router_cases_booking.jsonl",
    EVAL_DIR / "router_cases_other.jsonl",
]


# Кэш загруженных кейсов: грузим один раз при первом запросе. Файлы JSONL не
# меняются на лету (это часть репо), поэтому горячая перезагрузка не нужна.
_router_cases_cache: list[dict] | None = None


def _load_router_cases() -> list[dict]:
    global _router_cases_cache
    if _router_cases_cache is not None:
        return _router_cases_cache

    all_cases: list[dict] = []
    for path in ROUTER_CASE_FILES:
        if not path.is_file():
            logger.warning("Router case file not found: %s", path)
            continue
        with path.open(encoding="utf-8") as f:
            for line_no, raw in enumerate(f, 1):
                line = raw.strip()
                if not line:
                    continue
                try:
                    data = json.loads(line)
                except json.JSONDecodeError as e:
                    logger.warning("%s:%d JSON decode error: %s", path.name, line_no, e)
                    continue
                all_cases.append(data)

    _router_cases_cache = all_cases
    logger.info("Loaded %d router eval cases from %d file(s)", len(all_cases), len(ROUTER_CASE_FILES))
    return all_cases


@router.get("/router-cases")
def list_router_cases(intent_code: str | None = None, limit: int = 500):
    """Список кейсов маршрутизатора, опционально с фильтром по `expected_intent`.

    Сортировка — по `count` desc (самые частотные фразы вверху). `limit` ограничивает
    объём, чтобы UI не давился на 800+ опциях datalist.
    """
    cases = _load_router_cases()

    if intent_code:
        filtered = [c for c in cases if c.get("expected_intent") == intent_code]
    else:
        filtered = list(cases)

    filtered.sort(key=lambda c: c.get("count", 0), reverse=True)
    filtered = filtered[:max(1, min(limit, 5000))]

    items = [
        {
            "text": c.get("text", ""),
            "expected_intent": c.get("expected_intent", ""),
            "count": int(c.get("count", 0) or 0),
            "note": c.get("note") or None,
        }
        for c in filtered
    ]

    return {
        "intent_code": intent_code,
        "total": len(items),
        "cases": items,
    }


# ---------- Прогоны регрессии (Спринт 8a) ----------

class StartRouterRunRequest(BaseModel):
    suite: str = Field("router", description="Сейчас поддерживается только 'router'")
    text_hashes: list[str] = Field(..., min_length=1, description="sha256(text) выбранных кейсов")


def _run_to_info(run, router_config_version: int | None) -> EvalRunInfo:
    return EvalRunInfo(
        id=run.id,
        suite=run.suite,
        router_config_id=run.router_config_id,
        router_config_version=router_config_version,
        min_count=run.min_count,
        status=run.status,
        total=run.total,
        passed=run.passed,
        failed=run.failed,
        cache_hits=run.cache_hits,
        error_text=run.error_text,
        started_at=run.started_at.isoformat(),
        finished_at=run.finished_at.isoformat() if run.finished_at else None,
    )


def _case_to_info(c) -> EvalRunCaseInfo:
    return EvalRunCaseInfo(
        text=c.text,
        expected_intent=c.expected_intent,
        predicted_intent=c.predicted_intent,
        count_weight=c.count_weight,
        is_pass=c.is_pass,
    )


async def _config_version(session: AsyncSession, config_id: int | None) -> int | None:
    if config_id is None:
        return None
    cfg = await session.get(AgentConfig, config_id)
    return cfg.version if cfg else None


@router.post("/runs", response_model=EvalRunInfo)
async def start_run(req: StartRouterRunRequest, session: AsyncSession = Depends(get_session)):
    if req.suite != "router":
        raise HTTPException(status_code=400, detail="Only suite='router' is supported in 8a")
    try:
        run = await eval_run_service.start_router_run(session, req.text_hashes)
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    version = await _config_version(session, run.router_config_id)
    return _run_to_info(run, version)


@router.get("/router-cases-with-status")
async def router_cases_with_status(session: AsyncSession = Depends(get_session)):
    """Все кейсы из JSONL, отсортированы по count desc, + кэш на активной версии роутера.

    Под каждым кейсом — последний предсказанный intent для этой версии (если был),
    и pass/fail сравнение с expected. UI строит таблицу выбора с массовыми операциями.
    """
    cases = eval_run_service.load_all_router_cases()
    router_config_id = await eval_run_service._resolve_active_router_config_id(session)
    version = await _config_version(session, router_config_id)
    cache = await eval_run_service.cached_predictions(session, router_config_id)

    items = []
    for idx, c in enumerate(cases, 1):
        th = eval_run_service._text_hash(c.text)
        cached_predicted = cache.get(th)
        cached_is_pass = (
            None if cached_predicted is None else cached_predicted == c.expected_intent
        )
        items.append({
            "idx": idx,
            "text": c.text,
            "text_hash": th,
            "expected_intent": c.expected_intent,
            "count": c.count,
            "cached_predicted": cached_predicted,
            "cached_is_pass": cached_is_pass,
        })
    return {
        "router_config_id": router_config_id,
        "router_config_version": version,
        "total": len(items),
        "cases": items,
    }


# ---------- Branch runs (Спринт 8b) ----------

class StartBranchRunRequest(BaseModel):
    intent_code: str
    text_hashes: list[str] = Field(..., min_length=1)


def _branch_run_to_info(run, version: int | None) -> EvalBranchRunInfo:
    return EvalBranchRunInfo(
        id=run.id,
        suite=run.suite,
        intent_code=run.intent_code,
        branch_config_id=run.branch_config_id,
        branch_config_version=version,
        status=run.status,
        total=run.total,
        passed=run.passed,
        failed=run.failed,
        cache_hits=run.cache_hits,
        error_text=run.error_text,
        started_at=run.started_at.isoformat(),
        finished_at=run.finished_at.isoformat() if run.finished_at else None,
    )


def _branch_case_to_info(c) -> EvalBranchRunCaseInfo:
    return EvalBranchRunCaseInfo(
        text=c.text,
        coverage=c.coverage,
        expected_doc_section=c.expected_doc_section,
        expected_keywords=json.loads(c.expected_keywords_json or "[]"),
        expected_must_not=json.loads(c.expected_must_not_json or "[]"),
        keywords_min=c.keywords_min,
        predicted_answer=c.predicted_answer,
        predicted_sections=json.loads(c.predicted_sections_json or "[]"),
        is_pass=c.is_pass,
        fail_reasons=json.loads(c.fail_reasons_json or "[]"),
        count_weight=c.count_weight,
    )


@router.get("/branch-cases-with-status")
async def branch_cases_with_status(
    intent_code: str, session: AsyncSession = Depends(get_session)
):
    """Все кейсы JSONL для ветки + кэш на её активной версии."""
    cases = eval_branch_run_service.load_branch_cases(intent_code)
    branch_config_id = await eval_branch_run_service._resolve_active_branch_config_id(
        session, intent_code,
    )
    version = await _config_version(session, branch_config_id)
    cache = await eval_branch_run_service.cached_predictions(session, branch_config_id)

    items = []
    for idx, c in enumerate(cases, 1):
        th = eval_branch_run_service._text_hash(c.text)
        cached = cache.get(th)
        cached_is_pass = None
        cached_answer = None
        cached_fail_reasons: list[str] = []
        if cached is not None:
            is_pass, reasons = eval_branch_run_service._evaluate_case(
                c, cached["answer_text"], cached["retrieved_sections"],
            )
            cached_is_pass = is_pass
            cached_answer = cached["answer_text"]
            cached_fail_reasons = reasons
        items.append({
            "idx": idx,
            "text": c.text,
            "text_hash": th,
            "intent_code": c.intent_code,
            "coverage": c.coverage,
            "expected_doc_section": c.expected_doc_section,
            "expected_keywords": c.expected_keywords,
            "expected_must_not": c.expected_must_not,
            "keywords_min": c.keywords_min,
            "keywords_any": c.keywords_any,
            "count": c.count,
            "note": c.note,
            "cached_is_pass": cached_is_pass,
            "cached_answer": cached_answer,
            "cached_fail_reasons": cached_fail_reasons,
        })
    return {
        "intent_code": intent_code,
        "branch_config_id": branch_config_id,
        "branch_config_version": version,
        "total": len(items),
        "cases": items,
    }


@router.post("/branch-runs", response_model=EvalBranchRunInfo)
async def start_branch_run(
    req: StartBranchRunRequest, session: AsyncSession = Depends(get_session)
):
    try:
        run = await eval_branch_run_service.start_branch_run(
            session, req.intent_code, req.text_hashes,
        )
    except ValueError as e:
        raise HTTPException(status_code=400, detail=str(e))
    version = await _config_version(session, run.branch_config_id)
    return _branch_run_to_info(run, version)


@router.get("/branch-runs", response_model=EvalBranchRunListResponse)
async def list_branch_runs(
    intent_code: str | None = None, session: AsyncSession = Depends(get_session)
):
    runs = await eval_branch_run_service.list_runs(session, intent_code, limit=50)
    items = []
    for r in runs:
        version = await _config_version(session, r.branch_config_id)
        items.append(_branch_run_to_info(r, version))
    return EvalBranchRunListResponse(runs=items, total=len(items))


@router.get("/branch-runs/{run_id}", response_model=EvalBranchRunDetailResponse)
async def get_branch_run(run_id: int, session: AsyncSession = Depends(get_session)):
    run = await eval_branch_run_service.get_run(session, run_id)
    if run is None:
        raise HTTPException(status_code=404, detail="Branch run not found")
    version = await _config_version(session, run.branch_config_id)
    cases = await eval_branch_run_service.list_run_cases(session, run_id)
    return EvalBranchRunDetailResponse(
        run=_branch_run_to_info(run, version),
        cases=[_branch_case_to_info(c) for c in cases],
    )


@router.get("/runs", response_model=EvalRunListResponse)
async def list_runs(session: AsyncSession = Depends(get_session)):
    runs = await eval_run_service.list_runs(session, limit=50)
    items = []
    for r in runs:
        version = await _config_version(session, r.router_config_id)
        items.append(_run_to_info(r, version))
    return EvalRunListResponse(runs=items, total=len(items))


@router.get("/runs/{run_id}", response_model=EvalRunDetailResponse)
async def get_run(run_id: int, session: AsyncSession = Depends(get_session)):
    run = await eval_run_service.get_run(session, run_id)
    if run is None:
        raise HTTPException(status_code=404, detail="Run not found")
    version = await _config_version(session, run.router_config_id)
    cases = await eval_run_service.list_run_cases(session, run_id)
    diff = await eval_run_service.compute_diff_vs_previous(session, run)
    return EvalRunDetailResponse(
        run=_run_to_info(run, version),
        cases=[_case_to_info(c) for c in cases],
        diff=EvalRunDiffInfo(
            prev_run_id=diff.prev_run_id,
            new_fails=[_case_to_info(c) for c in diff.new_fails],
            new_passes=[_case_to_info(c) for c in diff.new_passes],
        ),
    )