Метрики агентів

import time
from prometheus_client import Counter, Histogram

RUN_TOTAL = Counter(
    "agent_run_total",
    "Total number of agent runs",
    ["status", "stop_reason", "release"],
)
# success_rate = RUN_TOTAL{status="ok"} / RUN_TOTAL

RUN_LATENCY_MS = Histogram(
    "agent_run_latency_ms",
    "Run latency in milliseconds",
    ["release"],
    buckets=(100, 250, 500, 1000, 2000, 5000, 10000),
)

STEP_COUNT = Histogram(
    "agent_steps_per_run",
    "Number of steps per run",
    ["release"],
    buckets=(1, 2, 4, 8, 12, 16, 24, 32),
)

TOOL_CALL_TOTAL = Counter(
    "agent_tool_call_total",
    "Total tool calls",
    ["tool", "status", "release"],
)

TOOL_ERROR_TOTAL = Counter(
    "agent_tool_error_total",
    "Total tool errors by class",
    ["tool", "error_class", "release"],
)

LLM_ERROR_TOTAL = Counter(
    "agent_llm_error_total",
    "Total LLM step errors by model and class",
    ["model", "error_class", "release"],
)

TOOL_LATENCY_MS = Histogram(
    "agent_tool_latency_ms",
    "Tool call latency in milliseconds",
    ["tool", "release"],
    buckets=(20, 50, 100, 250, 500, 1000, 2000, 5000),
)

TOKEN_USAGE_TOTAL = Counter(
    "agent_token_usage_total",
    "Total LLM tokens",
    ["model", "token_type", "release"],
)


def observe_llm_usage(model, token_usage, release):
    # більшість LLM-провайдерів повертають token usage у відповіді
    if not token_usage:
        return
    TOKEN_USAGE_TOTAL.labels(model=model, token_type="prompt", release=release).inc(
        token_usage.get("prompt_tokens", 0)
    )
    TOKEN_USAGE_TOTAL.labels(model=model, token_type="completion", release=release).inc(
        token_usage.get("completion_tokens", 0)
    )


def run_agent(agent, task, release="2026-03-21"):
    started_at = time.time()
    steps = 0
    stop_reason = "max_steps"
    run_status = "ok"

    try:
        for step in agent.iter(task):
            steps += 1
            step_type = step.type
            result = None  # може залишитись None для невідомих типів кроків (захищено перевіркою нижче)

            if step_type == "tool_call":
                tool_name = getattr(step, "tool_name", "unknown")
                tool_started_at = time.time()
                try:
                    result = step.execute()
                    TOOL_CALL_TOTAL.labels(tool=tool_name, status="ok", release=release).inc()
                    TOOL_LATENCY_MS.labels(tool=tool_name, release=release).observe(
                        (time.time() - tool_started_at) * 1000
                    )
                except Exception as error:
                    TOOL_CALL_TOTAL.labels(tool=tool_name, status="error", release=release).inc()
                    TOOL_ERROR_TOTAL.labels(
                        tool=tool_name,
                        error_class=type(error).__name__,
                        release=release,
                    ).inc()
                    TOOL_LATENCY_MS.labels(tool=tool_name, release=release).observe(
                        (time.time() - tool_started_at) * 1000
                    )
                    run_status = "error"
                    stop_reason = "tool_error"
                    raise
            else:
                try:
                    result = step.execute()
                    observe_llm_usage(
                        model=getattr(step, "model", "unknown"),
                        token_usage=getattr(result, "token_usage", None),
                        release=release,
                    )
                except Exception as error:
                    LLM_ERROR_TOTAL.labels(
                        model=getattr(step, "model", "unknown"),
                        error_class=type(error).__name__,
                        release=release,
                    ).inc()
                    run_status = "error"
                    stop_reason = "step_error"
                    raise

            if result and result.is_final:
                stop_reason = "completed"
                break

    finally:
        RUN_TOTAL.labels(status=run_status, stop_reason=stop_reason, release=release).inc()
        RUN_LATENCY_MS.labels(release=release).observe((time.time() - started_at) * 1000)
        STEP_COUNT.labels(release=release).observe(steps)

agent_tool_call_total{tool="search_docs",status="error",release="2026-03-21"} 47