{
  "schema_version": "agent_nemotron_contract_tuned_smoke_matrix_v1",
  "generated_at": "2026-06-02T10:27:22+08:00",
  "candidate_id": "nemo_nemotron_fabric",
  "candidate_variant_id": "nemo_nemotron_fabric_contract_tuned_v1",
  "run_id": "nemotron-replay-prod-20260602095438",
  "latency_budget_ms": 45000,
  "full_replay_allowed": false,
  "decision": "all_tested_nemotron_smokes_blocked_before_full_replay",
  "tested_models": [
    {
      "model": "nvidia/nemotron-3-super-120b-a12b",
      "tested_at": "2026-06-01",
      "requests": 5,
      "results": 5,
      "runner_valid": true,
      "external_error_records": 0,
      "fallback_used_records": 0,
      "trace_incomplete_records": 0,
      "retry_used_records": 1,
      "avg_latency_ms": 213890.3999,
      "p95_latency_ms": 374591.0851,
      "smoke_gate_decision": "blocked",
      "blocking_failures": [
        "latency_budget_exceeded"
      ],
      "runner_report": "docs/evaluations/agent_nemotron_contract_tuned_smoke_external_runner_report_2026-06-01.json",
      "smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_smoke_gate_2026-06-01.json"
    },
    {
      "model": "nvidia/nvidia-nemotron-nano-9b-v2",
      "tested_at": "2026-06-02",
      "requests": 5,
      "results": 5,
      "runner_valid": true,
      "external_error_records": 0,
      "fallback_used_records": 5,
      "trace_incomplete_records": 5,
      "retry_used_records": 0,
      "avg_latency_ms": 60103.0275,
      "p95_latency_ms": 60108.6491,
      "smoke_gate_decision": "blocked",
      "blocking_failures": [
        "fallbacks_present",
        "trace_incomplete_records_present",
        "latency_budget_exceeded"
      ],
      "runner_report": "docs/evaluations/agent_nemotron_contract_tuned_nano9b_smoke_external_runner_report_2026-06-02.json",
      "smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_nano9b_smoke_gate_2026-06-02.json"
    },
    {
      "model": "nvidia/nemotron-mini-4b-instruct",
      "tested_at": "2026-06-02",
      "requests": 5,
      "results": 5,
      "runner_valid": false,
      "external_error_records": 5,
      "fallback_used_records": 5,
      "trace_incomplete_records": 5,
      "retry_used_records": 0,
      "avg_latency_ms": 527.5488,
      "p95_latency_ms": 681.8552,
      "smoke_gate_decision": "blocked",
      "blocking_failures": [
        "runner_invalid",
        "external_errors_present",
        "fallbacks_present",
        "trace_incomplete_records_present"
      ],
      "runner_report": "docs/evaluations/agent_nemotron_contract_tuned_mini4b_smoke_external_runner_report_2026-06-02.json",
      "smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_mini4b_smoke_gate_2026-06-02.json"
    },
    {
      "model": "nvidia/nemotron-3-nano-30b-a3b",
      "tested_at": "2026-06-02",
      "requests": 5,
      "results": 5,
      "runner_valid": false,
      "external_error_records": 4,
      "fallback_used_records": 4,
      "trace_incomplete_records": 4,
      "retry_used_records": 5,
      "avg_latency_ms": 8836.9188,
      "p95_latency_ms": 11180.4184,
      "smoke_gate_decision": "blocked",
      "blocking_failures": [
        "runner_invalid",
        "external_errors_present",
        "fallbacks_present",
        "trace_incomplete_records_present"
      ],
      "runner_report": "docs/evaluations/agent_nemotron_contract_tuned_nemotron3nano30b_smoke_external_runner_report_2026-06-02.json",
      "smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_nemotron3nano30b_smoke_gate_2026-06-02.json"
    },
    {
      "model": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
      "tested_at": "2026-06-02",
      "requests": 5,
      "results": 5,
      "runner_valid": true,
      "external_error_records": 0,
      "fallback_used_records": 0,
      "trace_incomplete_records": 0,
      "retry_used_records": 2,
      "avg_latency_ms": 40121.8494,
      "p95_latency_ms": 67191.2835,
      "smoke_gate_decision": "blocked",
      "blocking_failures": [
        "latency_budget_exceeded"
      ],
      "runner_report": "docs/evaluations/agent_nemotron_contract_tuned_49b_v15_smoke_external_runner_report_2026-06-02.json",
      "smoke_gate": "docs/evaluations/agent_nemotron_contract_tuned_49b_v15_smoke_gate_2026-06-02.json"
    }
  ],
  "best_observed_models": {
    "best_contract_reliability": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
    "best_latency": "nvidia/nemotron-mini-4b-instruct",
    "best_balanced_candidate": "nvidia/llama-3.3-nemotron-super-49b-v1.5",
    "best_balanced_limit": "p95 latency still exceeds the 45s smoke-gate budget"
  },
  "professional_decision": {
    "may_replace_openclaw": false,
    "may_enter_shadow": false,
    "may_enter_canary": false,
    "may_run_full_50_replay": false,
    "recommended_role": [
      "offline specialist",
      "agent-fabric evaluator",
      "NIM runtime candidate after stricter JSON enforcement or latency reduction"
    ],
    "next_safe_steps": [
      "Do not run full replay until a Nemotron-family model passes the 5-record smoke gate.",
      "For Nemotron 3 Nano 30B, investigate stricter structured-output enforcement before another smoke.",
      "For Nemotron 49B v1.5, investigate latency reduction before another smoke."
    ]
  }
}
