{
 "about": "Untuned (off-the-shelf) models, measured before and during the fine-tune program through Ollama. 'routing' = the production command-routing prompt (~5.6k tokens); 'content' = 11 generation workloads (summaries, tagging, narration and similar) scored by a judge model. 'floor' tier = unambiguous cases every acceptable model must pass; 'hard' tier = deliberately ambiguous cases. Tournament numbers below were recomputed from the raw per-cycle score records by time window and checked against the program's written notes where they quote them.",
 "tournament_blocks": [
  {
   "block": "pre-0908 (12 workloads, no tier split)",
   "window": "2026-09-07T22:48Z to 2026-09-08T19:32Z",
   "source": "final_report_2026-09-08.txt; soak_scores.jsonl"
  },
  {
   "block": "pre-0908 hard",
   "window": "2026-09-08T21:06Z to 2026-09-09T01:41Z",
   "source": "final_report.txt; final_report_hard_2026-09-09.txt"
  },
  {
   "block": "floor1",
   "window": "2026-09-09T03:24Z to 04:21Z",
   "source": "tournament.log"
  },
  {
   "block": "hard1",
   "window": "2026-09-09T04:23Z to 07:20Z",
   "source": "tournament.log"
  },
  {
   "block": "route1",
   "window": "2026-09-09T07:23Z to 15:38Z",
   "note": "superseded by route1b; heavy contention (median s/case 10-38)",
   "source": "tournament.log"
  },
  {
   "block": "route1b",
   "window": "2026-09-09T15:38Z to 16:29Z",
   "note": "the routing result the program notes and ledger quote",
   "source": "tournament.log"
  },
  {
   "block": "content1",
   "window": "2026-09-09T17:07Z to 2026-09-10T04:14Z",
   "source": "tournament.log"
  },
  {
   "block": "e2bgranite_content1",
   "window": "2026-09-13T03:18Z to 10:38Z",
   "note": "filled the content gap for e2b and granite; ran under contention from the live router",
   "source": "tournament.log; LOCAL_LLM_PHASE2_ROADMAP.md:39-50"
  }
 ],
 "tournament": [
  {
   "model": "gemma4:12b-it-qat",
   "role": "production router before the fine-tune; baseline for every version",
   "routing_route1b": {
    "floor": "150/150",
    "hard": "130/150",
    "blended": 0.933,
    "median_s_per_case": {
     "floor": 3.0,
     "hard": 4.5
    }
   },
   "content_content1": {
    "floor": "490/490",
    "hard": "323/330",
    "blended": 0.991,
    "blended_count": "813/820"
   },
   "floor1": {
    "routing": "30/30",
    "content": "123/123"
   },
   "hard1": {
    "routing": "34/45",
    "content": "80/82"
   },
   "resident_vram": {
    "ledger_gb": 8.8,
    "soak_scores_max_size_vram_gb": 8.5,
    "soak_scores_typical_gb": 4.76
   },
   "source": "soak_scores.jsonl (by block window); LOCAL_LLM_PHASE2_ROADMAP.md:60-65; router_ledger.html",
   "measured_on": "2026-09-09"
  },
  {
   "model": "gemma4:e2b-it-qat",
   "role": "first-choice fine-tune base; abandoned for tooling reasons (see gotchas)",
   "routing_route1b": {
    "floor": "150/150",
    "hard": "120/150",
    "blended": 0.9,
    "median_s_per_case": {
     "floor": 1.0,
     "hard": 1.2
    }
   },
   "content_e2bgranite_content1": {
    "floor": "476/490",
    "hard": "315/330",
    "blended": 0.965,
    "blended_count": "791/820"
   },
   "hard1": {
    "routing": "35/45",
    "content": "77/82"
   },
   "floor1": "unavailable (load errors in that block)",
   "resident_vram": {
    "ledger_gb": null,
    "soak_scores_max_size_vram_gb": 1.79
   },
   "source": "soak_scores.jsonl; LOCAL_LLM_PHASE2_ROADMAP.md:63",
   "measured_on": "2026-09-13"
  },
  {
   "model": "granite4.2:3b-q8_0",
   "routing_route1b": {
    "floor": "140/150",
    "hard": "129/150",
    "blended": 0.897,
    "median_s_per_case": {
     "floor": 0.7,
     "hard": 0.9
    }
   },
   "content_e2bgranite_content1": {
    "floor": "488/490",
    "hard": "306/330",
    "blended": 0.968,
    "blended_count": "794/820"
   },
   "floor1": {
    "routing": "28/30",
    "content": "120/123"
   },
   "hard1": {
    "routing": "35/45",
    "content": "78/82"
   },
   "resident_vram": {
    "ledger_gb": 4.7,
    "soak_scores_max_size_vram_gb": 4.44
   },
   "timing_note": "hard-tier mean 116.2 s vs median 27.6 s per workload record under contention; one record took 1357.88 s",
   "source": "soak_scores.jsonl; LOCAL_LLM_PHASE2_ROADMAP.md:64,76-86",
   "measured_on": "2026-09-13"
  },
  {
   "model": "qwen3.5:9b",
   "role": "the original fine-tune plan, dropped",
   "architecture_note": "hybrid SSM/attention (qwen35.ssm.* fields in ollama show), not a plain dense model",
   "routing_route1b": {
    "floor": "140/150",
    "hard": "109/150",
    "blended": 0.83,
    "median_s_per_case": {
     "floor": 1.4,
     "hard": 1.5
    }
   },
   "content_content1": {
    "floor": "476/480",
    "hard": "304/330",
    "blended": 0.963,
    "blended_count": "780/810"
   },
   "hard1": {
    "routing": "34/45",
    "content": "76/82"
   },
   "resident_vram": {
    "ledger_gb": 5.8,
    "soak_scores_max_size_vram_gb": 6.32
   },
   "source": "soak_scores.jsonl; LOCAL_LLM_PHASE2_ROADMAP.md:62,169-176",
   "measured_on": "2026-09-09"
  },
  {
   "model": "qwen3.5:4b-q8_0",
   "routing_route1b": {
    "floor": "150/150",
    "hard": "120/150",
    "blended": 0.9,
    "median_s_per_case": {
     "floor": 1.2,
     "hard": 1.6
    }
   },
   "content_content1": {
    "floor": "489/490",
    "hard": "299/330",
    "blended": 0.961,
    "blended_count": "788/820"
   },
   "hard1": {
    "routing": "36/45",
    "content": "77/82"
   },
   "resident_vram": {
    "soak_scores_max_size_vram_gb": 5.31
   },
   "note": "not in the program notes's or ledger's comparison table; numbers are from the raw records only",
   "source": "soak_scores.jsonl",
   "measured_on": "2026-09-09"
  },
  {
   "model": "nemotron-3-nano:4b-q8_0",
   "floor1": {
    "routing": "26/30",
    "content": "122/123"
   },
   "resident_vram": {
    "soak_scores_max_size_vram_gb": 4.46
   },
   "note": "floor tier only; dropped after floor1",
   "source": "soak_scores.jsonl; tournament.log",
   "measured_on": "2026-09-09"
  },
  {
   "model": "qwen3.5:4b-q4_K_M, qwen3.5:9b-q8_0, LFM2.5-2.6B (Q8_0, Q4_K_M)",
   "pre0908_summary": [
    {
     "model": "qwen3.5:4b-q8_0",
     "pass": "778/781",
     "s_per_case": 8.7,
     "vram_gb": 5.1
    },
    {
     "model": "qwen3.5:4b-q4_K_M",
     "pass": "774/781",
     "s_per_case": 10.9,
     "vram_gb": 2.2
    },
    {
     "model": "qwen3.5:9b",
     "pass": "768/781",
     "s_per_case": 2.9,
     "vram_gb": 5.8
    },
    {
     "model": "qwen3.5:9b-q8_0",
     "pass": "760/778",
     "s_per_case": 6.3,
     "vram_gb": 9.1
    },
    {
     "model": "LFM2.5-2.6B Q8_0",
     "pass": "621/703",
     "s_per_case": 12.4,
     "vram_gb": 3.0,
     "routing_workload": 0.68
    },
    {
     "model": "LFM2.5-2.6B Q4_K_M",
     "pass": "607/703",
     "s_per_case": 6.5,
     "vram_gb": 1.8,
     "routing_workload": 0.58
    }
   ],
   "pre0908_hard_summary": [
    {
     "model": "qwen3.5:4b-q8_0",
     "pass": "530/607",
     "s_per_case": 9.9
    },
    {
     "model": "qwen3.5:9b",
     "pass": "520/607",
     "s_per_case": 10.0
    }
   ],
   "q8_vs_q4_note": "qwen3.5:9b vs its q8_0: same 98% pass rate, q8 2.2x slower (6.3 vs 2.9 s/case) and 3.3 GB more VRAM",
   "caveat": "Earlier, easier case set, all 12 workloads pooled; not comparable to the tiered blocks. LFM2 has no context-shift support and surfaced the num_ctx overflow as HTTP 400 errors (see gotchas), so its routing scores from this window are suspect.",
   "source": "final_report_2026-09-08.txt; final_report.txt; LOCAL_LLM_PHASE2_ROADMAP.md:1119-1124; CLAUDE.md (Lessons learned: Ollama default context)",
   "measured_on": "2026-09-08"
  }
 ],
 "warm_head_to_head": {
  "method": "evict, cold-load, then repeated warm calls through the production routing code path, under low system load",
  "results": [
   {
    "model": "qwen3.5:9b",
    "cold_load_s": 8.8,
    "warm_route_s": {
     "median": 6.03,
     "min": 6.03,
     "max": 6.79
    },
    "warm_200_token_generation_s": 6.73
   },
   {
    "model": "gemma4:e2b-it-qat",
    "cold_load_s": 5.3,
    "warm_route_s": {
     "median": 6.03,
     "min": 6.03,
     "max": 6.03
    },
    "warm_200_token_generation_s": 3.04
   },
   {
    "model": "granite4.2:3b-q8_0",
    "cold_load_s": 2.9,
    "warm_route_s": {
     "median": 6.03,
     "min": 6.02,
     "max": 6.03
    },
    "warm_200_token_generation_s": 4.08
   }
  ],
  "note": "warm-route times converge on ~6.03 s for all three with no speed-up across same-prefix calls, which the program notes reads as the static prompt prefix not being cached call to call. The 200-token generation column is the clean comparison.",
  "source": "LOCAL_LLM_PHASE2_ROADMAP.md:88-114",
  "measured_on": "2026-09-13"
 },
 "swa_cache_reuse_test": {
  "method": "6 trials per model, each with fresh ~300-char context through the real routing path; static ~5.6k tokens + dynamic tail = 6.2-6.5k tokens",
  "results": [
   {
    "model": "gemma4:12b-it-qat",
    "sliding_window": 1024,
    "trial0_s": 269.96,
    "trials1_5_s": "49-56",
    "median_s": 53.8,
    "fast_plateau": false
   },
   {
    "model": "gemma4:e2b-it-qat",
    "sliding_window": 512,
    "trial0_s": 60.74,
    "trials1_5_s": "11-12",
    "median_s": 11.76,
    "fast_plateau": true
   },
   {
    "model": "granite4.2:3b-q8_0",
    "trial0_s": 142.14,
    "trials1_5_s": "23-33",
    "median_s": 29.2,
    "fast_plateau": false
   },
   {
    "model": "qwen3.5:9b",
    "trial0_s": 142.79,
    "trials1_5_s": "24-36",
    "median_s": 26.0,
    "fast_plateau": false
   }
  ],
  "confound": "system load rose from ~3.0 to ~6-8 between the gemma runs and the granite/qwen runs; a clean test would run all four back to back",
  "source": "LOCAL_LLM_PHASE2_ROADMAP.md:123-214",
  "measured_on": "2026-09-13"
 },
 "untuned_12b_on_gold_sets": [
  {
   "test_set": "550 cases",
   "passed": 469,
   "accuracy": 0.8527,
   "wall_clock_s": 1012.9,
   "source": "bake_v2_summary.json:baseline-prod",
   "measured_on": "2026-09-14"
  },
  {
   "test_set": "865 cases",
   "passed": 691,
   "accuracy": 0.7988,
   "s_per_case": 2.124,
   "source": "bake_baseline_summary.json",
   "measured_on": "2026-09-17"
  },
  {
   "test_set": "865, train-flagged slice",
   "accuracy": 0.76,
   "source": "router_ledger.html (baseline table)",
   "measured_on": "2026-09-18",
   "unverified": true,
   "note": "Ledger-only. Recomputing with today's train flags gives 79.5% (667/839) because flags changed after 09-18."
  },
  {
   "test_set": "865, held-out slice",
   "accuracy": 0.941,
   "source": "router_ledger.html (baseline table)",
   "measured_on": "2026-09-18",
   "unverified": true,
   "note": "Ledger-only. With today's train flags the held-out slice is 26 cases and the 12B scores 24/26 (92.3%)."
  },
  {
   "test_set": "1,125 runnable, harness fixed",
   "passed": 924,
   "accuracy": 0.8213,
   "held_out_passed": 96,
   "held_out_total": 108,
   "held_out_accuracy": 0.8889,
   "s_per_case": 2.685,
   "source": "bake_12b_vs_v12_summary.json:untuned-gemma4-12b",
   "measured_on": "2026-09-19"
  }
 ],
 "comparison_to_finetunes": {
  "statement": "On the 1,125-case set the rank-112 fine-tunes route about 2x faster than the untuned 12B (1.22-1.34 vs 2.69 s/case) and are 11-14 pp more accurate overall; v9 was the first to beat it on held-out (91.7% vs 88.9%).",
  "source": "bake_12b_vs_v12_summary.json (speed_ratio 2.124, accuracy delta -0.112); bake_v11_vs_v12_summary.json",
  "measured_on": "2026-09-21"
 }
}