{
 "about": "The training recipe shared by every fresh-from-base version (v1-v3, v5-v17). v4 differs only in continuing from v3's adapter at 8 epochs. Values come from the training scripts and the run headers printed in each training log.",
 "task": {
  "description": "Supervised LoRA fine-tune. Input is the production routing prompt (system + user, ~5.6k-6.3k tokens incl. capability docs and recent context); target is the assistant's single JSON capability call (13-128 tokens, median 26).",
  "loss": "response-only: prompt tokens masked; labels and logits computed only for the last response_len + 1 positions via logits_to_keep",
  "example_length_tokens": {
   "min": 6277,
   "max": 6447,
   "corpus": "v1-v5 era",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:510-512; train_v4.py docstring",
   "measured_on": "2026-09-14"
  },
  "longest_example_later_corpora_tokens": [
   {
    "corpus": "2026-09-15",
    "tokens": 7037,
    "source": "probe_rank112.log",
    "measured_on": "2026-09-17"
   },
   {
    "corpus": "2026-09-18 unfiltered",
    "tokens": 7288,
    "source": "probe_rank112_ds0918.log",
    "measured_on": "2026-09-18"
   },
   {
    "corpus": "2026-09-18 filtered",
    "tokens": 6931,
    "source": "probe_rank112_ds0918f.log",
    "measured_on": "2026-09-18"
   },
   {
    "corpus": "2026-09-19 filtered",
    "tokens": 7068,
    "source": "probe_rank112_ds0919f.log",
    "measured_on": "2026-09-20"
   },
   {
    "corpus": "2026-09-21",
    "tokens": 7177,
    "source": "probe_sweep.log",
    "measured_on": "2026-09-23"
   }
  ],
  "response_span_tokens": {
   "min": 13,
   "max": 128,
   "median": 26,
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:582-583",
   "measured_on": "2026-09-13"
  }
 },
 "base_model": {
  "hf_repo": "unsloth/gemma-3-4b-it",
  "text_layers": 34,
  "vocab": 262208,
  "hidden": 2560,
  "source": "train_router.py:48,137-145; LOCAL_LLM_PHASE2_ROADMAP.md:580,682",
  "measured_on": "2026-09-14"
 },
 "quantization_for_training": {
  "method": "bitsandbytes 4-bit QLoRA",
  "load_in_4bit": true,
  "bnb_4bit_quant_type": "nf4",
  "bnb_4bit_use_double_quant": true,
  "bnb_4bit_compute_dtype": "bfloat16",
  "lm_head": "cast from fp32 to bf16 after get_peft_model",
  "source": "train_router.py:72-82,144-146",
  "measured_on": "2026-09-18"
 },
 "lora": {
  "target_modules": [
   "q_proj",
   "k_proj",
   "v_proj",
   "o_proj",
   "gate_proj",
   "up_proj",
   "down_proj"
  ],
  "rank": {
   "shipped": 112,
   "tried": [
    16,
    64,
    96,
    112
   ],
   "probed_oom": [
    128
   ],
   "source": "train_router.py:120-133; train logs",
   "measured_on": "2026-09-25"
  },
  "alpha": "equal to rank (scale 1.0) in every shipped version; rsLoRA runs used alpha 32 at r112 and 30 at r96",
  "dropout": "0 (0.1 tried in v14/v15, closed)",
  "rslora": "off (tried v13-v15, closed)",
  "dora": "off (OOM at r64 and r112)",
  "bias": "none",
  "source": "train_router.py:59-66,120-133; train.py:106-113",
  "measured_on": "2026-09-25"
 },
 "hyperparameters": {
  "epochs": 3,
  "learning_rate": 0.0002,
  "lr_scheduler": "linear",
  "warmup_steps": 5,
  "per_device_train_batch_size": 1,
  "gradient_accumulation_steps": 8,
  "effective_batch": 8,
  "optimizer": "paged_adamw_8bit",
  "weight_decay": 0.01,
  "seed": 3407,
  "bf16": true,
  "max_seq_length": 8192,
  "gradient_checkpointing": "on, use_reentrant=False, via prepare_model_for_kbit_training",
  "attn_implementation": "sdpa",
  "save_and_eval_steps": 25,
  "decoding_temperature_at_inference": 0,
  "source": "train_router.py:46-66,116-119,209-237; router_ledger.html (tried and closed: router decoding temperature)",
  "measured_on": "2026-09-18"
 },
 "memory_fixes": [
  {
   "fix": "chunked MLP: Gemma3MLP.forward monkey-patched to run gate/up/down over 2048-token chunks of the sequence",
   "why": "gate_proj(x) and up_proj(x) over ~6300 tokens must coexist for the gated product; that pair was the measured OOM point",
   "source": "train_router.py:86-114; LOCAL_LLM_PHASE2_ROADMAP.md:671-680",
   "measured_on": "2026-09-14"
  },
  {
   "fix": "torch.autograd.graph.save_on_cpu(pin_memory=True) around trainer.train()",
   "why": "gradient-checkpoint boundary tensors for 34 layers stayed on GPU; memory climbed 4.5 to 6.99 GB in one forward, 6.39 GB with the offload",
   "source": "train_router.py:253-264; LOCAL_LLM_PHASE2_ROADMAP.md:681-687",
   "measured_on": "2026-09-14"
  },
  {
   "fix": "lm_head cast fp32 to bf16",
   "why": "fp32 lm_head was 2.68 GB and forced a mixed-precision matmul; pre-training VRAM fell 4502 MB to 3221 MB",
   "source": "train_router.py:137-146; LOCAL_LLM_PHASE2_ROADMAP.md:688-695",
   "measured_on": "2026-09-14"
  },
  {
   "fix": "logits_to_keep = response_len + 1",
   "why": "full [1, 6390, 262208] fp32 logits would be 6.7 GB vs 135 MB for the response span; did not fix the first OOM (that was the MLP) but removes the tensor whenever the forward reaches lm_head",
   "source": "train_router.py:166-185; LOCAL_LLM_PHASE2_ROADMAP.md:579-600",
   "measured_on": "2026-09-14"
  },
  {
   "fix": "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
   "why": "allocator setting only; bought 3 steps in the v14 dropout runs, not a fix",
   "source": "train_router.py:35; LOCAL_LLM_PHASE2_ROADMAP.md:1268-1269",
   "measured_on": "2026-09-23"
  },
  {
   "fix": "pre-tokenize and mask by hand; no datasets.map()",
   "why": "map() crashed dill-pickling a closure over the Unsloth-patched tokenizer",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:517-524",
   "measured_on": "2026-09-14"
  },
  {
   "fix": "token_type_ids all zeros",
   "why": "plain transformers Gemma3 requires them during training (clear ValueError)",
   "source": "train_router.py:174; LOCAL_LLM_PHASE2_ROADMAP.md:533-535",
   "measured_on": "2026-09-14"
  }
 ],
 "did_not_help": [
  {
   "lever": "expandable_segments allocator tuning",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:541-542"
  },
  {
   "lever": "paged_adamw_8bit (kept, but the first OOM was in forward, before any optimizer step)",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:542-543"
  },
  {
   "lever": "non-reentrant gradient checkpointing alone",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:543-544"
  },
  {
   "lever": "splitting layers across both GPUs with device_map=auto (pipeline split does not shrink one layer's activations)",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:544-546,562-571"
  },
  {
   "lever": "attention-only LoRA targets (moved the OOM one line; the floor is the base MLP)",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:601-610"
  },
  {
   "lever": "Liger Kernel fused GEGLU (fuses only the elementwise step after both projections exist)",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:611-619"
  },
  {
   "lever": "flash-attn (attention was never the bottleneck; built from source after installing a CUDA 13.0 toolkit, then abandoned)",
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:653-667"
  }
 ],
 "why_plain_transformers_peft_not_unsloth": {
  "decision": "train with transformers.AutoModelForCausalLM + BitsAndBytesConfig + peft.get_peft_model; Unsloth's FastLanguageModel loader dropped on 2026-09-14",
  "reasons": [
   "Unsloth's patched Gemma 4 E2B MoE forward crashed whenever any part of the model sat off-GPU (embedding offloaded to CPU, index_select device mismatch); E2B needs 8-10 GB and does not fit one 8 GB card.",
   "Unsloth multi-GPU was not supported (PR #2074 closed unmerged); spanning both cards loaded cleanly but transformers' own Gemma 4 forward then hit a cuda:0/cuda:1 mismatch.",
   "Forcing device_map={'': 0} silently disabled Unsloth's 4-bit quantization ('Switching to 16bit LoRA') and OOMed.",
   "On gemma-3-4b-it, Unsloth's Gemma3 attention patch raised 'too many values to unpack' on real ~6300-token sequences (a 2048-token trial never reached that path).",
   "datasets.map() via Unsloth's train_on_responses_only crashed pickling the patched tokenizer."
  ],
  "source": "LOCAL_LLM_PHASE2_ROADMAP.md:346-439,515-535; train_v4.py docstring",
  "measured_on": "2026-09-14"
 },
 "libraries": {
  "versions": {
   "python": "3.12.13",
   "torch": "2.6.0+cu124",
   "transformers": "5.5.0",
   "peft": "0.20.0",
   "trl": "0.24.0",
   "accelerate": "1.15.0",
   "bitsandbytes": "0.50.2",
   "datasets": "4.3.0",
   "safetensors": "0.8.0",
   "xformers": "0.0.29.post3",
   "unsloth": "2026.9.4",
   "unsloth_zoo": "2026.9.3",
   "liger_kernel": "0.8.2",
   "flash_attn": "2.8.3.post1 (built, unused)",
   "torchao": "0.9.0 (pinned; not present in the venv on 2026-10-15)"
  },
  "note": "Read from the training venv's installed packages on 2026-10-15. Install dates are 2026-09-13; no later upgrade is recorded, so these are taken to be the training versions. torch was upgraded in place from 2.5.1+cu121 to 2.6.0+cu124 on 2026-09-13.",
  "inferred": true,
  "source": "training venv dist-info listing; LOCAL_LLM_PHASE2_ROADMAP.md:296-302,482-486",
  "measured_on": "2026-10-15"
 },
 "driver_and_hardware": {
  "nvidia_driver": "580.178.04",
  "cuda_toolkit_installed": "13.0 (toolkit only, for the flash-attn build)",
  "training_gpu": "CUDA device 1 under CUDA_DEVICE_ORDER=PCI_BUS_ID, which is an RTX 3070 Ti 8 GB on this box; torch reports 7.65 GiB total capacity",
  "other_gpu": "RTX 3060 Ti 8 GB. Not used for training. The live Ollama server shared the cards and repeatedly reloaded models onto the training GPU (see gotchas).",
  "inferred": true,
  "note": "GPU model names read from nvidia-smi on 2026-10-15; no September log names the card. The program notes confirms 'two 8GB GPUs' and ~7.65 GB usable.",
  "source": "train_router.py:33-34; nvidia-smi; probe_rank128.log; LOCAL_LLM_PHASE2_ROADMAP.md:413,439,653-657",
  "measured_on": "2026-10-15"
 },
 "runtime_per_run": {
  "rank112_full_corpus_hours": {
   "value": [
    11.8,
    12.5
   ],
   "runs": "v11 (1,167 ex, 438 steps), v17 (1,189 ex, 447 steps)",
   "source": "train_v11-r112_relaunch_1789910846.log; train_v17-r112-full_relaunch_1790269813.log (train_runtime)",
   "measured_on": "2026-09-25"
  },
  "rank112_two_epochs_hours": {
   "value": 7.9,
   "source": "train_v12-r112-e2_relaunch_1789974074.log",
   "measured_on": "2026-09-21"
  },
  "seconds_per_optimizer_step_r16": {
   "value": 81,
   "source": "LOCAL_LLM_PHASE2_ROADMAP.md:805-806",
   "measured_on": "2026-09-15"
  },
  "note": "HF Trainer's train_runtime covers only the last resumed segment when a run was killed and resumed (v2, v5)."
 },
 "export_path": {
  "steps": [
   "merge: load base in bf16 on CPU, PeftModel.from_pretrained, merge_and_unload, save safetensors (merge_adapter_generic.py)",
   "tensor-name fix: drop the 439 vision/projector tensors and rename the 444 text tensors from model.language_model.* to language_model.* (fix_tensor_names_generic.py; 883 tensors total)",
   "llama.cpp convert_hf_to_gguf.py --outtype f16",
   "llama.cpp llama-quantize ... Q8_0",
   "ollama create from a Modelfile (below)"
  ],
  "tensor_fix_reason": "not stated in the sources; the merged checkpoint carries Gemma 3's vision tower and a nested text-model prefix, and only the text tensors are kept",
  "tensor_fix_reason_inferred": true,
  "llama_cpp_checkout": "commit ad6c668, committed 2026-09-14 (checked 2026-10-15)",
  "gguf_sizes_gb": {
   "f16": 7.78,
   "Q8_0": 4.14,
   "Q4_0": 2.37
  },
  "verification_lesson": "a sed substitution once built the v3 Ollama tag from v2's GGUF; caught by comparing model-layer sha256 and Ollama model id before baking",
  "source": "v17_final_chain.sh:41-46; merge_adapter_generic.py; fix_tensor_names_generic.py; merge logs; GGUF file sizes; LOCAL_LLM_PHASE2_ROADMAP.md:810-815,947-951",
  "measured_on": "2026-09-25"
 },
 "ollama_modelfile": {
  "text": "FROM <path-to>/gemma3-4b-router-<version>.Q8_0.gguf\nTEMPLATE \"\"\"{{- range $i, $_ := .Messages }}\n{{- $last := eq (len (slice $.Messages $i)) 1 }}\n{{- if or (eq .Role \"user\") (eq .Role \"system\") }}<start_of_turn>user\n{{ .Content }}<end_of_turn>\n{{ if $last }}<start_of_turn>model\n{{ end }}\n{{- else if eq .Role \"assistant\" }}<start_of_turn>model\n{{ .Content }}{{ if not $last }}<end_of_turn>\n{{ end }}\n{{- end }}\n{{- end }}\"\"\"\nPARAMETER stop <end_of_turn>\nPARAMETER temperature 1\nPARAMETER top_k 64\nPARAMETER top_p 0.95\nPARAMETER num_ctx 8192",
  "notes": [
   "system and user turns both render as a Gemma user turn",
   "the router overrides temperature to 0 per request",
   "the client sends num_ctx 9216 per request in production (code default since 2026-09-19); the Modelfile's 8192 is a fallback"
  ],
  "source": "Modelfile-gemma3-4b-router-v17-full (path stripped); CLAUDE.md (OLLAMA_NUM_CTX); router_ledger.html",
  "measured_on": "2026-09-25"
 },
 "q8_0_vs_q4_0": {
  "model": "v5 (rank 16)",
  "test_set": 865,
  "q8_0": {
   "passed": 773,
   "accuracy": 0.8936,
   "s_per_case": 1.506
  },
  "q4_0": {
   "passed": 745,
   "accuracy": 0.8613,
   "s_per_case": 1.278
  },
  "delta_pp": -3.23,
  "speed_ratio_q4_over_q8": 0.849,
  "verdict": "Q8_0 stays: -3.2 pp is not worth ~15% speed.",
  "source": "bake_v5_q4vq8_summary.json",
  "measured_on": "2026-09-16"
 },
 "release_rule": {
  "rule": "veto any model that fails a held-out guard an incumbent passes; rank the rest by held-out accuracy; break ties within 0.5 pp by overall accuracy. For v17 the scripted rule was: beat the incumbent on the fair slice and add no new held-out guard breach.",
  "why": "the earlier 'highest overall accuracy wins' rule picked v13 (rsLoRA) on 22 Sep and had to be overruled",
  "source": "pick_release_winner.py docstring; v17_final_chain.sh:8-11",
  "measured_on": "2026-09-25"
 }
}