{
 "about": "Environment and tooling failures hit during the program, with symptom, cause, fix and versions. Versions are those recorded at the time; see recipe.json for the final stack.",
 "gotchas": [
  {
   "id": "G01",
   "title": "Ollama silently truncates prompts longer than its default context",
   "symptom": "Intermittent routing misses that looked like model flakiness; the model still returned plausible answers.",
   "cause": "Ollama assigned each model a VRAM-derived num_ctx of 4096 while the routing prompt had grown to ~5,650 tokens. For one model it pre-trimmed the prompt to 2,050 tokens (visible only as task.n_tokens in the llama-server log), dropping the capability list at the front of the prompt. The response's truncated flag stayed 0. A model without context-shift support (LFM2) surfaced the same overflow as HTTP 400, which is how it was noticed.",
   "fix": "Send num_ctx explicitly on every call (8192 from 2026-09-07, 9216 from 2026-09-19) and keep a deterministic test that bounds the measured prompt token count against it.",
   "versions": {
    "ollama": "0.32"
   },
   "sources": [
    "CLAUDE.md (Lessons learned: Ollama default context window)",
    "LOCAL_LLM_PHASE2_ROADMAP.md:1063"
   ],
   "measured_on": "2026-09-07"
  },
  {
   "id": "G02",
   "title": "NVIDIA driver/library mismatch blocks new CUDA processes but not running ones",
   "symptom": "torch.cuda.is_available() returned False with 'Error 804: forward compatibility was attempted on non supported HW'; nvidia-smi reported 'Driver/library version mismatch'. Already-running Ollama processes kept working.",
   "cause": "Kernel driver module and on-disk user-space libraries were at different versions after an update; processes that created a CUDA context before the change kept it, new processes could not create one.",
   "fix": "Reboot to reload the kernel module. torch.cuda.is_available() was True immediately after.",
   "versions": {
    "nvml_library": "580.178",
    "torch": "2.5.1+cu121 at the time"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:285-291,320-334"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G03",
   "title": "torchao too new for the pinned torch",
   "symptom": "import unsloth failed at module load, before touching the GPU.",
   "cause": "uv resolved torchao 0.18.0, which references torch.int1, a dtype missing from torch 2.5.1.",
   "fix": "Pin torchao==0.9.0, contemporary with torch 2.5.x.",
   "versions": {
    "torch": "2.5.1+cu121",
    "torchao_bad": "0.18.0",
    "torchao_good": "0.9.0"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:296-305"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G04",
   "title": "Unsloth's Gemma 4 E2B MoE forward patch breaks when any weight is off-GPU",
   "symptom": "RuntimeError: tensors on cpu and cuda:0 in index_select during the forward pass.",
   "cause": "Unsloth's loader offloaded the embedding table to CPU to fit, and the patched Gemma 4 MoE forward in unsloth_zoo does not move inputs to match. Reproduced with and without explicit fp32 CPU offload. Gemma 4 E2B LoRA needs 8-10 GB per Unsloth's own guide, so it never fits one 8 GB card without offload.",
   "fix": "None found. Switched base model to unsloth/gemma-3-4b-it, which loads in 4.39 GB with no offload.",
   "versions": {
    "unsloth": "2026.9.4",
    "unsloth_zoo": "2026.9.3",
    "transformers": "5.5.0 (the maximum Unsloth's metadata allowed)",
    "model": "unsloth/gemma-4-E2B-it"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:346-406,408-411,459-474"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G05",
   "title": "Forcing device_map={'': 0} silently disables Unsloth's 4-bit quantization",
   "symptom": "OOM loading a model that should fit in 4-bit; message 'Unsloth: QLoRA and full finetuning all not selected. Switching to 16bit LoRA'.",
   "cause": "An explicit single-device map bypasses Unsloth's quantization detection regardless of the quantization_config passed.",
   "fix": "Do not force device_map with Unsloth's loader. Plain transformers + BitsAndBytesConfig honours device_map={'': 0} and quantizes correctly.",
   "versions": {
    "unsloth": "2026.9.4"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:362-366,387-391"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G06",
   "title": "bitsandbytes refuses auto device maps with any CPU placement",
   "symptom": "ValueError: Some modules are dispatched on the CPU or the disk, even with the whole 8 GB card free.",
   "cause": "Accelerate's auto planner placed some modules on CPU; bnb's validate_environment refuses that unless fp32 CPU offload is opted in. Why more free VRAM gave a stricter failure than less was not understood.",
   "fix": "Not pursued; superseded by the base-model change.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:371-380"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G07",
   "title": "Unsloth multi-GPU is not supported; Gemma 4 crashes across two cards",
   "symptom": "With both GPUs visible the 4-bit split loaded cleanly (4.38 + 3.17 GB) but the forward raised 'tensors on cuda:0 and cuda:1' inside transformers' modeling_gemma4.py.",
   "cause": "Gemma 4's unified text/vision input path masks in a pad_embedding buffer that ends up on a different GPU from inputs_embeds; neither Unsloth's patches nor accelerate's dispatch hooks move it. Unsloth's multi-GPU PR #2074 was closed unmerged (2025-03-18).",
   "fix": "Single-GPU only. Every Gemma 4 size is unified multimodal, so none avoids this; stepped back to Gemma 3 4B.",
   "versions": {
    "transformers": "5.5.0",
    "unsloth": "2026.9.4"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:412-439,459-469"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G08",
   "title": "transformers 5.5.0 mask utilities require torch 2.6",
   "symptom": "ValueError: Using or_mask_function ... require torch>=2.6 on the first gemma-3-4b forward.",
   "cause": "torch was pinned at 2.5.1 from the earlier driver test.",
   "fix": "Upgrade in place to torch 2.6.0+cu124 with a matching xformers 0.0.29.post3, pinned together. A blind 'pip -U xformers' pulled torch 2.14.0, outside Unsloth's supported range (<2.13.0); reverted.",
   "versions": {
    "torch": "2.5.1+cu121 -> 2.6.0+cu124",
    "xformers": "0.0.29.post3",
    "transformers": "5.5.0"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:476-490"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G09",
   "title": "Unsloth's Gemma3 attention patch fails on long sequences",
   "symptom": "'too many values to unpack' on real ~6300-token examples; a 2048-token trial had passed.",
   "cause": "Length-dependent bug in unsloth_zoo/temporary_patches/gemma.py.",
   "fix": "Drop FastLanguageModel; use plain transformers AutoModelForCausalLM + BitsAndBytesConfig + peft with the same LoRA config.",
   "versions": {
    "unsloth_zoo": "2026.9.3"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:525-533",
    "train_v4.py docstring"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G10",
   "title": "datasets.map() crashes pickling an Unsloth-patched tokenizer",
   "symptom": "cannot pickle 'ConfigModuleInstance' object, from both SFTTrainer._prepare_dataset and Unsloth's train_on_responses_only.",
   "cause": "Dill tries to pickle a closure over the patched tokenizer that captures a torch._dynamo/_inductor-style config singleton.",
   "fix": "Pre-tokenize and mask by hand in the main process; pass dataset_kwargs={'skip_prepare_dataset': True}; no .map() anywhere.",
   "versions": {
    "datasets": "4.3.0",
    "trl": "0.24.0"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:517-524"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G11",
   "title": "SFTTrainer argument renames",
   "symptom": "TypeError on tokenizer= and max_seq_length=.",
   "cause": "Renamed to processing_class= and max_length= in this trl version.",
   "fix": "Use the new names (later replaced SFTTrainer with plain Trainer and a custom collator).",
   "versions": {
    "trl": "0.24.0"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:515-517",
    "train_router.py:209-237"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G12",
   "title": "Gemma3 training requires token_type_ids",
   "symptom": "Clear ValueError during training with plain transformers.",
   "cause": "Gemma3's forward expects token_type_ids in training mode (image/text segmenting).",
   "fix": "Supply all zeros for text-only data.",
   "versions": {
    "transformers": "5.5.0"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:533-535",
    "train_router.py:174"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G13",
   "title": "Gemma3 decoder moved under model.model.language_model",
   "symptom": "A trainer copy that iterated model.model.layers broke.",
   "cause": "transformers' Gemma3 code nests the decoder at model.model.language_model.layers.",
   "fix": "Find modules by type name (Gemma3MLP) via model.modules(), which ignores nesting depth.",
   "versions": {
    "transformers": "5.5.0"
   },
   "sources": [
    "train_router.py:19-28,109-114"
   ],
   "measured_on": "2026-09-16"
  },
  {
   "id": "G14",
   "title": "OOM in the MLP, not attention, on ~6.3k-token sequences",
   "symptom": "First real training step OOMed (7.39 GB used, tried 248 MB) inside an early decoder layer; flash-attn, Liger, attention-only LoRA, logits_to_keep, paged optimizer, allocator tuning and a two-GPU layer split all left the same ~200-250 MB gap.",
   "cause": "gate_proj(x) and up_proj(x) over the full sequence must coexist for the gated product (the base model's tensors, not LoRA's). Also: 34 checkpoint-boundary hidden states stayed on GPU, and lm_head shipped in fp32 (2.68 GB).",
   "fix": "Chunk the MLP over 2048-token slices; wrap training in save_on_cpu(pin_memory=True); cast lm_head to bf16. First clean run peaked at 4.93 GB.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:537-713",
    "train_router.py:86-146"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G15",
   "title": "uv served a stale cached wheel across rebuilds",
   "symptom": "flash-attn rebuilt with different CFLAGS three times and kept producing the same wheel despite --reinstall and --no-cache.",
   "cause": "uv's build cache is keyed on source content, not environment variables.",
   "fix": "Delete the package's directory under uv's sdists cache directly.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:662-667"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G16",
   "title": "flash-attn needs a system CUDA toolkit to build",
   "symptom": "pip install flash-attn failed immediately with 'CUDA_HOME environment variable is not set'.",
   "cause": "Only the pip-installed CUDA runtime was present, no nvcc.",
   "fix": "Installed the CUDA 13.0 toolkit only (no driver component, so the working 580.178.04 driver was untouched). The build then worked, but flash-attn turned out to be the wrong lever (G14).",
   "versions": {
    "cuda_toolkit": "13.0",
    "driver": "580.178.04",
    "flash_attn": "2.8.3.post1"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:551-555,653-667"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G17",
   "title": "PEFT LoRA dropout costs a backward mask, at any rate above zero",
   "symptom": "Rank-112 runs with lora_dropout=0.1 died at steps 236 and 239 with CUDA OOM (~6.9-7.0 GiB allocated of 7.65 GiB) where the same config at dropout 0 peaked at 6,653 MB.",
   "cause": "At lora_dropout=0 PEFT installs nn.Identity (no allocation); above 0 it installs nn.Dropout, which saves a mask per call for backward across 7 target modules on 34 layers. Estimated ~350 MB of peak. Lowering the rate to 0.05 would not help.",
   "fix": "Keep dropout at 0, or drop rank (rank 96 + dropout trained at 6,629 MB). expandable_segments bought only 3 steps.",
   "versions": {
    "peft": "0.20.0 (peft/tuners/lora/layer.py:251)"
   },
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1256-1273",
    "train_v14-r112-drop10_relaunch_1790126592.log",
    "train_v14-r112-drop10_relaunch_1790132778.log"
   ],
   "measured_on": "2026-09-23"
  },
  {
   "id": "G18",
   "title": "Xid 31 MMU fault during a near-full-memory training run",
   "symptom": "Training died at step 184 with 'CUDA error: an illegal memory access was encountered' inside bitsandbytes' optimizer step; the program notes records it as Xid 31 (MMU fault on the graphics engine). The kernel log itself was not re-checked for this dataset. The GPU recovered by itself.",
   "cause": "Not established. It happened on the same dropout config that later OOMed twice at ~0 free memory, the first Xid on the box in 7 days.",
   "fix": "None needed beyond abandoning the config; nothing wedged.",
   "versions": {
    "bitsandbytes": "0.50.2",
    "driver": "580.178.04"
   },
   "sources": [
    "train_v14-r112-drop10_relaunch_1790108172.log",
    "LOCAL_LLM_PHASE2_ROADMAP.md:1260-1261"
   ],
   "measured_on": "2026-09-22"
  },
  {
   "id": "G19",
   "title": "DoRA does not fit at rank 64 or 112 on 8 GB",
   "symptom": "OOM inside the DoRA weight-norm computation on the first step, with the GPU otherwise clear.",
   "cause": "DoRA's magnitude/weight-norm path allocates extra per-layer tensors on top of LoRA's.",
   "cause_inferred": true,
   "fix": "None on this card; needs shorter sequences or more VRAM.",
   "sources": [
    "probe_dora_r64.log",
    "probe_dora_r112.log"
   ],
   "measured_on": "2026-09-18"
  },
  {
   "id": "G20",
   "title": "A probe without model.train() silently disables gradient checkpointing",
   "symptom": "A one-step memory probe thrashed the host and later hit a 32 GB cgroup memory cap.",
   "cause": "HF only checkpoints when module.training is True. Without it, save_on_cpu spilled every activation into pinned host RAM.",
   "fix": "Call model.train() in the probe; run probes under systemd-run --scope -p MemoryMax=12G -p MemorySwapMax=0.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1003-1007",
    "probe_max_rank.py:77",
    "probe_sweep.sh"
   ],
   "measured_on": "2026-09-17"
  },
  {
   "id": "G21",
   "title": "A live Ollama server reloads models onto the training GPU within seconds",
   "symptom": "Training launches OOMed before reaching the loop, or were killed mid-run by the host's low-memory monitor (4 times for v5, 9 relaunches for v2).",
   "cause": "Production traffic reloaded an evicted model onto the training GPU within 1-2 s of 'ollama stop', winning the race against the trainer's allocation. save_on_cpu's pinned host memory also raised host-RAM pressure.",
   "fix": "A launcher that retries stop+launch up to 15 times and declares success only when the training loop starts; save a checkpoint every 25 steps and auto-resume from the latest. Do not evict the embedding model (its eviction livelocked its client).",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:719-723,931-945",
    "launch_router_retry.sh"
   ],
   "measured_on": "2026-09-15"
  },
  {
   "id": "G22",
   "title": "A partly CPU-resident model pollutes bake timings",
   "symptom": "Mean seconds per case jumped (1.71 for one model, ~2.8 s per case for split placement, up to 4.2 in a contended bake) while accuracy looked normal.",
   "cause": "With two other models resident, Ollama placed the new model split across CPU and GPU.",
   "fix": "Evict finished models between bake halves; read speed only from bakes with the model fully on GPU.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1021-1024",
    "bake_v7r112_vs_v6wide_summary.json",
    "bake_expanded_summary.json"
   ],
   "measured_on": "2026-09-18"
  },
  {
   "id": "G23",
   "title": "Merged Gemma 3 checkpoints need a tensor rename before GGUF conversion",
   "symptom": "Not recorded in the sources; the step exists in every export chain.",
   "cause": "The merged checkpoint carries the vision tower and projector plus a model.language_model.* prefix on text tensors.",
   "fix": "Before convert_hf_to_gguf.py, drop the 439 vision/projector tensors and rename the 444 text tensors to language_model.* (883 total). Verify the count every time.",
   "inferred": true,
   "sources": [
    "fix_tensor_names_generic.py",
    "merge logs",
    "LOCAL_LLM_PHASE2_ROADMAP.md:947-951"
   ],
   "measured_on": "2026-09-15"
  },
  {
   "id": "G24",
   "title": "Template substitution built a model tag from the wrong weights",
   "symptom": "The v3 Ollama tag would have served v2's GGUF.",
   "cause": "A sed substitution on the Modelfile assumed a filename pattern the new version did not follow.",
   "fix": "Verify the model-layer sha256 and Ollama model id differ from the previous version before baking; grep-verify every path.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:810-815,947-948"
   ],
   "measured_on": "2026-09-15"
  },
  {
   "id": "G25",
   "title": "Test harness routed on the wrong channel",
   "symptom": "Every context-dependent case scored as if it had no context, for every model, since July.",
   "cause": "Bake scripts routed on a default channel while each case's setup exchange seeded context on the case's own channel.",
   "fix": "Route each case on its setup's channel. v9's held-out rose from 86.8% to 91.7% on the same weights.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1054-1057"
   ],
   "measured_on": "2026-09-19"
  },
  {
   "id": "G26",
   "title": "Schema enum built from the visible docs dropped a hidden capability",
   "symptom": "Turning on grammar-constrained decoding broke 10 cases, 8 of them one capability forced to clarify.",
   "cause": "The JSON-schema enum came from the prompt's documented capability list, which hides one capability the router still accepts.",
   "fix": "Build the enum from every available capability, hidden ones included.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1181-1186"
   ],
   "measured_on": "2026-09-21"
  },
  {
   "id": "G27",
   "title": "rsLoRA with alpha left at rank is far too hot",
   "symptom": "Held-out eval loss flat at ~0.57 by step 50 (plain recipe ~0.33); train loss peaked at 3.85.",
   "cause": "rsLoRA scales by alpha/sqrt(r): 112/sqrt(112) = 10.6, versus 1.0 for the plain recipe at lr 2e-4.",
   "fix": "Lower alpha (32 gives scale 3.0). When averaging rsLoRA checkpoints by concatenation, also scale B by 1/sqrt(n).",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:1212-1217",
    "train_v13-r112-rslora_relaunch_1790010768.log",
    "average_checkpoints.py:23-31"
   ],
   "measured_on": "2026-09-21"
  },
  {
   "id": "G28",
   "title": "Static prompt prefix was not cached between calls",
   "symptom": "Warm routing calls took ~6.03 s for three different models with no speed-up across repeated same-prefix calls; one model (gemma4:12b) never reused cache once the dynamic tail varied (49-270 s per call in a reproduction).",
   "cause": "Partly sliding-window-attention / hybrid-memory cache restore limits in llama.cpp; partly contention from other models sharing the server. Not fully separated.",
   "fix": "Raised the router timeout (60 to 90 s) so a slow call is not turned into a wrong answer; planned (not done during the program) a dedicated GPU-pinned router instance.",
   "sources": [
    "LOCAL_LLM_PHASE2_ROADMAP.md:15-35,88-214,1066-1095"
   ],
   "measured_on": "2026-09-13"
  },
  {
   "id": "G29",
   "title": "AMD ROCm/Vulkan GPU wedges (before this program)",
   "symptom": "Three independent kernel wedge or device-loss incidents with the earlier AMD RX 7900 XTX serving Ollama.",
   "cause": "ROCm/Vulkan driver stack instability under an always-on local LLM.",
   "fix": "Hardware swap to NVIDIA/CUDA on 2026-08-07/08; no recurrence since. Context only: all fine-tuning ran on NVIDIA.",
   "sources": [
    "CLAUDE.md (Open items; Lessons learned: always-on local LLM backend)",
    "LOCAL_LLM_PHASE2_ROADMAP.md:1145-1147"
   ],
   "measured_on": "2026-08-08"
  },
  {
   "id": "G30",
   "title": "HF Trainer's train_runtime and train_loss cover only the last resumed segment",
   "symptom": "v2 reported train_loss 0.0274, far below every other run.",
   "cause": "After resume_from_checkpoint the reported aggregate covers only the steps run in that process.",
   "fix": "Read per-step losses from log_history across segments, or report peak and loss only for uninterrupted runs.",
   "inferred": true,
   "sources": [
    "train_v2n.log",
    "LOCAL_LLM_PHASE2_ROADMAP.md:721-724"
   ],
   "measured_on": "2026-09-14"
  },
  {
   "id": "G31",
   "title": "On Windows, a training step larger than VRAM spills to system memory instead of failing",
   "symptom": "With the memory fixes off, the probe on a 24 GB RX 7900 XTX reported a 35.8 to 36.6 GB peak and finished, with no out-of-memory error. The step took 58 to 69 s instead of about 14 s.",
   "cause": "The Windows display driver model lets GPU allocations overflow into shared system memory. The allocator's peak can exceed the card's capacity, and the only sign is the slowdown.",
   "fix": "On Windows, treat a peak above the card's total memory, or a step several times slower than expected, as a failure. A config that 'fits' on Windows this way runs out of memory on Linux.",
   "inferred": true,
   "versions": "Windows 11 build 26200, Adrenalin 26.7.1, ROCm 7.2.1, torch 2.9.1",
   "sources": [
    "vram.json cross_gpu"
   ],
   "measured_on": "2026-10-15"
  },
  {
   "id": "G32",
   "title": "Python hangs at exit after GPU work on Windows ROCm",
   "symptom": "A probe printed its result and then never exited; the process had to be killed.",
   "cause": "Interpreter shutdown hangs after ROCm GPU work on Windows. The cause is not known.",
   "fix": "Print results and flush, then call os._exit() to skip normal teardown, and run each probe in its own process with a watchdog that kills it shortly after the result line.",
   "versions": "Windows 11, ROCm 7.2.1, torch 2.9.1+rocm7.2.1, Python 3.12",
   "sources": [
    "probe_kit/probe_vram.py"
   ],
   "measured_on": "2026-10-15"
  }
 ]
}