{
  "at": "2026-10-14T18:09:43.432Z",
  "question": "Can visual and executable rewards improve a small model's responsive UI generation beyond supervised fine-tuning?",
  "briefs": 24,
  "seeds": [
    1701,
    2718
  ],
  "families": [
    "booking",
    "settings",
    "pricing",
    "search"
  ],
  "generatedCandidates": 288,
  "pairwise": [
    {
      "control": "sft",
      "n": 48,
      "wins": 31,
      "ties": 1,
      "losses": 16,
      "winRate": 0.65625,
      "ci": [
        0.5520833333333334,
        0.7604166666666666
      ]
    },
    {
      "control": "base",
      "n": 48,
      "wins": 26,
      "ties": 2,
      "losses": 20,
      "winRate": 0.5625,
      "ci": [
        0.4375,
        0.6875
      ]
    },
    {
      "control": "best",
      "n": 48,
      "wins": 14,
      "ties": 3,
      "losses": 31,
      "winRate": 0.3229166666666667,
      "ci": [
        0.19791666666666666,
        0.4583333333333333
      ]
    }
  ],
  "methods": [
    {
      "method": "base",
      "n": 48,
      "complete": 47,
      "coverage": 0.8127604166666664,
      "geometry": 0.7148157054571387,
      "visual": 0.5789930555555555,
      "responsive": 0.203125,
      "outputTokens": 1683.25,
      "inferenceCandidates": 1
    },
    {
      "method": "sft",
      "n": 48,
      "complete": 39,
      "coverage": 0.7758258928571428,
      "geometry": 0.7440207047604982,
      "visual": 0.43098958333333326,
      "responsive": 0.3658854166666667,
      "outputTokens": 2724.4791666666665,
      "inferenceCandidates": 1
    },
    {
      "method": "rl",
      "n": 48,
      "complete": 42,
      "coverage": 0.8314942956349203,
      "geometry": 0.8325517731058655,
      "visual": 0.502170138888889,
      "responsive": 0.41796875,
      "outputTokens": 2322.0625,
      "inferenceCandidates": 1
    },
    {
      "method": "best",
      "n": 48,
      "complete": 48,
      "coverage": 0.8908680555555554,
      "geometry": 0.8464559056206474,
      "visual": 0.6406249999999999,
      "responsive": 0.453125,
      "outputTokens": 1693.1041666666667,
      "inferenceCandidates": 4
    }
  ],
  "limitations": [
    "Synthetic pilot with four held-out task families.",
    "Two samples per brief, confidence intervals resample briefs, not individual generations.",
    "Training and blind evaluation use the same model family, different prompts. No human preference study.",
    "Best-of-four selection spends four generations plus visual ranking per delivered UI.",
    "The generator is already a visual-code model before this experiment."
  ]
}