{
  "name": "Can Jev do structured outputs?",
  "kind": "Synthetic micro-experiment, not a full benchmark",
  "date": "2026-09-30",
  "question": "Can breadth-first typed decisions assemble schema-valid, request-correct objects without text generation?",
  "unit": "One hand-authored request and its Zod schema, paired across two methods",
  "matrix": { "cases": 24, "tiers": 6, "casesPerTier": 4, "attemptsPerMethod": 1, "totalTrials": 48 },
  "methods": {
    "jev": { "model": "jev-1.13.0", "endpoint": "https://api.typesafe.ai/v1/systemone", "policy": "Greedy Choice selection. Batch all decisions on each breadth-first schema frontier in one request. Only visit selected branches and array elements." },
    "luna": { "model": "gpt-6-luna", "endpoint": "https://api.openai.com/v1/responses", "reasoningEffort": "high", "strict": true, "maxOutputTokens": 8192, "stream": true, "modelPin": "Only the undated native API alias is available. Record returned model IDs." }
  },
  "variables": "Both methods must use VARIABLE:<slug> for every open string/number/integer. Slugs enumerate all base-type-compatible memory values, not just values satisfying ranges or patterns. Integers exclude fractional numbers. The identical memory contains distractors and constraint-invalid values. Substitute once, after the whole object is complete, then validate the original schema. No repair or gold-guided filtering.",
  "subset": "Strict required objects; enums, literals, booleans, nullable unions; bounded arrays (maxItems <= 8); disjoint discriminated unions expressed as anyOf; constrained string/number/integer leaves via memory references. No recursive refs, unbounded arrays, dynamic keys, optional missing keys, or arbitrary JSON Schema composition. Not general free-text generation.",
  "cases": "Hand-authored synthetic cases: flat decisions, memory leaves, nested constraints and nulls, ordered arrays, branch-specific delivery, deep workflows with arrays of unions. Four per tier, including empty arrays, negations, alternate branches, and absent values. No target case selection from model outcomes.",
  "controls": "Same input request, original schema, memory, and output-reference convention. Different decoding/assembly policies are the treatment. Code fixes Jev object keys and constants; Luna strict decoding fixes its shape. Both see original constraints. Neither sees gold. No additional few-shot examples.",
  "preflight": "Two separate synthetic cases with different memory. Verify multiple layers, full request traces, substitution, usage, independent validators, scoring, and report rendering before freezing.",
  "order": "One trial at a time on the same Montreal Mac/controller. Case-major; arm order alternates Jev-first/Luna-first. No concurrency. One attempt, no model-error repairs or automatic transport retries. Halt on API/transport failure and disclose incomplete data.",
  "timing": "Wall time per object, including schema compilation, all API round trips, serialization, local trace writes, memory substitution, and Zod validation. Excludes offline scorer, smoke/setup, publishing, and human work. Includes schema cold-start if incurred. Native API paths differ; no production-latency claim.",
  "metrics": "Wire-schema validity (Ajv), resolved original-schema validity (Zod plus independent Ajv), exact whole-object match (ordered arrays, key-order independent), exact leaf match, median end-to-end seconds, requests per object, input/output tokens, estimated API cost per object and total. Missing or invalid outputs count as failures. No cherry-picked reruns.",
  "pricingUSDPerMillion": { "jevInput": 0.042, "jevOutput": 0, "lunaInput": 0.1, "lunaCachedInput": 0.01, "lunaOutputIncludingReasoning": 0.5 },
  "cost": "Estimate from recorded provider usage and documented Standard API rates, not an invoice. Report smoke/setup separately; exclude fixed controller cost, taxes, and human time. No cache control; record cached tokens and billed-rate effect.",
  "analysis": "Report each tier and all 24 pairs. Treat results as a feasibility demonstration with one attempt per synthetic request, no claim of broad superiority or statistically established quality differences.",
  "publication": "Audit must pass before final complete state. Publish only synthetic inputs, gold, request/response traces without headers, results, protocol, audit, and whitelisted source. Preserve all existing experiments."
}