{
  "model": "imajev-4b-mlx-8bit",
  "base": "mlx-community/Qwen3.5-4B-MLX-8bit@5319bbbe",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "36db20dd",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.01062062497298484,
      "probabilities": {
        "f1ded018": 0.0024545999805502213,
        "d65f5e07": 0.4763337384153213,
        "fa3bada5": 0.00271306571440695,
        "36db20dd": 0.5184985958897215
      },
      "ms": 11936,
      "tokens": 1114
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "feat",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.013067080638446676,
      "probabilities": {
        "refactor": 0.3043614479105549,
        "docs": 0.006849926346228064,
        "fix": 0.005298306813945578,
        "test": 0.004648628197559488,
        "feat": 0.678841690731712
      },
      "ms": 1460,
      "tokens": 449
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.004116137772267619,
      "probabilities": {
        "not_supported": 0.057377803960761184,
        "supported": 0.9426221960392388
      },
      "ms": 1425,
      "tokens": 644
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.00005445763298744156,
      "probabilities": {
        "none": 0.006346573686770052,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.9885440920006697,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.0040414788138018855,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.0010678554987583965
      },
      "ms": 1077,
      "tokens": 507
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "not_toxic",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.00406840260956039,
      "probabilities": {
        "toxic": 0.011968012347926098,
        "not_toxic": 0.9880319876520739
      },
      "ms": 789,
      "tokens": 348
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "jailbreak",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.002051555319920109,
      "probabilities": {
        "jailbreak": 0.8741230428007104,
        "injection": 0.11899407524234436,
        "benign": 0.006882881956945307
      },
      "ms": 1194,
      "tokens": 551
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.012509329834446845,
      "probabilities": {
        "student_loan": 0.00047650877237984785,
        "debt_collection": 0.0010514380067824537,
        "bank_account": 0.005552028749105153,
        "mortgage": 0.0006325624896662557,
        "credit_card": 0.0013722864316664436,
        "credit_reporting": 0.0009862917451133907,
        "money_transfer": 0.9899288838052865
      },
      "ms": 1150,
      "tokens": 552
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.016080745015602464,
      "probabilities": {
        "student_loan": 0.0012706985281166702,
        "credit_card": 0.03985625248145131,
        "credit_reporting": 0.007530949879406442,
        "bank_account": 0.945958202080844,
        "money_transfer": 0.002157422261483951,
        "debt_collection": 0.0015055887531912537,
        "mortgage": 0.0017208860155063658
      },
      "ms": 1068,
      "tokens": 496
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.8869789513142813,
      "probabilities": {
        "wall_art": 0.011571155135984924,
        "grocery": 0.009921326010016246,
        "chair": 0.032214557343166646,
        "lamp": 0.013005653278390294,
        "rug": 0.01419556548569754,
        "table": 0.012175226098597565,
        "shoes": 0.008601748662327071,
        "sofa": 0.8983147679858198
      },
      "ms": 1298,
      "tokens": 634
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.9076489374821851,
      "probabilities": {
        "rug": 0.009594538381040223,
        "lamp": 0.007773466229062744,
        "sofa": 0.9244272231340913,
        "chair": 0.02652217361030571,
        "grocery": 0.007938790276635889,
        "wall_art": 0.0068506973709621406,
        "shoes": 0.006603225164480814,
        "table": 0.010289885833421136
      },
      "ms": 1290,
      "tokens": 595
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.010158254277376984,
      "probabilities": {
        "risk_factors": 0.002640988747872033,
        "controls": 0.0029753032012356656,
        "legal_proceedings": 0.00210415334379488,
        "business": 0.9311526502160916,
        "mdna": 0.061126904491005794
      },
      "ms": 1220,
      "tokens": 592
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.8864122259545895,
      "probabilities": {
        "revenue": 0.009274748277734975,
        "income_tax_expense": 0.012155639609450692,
        "borrowings": 0.018086822357258243,
        "amortization": 0.012633204054062343,
        "share_based_compensation": 0.9354573173286507,
        "goodwill": 0.0123922683728431
      },
      "ms": 1201,
      "tokens": 588
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.04280073762422768,
      "probabilities": {
        "governing_law": 0.0010218323748469806,
        "termination_for_convenience": 0.9953535450997126,
        "anti_assignment": 0.0010602723330977708,
        "non_compete": 0.0008529549363948482,
        "license_grant": 0.0007325763432132844,
        "cap_on_liability": 0.000978818912734535
      },
      "ms": 1274,
      "tokens": 604
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "limitation_of_liability",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0012847780639486197,
      "probabilities": {
        "unilateral_termination": 0.0019534387160833585,
        "limitation_of_liability": 0.9129153789589042,
        "choice_of_law": 0.001969011712813373,
        "not_unfair": 0.028140991046218547,
        "contract_by_using": 0.0015610330854657671,
        "content_removal": 0.0497220760358452,
        "arbitration": 0.0014757637984650245,
        "unilateral_change": 0.0013661402369606627,
        "jurisdiction": 0.0008961664092438249
      },
      "ms": 1189,
      "tokens": 569
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.9927148824365649,
      "probabilities": {
        "B": 0.3809149486775431,
        "A": 0.6190850513224568
      },
      "ms": 1057,
      "tokens": 511
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0029910423039095865,
      "probabilities": {
        "minor": 0.5773562354168111,
        "major": 0.08537411945405268,
        "patch": 0.33726964512913615
      },
      "ms": 955,
      "tokens": 434
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.1989127595460664,
      "probabilities": {
        "category": 0.012692594900379451,
        "money": 0.8634622326993974,
        "identifier": 0.029850636848413065,
        "count_or_measurement": 0.055388845708129394,
        "date": 0.014835609277745515,
        "geography": 0.015602430129832253,
        "free_text": 0.008167650436102947
      },
      "ms": 1623,
      "tokens": 734
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.07705240862238988,
      "probabilities": {
        "money": 0.8859200307024981,
        "geography": 0.009344206374729613,
        "category": 0.0076470707402519775,
        "identifier": 0.014056932329165215,
        "free_text": 0.0050664976881357035,
        "count_or_measurement": 0.05212226952024006,
        "date": 0.025842992644979395
      },
      "ms": 2022,
      "tokens": 870
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "neither",
      "correct": false,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0050962429056762865,
      "probabilities": {
        "action_item": 0.17815820870421914,
        "neither": 0.45650900665044286,
        "decision": 0.365332784645338
      },
      "ms": 1572,
      "tokens": 646
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.002301583281511488,
      "probabilities": {
        "decision": 0.017842200538139563,
        "action_item": 0.013430159471672818,
        "neither": 0.9687276399901876
      },
      "ms": 1326,
      "tokens": 638
    }
  ]
}