{
  "model": "imajev-4b-mlx-4bit",
  "base": "mlx-community/Qwen3.5-4B-MLX-4bit@32f3e8ec",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "36db20dd",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.019647588377987225,
      "probabilities": {
        "f1ded018": 0.003121838515021863,
        "d65f5e07": 0.33997822341231915,
        "fa3bada5": 0.0031408884214305112,
        "36db20dd": 0.6537590496512284
      },
      "ms": 5624,
      "tokens": 1114
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "refactor",
      "correct": false,
      "same_as_bf16": false,
      "abstained": false,
      "unknown_probability": 0.20104468177570395,
      "probabilities": {
        "refactor": 0.5367821030296323,
        "docs": 0.010367737646386063,
        "fix": 0.006784116115275975,
        "test": 0.006905671686853539,
        "feat": 0.43916037152185217
      },
      "ms": 1001,
      "tokens": 449
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0044374937281403655,
      "probabilities": {
        "not_supported": 0.08231767178861138,
        "supported": 0.9176823282113886
      },
      "ms": 1365,
      "tokens": 644
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.00008311289398168969,
      "probabilities": {
        "none": 0.006679481413709458,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.9889362466186602,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.0033223630934570035,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.0010619088741733221
      },
      "ms": 1054,
      "tokens": 507
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "not_toxic",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.010428877384829298,
      "probabilities": {
        "toxic": 0.016325388497604492,
        "not_toxic": 0.9836746115023955
      },
      "ms": 803,
      "tokens": 348
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "jailbreak",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0015124220064162259,
      "probabilities": {
        "jailbreak": 0.9193240440410708,
        "injection": 0.07094117873452829,
        "benign": 0.009734777224400826
      },
      "ms": 1186,
      "tokens": 551
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.04492601940131245,
      "probabilities": {
        "student_loan": 0.0008963530206103706,
        "debt_collection": 0.0016891085035848523,
        "bank_account": 0.005966354494557554,
        "mortgage": 0.0010841829521743278,
        "credit_card": 0.0020799268171146465,
        "credit_reporting": 0.0019089266383133542,
        "money_transfer": 0.986375147573645
      },
      "ms": 1165,
      "tokens": 552
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.04409445262899312,
      "probabilities": {
        "student_loan": 0.0016834060105098134,
        "credit_card": 0.03510350864015815,
        "credit_reporting": 0.008111516069376684,
        "bank_account": 0.9497766457659339,
        "money_transfer": 0.0018443325270718627,
        "debt_collection": 0.0017407890443661594,
        "mortgage": 0.0017398019425834419
      },
      "ms": 1068,
      "tokens": 496
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.9903862665783549,
      "probabilities": {
        "wall_art": 0.035910516052017484,
        "grocery": 0.031041132578873882,
        "chair": 0.10693750015520105,
        "lamp": 0.0430796090048162,
        "rug": 0.040417322367583434,
        "table": 0.029766656943235135,
        "shoes": 0.023035400358711204,
        "sofa": 0.6898118625395616
      },
      "ms": 1404,
      "tokens": 634
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.9820224516411219,
      "probabilities": {
        "rug": 0.013859164778414465,
        "lamp": 0.010697174779718011,
        "sofa": 0.9044114120435324,
        "chair": 0.03112650033794433,
        "grocery": 0.012164203960172343,
        "wall_art": 0.00786007740616869,
        "shoes": 0.008260191192686832,
        "table": 0.011621275501362984
      },
      "ms": 1285,
      "tokens": 595
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.03853836297870491,
      "probabilities": {
        "risk_factors": 0.0030338654225172015,
        "controls": 0.003791727024116075,
        "legal_proceedings": 0.0027223899069865336,
        "business": 0.9202221605714903,
        "mdna": 0.07022985707488982
      },
      "ms": 1249,
      "tokens": 592
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.31774112244138314,
      "probabilities": {
        "revenue": 0.0038697912336317316,
        "income_tax_expense": 0.004906370988504401,
        "borrowings": 0.008471615146778669,
        "amortization": 0.005880581377254296,
        "share_based_compensation": 0.9704430703500789,
        "goodwill": 0.006428570903751875
      },
      "ms": 1264,
      "tokens": 588
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.038385115352113544,
      "probabilities": {
        "governing_law": 0.0011284076504257407,
        "termination_for_convenience": 0.9952391787187895,
        "anti_assignment": 0.001043787950243227,
        "non_compete": 0.0009510307614344485,
        "license_grant": 0.0006892692569431036,
        "cap_on_liability": 0.0009483256621639543
      },
      "ms": 1291,
      "tokens": 604
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "limitation_of_liability",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0018841587330860182,
      "probabilities": {
        "unilateral_termination": 0.0022092921778586916,
        "limitation_of_liability": 0.9132574232051051,
        "choice_of_law": 0.0021652551171686143,
        "not_unfair": 0.03745432159226857,
        "contract_by_using": 0.0018704058968168585,
        "content_removal": 0.03896346107655508,
        "arbitration": 0.0014656909938127458,
        "unilateral_change": 0.0017081771781362485,
        "jurisdiction": 0.0009059727622781305
      },
      "ms": 1248,
      "tokens": 569
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "same_as_bf16": true,
      "abstained": true,
      "unknown_probability": 0.9410522879545646,
      "probabilities": {
        "B": 0.32562220920481433,
        "A": 0.6743777907951858
      },
      "ms": 1097,
      "tokens": 511
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.003192769174930254,
      "probabilities": {
        "minor": 0.5779352358701669,
        "major": 0.12493647388725115,
        "patch": 0.2971282902425818
      },
      "ms": 952,
      "tokens": 434
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.09584475168643196,
      "probabilities": {
        "category": 0.009310613619429383,
        "money": 0.8923738498128552,
        "identifier": 0.028095129815152615,
        "count_or_measurement": 0.04262515829645705,
        "date": 0.015187521194274762,
        "geography": 0.006373671054955317,
        "free_text": 0.006034056206875722
      },
      "ms": 1556,
      "tokens": 734
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.062167965025225834,
      "probabilities": {
        "money": 0.8992750342357182,
        "geography": 0.008055321853270061,
        "category": 0.007638457465864982,
        "identifier": 0.015371519167971909,
        "free_text": 0.005645208481933926,
        "count_or_measurement": 0.04354730788578921,
        "date": 0.020467150909451712
      },
      "ms": 1838,
      "tokens": 870
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "neither",
      "correct": false,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0043626896886006,
      "probabilities": {
        "action_item": 0.3127547115183164,
        "neither": 0.46360094313696615,
        "decision": 0.22364434534471747
      },
      "ms": 1362,
      "tokens": 646
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "same_as_bf16": true,
      "abstained": false,
      "unknown_probability": 0.0024973617937099385,
      "probabilities": {
        "decision": 0.0192398488149209,
        "action_item": 0.018588194144252338,
        "neither": 0.9621719570408268
      },
      "ms": 1324,
      "tokens": 638
    }
  ]
}