{
  "date": "2026-09-28T20:23:06.067Z",
  "local": "imajev-4b MLX 1 rotation + calibration",
  "cloud": "jev-1.13.0",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "source": "CommitPackFT",
      "local": {
        "model": "imajev-4b",
        "label": "36db20dd",
        "correct": true,
        "confidence": 0.3647032204126109,
        "probabilities": {
          "f1ded018": 0.0025420576585180764,
          "d65f5e07": 0.4685273851718055,
          "fa3bada5": 0.002803736575908375,
          "36db20dd": 0.526126820593768
        },
        "abstained": false,
        "unknown_probability": 0.009413809490582067,
        "input_tokens": 1114,
        "ms": 2364
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "36db20dd",
        "correct": true,
        "confidence": 0.58,
        "probabilities": {
          "36db20dd": 0.69,
          "f1ded018": 0,
          "d65f5e07": 0.31,
          "fa3bada5": 0
        },
        "abstained": false,
        "input_tokens": 1307,
        "ms": 539
      }
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "source": "CommitPackFT",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "feat",
        "correct": true,
        "confidence": 0.99,
        "probabilities": {
          "refactor": 0,
          "feat": 1,
          "fix": 0,
          "docs": 0,
          "test": 0
        },
        "abstained": false,
        "input_tokens": 707,
        "ms": 214
      },
      "local": {
        "model": "imajev-4b",
        "label": "feat",
        "correct": true,
        "confidence": 0.5886765393610421,
        "probabilities": {
          "refactor": 0.30401908718478593,
          "docs": 0.007078240396088387,
          "fix": 0.005663039231755534,
          "test": 0.004901117681618557,
          "feat": 0.6783385155057516
        },
        "abstained": false,
        "unknown_probability": 0.015464537721986886,
        "input_tokens": 449,
        "ms": 917
      }
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "source": "HAGRID",
      "local": {
        "model": "imajev-4b",
        "label": "supported",
        "correct": false,
        "confidence": 0.8704062864517941,
        "probabilities": {
          "not_supported": 0.06284139474737926,
          "supported": 0.9371586052526206
        },
        "abstained": false,
        "unknown_probability": 0.004473118001631435,
        "input_tokens": 644,
        "ms": 1337
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "supported",
        "correct": false,
        "confidence": 0.79,
        "probabilities": {
          "supported": 0.9,
          "not_supported": 0.1
        },
        "abstained": false,
        "input_tokens": 872,
        "ms": 283
      }
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "source": "BFCL v3 Live (Berkeley Function-Calling Leaderboard)",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "project_api.ProjectApi.get_project_by_name_and_version",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0,
          "project_api.ProjectApi.get_project_by_name_and_version": 1,
          "badge_api.BadgeApi.get_project_policy_violations_badge1": 0,
          "none": 0
        },
        "abstained": false,
        "input_tokens": 776,
        "ms": 190
      },
      "local": {
        "model": "imajev-4b",
        "label": "project_api.ProjectApi.get_project_by_name_and_version",
        "correct": true,
        "confidence": 0.9840677639266333,
        "probabilities": {
          "none": 0.006596847649588046,
          "project_api.ProjectApi.get_project_by_name_and_version": 0.9880901411839121,
          "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.0042007389400160544,
          "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.0011122722264838064
        },
        "abstained": false,
        "unknown_probability": 0.00005327023996575821,
        "input_tokens": 507,
        "ms": 1062
      }
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "source": "google/civil_comments",
      "local": {
        "model": "imajev-4b",
        "label": "not_toxic",
        "correct": true,
        "confidence": 0.9721831389339433,
        "probabilities": {
          "toxic": 0.01171372791601833,
          "not_toxic": 0.9882862720839817
        },
        "abstained": false,
        "unknown_probability": 0.004494704730573657,
        "input_tokens": 348,
        "ms": 756
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "not_toxic",
        "correct": true,
        "confidence": 0.98,
        "probabilities": {
          "not_toxic": 0.99,
          "toxic": 0.01
        },
        "abstained": false,
        "input_tokens": 570,
        "ms": 207
      }
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "source": "TrustAIRLab/in-the-wild-jailbreak-prompts",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "jailbreak",
        "correct": true,
        "confidence": 0.96,
        "probabilities": {
          "injection": 0.03,
          "benign": 0,
          "jailbreak": 0.97
        },
        "abstained": false,
        "input_tokens": 777,
        "ms": 226
      },
      "local": {
        "model": "imajev-4b",
        "label": "jailbreak",
        "correct": true,
        "confidence": 0.8015579400703039,
        "probabilities": {
          "jailbreak": 0.8689161868284402,
          "injection": 0.12355703163068389,
          "benign": 0.007526781540875961
        },
        "abstained": false,
        "unknown_probability": 0.0022608891235697893,
        "input_tokens": 551,
        "ms": 1145
      }
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "source": "CFPB Consumer Complaint Database",
      "local": {
        "model": "imajev-4b",
        "label": "money_transfer",
        "correct": true,
        "confidence": 0.9740271855124178,
        "probabilities": {
          "student_loan": 0.0005058043843912817,
          "debt_collection": 0.0011106251723006905,
          "bank_account": 0.005770324148635935,
          "mortgage": 0.0006697089094081454,
          "credit_card": 0.0014365622318640355,
          "credit_reporting": 0.001035731549808239,
          "money_transfer": 0.9894712436035917
        },
        "abstained": false,
        "unknown_probability": 0.013859509322102993,
        "input_tokens": 552,
        "ms": 1146
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "money_transfer",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "bank_account": 0,
          "debt_collection": 0,
          "money_transfer": 1,
          "credit_reporting": 0,
          "credit_card": 0,
          "mortgage": 0,
          "student_loan": 0
        },
        "abstained": false,
        "input_tokens": 799,
        "ms": 159
      }
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "source": "CFPB Consumer Complaint Database",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "bank_account",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "credit_reporting": 0,
          "credit_card": 0,
          "debt_collection": 0,
          "bank_account": 1,
          "mortgage": 0,
          "money_transfer": 0,
          "student_loan": 0
        },
        "abstained": false,
        "input_tokens": 741,
        "ms": 181
      },
      "local": {
        "model": "imajev-4b",
        "label": "bank_account",
        "correct": true,
        "confidence": 0.9202666075067962,
        "probabilities": {
          "student_loan": 0.001312626399841494,
          "credit_card": 0.040346899852291326,
          "credit_reporting": 0.008036587491442716,
          "bank_account": 0.9447494642093499,
          "money_transfer": 0.002227404904520716,
          "debt_collection": 0.0015713521374282216,
          "mortgage": 0.0017556650051255185
        },
        "abstained": false,
        "unknown_probability": 0.01632684552782613,
        "input_tokens": 496,
        "ms": 1037
      }
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "source": "Amazon Berkeley Objects (ABO)",
      "local": {
        "model": "imajev-4b",
        "label": "sofa",
        "correct": true,
        "confidence": 0.1350923368184137,
        "probabilities": {
          "wall_art": 0.009746207442312884,
          "grocery": 0.008410928502078061,
          "chair": 0.02696705348883135,
          "lamp": 0.01113329600056534,
          "rug": 0.011879535369364601,
          "table": 0.010327913882960178,
          "shoes": 0.007198707945713215,
          "sofa": 0.9143363573681744
        },
        "abstained": true,
        "unknown_probability": 0.8502466108235065,
        "input_tokens": 634,
        "ms": 1258
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "sofa",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "rug": 0,
          "lamp": 0,
          "table": 0,
          "wall_art": 0,
          "shoes": 0,
          "sofa": 1,
          "grocery": 0,
          "chair": 0
        },
        "abstained": false,
        "input_tokens": 912,
        "ms": 176
      }
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "source": "Amazon Berkeley Objects (ABO)",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "sofa",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "grocery": 0,
          "lamp": 0,
          "shoes": 0,
          "table": 0,
          "sofa": 1,
          "rug": 0,
          "wall_art": 0,
          "chair": 0
        },
        "abstained": false,
        "input_tokens": 883,
        "ms": 169
      },
      "local": {
        "model": "imajev-4b",
        "label": "sofa",
        "correct": true,
        "confidence": 0.06404603508413775,
        "probabilities": {
          "rug": 0.010715783148590362,
          "lamp": 0.008544132536229833,
          "sofa": 0.9196835091956325,
          "chair": 0.02683890712333545,
          "grocery": 0.008716342562525502,
          "wall_art": 0.007512662928143676,
          "shoes": 0.007201582134461498,
          "table": 0.010787080371081283
        },
        "abstained": true,
        "unknown_probability": 0.9294810071554854,
        "input_tokens": 595,
        "ms": 1322
      }
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "source": "EDGAR-CORPUS",
      "local": {
        "model": "imajev-4b",
        "label": "business",
        "correct": true,
        "confidence": 0.9047944262239255,
        "probabilities": {
          "risk_factors": 0.002701002626846909,
          "controls": 0.003020732183345271,
          "legal_proceedings": 0.0021368449851778175,
          "business": 0.9315559636133808,
          "mdna": 0.0605854565912492
        },
        "abstained": false,
        "unknown_probability": 0.010553427240353252,
        "input_tokens": 592,
        "ms": 1223
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "business",
        "correct": true,
        "confidence": 0.99,
        "probabilities": {
          "business": 1,
          "controls": 0,
          "mdna": 0,
          "legal_proceedings": 0,
          "risk_factors": 0
        },
        "abstained": false,
        "input_tokens": 835,
        "ms": 254
      }
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "source": "FiNER-139",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "share_based_compensation",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "amortization": 0,
          "share_based_compensation": 1,
          "borrowings": 0,
          "revenue": 0,
          "income_tax_expense": 0,
          "goodwill": 0
        },
        "abstained": false,
        "input_tokens": 853,
        "ms": 190
      },
      "local": {
        "model": "imajev-4b",
        "label": "share_based_compensation",
        "correct": true,
        "confidence": 0.11619535998838353,
        "probabilities": {
          "revenue": 0.009373260285917291,
          "income_tax_expense": 0.012263766987349978,
          "borrowings": 0.01806761162387326,
          "amortization": 0.012886040509398821,
          "share_based_compensation": 0.9347740341493214,
          "goodwill": 0.012635286444139394
        },
        "abstained": true,
        "unknown_probability": 0.8739375889931522,
        "input_tokens": 588,
        "ms": 1321
      }
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "source": "CUAD (Contract Understanding Atticus Dataset) v1",
      "local": {
        "model": "imajev-4b",
        "label": "termination_for_convenience",
        "correct": true,
        "confidence": 0.9515247739335619,
        "probabilities": {
          "governing_law": 0.0010684057447115324,
          "termination_for_convenience": 0.9952307944637334,
          "anti_assignment": 0.001071325178976455,
          "non_compete": 0.0008765167759376863,
          "license_grant": 0.0007525666137827107,
          "cap_on_liability": 0.0010003912228581735
        },
        "abstained": false,
        "unknown_probability": 0.04299826047319659,
        "input_tokens": 604,
        "ms": 1252
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "termination_for_convenience",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "termination_for_convenience": 1,
          "cap_on_liability": 0,
          "non_compete": 0,
          "governing_law": 0,
          "anti_assignment": 0,
          "license_grant": 0
        },
        "abstained": false,
        "input_tokens": 858,
        "ms": 204
      }
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "source": "UNFAIR-ToS (LexGLUE)",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "limitation_of_liability",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "arbitration": 0,
          "not_unfair": 0,
          "unilateral_change": 0,
          "choice_of_law": 0,
          "contract_by_using": 0,
          "jurisdiction": 0,
          "unilateral_termination": 0,
          "limitation_of_liability": 1,
          "content_removal": 0
        },
        "abstained": false,
        "input_tokens": 851,
        "ms": 223
      },
      "local": {
        "model": "imajev-4b",
        "label": "limitation_of_liability",
        "correct": true,
        "confidence": 0.9045368463585932,
        "probabilities": {
          "unilateral_termination": 0.0018411431357306077,
          "limitation_of_liability": 0.9162918859007833,
          "choice_of_law": 0.001842772026741652,
          "not_unfair": 0.027456202888389923,
          "contract_by_using": 0.0014280514186018134,
          "content_removal": 0.04759343400661738,
          "arbitration": 0.0013880797009304634,
          "unilateral_change": 0.0012756773283236023,
          "jurisdiction": 0.0008827535938813379
        },
        "abstained": false,
        "unknown_probability": 0.001425794687190113,
        "input_tokens": 569,
        "ms": 1177
      }
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "source": "Upworthy Research Archive (exploratory packages)",
      "local": {
        "model": "imajev-4b",
        "label": "A",
        "correct": true,
        "confidence": 0.0021703522382150496,
        "probabilities": {
          "B": 0.3602827903280743,
          "A": 0.6397172096719258
        },
        "abstained": true,
        "unknown_probability": 0.9922330533106433,
        "input_tokens": 511,
        "ms": 1011
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "B",
        "correct": false,
        "confidence": 0.14,
        "probabilities": {
          "B": 0.57,
          "A": 0.43
        },
        "abstained": false,
        "input_tokens": 743,
        "ms": 178
      }
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "source": "ramsey/uuid CHANGELOG.md",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "minor",
        "correct": true,
        "confidence": 0.62,
        "probabilities": {
          "major": 0.2,
          "patch": 0.05,
          "minor": 0.75
        },
        "abstained": false,
        "input_tokens": 664,
        "ms": 208
      },
      "local": {
        "model": "imajev-4b",
        "label": "minor",
        "correct": true,
        "confidence": 0.3716536045101389,
        "probabilities": {
          "minor": 0.5818775174933603,
          "major": 0.08419835805593737,
          "patch": 0.3339241244507024
        },
        "abstained": false,
        "unknown_probability": 0.0031186184831505826,
        "input_tokens": 434,
        "ms": 930
      }
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "source": "FiveThirtyEight data repository",
      "local": {
        "model": "imajev-4b",
        "label": "money",
        "correct": true,
        "confidence": 0.6464969044881348,
        "probabilities": {
          "category": 0.013996135864342747,
          "money": 0.8622407721783174,
          "identifier": 0.02914259067547233,
          "count_or_measurement": 0.05424148767281725,
          "date": 0.015293125501785843,
          "geography": 0.016013868645414084,
          "free_text": 0.00907201946185025
        },
        "abstained": false,
        "unknown_probability": 0.22970139816794136,
        "input_tokens": 734,
        "ms": 1541
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "money",
        "correct": true,
        "confidence": 0.99,
        "probabilities": {
          "money": 0.99,
          "count_or_measurement": 0.01,
          "geography": 0,
          "free_text": 0,
          "category": 0,
          "identifier": 0,
          "date": 0
        },
        "abstained": false,
        "input_tokens": 1027,
        "ms": 224
      }
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "source": "U.S. Treasury Fiscal Data",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "money",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "geography": 0,
          "free_text": 0,
          "category": 0,
          "money": 1,
          "date": 0,
          "count_or_measurement": 0,
          "identifier": 0
        },
        "abstained": false,
        "input_tokens": 1154,
        "ms": 233
      },
      "local": {
        "model": "imajev-4b",
        "label": "money",
        "correct": true,
        "confidence": 0.7856531346599569,
        "probabilities": {
          "money": 0.8784012616808771,
          "geography": 0.009935825582250498,
          "category": 0.008213070671195371,
          "identifier": 0.015306256178076984,
          "free_text": 0.005520595391824716,
          "count_or_measurement": 0.05360601735017508,
          "date": 0.029016973145600308
        },
        "abstained": false,
        "unknown_probability": 0.084464200947469,
        "input_tokens": 870,
        "ms": 1822
      }
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "source": "AMI Meeting Corpus",
      "local": {
        "model": "imajev-4b",
        "label": "neither",
        "correct": false,
        "confidence": 0.18953492964041602,
        "probabilities": {
          "action_item": 0.18225287323700584,
          "neither": 0.4603537987859381,
          "decision": 0.35739332797705614
        },
        "abstained": false,
        "unknown_probability": 0.005226289243721004,
        "input_tokens": 646,
        "ms": 1325
      },
      "cloud": {
        "model": "jev-1.13.0",
        "label": "action_item",
        "correct": true,
        "confidence": 0.49,
        "probabilities": {
          "action_item": 0.66,
          "neither": 0.21,
          "decision": 0.13
        },
        "abstained": false,
        "input_tokens": 955,
        "ms": 211
      }
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "source": "AMI Meeting Corpus",
      "cloud": {
        "model": "jev-1.13.0",
        "label": "neither",
        "correct": true,
        "confidence": 1,
        "probabilities": {
          "decision": 0,
          "action_item": 0,
          "neither": 1
        },
        "abstained": false,
        "input_tokens": 948,
        "ms": 312
      },
      "local": {
        "model": "imajev-4b",
        "label": "neither",
        "correct": true,
        "confidence": 0.9502344433097549,
        "probabilities": {
          "decision": 0.017928831634072728,
          "action_item": 0.01379816757261834,
          "neither": 0.968273000793309
        },
        "abstained": false,
        "unknown_probability": 0.00228374231618965,
        "input_tokens": 638,
        "ms": 1291
      }
    }
  ]
}