{
  "model": "plumb-mlx",
  "base": "CogniSoft/Plumb-4B-MLX-8bit@b80e4b55",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "d65f5e07",
      "correct": false,
      "same_as_bf16": false,
      "probabilities": {
        "f1ded018": 0.00457264157012105,
        "d65f5e07": 0.507875382900238,
        "fa3bada5": 0.009437858127057552,
        "36db20dd": 0.47811412811279297
      },
      "ms": 10033,
      "tokens": 1124
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "feat",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "refactor": 0.060172080993652344,
        "docs": 0.02289743162691593,
        "fix": 0.03494342789053917,
        "test": 0.024322733283042908,
        "feat": 0.8576643466949463
      },
      "ms": 799,
      "tokens": 469
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "same_as_bf16": true,
      "probabilities": {
        "not_supported": 0.06935641914606094,
        "supported": 0.9306435585021973
      },
      "ms": 1021,
      "tokens": 639
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "none": 0.07044370472431183,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.8898437023162842,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.021053871139883995,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.01865869201719761
      },
      "ms": 864,
      "tokens": 518
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "not_toxic",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "toxic": 0.0866633877158165,
        "not_toxic": 0.9133366346359253
      },
      "ms": 569,
      "tokens": 344
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "jailbreak",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "jailbreak": 0.8199835419654846,
        "injection": 0.1423196941614151,
        "benign": 0.037696726620197296
      },
      "ms": 912,
      "tokens": 557
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "student_loan": 0.006060988642275333,
        "debt_collection": 0.006438261829316616,
        "bank_account": 0.006438261829316616,
        "mortgage": 0.003971592523157597,
        "credit_card": 0.0035197639372199774,
        "credit_reporting": 0.006438261829316616,
        "money_transfer": 0.9671328663825989
      },
      "ms": 967,
      "tokens": 589
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "student_loan": 0.005359961651265621,
        "credit_card": 0.014962201938033104,
        "credit_reporting": 0.007249185815453529,
        "bank_account": 0.965063750743866,
        "money_transfer": 0.0031126635149121284,
        "debt_collection": 0.0018076006090268493,
        "mortgage": 0.0024447261821478605
      },
      "ms": 864,
      "tokens": 533
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "wall_art": 0.01568157784640789,
        "grocery": 0.010275701060891151,
        "chair": 0.046499595046043396,
        "lamp": 0.011594772338867188,
        "rug": 0.006338801234960556,
        "table": 0.007597730960696936,
        "shoes": 0.005617668852210045,
        "sofa": 0.8963942527770996
      },
      "ms": 1109,
      "tokens": 680
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "rug": 0.019954146817326546,
        "lamp": 0.013889303430914879,
        "sofa": 0.8958607316017151,
        "chair": 0.041185036301612854,
        "grocery": 0.0071482472121715546,
        "wall_art": 0.0071482472121715546,
        "shoes": 0.004544731229543686,
        "table": 0.010269580408930779
      },
      "ms": 1060,
      "tokens": 641
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "risk_factors": 0.0253329798579216,
        "controls": 0.021135371178388596,
        "legal_proceedings": 0.023848501965403557,
        "business": 0.8932883143424988,
        "mdna": 0.03639480844140053
      },
      "ms": 1007,
      "tokens": 611
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "revenue": 0.00867258757352829,
        "income_tax_expense": 0.00867258757352829,
        "borrowings": 0.0076859574764966965,
        "amortization": 0.006036644335836172,
        "share_based_compensation": 0.9632493257522583,
        "goodwill": 0.005682900082319975
      },
      "ms": 1008,
      "tokens": 618
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "governing_law": 0.005039137788116932,
        "termination_for_convenience": 0.963775098323822,
        "anti_assignment": 0.011735815554857254,
        "non_compete": 0.006815281230956316,
        "license_grant": 0.00446585938334465,
        "cap_on_liability": 0.008168835192918777
      },
      "ms": 1011,
      "tokens": 633
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "not_unfair",
      "correct": false,
      "same_as_bf16": false,
      "probabilities": {
        "unilateral_termination": 0.01593012735247612,
        "limitation_of_liability": 0.2270604968070984,
        "choice_of_law": 0.01797504723072052,
        "not_unfair": 0.6338338255882263,
        "contract_by_using": 0.014117834158241749,
        "content_removal": 0.04446856305003166,
        "arbitration": 0.011778546497225761,
        "unilateral_change": 0.013290535658597946,
        "jurisdiction": 0.021545015275478363
      },
      "ms": 1010,
      "tokens": 626
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "B": 0.48490816354751587,
        "A": 0.5150918364524841
      },
      "ms": 818,
      "tokens": 506
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "minor": 0.8630980849266052,
        "major": 0.07257946580648422,
        "patch": 0.06432250887155533
      },
      "ms": 712,
      "tokens": 437
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "category": 0.01704101637005806,
        "money": 0.7202407717704773,
        "identifier": 0.07259523123502731,
        "count_or_measurement": 0.14105509221553802,
        "date": 0.020425481721758842,
        "geography": 0.01604243740439415,
        "free_text": 0.012599925510585308
      },
      "ms": 1257,
      "tokens": 772
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "money": 0.7694876194000244,
        "geography": 0.026156103238463402,
        "category": 0.03135085850954056,
        "identifier": 0.06470765918493271,
        "free_text": 0.013461465947329998,
        "count_or_measurement": 0.07301412522792816,
        "date": 0.021822107955813408
      },
      "ms": 1452,
      "tokens": 908
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "action_item",
      "correct": true,
      "same_as_bf16": false,
      "probabilities": {
        "action_item": 0.39649099111557007,
        "neither": 0.3732568025588989,
        "decision": 0.230252206325531
      },
      "ms": 1056,
      "tokens": 650
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "same_as_bf16": true,
      "probabilities": {
        "decision": 0.01789713464677334,
        "action_item": 0.019011182710528374,
        "neither": 0.9630917310714722
      },
      "ms": 1056,
      "tokens": 642
    }
  ]
}