{
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "same_decisions_across_runtimes": true,
  "variants": {
    "cpu": {
      "correct": 14,
      "valid": 20,
      "median_ms": 4816,
      "p95_ms": 7420,
      "total_ms": 104955,
      "max_rss_mib": 1594,
      "rows": [
        {
          "id": "eng-5-36db20dd946e",
          "label": "36db20dd",
          "confidence": 0.4069139063358307,
          "ms": 13826,
          "rss_mib": 1380,
          "gold": "36db20dd",
          "correct": true,
          "earlier_local": "36db20dd"
        },
        {
          "id": "eng-2-3f22797c158b",
          "label": "fix",
          "confidence": 0.5403599143028259,
          "ms": 3325,
          "rss_mib": 1566,
          "gold": "feat",
          "correct": false,
          "earlier_local": "feat"
        },
        {
          "id": "agt-4-hagrid-dev-983-0",
          "label": "supported",
          "confidence": 0.7246087193489075,
          "ms": 4327,
          "rss_mib": 1431,
          "gold": "not_supported",
          "correct": false,
          "earlier_local": "supported"
        },
        {
          "id": "agt-1-live-multiple-258-122-0",
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "confidence": 0.6174183487892151,
          "ms": 3915,
          "rss_mib": 770,
          "gold": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "earlier_local": "project_api.ProjectApi.get_project_by_name_and_version"
        },
        {
          "id": "saf-2-civil-1620015",
          "label": "toxic",
          "confidence": 0.5169689059257507,
          "ms": 2537,
          "rss_mib": 1390,
          "gold": "not_toxic",
          "correct": false,
          "earlier_local": "not_toxic"
        },
        {
          "id": "saf-1-itw-jailbreak-2023-12-25-1216",
          "label": "jailbreak",
          "confidence": 0.9061570763587952,
          "ms": 4341,
          "rss_mib": 1358,
          "gold": "jailbreak",
          "correct": true,
          "earlier_local": "jailbreak"
        },
        {
          "id": "sup-1-cfpb-2791756",
          "label": "money_transfer",
          "confidence": 0.3764740824699402,
          "ms": 4471,
          "rss_mib": 805,
          "gold": "money_transfer",
          "correct": true,
          "earlier_local": "money_transfer"
        },
        {
          "id": "sup-1-cfpb-1680253",
          "label": "bank_account",
          "confidence": 0.8182962536811829,
          "ms": 4764,
          "rss_mib": 1421,
          "gold": "bank_account",
          "correct": true,
          "earlier_local": "bank_account"
        },
        {
          "id": "com-2-abo-b07bw8q16k",
          "label": "sofa",
          "confidence": 0.948901355266571,
          "ms": 5196,
          "rss_mib": 1452,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "com-2-abo-b072m1wjc7",
          "label": "sofa",
          "confidence": 0.9570420980453491,
          "ms": 4868,
          "rss_mib": 1058,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "fin-1-edgar-1042642-2017",
          "label": "business",
          "confidence": 0.7096704244613647,
          "ms": 5013,
          "rss_mib": 1453,
          "gold": "business",
          "correct": true,
          "earlier_local": "business"
        },
        {
          "id": "fin-4-finer-test-1013526",
          "label": "share_based_compensation",
          "confidence": 0.9953267574310303,
          "ms": 4221,
          "rss_mib": 1459,
          "gold": "share_based_compensation",
          "correct": true,
          "earlier_local": "share_based_compensation"
        },
        {
          "id": "leg-2-cuad-95-termination-for-convenience",
          "label": "termination_for_convenience",
          "confidence": 0.949510395526886,
          "ms": 4962,
          "rss_mib": 640,
          "gold": "termination_for_convenience",
          "correct": true,
          "earlier_local": "termination_for_convenience"
        },
        {
          "id": "leg-3-unfair-tos-test-237",
          "label": "limitation_of_liability",
          "confidence": 0.46820035576820374,
          "ms": 4979,
          "rss_mib": 1379,
          "gold": "limitation_of_liability",
          "correct": true,
          "earlier_local": "limitation_of_liability"
        },
        {
          "id": "prd-1-53f253aaf1592f712e000017",
          "label": "B",
          "confidence": 0.5434107780456543,
          "ms": 3323,
          "rss_mib": 1435,
          "gold": "A",
          "correct": false,
          "earlier_local": "A"
        },
        {
          "id": "prd-2-ramsey-uuid-2.4.0",
          "label": "major",
          "confidence": 0.43923112750053406,
          "ms": 3488,
          "rss_mib": 1350,
          "gold": "minor",
          "correct": false,
          "earlier_local": "minor"
        },
        {
          "id": "dat-4-bechdel-6",
          "label": "money",
          "confidence": 0.6717451214790344,
          "ms": 6254,
          "rss_mib": 768,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "dat-4-debt-1",
          "label": "money",
          "confidence": 0.6650392413139343,
          "ms": 7420,
          "rss_mib": 1583,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
          "label": "neither",
          "confidence": 0.5588849186897278,
          "ms": 6803,
          "rss_mib": 1592,
          "gold": "action_item",
          "correct": false,
          "earlier_local": "neither"
        },
        {
          "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
          "label": "neither",
          "confidence": 0.503093957901001,
          "ms": 6922,
          "rss_mib": 1594,
          "gold": "neither",
          "correct": true,
          "earlier_local": "neither"
        }
      ]
    },
    "mlx": {
      "correct": 14,
      "valid": 20,
      "median_ms": 272,
      "p95_ms": 698,
      "total_ms": 7373,
      "max_rss_mib": 2081,
      "active_mib_last": 1856,
      "peak_mlx_mib": 3599,
      "rows": [
        {
          "id": "eng-5-36db20dd946e",
          "label": "36db20dd",
          "confidence": 0.4069146513938904,
          "ms": 1688,
          "rss_mib": 2063,
          "active_mib": 1856,
          "peak_mib": 3109,
          "gold": "36db20dd",
          "correct": true,
          "earlier_local": "36db20dd"
        },
        {
          "id": "eng-2-3f22797c158b",
          "label": "fix",
          "confidence": 0.5403594374656677,
          "ms": 182,
          "rss_mib": 2068,
          "active_mib": 1856,
          "peak_mib": 3109,
          "gold": "feat",
          "correct": false,
          "earlier_local": "feat"
        },
        {
          "id": "agt-4-hagrid-dev-983-0",
          "label": "supported",
          "confidence": 0.7246100306510925,
          "ms": 316,
          "rss_mib": 2073,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "not_supported",
          "correct": false,
          "earlier_local": "supported"
        },
        {
          "id": "agt-1-live-multiple-258-122-0",
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "confidence": 0.6174189448356628,
          "ms": 272,
          "rss_mib": 2075,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "earlier_local": "project_api.ProjectApi.get_project_by_name_and_version"
        },
        {
          "id": "saf-2-civil-1620015",
          "label": "toxic",
          "confidence": 0.5169726014137268,
          "ms": 103,
          "rss_mib": 2077,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "not_toxic",
          "correct": false,
          "earlier_local": "not_toxic"
        },
        {
          "id": "saf-1-itw-jailbreak-2023-12-25-1216",
          "label": "jailbreak",
          "confidence": 0.9061563014984131,
          "ms": 184,
          "rss_mib": 2081,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "jailbreak",
          "correct": true,
          "earlier_local": "jailbreak"
        },
        {
          "id": "sup-1-cfpb-2791756",
          "label": "money_transfer",
          "confidence": 0.37647297978401184,
          "ms": 165,
          "rss_mib": 2014,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "money_transfer",
          "correct": true,
          "earlier_local": "money_transfer"
        },
        {
          "id": "sup-1-cfpb-1680253",
          "label": "bank_account",
          "confidence": 0.8182961344718933,
          "ms": 183,
          "rss_mib": 2015,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "bank_account",
          "correct": true,
          "earlier_local": "bank_account"
        },
        {
          "id": "com-2-abo-b07bw8q16k",
          "label": "sofa",
          "confidence": 0.9489010572433472,
          "ms": 698,
          "rss_mib": 2018,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "com-2-abo-b072m1wjc7",
          "label": "sofa",
          "confidence": 0.95704185962677,
          "ms": 284,
          "rss_mib": 2018,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "fin-1-edgar-1042642-2017",
          "label": "business",
          "confidence": 0.7096716165542603,
          "ms": 260,
          "rss_mib": 2020,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "business",
          "correct": true,
          "earlier_local": "business"
        },
        {
          "id": "fin-4-finer-test-1013526",
          "label": "share_based_compensation",
          "confidence": 0.9953267574310303,
          "ms": 251,
          "rss_mib": 2022,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "share_based_compensation",
          "correct": true,
          "earlier_local": "share_based_compensation"
        },
        {
          "id": "leg-2-cuad-95-termination-for-convenience",
          "label": "termination_for_convenience",
          "confidence": 0.9495108127593994,
          "ms": 321,
          "rss_mib": 2024,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "termination_for_convenience",
          "correct": true,
          "earlier_local": "termination_for_convenience"
        },
        {
          "id": "leg-3-unfair-tos-test-237",
          "label": "limitation_of_liability",
          "confidence": 0.4681995213031769,
          "ms": 272,
          "rss_mib": 2025,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "limitation_of_liability",
          "correct": true,
          "earlier_local": "limitation_of_liability"
        },
        {
          "id": "prd-1-53f253aaf1592f712e000017",
          "label": "B",
          "confidence": 0.5434123873710632,
          "ms": 258,
          "rss_mib": 2025,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "A",
          "correct": false,
          "earlier_local": "A"
        },
        {
          "id": "prd-2-ramsey-uuid-2.4.0",
          "label": "major",
          "confidence": 0.4392293691635132,
          "ms": 179,
          "rss_mib": 2026,
          "active_mib": 1856,
          "peak_mib": 3240,
          "gold": "minor",
          "correct": false,
          "earlier_local": "minor"
        },
        {
          "id": "dat-4-bechdel-6",
          "label": "money",
          "confidence": 0.6717460751533508,
          "ms": 400,
          "rss_mib": 2027,
          "active_mib": 1856,
          "peak_mib": 3304,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "dat-4-debt-1",
          "label": "money",
          "confidence": 0.6650403141975403,
          "ms": 564,
          "rss_mib": 2027,
          "active_mib": 1856,
          "peak_mib": 3599,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
          "label": "neither",
          "confidence": 0.5588817000389099,
          "ms": 396,
          "rss_mib": 2027,
          "active_mib": 1856,
          "peak_mib": 3599,
          "gold": "action_item",
          "correct": false,
          "earlier_local": "neither"
        },
        {
          "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
          "label": "neither",
          "confidence": 0.5030925869941711,
          "ms": 397,
          "rss_mib": 2028,
          "active_mib": 1856,
          "peak_mib": 3599,
          "gold": "neither",
          "correct": true,
          "earlier_local": "neither"
        }
      ]
    },
    "mlx8": {
      "correct": 14,
      "valid": 20,
      "median_ms": 227,
      "p95_ms": 425,
      "total_ms": 5214,
      "max_rss_mib": 373,
      "active_mib_last": 668,
      "peak_mlx_mib": 2328,
      "rows": [
        {
          "id": "eng-5-36db20dd946e",
          "label": "36db20dd",
          "confidence": 0.41320520639419556,
          "ms": 770,
          "rss_mib": 341,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "36db20dd",
          "correct": true,
          "earlier_local": "36db20dd"
        },
        {
          "id": "eng-2-3f22797c158b",
          "label": "fix",
          "confidence": 0.5390995144844055,
          "ms": 189,
          "rss_mib": 345,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "feat",
          "correct": false,
          "earlier_local": "feat"
        },
        {
          "id": "agt-4-hagrid-dev-983-0",
          "label": "supported",
          "confidence": 0.7262895107269287,
          "ms": 234,
          "rss_mib": 351,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "not_supported",
          "correct": false,
          "earlier_local": "supported"
        },
        {
          "id": "agt-1-live-multiple-258-122-0",
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "confidence": 0.6167898774147034,
          "ms": 235,
          "rss_mib": 352,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "earlier_local": "project_api.ProjectApi.get_project_by_name_and_version"
        },
        {
          "id": "saf-2-civil-1620015",
          "label": "toxic",
          "confidence": 0.512905478477478,
          "ms": 96,
          "rss_mib": 354,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "not_toxic",
          "correct": false,
          "earlier_local": "not_toxic"
        },
        {
          "id": "saf-1-itw-jailbreak-2023-12-25-1216",
          "label": "jailbreak",
          "confidence": 0.9054470062255859,
          "ms": 196,
          "rss_mib": 357,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "jailbreak",
          "correct": true,
          "earlier_local": "jailbreak"
        },
        {
          "id": "sup-1-cfpb-2791756",
          "label": "money_transfer",
          "confidence": 0.3763797879219055,
          "ms": 192,
          "rss_mib": 361,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "money_transfer",
          "correct": true,
          "earlier_local": "money_transfer"
        },
        {
          "id": "sup-1-cfpb-1680253",
          "label": "bank_account",
          "confidence": 0.8163550496101379,
          "ms": 153,
          "rss_mib": 361,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "bank_account",
          "correct": true,
          "earlier_local": "bank_account"
        },
        {
          "id": "com-2-abo-b07bw8q16k",
          "label": "sofa",
          "confidence": 0.9473903179168701,
          "ms": 250,
          "rss_mib": 363,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "com-2-abo-b072m1wjc7",
          "label": "sofa",
          "confidence": 0.9558225274085999,
          "ms": 228,
          "rss_mib": 363,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "sofa",
          "correct": true,
          "earlier_local": "sofa"
        },
        {
          "id": "fin-1-edgar-1042642-2017",
          "label": "business",
          "confidence": 0.7126118540763855,
          "ms": 226,
          "rss_mib": 366,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "business",
          "correct": true,
          "earlier_local": "business"
        },
        {
          "id": "fin-4-finer-test-1013526",
          "label": "share_based_compensation",
          "confidence": 0.9953306913375854,
          "ms": 214,
          "rss_mib": 368,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "share_based_compensation",
          "correct": true,
          "earlier_local": "share_based_compensation"
        },
        {
          "id": "leg-2-cuad-95-termination-for-convenience",
          "label": "termination_for_convenience",
          "confidence": 0.9496278762817383,
          "ms": 270,
          "rss_mib": 369,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "termination_for_convenience",
          "correct": true,
          "earlier_local": "termination_for_convenience"
        },
        {
          "id": "leg-3-unfair-tos-test-237",
          "label": "limitation_of_liability",
          "confidence": 0.46618595719337463,
          "ms": 218,
          "rss_mib": 370,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "limitation_of_liability",
          "correct": true,
          "earlier_local": "limitation_of_liability"
        },
        {
          "id": "prd-1-53f253aaf1592f712e000017",
          "label": "B",
          "confidence": 0.544985830783844,
          "ms": 189,
          "rss_mib": 370,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "A",
          "correct": false,
          "earlier_local": "A"
        },
        {
          "id": "prd-2-ramsey-uuid-2.4.0",
          "label": "major",
          "confidence": 0.44109681248664856,
          "ms": 140,
          "rss_mib": 371,
          "active_mib": 668,
          "peak_mib": 2320,
          "gold": "minor",
          "correct": false,
          "earlier_local": "minor"
        },
        {
          "id": "dat-4-bechdel-6",
          "label": "money",
          "confidence": 0.6669921875,
          "ms": 295,
          "rss_mib": 372,
          "active_mib": 672,
          "peak_mib": 2320,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "dat-4-debt-1",
          "label": "money",
          "confidence": 0.6616989970207214,
          "ms": 425,
          "rss_mib": 372,
          "active_mib": 668,
          "peak_mib": 2328,
          "gold": "money",
          "correct": true,
          "earlier_local": "money"
        },
        {
          "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
          "label": "neither",
          "confidence": 0.5532903671264648,
          "ms": 370,
          "rss_mib": 372,
          "active_mib": 668,
          "peak_mib": 2328,
          "gold": "action_item",
          "correct": false,
          "earlier_local": "neither"
        },
        {
          "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
          "label": "neither",
          "confidence": 0.5000563263893127,
          "ms": 324,
          "rss_mib": 373,
          "active_mib": 668,
          "peak_mib": 2328,
          "gold": "neither",
          "correct": true,
          "earlier_local": "neither"
        }
      ]
    }
  }
}