{
  "title": "Comparing local decision models",
  "date": "2026-09-29",
  "source": {
    "dataset": "Decision Bench bench-v4",
    "url": "https://decisionbench.ai",
    "corpusSha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
    "sampleSha256": "b89a790bd832223630fd2d757e38e6f24856002e757982be189534d00a5b2785",
    "machine": "M3 Pro MacBook Pro, 36 GiB unified memory",
    "size": 20
  },
  "methods": [
    {
      "id": "jev",
      "name": "Jev Cloud 1.13",
      "remote": true,
      "images": false,
      "correct": 18,
      "valid": 20,
      "medianMs": 209.5,
      "p95Ms": 312,
      "memoryGiB": null,
      "memoryMethod": null,
      "coldSeconds": null,
      "raw": "raw/results.json"
    },
    {
      "id": "imajev-bf16",
      "name": "Imajev 4B BF16",
      "remote": false,
      "images": true,
      "correct": 18,
      "valid": 20,
      "medianMs": 1237.5,
      "p95Ms": 1822,
      "memoryGiB": 9.9,
      "memoryMethod": "MLX process physical footprint after run",
      "coldSeconds": null,
      "raw": "raw/results.json"
    },
    {
      "id": "imajev-4bit",
      "name": "Imajev 4B 4-bit",
      "remote": false,
      "images": true,
      "correct": 17,
      "valid": 20,
      "medianMs": 1256.5,
      "p95Ms": 1838,
      "memoryGiB": 4.4,
      "memoryMethod": "MLX process loaded footprint after run",
      "coldSeconds": null,
      "raw": "raw/results-4bit.json"
    },
    {
      "id": "imajev-8bit",
      "name": "Imajev 4B 8-bit",
      "remote": false,
      "images": true,
      "correct": 18,
      "valid": 20,
      "medianMs": 1247,
      "p95Ms": 2022,
      "memoryGiB": 6.1,
      "memoryMethod": "MLX process loaded footprint after run",
      "coldSeconds": null,
      "raw": "raw/results-8bit.json"
    },
    {
      "id": "plumb",
      "name": "Plumb 4B 8-bit",
      "remote": false,
      "images": false,
      "correct": 17,
      "valid": 20,
      "medianMs": 1009,
      "p95Ms": 1452,
      "memoryGiB": 5.1,
      "memoryMethod": "MLX process loaded footprint after run",
      "coldSeconds": null,
      "raw": "raw/results-plumb.json"
    },
    {
      "id": "gliner-cpu",
      "name": "GLiNER2.5 CPU",
      "remote": false,
      "images": false,
      "correct": 14,
      "valid": 20,
      "medianMs": 4816,
      "p95Ms": 7420,
      "memoryGiB": 1.556640625,
      "memoryMethod": "Peak sampled process RSS",
      "coldSeconds": 25,
      "raw": "raw/results-gliner.json"
    },
    {
      "id": "gliner-mlx",
      "name": "GLiNER2.5 MLX",
      "remote": false,
      "images": false,
      "correct": 14,
      "valid": 20,
      "medianMs": 272,
      "p95Ms": 698,
      "memoryGiB": 3.5146484375,
      "memoryMethod": "Peak MLX allocation, excludes other process memory",
      "coldSeconds": 25,
      "raw": "raw/results-gliner.json"
    },
    {
      "id": "gliner-int8",
      "name": "GLiNER2.5 MLX int8",
      "remote": false,
      "images": false,
      "correct": 14,
      "valid": 20,
      "medianMs": 227,
      "p95Ms": 425,
      "memoryGiB": 2.2734375,
      "memoryMethod": "Peak MLX allocation, excludes other process memory",
      "coldSeconds": 25,
      "raw": "raw/results-gliner.json"
    },
    {
      "id": "tev-4b",
      "name": "Tev1 4B",
      "remote": false,
      "images": false,
      "correct": 17,
      "valid": 20,
      "medianMs": 1363,
      "p95Ms": 1861,
      "memoryGiB": 5.1,
      "memoryMethod": "Runner RSS after 20 requests",
      "coldSeconds": 93,
      "raw": "raw/results-tev1.json"
    },
    {
      "id": "tev-08b",
      "name": "Tev1 0.8B",
      "remote": false,
      "images": false,
      "correct": 16,
      "valid": 20,
      "medianMs": 219,
      "p95Ms": 303,
      "memoryGiB": 2.7,
      "memoryMethod": "Runner RSS after 20 requests",
      "coldSeconds": 0.9,
      "raw": "raw/results-tev1-0.8b.json"
    },
    {
      "id": "nimble",
      "name": "Nimble 9B Q8",
      "remote": false,
      "images": false,
      "correct": 19,
      "valid": 20,
      "medianMs": 1910,
      "p95Ms": 2762,
      "memoryGiB": 10.5,
      "memoryMethod": "Runner RSS after 20 requests",
      "coldSeconds": 2.6,
      "raw": "raw/results-nimble.json"
    }
  ],
  "cases": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "source": "CommitPackFT",
      "gold": "36db20dd",
      "predictions": {
        "jev": {
          "label": "36db20dd",
          "correct": true,
          "ms": 539
        },
        "imajev-bf16": {
          "label": "36db20dd",
          "correct": true,
          "ms": 2364
        },
        "imajev-4bit": {
          "label": "36db20dd",
          "correct": true,
          "ms": 5624
        },
        "imajev-8bit": {
          "label": "36db20dd",
          "correct": true,
          "ms": 11936
        },
        "plumb": {
          "label": "d65f5e07",
          "correct": false,
          "ms": 10033
        },
        "gliner-cpu": {
          "label": "36db20dd",
          "correct": true,
          "ms": 13826
        },
        "gliner-mlx": {
          "label": "36db20dd",
          "correct": true,
          "ms": 1688
        },
        "gliner-int8": {
          "label": "36db20dd",
          "correct": true,
          "ms": 770
        },
        "tev-4b": {
          "label": "d65f5e07",
          "correct": false,
          "ms": 3626
        },
        "tev-08b": {
          "label": "36db20dd",
          "correct": true,
          "ms": 649
        },
        "nimble": {
          "label": "36db20dd",
          "correct": true,
          "ms": 3925
        }
      }
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "source": "CommitPackFT",
      "gold": "feat",
      "predictions": {
        "jev": {
          "label": "feat",
          "correct": true,
          "ms": 214
        },
        "imajev-bf16": {
          "label": "feat",
          "correct": true,
          "ms": 917
        },
        "imajev-4bit": {
          "label": "refactor",
          "correct": false,
          "ms": 1001
        },
        "imajev-8bit": {
          "label": "feat",
          "correct": true,
          "ms": 1460
        },
        "plumb": {
          "label": "feat",
          "correct": true,
          "ms": 799
        },
        "gliner-cpu": {
          "label": "fix",
          "correct": false,
          "ms": 3325
        },
        "gliner-mlx": {
          "label": "fix",
          "correct": false,
          "ms": 182
        },
        "gliner-int8": {
          "label": "fix",
          "correct": false,
          "ms": 189
        },
        "tev-4b": {
          "label": "feat",
          "correct": true,
          "ms": 1009
        },
        "tev-08b": {
          "label": "feat",
          "correct": true,
          "ms": 170
        },
        "nimble": {
          "label": "feat",
          "correct": true,
          "ms": 1684
        }
      }
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "source": "HAGRID",
      "gold": "not_supported",
      "predictions": {
        "jev": {
          "label": "supported",
          "correct": false,
          "ms": 283
        },
        "imajev-bf16": {
          "label": "supported",
          "correct": false,
          "ms": 1337
        },
        "imajev-4bit": {
          "label": "supported",
          "correct": false,
          "ms": 1365
        },
        "imajev-8bit": {
          "label": "supported",
          "correct": false,
          "ms": 1425
        },
        "plumb": {
          "label": "supported",
          "correct": false,
          "ms": 1021
        },
        "gliner-cpu": {
          "label": "supported",
          "correct": false,
          "ms": 4327
        },
        "gliner-mlx": {
          "label": "supported",
          "correct": false,
          "ms": 316
        },
        "gliner-int8": {
          "label": "supported",
          "correct": false,
          "ms": 234
        },
        "tev-4b": {
          "label": "supported",
          "correct": false,
          "ms": 1283
        },
        "tev-08b": {
          "label": "supported",
          "correct": false,
          "ms": 228
        },
        "nimble": {
          "label": "supported",
          "correct": false,
          "ms": 2151
        }
      }
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "source": "BFCL v3 Live (Berkeley Function-Calling Leaderboard)",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "predictions": {
        "jev": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 190
        },
        "imajev-bf16": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 1062
        },
        "imajev-4bit": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 1054
        },
        "imajev-8bit": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 1077
        },
        "plumb": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 864
        },
        "gliner-cpu": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 3915
        },
        "gliner-mlx": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 272
        },
        "gliner-int8": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 235
        },
        "tev-4b": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 1156
        },
        "tev-08b": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 190
        },
        "nimble": {
          "label": "project_api.ProjectApi.get_project_by_name_and_version",
          "correct": true,
          "ms": 1794
        }
      }
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "source": "google/civil_comments",
      "gold": "not_toxic",
      "predictions": {
        "jev": {
          "label": "not_toxic",
          "correct": true,
          "ms": 207
        },
        "imajev-bf16": {
          "label": "not_toxic",
          "correct": true,
          "ms": 756
        },
        "imajev-4bit": {
          "label": "not_toxic",
          "correct": true,
          "ms": 803
        },
        "imajev-8bit": {
          "label": "not_toxic",
          "correct": true,
          "ms": 789
        },
        "plumb": {
          "label": "not_toxic",
          "correct": true,
          "ms": 569
        },
        "gliner-cpu": {
          "label": "toxic",
          "correct": false,
          "ms": 2537
        },
        "gliner-mlx": {
          "label": "toxic",
          "correct": false,
          "ms": 103
        },
        "gliner-int8": {
          "label": "toxic",
          "correct": false,
          "ms": 96
        },
        "tev-4b": {
          "label": "not_toxic",
          "correct": true,
          "ms": 774
        },
        "tev-08b": {
          "label": "toxic",
          "correct": false,
          "ms": 129
        },
        "nimble": {
          "label": "not_toxic",
          "correct": true,
          "ms": 1148
        }
      }
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "source": "TrustAIRLab/in-the-wild-jailbreak-prompts",
      "gold": "jailbreak",
      "predictions": {
        "jev": {
          "label": "jailbreak",
          "correct": true,
          "ms": 226
        },
        "imajev-bf16": {
          "label": "jailbreak",
          "correct": true,
          "ms": 1145
        },
        "imajev-4bit": {
          "label": "jailbreak",
          "correct": true,
          "ms": 1186
        },
        "imajev-8bit": {
          "label": "jailbreak",
          "correct": true,
          "ms": 1194
        },
        "plumb": {
          "label": "jailbreak",
          "correct": true,
          "ms": 912
        },
        "gliner-cpu": {
          "label": "jailbreak",
          "correct": true,
          "ms": 4341
        },
        "gliner-mlx": {
          "label": "jailbreak",
          "correct": true,
          "ms": 184
        },
        "gliner-int8": {
          "label": "jailbreak",
          "correct": true,
          "ms": 196
        },
        "tev-4b": {
          "label": "jailbreak",
          "correct": true,
          "ms": 1362
        },
        "tev-08b": {
          "label": "injection",
          "correct": false,
          "ms": 203
        },
        "nimble": {
          "label": "jailbreak",
          "correct": true,
          "ms": 1823
        }
      }
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "source": "CFPB Consumer Complaint Database",
      "gold": "money_transfer",
      "predictions": {
        "jev": {
          "label": "money_transfer",
          "correct": true,
          "ms": 159
        },
        "imajev-bf16": {
          "label": "money_transfer",
          "correct": true,
          "ms": 1146
        },
        "imajev-4bit": {
          "label": "money_transfer",
          "correct": true,
          "ms": 1165
        },
        "imajev-8bit": {
          "label": "money_transfer",
          "correct": true,
          "ms": 1150
        },
        "plumb": {
          "label": "money_transfer",
          "correct": true,
          "ms": 967
        },
        "gliner-cpu": {
          "label": "money_transfer",
          "correct": true,
          "ms": 4471
        },
        "gliner-mlx": {
          "label": "money_transfer",
          "correct": true,
          "ms": 165
        },
        "gliner-int8": {
          "label": "money_transfer",
          "correct": true,
          "ms": 192
        },
        "tev-4b": {
          "label": "money_transfer",
          "correct": true,
          "ms": 1374
        },
        "tev-08b": {
          "label": "money_transfer",
          "correct": true,
          "ms": 216
        },
        "nimble": {
          "label": "money_transfer",
          "correct": true,
          "ms": 1878
        }
      }
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "source": "CFPB Consumer Complaint Database",
      "gold": "bank_account",
      "predictions": {
        "jev": {
          "label": "bank_account",
          "correct": true,
          "ms": 181
        },
        "imajev-bf16": {
          "label": "bank_account",
          "correct": true,
          "ms": 1037
        },
        "imajev-4bit": {
          "label": "bank_account",
          "correct": true,
          "ms": 1068
        },
        "imajev-8bit": {
          "label": "bank_account",
          "correct": true,
          "ms": 1068
        },
        "plumb": {
          "label": "bank_account",
          "correct": true,
          "ms": 864
        },
        "gliner-cpu": {
          "label": "bank_account",
          "correct": true,
          "ms": 4764
        },
        "gliner-mlx": {
          "label": "bank_account",
          "correct": true,
          "ms": 183
        },
        "gliner-int8": {
          "label": "bank_account",
          "correct": true,
          "ms": 153
        },
        "tev-4b": {
          "label": "bank_account",
          "correct": true,
          "ms": 1233
        },
        "tev-08b": {
          "label": "bank_account",
          "correct": true,
          "ms": 199
        },
        "nimble": {
          "label": "bank_account",
          "correct": true,
          "ms": 1729
        }
      }
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "source": "Amazon Berkeley Objects (ABO)",
      "gold": "sofa",
      "predictions": {
        "jev": {
          "label": "sofa",
          "correct": true,
          "ms": 176
        },
        "imajev-bf16": {
          "label": "sofa",
          "correct": true,
          "ms": 1258
        },
        "imajev-4bit": {
          "label": "sofa",
          "correct": true,
          "ms": 1404
        },
        "imajev-8bit": {
          "label": "sofa",
          "correct": true,
          "ms": 1298
        },
        "plumb": {
          "label": "sofa",
          "correct": true,
          "ms": 1109
        },
        "gliner-cpu": {
          "label": "sofa",
          "correct": true,
          "ms": 5196
        },
        "gliner-mlx": {
          "label": "sofa",
          "correct": true,
          "ms": 698
        },
        "gliner-int8": {
          "label": "sofa",
          "correct": true,
          "ms": 250
        },
        "tev-4b": {
          "label": "sofa",
          "correct": true,
          "ms": 1510
        },
        "tev-08b": {
          "label": "sofa",
          "correct": true,
          "ms": 241
        },
        "nimble": {
          "label": "sofa",
          "correct": true,
          "ms": 2087
        }
      }
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "source": "Amazon Berkeley Objects (ABO)",
      "gold": "sofa",
      "predictions": {
        "jev": {
          "label": "sofa",
          "correct": true,
          "ms": 169
        },
        "imajev-bf16": {
          "label": "sofa",
          "correct": true,
          "ms": 1322
        },
        "imajev-4bit": {
          "label": "sofa",
          "correct": true,
          "ms": 1285
        },
        "imajev-8bit": {
          "label": "sofa",
          "correct": true,
          "ms": 1290
        },
        "plumb": {
          "label": "sofa",
          "correct": true,
          "ms": 1060
        },
        "gliner-cpu": {
          "label": "sofa",
          "correct": true,
          "ms": 4868
        },
        "gliner-mlx": {
          "label": "sofa",
          "correct": true,
          "ms": 284
        },
        "gliner-int8": {
          "label": "sofa",
          "correct": true,
          "ms": 228
        },
        "tev-4b": {
          "label": "sofa",
          "correct": true,
          "ms": 1386
        },
        "tev-08b": {
          "label": "sofa",
          "correct": true,
          "ms": 220
        },
        "nimble": {
          "label": "sofa",
          "correct": true,
          "ms": 1904
        }
      }
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "source": "EDGAR-CORPUS",
      "gold": "business",
      "predictions": {
        "jev": {
          "label": "business",
          "correct": true,
          "ms": 254
        },
        "imajev-bf16": {
          "label": "business",
          "correct": true,
          "ms": 1223
        },
        "imajev-4bit": {
          "label": "business",
          "correct": true,
          "ms": 1249
        },
        "imajev-8bit": {
          "label": "business",
          "correct": true,
          "ms": 1220
        },
        "plumb": {
          "label": "business",
          "correct": true,
          "ms": 1007
        },
        "gliner-cpu": {
          "label": "business",
          "correct": true,
          "ms": 5013
        },
        "gliner-mlx": {
          "label": "business",
          "correct": true,
          "ms": 260
        },
        "gliner-int8": {
          "label": "business",
          "correct": true,
          "ms": 226
        },
        "tev-4b": {
          "label": "business",
          "correct": true,
          "ms": 1503
        },
        "tev-08b": {
          "label": "business",
          "correct": true,
          "ms": 217
        },
        "nimble": {
          "label": "business",
          "correct": true,
          "ms": 1905
        }
      }
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "source": "FiNER-139",
      "gold": "share_based_compensation",
      "predictions": {
        "jev": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 190
        },
        "imajev-bf16": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1321
        },
        "imajev-4bit": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1264
        },
        "imajev-8bit": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1201
        },
        "plumb": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1008
        },
        "gliner-cpu": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 4221
        },
        "gliner-mlx": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 251
        },
        "gliner-int8": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 214
        },
        "tev-4b": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1386
        },
        "tev-08b": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 218
        },
        "nimble": {
          "label": "share_based_compensation",
          "correct": true,
          "ms": 1915
        }
      }
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "source": "CUAD (Contract Understanding Atticus Dataset) v1",
      "gold": "termination_for_convenience",
      "predictions": {
        "jev": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 204
        },
        "imajev-bf16": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1252
        },
        "imajev-4bit": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1291
        },
        "imajev-8bit": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1274
        },
        "plumb": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1011
        },
        "gliner-cpu": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 4962
        },
        "gliner-mlx": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 321
        },
        "gliner-int8": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 270
        },
        "tev-4b": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1472
        },
        "tev-08b": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 232
        },
        "nimble": {
          "label": "termination_for_convenience",
          "correct": true,
          "ms": 1999
        }
      }
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "source": "UNFAIR-ToS (LexGLUE)",
      "gold": "limitation_of_liability",
      "predictions": {
        "jev": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 223
        },
        "imajev-bf16": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 1177
        },
        "imajev-4bit": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 1248
        },
        "imajev-8bit": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 1189
        },
        "plumb": {
          "label": "not_unfair",
          "correct": false,
          "ms": 1010
        },
        "gliner-cpu": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 4979
        },
        "gliner-mlx": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 272
        },
        "gliner-int8": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 218
        },
        "tev-4b": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 1359
        },
        "tev-08b": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 223
        },
        "nimble": {
          "label": "limitation_of_liability",
          "correct": true,
          "ms": 1924
        }
      }
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "source": "Upworthy Research Archive (exploratory packages)",
      "gold": "A",
      "predictions": {
        "jev": {
          "label": "B",
          "correct": false,
          "ms": 178
        },
        "imajev-bf16": {
          "label": "A",
          "correct": true,
          "ms": 1011
        },
        "imajev-4bit": {
          "label": "A",
          "correct": true,
          "ms": 1097
        },
        "imajev-8bit": {
          "label": "A",
          "correct": true,
          "ms": 1057
        },
        "plumb": {
          "label": "A",
          "correct": true,
          "ms": 818
        },
        "gliner-cpu": {
          "label": "B",
          "correct": false,
          "ms": 3323
        },
        "gliner-mlx": {
          "label": "B",
          "correct": false,
          "ms": 258
        },
        "gliner-int8": {
          "label": "B",
          "correct": false,
          "ms": 189
        },
        "tev-4b": {
          "label": "A",
          "correct": true,
          "ms": 1204
        },
        "tev-08b": {
          "label": "A",
          "correct": true,
          "ms": 181
        },
        "nimble": {
          "label": "A",
          "correct": true,
          "ms": 1639
        }
      }
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "source": "ramsey/uuid CHANGELOG.md",
      "gold": "minor",
      "predictions": {
        "jev": {
          "label": "minor",
          "correct": true,
          "ms": 208
        },
        "imajev-bf16": {
          "label": "minor",
          "correct": true,
          "ms": 930
        },
        "imajev-4bit": {
          "label": "minor",
          "correct": true,
          "ms": 952
        },
        "imajev-8bit": {
          "label": "minor",
          "correct": true,
          "ms": 955
        },
        "plumb": {
          "label": "minor",
          "correct": true,
          "ms": 712
        },
        "gliner-cpu": {
          "label": "major",
          "correct": false,
          "ms": 3488
        },
        "gliner-mlx": {
          "label": "major",
          "correct": false,
          "ms": 179
        },
        "gliner-int8": {
          "label": "major",
          "correct": false,
          "ms": 140
        },
        "tev-4b": {
          "label": "minor",
          "correct": true,
          "ms": 1080
        },
        "tev-08b": {
          "label": "minor",
          "correct": true,
          "ms": 159
        },
        "nimble": {
          "label": "minor",
          "correct": true,
          "ms": 1446
        }
      }
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "source": "FiveThirtyEight data repository",
      "gold": "money",
      "predictions": {
        "jev": {
          "label": "money",
          "correct": true,
          "ms": 224
        },
        "imajev-bf16": {
          "label": "money",
          "correct": true,
          "ms": 1541
        },
        "imajev-4bit": {
          "label": "money",
          "correct": true,
          "ms": 1556
        },
        "imajev-8bit": {
          "label": "money",
          "correct": true,
          "ms": 1623
        },
        "plumb": {
          "label": "money",
          "correct": true,
          "ms": 1257
        },
        "gliner-cpu": {
          "label": "money",
          "correct": true,
          "ms": 6254
        },
        "gliner-mlx": {
          "label": "money",
          "correct": true,
          "ms": 400
        },
        "gliner-int8": {
          "label": "money",
          "correct": true,
          "ms": 295
        },
        "tev-4b": {
          "label": "money",
          "correct": true,
          "ms": 1523
        },
        "tev-08b": {
          "label": "identifier",
          "correct": false,
          "ms": 256
        },
        "nimble": {
          "label": "money",
          "correct": true,
          "ms": 2283
        }
      }
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "source": "U.S. Treasury Fiscal Data",
      "gold": "money",
      "predictions": {
        "jev": {
          "label": "money",
          "correct": true,
          "ms": 233
        },
        "imajev-bf16": {
          "label": "money",
          "correct": true,
          "ms": 1822
        },
        "imajev-4bit": {
          "label": "money",
          "correct": true,
          "ms": 1838
        },
        "imajev-8bit": {
          "label": "money",
          "correct": true,
          "ms": 2022
        },
        "plumb": {
          "label": "money",
          "correct": true,
          "ms": 1452
        },
        "gliner-cpu": {
          "label": "money",
          "correct": true,
          "ms": 7420
        },
        "gliner-mlx": {
          "label": "money",
          "correct": true,
          "ms": 564
        },
        "gliner-int8": {
          "label": "money",
          "correct": true,
          "ms": 425
        },
        "tev-4b": {
          "label": "money",
          "correct": true,
          "ms": 1861
        },
        "tev-08b": {
          "label": "money",
          "correct": true,
          "ms": 303
        },
        "nimble": {
          "label": "money",
          "correct": true,
          "ms": 2762
        }
      }
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "source": "AMI Meeting Corpus",
      "gold": "action_item",
      "predictions": {
        "jev": {
          "label": "action_item",
          "correct": true,
          "ms": 211
        },
        "imajev-bf16": {
          "label": "neither",
          "correct": false,
          "ms": 1325
        },
        "imajev-4bit": {
          "label": "neither",
          "correct": false,
          "ms": 1362
        },
        "imajev-8bit": {
          "label": "neither",
          "correct": false,
          "ms": 1572
        },
        "plumb": {
          "label": "action_item",
          "correct": true,
          "ms": 1056
        },
        "gliner-cpu": {
          "label": "neither",
          "correct": false,
          "ms": 6803
        },
        "gliner-mlx": {
          "label": "neither",
          "correct": false,
          "ms": 396
        },
        "gliner-int8": {
          "label": "neither",
          "correct": false,
          "ms": 370
        },
        "tev-4b": {
          "label": "decision",
          "correct": false,
          "ms": 1327
        },
        "tev-08b": {
          "label": "action_item",
          "correct": true,
          "ms": 225
        },
        "nimble": {
          "label": "action_item",
          "correct": true,
          "ms": 2048
        }
      }
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "source": "AMI Meeting Corpus",
      "gold": "neither",
      "predictions": {
        "jev": {
          "label": "neither",
          "correct": true,
          "ms": 312
        },
        "imajev-bf16": {
          "label": "neither",
          "correct": true,
          "ms": 1291
        },
        "imajev-4bit": {
          "label": "neither",
          "correct": true,
          "ms": 1324
        },
        "imajev-8bit": {
          "label": "neither",
          "correct": true,
          "ms": 1326
        },
        "plumb": {
          "label": "neither",
          "correct": true,
          "ms": 1056
        },
        "gliner-cpu": {
          "label": "neither",
          "correct": true,
          "ms": 6922
        },
        "gliner-mlx": {
          "label": "neither",
          "correct": true,
          "ms": 397
        },
        "gliner-int8": {
          "label": "neither",
          "correct": true,
          "ms": 324
        },
        "tev-4b": {
          "label": "neither",
          "correct": true,
          "ms": 1364
        },
        "tev-08b": {
          "label": "neither",
          "correct": true,
          "ms": 227
        },
        "nimble": {
          "label": "neither",
          "correct": true,
          "ms": 2025
        }
      }
    }
  ]
}