{
  "model": "nimble",
  "runner": "ollama 0.35.0",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "36db20dd",
      "correct": true,
      "probabilities": {
        "f1ded018": 0.0002322403359260553,
        "d65f5e07": 0.19169953669376466,
        "fa3bada5": 0.00013282318383694655,
        "36db20dd": 0.8079353997864723
      },
      "confidence": 0.6450290643889062,
      "ms": 3925,
      "usage": {
        "input_tokens": 1287,
        "output_tokens": 1
      }
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "feat",
      "correct": true,
      "probabilities": {
        "refactor": 0.15173236881663138,
        "docs": 0.0022384403802687344,
        "fix": 0.008849894377701138,
        "test": 0.0009351300924875149,
        "feat": 0.8362441663329113
      },
      "confidence": 0.6907743861653424,
      "ms": 1684,
      "usage": {
        "input_tokens": 521,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "probabilities": {
        "not_supported": 0.006352005088183932,
        "supported": 0.9936479949118161
      },
      "confidence": 0.9445045926321403,
      "ms": 2151,
      "usage": {
        "input_tokens": 688,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "probabilities": {
        "none": 0.0004371264070207282,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.9985157840888433,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.0006675305427245548,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.0003795589614114883
      },
      "confidence": 0.9908136807617098,
      "ms": 1794,
      "usage": {
        "input_tokens": 556,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "not_toxic",
      "correct": true,
      "probabilities": {
        "toxic": 0.022645062717450756,
        "not_toxic": 0.9773549372825493
      },
      "confidence": 0.8439552501301906,
      "ms": 1148,
      "usage": {
        "input_tokens": 391,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "jailbreak",
      "correct": true,
      "probabilities": {
        "jailbreak": 0.9701138530936572,
        "injection": 0.020392085373903374,
        "benign": 0.009494061532439462
      },
      "confidence": 0.8607077931938936,
      "ms": 1823,
      "usage": {
        "input_tokens": 604,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "probabilities": {
        "student_loan": 0.0003922355801885625,
        "debt_collection": 0.00040395539467498275,
        "bank_account": 0.001459022977381231,
        "mortgage": 0.0003096477379023601,
        "credit_card": 0.00043329161150024934,
        "credit_reporting": 0.00031983063313507167,
        "money_transfer": 0.9966820160652176
      },
      "confidence": 0.9858655538522656,
      "ms": 1878,
      "usage": {
        "input_tokens": 625,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "probabilities": {
        "student_loan": 0.00012295599599684208,
        "credit_card": 0.0006503475704317081,
        "credit_reporting": 0.00010358752715344038,
        "bank_account": 0.9989870402201526,
        "money_transfer": 0.00007537108883210737,
        "debt_collection": 0.000034846833680520314,
        "mortgage": 0.000025850763752753415
      },
      "confidence": 0.9952780764804147,
      "ms": 1729,
      "usage": {
        "input_tokens": 568,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "wall_art": 0.00008748812523660505,
        "grocery": 0.00012904197696691125,
        "chair": 0.0010650599699000367,
        "lamp": 0.0001425218554110854,
        "rug": 0.0000702145961164089,
        "table": 0.00010031720515181528,
        "shoes": 0.000026655538326983334,
        "sofa": 0.99837870073289
      },
      "confidence": 0.9932572149663863,
      "ms": 2087,
      "usage": {
        "input_tokens": 724,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "rug": 0.00009703725843826387,
        "lamp": 0.00011643116180375015,
        "sofa": 0.9973888672673453,
        "chair": 0.0021868289247056233,
        "grocery": 0.000029032908814261405,
        "wall_art": 0.00001836468983474412,
        "shoes": 0.000015101772399978309,
        "table": 0.00014833601665814288
      },
      "confidence": 0.9904142603739918,
      "ms": 1904,
      "usage": {
        "input_tokens": 663,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "probabilities": {
        "risk_factors": 0.002225240811098694,
        "controls": 0.000346271475812036,
        "legal_proceedings": 0.0003396802163099419,
        "business": 0.9826324488063949,
        "mdna": 0.014456358690384526
      },
      "confidence": 0.9394037631817134,
      "ms": 1905,
      "usage": {
        "input_tokens": 653,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "probabilities": {
        "revenue": 0.00008889434439978135,
        "income_tax_expense": 0.0000876330900566281,
        "borrowings": 0.00014068193505136143,
        "amortization": 0.00016863520466052376,
        "share_based_compensation": 0.9993463979527593,
        "goodwill": 0.00016775747307254675
      },
      "confidence": 0.9963876917855231,
      "ms": 1915,
      "usage": {
        "input_tokens": 659,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "probabilities": {
        "governing_law": 0.00044974855488863626,
        "termination_for_convenience": 0.9986003243429524,
        "anti_assignment": 0.000358584489265818,
        "non_compete": 0.00042304477620538845,
        "license_grant": 0.0001009506194740704,
        "cap_on_liability": 0.00006734721721389676
      },
      "confidence": 0.9929836696561122,
      "ms": 1999,
      "usage": {
        "input_tokens": 675,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "limitation_of_liability",
      "correct": true,
      "probabilities": {
        "unilateral_termination": 0.0006115923446364693,
        "limitation_of_liability": 0.9792753257781388,
        "choice_of_law": 0.0003433682737051518,
        "not_unfair": 0.017285428947947358,
        "contract_by_using": 0.00012990890500801157,
        "content_removal": 0.0021419267161301462,
        "arbitration": 0.00005331532071416871,
        "unilateral_change": 0.00007696143262601514,
        "jurisdiction": 0.00008217228109418464
      },
      "confidence": 0.9479940947103574,
      "ms": 1924,
      "usage": {
        "input_tokens": 662,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "probabilities": {
        "B": 0.2309848121657365,
        "A": 0.7690151878342635
      },
      "confidence": 0.2202758940550662,
      "ms": 1639,
      "usage": {
        "input_tokens": 557,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "probabilities": {
        "minor": 0.7798092335284222,
        "major": 0.06276777864062302,
        "patch": 0.15742298783095474
      },
      "confidence": 0.40037928574334547,
      "ms": 1446,
      "usage": {
        "input_tokens": 486,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "probabilities": {
        "category": 0.001254026733407067,
        "money": 0.9819319372933855,
        "identifier": 0.006340257944492773,
        "count_or_measurement": 0.009877992920760281,
        "date": 0.00039784968894672625,
        "geography": 0.0001249389685689687,
        "free_text": 0.00007299645043873266
      },
      "confidence": 0.9440294075027617,
      "ms": 2283,
      "usage": {
        "input_tokens": 785,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "probabilities": {
        "money": 0.9406493493046015,
        "geography": 0.0009541824782049922,
        "category": 0.0005835827328627076,
        "identifier": 0.04952807366522637,
        "free_text": 0.00015016258056446498,
        "count_or_measurement": 0.007729066398030607,
        "date": 0.00040558284050929057
      },
      "confidence": 0.8666680933707167,
      "ms": 2762,
      "usage": {
        "input_tokens": 922,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "action_item",
      "correct": true,
      "probabilities": {
        "action_item": 0.4321777436330071,
        "neither": 0.23278162895939003,
        "decision": 0.33504062740760293
      },
      "confidence": 0.027641399492310814,
      "ms": 2048,
      "usage": {
        "input_tokens": 688,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "probabilities": {
        "decision": 0.0015767692298627647,
        "action_item": 0.0015692804782223041,
        "neither": 0.9968539502919149
      },
      "confidence": 0.978656650166037,
      "ms": 2025,
      "usage": {
        "input_tokens": 681,
        "output_tokens": 1
      }
    }
  ]
}