{
  "model": "tev1:0.8b",
  "runner": "ollama 0.35.0",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "36db20dd",
      "correct": true,
      "probabilities": {
        "f1ded018": 0.017136787128543496,
        "d65f5e07": 0.47323253331368637,
        "fa3bada5": 0.025921796840254776,
        "36db20dd": 0.48370888271751533
      },
      "confidence": 0.37262034322359927,
      "ms": 649,
      "usage": {
        "input_tokens": 1249,
        "output_tokens": 1
      }
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "feat",
      "correct": true,
      "probabilities": {
        "refactor": 0.037373603663094855,
        "docs": 0.026133613551321815,
        "fix": 0.016907410528364173,
        "test": 0.0032924632432436577,
        "feat": 0.9162929090139754
      },
      "confidence": 0.7601721611674319,
      "ms": 170,
      "usage": {
        "input_tokens": 483,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "probabilities": {
        "not_supported": 0.008531542266220604,
        "supported": 0.9914684577337793
      },
      "confidence": 0.9291071411979378,
      "ms": 228,
      "usage": {
        "input_tokens": 650,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "probabilities": {
        "none": 0.04539491171110718,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.9529102979368723,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.0012439136400760411,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.0004508767119443812
      },
      "confidence": 0.8570756652845395,
      "ms": 190,
      "usage": {
        "input_tokens": 518,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "toxic",
      "correct": false,
      "probabilities": {
        "toxic": 0.5102162594495762,
        "not_toxic": 0.48978374055042384
      },
      "confidence": 0.00030117476812574484,
      "ms": 129,
      "usage": {
        "input_tokens": 353,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "injection",
      "correct": false,
      "probabilities": {
        "jailbreak": 0.295320335480068,
        "injection": 0.5563044490742375,
        "benign": 0.14837521544569446
      },
      "confidence": 0.11748565435964786,
      "ms": 203,
      "usage": {
        "input_tokens": 566,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "probabilities": {
        "student_loan": 0.0727000817074701,
        "debt_collection": 0.16633725337004351,
        "bank_account": 0.04796464670404585,
        "mortgage": 0.0017562934720856385,
        "credit_card": 0.03650990736230205,
        "credit_reporting": 0.02451477159344102,
        "money_transfer": 0.6502170457906118
      },
      "confidence": 0.4154818803600012,
      "ms": 216,
      "usage": {
        "input_tokens": 587,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "probabilities": {
        "student_loan": 0.003819546439852688,
        "credit_card": 0.4884795040214761,
        "credit_reporting": 0.01832590406234114,
        "bank_account": 0.4889194356283845,
        "money_transfer": 0.0002507223567771372,
        "debt_collection": 0.000042121024366073265,
        "mortgage": 0.0001627664668022424
      },
      "confidence": 0.589751321499701,
      "ms": 199,
      "usage": {
        "input_tokens": 530,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "wall_art": 0.00009539208410039647,
        "grocery": 0.00019573334114056015,
        "chair": 0.000059879120666496035,
        "lamp": 0.00010392702378874902,
        "rug": 0.000029281998896374995,
        "table": 0.0001571406252910894,
        "shoes": 0.0005554527186483076,
        "sofa": 0.9988031930874681
      },
      "confidence": 0.9946469289931201,
      "ms": 241,
      "usage": {
        "input_tokens": 686,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "rug": 0.00010883308154388642,
        "lamp": 0.00014096947447527652,
        "sofa": 0.9991904668055004,
        "chair": 0.0003753160730948876,
        "grocery": 0.000012367401068499558,
        "wall_art": 0.000017368064678480627,
        "shoes": 0.000005700439007597993,
        "table": 0.00014897866063093691
      },
      "confidence": 0.9962853168738577,
      "ms": 220,
      "usage": {
        "input_tokens": 625,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "probabilities": {
        "risk_factors": 0.0048150123769398435,
        "controls": 0.06722602160282665,
        "legal_proceedings": 0.001029344769433605,
        "business": 0.5755922120679176,
        "mdna": 0.35133740918288225
      },
      "confidence": 0.4409868458755548,
      "ms": 217,
      "usage": {
        "input_tokens": 615,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "probabilities": {
        "revenue": 0.0003581598225446793,
        "income_tax_expense": 0.0001651955347171027,
        "borrowings": 0.00008284717868890482,
        "amortization": 0.00010805144229439669,
        "share_based_compensation": 0.9988344036156697,
        "goodwill": 0.00045134240608514655
      },
      "confidence": 0.9940351283726131,
      "ms": 218,
      "usage": {
        "input_tokens": 621,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "probabilities": {
        "governing_law": 0.014447152042704497,
        "termination_for_convenience": 0.9104973550853585,
        "anti_assignment": 0.02306430077073423,
        "non_compete": 0.014093224402269452,
        "license_grant": 0.010930187904050887,
        "cap_on_liability": 0.02696777979488232
      },
      "confidence": 0.7542105193608692,
      "ms": 232,
      "usage": {
        "input_tokens": 637,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "limitation_of_liability",
      "correct": true,
      "probabilities": {
        "unilateral_termination": 0.008164155683542656,
        "limitation_of_liability": 0.8046609983779196,
        "choice_of_law": 0.0016574850277188606,
        "not_unfair": 0.11824484882317123,
        "contract_by_using": 0.007079301108068626,
        "content_removal": 0.05295423219010688,
        "arbitration": 0.0015351467743661147,
        "unilateral_change": 0.0022764288667435732,
        "jurisdiction": 0.0034274031483622776
      },
      "confidence": 0.6763670766019634,
      "ms": 223,
      "usage": {
        "input_tokens": 624,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "probabilities": {
        "B": 0.46285999172169107,
        "A": 0.5371400082783089
      },
      "confidence": 0.003983718094197064,
      "ms": 181,
      "usage": {
        "input_tokens": 519,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "probabilities": {
        "minor": 0.7754519979761727,
        "major": 0.12153441653705097,
        "patch": 0.10301358548677621
      },
      "confidence": 0.374224747023693,
      "ms": 159,
      "usage": {
        "input_tokens": 448,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "identifier",
      "correct": false,
      "probabilities": {
        "category": 0.20801945407866088,
        "money": 0.16823967460767816,
        "identifier": 0.37729296606569,
        "count_or_measurement": 0.13157512952841907,
        "date": 0.025908283838595617,
        "geography": 0.02241302824292926,
        "free_text": 0.06655146363802691
      },
      "confidence": 0.1668610379729698,
      "ms": 256,
      "usage": {
        "input_tokens": 747,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "probabilities": {
        "money": 0.9401493907964129,
        "geography": 0.013001879498497402,
        "category": 0.005337182039117294,
        "identifier": 0.01669780091954478,
        "free_text": 0.003531809099396596,
        "count_or_measurement": 0.013663328687827068,
        "date": 0.007618608959204158
      },
      "confidence": 0.8322093918937032,
      "ms": 303,
      "usage": {
        "input_tokens": 884,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "action_item",
      "correct": true,
      "probabilities": {
        "action_item": 0.6099184733152491,
        "neither": 0.2678702914282763,
        "decision": 0.1222112352564746
      },
      "confidence": 0.17049598385743958,
      "ms": 225,
      "usage": {
        "input_tokens": 650,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "probabilities": {
        "decision": 0.15374793255028904,
        "action_item": 0.027447335906187764,
        "neither": 0.8188047315435233
      },
      "confidence": 0.49913414262528455,
      "ms": 227,
      "usage": {
        "input_tokens": 643,
        "output_tokens": 1
      }
    }
  ]
}