{
  "model": "tev1:4b",
  "runner": "ollama 0.35.0",
  "corpus_sha256": "3599baea0d9c7e4e3d86c8b06af96e6850edca037b334bbc6b4e1a0033021725",
  "rows": [
    {
      "id": "eng-5-36db20dd946e",
      "category": "engineering",
      "gold": "36db20dd",
      "label": "d65f5e07",
      "correct": false,
      "probabilities": {
        "f1ded018": 0.000059642715944089166,
        "d65f5e07": 0.7436385611765315,
        "fa3bada5": 0.00005704319194875583,
        "36db20dd": 0.25624475291557586
      },
      "confidence": 0.5886068682644567,
      "ms": 3626,
      "usage": {
        "input_tokens": 1249,
        "output_tokens": 1
      }
    },
    {
      "id": "eng-2-3f22797c158b",
      "category": "engineering",
      "gold": "feat",
      "label": "feat",
      "correct": true,
      "probabilities": {
        "refactor": 0.08208012718820576,
        "docs": 0.003210369532820205,
        "fix": 0.0051106621848876375,
        "test": 0.00027406047322940755,
        "feat": 0.9093247806208571
      },
      "confidence": 0.7891903558962123,
      "ms": 1009,
      "usage": {
        "input_tokens": 483,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-4-hagrid-dev-983-0",
      "category": "agents",
      "gold": "not_supported",
      "label": "supported",
      "correct": false,
      "probabilities": {
        "not_supported": 0.030179504748779794,
        "supported": 0.9698204952512203
      },
      "confidence": 0.8047087319643664,
      "ms": 1283,
      "usage": {
        "input_tokens": 650,
        "output_tokens": 1
      }
    },
    {
      "id": "agt-1-live-multiple-258-122-0",
      "category": "agents",
      "gold": "project_api.ProjectApi.get_project_by_name_and_version",
      "label": "project_api.ProjectApi.get_project_by_name_and_version",
      "correct": true,
      "probabilities": {
        "none": 0.002783248009234378,
        "project_api.ProjectApi.get_project_by_name_and_version": 0.997001997809351,
        "badge_api.BadgeApi.get_project_vulnerabilities_badge": 0.00016707571686828336,
        "badge_api.BadgeApi.get_project_policy_violations_badge1": 0.00004767846454620845
      },
      "confidence": 0.9846367164343443,
      "ms": 1156,
      "usage": {
        "input_tokens": 518,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-2-civil-1620015",
      "category": "safety",
      "gold": "not_toxic",
      "label": "not_toxic",
      "correct": true,
      "probabilities": {
        "toxic": 0.04076151284372169,
        "not_toxic": 0.9592384871562784
      },
      "confidence": 0.7542271309694691,
      "ms": 774,
      "usage": {
        "input_tokens": 353,
        "output_tokens": 1
      }
    },
    {
      "id": "saf-1-itw-jailbreak-2023-12-25-1216",
      "category": "safety",
      "gold": "jailbreak",
      "label": "jailbreak",
      "correct": true,
      "probabilities": {
        "jailbreak": 0.8774684854690886,
        "injection": 0.08287411688419939,
        "benign": 0.03965739764671209
      },
      "confidence": 0.5912266849625817,
      "ms": 1362,
      "usage": {
        "input_tokens": 566,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-2791756",
      "category": "support",
      "gold": "money_transfer",
      "label": "money_transfer",
      "correct": true,
      "probabilities": {
        "student_loan": 0.00020851588112684246,
        "debt_collection": 0.0007354894102217248,
        "bank_account": 0.00043554700820051806,
        "mortgage": 0.00008339811785967221,
        "credit_card": 0.00009315949249583527,
        "credit_reporting": 0.00007611639957417831,
        "money_transfer": 0.9983677736905212
      },
      "confidence": 0.9925766911206529,
      "ms": 1374,
      "usage": {
        "input_tokens": 587,
        "output_tokens": 1
      }
    },
    {
      "id": "sup-1-cfpb-1680253",
      "category": "support",
      "gold": "bank_account",
      "label": "bank_account",
      "correct": true,
      "probabilities": {
        "student_loan": 0.00009993982602328802,
        "credit_card": 0.006098587504831841,
        "credit_reporting": 0.0001332825215102742,
        "bank_account": 0.993592118618666,
        "money_transfer": 0.000050532406074730214,
        "debt_collection": 0.000011131536358092303,
        "mortgage": 0.000014407586535768733
      },
      "confidence": 0.979245920208416,
      "ms": 1233,
      "usage": {
        "input_tokens": 530,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b07bw8q16k",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "wall_art": 0.00010723461537866432,
        "grocery": 0.000057983946972031684,
        "chair": 0.0007631832827177872,
        "lamp": 0.00008486128154195725,
        "rug": 0.00001367126521758636,
        "table": 0.000014728913948346298,
        "shoes": 0.000014085020751095046,
        "sofa": 0.9989442516734724
      },
      "confidence": 0.995504072884884,
      "ms": 1510,
      "usage": {
        "input_tokens": 686,
        "output_tokens": 1
      }
    },
    {
      "id": "com-2-abo-b072m1wjc7",
      "category": "commerce",
      "gold": "sofa",
      "label": "sofa",
      "correct": true,
      "probabilities": {
        "rug": 0.00029594254007811585,
        "lamp": 0.00012397484029864217,
        "sofa": 0.9981843692442551,
        "chair": 0.0013489272148319322,
        "grocery": 0.0000050708441061136036,
        "wall_art": 0.0000024489216799363944,
        "shoes": 0.0000033779104021610156,
        "table": 0.00003588848434816392
      },
      "confidence": 0.9929060388522716,
      "ms": 1386,
      "usage": {
        "input_tokens": 625,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-1-edgar-1042642-2017",
      "category": "finance",
      "gold": "business",
      "label": "business",
      "correct": true,
      "probabilities": {
        "risk_factors": 0.0006701061112795464,
        "controls": 0.0010843008197828205,
        "legal_proceedings": 0.00044411770974842247,
        "business": 0.9897765426645386,
        "mdna": 0.008024932694650523
      },
      "confidence": 0.9598488497849623,
      "ms": 1503,
      "usage": {
        "input_tokens": 615,
        "output_tokens": 1
      }
    },
    {
      "id": "fin-4-finer-test-1013526",
      "category": "finance",
      "gold": "share_based_compensation",
      "label": "share_based_compensation",
      "correct": true,
      "probabilities": {
        "revenue": 0.00003545278555590302,
        "income_tax_expense": 0.000050530238745220986,
        "borrowings": 0.00006339120814131246,
        "amortization": 0.000035228724968973283,
        "share_based_compensation": 0.9997916818121485,
        "goodwill": 0.00002371523044003514
      },
      "confidence": 0.9987174552896843,
      "ms": 1386,
      "usage": {
        "input_tokens": 621,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-2-cuad-95-termination-for-convenience",
      "category": "legal",
      "gold": "termination_for_convenience",
      "label": "termination_for_convenience",
      "correct": true,
      "probabilities": {
        "governing_law": 0.000007457142035751145,
        "termination_for_convenience": 0.9999205945778319,
        "anti_assignment": 0.00004355808485366566,
        "non_compete": 0.00001389907626494949,
        "license_grant": 0.0000042756818980723745,
        "cap_on_liability": 0.000010215437115520125
      },
      "confidence": 0.9994806659000843,
      "ms": 1472,
      "usage": {
        "input_tokens": 637,
        "output_tokens": 1
      }
    },
    {
      "id": "leg-3-unfair-tos-test-237",
      "category": "legal",
      "gold": "limitation_of_liability",
      "label": "limitation_of_liability",
      "correct": true,
      "probabilities": {
        "unilateral_termination": 0.0000984616521899401,
        "limitation_of_liability": 0.9861074348132994,
        "choice_of_law": 0.00008969687708328511,
        "not_unfair": 0.012817075677043146,
        "contract_by_using": 0.00004007240598906035,
        "content_removal": 0.0006636509379720604,
        "arbitration": 0.00002685432913941973,
        "unilateral_change": 0.000034124148106149585,
        "jurisdiction": 0.00012262915917749312
      },
      "confidence": 0.964325981685981,
      "ms": 1359,
      "usage": {
        "input_tokens": 624,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-1-53f253aaf1592f712e000017",
      "category": "product",
      "gold": "A",
      "label": "A",
      "correct": true,
      "probabilities": {
        "B": 0.4206702587351086,
        "A": 0.5793297412648913
      },
      "confidence": 0.018235320002659816,
      "ms": 1204,
      "usage": {
        "input_tokens": 519,
        "output_tokens": 1
      }
    },
    {
      "id": "prd-2-ramsey-uuid-2.4.0",
      "category": "product",
      "gold": "minor",
      "label": "minor",
      "correct": true,
      "probabilities": {
        "minor": 0.9102470093834009,
        "major": 0.007324026706955506,
        "patch": 0.08242896390964347
      },
      "confidence": 0.7020459607387693,
      "ms": 1080,
      "usage": {
        "input_tokens": 448,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-bechdel-6",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "probabilities": {
        "category": 0.0016104356489515742,
        "money": 0.9720629695129132,
        "identifier": 0.008542235982771016,
        "count_or_measurement": 0.01712241979638802,
        "date": 0.0004274724199372153,
        "geography": 0.00007728749000120763,
        "free_text": 0.00015717914903792682
      },
      "confidence": 0.9210383461473306,
      "ms": 1523,
      "usage": {
        "input_tokens": 747,
        "output_tokens": 1
      }
    },
    {
      "id": "dat-4-debt-1",
      "category": "data",
      "gold": "money",
      "label": "money",
      "correct": true,
      "probabilities": {
        "money": 0.975913123235073,
        "geography": 0.0011569934983495095,
        "category": 0.0014253103608478437,
        "identifier": 0.010165471451248555,
        "free_text": 0.00007603304119684212,
        "count_or_measurement": 0.010710859982637782,
        "date": 0.000552208430646518
      },
      "confidence": 0.9275101369806118,
      "ms": 1861,
      "usage": {
        "input_tokens": 884,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015a.a.dialog-act.dharshi.169",
      "category": "documents",
      "gold": "action_item",
      "label": "decision",
      "correct": false,
      "probabilities": {
        "action_item": 0.15647286087258447,
        "neither": 0.24269776606042157,
        "decision": 0.600829373066994
      },
      "confidence": 0.14440115054146418,
      "ms": 1327,
      "usage": {
        "input_tokens": 650,
        "output_tokens": 1
      }
    },
    {
      "id": "doc-2-es2015d.b.dialog-act.dharshi.87",
      "category": "documents",
      "gold": "neither",
      "label": "neither",
      "correct": true,
      "probabilities": {
        "decision": 0.01803596307775686,
        "action_item": 0.007525161305437731,
        "neither": 0.9744388756168053
      },
      "confidence": 0.8776207935075829,
      "ms": 1364,
      "usage": {
        "input_tokens": 643,
        "output_tokens": 1
      }
    }
  ]
}