Hugging Face's logo

Berk
/
assay-0.6b

Zero-Shot Classification
Transformers
Safetensors
English
qwen3
text-generation
assay
decision-model
calibrated
conformal-prediction
text-classification
structured-output
text-generation-inference
Model card Files
xet
 Community
assay-0.6b / eval-transfer-v4-scaled.json
Berk's picture
Berk
Upload folder using huggingface_hub
c6d4f7d verified
Raw
History Blame Contribute Delete
6.7 kB
Invalid JSON:Unexpected token 'N', ..."fidence": NaN, "... is not valid JSON
{
 "overall": {
 "count": 764,
 "accuracy": 0.6361256544502618,
 "brier": 0.49852247078751044,
 "brier_soft": 0.49852247078751044,
 "nll": 0.850876258257955,
 "nll_soft": 0.850876258257955,
 "ece": 0.12407712725654348,
 "mean_confidence": 0.7596816331834112,
 "confident_error_rate": 0.04450261780104712,
 "score_mae": 0.2973544131495921
 },
 "by_type": {
 "bool": {
 "count": 280,
 "accuracy": 0.6642857142857143,
 "brier": 0.49259952159686327,
 "brier_soft": 0.49259952159686327,
 "nll": 0.7325980517166166,
 "nll_soft": 0.7325980517166166,
 "ece": 0.16049088636076536,
 "mean_confidence": 0.8200732964531596,
 "confident_error_rate": 0.07857142857142857
 },
 "choice": {
 "count": 444,
 "accuracy": 0.5923423423423423,
 "brier": 0.5278777983765222,
 "brier_soft": 0.5278777983765222,
 "nll": 0.9644794938837881,
 "nll_soft": 0.9644794938837881,
 "ece": 0.1394493857168115,
 "mean_confidence": 0.72885617509156,
 "confident_error_rate": 0.02702702702702703
 },
 "score": {
 "count": 40,
 "accuracy": 0.925,
 "brier": 0.21413897888400896,
 "brier_soft": 0.21413897888400896,
 "nll": 0.41782778860057623,
 "nll_soft": 0.41782778860057623,
 "ece": 0.268528126028744,
 "mean_confidence": 0.6791025751147204,
 "confident_error_rate": 0.0,
 "score_mae": 0.2973544131495921
 }
 },
 "by_source": {
 "composition_held_and_or": {
 "count": 32,
 "accuracy": 0.5,
 "brier": 0.667279448814107,
 "brier_soft": 0.667279448814107,
 "nll": 0.907676834819235,
 "nll_soft": 0.907676834819235,
 "ece": 0.3314446704480186,
 "mean_confidence": 0.7510849772790604,
 "confident_error_rate": 0.0
 },
 "composition_held_conditional": {
 "count": 32,
 "accuracy": 0.625,
 "brier": 0.6157411019594652,
 "brier_soft": 0.6157411019594652,
 "nll": 0.8572283897434516,
 "nll_soft": 0.8572283897434516,
 "ece": 0.34807412831864337,
 "mean_confidence": 0.7421134787354795,
 "confident_error_rate": 0.03125
 },
 "composition_held_or_not": {
 "count": 32,
 "accuracy": 0.5625,
 "brier": 0.4698421720914693,
 "brier_soft": 0.4698421720914693,
 "nll": 0.6583405822725403,
 "nll_soft": 0.6583405822725403,
 "ece": 0.27331303647467814,
 "mean_confidence": 0.7097878951663201,
 "confident_error_rate": 0.0
 },
 "contrastive_authorization": {
 "count": 40,
 "accuracy": 0.5,
 "brier": 0.836510913803445,
 "brier_soft": 0.836510913803445,
 "nll": 1.2750964423387703,
 "nll_soft": 1.2750964423387703,
 "ece": 0.44958833389484043,
 "mean_confidence": 0.9495883338948404,
 "confident_error_rate": 0.325
 },
 "contrastive_deadline": {
 "count": 40,
 "accuracy": 0.925,
 "brier": 0.21413897888400896,
 "brier_soft": 0.21413897888400896,
 "nll": 0.41782778860057623,
 "nll_soft": 0.41782778860057623,
 "ece": 0.268528126028744,
 "mean_confidence": 0.6791025751147204,
 "confident_error_rate": 0.0,
 "score_mae": 0.2973544131495921
 },
 "emotion": {
 "count": 116,
 "accuracy": 0.45689655172413796,
 "brier": 0.7078152200897051,
 "brier_soft": 0.7078152200897051,
 "nll": 1.376784251217549,
 "nll_soft": 1.376784251217549,
 "ece": 0.2352329817213669,
 "mean_confidence": 0.6614128736301557,
 "confident_error_rate": 0.04310344827586207
 },
 "mmlu": {
 "count": 116,
 "accuracy": 0.45689655172413796,
 "brier": 0.6360155342312662,
 "brier_soft": 0.6360155342312662,
 "nll": 1.2028071667685185,
 "nll_soft": 1.2028071667685185,
 "ece": 0.16795157187610193,
 "mean_confidence": 0.6082772615480655,
 "confident_error_rate": 0.008620689655172414
 },
 "paws": {
 "count": 80,
 "accuracy": 0.5875,
 "brier": 0.5475383162511623,
 "brier_soft": 0.5475383162511623,
 "nll": 0.7795850558186694,
 "nll_soft": 0.7795850558186694,
 "ece": 0.266156270517573,
 "mean_confidence": 0.7797887976199636,
 "confident_error_rate": 0.0375
 },
 "qnli": {
 "count": 80,
 "accuracy": 0.775,
 "brier": 0.31303704237048013,
 "brier_soft": 0.31303704237048013,
 "nll": 0.4808854176032183,
 "nll_soft": 0.4808854176032183,
 "ece": 0.10685060871150737,
 "mean_confidence": 0.7574151574083786,
 "confident_error_rate": 0.0
 },
 "sciq": {
 "count": 116,
 "accuracy": 0.8879310344827587,
 "brier": 0.19311868832984344,
 "brier_soft": 0.19311868832984344,
 "nll": 0.4435553877525076,
 "nll_soft": 0.4435553877525076,
 "ece": 0.06831785607105256,
 "mean_confidence": 0.9123493346740643,
 "confident_error_rate": 0.04310344827586207
 },
 "tweet_offensive": {
 "count": 80,
 "accuracy": 0.7125,
 "brier": 0.4452675100656568,
 "brier_soft": 0.4452675100656568,
 "nll": 0.6660744864168855,
 "nll_soft": 0.6660744864168855,
 "ece": 0.17723213008556796,
 "mean_confidence": 0.858258415610296,
 "confident_error_rate": 0.075
 }
 },
 "reliability": [
 {
 "bin": "0.0-0.1",
 "count": 0,
 "confidence": NaN,
 "accuracy": NaN
 },
 {
 "bin": "0.1-0.2",
 "count": 0,
 "confidence": NaN,
 "accuracy": NaN
 },
 {
 "bin": "0.2-0.3",
 "count": 5,
 "confidence": 0.2889275957069053,
 "accuracy": 0.0
 },
 {
 "bin": "0.3-0.4",
 "count": 28,
 "confidence": 0.35784894162284964,
 "accuracy": 0.17857142857142858
 },
 {
 "bin": "0.4-0.5",
 "count": 54,
 "confidence": 0.46309241565115256,
 "accuracy": 0.42592592592592593
 },
 {
 "bin": "0.5-0.6",
 "count": 87,
 "confidence": 0.5474811452572101,
 "accuracy": 0.5402298850574713
 },
 {
 "bin": "0.6-0.7",
 "count": 85,
 "confidence": 0.6506189554618936,
 "accuracy": 0.5529411764705883
 },
 {
 "bin": "0.7-0.8",
 "count": 126,
 "confidence": 0.7569211789474528,
 "accuracy": 0.6507936507936508
 },
 {
 "bin": "0.8-0.9",
 "count": 161,
 "confidence": 0.8494863628752396,
 "accuracy": 0.6086956521739131
 },
 {
 "bin": "0.9-1.0",
 "count": 218,
 "confidence": 0.9580391061516451,
 "accuracy": 0.8440366972477065
 }
 ],
 "seconds": 1.7002065181732178,
 "model": "runs/assay-0.6b-v4",
 "data": "data/suites/kev-transfer-v4-dev.jsonl"
}