File size: 1,543 Bytes
ecfd1e5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | {
"description": "Out-of-domain evaluation on datasets NOT in the training mix",
"stock_news_ood": {
"dataset": "ic-fspml/stock_news_sentiment",
"n_samples": 30150,
"description": "Real stock news headlines, 5-class mapped to 3-class",
"label_distribution": {"negative": 6148, "neutral": 10787, "positive": 13215},
"results": {
"finbert_finetuned": {"accuracy": 0.6781, "f1_macro": 0.6765, "f1_weighted": 0.6770},
"roberta_large_finetuned": {"accuracy": 0.7211, "f1_macro": 0.7265, "f1_weighted": 0.7221},
"climatebert_finetuned": {"accuracy": 0.6472, "f1_macro": 0.6441, "f1_weighted": 0.6448}
}
},
"jb_phrasebank_ood": {
"dataset": "Jean-Baptiste/financial_news_sentiment_mixte_with_phrasebank_75",
"n_samples": 785,
"description": "Mixed PhraseBank (75% agreement) + financial news articles",
"label_distribution": {"negative": 79, "neutral": 481, "positive": 225},
"results": {
"finbert_finetuned": {"accuracy": 0.9236, "f1_macro": 0.9134, "f1_weighted": 0.9228},
"roberta_large_finetuned": {"accuracy": 0.9414, "f1_macro": 0.9357, "f1_weighted": 0.9414},
"climatebert_finetuned": {"accuracy": 0.9248, "f1_macro": 0.9213, "f1_weighted": 0.9241}
}
},
"in_domain_reference": {
"n_samples": 4333,
"results": {
"finbert_finetuned": {"accuracy": 0.8973, "f1_macro": 0.8813},
"roberta_large_finetuned": {"accuracy": 0.9130, "f1_macro": 0.9023},
"climatebert_finetuned": {"accuracy": 0.8885, "f1_macro": 0.8716}
}
}
}
|