peyterho commited on
Commit
ecfd1e5
·
verified ·
1 Parent(s): 1797ac0

Add out-of-domain evaluation results

Browse files
Files changed (1) hide show
  1. eval_ood_results.json +33 -0
eval_ood_results.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "description": "Out-of-domain evaluation on datasets NOT in the training mix",
3
+ "stock_news_ood": {
4
+ "dataset": "ic-fspml/stock_news_sentiment",
5
+ "n_samples": 30150,
6
+ "description": "Real stock news headlines, 5-class mapped to 3-class",
7
+ "label_distribution": {"negative": 6148, "neutral": 10787, "positive": 13215},
8
+ "results": {
9
+ "finbert_finetuned": {"accuracy": 0.6781, "f1_macro": 0.6765, "f1_weighted": 0.6770},
10
+ "roberta_large_finetuned": {"accuracy": 0.7211, "f1_macro": 0.7265, "f1_weighted": 0.7221},
11
+ "climatebert_finetuned": {"accuracy": 0.6472, "f1_macro": 0.6441, "f1_weighted": 0.6448}
12
+ }
13
+ },
14
+ "jb_phrasebank_ood": {
15
+ "dataset": "Jean-Baptiste/financial_news_sentiment_mixte_with_phrasebank_75",
16
+ "n_samples": 785,
17
+ "description": "Mixed PhraseBank (75% agreement) + financial news articles",
18
+ "label_distribution": {"negative": 79, "neutral": 481, "positive": 225},
19
+ "results": {
20
+ "finbert_finetuned": {"accuracy": 0.9236, "f1_macro": 0.9134, "f1_weighted": 0.9228},
21
+ "roberta_large_finetuned": {"accuracy": 0.9414, "f1_macro": 0.9357, "f1_weighted": 0.9414},
22
+ "climatebert_finetuned": {"accuracy": 0.9248, "f1_macro": 0.9213, "f1_weighted": 0.9241}
23
+ }
24
+ },
25
+ "in_domain_reference": {
26
+ "n_samples": 4333,
27
+ "results": {
28
+ "finbert_finetuned": {"accuracy": 0.8973, "f1_macro": 0.8813},
29
+ "roberta_large_finetuned": {"accuracy": 0.9130, "f1_macro": 0.9023},
30
+ "climatebert_finetuned": {"accuracy": 0.8885, "f1_macro": 0.8716}
31
+ }
32
+ }
33
+ }