| --- |
| model-index: |
| - name: no_model_name_available |
| results: |
| - dataset: |
| config: default |
| name: MTEB AFQMC (default) |
| revision: b44c3b011063adb25877c13823db83bb193913c4 |
| split: validation |
| type: C-MTEB/AFQMC |
| metrics: |
| - type: cosine_pearson |
| value: 31.60219205269865 |
| - type: cosine_spearman |
| value: 32.26566089398552 |
| - type: euclidean_pearson |
| value: 31.38659295608159 |
| - type: euclidean_spearman |
| value: 32.265680997074284 |
| - type: main_score |
| value: 32.26566089398552 |
| - type: manhattan_pearson |
| value: 31.012318343485934 |
| - type: manhattan_spearman |
| value: 31.881347232593882 |
| - type: pearson |
| value: 31.60219205269865 |
| - type: spearman |
| value: 32.26566089398552 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB ATEC (default) |
| revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 |
| split: test |
| type: C-MTEB/ATEC |
| metrics: |
| - type: cosine_pearson |
| value: 40.89963324512739 |
| - type: cosine_spearman |
| value: 40.342262626966686 |
| - type: euclidean_pearson |
| value: 43.26579075620696 |
| - type: euclidean_spearman |
| value: 40.34226375259283 |
| - type: main_score |
| value: 40.342262626966686 |
| - type: manhattan_pearson |
| value: 43.09428997760782 |
| - type: manhattan_spearman |
| value: 40.132604575720485 |
| - type: pearson |
| value: 40.89963324512739 |
| - type: spearman |
| value: 40.342262626966686 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB ATEC (default) |
| revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 |
| split: validation |
| type: C-MTEB/ATEC |
| metrics: |
| - type: cosine_pearson |
| value: 40.97674579633659 |
| - type: cosine_spearman |
| value: 41.15073385665892 |
| - type: euclidean_pearson |
| value: 43.12674145119401 |
| - type: euclidean_spearman |
| value: 41.15073497290901 |
| - type: main_score |
| value: 41.15073385665892 |
| - type: manhattan_pearson |
| value: 43.016332350517416 |
| - type: manhattan_spearman |
| value: 40.99128368771293 |
| - type: pearson |
| value: 40.97674579633659 |
| - type: spearman |
| value: 41.15073385665892 |
| task: |
| type: STS |
| - dataset: |
| config: en |
| name: MTEB AmazonCounterfactualClassification (en) |
| revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
| split: test |
| type: mteb/amazon_counterfactual |
| metrics: |
| - type: accuracy |
| value: 74.1492537313433 |
| - type: ap |
| value: 36.58820102143676 |
| - type: ap_weighted |
| value: 36.58820102143676 |
| - type: f1 |
| value: 67.93641050300623 |
| - type: f1_weighted |
| value: 76.47946936836382 |
| - type: main_score |
| value: 74.1492537313433 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB AmazonPolarityClassification (default) |
| revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
| split: test |
| type: mteb/amazon_polarity |
| metrics: |
| - type: accuracy |
| value: 92.57937499999998 |
| - type: ap |
| value: 89.09881932276382 |
| - type: ap_weighted |
| value: 89.09881932276382 |
| - type: f1 |
| value: 92.57389464257594 |
| - type: f1_weighted |
| value: 92.57389464257594 |
| - type: main_score |
| value: 92.57937499999998 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB AmazonReviewsClassification (en) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: test |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 47.95399999999999 |
| - type: f1 |
| value: 45.23480325168402 |
| - type: f1_weighted |
| value: 45.23480325168403 |
| - type: main_score |
| value: 47.95399999999999 |
| task: |
| type: Classification |
| - dataset: |
| config: zh |
| name: MTEB AmazonReviewsClassification (zh) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: test |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 43.916000000000004 |
| - type: f1 |
| value: 40.79038102586015 |
| - type: f1_weighted |
| value: 40.79038102586015 |
| - type: main_score |
| value: 43.916000000000004 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB AmazonReviewsClassification (en) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: validation |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 47.716 |
| - type: f1 |
| value: 44.97469896514136 |
| - type: f1_weighted |
| value: 44.97469896514136 |
| - type: main_score |
| value: 47.716 |
| task: |
| type: Classification |
| - dataset: |
| config: zh |
| name: MTEB AmazonReviewsClassification (zh) |
| revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
| split: validation |
| type: mteb/amazon_reviews_multi |
| metrics: |
| - type: accuracy |
| value: 43.016000000000005 |
| - type: f1 |
| value: 39.88062282479835 |
| - type: f1_weighted |
| value: 39.88062282479835 |
| - type: main_score |
| value: 43.016000000000005 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB ArguAna (default) |
| revision: c22ab2a51041ffd869aaddef7af8d8215647e41a |
| split: test |
| type: mteb/arguana |
| metrics: |
| - type: main_score |
| value: 55.97299999999999 |
| - type: map_at_1 |
| value: 31.009999999999998 |
| - type: map_at_10 |
| value: 46.951 |
| - type: map_at_100 |
| value: 47.788000000000004 |
| - type: map_at_1000 |
| value: 47.794 |
| - type: map_at_20 |
| value: 47.656 |
| - type: map_at_3 |
| value: 41.69 |
| - type: map_at_5 |
| value: 44.795 |
| - type: mrr_at_1 |
| value: 31.57894736842105 |
| - type: mrr_at_10 |
| value: 47.150336426652245 |
| - type: mrr_at_100 |
| value: 48.00003421265431 |
| - type: mrr_at_1000 |
| value: 48.006517491673485 |
| - type: mrr_at_20 |
| value: 47.86823495425013 |
| - type: mrr_at_3 |
| value: 41.90374585111427 |
| - type: mrr_at_5 |
| value: 45.00474158368897 |
| - type: nauc_map_at_1000_diff1 |
| value: 14.400156277962079 |
| - type: nauc_map_at_1000_max |
| value: -6.074701279893042 |
| - type: nauc_map_at_1000_std |
| value: -12.047730490841793 |
| - type: nauc_map_at_100_diff1 |
| value: 14.400167976253817 |
| - type: nauc_map_at_100_max |
| value: -6.0697710559623825 |
| - type: nauc_map_at_100_std |
| value: -12.03623231778573 |
| - type: nauc_map_at_10_diff1 |
| value: 14.39390977335818 |
| - type: nauc_map_at_10_max |
| value: -5.937292882369333 |
| - type: nauc_map_at_10_std |
| value: -11.955448521986341 |
| - type: nauc_map_at_1_diff1 |
| value: 18.2188090059407 |
| - type: nauc_map_at_1_max |
| value: -6.90680836409332 |
| - type: nauc_map_at_1_std |
| value: -11.42044016086847 |
| - type: nauc_map_at_20_diff1 |
| value: 14.25797265657041 |
| - type: nauc_map_at_20_max |
| value: -6.136254023725178 |
| - type: nauc_map_at_20_std |
| value: -12.095812481204513 |
| - type: nauc_map_at_3_diff1 |
| value: 14.694055542759067 |
| - type: nauc_map_at_3_max |
| value: -5.922208526639951 |
| - type: nauc_map_at_3_std |
| value: -12.637146606706324 |
| - type: nauc_map_at_5_diff1 |
| value: 14.034909746881796 |
| - type: nauc_map_at_5_max |
| value: -6.037648673220035 |
| - type: nauc_map_at_5_std |
| value: -12.488119466760367 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 12.907349893032888 |
| - type: nauc_mrr_at_1000_max |
| value: -6.476631933744489 |
| - type: nauc_mrr_at_1000_std |
| value: -12.135655638319898 |
| - type: nauc_mrr_at_100_diff1 |
| value: 12.90767904668398 |
| - type: nauc_mrr_at_100_max |
| value: -6.471625560815013 |
| - type: nauc_mrr_at_100_std |
| value: -12.124160525865376 |
| - type: nauc_mrr_at_10_diff1 |
| value: 12.898084989549307 |
| - type: nauc_mrr_at_10_max |
| value: -6.371999485392878 |
| - type: nauc_mrr_at_10_std |
| value: -12.060712822104344 |
| - type: nauc_mrr_at_1_diff1 |
| value: 16.534028417854632 |
| - type: nauc_mrr_at_1_max |
| value: -6.531221880816804 |
| - type: nauc_mrr_at_1_std |
| value: -11.427032725801363 |
| - type: nauc_mrr_at_20_diff1 |
| value: 12.772149932536516 |
| - type: nauc_mrr_at_20_max |
| value: -6.536237532046593 |
| - type: nauc_mrr_at_20_std |
| value: -12.18322445801735 |
| - type: nauc_mrr_at_3_diff1 |
| value: 13.294722540439723 |
| - type: nauc_mrr_at_3_max |
| value: -6.270285589254632 |
| - type: nauc_mrr_at_3_std |
| value: -12.590739373950477 |
| - type: nauc_mrr_at_5_diff1 |
| value: 12.701572066028916 |
| - type: nauc_mrr_at_5_max |
| value: -6.35025779804965 |
| - type: nauc_mrr_at_5_std |
| value: -12.567997847961006 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 14.04477346308097 |
| - type: nauc_ndcg_at_1000_max |
| value: -5.805803656284627 |
| - type: nauc_ndcg_at_1000_std |
| value: -11.903389341799974 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 14.046024694124535 |
| - type: nauc_ndcg_at_100_max |
| value: -5.638595406841976 |
| - type: nauc_ndcg_at_100_std |
| value: -11.563718937605266 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 13.774482728152659 |
| - type: nauc_ndcg_at_10_max |
| value: -5.112671934691593 |
| - type: nauc_ndcg_at_10_std |
| value: -11.45598979914733 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 18.2188090059407 |
| - type: nauc_ndcg_at_1_max |
| value: -6.90680836409332 |
| - type: nauc_ndcg_at_1_std |
| value: -11.42044016086847 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 13.19308743032763 |
| - type: nauc_ndcg_at_20_max |
| value: -5.925869069550241 |
| - type: nauc_ndcg_at_20_std |
| value: -12.002174058926709 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 14.098445595476438 |
| - type: nauc_ndcg_at_3_max |
| value: -5.438990657735945 |
| - type: nauc_ndcg_at_3_std |
| value: -13.026198448199588 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 12.887695825204021 |
| - type: nauc_ndcg_at_5_max |
| value: -5.527892954283733 |
| - type: nauc_ndcg_at_5_std |
| value: -12.79674424315614 |
| - type: nauc_precision_at_1000_diff1 |
| value: 15.720975272424962 |
| - type: nauc_precision_at_1000_max |
| value: -9.434922353859656 |
| - type: nauc_precision_at_1000_std |
| value: -12.201774463835351 |
| - type: nauc_precision_at_100_diff1 |
| value: 14.822568320368415 |
| - type: nauc_precision_at_100_max |
| value: 16.970591395955335 |
| - type: nauc_precision_at_100_std |
| value: 34.44303415297543 |
| - type: nauc_precision_at_10_diff1 |
| value: 10.924572747165758 |
| - type: nauc_precision_at_10_max |
| value: 0.7245336905113386 |
| - type: nauc_precision_at_10_std |
| value: -7.246984906362029 |
| - type: nauc_precision_at_1_diff1 |
| value: 18.2188090059407 |
| - type: nauc_precision_at_1_max |
| value: -6.90680836409332 |
| - type: nauc_precision_at_1_std |
| value: -11.42044016086847 |
| - type: nauc_precision_at_20_diff1 |
| value: -3.338584460694707 |
| - type: nauc_precision_at_20_max |
| value: -4.566280243136391 |
| - type: nauc_precision_at_20_std |
| value: -10.006136097038183 |
| - type: nauc_precision_at_3_diff1 |
| value: 12.491306916226456 |
| - type: nauc_precision_at_3_max |
| value: -3.939014391748743 |
| - type: nauc_precision_at_3_std |
| value: -14.18952698929006 |
| - type: nauc_precision_at_5_diff1 |
| value: 8.856000600248196 |
| - type: nauc_precision_at_5_max |
| value: -3.5855091847389 |
| - type: nauc_precision_at_5_std |
| value: -13.869699312071923 |
| - type: nauc_recall_at_1000_diff1 |
| value: 15.720975272417975 |
| - type: nauc_recall_at_1000_max |
| value: -9.434922353860903 |
| - type: nauc_recall_at_1000_std |
| value: -12.201774463832038 |
| - type: nauc_recall_at_100_diff1 |
| value: 14.822568320369559 |
| - type: nauc_recall_at_100_max |
| value: 16.970591395954745 |
| - type: nauc_recall_at_100_std |
| value: 34.443034152975024 |
| - type: nauc_recall_at_10_diff1 |
| value: 10.924572747165762 |
| - type: nauc_recall_at_10_max |
| value: 0.724533690511315 |
| - type: nauc_recall_at_10_std |
| value: -7.246984906362018 |
| - type: nauc_recall_at_1_diff1 |
| value: 18.2188090059407 |
| - type: nauc_recall_at_1_max |
| value: -6.90680836409332 |
| - type: nauc_recall_at_1_std |
| value: -11.42044016086847 |
| - type: nauc_recall_at_20_diff1 |
| value: -3.3385844606947677 |
| - type: nauc_recall_at_20_max |
| value: -4.566280243136629 |
| - type: nauc_recall_at_20_std |
| value: -10.006136097038366 |
| - type: nauc_recall_at_3_diff1 |
| value: 12.491306916226472 |
| - type: nauc_recall_at_3_max |
| value: -3.939014391748735 |
| - type: nauc_recall_at_3_std |
| value: -14.189526989290059 |
| - type: nauc_recall_at_5_diff1 |
| value: 8.856000600248263 |
| - type: nauc_recall_at_5_max |
| value: -3.5855091847388603 |
| - type: nauc_recall_at_5_std |
| value: -13.869699312071909 |
| - type: ndcg_at_1 |
| value: 31.009999999999998 |
| - type: ndcg_at_10 |
| value: 55.97299999999999 |
| - type: ndcg_at_100 |
| value: 59.272000000000006 |
| - type: ndcg_at_1000 |
| value: 59.407 |
| - type: ndcg_at_20 |
| value: 58.449 |
| - type: ndcg_at_3 |
| value: 45.227000000000004 |
| - type: ndcg_at_5 |
| value: 50.792 |
| - type: precision_at_1 |
| value: 31.009999999999998 |
| - type: precision_at_10 |
| value: 8.485 |
| - type: precision_at_100 |
| value: 0.9860000000000001 |
| - type: precision_at_1000 |
| value: 0.1 |
| - type: precision_at_20 |
| value: 4.723 |
| - type: precision_at_3 |
| value: 18.492 |
| - type: precision_at_5 |
| value: 13.783999999999999 |
| - type: recall_at_1 |
| value: 31.009999999999998 |
| - type: recall_at_10 |
| value: 84.851 |
| - type: recall_at_100 |
| value: 98.649 |
| - type: recall_at_1000 |
| value: 99.644 |
| - type: recall_at_20 |
| value: 94.452 |
| - type: recall_at_3 |
| value: 55.477 |
| - type: recall_at_5 |
| value: 68.919 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB ArxivClusteringP2P (default) |
| revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
| split: test |
| type: mteb/arxiv-clustering-p2p |
| metrics: |
| - type: main_score |
| value: 48.31683216128774 |
| - type: v_measure |
| value: 48.31683216128774 |
| - type: v_measure_std |
| value: 13.795207109799703 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB ArxivClusteringS2S (default) |
| revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
| split: test |
| type: mteb/arxiv-clustering-s2s |
| metrics: |
| - type: main_score |
| value: 40.2951016935384 |
| - type: v_measure |
| value: 40.2951016935384 |
| - type: v_measure_std |
| value: 14.193710444297869 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB AskUbuntuDupQuestions (default) |
| revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
| split: test |
| type: mteb/askubuntudupquestions-reranking |
| metrics: |
| - type: main_score |
| value: 60.45095169935259 |
| - type: map |
| value: 60.45095169935259 |
| - type: mrr |
| value: 73.43567251461988 |
| - type: nAUC_map_diff1 |
| value: 15.357222913791704 |
| - type: nAUC_map_max |
| value: 24.301239659848346 |
| - type: nAUC_map_std |
| value: 18.26732583044278 |
| - type: nAUC_mrr_diff1 |
| value: 24.108010981589057 |
| - type: nAUC_mrr_max |
| value: 34.90261214387396 |
| - type: nAUC_mrr_std |
| value: 20.350034497982126 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB BIOSSES (default) |
| revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
| split: test |
| type: mteb/biosses-sts |
| metrics: |
| - type: cosine_pearson |
| value: 90.16604991710759 |
| - type: cosine_spearman |
| value: 88.4670760233051 |
| - type: euclidean_pearson |
| value: 89.02378164860428 |
| - type: euclidean_spearman |
| value: 88.4670760233051 |
| - type: main_score |
| value: 88.4670760233051 |
| - type: manhattan_pearson |
| value: 88.8866912507422 |
| - type: manhattan_spearman |
| value: 88.2755053931781 |
| - type: pearson |
| value: 90.16604991710759 |
| - type: spearman |
| value: 88.4670760233051 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB BQ (default) |
| revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 |
| split: test |
| type: C-MTEB/BQ |
| metrics: |
| - type: cosine_pearson |
| value: 49.45233904713201 |
| - type: cosine_spearman |
| value: 49.77342815602789 |
| - type: euclidean_pearson |
| value: 49.13579036236359 |
| - type: euclidean_spearman |
| value: 49.77342122767529 |
| - type: main_score |
| value: 49.77342815602789 |
| - type: manhattan_pearson |
| value: 49.01322677955527 |
| - type: manhattan_spearman |
| value: 49.702538779772226 |
| - type: pearson |
| value: 49.45233904713201 |
| - type: spearman |
| value: 49.77342815602789 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB BQ (default) |
| revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 |
| split: validation |
| type: C-MTEB/BQ |
| metrics: |
| - type: cosine_pearson |
| value: 53.43473222697715 |
| - type: cosine_spearman |
| value: 54.24325202324013 |
| - type: euclidean_pearson |
| value: 53.4053341221681 |
| - type: euclidean_spearman |
| value: 54.2432485591385 |
| - type: main_score |
| value: 54.24325202324013 |
| - type: manhattan_pearson |
| value: 53.31602762068146 |
| - type: manhattan_spearman |
| value: 54.180811590825925 |
| - type: pearson |
| value: 53.43473222697715 |
| - type: spearman |
| value: 54.24325202324013 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB Banking77Classification (default) |
| revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
| split: test |
| type: mteb/banking77 |
| metrics: |
| - type: accuracy |
| value: 82.11038961038962 |
| - type: f1 |
| value: 81.50275371635729 |
| - type: f1_weighted |
| value: 81.50275371635732 |
| - type: main_score |
| value: 82.11038961038962 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB BiorxivClusteringP2P (default) |
| revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
| split: test |
| type: mteb/biorxiv-clustering-p2p |
| metrics: |
| - type: main_score |
| value: 39.85718105201471 |
| - type: v_measure |
| value: 39.85718105201471 |
| - type: v_measure_std |
| value: 0.9098592525717781 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB BiorxivClusteringS2S (default) |
| revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
| split: test |
| type: mteb/biorxiv-clustering-s2s |
| metrics: |
| - type: main_score |
| value: 34.901371726743854 |
| - type: v_measure |
| value: 34.901371726743854 |
| - type: v_measure_std |
| value: 0.49131958662099773 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB CLSClusteringP2P (default) |
| revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 |
| split: test |
| type: C-MTEB/CLSClusteringP2P |
| metrics: |
| - type: main_score |
| value: 42.580911514601844 |
| - type: v_measure |
| value: 42.580911514601844 |
| - type: v_measure_std |
| value: 1.3262494874619402 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB CLSClusteringS2S (default) |
| revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f |
| split: test |
| type: C-MTEB/CLSClusteringS2S |
| metrics: |
| - type: main_score |
| value: 38.36369670561906 |
| - type: v_measure |
| value: 38.36369670561906 |
| - type: v_measure_std |
| value: 1.3030031287521193 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB CMedQAv1-reranking (default) |
| revision: 8d7f1e942507dac42dc58017c1a001c3717da7df |
| split: test |
| type: C-MTEB/CMedQAv1-reranking |
| metrics: |
| - type: main_score |
| value: 82.23318409776884 |
| - type: map |
| value: 82.23318409776884 |
| - type: mrr |
| value: 85.05289682539681 |
| - type: nAUC_map_diff1 |
| value: 53.922817335441664 |
| - type: nAUC_map_max |
| value: 63.38587877583035 |
| - type: nAUC_map_std |
| value: 26.58945323149115 |
| - type: nAUC_mrr_diff1 |
| value: 61.2457871312172 |
| - type: nAUC_mrr_max |
| value: 71.77558608272952 |
| - type: nAUC_mrr_std |
| value: 35.945961549335976 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB CMedQAv2-reranking (default) |
| revision: 23d186750531a14a0357ca22cd92d712fd512ea0 |
| split: test |
| type: C-MTEB/CMedQAv2-reranking |
| metrics: |
| - type: main_score |
| value: 83.28208766373744 |
| - type: map |
| value: 83.28208766373744 |
| - type: mrr |
| value: 85.81444444444443 |
| - type: nAUC_map_diff1 |
| value: 59.23043241198723 |
| - type: nAUC_map_max |
| value: 63.96198552688328 |
| - type: nAUC_map_std |
| value: 17.563221080927807 |
| - type: nAUC_mrr_diff1 |
| value: 66.27403933527562 |
| - type: nAUC_mrr_max |
| value: 74.24319995478142 |
| - type: nAUC_mrr_std |
| value: 26.84913877864022 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB CQADupstackAndroidRetrieval (default) |
| revision: f46a197baaae43b4f621051089b82a364682dfeb |
| split: test |
| type: mteb/cqadupstack-android |
| metrics: |
| - type: main_score |
| value: 51.791 |
| - type: map_at_1 |
| value: 33.489000000000004 |
| - type: map_at_10 |
| value: 45.362 |
| - type: map_at_100 |
| value: 46.847 |
| - type: map_at_1000 |
| value: 46.963 |
| - type: map_at_20 |
| value: 46.167 |
| - type: map_at_3 |
| value: 41.737 |
| - type: map_at_5 |
| value: 43.747 |
| - type: mrr_at_1 |
| value: 40.486409155937054 |
| - type: mrr_at_10 |
| value: 51.12570111497148 |
| - type: mrr_at_100 |
| value: 51.86187493461626 |
| - type: mrr_at_1000 |
| value: 51.89536424646558 |
| - type: mrr_at_20 |
| value: 51.54190377431117 |
| - type: mrr_at_3 |
| value: 48.56938483547925 |
| - type: mrr_at_5 |
| value: 50.171673819742466 |
| - type: nauc_map_at_1000_diff1 |
| value: 45.83742367768875 |
| - type: nauc_map_at_1000_max |
| value: 36.666030418631365 |
| - type: nauc_map_at_1000_std |
| value: -3.0749754490409598 |
| - type: nauc_map_at_100_diff1 |
| value: 45.81723006290297 |
| - type: nauc_map_at_100_max |
| value: 36.669471954500835 |
| - type: nauc_map_at_100_std |
| value: -3.0711605055120037 |
| - type: nauc_map_at_10_diff1 |
| value: 46.11671975824962 |
| - type: nauc_map_at_10_max |
| value: 36.41961760572779 |
| - type: nauc_map_at_10_std |
| value: -3.5676307490322294 |
| - type: nauc_map_at_1_diff1 |
| value: 48.99600869130432 |
| - type: nauc_map_at_1_max |
| value: 30.72533190025592 |
| - type: nauc_map_at_1_std |
| value: -7.210226805142472 |
| - type: nauc_map_at_20_diff1 |
| value: 45.730620597411416 |
| - type: nauc_map_at_20_max |
| value: 36.67067673690639 |
| - type: nauc_map_at_20_std |
| value: -3.0616760792842874 |
| - type: nauc_map_at_3_diff1 |
| value: 46.3900637210476 |
| - type: nauc_map_at_3_max |
| value: 35.04691686861482 |
| - type: nauc_map_at_3_std |
| value: -4.855804907542516 |
| - type: nauc_map_at_5_diff1 |
| value: 46.30354693063511 |
| - type: nauc_map_at_5_max |
| value: 36.160207495289946 |
| - type: nauc_map_at_5_std |
| value: -3.7612546075044024 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 44.94342955084924 |
| - type: nauc_mrr_at_1000_max |
| value: 36.5868635648845 |
| - type: nauc_mrr_at_1000_std |
| value: -3.7279540299450598 |
| - type: nauc_mrr_at_100_diff1 |
| value: 44.9241145632844 |
| - type: nauc_mrr_at_100_max |
| value: 36.58379839831864 |
| - type: nauc_mrr_at_100_std |
| value: -3.7418032288649385 |
| - type: nauc_mrr_at_10_diff1 |
| value: 45.00805694123448 |
| - type: nauc_mrr_at_10_max |
| value: 36.705567574937454 |
| - type: nauc_mrr_at_10_std |
| value: -3.602116114964355 |
| - type: nauc_mrr_at_1_diff1 |
| value: 47.14298489978003 |
| - type: nauc_mrr_at_1_max |
| value: 33.38843521905287 |
| - type: nauc_mrr_at_1_std |
| value: -8.505210257231145 |
| - type: nauc_mrr_at_20_diff1 |
| value: 44.83329863262661 |
| - type: nauc_mrr_at_20_max |
| value: 36.589698139628496 |
| - type: nauc_mrr_at_20_std |
| value: -3.620200313971379 |
| - type: nauc_mrr_at_3_diff1 |
| value: 44.95899691734053 |
| - type: nauc_mrr_at_3_max |
| value: 36.61014661536669 |
| - type: nauc_mrr_at_3_std |
| value: -4.235751267084451 |
| - type: nauc_mrr_at_5_diff1 |
| value: 45.43301143912572 |
| - type: nauc_mrr_at_5_max |
| value: 37.016764711532716 |
| - type: nauc_mrr_at_5_std |
| value: -3.7811565499003232 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 44.56347509930279 |
| - type: nauc_ndcg_at_1000_max |
| value: 37.58231608565612 |
| - type: nauc_ndcg_at_1000_std |
| value: -1.0148805105229683 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 44.21798254097979 |
| - type: nauc_ndcg_at_100_max |
| value: 37.55836639241636 |
| - type: nauc_ndcg_at_100_std |
| value: -1.119038291236023 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 44.77884245032202 |
| - type: nauc_ndcg_at_10_max |
| value: 37.800051548342246 |
| - type: nauc_ndcg_at_10_std |
| value: -1.48841695838196 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 47.14298489978003 |
| - type: nauc_ndcg_at_1_max |
| value: 33.38843521905287 |
| - type: nauc_ndcg_at_1_std |
| value: -8.505210257231145 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 43.65031596123121 |
| - type: nauc_ndcg_at_20_max |
| value: 37.69836062122585 |
| - type: nauc_ndcg_at_20_std |
| value: -0.8253052163035528 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 45.00478060029277 |
| - type: nauc_ndcg_at_3_max |
| value: 36.75297532264166 |
| - type: nauc_ndcg_at_3_std |
| value: -3.0054585641131655 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 45.24437062894877 |
| - type: nauc_ndcg_at_5_max |
| value: 37.88266316994465 |
| - type: nauc_ndcg_at_5_std |
| value: -1.701786097430671 |
| - type: nauc_precision_at_1000_diff1 |
| value: -11.911798432587343 |
| - type: nauc_precision_at_1000_max |
| value: -10.189977280120303 |
| - type: nauc_precision_at_1000_std |
| value: -5.213316467405967 |
| - type: nauc_precision_at_100_diff1 |
| value: -6.795008520695643 |
| - type: nauc_precision_at_100_max |
| value: 1.308872758510908 |
| - type: nauc_precision_at_100_std |
| value: 3.1390422505657627 |
| - type: nauc_precision_at_10_diff1 |
| value: 12.648590902867074 |
| - type: nauc_precision_at_10_max |
| value: 24.68660171555869 |
| - type: nauc_precision_at_10_std |
| value: 7.893487447107204 |
| - type: nauc_precision_at_1_diff1 |
| value: 47.14298489978003 |
| - type: nauc_precision_at_1_max |
| value: 33.38843521905287 |
| - type: nauc_precision_at_1_std |
| value: -8.505210257231145 |
| - type: nauc_precision_at_20_diff1 |
| value: 2.7434758735468048 |
| - type: nauc_precision_at_20_max |
| value: 17.55565926646876 |
| - type: nauc_precision_at_20_std |
| value: 10.321439048951452 |
| - type: nauc_precision_at_3_diff1 |
| value: 29.566919929400875 |
| - type: nauc_precision_at_3_max |
| value: 33.95479571575024 |
| - type: nauc_precision_at_3_std |
| value: 1.7592238216915597 |
| - type: nauc_precision_at_5_diff1 |
| value: 22.428208270307856 |
| - type: nauc_precision_at_5_max |
| value: 31.004215116158413 |
| - type: nauc_precision_at_5_std |
| value: 5.279489297223801 |
| - type: nauc_recall_at_1000_diff1 |
| value: 31.890454093099407 |
| - type: nauc_recall_at_1000_max |
| value: 51.376825921063386 |
| - type: nauc_recall_at_1000_std |
| value: 59.90888686683735 |
| - type: nauc_recall_at_100_diff1 |
| value: 31.697335059128505 |
| - type: nauc_recall_at_100_max |
| value: 38.760900054389786 |
| - type: nauc_recall_at_100_std |
| value: 14.477418407176682 |
| - type: nauc_recall_at_10_diff1 |
| value: 37.593976107308166 |
| - type: nauc_recall_at_10_max |
| value: 37.120867787083576 |
| - type: nauc_recall_at_10_std |
| value: 4.0458731062140165 |
| - type: nauc_recall_at_1_diff1 |
| value: 48.99600869130432 |
| - type: nauc_recall_at_1_max |
| value: 30.72533190025592 |
| - type: nauc_recall_at_1_std |
| value: -7.210226805142472 |
| - type: nauc_recall_at_20_diff1 |
| value: 31.75084814121109 |
| - type: nauc_recall_at_20_max |
| value: 36.78465637755701 |
| - type: nauc_recall_at_20_std |
| value: 7.600404385507733 |
| - type: nauc_recall_at_3_diff1 |
| value: 40.91244393504077 |
| - type: nauc_recall_at_3_max |
| value: 35.611100064289175 |
| - type: nauc_recall_at_3_std |
| value: -1.7314625087631257 |
| - type: nauc_recall_at_5_diff1 |
| value: 40.48529204446073 |
| - type: nauc_recall_at_5_max |
| value: 37.96938179146327 |
| - type: nauc_recall_at_5_std |
| value: 2.243463426136501 |
| - type: ndcg_at_1 |
| value: 40.486 |
| - type: ndcg_at_10 |
| value: 51.791 |
| - type: ndcg_at_100 |
| value: 57.218999999999994 |
| - type: ndcg_at_1000 |
| value: 58.846 |
| - type: ndcg_at_20 |
| value: 53.82900000000001 |
| - type: ndcg_at_3 |
| value: 46.727999999999994 |
| - type: ndcg_at_5 |
| value: 49.126 |
| - type: precision_at_1 |
| value: 40.486 |
| - type: precision_at_10 |
| value: 9.771 |
| - type: precision_at_100 |
| value: 1.562 |
| - type: precision_at_1000 |
| value: 0.202 |
| - type: precision_at_20 |
| value: 5.7509999999999994 |
| - type: precision_at_3 |
| value: 22.556 |
| - type: precision_at_5 |
| value: 16.052 |
| - type: recall_at_1 |
| value: 33.489000000000004 |
| - type: recall_at_10 |
| value: 64.071 |
| - type: recall_at_100 |
| value: 86.47500000000001 |
| - type: recall_at_1000 |
| value: 96.408 |
| - type: recall_at_20 |
| value: 71.273 |
| - type: recall_at_3 |
| value: 49.547999999999995 |
| - type: recall_at_5 |
| value: 56.393 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackEnglishRetrieval (default) |
| revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 |
| split: test |
| type: mteb/cqadupstack-english |
| metrics: |
| - type: main_score |
| value: 49.274 |
| - type: map_at_1 |
| value: 33.019 |
| - type: map_at_10 |
| value: 43.469 |
| - type: map_at_100 |
| value: 44.818999999999996 |
| - type: map_at_1000 |
| value: 44.944 |
| - type: map_at_20 |
| value: 44.204 |
| - type: map_at_3 |
| value: 40.215 |
| - type: map_at_5 |
| value: 42.138999999999996 |
| - type: mrr_at_1 |
| value: 41.082802547770704 |
| - type: mrr_at_10 |
| value: 49.50763320190077 |
| - type: mrr_at_100 |
| value: 50.15386440914099 |
| - type: mrr_at_1000 |
| value: 50.1948078078438 |
| - type: mrr_at_20 |
| value: 49.86890003378296 |
| - type: mrr_at_3 |
| value: 47.250530785562646 |
| - type: mrr_at_5 |
| value: 48.65817409766459 |
| - type: nauc_map_at_1000_diff1 |
| value: 53.82535875039235 |
| - type: nauc_map_at_1000_max |
| value: 45.453250348612215 |
| - type: nauc_map_at_1000_std |
| value: -1.9559612984873571 |
| - type: nauc_map_at_100_diff1 |
| value: 53.81013847448271 |
| - type: nauc_map_at_100_max |
| value: 45.392209330066066 |
| - type: nauc_map_at_100_std |
| value: -2.0524451381485234 |
| - type: nauc_map_at_10_diff1 |
| value: 54.209459779949384 |
| - type: nauc_map_at_10_max |
| value: 44.883275752243065 |
| - type: nauc_map_at_10_std |
| value: -3.6109937791207094 |
| - type: nauc_map_at_1_diff1 |
| value: 58.94514805782117 |
| - type: nauc_map_at_1_max |
| value: 39.37520774150509 |
| - type: nauc_map_at_1_std |
| value: -8.720964154916928 |
| - type: nauc_map_at_20_diff1 |
| value: 53.8348887034513 |
| - type: nauc_map_at_20_max |
| value: 44.99782089147465 |
| - type: nauc_map_at_20_std |
| value: -2.718742980010167 |
| - type: nauc_map_at_3_diff1 |
| value: 56.02884388647345 |
| - type: nauc_map_at_3_max |
| value: 43.415666030670124 |
| - type: nauc_map_at_3_std |
| value: -6.731028873830273 |
| - type: nauc_map_at_5_diff1 |
| value: 54.723746443656566 |
| - type: nauc_map_at_5_max |
| value: 44.58690708846215 |
| - type: nauc_map_at_5_std |
| value: -5.030535383171446 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 53.153007923698894 |
| - type: nauc_mrr_at_1000_max |
| value: 47.498466648364534 |
| - type: nauc_mrr_at_1000_std |
| value: 1.2882577043538435 |
| - type: nauc_mrr_at_100_diff1 |
| value: 53.135489251238056 |
| - type: nauc_mrr_at_100_max |
| value: 47.48916134974268 |
| - type: nauc_mrr_at_100_std |
| value: 1.2889395420272438 |
| - type: nauc_mrr_at_10_diff1 |
| value: 53.1220415513986 |
| - type: nauc_mrr_at_10_max |
| value: 47.490791997767964 |
| - type: nauc_mrr_at_10_std |
| value: 1.1444407350516157 |
| - type: nauc_mrr_at_1_diff1 |
| value: 57.559058682171504 |
| - type: nauc_mrr_at_1_max |
| value: 46.89026874220749 |
| - type: nauc_mrr_at_1_std |
| value: -1.9116043469494446 |
| - type: nauc_mrr_at_20_diff1 |
| value: 53.034500689960275 |
| - type: nauc_mrr_at_20_max |
| value: 47.41450821815849 |
| - type: nauc_mrr_at_20_std |
| value: 1.240765437252736 |
| - type: nauc_mrr_at_3_diff1 |
| value: 54.25315882717826 |
| - type: nauc_mrr_at_3_max |
| value: 47.428006007217235 |
| - type: nauc_mrr_at_3_std |
| value: -0.12495431309209105 |
| - type: nauc_mrr_at_5_diff1 |
| value: 53.5054857141475 |
| - type: nauc_mrr_at_5_max |
| value: 47.83146647409837 |
| - type: nauc_mrr_at_5_std |
| value: 0.5629970448268111 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 51.261194449319504 |
| - type: nauc_ndcg_at_1000_max |
| value: 46.994312489862835 |
| - type: nauc_ndcg_at_1000_std |
| value: 3.2428209322165067 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 50.84368410402597 |
| - type: nauc_ndcg_at_100_max |
| value: 46.73298393365377 |
| - type: nauc_ndcg_at_100_std |
| value: 2.904073356585609 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 51.72255521298621 |
| - type: nauc_ndcg_at_10_max |
| value: 46.31005929924904 |
| - type: nauc_ndcg_at_10_std |
| value: 0.2715351422503746 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 57.559058682171504 |
| - type: nauc_ndcg_at_1_max |
| value: 46.89026874220749 |
| - type: nauc_ndcg_at_1_std |
| value: -1.9116043469494446 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 50.8506271301813 |
| - type: nauc_ndcg_at_20_max |
| value: 46.0583706384306 |
| - type: nauc_ndcg_at_20_std |
| value: 1.6396894489539218 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 54.00038574913631 |
| - type: nauc_ndcg_at_3_max |
| value: 46.076178038905404 |
| - type: nauc_ndcg_at_3_std |
| value: -2.211424037505318 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 52.628195775092316 |
| - type: nauc_ndcg_at_5_max |
| value: 46.78093894422556 |
| - type: nauc_ndcg_at_5_std |
| value: -1.3380283106634656 |
| - type: nauc_precision_at_1000_diff1 |
| value: -12.938958862510566 |
| - type: nauc_precision_at_1000_max |
| value: 8.556158319175314 |
| - type: nauc_precision_at_1000_std |
| value: 28.485389071197346 |
| - type: nauc_precision_at_100_diff1 |
| value: -8.770372899573491 |
| - type: nauc_precision_at_100_max |
| value: 18.05611676926777 |
| - type: nauc_precision_at_100_std |
| value: 33.603692427049545 |
| - type: nauc_precision_at_10_diff1 |
| value: 10.17936772396029 |
| - type: nauc_precision_at_10_max |
| value: 33.28847244292926 |
| - type: nauc_precision_at_10_std |
| value: 24.05529615188066 |
| - type: nauc_precision_at_1_diff1 |
| value: 57.559058682171504 |
| - type: nauc_precision_at_1_max |
| value: 46.89026874220749 |
| - type: nauc_precision_at_1_std |
| value: -1.9116043469494446 |
| - type: nauc_precision_at_20_diff1 |
| value: 0.46596639548970015 |
| - type: nauc_precision_at_20_max |
| value: 26.34396955936117 |
| - type: nauc_precision_at_20_std |
| value: 29.960110998616308 |
| - type: nauc_precision_at_3_diff1 |
| value: 32.1884032130926 |
| - type: nauc_precision_at_3_max |
| value: 42.9623864532112 |
| - type: nauc_precision_at_3_std |
| value: 9.406319207236965 |
| - type: nauc_precision_at_5_diff1 |
| value: 20.663922808040514 |
| - type: nauc_precision_at_5_max |
| value: 40.23784932763058 |
| - type: nauc_precision_at_5_std |
| value: 16.15485535812318 |
| - type: nauc_recall_at_1000_diff1 |
| value: 34.02276539506821 |
| - type: nauc_recall_at_1000_max |
| value: 51.78898549190249 |
| - type: nauc_recall_at_1000_std |
| value: 38.51821109938462 |
| - type: nauc_recall_at_100_diff1 |
| value: 35.11970287568031 |
| - type: nauc_recall_at_100_max |
| value: 45.26179169180922 |
| - type: nauc_recall_at_100_std |
| value: 19.468341893615374 |
| - type: nauc_recall_at_10_diff1 |
| value: 42.731604441196666 |
| - type: nauc_recall_at_10_max |
| value: 42.89410379930046 |
| - type: nauc_recall_at_10_std |
| value: 3.5259768753999587 |
| - type: nauc_recall_at_1_diff1 |
| value: 58.94514805782117 |
| - type: nauc_recall_at_1_max |
| value: 39.37520774150509 |
| - type: nauc_recall_at_1_std |
| value: -8.720964154916928 |
| - type: nauc_recall_at_20_diff1 |
| value: 38.6527326827719 |
| - type: nauc_recall_at_20_max |
| value: 41.81381796149285 |
| - type: nauc_recall_at_20_std |
| value: 9.447128423015046 |
| - type: nauc_recall_at_3_diff1 |
| value: 51.06019004682993 |
| - type: nauc_recall_at_3_max |
| value: 42.099338080420274 |
| - type: nauc_recall_at_3_std |
| value: -6.020642288695232 |
| - type: nauc_recall_at_5_diff1 |
| value: 46.14582217531629 |
| - type: nauc_recall_at_5_max |
| value: 43.94158387704093 |
| - type: nauc_recall_at_5_std |
| value: -2.0041618732754696 |
| - type: ndcg_at_1 |
| value: 41.083 |
| - type: ndcg_at_10 |
| value: 49.274 |
| - type: ndcg_at_100 |
| value: 53.835 |
| - type: ndcg_at_1000 |
| value: 55.69499999999999 |
| - type: ndcg_at_20 |
| value: 50.983000000000004 |
| - type: ndcg_at_3 |
| value: 44.912 |
| - type: ndcg_at_5 |
| value: 47.121 |
| - type: precision_at_1 |
| value: 41.083 |
| - type: precision_at_10 |
| value: 9.274000000000001 |
| - type: precision_at_100 |
| value: 1.488 |
| - type: precision_at_1000 |
| value: 0.193 |
| - type: precision_at_20 |
| value: 5.449 |
| - type: precision_at_3 |
| value: 21.741 |
| - type: precision_at_5 |
| value: 15.439 |
| - type: recall_at_1 |
| value: 33.019 |
| - type: recall_at_10 |
| value: 59.294999999999995 |
| - type: recall_at_100 |
| value: 78.545 |
| - type: recall_at_1000 |
| value: 90.12400000000001 |
| - type: recall_at_20 |
| value: 65.443 |
| - type: recall_at_3 |
| value: 46.21 |
| - type: recall_at_5 |
| value: 52.575 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackGamingRetrieval (default) |
| revision: 4885aa143210c98657558c04aaf3dc47cfb54340 |
| split: test |
| type: mteb/cqadupstack-gaming |
| metrics: |
| - type: main_score |
| value: 59.83500000000001 |
| - type: map_at_1 |
| value: 41.743 |
| - type: map_at_10 |
| value: 54.081999999999994 |
| - type: map_at_100 |
| value: 55.135999999999996 |
| - type: map_at_1000 |
| value: 55.184 |
| - type: map_at_20 |
| value: 54.767999999999994 |
| - type: map_at_3 |
| value: 50.89 |
| - type: map_at_5 |
| value: 52.636 |
| - type: mrr_at_1 |
| value: 47.39811912225706 |
| - type: mrr_at_10 |
| value: 57.38179827835008 |
| - type: mrr_at_100 |
| value: 58.01643316296891 |
| - type: mrr_at_1000 |
| value: 58.04110233372705 |
| - type: mrr_at_20 |
| value: 57.82176911544285 |
| - type: mrr_at_3 |
| value: 54.98432601880885 |
| - type: mrr_at_5 |
| value: 56.33542319749226 |
| - type: nauc_map_at_1000_diff1 |
| value: 56.38274182942337 |
| - type: nauc_map_at_1000_max |
| value: 39.63215709105948 |
| - type: nauc_map_at_1000_std |
| value: -6.245907717300131 |
| - type: nauc_map_at_100_diff1 |
| value: 56.36311874132528 |
| - type: nauc_map_at_100_max |
| value: 39.62470000319664 |
| - type: nauc_map_at_100_std |
| value: -6.271622755681494 |
| - type: nauc_map_at_10_diff1 |
| value: 56.410565627073225 |
| - type: nauc_map_at_10_max |
| value: 39.16425951389524 |
| - type: nauc_map_at_10_std |
| value: -7.206521474602716 |
| - type: nauc_map_at_1_diff1 |
| value: 58.34604316308072 |
| - type: nauc_map_at_1_max |
| value: 31.305799393516853 |
| - type: nauc_map_at_1_std |
| value: -9.67195266691713 |
| - type: nauc_map_at_20_diff1 |
| value: 56.38143625487464 |
| - type: nauc_map_at_20_max |
| value: 39.462438789562455 |
| - type: nauc_map_at_20_std |
| value: -6.599407894095691 |
| - type: nauc_map_at_3_diff1 |
| value: 56.90332449245052 |
| - type: nauc_map_at_3_max |
| value: 37.454195451703995 |
| - type: nauc_map_at_3_std |
| value: -9.382786205944821 |
| - type: nauc_map_at_5_diff1 |
| value: 56.538604915661004 |
| - type: nauc_map_at_5_max |
| value: 38.6588144327087 |
| - type: nauc_map_at_5_std |
| value: -7.932442776531816 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 56.1537707758201 |
| - type: nauc_mrr_at_1000_max |
| value: 40.87392514538646 |
| - type: nauc_mrr_at_1000_std |
| value: -5.108268246986718 |
| - type: nauc_mrr_at_100_diff1 |
| value: 56.14434800759561 |
| - type: nauc_mrr_at_100_max |
| value: 40.88497861437684 |
| - type: nauc_mrr_at_100_std |
| value: -5.100160912125043 |
| - type: nauc_mrr_at_10_diff1 |
| value: 56.091546352822434 |
| - type: nauc_mrr_at_10_max |
| value: 41.04917579584731 |
| - type: nauc_mrr_at_10_std |
| value: -5.096011574407418 |
| - type: nauc_mrr_at_1_diff1 |
| value: 58.89486283556674 |
| - type: nauc_mrr_at_1_max |
| value: 36.877138420765164 |
| - type: nauc_mrr_at_1_std |
| value: -8.010727906497483 |
| - type: nauc_mrr_at_20_diff1 |
| value: 56.15532215594925 |
| - type: nauc_mrr_at_20_max |
| value: 40.91911784659166 |
| - type: nauc_mrr_at_20_std |
| value: -5.159856708038148 |
| - type: nauc_mrr_at_3_diff1 |
| value: 56.41304554774757 |
| - type: nauc_mrr_at_3_max |
| value: 40.599408683012975 |
| - type: nauc_mrr_at_3_std |
| value: -5.966503192813791 |
| - type: nauc_mrr_at_5_diff1 |
| value: 56.178462641991004 |
| - type: nauc_mrr_at_5_max |
| value: 40.88639915714814 |
| - type: nauc_mrr_at_5_std |
| value: -5.4712972818244205 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 55.46084562015493 |
| - type: nauc_ndcg_at_1000_max |
| value: 42.11339231750283 |
| - type: nauc_ndcg_at_1000_std |
| value: -2.933574308921646 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 55.244408030279644 |
| - type: nauc_ndcg_at_100_max |
| value: 42.51902459556891 |
| - type: nauc_ndcg_at_100_std |
| value: -2.681903058600699 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 55.07975132155747 |
| - type: nauc_ndcg_at_10_max |
| value: 41.86638367277626 |
| - type: nauc_ndcg_at_10_std |
| value: -4.574212407886393 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 58.89486283556674 |
| - type: nauc_ndcg_at_1_max |
| value: 36.877138420765164 |
| - type: nauc_ndcg_at_1_std |
| value: -8.010727906497483 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 55.239108306400865 |
| - type: nauc_ndcg_at_20_max |
| value: 42.19784330055704 |
| - type: nauc_ndcg_at_20_std |
| value: -3.690456034599944 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 56.094939697467325 |
| - type: nauc_ndcg_at_3_max |
| value: 39.75116550436197 |
| - type: nauc_ndcg_at_3_std |
| value: -7.375673693822571 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 55.377651199567794 |
| - type: nauc_ndcg_at_5_max |
| value: 41.20722954879245 |
| - type: nauc_ndcg_at_5_std |
| value: -5.679020392514973 |
| - type: nauc_precision_at_1000_diff1 |
| value: -10.756112623603697 |
| - type: nauc_precision_at_1000_max |
| value: 17.64732842181831 |
| - type: nauc_precision_at_1000_std |
| value: 32.742279334654306 |
| - type: nauc_precision_at_100_diff1 |
| value: -4.896852655342983 |
| - type: nauc_precision_at_100_max |
| value: 24.707372714988725 |
| - type: nauc_precision_at_100_std |
| value: 32.19414457350063 |
| - type: nauc_precision_at_10_diff1 |
| value: 16.228966073160773 |
| - type: nauc_precision_at_10_max |
| value: 35.39971659325401 |
| - type: nauc_precision_at_10_std |
| value: 15.975657844520837 |
| - type: nauc_precision_at_1_diff1 |
| value: 58.89486283556674 |
| - type: nauc_precision_at_1_max |
| value: 36.877138420765164 |
| - type: nauc_precision_at_1_std |
| value: -8.010727906497483 |
| - type: nauc_precision_at_20_diff1 |
| value: 6.765510087471395 |
| - type: nauc_precision_at_20_max |
| value: 31.77369794420453 |
| - type: nauc_precision_at_20_std |
| value: 24.487726333260845 |
| - type: nauc_precision_at_3_diff1 |
| value: 37.01533500883528 |
| - type: nauc_precision_at_3_max |
| value: 40.28829957277282 |
| - type: nauc_precision_at_3_std |
| value: 0.15790828521244832 |
| - type: nauc_precision_at_5_diff1 |
| value: 27.325187065547695 |
| - type: nauc_precision_at_5_max |
| value: 39.67710773459586 |
| - type: nauc_precision_at_5_std |
| value: 8.307845112173677 |
| - type: nauc_recall_at_1000_diff1 |
| value: 34.97259871293003 |
| - type: nauc_recall_at_1000_max |
| value: 73.36153616209499 |
| - type: nauc_recall_at_1000_std |
| value: 63.52466639318273 |
| - type: nauc_recall_at_100_diff1 |
| value: 43.84585939706463 |
| - type: nauc_recall_at_100_max |
| value: 58.75253788214712 |
| - type: nauc_recall_at_100_std |
| value: 23.779812502563956 |
| - type: nauc_recall_at_10_diff1 |
| value: 47.80161773501786 |
| - type: nauc_recall_at_10_max |
| value: 46.2174264798925 |
| - type: nauc_recall_at_10_std |
| value: 0.8663876046028921 |
| - type: nauc_recall_at_1_diff1 |
| value: 58.34604316308072 |
| - type: nauc_recall_at_1_max |
| value: 31.305799393516853 |
| - type: nauc_recall_at_1_std |
| value: -9.67195266691713 |
| - type: nauc_recall_at_20_diff1 |
| value: 46.90388293555046 |
| - type: nauc_recall_at_20_max |
| value: 49.28144135226787 |
| - type: nauc_recall_at_20_std |
| value: 7.537105099790044 |
| - type: nauc_recall_at_3_diff1 |
| value: 52.97073509767745 |
| - type: nauc_recall_at_3_max |
| value: 40.42098227210626 |
| - type: nauc_recall_at_3_std |
| value: -8.28013314935897 |
| - type: nauc_recall_at_5_diff1 |
| value: 50.35991406369175 |
| - type: nauc_recall_at_5_max |
| value: 43.442736162816395 |
| - type: nauc_recall_at_5_std |
| value: -3.893478526464003 |
| - type: ndcg_at_1 |
| value: 47.398 |
| - type: ndcg_at_10 |
| value: 59.83500000000001 |
| - type: ndcg_at_100 |
| value: 63.743 |
| - type: ndcg_at_1000 |
| value: 64.75800000000001 |
| - type: ndcg_at_20 |
| value: 61.78399999999999 |
| - type: ndcg_at_3 |
| value: 54.481 |
| - type: ndcg_at_5 |
| value: 57.034 |
| - type: precision_at_1 |
| value: 47.398 |
| - type: precision_at_10 |
| value: 9.504999999999999 |
| - type: precision_at_100 |
| value: 1.2449999999999999 |
| - type: precision_at_1000 |
| value: 0.13699999999999998 |
| - type: precision_at_20 |
| value: 5.357 |
| - type: precision_at_3 |
| value: 24.18 |
| - type: precision_at_5 |
| value: 16.439 |
| - type: recall_at_1 |
| value: 41.743 |
| - type: recall_at_10 |
| value: 73.476 |
| - type: recall_at_100 |
| value: 89.875 |
| - type: recall_at_1000 |
| value: 97.311 |
| - type: recall_at_20 |
| value: 80.61500000000001 |
| - type: recall_at_3 |
| value: 59.192 |
| - type: recall_at_5 |
| value: 65.43299999999999 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackGisRetrieval (default) |
| revision: 5003b3064772da1887988e05400cf3806fe491f2 |
| split: test |
| type: mteb/cqadupstack-gis |
| metrics: |
| - type: main_score |
| value: 42.451 |
| - type: map_at_1 |
| value: 28.996 |
| - type: map_at_10 |
| value: 37.616 |
| - type: map_at_100 |
| value: 38.702999999999996 |
| - type: map_at_1000 |
| value: 38.785 |
| - type: map_at_20 |
| value: 38.248 |
| - type: map_at_3 |
| value: 34.906 |
| - type: map_at_5 |
| value: 36.313 |
| - type: mrr_at_1 |
| value: 30.847457627118647 |
| - type: mrr_at_10 |
| value: 39.38054882970136 |
| - type: mrr_at_100 |
| value: 40.366915853040304 |
| - type: mrr_at_1000 |
| value: 40.422138866370375 |
| - type: mrr_at_20 |
| value: 39.963305509876314 |
| - type: mrr_at_3 |
| value: 36.81732580037664 |
| - type: mrr_at_5 |
| value: 38.28060263653482 |
| - type: nauc_map_at_1000_diff1 |
| value: 46.412845971748965 |
| - type: nauc_map_at_1000_max |
| value: 35.71820582656466 |
| - type: nauc_map_at_1000_std |
| value: -3.4396952487244543 |
| - type: nauc_map_at_100_diff1 |
| value: 46.395516885783515 |
| - type: nauc_map_at_100_max |
| value: 35.699005377624786 |
| - type: nauc_map_at_100_std |
| value: -3.4295307929848815 |
| - type: nauc_map_at_10_diff1 |
| value: 46.60661423872333 |
| - type: nauc_map_at_10_max |
| value: 35.76865437824633 |
| - type: nauc_map_at_10_std |
| value: -3.7286516914981194 |
| - type: nauc_map_at_1_diff1 |
| value: 50.67584728744242 |
| - type: nauc_map_at_1_max |
| value: 33.31838096723387 |
| - type: nauc_map_at_1_std |
| value: -7.017496210052664 |
| - type: nauc_map_at_20_diff1 |
| value: 46.336180127932245 |
| - type: nauc_map_at_20_max |
| value: 35.67863259884862 |
| - type: nauc_map_at_20_std |
| value: -3.532643797779482 |
| - type: nauc_map_at_3_diff1 |
| value: 47.60693220558914 |
| - type: nauc_map_at_3_max |
| value: 34.51587922644232 |
| - type: nauc_map_at_3_std |
| value: -5.094395358598097 |
| - type: nauc_map_at_5_diff1 |
| value: 47.06590116277457 |
| - type: nauc_map_at_5_max |
| value: 35.09758567281723 |
| - type: nauc_map_at_5_std |
| value: -4.594804514448893 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 45.22776158670323 |
| - type: nauc_mrr_at_1000_max |
| value: 36.86081533470028 |
| - type: nauc_mrr_at_1000_std |
| value: -2.033205148222453 |
| - type: nauc_mrr_at_100_diff1 |
| value: 45.204577420420954 |
| - type: nauc_mrr_at_100_max |
| value: 36.849578433404155 |
| - type: nauc_mrr_at_100_std |
| value: -2.016257960786726 |
| - type: nauc_mrr_at_10_diff1 |
| value: 45.2359210975849 |
| - type: nauc_mrr_at_10_max |
| value: 37.01690402885584 |
| - type: nauc_mrr_at_10_std |
| value: -2.2602767431608597 |
| - type: nauc_mrr_at_1_diff1 |
| value: 48.87088666432611 |
| - type: nauc_mrr_at_1_max |
| value: 35.58051752132078 |
| - type: nauc_mrr_at_1_std |
| value: -4.731264758679752 |
| - type: nauc_mrr_at_20_diff1 |
| value: 45.107901559758574 |
| - type: nauc_mrr_at_20_max |
| value: 36.871010473007566 |
| - type: nauc_mrr_at_20_std |
| value: -2.09198313309596 |
| - type: nauc_mrr_at_3_diff1 |
| value: 46.532099561607964 |
| - type: nauc_mrr_at_3_max |
| value: 36.533535412036436 |
| - type: nauc_mrr_at_3_std |
| value: -3.1250129413210814 |
| - type: nauc_mrr_at_5_diff1 |
| value: 45.57186948675289 |
| - type: nauc_mrr_at_5_max |
| value: 36.46221116432317 |
| - type: nauc_mrr_at_5_std |
| value: -2.8206584854678916 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 44.329221962893975 |
| - type: nauc_ndcg_at_1000_max |
| value: 36.91867297213294 |
| - type: nauc_ndcg_at_1000_std |
| value: -0.4934939008290994 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 44.02704131900571 |
| - type: nauc_ndcg_at_100_max |
| value: 36.73741523697531 |
| - type: nauc_ndcg_at_100_std |
| value: 0.056585087009301434 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 44.46275070065777 |
| - type: nauc_ndcg_at_10_max |
| value: 37.08165048296797 |
| - type: nauc_ndcg_at_10_std |
| value: -1.4504178730008903 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 48.87088666432611 |
| - type: nauc_ndcg_at_1_max |
| value: 35.58051752132078 |
| - type: nauc_ndcg_at_1_std |
| value: -4.731264758679752 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 43.715351338600854 |
| - type: nauc_ndcg_at_20_max |
| value: 36.597558579484286 |
| - type: nauc_ndcg_at_20_std |
| value: -0.7442166823850342 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 46.6559452141376 |
| - type: nauc_ndcg_at_3_max |
| value: 35.303431090059576 |
| - type: nauc_ndcg_at_3_std |
| value: -4.245048423792951 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 45.46364843701738 |
| - type: nauc_ndcg_at_5_max |
| value: 35.786069703721715 |
| - type: nauc_ndcg_at_5_std |
| value: -3.225507760537463 |
| - type: nauc_precision_at_1000_diff1 |
| value: -8.813657843193829 |
| - type: nauc_precision_at_1000_max |
| value: 19.341916147889847 |
| - type: nauc_precision_at_1000_std |
| value: 11.83125844170699 |
| - type: nauc_precision_at_100_diff1 |
| value: 8.781439905664739 |
| - type: nauc_precision_at_100_max |
| value: 29.44860083085914 |
| - type: nauc_precision_at_100_std |
| value: 13.776934250429376 |
| - type: nauc_precision_at_10_diff1 |
| value: 28.889666145944 |
| - type: nauc_precision_at_10_max |
| value: 41.11966477643234 |
| - type: nauc_precision_at_10_std |
| value: 6.963197458201788 |
| - type: nauc_precision_at_1_diff1 |
| value: 48.87088666432611 |
| - type: nauc_precision_at_1_max |
| value: 35.58051752132078 |
| - type: nauc_precision_at_1_std |
| value: -4.731264758679752 |
| - type: nauc_precision_at_20_diff1 |
| value: 21.46418782701143 |
| - type: nauc_precision_at_20_max |
| value: 37.04050243855216 |
| - type: nauc_precision_at_20_std |
| value: 8.967545775130677 |
| - type: nauc_precision_at_3_diff1 |
| value: 39.977903525162525 |
| - type: nauc_precision_at_3_max |
| value: 37.8324727688519 |
| - type: nauc_precision_at_3_std |
| value: -0.09362980766141979 |
| - type: nauc_precision_at_5_diff1 |
| value: 36.05449702608607 |
| - type: nauc_precision_at_5_max |
| value: 39.31263152685144 |
| - type: nauc_precision_at_5_std |
| value: 1.4853599728966675 |
| - type: nauc_recall_at_1000_diff1 |
| value: 23.131220881305328 |
| - type: nauc_recall_at_1000_max |
| value: 43.09488375414571 |
| - type: nauc_recall_at_1000_std |
| value: 34.32484643072848 |
| - type: nauc_recall_at_100_diff1 |
| value: 32.509347146711775 |
| - type: nauc_recall_at_100_max |
| value: 38.31293004210284 |
| - type: nauc_recall_at_100_std |
| value: 20.31295020880922 |
| - type: nauc_recall_at_10_diff1 |
| value: 38.162435666945825 |
| - type: nauc_recall_at_10_max |
| value: 39.05783231051994 |
| - type: nauc_recall_at_10_std |
| value: 4.737164462571157 |
| - type: nauc_recall_at_1_diff1 |
| value: 50.67584728744242 |
| - type: nauc_recall_at_1_max |
| value: 33.31838096723387 |
| - type: nauc_recall_at_1_std |
| value: -7.017496210052664 |
| - type: nauc_recall_at_20_diff1 |
| value: 34.36040334628013 |
| - type: nauc_recall_at_20_max |
| value: 36.688387172616835 |
| - type: nauc_recall_at_20_std |
| value: 8.670145039799666 |
| - type: nauc_recall_at_3_diff1 |
| value: 44.33263333615946 |
| - type: nauc_recall_at_3_max |
| value: 34.21104932799129 |
| - type: nauc_recall_at_3_std |
| value: -3.4348954541060057 |
| - type: nauc_recall_at_5_diff1 |
| value: 41.3941366549961 |
| - type: nauc_recall_at_5_max |
| value: 35.61498401814357 |
| - type: nauc_recall_at_5_std |
| value: -0.5242808474696788 |
| - type: ndcg_at_1 |
| value: 30.847 |
| - type: ndcg_at_10 |
| value: 42.451 |
| - type: ndcg_at_100 |
| value: 47.666 |
| - type: ndcg_at_1000 |
| value: 49.559 |
| - type: ndcg_at_20 |
| value: 44.564 |
| - type: ndcg_at_3 |
| value: 37.141000000000005 |
| - type: ndcg_at_5 |
| value: 39.615 |
| - type: precision_at_1 |
| value: 30.847 |
| - type: precision_at_10 |
| value: 6.361999999999999 |
| - type: precision_at_100 |
| value: 0.9440000000000001 |
| - type: precision_at_1000 |
| value: 0.11399999999999999 |
| - type: precision_at_20 |
| value: 3.695 |
| - type: precision_at_3 |
| value: 15.292 |
| - type: precision_at_5 |
| value: 10.644 |
| - type: recall_at_1 |
| value: 28.996 |
| - type: recall_at_10 |
| value: 55.584 |
| - type: recall_at_100 |
| value: 79.137 |
| - type: recall_at_1000 |
| value: 93.133 |
| - type: recall_at_20 |
| value: 63.344 |
| - type: recall_at_3 |
| value: 41.388999999999996 |
| - type: recall_at_5 |
| value: 47.302 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackMathematicaRetrieval (default) |
| revision: 90fceea13679c63fe563ded68f3b6f06e50061de |
| split: test |
| type: mteb/cqadupstack-mathematica |
| metrics: |
| - type: main_score |
| value: 34.095 |
| - type: map_at_1 |
| value: 19.73 |
| - type: map_at_10 |
| value: 28.621999999999996 |
| - type: map_at_100 |
| value: 29.951 |
| - type: map_at_1000 |
| value: 30.063000000000002 |
| - type: map_at_20 |
| value: 29.309 |
| - type: map_at_3 |
| value: 25.667 |
| - type: map_at_5 |
| value: 27.594 |
| - type: mrr_at_1 |
| value: 24.502487562189053 |
| - type: mrr_at_10 |
| value: 33.665255073837145 |
| - type: mrr_at_100 |
| value: 34.59932347722826 |
| - type: mrr_at_1000 |
| value: 34.66003643326513 |
| - type: mrr_at_20 |
| value: 34.11376652638897 |
| - type: mrr_at_3 |
| value: 31.05306799336651 |
| - type: mrr_at_5 |
| value: 32.76326699834162 |
| - type: nauc_map_at_1000_diff1 |
| value: 34.72907872454501 |
| - type: nauc_map_at_1000_max |
| value: 28.254188806716968 |
| - type: nauc_map_at_1000_std |
| value: 1.766585437449934 |
| - type: nauc_map_at_100_diff1 |
| value: 34.72932621462264 |
| - type: nauc_map_at_100_max |
| value: 28.27419759099569 |
| - type: nauc_map_at_100_std |
| value: 1.7699849561943597 |
| - type: nauc_map_at_10_diff1 |
| value: 34.78565974033627 |
| - type: nauc_map_at_10_max |
| value: 27.986939554161456 |
| - type: nauc_map_at_10_std |
| value: 1.167749138251006 |
| - type: nauc_map_at_1_diff1 |
| value: 38.91003571707319 |
| - type: nauc_map_at_1_max |
| value: 26.48670439569984 |
| - type: nauc_map_at_1_std |
| value: -0.6581147831046584 |
| - type: nauc_map_at_20_diff1 |
| value: 34.930356018900085 |
| - type: nauc_map_at_20_max |
| value: 28.11826713770072 |
| - type: nauc_map_at_20_std |
| value: 1.4222869706417194 |
| - type: nauc_map_at_3_diff1 |
| value: 36.0762128105621 |
| - type: nauc_map_at_3_max |
| value: 28.565191344891815 |
| - type: nauc_map_at_3_std |
| value: 0.7825139863346278 |
| - type: nauc_map_at_5_diff1 |
| value: 35.51997355447966 |
| - type: nauc_map_at_5_max |
| value: 27.79640533393062 |
| - type: nauc_map_at_5_std |
| value: 0.4033822753367694 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 35.086631245748286 |
| - type: nauc_mrr_at_1000_max |
| value: 28.00090704456733 |
| - type: nauc_mrr_at_1000_std |
| value: 2.7443538042856495 |
| - type: nauc_mrr_at_100_diff1 |
| value: 35.08022882692694 |
| - type: nauc_mrr_at_100_max |
| value: 28.02518055725871 |
| - type: nauc_mrr_at_100_std |
| value: 2.756913025485739 |
| - type: nauc_mrr_at_10_diff1 |
| value: 35.189138304228955 |
| - type: nauc_mrr_at_10_max |
| value: 27.893789610020132 |
| - type: nauc_mrr_at_10_std |
| value: 2.5277514271816273 |
| - type: nauc_mrr_at_1_diff1 |
| value: 38.49246887300505 |
| - type: nauc_mrr_at_1_max |
| value: 25.42106416145382 |
| - type: nauc_mrr_at_1_std |
| value: -0.3166610087713868 |
| - type: nauc_mrr_at_20_diff1 |
| value: 35.27168804507115 |
| - type: nauc_mrr_at_20_max |
| value: 28.012190140962623 |
| - type: nauc_mrr_at_20_std |
| value: 2.6699643794051733 |
| - type: nauc_mrr_at_3_diff1 |
| value: 35.244407269705356 |
| - type: nauc_mrr_at_3_max |
| value: 27.901137842346667 |
| - type: nauc_mrr_at_3_std |
| value: 1.536344232061536 |
| - type: nauc_mrr_at_5_diff1 |
| value: 35.60496636899887 |
| - type: nauc_mrr_at_5_max |
| value: 27.646092417250294 |
| - type: nauc_mrr_at_5_std |
| value: 1.7849129602744565 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 33.00641553083242 |
| - type: nauc_ndcg_at_1000_max |
| value: 29.281184042576324 |
| - type: nauc_ndcg_at_1000_std |
| value: 4.705354777869887 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 32.73299739191785 |
| - type: nauc_ndcg_at_100_max |
| value: 29.733498550725486 |
| - type: nauc_ndcg_at_100_std |
| value: 5.051380591295473 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 33.42778333197981 |
| - type: nauc_ndcg_at_10_max |
| value: 28.500230808790462 |
| - type: nauc_ndcg_at_10_std |
| value: 2.6279521120828426 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 38.49246887300505 |
| - type: nauc_ndcg_at_1_max |
| value: 25.42106416145382 |
| - type: nauc_ndcg_at_1_std |
| value: -0.3166610087713868 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 33.932374714340305 |
| - type: nauc_ndcg_at_20_max |
| value: 28.97338117740232 |
| - type: nauc_ndcg_at_20_std |
| value: 3.382234056656039 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 35.06726185470219 |
| - type: nauc_ndcg_at_3_max |
| value: 28.769824175873655 |
| - type: nauc_ndcg_at_3_std |
| value: 0.9778290393744915 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 34.73183576563172 |
| - type: nauc_ndcg_at_5_max |
| value: 27.92235378893707 |
| - type: nauc_ndcg_at_5_std |
| value: 0.931888346245052 |
| - type: nauc_precision_at_1000_diff1 |
| value: -4.969051807978748 |
| - type: nauc_precision_at_1000_max |
| value: 0.14144278477866445 |
| - type: nauc_precision_at_1000_std |
| value: 4.867244664069488 |
| - type: nauc_precision_at_100_diff1 |
| value: 3.4485901120482914 |
| - type: nauc_precision_at_100_max |
| value: 12.881970758272205 |
| - type: nauc_precision_at_100_std |
| value: 11.70053444498138 |
| - type: nauc_precision_at_10_diff1 |
| value: 19.652560943517372 |
| - type: nauc_precision_at_10_max |
| value: 22.721397508432503 |
| - type: nauc_precision_at_10_std |
| value: 6.4517755635275025 |
| - type: nauc_precision_at_1_diff1 |
| value: 38.49246887300505 |
| - type: nauc_precision_at_1_max |
| value: 25.42106416145382 |
| - type: nauc_precision_at_1_std |
| value: -0.3166610087713868 |
| - type: nauc_precision_at_20_diff1 |
| value: 17.228427222424315 |
| - type: nauc_precision_at_20_max |
| value: 20.728777641636476 |
| - type: nauc_precision_at_20_std |
| value: 7.817118735958645 |
| - type: nauc_precision_at_3_diff1 |
| value: 30.223066194086307 |
| - type: nauc_precision_at_3_max |
| value: 27.412166459133786 |
| - type: nauc_precision_at_3_std |
| value: 1.698402524212445 |
| - type: nauc_precision_at_5_diff1 |
| value: 26.619771134350295 |
| - type: nauc_precision_at_5_max |
| value: 23.208486114756507 |
| - type: nauc_precision_at_5_std |
| value: 1.214970586733223 |
| - type: nauc_recall_at_1000_diff1 |
| value: 11.623462125104215 |
| - type: nauc_recall_at_1000_max |
| value: 36.091211213022106 |
| - type: nauc_recall_at_1000_std |
| value: 32.23113490590334 |
| - type: nauc_recall_at_100_diff1 |
| value: 21.087538105716423 |
| - type: nauc_recall_at_100_max |
| value: 34.78408730230787 |
| - type: nauc_recall_at_100_std |
| value: 18.502764053088498 |
| - type: nauc_recall_at_10_diff1 |
| value: 28.025850341314616 |
| - type: nauc_recall_at_10_max |
| value: 28.278332371196424 |
| - type: nauc_recall_at_10_std |
| value: 5.215906443076799 |
| - type: nauc_recall_at_1_diff1 |
| value: 38.91003571707319 |
| - type: nauc_recall_at_1_max |
| value: 26.48670439569984 |
| - type: nauc_recall_at_1_std |
| value: -0.6581147831046584 |
| - type: nauc_recall_at_20_diff1 |
| value: 29.473435609654423 |
| - type: nauc_recall_at_20_max |
| value: 29.49664949068959 |
| - type: nauc_recall_at_20_std |
| value: 7.462607204613657 |
| - type: nauc_recall_at_3_diff1 |
| value: 32.75851316103734 |
| - type: nauc_recall_at_3_max |
| value: 30.05729454718181 |
| - type: nauc_recall_at_3_std |
| value: 1.9302697513077123 |
| - type: nauc_recall_at_5_diff1 |
| value: 31.4784165636263 |
| - type: nauc_recall_at_5_max |
| value: 27.066581269469197 |
| - type: nauc_recall_at_5_std |
| value: 1.3182034896545982 |
| - type: ndcg_at_1 |
| value: 24.502 |
| - type: ndcg_at_10 |
| value: 34.095 |
| - type: ndcg_at_100 |
| value: 40.278000000000006 |
| - type: ndcg_at_1000 |
| value: 42.845 |
| - type: ndcg_at_20 |
| value: 36.158 |
| - type: ndcg_at_3 |
| value: 29.002 |
| - type: ndcg_at_5 |
| value: 31.952 |
| - type: precision_at_1 |
| value: 24.502 |
| - type: precision_at_10 |
| value: 6.219 |
| - type: precision_at_100 |
| value: 1.082 |
| - type: precision_at_1000 |
| value: 0.14300000000000002 |
| - type: precision_at_20 |
| value: 3.7560000000000002 |
| - type: precision_at_3 |
| value: 13.764999999999999 |
| - type: precision_at_5 |
| value: 10.323 |
| - type: recall_at_1 |
| value: 19.73 |
| - type: recall_at_10 |
| value: 45.832 |
| - type: recall_at_100 |
| value: 72.90299999999999 |
| - type: recall_at_1000 |
| value: 91.12400000000001 |
| - type: recall_at_20 |
| value: 52.941 |
| - type: recall_at_3 |
| value: 32.147999999999996 |
| - type: recall_at_5 |
| value: 39.572 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackPhysicsRetrieval (default) |
| revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 |
| split: test |
| type: mteb/cqadupstack-physics |
| metrics: |
| - type: main_score |
| value: 48.891 |
| - type: map_at_1 |
| value: 31.075999999999997 |
| - type: map_at_10 |
| value: 42.577999999999996 |
| - type: map_at_100 |
| value: 43.998 |
| - type: map_at_1000 |
| value: 44.107 |
| - type: map_at_20 |
| value: 43.394 |
| - type: map_at_3 |
| value: 39.237 |
| - type: map_at_5 |
| value: 41.213 |
| - type: mrr_at_1 |
| value: 37.632338787295474 |
| - type: mrr_at_10 |
| value: 48.028705867974345 |
| - type: mrr_at_100 |
| value: 48.89056715596661 |
| - type: mrr_at_1000 |
| value: 48.92751183933152 |
| - type: mrr_at_20 |
| value: 48.572019107385856 |
| - type: mrr_at_3 |
| value: 45.58870709015074 |
| - type: mrr_at_5 |
| value: 47.090150786012124 |
| - type: nauc_map_at_1000_diff1 |
| value: 50.78783234215205 |
| - type: nauc_map_at_1000_max |
| value: 33.542131730312164 |
| - type: nauc_map_at_1000_std |
| value: -0.3678105032092534 |
| - type: nauc_map_at_100_diff1 |
| value: 50.801030214261935 |
| - type: nauc_map_at_100_max |
| value: 33.49117253773047 |
| - type: nauc_map_at_100_std |
| value: -0.424437332181341 |
| - type: nauc_map_at_10_diff1 |
| value: 50.665593124786014 |
| - type: nauc_map_at_10_max |
| value: 32.785196057455686 |
| - type: nauc_map_at_10_std |
| value: -1.1779549158534983 |
| - type: nauc_map_at_1_diff1 |
| value: 55.868642241264645 |
| - type: nauc_map_at_1_max |
| value: 30.544699698856615 |
| - type: nauc_map_at_1_std |
| value: -3.824717473245085 |
| - type: nauc_map_at_20_diff1 |
| value: 50.77114941389146 |
| - type: nauc_map_at_20_max |
| value: 33.26827708180765 |
| - type: nauc_map_at_20_std |
| value: -0.734677624886567 |
| - type: nauc_map_at_3_diff1 |
| value: 51.03832030578005 |
| - type: nauc_map_at_3_max |
| value: 32.39458212663325 |
| - type: nauc_map_at_3_std |
| value: -1.6494237804803646 |
| - type: nauc_map_at_5_diff1 |
| value: 50.97104795265703 |
| - type: nauc_map_at_5_max |
| value: 32.963257618296986 |
| - type: nauc_map_at_5_std |
| value: -1.2954427188265398 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 50.087825368297565 |
| - type: nauc_mrr_at_1000_max |
| value: 35.696912235935315 |
| - type: nauc_mrr_at_1000_std |
| value: 0.9517029361871309 |
| - type: nauc_mrr_at_100_diff1 |
| value: 50.091410892116386 |
| - type: nauc_mrr_at_100_max |
| value: 35.701167670781956 |
| - type: nauc_mrr_at_100_std |
| value: 0.9492584917140756 |
| - type: nauc_mrr_at_10_diff1 |
| value: 49.88389091064117 |
| - type: nauc_mrr_at_10_max |
| value: 35.6067947110772 |
| - type: nauc_mrr_at_10_std |
| value: 0.7626165780679156 |
| - type: nauc_mrr_at_1_diff1 |
| value: 55.01931926385987 |
| - type: nauc_mrr_at_1_max |
| value: 35.731630359671044 |
| - type: nauc_mrr_at_1_std |
| value: 0.4765227639052635 |
| - type: nauc_mrr_at_20_diff1 |
| value: 50.04232795868649 |
| - type: nauc_mrr_at_20_max |
| value: 35.64757803934064 |
| - type: nauc_mrr_at_20_std |
| value: 0.8038895849793868 |
| - type: nauc_mrr_at_3_diff1 |
| value: 49.29102858426895 |
| - type: nauc_mrr_at_3_max |
| value: 35.511749287022596 |
| - type: nauc_mrr_at_3_std |
| value: 0.9607913501181212 |
| - type: nauc_mrr_at_5_diff1 |
| value: 49.90634335653725 |
| - type: nauc_mrr_at_5_max |
| value: 35.57725666069228 |
| - type: nauc_mrr_at_5_std |
| value: 0.5886034889984604 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 49.227101169579974 |
| - type: nauc_ndcg_at_1000_max |
| value: 35.304422697207904 |
| - type: nauc_ndcg_at_1000_std |
| value: 2.3564962090430357 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 49.33636342826304 |
| - type: nauc_ndcg_at_100_max |
| value: 34.93271239347418 |
| - type: nauc_ndcg_at_100_std |
| value: 2.304638273222096 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 48.62225183717284 |
| - type: nauc_ndcg_at_10_max |
| value: 33.013586201737816 |
| - type: nauc_ndcg_at_10_std |
| value: -0.3811388147797492 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 55.01931926385987 |
| - type: nauc_ndcg_at_1_max |
| value: 35.731630359671044 |
| - type: nauc_ndcg_at_1_std |
| value: 0.4765227639052635 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 49.02938009186652 |
| - type: nauc_ndcg_at_20_max |
| value: 34.07537935061685 |
| - type: nauc_ndcg_at_20_std |
| value: 0.7596556118589683 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 48.53275134328913 |
| - type: nauc_ndcg_at_3_max |
| value: 33.72246853040944 |
| - type: nauc_ndcg_at_3_std |
| value: 0.07148157187994036 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 49.125387965082595 |
| - type: nauc_ndcg_at_5_max |
| value: 33.89755823168926 |
| - type: nauc_ndcg_at_5_std |
| value: -0.23484468412288975 |
| - type: nauc_precision_at_1000_diff1 |
| value: -16.388718759022847 |
| - type: nauc_precision_at_1000_max |
| value: 5.237181961139354 |
| - type: nauc_precision_at_1000_std |
| value: 12.481420642405105 |
| - type: nauc_precision_at_100_diff1 |
| value: -5.613297466581972 |
| - type: nauc_precision_at_100_max |
| value: 13.871852332913598 |
| - type: nauc_precision_at_100_std |
| value: 15.784270811182186 |
| - type: nauc_precision_at_10_diff1 |
| value: 14.380456681659199 |
| - type: nauc_precision_at_10_max |
| value: 24.28938422113675 |
| - type: nauc_precision_at_10_std |
| value: 9.104016210929833 |
| - type: nauc_precision_at_1_diff1 |
| value: 55.01931926385987 |
| - type: nauc_precision_at_1_max |
| value: 35.731630359671044 |
| - type: nauc_precision_at_1_std |
| value: 0.4765227639052635 |
| - type: nauc_precision_at_20_diff1 |
| value: 6.997723624231359 |
| - type: nauc_precision_at_20_max |
| value: 22.242975253242793 |
| - type: nauc_precision_at_20_std |
| value: 12.460553518097337 |
| - type: nauc_precision_at_3_diff1 |
| value: 31.93565478138394 |
| - type: nauc_precision_at_3_max |
| value: 32.245381961758554 |
| - type: nauc_precision_at_3_std |
| value: 6.3778575720255635 |
| - type: nauc_precision_at_5_diff1 |
| value: 25.360806939232344 |
| - type: nauc_precision_at_5_max |
| value: 29.95777944809185 |
| - type: nauc_precision_at_5_std |
| value: 8.192950259472545 |
| - type: nauc_recall_at_1000_diff1 |
| value: 33.00760032876783 |
| - type: nauc_recall_at_1000_max |
| value: 52.825856604033994 |
| - type: nauc_recall_at_1000_std |
| value: 45.239442029547384 |
| - type: nauc_recall_at_100_diff1 |
| value: 40.32600076465021 |
| - type: nauc_recall_at_100_max |
| value: 35.20651551017542 |
| - type: nauc_recall_at_100_std |
| value: 18.2866715724604 |
| - type: nauc_recall_at_10_diff1 |
| value: 40.19090180531315 |
| - type: nauc_recall_at_10_max |
| value: 27.727160089866675 |
| - type: nauc_recall_at_10_std |
| value: -0.34152382508922086 |
| - type: nauc_recall_at_1_diff1 |
| value: 55.868642241264645 |
| - type: nauc_recall_at_1_max |
| value: 30.544699698856615 |
| - type: nauc_recall_at_1_std |
| value: -3.824717473245085 |
| - type: nauc_recall_at_20_diff1 |
| value: 40.53509773756395 |
| - type: nauc_recall_at_20_max |
| value: 30.879328024854107 |
| - type: nauc_recall_at_20_std |
| value: 4.5165469550975255 |
| - type: nauc_recall_at_3_diff1 |
| value: 43.27936784610322 |
| - type: nauc_recall_at_3_max |
| value: 30.443511585383586 |
| - type: nauc_recall_at_3_std |
| value: -0.4500440621385532 |
| - type: nauc_recall_at_5_diff1 |
| value: 42.84235237573527 |
| - type: nauc_recall_at_5_max |
| value: 30.6861143937192 |
| - type: nauc_recall_at_5_std |
| value: -0.6079883050754419 |
| - type: ndcg_at_1 |
| value: 37.632 |
| - type: ndcg_at_10 |
| value: 48.891 |
| - type: ndcg_at_100 |
| value: 54.44 |
| - type: ndcg_at_1000 |
| value: 56.218 |
| - type: ndcg_at_20 |
| value: 51.242 |
| - type: ndcg_at_3 |
| value: 43.618 |
| - type: ndcg_at_5 |
| value: 46.321 |
| - type: precision_at_1 |
| value: 37.632 |
| - type: precision_at_10 |
| value: 8.884 |
| - type: precision_at_100 |
| value: 1.362 |
| - type: precision_at_1000 |
| value: 0.169 |
| - type: precision_at_20 |
| value: 5.221 |
| - type: precision_at_3 |
| value: 20.788999999999998 |
| - type: precision_at_5 |
| value: 14.802999999999999 |
| - type: recall_at_1 |
| value: 31.075999999999997 |
| - type: recall_at_10 |
| value: 62.087 |
| - type: recall_at_100 |
| value: 84.615 |
| - type: recall_at_1000 |
| value: 95.809 |
| - type: recall_at_20 |
| value: 70.092 |
| - type: recall_at_3 |
| value: 47.49 |
| - type: recall_at_5 |
| value: 54.359 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackProgrammersRetrieval (default) |
| revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 |
| split: test |
| type: mteb/cqadupstack-programmers |
| metrics: |
| - type: main_score |
| value: 45.334 |
| - type: map_at_1 |
| value: 27.275 |
| - type: map_at_10 |
| value: 38.818000000000005 |
| - type: map_at_100 |
| value: 40.245999999999995 |
| - type: map_at_1000 |
| value: 40.348 |
| - type: map_at_20 |
| value: 39.641 |
| - type: map_at_3 |
| value: 35.115 |
| - type: map_at_5 |
| value: 37.232 |
| - type: mrr_at_1 |
| value: 33.789954337899545 |
| - type: mrr_at_10 |
| value: 44.11923787779952 |
| - type: mrr_at_100 |
| value: 45.028806593226676 |
| - type: mrr_at_1000 |
| value: 45.0740604466411 |
| - type: mrr_at_20 |
| value: 44.67688886801231 |
| - type: mrr_at_3 |
| value: 41.248097412480966 |
| - type: mrr_at_5 |
| value: 42.97754946727547 |
| - type: nauc_map_at_1000_diff1 |
| value: 43.013762963519305 |
| - type: nauc_map_at_1000_max |
| value: 39.10459163352522 |
| - type: nauc_map_at_1000_std |
| value: 0.5686019526390734 |
| - type: nauc_map_at_100_diff1 |
| value: 43.01344839983274 |
| - type: nauc_map_at_100_max |
| value: 39.12155667806109 |
| - type: nauc_map_at_100_std |
| value: 0.5850312262411536 |
| - type: nauc_map_at_10_diff1 |
| value: 43.40889409121977 |
| - type: nauc_map_at_10_max |
| value: 38.86352881392226 |
| - type: nauc_map_at_10_std |
| value: -0.09138731580615166 |
| - type: nauc_map_at_1_diff1 |
| value: 49.580098743143 |
| - type: nauc_map_at_1_max |
| value: 33.624185967920326 |
| - type: nauc_map_at_1_std |
| value: -7.896295502496881 |
| - type: nauc_map_at_20_diff1 |
| value: 43.11936673331683 |
| - type: nauc_map_at_20_max |
| value: 39.07709188651765 |
| - type: nauc_map_at_20_std |
| value: 0.4602382023590104 |
| - type: nauc_map_at_3_diff1 |
| value: 43.67107257453258 |
| - type: nauc_map_at_3_max |
| value: 36.84244693065489 |
| - type: nauc_map_at_3_std |
| value: -3.289874933863321 |
| - type: nauc_map_at_5_diff1 |
| value: 43.758122467637826 |
| - type: nauc_map_at_5_max |
| value: 38.294511650248126 |
| - type: nauc_map_at_5_std |
| value: -1.4279289313215355 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 41.19785571847013 |
| - type: nauc_mrr_at_1000_max |
| value: 38.55497179205239 |
| - type: nauc_mrr_at_1000_std |
| value: 1.7188770740469619 |
| - type: nauc_mrr_at_100_diff1 |
| value: 41.177608254142875 |
| - type: nauc_mrr_at_100_max |
| value: 38.55707450419509 |
| - type: nauc_mrr_at_100_std |
| value: 1.742333253511747 |
| - type: nauc_mrr_at_10_diff1 |
| value: 41.16178606855569 |
| - type: nauc_mrr_at_10_max |
| value: 38.53198828945776 |
| - type: nauc_mrr_at_10_std |
| value: 1.4657516877125125 |
| - type: nauc_mrr_at_1_diff1 |
| value: 47.42346510865722 |
| - type: nauc_mrr_at_1_max |
| value: 36.48815188158201 |
| - type: nauc_mrr_at_1_std |
| value: -2.34134882449636 |
| - type: nauc_mrr_at_20_diff1 |
| value: 41.246202514418584 |
| - type: nauc_mrr_at_20_max |
| value: 38.69180784192216 |
| - type: nauc_mrr_at_20_std |
| value: 1.8205983742560619 |
| - type: nauc_mrr_at_3_diff1 |
| value: 41.09603949294592 |
| - type: nauc_mrr_at_3_max |
| value: 37.95896498227977 |
| - type: nauc_mrr_at_3_std |
| value: 0.2874075190886481 |
| - type: nauc_mrr_at_5_diff1 |
| value: 41.18455834868946 |
| - type: nauc_mrr_at_5_max |
| value: 38.456998347163065 |
| - type: nauc_mrr_at_5_std |
| value: 0.9867811075887676 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 40.54615364663546 |
| - type: nauc_ndcg_at_1000_max |
| value: 40.42616803864886 |
| - type: nauc_ndcg_at_1000_std |
| value: 4.363693436984652 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 40.44224861178897 |
| - type: nauc_ndcg_at_100_max |
| value: 40.94806712564172 |
| - type: nauc_ndcg_at_100_std |
| value: 5.196573771400126 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 40.92593737099367 |
| - type: nauc_ndcg_at_10_max |
| value: 40.26823363364135 |
| - type: nauc_ndcg_at_10_std |
| value: 3.192020901707987 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 47.42346510865722 |
| - type: nauc_ndcg_at_1_max |
| value: 36.48815188158201 |
| - type: nauc_ndcg_at_1_std |
| value: -2.34134882449636 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 40.70844796238177 |
| - type: nauc_ndcg_at_20_max |
| value: 41.066915934122356 |
| - type: nauc_ndcg_at_20_std |
| value: 4.941739690696084 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 40.22388347943839 |
| - type: nauc_ndcg_at_3_max |
| value: 37.97075355659086 |
| - type: nauc_ndcg_at_3_std |
| value: -0.3952100142870558 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 40.95268317695563 |
| - type: nauc_ndcg_at_5_max |
| value: 39.3554650798222 |
| - type: nauc_ndcg_at_5_std |
| value: 1.02690752358091 |
| - type: nauc_precision_at_1000_diff1 |
| value: -21.508622681866868 |
| - type: nauc_precision_at_1000_max |
| value: -5.391055753734811 |
| - type: nauc_precision_at_1000_std |
| value: 7.148967890675029 |
| - type: nauc_precision_at_100_diff1 |
| value: -9.555610415584772 |
| - type: nauc_precision_at_100_max |
| value: 12.841520305380632 |
| - type: nauc_precision_at_100_std |
| value: 19.88687702744806 |
| - type: nauc_precision_at_10_diff1 |
| value: 11.710375921485369 |
| - type: nauc_precision_at_10_max |
| value: 34.61710718960949 |
| - type: nauc_precision_at_10_std |
| value: 21.07494229065057 |
| - type: nauc_precision_at_1_diff1 |
| value: 47.42346510865722 |
| - type: nauc_precision_at_1_max |
| value: 36.48815188158201 |
| - type: nauc_precision_at_1_std |
| value: -2.34134882449636 |
| - type: nauc_precision_at_20_diff1 |
| value: 4.261943900088042 |
| - type: nauc_precision_at_20_max |
| value: 29.277336528563648 |
| - type: nauc_precision_at_20_std |
| value: 23.809798696946697 |
| - type: nauc_precision_at_3_diff1 |
| value: 24.180190068545883 |
| - type: nauc_precision_at_3_max |
| value: 37.86395654258292 |
| - type: nauc_precision_at_3_std |
| value: 9.925473230392306 |
| - type: nauc_precision_at_5_diff1 |
| value: 18.51298619639024 |
| - type: nauc_precision_at_5_max |
| value: 36.483902995937235 |
| - type: nauc_precision_at_5_std |
| value: 15.45543901748184 |
| - type: nauc_recall_at_1000_diff1 |
| value: 19.402855614334317 |
| - type: nauc_recall_at_1000_max |
| value: 54.58840809219886 |
| - type: nauc_recall_at_1000_std |
| value: 53.59980637963878 |
| - type: nauc_recall_at_100_diff1 |
| value: 27.63391689753813 |
| - type: nauc_recall_at_100_max |
| value: 48.11832053014399 |
| - type: nauc_recall_at_100_std |
| value: 30.476790377619945 |
| - type: nauc_recall_at_10_diff1 |
| value: 34.00655805236221 |
| - type: nauc_recall_at_10_max |
| value: 41.78819015238207 |
| - type: nauc_recall_at_10_std |
| value: 11.709621782547302 |
| - type: nauc_recall_at_1_diff1 |
| value: 49.580098743143 |
| - type: nauc_recall_at_1_max |
| value: 33.624185967920326 |
| - type: nauc_recall_at_1_std |
| value: -7.896295502496881 |
| - type: nauc_recall_at_20_diff1 |
| value: 32.58237251319437 |
| - type: nauc_recall_at_20_max |
| value: 45.64540237392343 |
| - type: nauc_recall_at_20_std |
| value: 20.49216050873925 |
| - type: nauc_recall_at_3_diff1 |
| value: 35.68042917162092 |
| - type: nauc_recall_at_3_max |
| value: 36.41986013001979 |
| - type: nauc_recall_at_3_std |
| value: -0.24966469870022118 |
| - type: nauc_recall_at_5_diff1 |
| value: 35.53479753080461 |
| - type: nauc_recall_at_5_max |
| value: 39.57047856279735 |
| - type: nauc_recall_at_5_std |
| value: 3.999123969896682 |
| - type: ndcg_at_1 |
| value: 33.79 |
| - type: ndcg_at_10 |
| value: 45.334 |
| - type: ndcg_at_100 |
| value: 51.06 |
| - type: ndcg_at_1000 |
| value: 52.908 |
| - type: ndcg_at_20 |
| value: 47.776 |
| - type: ndcg_at_3 |
| value: 39.503 |
| - type: ndcg_at_5 |
| value: 42.308 |
| - type: precision_at_1 |
| value: 33.79 |
| - type: precision_at_10 |
| value: 8.505 |
| - type: precision_at_100 |
| value: 1.307 |
| - type: precision_at_1000 |
| value: 0.165 |
| - type: precision_at_20 |
| value: 4.994 |
| - type: precision_at_3 |
| value: 19.33 |
| - type: precision_at_5 |
| value: 14.063999999999998 |
| - type: recall_at_1 |
| value: 27.275 |
| - type: recall_at_10 |
| value: 59.453 |
| - type: recall_at_100 |
| value: 83.417 |
| - type: recall_at_1000 |
| value: 95.174 |
| - type: recall_at_20 |
| value: 68.195 |
| - type: recall_at_3 |
| value: 43.206 |
| - type: recall_at_5 |
| value: 50.397000000000006 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackRetrieval (default) |
| revision: CQADupstackRetrieval_is_a_combined_dataset |
| split: test |
| type: CQADupstackRetrieval_is_a_combined_dataset |
| metrics: |
| - type: main_score |
| value: 43.816 |
| - type: ndcg_at_10 |
| value: 43.816 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackStatsRetrieval (default) |
| revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a |
| split: test |
| type: mteb/cqadupstack-stats |
| metrics: |
| - type: main_score |
| value: 38.894 |
| - type: map_at_1 |
| value: 26.427 |
| - type: map_at_10 |
| value: 34.255 |
| - type: map_at_100 |
| value: 35.303000000000004 |
| - type: map_at_1000 |
| value: 35.404 |
| - type: map_at_20 |
| value: 34.774 |
| - type: map_at_3 |
| value: 31.695 |
| - type: map_at_5 |
| value: 33.202999999999996 |
| - type: mrr_at_1 |
| value: 29.601226993865033 |
| - type: mrr_at_10 |
| value: 37.08016846820527 |
| - type: mrr_at_100 |
| value: 37.98796986670183 |
| - type: mrr_at_1000 |
| value: 38.05125035681772 |
| - type: mrr_at_20 |
| value: 37.55764205112128 |
| - type: mrr_at_3 |
| value: 34.81595092024541 |
| - type: mrr_at_5 |
| value: 36.019938650306756 |
| - type: nauc_map_at_1000_diff1 |
| value: 56.60072218558477 |
| - type: nauc_map_at_1000_max |
| value: 38.11726050477455 |
| - type: nauc_map_at_1000_std |
| value: 1.0589013948647812 |
| - type: nauc_map_at_100_diff1 |
| value: 56.592212691088264 |
| - type: nauc_map_at_100_max |
| value: 38.09568149860661 |
| - type: nauc_map_at_100_std |
| value: 1.0392153444561998 |
| - type: nauc_map_at_10_diff1 |
| value: 56.86378015345323 |
| - type: nauc_map_at_10_max |
| value: 37.875244017016946 |
| - type: nauc_map_at_10_std |
| value: 0.6492500472958144 |
| - type: nauc_map_at_1_diff1 |
| value: 61.06511889290507 |
| - type: nauc_map_at_1_max |
| value: 37.14377732406466 |
| - type: nauc_map_at_1_std |
| value: -3.0410115573638064 |
| - type: nauc_map_at_20_diff1 |
| value: 56.65960048389336 |
| - type: nauc_map_at_20_max |
| value: 38.01063372743232 |
| - type: nauc_map_at_20_std |
| value: 0.7887134640638815 |
| - type: nauc_map_at_3_diff1 |
| value: 58.07356810929091 |
| - type: nauc_map_at_3_max |
| value: 37.49068261785256 |
| - type: nauc_map_at_3_std |
| value: -1.1929095993889525 |
| - type: nauc_map_at_5_diff1 |
| value: 57.50901814735278 |
| - type: nauc_map_at_5_max |
| value: 37.85923289090272 |
| - type: nauc_map_at_5_std |
| value: 0.059903065225492776 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 54.58824792518784 |
| - type: nauc_mrr_at_1000_max |
| value: 38.86931059709252 |
| - type: nauc_mrr_at_1000_std |
| value: 2.9986997791166368 |
| - type: nauc_mrr_at_100_diff1 |
| value: 54.57585597713184 |
| - type: nauc_mrr_at_100_max |
| value: 38.87313557690555 |
| - type: nauc_mrr_at_100_std |
| value: 3.004154480090834 |
| - type: nauc_mrr_at_10_diff1 |
| value: 54.750538678542725 |
| - type: nauc_mrr_at_10_max |
| value: 38.91736870335598 |
| - type: nauc_mrr_at_10_std |
| value: 2.827831779250098 |
| - type: nauc_mrr_at_1_diff1 |
| value: 58.42689852509982 |
| - type: nauc_mrr_at_1_max |
| value: 38.738304414401156 |
| - type: nauc_mrr_at_1_std |
| value: 0.20380762325184898 |
| - type: nauc_mrr_at_20_diff1 |
| value: 54.571333128033274 |
| - type: nauc_mrr_at_20_max |
| value: 38.82683538226168 |
| - type: nauc_mrr_at_20_std |
| value: 2.81272631376222 |
| - type: nauc_mrr_at_3_diff1 |
| value: 55.402618824410055 |
| - type: nauc_mrr_at_3_max |
| value: 38.770457076566686 |
| - type: nauc_mrr_at_3_std |
| value: 2.053522695739241 |
| - type: nauc_mrr_at_5_diff1 |
| value: 55.247338994146354 |
| - type: nauc_mrr_at_5_max |
| value: 39.03504319610805 |
| - type: nauc_mrr_at_5_std |
| value: 2.625757410773132 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 53.96113307294218 |
| - type: nauc_ndcg_at_1000_max |
| value: 39.50706897713246 |
| - type: nauc_ndcg_at_1000_std |
| value: 4.9387998806714934 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 53.85402259839868 |
| - type: nauc_ndcg_at_100_max |
| value: 39.56983171505153 |
| - type: nauc_ndcg_at_100_std |
| value: 4.972045278289709 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 54.71242559860603 |
| - type: nauc_ndcg_at_10_max |
| value: 38.581472160487685 |
| - type: nauc_ndcg_at_10_std |
| value: 2.839169333745226 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 58.42689852509982 |
| - type: nauc_ndcg_at_1_max |
| value: 38.738304414401156 |
| - type: nauc_ndcg_at_1_std |
| value: 0.20380762325184898 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 53.978219129570896 |
| - type: nauc_ndcg_at_20_max |
| value: 38.862218171161544 |
| - type: nauc_ndcg_at_20_std |
| value: 3.239351254035964 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 56.19488839726825 |
| - type: nauc_ndcg_at_3_max |
| value: 38.43663271574053 |
| - type: nauc_ndcg_at_3_std |
| value: 0.963285267513604 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 55.92862198714638 |
| - type: nauc_ndcg_at_5_max |
| value: 38.680176574203585 |
| - type: nauc_ndcg_at_5_std |
| value: 2.0517484488591657 |
| - type: nauc_precision_at_1000_diff1 |
| value: -10.093484727725837 |
| - type: nauc_precision_at_1000_max |
| value: 11.599506756878041 |
| - type: nauc_precision_at_1000_std |
| value: 16.104303375916956 |
| - type: nauc_precision_at_100_diff1 |
| value: 8.969090844678053 |
| - type: nauc_precision_at_100_max |
| value: 27.083136012889142 |
| - type: nauc_precision_at_100_std |
| value: 21.583675042204572 |
| - type: nauc_precision_at_10_diff1 |
| value: 33.02398417235467 |
| - type: nauc_precision_at_10_max |
| value: 36.19574777774318 |
| - type: nauc_precision_at_10_std |
| value: 15.536283231055586 |
| - type: nauc_precision_at_1_diff1 |
| value: 58.42689852509982 |
| - type: nauc_precision_at_1_max |
| value: 38.738304414401156 |
| - type: nauc_precision_at_1_std |
| value: 0.20380762325184898 |
| - type: nauc_precision_at_20_diff1 |
| value: 25.782064865016064 |
| - type: nauc_precision_at_20_max |
| value: 34.40259494180231 |
| - type: nauc_precision_at_20_std |
| value: 16.217527374266183 |
| - type: nauc_precision_at_3_diff1 |
| value: 47.01043944309824 |
| - type: nauc_precision_at_3_max |
| value: 38.470771808417766 |
| - type: nauc_precision_at_3_std |
| value: 7.132839594950563 |
| - type: nauc_precision_at_5_diff1 |
| value: 41.11616429779191 |
| - type: nauc_precision_at_5_max |
| value: 37.09283603644687 |
| - type: nauc_precision_at_5_std |
| value: 11.627051542109017 |
| - type: nauc_recall_at_1000_diff1 |
| value: 29.344095205506555 |
| - type: nauc_recall_at_1000_max |
| value: 46.58735252578747 |
| - type: nauc_recall_at_1000_std |
| value: 43.34763296426759 |
| - type: nauc_recall_at_100_diff1 |
| value: 40.43843747747295 |
| - type: nauc_recall_at_100_max |
| value: 42.50706821532735 |
| - type: nauc_recall_at_100_std |
| value: 21.22093617475044 |
| - type: nauc_recall_at_10_diff1 |
| value: 48.26433832406352 |
| - type: nauc_recall_at_10_max |
| value: 37.79745160062501 |
| - type: nauc_recall_at_10_std |
| value: 6.695186585419338 |
| - type: nauc_recall_at_1_diff1 |
| value: 61.06511889290507 |
| - type: nauc_recall_at_1_max |
| value: 37.14377732406466 |
| - type: nauc_recall_at_1_std |
| value: -3.0410115573638064 |
| - type: nauc_recall_at_20_diff1 |
| value: 44.50773149894022 |
| - type: nauc_recall_at_20_max |
| value: 38.219843285381856 |
| - type: nauc_recall_at_20_std |
| value: 8.199016503969196 |
| - type: nauc_recall_at_3_diff1 |
| value: 54.15714160224081 |
| - type: nauc_recall_at_3_max |
| value: 37.840736226935725 |
| - type: nauc_recall_at_3_std |
| value: 1.4933386616317446 |
| - type: nauc_recall_at_5_diff1 |
| value: 52.58026028311702 |
| - type: nauc_recall_at_5_max |
| value: 38.484030122838305 |
| - type: nauc_recall_at_5_std |
| value: 4.460832900300881 |
| - type: ndcg_at_1 |
| value: 29.601 |
| - type: ndcg_at_10 |
| value: 38.894 |
| - type: ndcg_at_100 |
| value: 44.04 |
| - type: ndcg_at_1000 |
| value: 46.382 |
| - type: ndcg_at_20 |
| value: 40.663 |
| - type: ndcg_at_3 |
| value: 34.236 |
| - type: ndcg_at_5 |
| value: 36.52 |
| - type: precision_at_1 |
| value: 29.601 |
| - type: precision_at_10 |
| value: 6.181 |
| - type: precision_at_100 |
| value: 0.9570000000000001 |
| - type: precision_at_1000 |
| value: 0.124 |
| - type: precision_at_20 |
| value: 3.566 |
| - type: precision_at_3 |
| value: 14.571000000000002 |
| - type: precision_at_5 |
| value: 10.337 |
| - type: recall_at_1 |
| value: 26.427 |
| - type: recall_at_10 |
| value: 50.214000000000006 |
| - type: recall_at_100 |
| value: 73.598 |
| - type: recall_at_1000 |
| value: 90.659 |
| - type: recall_at_20 |
| value: 56.842000000000006 |
| - type: recall_at_3 |
| value: 37.509 |
| - type: recall_at_5 |
| value: 43.061 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackTexRetrieval (default) |
| revision: 46989137a86843e03a6195de44b09deda022eec7 |
| split: test |
| type: mteb/cqadupstack-tex |
| metrics: |
| - type: main_score |
| value: 31.662000000000003 |
| - type: map_at_1 |
| value: 18.723 |
| - type: map_at_10 |
| value: 26.701000000000004 |
| - type: map_at_100 |
| value: 27.828000000000003 |
| - type: map_at_1000 |
| value: 27.954 |
| - type: map_at_20 |
| value: 27.278000000000002 |
| - type: map_at_3 |
| value: 24.154 |
| - type: map_at_5 |
| value: 25.55 |
| - type: mrr_at_1 |
| value: 22.40192704748796 |
| - type: mrr_at_10 |
| value: 30.449357650837396 |
| - type: mrr_at_100 |
| value: 31.366622972747187 |
| - type: mrr_at_1000 |
| value: 31.435964919479986 |
| - type: mrr_at_20 |
| value: 30.927227547273077 |
| - type: mrr_at_3 |
| value: 28.05115852259698 |
| - type: mrr_at_5 |
| value: 29.43106217022262 |
| - type: nauc_map_at_1000_diff1 |
| value: 40.29294330784634 |
| - type: nauc_map_at_1000_max |
| value: 30.36951944693726 |
| - type: nauc_map_at_1000_std |
| value: -0.3414834335787859 |
| - type: nauc_map_at_100_diff1 |
| value: 40.28120265458076 |
| - type: nauc_map_at_100_max |
| value: 30.36471186375651 |
| - type: nauc_map_at_100_std |
| value: -0.3335024521355652 |
| - type: nauc_map_at_10_diff1 |
| value: 40.54922279010274 |
| - type: nauc_map_at_10_max |
| value: 30.06425681128433 |
| - type: nauc_map_at_10_std |
| value: -0.9498753795017445 |
| - type: nauc_map_at_1_diff1 |
| value: 46.531783062841534 |
| - type: nauc_map_at_1_max |
| value: 27.458325853105315 |
| - type: nauc_map_at_1_std |
| value: -4.597119334637891 |
| - type: nauc_map_at_20_diff1 |
| value: 40.382854954927524 |
| - type: nauc_map_at_20_max |
| value: 30.250152473037033 |
| - type: nauc_map_at_20_std |
| value: -0.612621247842456 |
| - type: nauc_map_at_3_diff1 |
| value: 41.805903548458296 |
| - type: nauc_map_at_3_max |
| value: 29.902476093359216 |
| - type: nauc_map_at_3_std |
| value: -1.7418548848229358 |
| - type: nauc_map_at_5_diff1 |
| value: 40.971548027716956 |
| - type: nauc_map_at_5_max |
| value: 30.02180838754201 |
| - type: nauc_map_at_5_std |
| value: -1.341795240943666 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 39.95697123995655 |
| - type: nauc_mrr_at_1000_max |
| value: 31.997575460481613 |
| - type: nauc_mrr_at_1000_std |
| value: 0.4064232742934565 |
| - type: nauc_mrr_at_100_diff1 |
| value: 39.93042399360589 |
| - type: nauc_mrr_at_100_max |
| value: 31.996106010277902 |
| - type: nauc_mrr_at_100_std |
| value: 0.42019195064055487 |
| - type: nauc_mrr_at_10_diff1 |
| value: 40.07298006475225 |
| - type: nauc_mrr_at_10_max |
| value: 31.919593394912855 |
| - type: nauc_mrr_at_10_std |
| value: 0.02350450115938819 |
| - type: nauc_mrr_at_1_diff1 |
| value: 45.92456630155256 |
| - type: nauc_mrr_at_1_max |
| value: 30.624291065723035 |
| - type: nauc_mrr_at_1_std |
| value: -3.080621621197733 |
| - type: nauc_mrr_at_20_diff1 |
| value: 39.961455846237456 |
| - type: nauc_mrr_at_20_max |
| value: 32.006415548052416 |
| - type: nauc_mrr_at_20_std |
| value: 0.3198094423486476 |
| - type: nauc_mrr_at_3_diff1 |
| value: 41.32816822053059 |
| - type: nauc_mrr_at_3_max |
| value: 32.41066911321068 |
| - type: nauc_mrr_at_3_std |
| value: -0.6529950528921229 |
| - type: nauc_mrr_at_5_diff1 |
| value: 40.34219346934063 |
| - type: nauc_mrr_at_5_max |
| value: 32.04615580231512 |
| - type: nauc_mrr_at_5_std |
| value: -0.2914250580085147 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 37.474994751920576 |
| - type: nauc_ndcg_at_1000_max |
| value: 31.41222657464391 |
| - type: nauc_ndcg_at_1000_std |
| value: 3.240693443312849 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 37.03474261474229 |
| - type: nauc_ndcg_at_100_max |
| value: 31.497431680733584 |
| - type: nauc_ndcg_at_100_std |
| value: 3.700027399857788 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 38.03921533314436 |
| - type: nauc_ndcg_at_10_max |
| value: 30.78682453138251 |
| - type: nauc_ndcg_at_10_std |
| value: 0.8769594573808579 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 45.92456630155256 |
| - type: nauc_ndcg_at_1_max |
| value: 30.624291065723035 |
| - type: nauc_ndcg_at_1_std |
| value: -3.080621621197733 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 37.62104689563685 |
| - type: nauc_ndcg_at_20_max |
| value: 31.221003974077853 |
| - type: nauc_ndcg_at_20_std |
| value: 1.9883412769611548 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 40.17572316262669 |
| - type: nauc_ndcg_at_3_max |
| value: 31.203439927044585 |
| - type: nauc_ndcg_at_3_std |
| value: -0.712868414940749 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 38.848965800200695 |
| - type: nauc_ndcg_at_5_max |
| value: 30.90409092278334 |
| - type: nauc_ndcg_at_5_std |
| value: -0.07380105331601196 |
| - type: nauc_precision_at_1000_diff1 |
| value: 0.3488459536942616 |
| - type: nauc_precision_at_1000_max |
| value: 11.974221111911714 |
| - type: nauc_precision_at_1000_std |
| value: 5.545029664089995 |
| - type: nauc_precision_at_100_diff1 |
| value: 8.558130903347076 |
| - type: nauc_precision_at_100_max |
| value: 23.313159347579884 |
| - type: nauc_precision_at_100_std |
| value: 12.667615203365548 |
| - type: nauc_precision_at_10_diff1 |
| value: 23.1055686548991 |
| - type: nauc_precision_at_10_max |
| value: 30.62764918957524 |
| - type: nauc_precision_at_10_std |
| value: 5.655860099998371 |
| - type: nauc_precision_at_1_diff1 |
| value: 45.92456630155256 |
| - type: nauc_precision_at_1_max |
| value: 30.624291065723035 |
| - type: nauc_precision_at_1_std |
| value: -3.080621621197733 |
| - type: nauc_precision_at_20_diff1 |
| value: 19.02845795878823 |
| - type: nauc_precision_at_20_max |
| value: 29.986698288034308 |
| - type: nauc_precision_at_20_std |
| value: 8.65839413322005 |
| - type: nauc_precision_at_3_diff1 |
| value: 33.537119810716284 |
| - type: nauc_precision_at_3_max |
| value: 33.88768604457864 |
| - type: nauc_precision_at_3_std |
| value: 2.2581668899844054 |
| - type: nauc_precision_at_5_diff1 |
| value: 28.667111448412143 |
| - type: nauc_precision_at_5_max |
| value: 32.707947446614234 |
| - type: nauc_precision_at_5_std |
| value: 3.633065285428966 |
| - type: nauc_recall_at_1000_diff1 |
| value: 19.260215950407126 |
| - type: nauc_recall_at_1000_max |
| value: 29.880298126037186 |
| - type: nauc_recall_at_1000_std |
| value: 29.313220294243376 |
| - type: nauc_recall_at_100_diff1 |
| value: 22.618647334080375 |
| - type: nauc_recall_at_100_max |
| value: 30.06708168274523 |
| - type: nauc_recall_at_100_std |
| value: 19.578709404274342 |
| - type: nauc_recall_at_10_diff1 |
| value: 29.745906783751813 |
| - type: nauc_recall_at_10_max |
| value: 28.613864193571125 |
| - type: nauc_recall_at_10_std |
| value: 4.836841344636072 |
| - type: nauc_recall_at_1_diff1 |
| value: 46.531783062841534 |
| - type: nauc_recall_at_1_max |
| value: 27.458325853105315 |
| - type: nauc_recall_at_1_std |
| value: -4.597119334637891 |
| - type: nauc_recall_at_20_diff1 |
| value: 28.092320196353327 |
| - type: nauc_recall_at_20_max |
| value: 29.617127996080235 |
| - type: nauc_recall_at_20_std |
| value: 8.59271280643495 |
| - type: nauc_recall_at_3_diff1 |
| value: 35.81724087499039 |
| - type: nauc_recall_at_3_max |
| value: 30.1701581709378 |
| - type: nauc_recall_at_3_std |
| value: 1.038654228057759 |
| - type: nauc_recall_at_5_diff1 |
| value: 32.38568644423286 |
| - type: nauc_recall_at_5_max |
| value: 29.263454173692914 |
| - type: nauc_recall_at_5_std |
| value: 2.1188458895997964 |
| - type: ndcg_at_1 |
| value: 22.402 |
| - type: ndcg_at_10 |
| value: 31.662000000000003 |
| - type: ndcg_at_100 |
| value: 37.065 |
| - type: ndcg_at_1000 |
| value: 39.864 |
| - type: ndcg_at_20 |
| value: 33.533 |
| - type: ndcg_at_3 |
| value: 27.131 |
| - type: ndcg_at_5 |
| value: 29.223 |
| - type: precision_at_1 |
| value: 22.402 |
| - type: precision_at_10 |
| value: 5.7669999999999995 |
| - type: precision_at_100 |
| value: 0.992 |
| - type: precision_at_1000 |
| value: 0.14200000000000002 |
| - type: precision_at_20 |
| value: 3.45 |
| - type: precision_at_3 |
| value: 12.801000000000002 |
| - type: precision_at_5 |
| value: 9.277000000000001 |
| - type: recall_at_1 |
| value: 18.723 |
| - type: recall_at_10 |
| value: 42.738 |
| - type: recall_at_100 |
| value: 67.066 |
| - type: recall_at_1000 |
| value: 86.825 |
| - type: recall_at_20 |
| value: 49.641999999999996 |
| - type: recall_at_3 |
| value: 30.176 |
| - type: recall_at_5 |
| value: 35.5 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackUnixRetrieval (default) |
| revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 |
| split: test |
| type: mteb/cqadupstack-unix |
| metrics: |
| - type: main_score |
| value: 44.449 |
| - type: map_at_1 |
| value: 28.502 |
| - type: map_at_10 |
| value: 38.763 |
| - type: map_at_100 |
| value: 39.904 |
| - type: map_at_1000 |
| value: 40.003 |
| - type: map_at_20 |
| value: 39.379 |
| - type: map_at_3 |
| value: 35.367 |
| - type: map_at_5 |
| value: 37.480000000000004 |
| - type: mrr_at_1 |
| value: 33.2089552238806 |
| - type: mrr_at_10 |
| value: 42.56833392561001 |
| - type: mrr_at_100 |
| value: 43.42198225922794 |
| - type: mrr_at_1000 |
| value: 43.47454573766307 |
| - type: mrr_at_20 |
| value: 43.07361302641885 |
| - type: mrr_at_3 |
| value: 39.69216417910445 |
| - type: mrr_at_5 |
| value: 41.55783582089548 |
| - type: nauc_map_at_1000_diff1 |
| value: 51.09337515742835 |
| - type: nauc_map_at_1000_max |
| value: 45.11970808684597 |
| - type: nauc_map_at_1000_std |
| value: -0.3050907542347147 |
| - type: nauc_map_at_100_diff1 |
| value: 51.06721661390107 |
| - type: nauc_map_at_100_max |
| value: 45.10647782134187 |
| - type: nauc_map_at_100_std |
| value: -0.3098521100683701 |
| - type: nauc_map_at_10_diff1 |
| value: 51.09634307647701 |
| - type: nauc_map_at_10_max |
| value: 44.88824042512123 |
| - type: nauc_map_at_10_std |
| value: -0.738023844952336 |
| - type: nauc_map_at_1_diff1 |
| value: 58.91703951287665 |
| - type: nauc_map_at_1_max |
| value: 45.72426414838986 |
| - type: nauc_map_at_1_std |
| value: -4.450728836265055 |
| - type: nauc_map_at_20_diff1 |
| value: 51.07388859373564 |
| - type: nauc_map_at_20_max |
| value: 45.00318357068444 |
| - type: nauc_map_at_20_std |
| value: -0.4592556029173754 |
| - type: nauc_map_at_3_diff1 |
| value: 52.42891770025886 |
| - type: nauc_map_at_3_max |
| value: 44.64071416768749 |
| - type: nauc_map_at_3_std |
| value: -1.973140517009083 |
| - type: nauc_map_at_5_diff1 |
| value: 51.46402142721789 |
| - type: nauc_map_at_5_max |
| value: 44.626241564092766 |
| - type: nauc_map_at_5_std |
| value: -1.3987944859200176 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 49.56702747138606 |
| - type: nauc_mrr_at_1000_max |
| value: 44.979023748989455 |
| - type: nauc_mrr_at_1000_std |
| value: 0.25357932059734145 |
| - type: nauc_mrr_at_100_diff1 |
| value: 49.55224379363242 |
| - type: nauc_mrr_at_100_max |
| value: 44.97552508561541 |
| - type: nauc_mrr_at_100_std |
| value: 0.2748073187838927 |
| - type: nauc_mrr_at_10_diff1 |
| value: 49.39262295091568 |
| - type: nauc_mrr_at_10_max |
| value: 44.86831322043138 |
| - type: nauc_mrr_at_10_std |
| value: -0.04250684838053287 |
| - type: nauc_mrr_at_1_diff1 |
| value: 56.601138443656374 |
| - type: nauc_mrr_at_1_max |
| value: 46.155192599962 |
| - type: nauc_mrr_at_1_std |
| value: -3.841997988555605 |
| - type: nauc_mrr_at_20_diff1 |
| value: 49.48965201514485 |
| - type: nauc_mrr_at_20_max |
| value: 44.95960437502683 |
| - type: nauc_mrr_at_20_std |
| value: 0.26731422621033557 |
| - type: nauc_mrr_at_3_diff1 |
| value: 50.29192393046979 |
| - type: nauc_mrr_at_3_max |
| value: 45.211752965469316 |
| - type: nauc_mrr_at_3_std |
| value: -0.815057190995277 |
| - type: nauc_mrr_at_5_diff1 |
| value: 49.351603311309944 |
| - type: nauc_mrr_at_5_max |
| value: 44.88983601960641 |
| - type: nauc_mrr_at_5_std |
| value: -0.20982880810105417 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 48.58354462551937 |
| - type: nauc_ndcg_at_1000_max |
| value: 45.35705584395072 |
| - type: nauc_ndcg_at_1000_std |
| value: 2.54888435337591 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 47.83163408000412 |
| - type: nauc_ndcg_at_100_max |
| value: 45.0343949365134 |
| - type: nauc_ndcg_at_100_std |
| value: 2.980663545406531 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 47.7815366065242 |
| - type: nauc_ndcg_at_10_max |
| value: 44.36773394568082 |
| - type: nauc_ndcg_at_10_std |
| value: 1.02609790224527 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 56.601138443656374 |
| - type: nauc_ndcg_at_1_max |
| value: 46.155192599962 |
| - type: nauc_ndcg_at_1_std |
| value: -3.841997988555605 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 47.811909658082875 |
| - type: nauc_ndcg_at_20_max |
| value: 44.75137852464418 |
| - type: nauc_ndcg_at_20_std |
| value: 2.134275377210533 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 49.47165833829449 |
| - type: nauc_ndcg_at_3_max |
| value: 44.262246595483504 |
| - type: nauc_ndcg_at_3_std |
| value: -0.7284730096045571 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 48.32213730788881 |
| - type: nauc_ndcg_at_5_max |
| value: 44.132802200940915 |
| - type: nauc_ndcg_at_5_std |
| value: -0.08748854908072565 |
| - type: nauc_precision_at_1000_diff1 |
| value: -12.118988897199308 |
| - type: nauc_precision_at_1000_max |
| value: -0.7874363151972603 |
| - type: nauc_precision_at_1000_std |
| value: 8.882438027481804 |
| - type: nauc_precision_at_100_diff1 |
| value: -1.5152805469221087 |
| - type: nauc_precision_at_100_max |
| value: 14.090477325838059 |
| - type: nauc_precision_at_100_std |
| value: 14.149937999086665 |
| - type: nauc_precision_at_10_diff1 |
| value: 17.801742598469346 |
| - type: nauc_precision_at_10_max |
| value: 30.090739958907363 |
| - type: nauc_precision_at_10_std |
| value: 8.436791801910433 |
| - type: nauc_precision_at_1_diff1 |
| value: 56.601138443656374 |
| - type: nauc_precision_at_1_max |
| value: 46.155192599962 |
| - type: nauc_precision_at_1_std |
| value: -3.841997988555605 |
| - type: nauc_precision_at_20_diff1 |
| value: 12.84761699215353 |
| - type: nauc_precision_at_20_max |
| value: 26.67211391302849 |
| - type: nauc_precision_at_20_std |
| value: 11.133320866028658 |
| - type: nauc_precision_at_3_diff1 |
| value: 34.16116836040259 |
| - type: nauc_precision_at_3_max |
| value: 38.22148520643311 |
| - type: nauc_precision_at_3_std |
| value: 2.5818944979518905 |
| - type: nauc_precision_at_5_diff1 |
| value: 26.530376251979483 |
| - type: nauc_precision_at_5_max |
| value: 34.69034452388472 |
| - type: nauc_precision_at_5_std |
| value: 4.676074349833495 |
| - type: nauc_recall_at_1000_diff1 |
| value: 28.911934383429955 |
| - type: nauc_recall_at_1000_max |
| value: 50.212785017522506 |
| - type: nauc_recall_at_1000_std |
| value: 42.3629198766138 |
| - type: nauc_recall_at_100_diff1 |
| value: 31.381571692996857 |
| - type: nauc_recall_at_100_max |
| value: 41.01191885765792 |
| - type: nauc_recall_at_100_std |
| value: 20.857143634593037 |
| - type: nauc_recall_at_10_diff1 |
| value: 37.163994333372706 |
| - type: nauc_recall_at_10_max |
| value: 39.94892539019631 |
| - type: nauc_recall_at_10_std |
| value: 5.290418976361259 |
| - type: nauc_recall_at_1_diff1 |
| value: 58.91703951287665 |
| - type: nauc_recall_at_1_max |
| value: 45.72426414838986 |
| - type: nauc_recall_at_1_std |
| value: -4.450728836265055 |
| - type: nauc_recall_at_20_diff1 |
| value: 35.99281443407049 |
| - type: nauc_recall_at_20_max |
| value: 40.83481293624789 |
| - type: nauc_recall_at_20_std |
| value: 10.3889242981396 |
| - type: nauc_recall_at_3_diff1 |
| value: 44.15971877810932 |
| - type: nauc_recall_at_3_max |
| value: 41.75661191827119 |
| - type: nauc_recall_at_3_std |
| value: 0.22409370715719445 |
| - type: nauc_recall_at_5_diff1 |
| value: 39.79497306179428 |
| - type: nauc_recall_at_5_max |
| value: 40.39551747161536 |
| - type: nauc_recall_at_5_std |
| value: 2.3509968624532975 |
| - type: ndcg_at_1 |
| value: 33.209 |
| - type: ndcg_at_10 |
| value: 44.449 |
| - type: ndcg_at_100 |
| value: 49.541000000000004 |
| - type: ndcg_at_1000 |
| value: 51.66 |
| - type: ndcg_at_20 |
| value: 46.361000000000004 |
| - type: ndcg_at_3 |
| value: 38.61 |
| - type: ndcg_at_5 |
| value: 41.802 |
| - type: precision_at_1 |
| value: 33.209 |
| - type: precision_at_10 |
| value: 7.556 |
| - type: precision_at_100 |
| value: 1.124 |
| - type: precision_at_1000 |
| value: 0.14200000000000002 |
| - type: precision_at_20 |
| value: 4.314 |
| - type: precision_at_3 |
| value: 17.506 |
| - type: precision_at_5 |
| value: 12.668 |
| - type: recall_at_1 |
| value: 28.502 |
| - type: recall_at_10 |
| value: 57.781000000000006 |
| - type: recall_at_100 |
| value: 79.831 |
| - type: recall_at_1000 |
| value: 94.462 |
| - type: recall_at_20 |
| value: 64.565 |
| - type: recall_at_3 |
| value: 42.229 |
| - type: recall_at_5 |
| value: 50.144 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackWebmastersRetrieval (default) |
| revision: 160c094312a0e1facb97e55eeddb698c0abe3571 |
| split: test |
| type: mteb/cqadupstack-webmasters |
| metrics: |
| - type: main_score |
| value: 43.342000000000006 |
| - type: map_at_1 |
| value: 26.924999999999997 |
| - type: map_at_10 |
| value: 37.29 |
| - type: map_at_100 |
| value: 38.906 |
| - type: map_at_1000 |
| value: 39.129999999999995 |
| - type: map_at_20 |
| value: 38.1 |
| - type: map_at_3 |
| value: 34.35 |
| - type: map_at_5 |
| value: 35.955999999999996 |
| - type: mrr_at_1 |
| value: 32.21343873517787 |
| - type: mrr_at_10 |
| value: 41.73168015559322 |
| - type: mrr_at_100 |
| value: 42.81722926727115 |
| - type: mrr_at_1000 |
| value: 42.8578246510941 |
| - type: mrr_at_20 |
| value: 42.40869869880568 |
| - type: mrr_at_3 |
| value: 39.32806324110674 |
| - type: mrr_at_5 |
| value: 40.632411067193694 |
| - type: nauc_map_at_1000_diff1 |
| value: 39.56475855970276 |
| - type: nauc_map_at_1000_max |
| value: 38.177848390247235 |
| - type: nauc_map_at_1000_std |
| value: 6.9335353798460675 |
| - type: nauc_map_at_100_diff1 |
| value: 39.516419907405805 |
| - type: nauc_map_at_100_max |
| value: 38.36036331899402 |
| - type: nauc_map_at_100_std |
| value: 6.831480671192796 |
| - type: nauc_map_at_10_diff1 |
| value: 39.48256807532493 |
| - type: nauc_map_at_10_max |
| value: 38.197849516463194 |
| - type: nauc_map_at_10_std |
| value: 5.253983146776727 |
| - type: nauc_map_at_1_diff1 |
| value: 46.567254266614846 |
| - type: nauc_map_at_1_max |
| value: 37.732540483896635 |
| - type: nauc_map_at_1_std |
| value: 2.2489282023963955 |
| - type: nauc_map_at_20_diff1 |
| value: 39.378259059028046 |
| - type: nauc_map_at_20_max |
| value: 38.2189463642111 |
| - type: nauc_map_at_20_std |
| value: 6.056542688093049 |
| - type: nauc_map_at_3_diff1 |
| value: 40.40449060760161 |
| - type: nauc_map_at_3_max |
| value: 37.99871952048906 |
| - type: nauc_map_at_3_std |
| value: 3.4100661197624476 |
| - type: nauc_map_at_5_diff1 |
| value: 40.1519126124995 |
| - type: nauc_map_at_5_max |
| value: 37.95919343694378 |
| - type: nauc_map_at_5_std |
| value: 4.571457569129526 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 38.64403309308046 |
| - type: nauc_mrr_at_1000_max |
| value: 37.17525534091487 |
| - type: nauc_mrr_at_1000_std |
| value: 8.438248626531607 |
| - type: nauc_mrr_at_100_diff1 |
| value: 38.62191111052577 |
| - type: nauc_mrr_at_100_max |
| value: 37.16381346460307 |
| - type: nauc_mrr_at_100_std |
| value: 8.473494626840806 |
| - type: nauc_mrr_at_10_diff1 |
| value: 38.66598566082418 |
| - type: nauc_mrr_at_10_max |
| value: 37.356872781907384 |
| - type: nauc_mrr_at_10_std |
| value: 8.494041634436822 |
| - type: nauc_mrr_at_1_diff1 |
| value: 43.817215062943916 |
| - type: nauc_mrr_at_1_max |
| value: 37.39185593941398 |
| - type: nauc_mrr_at_1_std |
| value: 7.28642602050739 |
| - type: nauc_mrr_at_20_diff1 |
| value: 38.44686926468191 |
| - type: nauc_mrr_at_20_max |
| value: 37.09242707803003 |
| - type: nauc_mrr_at_20_std |
| value: 8.336904051478186 |
| - type: nauc_mrr_at_3_diff1 |
| value: 38.15060994005348 |
| - type: nauc_mrr_at_3_max |
| value: 36.815987651583306 |
| - type: nauc_mrr_at_3_std |
| value: 6.854787905916098 |
| - type: nauc_mrr_at_5_diff1 |
| value: 38.89757601751886 |
| - type: nauc_mrr_at_5_max |
| value: 37.19178420763993 |
| - type: nauc_mrr_at_5_std |
| value: 7.704930194711135 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 37.52136803315935 |
| - type: nauc_ndcg_at_1000_max |
| value: 38.92408944416557 |
| - type: nauc_ndcg_at_1000_std |
| value: 10.871928230197692 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 37.13360414141896 |
| - type: nauc_ndcg_at_100_max |
| value: 39.0053807375677 |
| - type: nauc_ndcg_at_100_std |
| value: 11.489300764908352 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 36.90485505709437 |
| - type: nauc_ndcg_at_10_max |
| value: 37.617894869105406 |
| - type: nauc_ndcg_at_10_std |
| value: 8.905497675458868 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 43.817215062943916 |
| - type: nauc_ndcg_at_1_max |
| value: 37.39185593941398 |
| - type: nauc_ndcg_at_1_std |
| value: 7.28642602050739 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 36.48691469681143 |
| - type: nauc_ndcg_at_20_max |
| value: 37.621472858058546 |
| - type: nauc_ndcg_at_20_std |
| value: 9.632107687173814 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 37.5454366452348 |
| - type: nauc_ndcg_at_3_max |
| value: 37.26941098955138 |
| - type: nauc_ndcg_at_3_std |
| value: 6.299967228476719 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 38.11812602665602 |
| - type: nauc_ndcg_at_5_max |
| value: 37.1666041307787 |
| - type: nauc_ndcg_at_5_std |
| value: 7.918994950799998 |
| - type: nauc_precision_at_1000_diff1 |
| value: -2.2969824543205806 |
| - type: nauc_precision_at_1000_max |
| value: -15.419366952284975 |
| - type: nauc_precision_at_1000_std |
| value: 19.12966399374656 |
| - type: nauc_precision_at_100_diff1 |
| value: -1.021770567948756 |
| - type: nauc_precision_at_100_max |
| value: -1.8775299175206996 |
| - type: nauc_precision_at_100_std |
| value: 27.24690244968834 |
| - type: nauc_precision_at_10_diff1 |
| value: 10.980118436692694 |
| - type: nauc_precision_at_10_max |
| value: 22.43559969209056 |
| - type: nauc_precision_at_10_std |
| value: 23.820891112348573 |
| - type: nauc_precision_at_1_diff1 |
| value: 43.817215062943916 |
| - type: nauc_precision_at_1_max |
| value: 37.39185593941398 |
| - type: nauc_precision_at_1_std |
| value: 7.28642602050739 |
| - type: nauc_precision_at_20_diff1 |
| value: 4.804175264538657 |
| - type: nauc_precision_at_20_max |
| value: 15.499790519728988 |
| - type: nauc_precision_at_20_std |
| value: 29.509462091568256 |
| - type: nauc_precision_at_3_diff1 |
| value: 21.43695233004016 |
| - type: nauc_precision_at_3_max |
| value: 31.880319956722815 |
| - type: nauc_precision_at_3_std |
| value: 13.059502909551176 |
| - type: nauc_precision_at_5_diff1 |
| value: 18.363478978651912 |
| - type: nauc_precision_at_5_max |
| value: 27.088121521248816 |
| - type: nauc_precision_at_5_std |
| value: 18.341614521330147 |
| - type: nauc_recall_at_1000_diff1 |
| value: 31.26577486561114 |
| - type: nauc_recall_at_1000_max |
| value: 64.08514957152025 |
| - type: nauc_recall_at_1000_std |
| value: 59.55425703698939 |
| - type: nauc_recall_at_100_diff1 |
| value: 26.66049572028577 |
| - type: nauc_recall_at_100_max |
| value: 43.36087610846491 |
| - type: nauc_recall_at_100_std |
| value: 35.593597922216865 |
| - type: nauc_recall_at_10_diff1 |
| value: 27.7772025462008 |
| - type: nauc_recall_at_10_max |
| value: 35.99035214574843 |
| - type: nauc_recall_at_10_std |
| value: 12.180058133691604 |
| - type: nauc_recall_at_1_diff1 |
| value: 46.567254266614846 |
| - type: nauc_recall_at_1_max |
| value: 37.732540483896635 |
| - type: nauc_recall_at_1_std |
| value: 2.2489282023963955 |
| - type: nauc_recall_at_20_diff1 |
| value: 25.280727909671363 |
| - type: nauc_recall_at_20_max |
| value: 34.24681065861685 |
| - type: nauc_recall_at_20_std |
| value: 16.674472276356063 |
| - type: nauc_recall_at_3_diff1 |
| value: 32.639943281033354 |
| - type: nauc_recall_at_3_max |
| value: 35.48634586230576 |
| - type: nauc_recall_at_3_std |
| value: 2.7588471369487557 |
| - type: nauc_recall_at_5_diff1 |
| value: 32.46681634072349 |
| - type: nauc_recall_at_5_max |
| value: 35.526045994502745 |
| - type: nauc_recall_at_5_std |
| value: 6.660060598477094 |
| - type: ndcg_at_1 |
| value: 32.213 |
| - type: ndcg_at_10 |
| value: 43.342000000000006 |
| - type: ndcg_at_100 |
| value: 49.484 |
| - type: ndcg_at_1000 |
| value: 51.507999999999996 |
| - type: ndcg_at_20 |
| value: 45.614 |
| - type: ndcg_at_3 |
| value: 38.84 |
| - type: ndcg_at_5 |
| value: 40.894999999999996 |
| - type: precision_at_1 |
| value: 32.213 |
| - type: precision_at_10 |
| value: 8.103 |
| - type: precision_at_100 |
| value: 1.625 |
| - type: precision_at_1000 |
| value: 0.246 |
| - type: precision_at_20 |
| value: 5.0889999999999995 |
| - type: precision_at_3 |
| value: 18.379 |
| - type: precision_at_5 |
| value: 13.123000000000001 |
| - type: recall_at_1 |
| value: 26.924999999999997 |
| - type: recall_at_10 |
| value: 55.249 |
| - type: recall_at_100 |
| value: 82.34 |
| - type: recall_at_1000 |
| value: 94.368 |
| - type: recall_at_20 |
| value: 63.757 |
| - type: recall_at_3 |
| value: 42.062 |
| - type: recall_at_5 |
| value: 47.615 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CQADupstackWordpressRetrieval (default) |
| revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 |
| split: test |
| type: mteb/cqadupstack-wordpress |
| metrics: |
| - type: main_score |
| value: 35.774 |
| - type: map_at_1 |
| value: 22.779 |
| - type: map_at_10 |
| value: 30.747000000000003 |
| - type: map_at_100 |
| value: 31.783 |
| - type: map_at_1000 |
| value: 31.872 |
| - type: map_at_20 |
| value: 31.274 |
| - type: map_at_3 |
| value: 27.96 |
| - type: map_at_5 |
| value: 29.537999999999997 |
| - type: mrr_at_1 |
| value: 25.13863216266174 |
| - type: mrr_at_10 |
| value: 33.15311152187305 |
| - type: mrr_at_100 |
| value: 34.0591979023387 |
| - type: mrr_at_1000 |
| value: 34.120150414093445 |
| - type: mrr_at_20 |
| value: 33.62239000549977 |
| - type: mrr_at_3 |
| value: 30.499075785582253 |
| - type: mrr_at_5 |
| value: 32.01478743068391 |
| - type: nauc_map_at_1000_diff1 |
| value: 40.9666267178634 |
| - type: nauc_map_at_1000_max |
| value: 35.512382177489464 |
| - type: nauc_map_at_1000_std |
| value: 0.6224247525822328 |
| - type: nauc_map_at_100_diff1 |
| value: 40.935071530613016 |
| - type: nauc_map_at_100_max |
| value: 35.4689334665505 |
| - type: nauc_map_at_100_std |
| value: 0.5881898556397818 |
| - type: nauc_map_at_10_diff1 |
| value: 41.09587027828798 |
| - type: nauc_map_at_10_max |
| value: 35.57960780251561 |
| - type: nauc_map_at_10_std |
| value: 0.21238247793288179 |
| - type: nauc_map_at_1_diff1 |
| value: 46.79723740072981 |
| - type: nauc_map_at_1_max |
| value: 37.68968438458517 |
| - type: nauc_map_at_1_std |
| value: -2.785325878591901 |
| - type: nauc_map_at_20_diff1 |
| value: 41.02661398711254 |
| - type: nauc_map_at_20_max |
| value: 35.61017903374831 |
| - type: nauc_map_at_20_std |
| value: 0.45478618525492803 |
| - type: nauc_map_at_3_diff1 |
| value: 41.89999642256378 |
| - type: nauc_map_at_3_max |
| value: 35.97333460925634 |
| - type: nauc_map_at_3_std |
| value: -1.0669866710282385 |
| - type: nauc_map_at_5_diff1 |
| value: 41.18936334778094 |
| - type: nauc_map_at_5_max |
| value: 35.651730615108626 |
| - type: nauc_map_at_5_std |
| value: 0.011285859606637189 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 41.39497842969287 |
| - type: nauc_mrr_at_1000_max |
| value: 36.819933081607815 |
| - type: nauc_mrr_at_1000_std |
| value: 1.8448453831538831 |
| - type: nauc_mrr_at_100_diff1 |
| value: 41.37543086993865 |
| - type: nauc_mrr_at_100_max |
| value: 36.79132840589643 |
| - type: nauc_mrr_at_100_std |
| value: 1.838173324273364 |
| - type: nauc_mrr_at_10_diff1 |
| value: 41.470486495069444 |
| - type: nauc_mrr_at_10_max |
| value: 36.94185193360758 |
| - type: nauc_mrr_at_10_std |
| value: 1.594158944434542 |
| - type: nauc_mrr_at_1_diff1 |
| value: 46.9714558985573 |
| - type: nauc_mrr_at_1_max |
| value: 39.30041031657009 |
| - type: nauc_mrr_at_1_std |
| value: -1.4033670246089662 |
| - type: nauc_mrr_at_20_diff1 |
| value: 41.43921225771939 |
| - type: nauc_mrr_at_20_max |
| value: 36.940835903316156 |
| - type: nauc_mrr_at_20_std |
| value: 1.8059880944253215 |
| - type: nauc_mrr_at_3_diff1 |
| value: 42.56076877140861 |
| - type: nauc_mrr_at_3_max |
| value: 37.4774293466681 |
| - type: nauc_mrr_at_3_std |
| value: 0.38144918993605603 |
| - type: nauc_mrr_at_5_diff1 |
| value: 41.6764265802116 |
| - type: nauc_mrr_at_5_max |
| value: 37.16536369010265 |
| - type: nauc_mrr_at_5_std |
| value: 1.5570583318968474 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 38.41316402205857 |
| - type: nauc_ndcg_at_1000_max |
| value: 34.85354630049824 |
| - type: nauc_ndcg_at_1000_std |
| value: 4.1987917490658795 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 37.86931389576125 |
| - type: nauc_ndcg_at_100_max |
| value: 33.82378543079163 |
| - type: nauc_ndcg_at_100_std |
| value: 3.71084103573832 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 38.69891586370789 |
| - type: nauc_ndcg_at_10_max |
| value: 34.69158263560064 |
| - type: nauc_ndcg_at_10_std |
| value: 2.1218981018673686 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 46.9714558985573 |
| - type: nauc_ndcg_at_1_max |
| value: 39.30041031657009 |
| - type: nauc_ndcg_at_1_std |
| value: -1.4033670246089662 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 38.363883413392486 |
| - type: nauc_ndcg_at_20_max |
| value: 34.667105813813535 |
| - type: nauc_ndcg_at_20_std |
| value: 2.8624626654781267 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 40.5184686636588 |
| - type: nauc_ndcg_at_3_max |
| value: 36.186749852210276 |
| - type: nauc_ndcg_at_3_std |
| value: 0.09474904645558901 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 39.24674105485247 |
| - type: nauc_ndcg_at_5_max |
| value: 35.322707726631805 |
| - type: nauc_ndcg_at_5_std |
| value: 1.7788731747792517 |
| - type: nauc_precision_at_1000_diff1 |
| value: 0.840976692854083 |
| - type: nauc_precision_at_1000_max |
| value: 3.261240112540733 |
| - type: nauc_precision_at_1000_std |
| value: 13.248030938023359 |
| - type: nauc_precision_at_100_diff1 |
| value: 10.072671120295702 |
| - type: nauc_precision_at_100_max |
| value: 17.240545350712175 |
| - type: nauc_precision_at_100_std |
| value: 20.314577652155904 |
| - type: nauc_precision_at_10_diff1 |
| value: 27.23270077955099 |
| - type: nauc_precision_at_10_max |
| value: 31.79041137664875 |
| - type: nauc_precision_at_10_std |
| value: 12.36209307812828 |
| - type: nauc_precision_at_1_diff1 |
| value: 46.9714558985573 |
| - type: nauc_precision_at_1_max |
| value: 39.30041031657009 |
| - type: nauc_precision_at_1_std |
| value: -1.4033670246089662 |
| - type: nauc_precision_at_20_diff1 |
| value: 23.795751404068003 |
| - type: nauc_precision_at_20_max |
| value: 29.82598945857867 |
| - type: nauc_precision_at_20_std |
| value: 14.92149587103534 |
| - type: nauc_precision_at_3_diff1 |
| value: 35.61737074241893 |
| - type: nauc_precision_at_3_max |
| value: 36.40376544125899 |
| - type: nauc_precision_at_3_std |
| value: 3.957970514402529 |
| - type: nauc_precision_at_5_diff1 |
| value: 30.87385523346844 |
| - type: nauc_precision_at_5_max |
| value: 34.27637004357153 |
| - type: nauc_precision_at_5_std |
| value: 9.030928793088314 |
| - type: nauc_recall_at_1000_diff1 |
| value: 11.601671012375652 |
| - type: nauc_recall_at_1000_max |
| value: 26.78951022752499 |
| - type: nauc_recall_at_1000_std |
| value: 40.83415411964083 |
| - type: nauc_recall_at_100_diff1 |
| value: 21.74556181581925 |
| - type: nauc_recall_at_100_max |
| value: 20.184610136900506 |
| - type: nauc_recall_at_100_std |
| value: 14.965834834698247 |
| - type: nauc_recall_at_10_diff1 |
| value: 30.115838102082716 |
| - type: nauc_recall_at_10_max |
| value: 29.06496783929028 |
| - type: nauc_recall_at_10_std |
| value: 5.597874206979672 |
| - type: nauc_recall_at_1_diff1 |
| value: 46.79723740072981 |
| - type: nauc_recall_at_1_max |
| value: 37.68968438458517 |
| - type: nauc_recall_at_1_std |
| value: -2.785325878591901 |
| - type: nauc_recall_at_20_diff1 |
| value: 28.02766014573457 |
| - type: nauc_recall_at_20_max |
| value: 28.239856197087665 |
| - type: nauc_recall_at_20_std |
| value: 8.29181316012612 |
| - type: nauc_recall_at_3_diff1 |
| value: 35.432867785333514 |
| - type: nauc_recall_at_3_max |
| value: 34.103779675298135 |
| - type: nauc_recall_at_3_std |
| value: 0.7732759979316737 |
| - type: nauc_recall_at_5_diff1 |
| value: 32.427691466534284 |
| - type: nauc_recall_at_5_max |
| value: 31.865805435351113 |
| - type: nauc_recall_at_5_std |
| value: 4.798978447571004 |
| - type: ndcg_at_1 |
| value: 25.139 |
| - type: ndcg_at_10 |
| value: 35.774 |
| - type: ndcg_at_100 |
| value: 40.96 |
| - type: ndcg_at_1000 |
| value: 43.246 |
| - type: ndcg_at_20 |
| value: 37.556 |
| - type: ndcg_at_3 |
| value: 30.312 |
| - type: ndcg_at_5 |
| value: 32.99 |
| - type: precision_at_1 |
| value: 25.139 |
| - type: precision_at_10 |
| value: 5.638 |
| - type: precision_at_100 |
| value: 0.889 |
| - type: precision_at_1000 |
| value: 0.11900000000000001 |
| - type: precision_at_20 |
| value: 3.235 |
| - type: precision_at_3 |
| value: 12.815999999999999 |
| - type: precision_at_5 |
| value: 9.316 |
| - type: recall_at_1 |
| value: 22.779 |
| - type: recall_at_10 |
| value: 49.199 |
| - type: recall_at_100 |
| value: 73.063 |
| - type: recall_at_1000 |
| value: 90.239 |
| - type: recall_at_20 |
| value: 55.92700000000001 |
| - type: recall_at_3 |
| value: 34.187 |
| - type: recall_at_5 |
| value: 40.792 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB ClimateFEVER (default) |
| revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 |
| split: test |
| type: mteb/climate-fever |
| metrics: |
| - type: main_score |
| value: 38.916000000000004 |
| - type: map_at_1 |
| value: 17.651 |
| - type: map_at_10 |
| value: 29.168 |
| - type: map_at_100 |
| value: 31.227 |
| - type: map_at_1000 |
| value: 31.408 |
| - type: map_at_20 |
| value: 30.307000000000002 |
| - type: map_at_3 |
| value: 24.647 |
| - type: map_at_5 |
| value: 26.951999999999998 |
| - type: mrr_at_1 |
| value: 40.78175895765472 |
| - type: mrr_at_10 |
| value: 51.68646915878195 |
| - type: mrr_at_100 |
| value: 52.40689532965702 |
| - type: mrr_at_1000 |
| value: 52.43209269167376 |
| - type: mrr_at_20 |
| value: 52.154150109013976 |
| - type: mrr_at_3 |
| value: 48.88165038002178 |
| - type: mrr_at_5 |
| value: 50.595005428881755 |
| - type: nauc_map_at_1000_diff1 |
| value: 25.616803628427053 |
| - type: nauc_map_at_1000_max |
| value: 38.197304271991534 |
| - type: nauc_map_at_1000_std |
| value: 19.5031903830227 |
| - type: nauc_map_at_100_diff1 |
| value: 25.57833406545184 |
| - type: nauc_map_at_100_max |
| value: 38.14502692959517 |
| - type: nauc_map_at_100_std |
| value: 19.44077348129036 |
| - type: nauc_map_at_10_diff1 |
| value: 25.95091383785147 |
| - type: nauc_map_at_10_max |
| value: 37.4399376489563 |
| - type: nauc_map_at_10_std |
| value: 17.947548047679657 |
| - type: nauc_map_at_1_diff1 |
| value: 34.7369005540742 |
| - type: nauc_map_at_1_max |
| value: 31.66881962992226 |
| - type: nauc_map_at_1_std |
| value: 8.246764177334914 |
| - type: nauc_map_at_20_diff1 |
| value: 25.495779918015018 |
| - type: nauc_map_at_20_max |
| value: 37.929125739632724 |
| - type: nauc_map_at_20_std |
| value: 18.793855321849914 |
| - type: nauc_map_at_3_diff1 |
| value: 27.187355367399856 |
| - type: nauc_map_at_3_max |
| value: 34.81150705282639 |
| - type: nauc_map_at_3_std |
| value: 14.014713401966459 |
| - type: nauc_map_at_5_diff1 |
| value: 26.08862309808681 |
| - type: nauc_map_at_5_max |
| value: 36.53941111535009 |
| - type: nauc_map_at_5_std |
| value: 16.116511338225646 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 26.59855134120458 |
| - type: nauc_mrr_at_1000_max |
| value: 35.77488039055326 |
| - type: nauc_mrr_at_1000_std |
| value: 20.7389528120806 |
| - type: nauc_mrr_at_100_diff1 |
| value: 26.590778113868673 |
| - type: nauc_mrr_at_100_max |
| value: 35.78358772121562 |
| - type: nauc_mrr_at_100_std |
| value: 20.744288172404808 |
| - type: nauc_mrr_at_10_diff1 |
| value: 26.5870707300715 |
| - type: nauc_mrr_at_10_max |
| value: 35.913843868573636 |
| - type: nauc_mrr_at_10_std |
| value: 20.976090226892623 |
| - type: nauc_mrr_at_1_diff1 |
| value: 29.56564017983464 |
| - type: nauc_mrr_at_1_max |
| value: 31.301768011417288 |
| - type: nauc_mrr_at_1_std |
| value: 14.75858762264703 |
| - type: nauc_mrr_at_20_diff1 |
| value: 26.56231125681433 |
| - type: nauc_mrr_at_20_max |
| value: 35.86261857417216 |
| - type: nauc_mrr_at_20_std |
| value: 20.800435951726282 |
| - type: nauc_mrr_at_3_diff1 |
| value: 25.559942762135485 |
| - type: nauc_mrr_at_3_max |
| value: 33.97715426818164 |
| - type: nauc_mrr_at_3_std |
| value: 19.351416325209865 |
| - type: nauc_mrr_at_5_diff1 |
| value: 26.141041525037817 |
| - type: nauc_mrr_at_5_max |
| value: 35.71438745282619 |
| - type: nauc_mrr_at_5_std |
| value: 20.74875586641808 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 25.041788617432932 |
| - type: nauc_ndcg_at_1000_max |
| value: 41.54576923132739 |
| - type: nauc_ndcg_at_1000_std |
| value: 26.51151915620546 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 24.43191211493594 |
| - type: nauc_ndcg_at_100_max |
| value: 40.847650283984564 |
| - type: nauc_ndcg_at_100_std |
| value: 25.75277697297615 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 25.233390869628174 |
| - type: nauc_ndcg_at_10_max |
| value: 39.62949324017721 |
| - type: nauc_ndcg_at_10_std |
| value: 22.2244036894323 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 29.56564017983464 |
| - type: nauc_ndcg_at_1_max |
| value: 31.301768011417288 |
| - type: nauc_ndcg_at_1_std |
| value: 14.75858762264703 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 24.27597965113978 |
| - type: nauc_ndcg_at_20_max |
| value: 40.393728924358356 |
| - type: nauc_ndcg_at_20_std |
| value: 23.674954170697884 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 24.922976501121497 |
| - type: nauc_ndcg_at_3_max |
| value: 35.03015688782362 |
| - type: nauc_ndcg_at_3_std |
| value: 17.155078928887757 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 24.781977823206624 |
| - type: nauc_ndcg_at_5_max |
| value: 38.07227204290295 |
| - type: nauc_ndcg_at_5_std |
| value: 19.693694672125837 |
| - type: nauc_precision_at_1000_diff1 |
| value: -4.115704930962564 |
| - type: nauc_precision_at_1000_max |
| value: 11.647989646622849 |
| - type: nauc_precision_at_1000_std |
| value: 25.566852614568838 |
| - type: nauc_precision_at_100_diff1 |
| value: 0.5157774949932177 |
| - type: nauc_precision_at_100_max |
| value: 21.45532828240429 |
| - type: nauc_precision_at_100_std |
| value: 30.553114749973965 |
| - type: nauc_precision_at_10_diff1 |
| value: 9.34584765889552 |
| - type: nauc_precision_at_10_max |
| value: 32.16000278371526 |
| - type: nauc_precision_at_10_std |
| value: 29.35892375659281 |
| - type: nauc_precision_at_1_diff1 |
| value: 29.56564017983464 |
| - type: nauc_precision_at_1_max |
| value: 31.301768011417288 |
| - type: nauc_precision_at_1_std |
| value: 14.75858762264703 |
| - type: nauc_precision_at_20_diff1 |
| value: 4.9990736206660396 |
| - type: nauc_precision_at_20_max |
| value: 29.872088450680923 |
| - type: nauc_precision_at_20_std |
| value: 30.216489116173488 |
| - type: nauc_precision_at_3_diff1 |
| value: 12.798858008292857 |
| - type: nauc_precision_at_3_max |
| value: 32.78603926269799 |
| - type: nauc_precision_at_3_std |
| value: 23.721222519146444 |
| - type: nauc_precision_at_5_diff1 |
| value: 10.229001376896228 |
| - type: nauc_precision_at_5_max |
| value: 34.26562428041649 |
| - type: nauc_precision_at_5_std |
| value: 27.123249202755378 |
| - type: nauc_recall_at_1000_diff1 |
| value: 15.410365830541176 |
| - type: nauc_recall_at_1000_max |
| value: 49.072553664240615 |
| - type: nauc_recall_at_1000_std |
| value: 49.891439906063205 |
| - type: nauc_recall_at_100_diff1 |
| value: 14.456552580056254 |
| - type: nauc_recall_at_100_max |
| value: 39.36987722093516 |
| - type: nauc_recall_at_100_std |
| value: 34.38344967422128 |
| - type: nauc_recall_at_10_diff1 |
| value: 20.23507425095135 |
| - type: nauc_recall_at_10_max |
| value: 39.51589936709692 |
| - type: nauc_recall_at_10_std |
| value: 24.500141888364887 |
| - type: nauc_recall_at_1_diff1 |
| value: 34.7369005540742 |
| - type: nauc_recall_at_1_max |
| value: 31.66881962992226 |
| - type: nauc_recall_at_1_std |
| value: 8.246764177334914 |
| - type: nauc_recall_at_20_diff1 |
| value: 16.101425670461474 |
| - type: nauc_recall_at_20_max |
| value: 39.169188223543586 |
| - type: nauc_recall_at_20_std |
| value: 26.926527703712676 |
| - type: nauc_recall_at_3_diff1 |
| value: 22.562156632821342 |
| - type: nauc_recall_at_3_max |
| value: 35.43366423709469 |
| - type: nauc_recall_at_3_std |
| value: 17.267094045670074 |
| - type: nauc_recall_at_5_diff1 |
| value: 20.62436789996695 |
| - type: nauc_recall_at_5_max |
| value: 38.89822406895274 |
| - type: nauc_recall_at_5_std |
| value: 21.051016860426518 |
| - type: ndcg_at_1 |
| value: 40.782000000000004 |
| - type: ndcg_at_10 |
| value: 38.916000000000004 |
| - type: ndcg_at_100 |
| value: 46.146 |
| - type: ndcg_at_1000 |
| value: 49.107 |
| - type: ndcg_at_20 |
| value: 41.888999999999996 |
| - type: ndcg_at_3 |
| value: 32.963 |
| - type: ndcg_at_5 |
| value: 34.872 |
| - type: precision_at_1 |
| value: 40.782000000000004 |
| - type: precision_at_10 |
| value: 11.87 |
| - type: precision_at_100 |
| value: 1.967 |
| - type: precision_at_1000 |
| value: 0.252 |
| - type: precision_at_20 |
| value: 7.234999999999999 |
| - type: precision_at_3 |
| value: 24.343 |
| - type: precision_at_5 |
| value: 18.279999999999998 |
| - type: recall_at_1 |
| value: 17.651 |
| - type: recall_at_10 |
| value: 44.321 |
| - type: recall_at_100 |
| value: 68.74 |
| - type: recall_at_1000 |
| value: 85.052 |
| - type: recall_at_20 |
| value: 52.693999999999996 |
| - type: recall_at_3 |
| value: 29.206 |
| - type: recall_at_5 |
| value: 35.363 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB CmedqaRetrieval (default) |
| revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 |
| split: test |
| type: C-MTEB/CmedqaRetrieval |
| metrics: |
| - type: main_score |
| value: 39.815 |
| - type: map_at_1 |
| value: 22.969 |
| - type: map_at_10 |
| value: 33.717000000000006 |
| - type: map_at_100 |
| value: 35.527 |
| - type: map_at_1000 |
| value: 35.669000000000004 |
| - type: map_at_20 |
| value: 34.701 |
| - type: map_at_3 |
| value: 30.145 |
| - type: map_at_5 |
| value: 32.101 |
| - type: mrr_at_1 |
| value: 35.408852213053265 |
| - type: mrr_at_10 |
| value: 42.79636377348293 |
| - type: mrr_at_100 |
| value: 43.799505469431466 |
| - type: mrr_at_1000 |
| value: 43.86305911334322 |
| - type: mrr_at_20 |
| value: 43.37854533926947 |
| - type: mrr_at_3 |
| value: 40.49762440610144 |
| - type: mrr_at_5 |
| value: 41.77294323580891 |
| - type: nauc_map_at_1000_diff1 |
| value: 48.1112957243525 |
| - type: nauc_map_at_1000_max |
| value: 45.10562447159609 |
| - type: nauc_map_at_1000_std |
| value: -3.612447606339434 |
| - type: nauc_map_at_100_diff1 |
| value: 48.070912563729344 |
| - type: nauc_map_at_100_max |
| value: 45.07885363880964 |
| - type: nauc_map_at_100_std |
| value: -3.632303318030044 |
| - type: nauc_map_at_10_diff1 |
| value: 48.018021707958944 |
| - type: nauc_map_at_10_max |
| value: 44.065132802797564 |
| - type: nauc_map_at_10_std |
| value: -4.847442985423416 |
| - type: nauc_map_at_1_diff1 |
| value: 52.57342490380559 |
| - type: nauc_map_at_1_max |
| value: 36.33892790190369 |
| - type: nauc_map_at_1_std |
| value: -7.554830062634192 |
| - type: nauc_map_at_20_diff1 |
| value: 48.000558518340675 |
| - type: nauc_map_at_20_max |
| value: 44.67149150982179 |
| - type: nauc_map_at_20_std |
| value: -4.281837169639415 |
| - type: nauc_map_at_3_diff1 |
| value: 48.907586769875444 |
| - type: nauc_map_at_3_max |
| value: 41.84368493765325 |
| - type: nauc_map_at_3_std |
| value: -6.132808996468529 |
| - type: nauc_map_at_5_diff1 |
| value: 48.41430811303262 |
| - type: nauc_map_at_5_max |
| value: 43.275175700700395 |
| - type: nauc_map_at_5_std |
| value: -5.606090404923237 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 54.23928137454549 |
| - type: nauc_mrr_at_1000_max |
| value: 51.52141320015717 |
| - type: nauc_mrr_at_1000_std |
| value: 0.45535832049480274 |
| - type: nauc_mrr_at_100_diff1 |
| value: 54.21443316125873 |
| - type: nauc_mrr_at_100_max |
| value: 51.51643391109282 |
| - type: nauc_mrr_at_100_std |
| value: 0.4736988328304099 |
| - type: nauc_mrr_at_10_diff1 |
| value: 54.21743294084247 |
| - type: nauc_mrr_at_10_max |
| value: 51.30368757017215 |
| - type: nauc_mrr_at_10_std |
| value: 0.15181063835569913 |
| - type: nauc_mrr_at_1_diff1 |
| value: 58.86488915793501 |
| - type: nauc_mrr_at_1_max |
| value: 52.31620000332108 |
| - type: nauc_mrr_at_1_std |
| value: -1.2963965803823345 |
| - type: nauc_mrr_at_20_diff1 |
| value: 54.17613850147242 |
| - type: nauc_mrr_at_20_max |
| value: 51.44963601888931 |
| - type: nauc_mrr_at_20_std |
| value: 0.33917399702518963 |
| - type: nauc_mrr_at_3_diff1 |
| value: 55.31466096640199 |
| - type: nauc_mrr_at_3_max |
| value: 52.070134962817136 |
| - type: nauc_mrr_at_3_std |
| value: -0.08530830198238608 |
| - type: nauc_mrr_at_5_diff1 |
| value: 54.57731938989671 |
| - type: nauc_mrr_at_5_max |
| value: 51.64739472086174 |
| - type: nauc_mrr_at_5_std |
| value: 0.18268948774575638 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 48.29500205761512 |
| - type: nauc_ndcg_at_1000_max |
| value: 47.88171483119307 |
| - type: nauc_ndcg_at_1000_std |
| value: 0.21403741411733387 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 47.37366257013038 |
| - type: nauc_ndcg_at_100_max |
| value: 47.76387826206963 |
| - type: nauc_ndcg_at_100_std |
| value: 0.826427463545558 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 47.34506821207949 |
| - type: nauc_ndcg_at_10_max |
| value: 45.488029750609286 |
| - type: nauc_ndcg_at_10_std |
| value: -2.944846404404074 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 58.86488915793501 |
| - type: nauc_ndcg_at_1_max |
| value: 52.31620000332108 |
| - type: nauc_ndcg_at_1_std |
| value: -1.2963965803823345 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 47.12752770930654 |
| - type: nauc_ndcg_at_20_max |
| value: 46.47247388716809 |
| - type: nauc_ndcg_at_20_std |
| value: -1.7736602031427529 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 49.34262730364437 |
| - type: nauc_ndcg_at_3_max |
| value: 47.22347634095395 |
| - type: nauc_ndcg_at_3_std |
| value: -2.563363733347789 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 48.284555734671144 |
| - type: nauc_ndcg_at_5_max |
| value: 46.07891305494883 |
| - type: nauc_ndcg_at_5_std |
| value: -3.107232535187627 |
| - type: nauc_precision_at_1000_diff1 |
| value: 5.125101705233774 |
| - type: nauc_precision_at_1000_max |
| value: 26.065307083522 |
| - type: nauc_precision_at_1000_std |
| value: 20.610223746634322 |
| - type: nauc_precision_at_100_diff1 |
| value: 11.866045880453454 |
| - type: nauc_precision_at_100_max |
| value: 36.42189620035723 |
| - type: nauc_precision_at_100_std |
| value: 22.521956326496763 |
| - type: nauc_precision_at_10_diff1 |
| value: 26.505548355872428 |
| - type: nauc_precision_at_10_max |
| value: 47.29524117494792 |
| - type: nauc_precision_at_10_std |
| value: 10.1116614235421 |
| - type: nauc_precision_at_1_diff1 |
| value: 58.86488915793501 |
| - type: nauc_precision_at_1_max |
| value: 52.31620000332108 |
| - type: nauc_precision_at_1_std |
| value: -1.2963965803823345 |
| - type: nauc_precision_at_20_diff1 |
| value: 20.886976295880487 |
| - type: nauc_precision_at_20_max |
| value: 44.30883416965209 |
| - type: nauc_precision_at_20_std |
| value: 14.011145517217743 |
| - type: nauc_precision_at_3_diff1 |
| value: 38.031908830120805 |
| - type: nauc_precision_at_3_max |
| value: 51.6114119909547 |
| - type: nauc_precision_at_3_std |
| value: 4.32752822701211 |
| - type: nauc_precision_at_5_diff1 |
| value: 32.521121686482275 |
| - type: nauc_precision_at_5_max |
| value: 50.65631029971074 |
| - type: nauc_precision_at_5_std |
| value: 6.649966273827001 |
| - type: nauc_recall_at_1000_diff1 |
| value: 22.845413165121183 |
| - type: nauc_recall_at_1000_max |
| value: 48.373939794348146 |
| - type: nauc_recall_at_1000_std |
| value: 40.974710828793064 |
| - type: nauc_recall_at_100_diff1 |
| value: 28.38725602654593 |
| - type: nauc_recall_at_100_max |
| value: 42.48910788250242 |
| - type: nauc_recall_at_100_std |
| value: 16.35920233861213 |
| - type: nauc_recall_at_10_diff1 |
| value: 35.870608122691 |
| - type: nauc_recall_at_10_max |
| value: 37.03672822253722 |
| - type: nauc_recall_at_10_std |
| value: -3.0810688213417867 |
| - type: nauc_recall_at_1_diff1 |
| value: 52.57342490380559 |
| - type: nauc_recall_at_1_max |
| value: 36.33892790190369 |
| - type: nauc_recall_at_1_std |
| value: -7.554830062634192 |
| - type: nauc_recall_at_20_diff1 |
| value: 33.42775765293211 |
| - type: nauc_recall_at_20_max |
| value: 38.55476461511651 |
| - type: nauc_recall_at_20_std |
| value: 0.4517674601589859 |
| - type: nauc_recall_at_3_diff1 |
| value: 43.48789481619157 |
| - type: nauc_recall_at_3_max |
| value: 39.17833917043277 |
| - type: nauc_recall_at_3_std |
| value: -5.279192048237245 |
| - type: nauc_recall_at_5_diff1 |
| value: 39.99694394881568 |
| - type: nauc_recall_at_5_max |
| value: 38.8498445921524 |
| - type: nauc_recall_at_5_std |
| value: -4.480536508614665 |
| - type: ndcg_at_1 |
| value: 35.409 |
| - type: ndcg_at_10 |
| value: 39.815 |
| - type: ndcg_at_100 |
| value: 47.034 |
| - type: ndcg_at_1000 |
| value: 49.697 |
| - type: ndcg_at_20 |
| value: 42.565 |
| - type: ndcg_at_3 |
| value: 35.249 |
| - type: ndcg_at_5 |
| value: 37.074 |
| - type: precision_at_1 |
| value: 35.409 |
| - type: precision_at_10 |
| value: 8.85 |
| - type: precision_at_100 |
| value: 1.469 |
| - type: precision_at_1000 |
| value: 0.18 |
| - type: precision_at_20 |
| value: 5.335 |
| - type: precision_at_3 |
| value: 19.947 |
| - type: precision_at_5 |
| value: 14.404 |
| - type: recall_at_1 |
| value: 22.969 |
| - type: recall_at_10 |
| value: 48.884 |
| - type: recall_at_100 |
| value: 78.777 |
| - type: recall_at_1000 |
| value: 96.914 |
| - type: recall_at_20 |
| value: 58.208000000000006 |
| - type: recall_at_3 |
| value: 34.929 |
| - type: recall_at_5 |
| value: 40.772000000000006 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB Cmnli (default) |
| revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 |
| split: validation |
| type: C-MTEB/CMNLI |
| metrics: |
| - type: cosine_accuracy |
| value: 63.042693926638606 |
| - type: cosine_accuracy_threshold |
| value: 76.46265029907227 |
| - type: cosine_ap |
| value: 69.87113533991577 |
| - type: cosine_f1 |
| value: 68.94233255753814 |
| - type: cosine_f1_threshold |
| value: 56.28046989440918 |
| - type: cosine_precision |
| value: 54.60131075914801 |
| - type: cosine_recall |
| value: 93.50011690437222 |
| - type: dot_accuracy |
| value: 63.042693926638606 |
| - type: dot_accuracy_threshold |
| value: 76.46265029907227 |
| - type: dot_ap |
| value: 69.8933256876927 |
| - type: dot_f1 |
| value: 68.94233255753814 |
| - type: dot_f1_threshold |
| value: 56.28047585487366 |
| - type: dot_precision |
| value: 54.60131075914801 |
| - type: dot_recall |
| value: 93.50011690437222 |
| - type: euclidean_accuracy |
| value: 63.042693926638606 |
| - type: euclidean_accuracy_threshold |
| value: 68.61100196838379 |
| - type: euclidean_ap |
| value: 69.87119389082798 |
| - type: euclidean_f1 |
| value: 68.94233255753814 |
| - type: euclidean_f1_threshold |
| value: 93.50885152816772 |
| - type: euclidean_precision |
| value: 54.60131075914801 |
| - type: euclidean_recall |
| value: 93.50011690437222 |
| - type: main_score |
| value: 63.042693926638606 |
| - type: manhattan_accuracy |
| value: 62.657847263980756 |
| - type: manhattan_accuracy_threshold |
| value: 1850.9063720703125 |
| - type: manhattan_ap |
| value: 69.61681898409992 |
| - type: manhattan_f1 |
| value: 68.82984159427696 |
| - type: manhattan_f1_threshold |
| value: 2378.0029296875 |
| - type: manhattan_precision |
| value: 54.13261888814468 |
| - type: manhattan_recall |
| value: 94.4821136310498 |
| - type: max_accuracy |
| value: 63.042693926638606 |
| - type: max_ap |
| value: 69.8933256876927 |
| - type: max_f1 |
| value: 68.94233255753814 |
| - type: max_precision |
| value: 54.60131075914801 |
| - type: max_recall |
| value: 94.4821136310498 |
| - type: similarity_accuracy |
| value: 63.042693926638606 |
| - type: similarity_accuracy_threshold |
| value: 76.46265029907227 |
| - type: similarity_ap |
| value: 69.87113533991577 |
| - type: similarity_f1 |
| value: 68.94233255753814 |
| - type: similarity_f1_threshold |
| value: 56.28046989440918 |
| - type: similarity_precision |
| value: 54.60131075914801 |
| - type: similarity_recall |
| value: 93.50011690437222 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB CovidRetrieval (default) |
| revision: 1271c7809071a13532e05f25fb53511ffce77117 |
| split: test |
| type: C-MTEB/CovidRetrieval |
| metrics: |
| - type: main_score |
| value: 84.379 |
| - type: map_at_1 |
| value: 73.604 |
| - type: map_at_10 |
| value: 81.03099999999999 |
| - type: map_at_100 |
| value: 81.274 |
| - type: map_at_1000 |
| value: 81.279 |
| - type: map_at_20 |
| value: 81.24 |
| - type: map_at_3 |
| value: 79.47 |
| - type: map_at_5 |
| value: 80.351 |
| - type: mrr_at_1 |
| value: 73.76185458377239 |
| - type: mrr_at_10 |
| value: 81.04311974174986 |
| - type: mrr_at_100 |
| value: 81.28215454043445 |
| - type: mrr_at_1000 |
| value: 81.28770523087694 |
| - type: mrr_at_20 |
| value: 81.24925687445058 |
| - type: mrr_at_3 |
| value: 79.53986652616791 |
| - type: mrr_at_5 |
| value: 80.4039339655778 |
| - type: nauc_map_at_1000_diff1 |
| value: 85.46427375215013 |
| - type: nauc_map_at_1000_max |
| value: 40.21490219185455 |
| - type: nauc_map_at_1000_std |
| value: -48.96793139327797 |
| - type: nauc_map_at_100_diff1 |
| value: 85.4644446768987 |
| - type: nauc_map_at_100_max |
| value: 40.23035336715615 |
| - type: nauc_map_at_100_std |
| value: -48.95308048383819 |
| - type: nauc_map_at_10_diff1 |
| value: 85.26224356683778 |
| - type: nauc_map_at_10_max |
| value: 40.25661447376166 |
| - type: nauc_map_at_10_std |
| value: -49.29368521251329 |
| - type: nauc_map_at_1_diff1 |
| value: 88.21629957013796 |
| - type: nauc_map_at_1_max |
| value: 37.83080773532884 |
| - type: nauc_map_at_1_std |
| value: -46.99042808069899 |
| - type: nauc_map_at_20_diff1 |
| value: 85.4572363555803 |
| - type: nauc_map_at_20_max |
| value: 40.30192541144375 |
| - type: nauc_map_at_20_std |
| value: -48.91376074777295 |
| - type: nauc_map_at_3_diff1 |
| value: 85.28229181056648 |
| - type: nauc_map_at_3_max |
| value: 39.60815875036543 |
| - type: nauc_map_at_3_std |
| value: -50.63770740326208 |
| - type: nauc_map_at_5_diff1 |
| value: 85.09928505788696 |
| - type: nauc_map_at_5_max |
| value: 39.896858679730634 |
| - type: nauc_map_at_5_std |
| value: -49.88022568110867 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 85.45890746495375 |
| - type: nauc_mrr_at_1000_max |
| value: 40.05596978582016 |
| - type: nauc_mrr_at_1000_std |
| value: -48.90941955475384 |
| - type: nauc_mrr_at_100_diff1 |
| value: 85.45908015824078 |
| - type: nauc_mrr_at_100_max |
| value: 40.07147434757591 |
| - type: nauc_mrr_at_100_std |
| value: -48.89458630706267 |
| - type: nauc_mrr_at_10_diff1 |
| value: 85.25764726917897 |
| - type: nauc_mrr_at_10_max |
| value: 40.099603315928206 |
| - type: nauc_mrr_at_10_std |
| value: -49.22980069856444 |
| - type: nauc_mrr_at_1_diff1 |
| value: 88.06183536385691 |
| - type: nauc_mrr_at_1_max |
| value: 37.83536214932872 |
| - type: nauc_mrr_at_1_std |
| value: -46.5713429052071 |
| - type: nauc_mrr_at_20_diff1 |
| value: 85.45090467444484 |
| - type: nauc_mrr_at_20_max |
| value: 40.13935028363459 |
| - type: nauc_mrr_at_20_std |
| value: -48.86537813167268 |
| - type: nauc_mrr_at_3_diff1 |
| value: 85.24167310516863 |
| - type: nauc_mrr_at_3_max |
| value: 39.682497832837186 |
| - type: nauc_mrr_at_3_std |
| value: -50.3559548925 |
| - type: nauc_mrr_at_5_diff1 |
| value: 85.09268565431421 |
| - type: nauc_mrr_at_5_max |
| value: 39.89031371475337 |
| - type: nauc_mrr_at_5_std |
| value: -49.655551830291884 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 85.12411679630183 |
| - type: nauc_ndcg_at_1000_max |
| value: 40.899838982860544 |
| - type: nauc_ndcg_at_1000_std |
| value: -48.61715011026588 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 85.12154652091637 |
| - type: nauc_ndcg_at_100_max |
| value: 41.32723015786323 |
| - type: nauc_ndcg_at_100_std |
| value: -48.16002090072688 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 84.25132198886159 |
| - type: nauc_ndcg_at_10_max |
| value: 41.78100587782578 |
| - type: nauc_ndcg_at_10_std |
| value: -49.16607901207903 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 88.06183536385691 |
| - type: nauc_ndcg_at_1_max |
| value: 37.83536214932872 |
| - type: nauc_ndcg_at_1_std |
| value: -46.5713429052071 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 85.0475628940421 |
| - type: nauc_ndcg_at_20_max |
| value: 41.96174817137773 |
| - type: nauc_ndcg_at_20_std |
| value: -47.58844892697574 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 84.27960098412159 |
| - type: nauc_ndcg_at_3_max |
| value: 40.33786907741922 |
| - type: nauc_ndcg_at_3_std |
| value: -52.004340720165864 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 83.84602758477916 |
| - type: nauc_ndcg_at_5_max |
| value: 40.85719695462724 |
| - type: nauc_ndcg_at_5_std |
| value: -50.58889323761097 |
| - type: nauc_precision_at_1000_diff1 |
| value: -48.76476929204374 |
| - type: nauc_precision_at_1000_max |
| value: -0.1666337888458482 |
| - type: nauc_precision_at_1000_std |
| value: 53.81780659847315 |
| - type: nauc_precision_at_100_diff1 |
| value: -3.2951420875662727 |
| - type: nauc_precision_at_100_max |
| value: 29.2186017772724 |
| - type: nauc_precision_at_100_std |
| value: 35.55529557180356 |
| - type: nauc_precision_at_10_diff1 |
| value: 48.4879108305862 |
| - type: nauc_precision_at_10_max |
| value: 46.36995270449671 |
| - type: nauc_precision_at_10_std |
| value: -22.72318400911746 |
| - type: nauc_precision_at_1_diff1 |
| value: 88.06183536385691 |
| - type: nauc_precision_at_1_max |
| value: 37.83536214932872 |
| - type: nauc_precision_at_1_std |
| value: -46.5713429052071 |
| - type: nauc_precision_at_20_diff1 |
| value: 29.69237300240173 |
| - type: nauc_precision_at_20_max |
| value: 48.71484019554503 |
| - type: nauc_precision_at_20_std |
| value: 20.65240722122367 |
| - type: nauc_precision_at_3_diff1 |
| value: 75.05731813510343 |
| - type: nauc_precision_at_3_max |
| value: 41.16734979850893 |
| - type: nauc_precision_at_3_std |
| value: -52.92424557581844 |
| - type: nauc_precision_at_5_diff1 |
| value: 66.39813557698707 |
| - type: nauc_precision_at_5_max |
| value: 42.52370016987382 |
| - type: nauc_precision_at_5_std |
| value: -44.1213251901674 |
| - type: nauc_recall_at_1000_diff1 |
| value: .nan |
| - type: nauc_recall_at_1000_max |
| value: .nan |
| - type: nauc_recall_at_1000_std |
| value: .nan |
| - type: nauc_recall_at_100_diff1 |
| value: 84.51287278881901 |
| - type: nauc_recall_at_100_max |
| value: 85.96547568790173 |
| - type: nauc_recall_at_100_std |
| value: 0.28791075910543246 |
| - type: nauc_recall_at_10_diff1 |
| value: 75.47811216865622 |
| - type: nauc_recall_at_10_max |
| value: 57.077070588344036 |
| - type: nauc_recall_at_10_std |
| value: -46.27338964199063 |
| - type: nauc_recall_at_1_diff1 |
| value: 88.21629957013796 |
| - type: nauc_recall_at_1_max |
| value: 37.83080773532884 |
| - type: nauc_recall_at_1_std |
| value: -46.99042808069899 |
| - type: nauc_recall_at_20_diff1 |
| value: 82.38786919086273 |
| - type: nauc_recall_at_20_max |
| value: 77.62368007507331 |
| - type: nauc_recall_at_20_std |
| value: -10.317865962550622 |
| - type: nauc_recall_at_3_diff1 |
| value: 80.3430059663806 |
| - type: nauc_recall_at_3_max |
| value: 43.14918120883788 |
| - type: nauc_recall_at_3_std |
| value: -57.95792751840148 |
| - type: nauc_recall_at_5_diff1 |
| value: 77.33264062978374 |
| - type: nauc_recall_at_5_max |
| value: 45.54667757514487 |
| - type: nauc_recall_at_5_std |
| value: -54.44876978926033 |
| - type: ndcg_at_1 |
| value: 73.762 |
| - type: ndcg_at_10 |
| value: 84.379 |
| - type: ndcg_at_100 |
| value: 85.383 |
| - type: ndcg_at_1000 |
| value: 85.508 |
| - type: ndcg_at_20 |
| value: 85.114 |
| - type: ndcg_at_3 |
| value: 81.255 |
| - type: ndcg_at_5 |
| value: 82.83 |
| - type: precision_at_1 |
| value: 73.762 |
| - type: precision_at_10 |
| value: 9.557 |
| - type: precision_at_100 |
| value: 1.001 |
| - type: precision_at_1000 |
| value: 0.101 |
| - type: precision_at_20 |
| value: 4.926 |
| - type: precision_at_3 |
| value: 28.908 |
| - type: precision_at_5 |
| value: 18.145 |
| - type: recall_at_1 |
| value: 73.604 |
| - type: recall_at_10 |
| value: 94.731 |
| - type: recall_at_100 |
| value: 99.05199999999999 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 97.576 |
| - type: recall_at_3 |
| value: 86.301 |
| - type: recall_at_5 |
| value: 90.095 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB DBPedia (default) |
| revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 |
| split: test |
| type: mteb/dbpedia |
| metrics: |
| - type: main_score |
| value: 45.153 |
| - type: map_at_1 |
| value: 9.046999999999999 |
| - type: map_at_10 |
| value: 20.629 |
| - type: map_at_100 |
| value: 29.964000000000002 |
| - type: map_at_1000 |
| value: 31.912000000000003 |
| - type: map_at_20 |
| value: 24.342 |
| - type: map_at_3 |
| value: 14.399999999999999 |
| - type: map_at_5 |
| value: 16.933999999999997 |
| - type: mrr_at_1 |
| value: 70.5 |
| - type: mrr_at_10 |
| value: 77.9640873015873 |
| - type: mrr_at_100 |
| value: 78.30766270365284 |
| - type: mrr_at_1000 |
| value: 78.31022444762601 |
| - type: mrr_at_20 |
| value: 78.22993596960315 |
| - type: mrr_at_3 |
| value: 76.75 |
| - type: mrr_at_5 |
| value: 77.36250000000001 |
| - type: nauc_map_at_1000_diff1 |
| value: 28.045576396300202 |
| - type: nauc_map_at_1000_max |
| value: 26.153471210607186 |
| - type: nauc_map_at_1000_std |
| value: 22.391104663325024 |
| - type: nauc_map_at_100_diff1 |
| value: 27.708397480512936 |
| - type: nauc_map_at_100_max |
| value: 23.341985202750255 |
| - type: nauc_map_at_100_std |
| value: 19.14393027622429 |
| - type: nauc_map_at_10_diff1 |
| value: 29.604187778342343 |
| - type: nauc_map_at_10_max |
| value: 9.607565758548718 |
| - type: nauc_map_at_10_std |
| value: -6.812067636191434 |
| - type: nauc_map_at_1_diff1 |
| value: 33.78958074380661 |
| - type: nauc_map_at_1_max |
| value: -4.668230310194967 |
| - type: nauc_map_at_1_std |
| value: -23.529326539297614 |
| - type: nauc_map_at_20_diff1 |
| value: 28.67331151769839 |
| - type: nauc_map_at_20_max |
| value: 14.895897875509375 |
| - type: nauc_map_at_20_std |
| value: 3.1963949053883187 |
| - type: nauc_map_at_3_diff1 |
| value: 29.722385417980412 |
| - type: nauc_map_at_3_max |
| value: -0.2811142783569912 |
| - type: nauc_map_at_3_std |
| value: -19.299586690821332 |
| - type: nauc_map_at_5_diff1 |
| value: 28.66624240695108 |
| - type: nauc_map_at_5_max |
| value: 3.360052191036737 |
| - type: nauc_map_at_5_std |
| value: -14.851723430211013 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 56.423221856530894 |
| - type: nauc_mrr_at_1000_max |
| value: 60.317065501501 |
| - type: nauc_mrr_at_1000_std |
| value: 33.698591024900175 |
| - type: nauc_mrr_at_100_diff1 |
| value: 56.425561584153606 |
| - type: nauc_mrr_at_100_max |
| value: 60.31984977402958 |
| - type: nauc_mrr_at_100_std |
| value: 33.70006799711308 |
| - type: nauc_mrr_at_10_diff1 |
| value: 56.43259894878052 |
| - type: nauc_mrr_at_10_max |
| value: 60.374499288909945 |
| - type: nauc_mrr_at_10_std |
| value: 33.4294830409633 |
| - type: nauc_mrr_at_1_diff1 |
| value: 55.75393019283295 |
| - type: nauc_mrr_at_1_max |
| value: 56.564175641482315 |
| - type: nauc_mrr_at_1_std |
| value: 28.295104729019933 |
| - type: nauc_mrr_at_20_diff1 |
| value: 56.33520950522991 |
| - type: nauc_mrr_at_20_max |
| value: 60.28399700243428 |
| - type: nauc_mrr_at_20_std |
| value: 33.63189278260014 |
| - type: nauc_mrr_at_3_diff1 |
| value: 56.7275475900834 |
| - type: nauc_mrr_at_3_max |
| value: 60.933993343835155 |
| - type: nauc_mrr_at_3_std |
| value: 35.25440863470142 |
| - type: nauc_mrr_at_5_diff1 |
| value: 56.627733469260036 |
| - type: nauc_mrr_at_5_max |
| value: 60.52601047103946 |
| - type: nauc_mrr_at_5_std |
| value: 34.06919416028891 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 35.51891017935117 |
| - type: nauc_ndcg_at_1000_max |
| value: 43.63290111887676 |
| - type: nauc_ndcg_at_1000_std |
| value: 38.27645609360528 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 34.95565666939815 |
| - type: nauc_ndcg_at_100_max |
| value: 35.603879842392054 |
| - type: nauc_ndcg_at_100_std |
| value: 29.535182565117623 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 34.25164503584335 |
| - type: nauc_ndcg_at_10_max |
| value: 36.161839357245015 |
| - type: nauc_ndcg_at_10_std |
| value: 22.057343756689214 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 46.86872053620517 |
| - type: nauc_ndcg_at_1_max |
| value: 39.03060882424493 |
| - type: nauc_ndcg_at_1_std |
| value: 20.21898747028476 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 34.39534638745961 |
| - type: nauc_ndcg_at_20_max |
| value: 33.42062258555372 |
| - type: nauc_ndcg_at_20_std |
| value: 21.677461411920135 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 35.54249517020183 |
| - type: nauc_ndcg_at_3_max |
| value: 38.5502021300953 |
| - type: nauc_ndcg_at_3_std |
| value: 20.87941638879022 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 33.139218138659594 |
| - type: nauc_ndcg_at_5_max |
| value: 37.74145771932368 |
| - type: nauc_ndcg_at_5_std |
| value: 21.60307300259375 |
| - type: nauc_precision_at_1000_diff1 |
| value: -3.4926442688521444 |
| - type: nauc_precision_at_1000_max |
| value: 9.33810183416714 |
| - type: nauc_precision_at_1000_std |
| value: 9.091298908761424 |
| - type: nauc_precision_at_100_diff1 |
| value: -0.8681013692695503 |
| - type: nauc_precision_at_100_max |
| value: 29.92488145588432 |
| - type: nauc_precision_at_100_std |
| value: 43.243564317268365 |
| - type: nauc_precision_at_10_diff1 |
| value: 8.354685886799782 |
| - type: nauc_precision_at_10_max |
| value: 40.88350345790237 |
| - type: nauc_precision_at_10_std |
| value: 43.53467360875934 |
| - type: nauc_precision_at_1_diff1 |
| value: 55.75393019283295 |
| - type: nauc_precision_at_1_max |
| value: 56.564175641482315 |
| - type: nauc_precision_at_1_std |
| value: 28.295104729019933 |
| - type: nauc_precision_at_20_diff1 |
| value: 3.7269285981427953 |
| - type: nauc_precision_at_20_max |
| value: 36.999904619801605 |
| - type: nauc_precision_at_20_std |
| value: 47.03245724966235 |
| - type: nauc_precision_at_3_diff1 |
| value: 19.58602295951204 |
| - type: nauc_precision_at_3_max |
| value: 40.774756975430684 |
| - type: nauc_precision_at_3_std |
| value: 30.313382731334386 |
| - type: nauc_precision_at_5_diff1 |
| value: 11.501462854603371 |
| - type: nauc_precision_at_5_max |
| value: 41.11491741352496 |
| - type: nauc_precision_at_5_std |
| value: 36.306292126509184 |
| - type: nauc_recall_at_1000_diff1 |
| value: 21.965267428294624 |
| - type: nauc_recall_at_1000_max |
| value: 37.73121016970661 |
| - type: nauc_recall_at_1000_std |
| value: 49.67514738459122 |
| - type: nauc_recall_at_100_diff1 |
| value: 23.36978996552894 |
| - type: nauc_recall_at_100_max |
| value: 25.803297478273763 |
| - type: nauc_recall_at_100_std |
| value: 28.323018838882152 |
| - type: nauc_recall_at_10_diff1 |
| value: 25.191581940489176 |
| - type: nauc_recall_at_10_max |
| value: 5.481367733091858 |
| - type: nauc_recall_at_10_std |
| value: -9.302647109645827 |
| - type: nauc_recall_at_1_diff1 |
| value: 33.78958074380661 |
| - type: nauc_recall_at_1_max |
| value: -4.668230310194967 |
| - type: nauc_recall_at_1_std |
| value: -23.529326539297614 |
| - type: nauc_recall_at_20_diff1 |
| value: 22.78382683996787 |
| - type: nauc_recall_at_20_max |
| value: 10.59760940055021 |
| - type: nauc_recall_at_20_std |
| value: 0.5482029877052178 |
| - type: nauc_recall_at_3_diff1 |
| value: 26.517579502576506 |
| - type: nauc_recall_at_3_max |
| value: -1.9201875876437906 |
| - type: nauc_recall_at_3_std |
| value: -19.530894582297815 |
| - type: nauc_recall_at_5_diff1 |
| value: 24.999441835514016 |
| - type: nauc_recall_at_5_max |
| value: 0.5801717047366033 |
| - type: nauc_recall_at_5_std |
| value: -16.443290167984774 |
| - type: ndcg_at_1 |
| value: 58.875 |
| - type: ndcg_at_10 |
| value: 45.153 |
| - type: ndcg_at_100 |
| value: 49.58 |
| - type: ndcg_at_1000 |
| value: 56.667 |
| - type: ndcg_at_20 |
| value: 44.497 |
| - type: ndcg_at_3 |
| value: 49.856 |
| - type: ndcg_at_5 |
| value: 47.043 |
| - type: precision_at_1 |
| value: 70.5 |
| - type: precision_at_10 |
| value: 36.65 |
| - type: precision_at_100 |
| value: 11.975 |
| - type: precision_at_1000 |
| value: 2.375 |
| - type: precision_at_20 |
| value: 28.337 |
| - type: precision_at_3 |
| value: 52.917 |
| - type: precision_at_5 |
| value: 45.25 |
| - type: recall_at_1 |
| value: 9.046999999999999 |
| - type: recall_at_10 |
| value: 26.662999999999997 |
| - type: recall_at_100 |
| value: 55.293000000000006 |
| - type: recall_at_1000 |
| value: 78.224 |
| - type: recall_at_20 |
| value: 35.278999999999996 |
| - type: recall_at_3 |
| value: 15.549 |
| - type: recall_at_5 |
| value: 19.657 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB DuRetrieval (default) |
| revision: a1a333e290fe30b10f3f56498e3a0d911a693ced |
| split: test |
| type: C-MTEB/DuRetrieval |
| metrics: |
| - type: main_score |
| value: 89.229 |
| - type: map_at_1 |
| value: 27.054000000000002 |
| - type: map_at_10 |
| value: 82.759 |
| - type: map_at_100 |
| value: 85.296 |
| - type: map_at_1000 |
| value: 85.33699999999999 |
| - type: map_at_20 |
| value: 84.75399999999999 |
| - type: map_at_3 |
| value: 57.766 |
| - type: map_at_5 |
| value: 72.907 |
| - type: mrr_at_1 |
| value: 92.15 |
| - type: mrr_at_10 |
| value: 94.4686309523809 |
| - type: mrr_at_100 |
| value: 94.534648802311 |
| - type: mrr_at_1000 |
| value: 94.53710776939944 |
| - type: mrr_at_20 |
| value: 94.52034224031897 |
| - type: mrr_at_3 |
| value: 94.24999999999997 |
| - type: mrr_at_5 |
| value: 94.37999999999997 |
| - type: nauc_map_at_1000_diff1 |
| value: -0.02796039048781859 |
| - type: nauc_map_at_1000_max |
| value: 49.7576338418727 |
| - type: nauc_map_at_1000_std |
| value: 23.05361731789902 |
| - type: nauc_map_at_100_diff1 |
| value: 0.008194529679268515 |
| - type: nauc_map_at_100_max |
| value: 49.74922545472186 |
| - type: nauc_map_at_100_std |
| value: 22.9835777977635 |
| - type: nauc_map_at_10_diff1 |
| value: 4.783611792072549 |
| - type: nauc_map_at_10_max |
| value: 45.78791767479075 |
| - type: nauc_map_at_10_std |
| value: 9.771217776003757 |
| - type: nauc_map_at_1_diff1 |
| value: 45.840411752765256 |
| - type: nauc_map_at_1_max |
| value: -12.570873671746416 |
| - type: nauc_map_at_1_std |
| value: -36.84331524265176 |
| - type: nauc_map_at_20_diff1 |
| value: 0.8823743065735505 |
| - type: nauc_map_at_20_max |
| value: 49.26910137511861 |
| - type: nauc_map_at_20_std |
| value: 20.3235967552421 |
| - type: nauc_map_at_3_diff1 |
| value: 27.914571236903722 |
| - type: nauc_map_at_3_max |
| value: 4.499387162458829 |
| - type: nauc_map_at_3_std |
| value: -30.155523730943536 |
| - type: nauc_map_at_5_diff1 |
| value: 16.7111462880928 |
| - type: nauc_map_at_5_max |
| value: 23.207688391011054 |
| - type: nauc_map_at_5_std |
| value: -16.489281202052332 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 31.30664037709492 |
| - type: nauc_mrr_at_1000_max |
| value: 84.27608202330632 |
| - type: nauc_mrr_at_1000_std |
| value: 51.23558134732731 |
| - type: nauc_mrr_at_100_diff1 |
| value: 31.306386401837504 |
| - type: nauc_mrr_at_100_max |
| value: 84.2804599579358 |
| - type: nauc_mrr_at_100_std |
| value: 51.22749075580445 |
| - type: nauc_mrr_at_10_diff1 |
| value: 31.437829554947523 |
| - type: nauc_mrr_at_10_max |
| value: 84.43324944772354 |
| - type: nauc_mrr_at_10_std |
| value: 51.46000653619227 |
| - type: nauc_mrr_at_1_diff1 |
| value: 31.36273617727332 |
| - type: nauc_mrr_at_1_max |
| value: 80.81143285339608 |
| - type: nauc_mrr_at_1_std |
| value: 46.5075202055344 |
| - type: nauc_mrr_at_20_diff1 |
| value: 31.158185456044674 |
| - type: nauc_mrr_at_20_max |
| value: 84.3105159105071 |
| - type: nauc_mrr_at_20_std |
| value: 51.295225958764725 |
| - type: nauc_mrr_at_3_diff1 |
| value: 31.774042950513493 |
| - type: nauc_mrr_at_3_max |
| value: 84.4039838157486 |
| - type: nauc_mrr_at_3_std |
| value: 51.11063749171155 |
| - type: nauc_mrr_at_5_diff1 |
| value: 31.96778711484582 |
| - type: nauc_mrr_at_5_max |
| value: 84.57777733473773 |
| - type: nauc_mrr_at_5_std |
| value: 51.642536215751576 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 3.0995160786463867 |
| - type: nauc_ndcg_at_1000_max |
| value: 60.44661643509836 |
| - type: nauc_ndcg_at_1000_std |
| value: 36.19905274044387 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 2.7767404830386506 |
| - type: nauc_ndcg_at_100_max |
| value: 60.1441312933469 |
| - type: nauc_ndcg_at_100_std |
| value: 36.17340932069341 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 3.494723781736116 |
| - type: nauc_ndcg_at_10_max |
| value: 55.33863484422592 |
| - type: nauc_ndcg_at_10_std |
| value: 27.925059533697226 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 31.36273617727332 |
| - type: nauc_ndcg_at_1_max |
| value: 80.81143285339608 |
| - type: nauc_ndcg_at_1_std |
| value: 46.5075202055344 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 3.1373926565607406 |
| - type: nauc_ndcg_at_20_max |
| value: 58.49392402871737 |
| - type: nauc_ndcg_at_20_std |
| value: 32.094872831601 |
| - type: nauc_ndcg_at_3_diff1 |
| value: -3.5044344394018196 |
| - type: nauc_ndcg_at_3_max |
| value: 56.21333251222252 |
| - type: nauc_ndcg_at_3_std |
| value: 33.93829033390993 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 1.658983298277881 |
| - type: nauc_ndcg_at_5_max |
| value: 48.94373808616266 |
| - type: nauc_ndcg_at_5_std |
| value: 23.803470422940855 |
| - type: nauc_precision_at_1000_diff1 |
| value: -32.902170215030935 |
| - type: nauc_precision_at_1000_max |
| value: 17.300907716587215 |
| - type: nauc_precision_at_1000_std |
| value: 52.51296253560843 |
| - type: nauc_precision_at_100_diff1 |
| value: -33.573728937434666 |
| - type: nauc_precision_at_100_max |
| value: 19.513743085739247 |
| - type: nauc_precision_at_100_std |
| value: 54.49616149633364 |
| - type: nauc_precision_at_10_diff1 |
| value: -35.16204430421235 |
| - type: nauc_precision_at_10_max |
| value: 32.18559538422582 |
| - type: nauc_precision_at_10_std |
| value: 54.167767973795286 |
| - type: nauc_precision_at_1_diff1 |
| value: 31.36273617727332 |
| - type: nauc_precision_at_1_max |
| value: 80.81143285339608 |
| - type: nauc_precision_at_1_std |
| value: 46.5075202055344 |
| - type: nauc_precision_at_20_diff1 |
| value: -34.2922112064245 |
| - type: nauc_precision_at_20_max |
| value: 24.67123112050235 |
| - type: nauc_precision_at_20_std |
| value: 55.38739984439128 |
| - type: nauc_precision_at_3_diff1 |
| value: -38.89657014112433 |
| - type: nauc_precision_at_3_max |
| value: 51.718392836961435 |
| - type: nauc_precision_at_3_std |
| value: 51.76733613564855 |
| - type: nauc_precision_at_5_diff1 |
| value: -38.297772070172165 |
| - type: nauc_precision_at_5_max |
| value: 41.64917637118582 |
| - type: nauc_precision_at_5_std |
| value: 51.161765176162476 |
| - type: nauc_recall_at_1000_diff1 |
| value: -0.17513872888091864 |
| - type: nauc_recall_at_1000_max |
| value: 72.38708288076182 |
| - type: nauc_recall_at_1000_std |
| value: 73.03296451601094 |
| - type: nauc_recall_at_100_diff1 |
| value: -7.289991660683619 |
| - type: nauc_recall_at_100_max |
| value: 60.30163206236221 |
| - type: nauc_recall_at_100_std |
| value: 52.57173609584166 |
| - type: nauc_recall_at_10_diff1 |
| value: 6.444643365227589 |
| - type: nauc_recall_at_10_max |
| value: 44.23969322390307 |
| - type: nauc_recall_at_10_std |
| value: 6.639619762390987 |
| - type: nauc_recall_at_1_diff1 |
| value: 45.840411752765256 |
| - type: nauc_recall_at_1_max |
| value: -12.570873671746416 |
| - type: nauc_recall_at_1_std |
| value: -36.84331524265176 |
| - type: nauc_recall_at_20_diff1 |
| value: -1.6465114691507572 |
| - type: nauc_recall_at_20_max |
| value: 52.55212477208588 |
| - type: nauc_recall_at_20_std |
| value: 29.282880316927745 |
| - type: nauc_recall_at_3_diff1 |
| value: 27.984237671618846 |
| - type: nauc_recall_at_3_max |
| value: -0.9271310366095001 |
| - type: nauc_recall_at_3_std |
| value: -34.16035939832247 |
| - type: nauc_recall_at_5_diff1 |
| value: 19.547943686458623 |
| - type: nauc_recall_at_5_max |
| value: 15.20222704175238 |
| - type: nauc_recall_at_5_std |
| value: -24.074202172178282 |
| - type: ndcg_at_1 |
| value: 92.15 |
| - type: ndcg_at_10 |
| value: 89.229 |
| - type: ndcg_at_100 |
| value: 91.515 |
| - type: ndcg_at_1000 |
| value: 91.872 |
| - type: ndcg_at_20 |
| value: 90.51 |
| - type: ndcg_at_3 |
| value: 88.765 |
| - type: ndcg_at_5 |
| value: 87.543 |
| - type: precision_at_1 |
| value: 92.15 |
| - type: precision_at_10 |
| value: 42.375 |
| - type: precision_at_100 |
| value: 4.798 |
| - type: precision_at_1000 |
| value: 0.48900000000000005 |
| - type: precision_at_20 |
| value: 22.888 |
| - type: precision_at_3 |
| value: 79.617 |
| - type: precision_at_5 |
| value: 67.02 |
| - type: recall_at_1 |
| value: 27.054000000000002 |
| - type: recall_at_10 |
| value: 89.815 |
| - type: recall_at_100 |
| value: 97.592 |
| - type: recall_at_1000 |
| value: 99.41799999999999 |
| - type: recall_at_20 |
| value: 94.293 |
| - type: recall_at_3 |
| value: 59.602 |
| - type: recall_at_5 |
| value: 76.706 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB EcomRetrieval (default) |
| revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 |
| split: test |
| type: C-MTEB/EcomRetrieval |
| metrics: |
| - type: main_score |
| value: 65.84100000000001 |
| - type: map_at_1 |
| value: 52.5 |
| - type: map_at_10 |
| value: 61.192 |
| - type: map_at_100 |
| value: 61.83 |
| - type: map_at_1000 |
| value: 61.844 |
| - type: map_at_20 |
| value: 61.617 |
| - type: map_at_3 |
| value: 58.75 |
| - type: map_at_5 |
| value: 60.245000000000005 |
| - type: mrr_at_1 |
| value: 52.5 |
| - type: mrr_at_10 |
| value: 61.192380952380944 |
| - type: mrr_at_100 |
| value: 61.829977663371 |
| - type: mrr_at_1000 |
| value: 61.84414119129694 |
| - type: mrr_at_20 |
| value: 61.6173393067201 |
| - type: mrr_at_3 |
| value: 58.74999999999999 |
| - type: mrr_at_5 |
| value: 60.24499999999999 |
| - type: nauc_map_at_1000_diff1 |
| value: 68.63381898499689 |
| - type: nauc_map_at_1000_max |
| value: 29.906802700314362 |
| - type: nauc_map_at_1000_std |
| value: -13.043778448362591 |
| - type: nauc_map_at_100_diff1 |
| value: 68.61922846433136 |
| - type: nauc_map_at_100_max |
| value: 29.911914135083894 |
| - type: nauc_map_at_100_std |
| value: -13.03583241535797 |
| - type: nauc_map_at_10_diff1 |
| value: 68.42254379324686 |
| - type: nauc_map_at_10_max |
| value: 30.11120757393897 |
| - type: nauc_map_at_10_std |
| value: -13.140082134227866 |
| - type: nauc_map_at_1_diff1 |
| value: 72.79827709978464 |
| - type: nauc_map_at_1_max |
| value: 26.28489385704028 |
| - type: nauc_map_at_1_std |
| value: -15.273100194851812 |
| - type: nauc_map_at_20_diff1 |
| value: 68.5543044735401 |
| - type: nauc_map_at_20_max |
| value: 29.880811074859288 |
| - type: nauc_map_at_20_std |
| value: -13.119173248800491 |
| - type: nauc_map_at_3_diff1 |
| value: 68.78227539274147 |
| - type: nauc_map_at_3_max |
| value: 28.160898614800654 |
| - type: nauc_map_at_3_std |
| value: -15.797291523471626 |
| - type: nauc_map_at_5_diff1 |
| value: 68.3493580349966 |
| - type: nauc_map_at_5_max |
| value: 29.462226781090628 |
| - type: nauc_map_at_5_std |
| value: -13.823334723010062 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 68.63381898499689 |
| - type: nauc_mrr_at_1000_max |
| value: 29.906802700314362 |
| - type: nauc_mrr_at_1000_std |
| value: -13.043778448362591 |
| - type: nauc_mrr_at_100_diff1 |
| value: 68.61922846433136 |
| - type: nauc_mrr_at_100_max |
| value: 29.911914135083894 |
| - type: nauc_mrr_at_100_std |
| value: -13.03583241535797 |
| - type: nauc_mrr_at_10_diff1 |
| value: 68.42254379324686 |
| - type: nauc_mrr_at_10_max |
| value: 30.11120757393897 |
| - type: nauc_mrr_at_10_std |
| value: -13.140082134227866 |
| - type: nauc_mrr_at_1_diff1 |
| value: 72.79827709978464 |
| - type: nauc_mrr_at_1_max |
| value: 26.28489385704028 |
| - type: nauc_mrr_at_1_std |
| value: -15.273100194851812 |
| - type: nauc_mrr_at_20_diff1 |
| value: 68.5543044735401 |
| - type: nauc_mrr_at_20_max |
| value: 29.880811074859288 |
| - type: nauc_mrr_at_20_std |
| value: -13.119173248800491 |
| - type: nauc_mrr_at_3_diff1 |
| value: 68.78227539274147 |
| - type: nauc_mrr_at_3_max |
| value: 28.160898614800654 |
| - type: nauc_mrr_at_3_std |
| value: -15.797291523471626 |
| - type: nauc_mrr_at_5_diff1 |
| value: 68.3493580349966 |
| - type: nauc_mrr_at_5_max |
| value: 29.462226781090628 |
| - type: nauc_mrr_at_5_std |
| value: -13.823334723010062 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 67.90196996106812 |
| - type: nauc_ndcg_at_1000_max |
| value: 32.36822400000294 |
| - type: nauc_ndcg_at_1000_std |
| value: -9.824494007845882 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 67.54486587995649 |
| - type: nauc_ndcg_at_100_max |
| value: 32.7718926705024 |
| - type: nauc_ndcg_at_100_std |
| value: -9.26575359100604 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 66.64847353850341 |
| - type: nauc_ndcg_at_10_max |
| value: 33.223671665163614 |
| - type: nauc_ndcg_at_10_std |
| value: -10.27829867720837 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 72.79827709978464 |
| - type: nauc_ndcg_at_1_max |
| value: 26.28489385704028 |
| - type: nauc_ndcg_at_1_std |
| value: -15.273100194851812 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 67.0754334299387 |
| - type: nauc_ndcg_at_20_max |
| value: 32.456199571793995 |
| - type: nauc_ndcg_at_20_std |
| value: -9.931114874548891 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 67.44808597891617 |
| - type: nauc_ndcg_at_3_max |
| value: 28.81312271324233 |
| - type: nauc_ndcg_at_3_std |
| value: -15.900590447226456 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 66.57742283926243 |
| - type: nauc_ndcg_at_5_max |
| value: 31.40058618065593 |
| - type: nauc_ndcg_at_5_std |
| value: -12.091744743636507 |
| - type: nauc_precision_at_1000_diff1 |
| value: 69.24992219109824 |
| - type: nauc_precision_at_1000_max |
| value: 92.99097416744418 |
| - type: nauc_precision_at_1000_std |
| value: 80.41083099906578 |
| - type: nauc_precision_at_100_diff1 |
| value: 58.40336134453783 |
| - type: nauc_precision_at_100_max |
| value: 75.6873119618219 |
| - type: nauc_precision_at_100_std |
| value: 53.056037229706746 |
| - type: nauc_precision_at_10_diff1 |
| value: 58.16256795502892 |
| - type: nauc_precision_at_10_max |
| value: 49.778279054949756 |
| - type: nauc_precision_at_10_std |
| value: 5.275571130545513 |
| - type: nauc_precision_at_1_diff1 |
| value: 72.79827709978464 |
| - type: nauc_precision_at_1_max |
| value: 26.28489385704028 |
| - type: nauc_precision_at_1_std |
| value: -15.273100194851812 |
| - type: nauc_precision_at_20_diff1 |
| value: 58.04381286465974 |
| - type: nauc_precision_at_20_max |
| value: 50.42452661345172 |
| - type: nauc_precision_at_20_std |
| value: 14.55202777678338 |
| - type: nauc_precision_at_3_diff1 |
| value: 63.10303988700744 |
| - type: nauc_precision_at_3_max |
| value: 30.947952093016234 |
| - type: nauc_precision_at_3_std |
| value: -16.210759642302556 |
| - type: nauc_precision_at_5_diff1 |
| value: 59.82134123823496 |
| - type: nauc_precision_at_5_max |
| value: 39.221486959906464 |
| - type: nauc_precision_at_5_std |
| value: -4.794642233334347 |
| - type: nauc_recall_at_1000_diff1 |
| value: 69.24992219109895 |
| - type: nauc_recall_at_1000_max |
| value: 92.99097416744485 |
| - type: nauc_recall_at_1000_std |
| value: 80.41083099906643 |
| - type: nauc_recall_at_100_diff1 |
| value: 58.40336134453772 |
| - type: nauc_recall_at_100_max |
| value: 75.6873119618217 |
| - type: nauc_recall_at_100_std |
| value: 53.056037229706696 |
| - type: nauc_recall_at_10_diff1 |
| value: 58.16256795502893 |
| - type: nauc_recall_at_10_max |
| value: 49.77827905494972 |
| - type: nauc_recall_at_10_std |
| value: 5.275571130545528 |
| - type: nauc_recall_at_1_diff1 |
| value: 72.79827709978464 |
| - type: nauc_recall_at_1_max |
| value: 26.28489385704028 |
| - type: nauc_recall_at_1_std |
| value: -15.273100194851812 |
| - type: nauc_recall_at_20_diff1 |
| value: 58.043812864659714 |
| - type: nauc_recall_at_20_max |
| value: 50.42452661345165 |
| - type: nauc_recall_at_20_std |
| value: 14.552027776783477 |
| - type: nauc_recall_at_3_diff1 |
| value: 63.10303988700737 |
| - type: nauc_recall_at_3_max |
| value: 30.9479520930162 |
| - type: nauc_recall_at_3_std |
| value: -16.21075964230267 |
| - type: nauc_recall_at_5_diff1 |
| value: 59.82134123823499 |
| - type: nauc_recall_at_5_max |
| value: 39.221486959906535 |
| - type: nauc_recall_at_5_std |
| value: -4.79464223333429 |
| - type: ndcg_at_1 |
| value: 52.5 |
| - type: ndcg_at_10 |
| value: 65.84100000000001 |
| - type: ndcg_at_100 |
| value: 68.738 |
| - type: ndcg_at_1000 |
| value: 69.148 |
| - type: ndcg_at_20 |
| value: 67.352 |
| - type: ndcg_at_3 |
| value: 60.839 |
| - type: ndcg_at_5 |
| value: 63.546 |
| - type: precision_at_1 |
| value: 52.5 |
| - type: precision_at_10 |
| value: 8.06 |
| - type: precision_at_100 |
| value: 0.9369999999999999 |
| - type: precision_at_1000 |
| value: 0.097 |
| - type: precision_at_20 |
| value: 4.324999999999999 |
| - type: precision_at_3 |
| value: 22.3 |
| - type: precision_at_5 |
| value: 14.7 |
| - type: recall_at_1 |
| value: 52.5 |
| - type: recall_at_10 |
| value: 80.60000000000001 |
| - type: recall_at_100 |
| value: 93.7 |
| - type: recall_at_1000 |
| value: 97.0 |
| - type: recall_at_20 |
| value: 86.5 |
| - type: recall_at_3 |
| value: 66.9 |
| - type: recall_at_5 |
| value: 73.5 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB EmotionClassification (default) |
| revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
| split: test |
| type: mteb/emotion |
| metrics: |
| - type: accuracy |
| value: 52.195 |
| - type: f1 |
| value: 46.11956692776424 |
| - type: f1_weighted |
| value: 53.928609352293456 |
| - type: main_score |
| value: 52.195 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB FEVER (default) |
| revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 |
| split: test |
| type: mteb/fever |
| metrics: |
| - type: main_score |
| value: 88.48700000000001 |
| - type: map_at_1 |
| value: 76.75699999999999 |
| - type: map_at_10 |
| value: 85.026 |
| - type: map_at_100 |
| value: 85.222 |
| - type: map_at_1000 |
| value: 85.233 |
| - type: map_at_20 |
| value: 85.153 |
| - type: map_at_3 |
| value: 83.995 |
| - type: map_at_5 |
| value: 84.72 |
| - type: mrr_at_1 |
| value: 82.74827482748275 |
| - type: mrr_at_10 |
| value: 89.36737721391175 |
| - type: mrr_at_100 |
| value: 89.42281791031236 |
| - type: mrr_at_1000 |
| value: 89.42338403651277 |
| - type: mrr_at_20 |
| value: 89.41000444854201 |
| - type: mrr_at_3 |
| value: 88.77887788778868 |
| - type: mrr_at_5 |
| value: 89.22892289228906 |
| - type: nauc_map_at_1000_diff1 |
| value: 54.25105398911303 |
| - type: nauc_map_at_1000_max |
| value: 23.429223523468487 |
| - type: nauc_map_at_1000_std |
| value: -1.7334687095777817 |
| - type: nauc_map_at_100_diff1 |
| value: 54.21667352719557 |
| - type: nauc_map_at_100_max |
| value: 23.41584289756298 |
| - type: nauc_map_at_100_std |
| value: -1.7268147416914834 |
| - type: nauc_map_at_10_diff1 |
| value: 53.891546320220954 |
| - type: nauc_map_at_10_max |
| value: 23.331511148674064 |
| - type: nauc_map_at_10_std |
| value: -1.7196488084463741 |
| - type: nauc_map_at_1_diff1 |
| value: 58.25983460638938 |
| - type: nauc_map_at_1_max |
| value: 16.201489550987855 |
| - type: nauc_map_at_1_std |
| value: -6.62306733738586 |
| - type: nauc_map_at_20_diff1 |
| value: 54.09715298373297 |
| - type: nauc_map_at_20_max |
| value: 23.404623777099566 |
| - type: nauc_map_at_20_std |
| value: -1.69489694977684 |
| - type: nauc_map_at_3_diff1 |
| value: 53.99844560192416 |
| - type: nauc_map_at_3_max |
| value: 23.394893454546985 |
| - type: nauc_map_at_3_std |
| value: -2.5898966262447085 |
| - type: nauc_map_at_5_diff1 |
| value: 53.71863187650913 |
| - type: nauc_map_at_5_max |
| value: 23.301072622171013 |
| - type: nauc_map_at_5_std |
| value: -2.0469972205599007 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 71.17515639341956 |
| - type: nauc_mrr_at_1000_max |
| value: 25.0708046486158 |
| - type: nauc_mrr_at_1000_std |
| value: -9.416121374883016 |
| - type: nauc_mrr_at_100_diff1 |
| value: 71.1753164464445 |
| - type: nauc_mrr_at_100_max |
| value: 25.07419346909715 |
| - type: nauc_mrr_at_100_std |
| value: -9.412863113733295 |
| - type: nauc_mrr_at_10_diff1 |
| value: 71.09405552697164 |
| - type: nauc_mrr_at_10_max |
| value: 25.26590564954804 |
| - type: nauc_mrr_at_10_std |
| value: -9.298704438200227 |
| - type: nauc_mrr_at_1_diff1 |
| value: 72.03214906017645 |
| - type: nauc_mrr_at_1_max |
| value: 19.686864438615697 |
| - type: nauc_mrr_at_1_std |
| value: -11.46718406152579 |
| - type: nauc_mrr_at_20_diff1 |
| value: 71.16991459665647 |
| - type: nauc_mrr_at_20_max |
| value: 25.145335855346197 |
| - type: nauc_mrr_at_20_std |
| value: -9.37386189687834 |
| - type: nauc_mrr_at_3_diff1 |
| value: 70.94963146032211 |
| - type: nauc_mrr_at_3_max |
| value: 26.231208166551717 |
| - type: nauc_mrr_at_3_std |
| value: -9.356935646618718 |
| - type: nauc_mrr_at_5_diff1 |
| value: 71.19526205241235 |
| - type: nauc_mrr_at_5_max |
| value: 25.695147330404268 |
| - type: nauc_mrr_at_5_std |
| value: -9.3075800123897 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 56.06544294033199 |
| - type: nauc_ndcg_at_1000_max |
| value: 25.610054392303667 |
| - type: nauc_ndcg_at_1000_std |
| value: -0.6867283406842567 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 55.25185300646874 |
| - type: nauc_ndcg_at_100_max |
| value: 25.399170365704 |
| - type: nauc_ndcg_at_100_std |
| value: -0.46547332840855044 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 54.016224221161245 |
| - type: nauc_ndcg_at_10_max |
| value: 25.442317454780277 |
| - type: nauc_ndcg_at_10_std |
| value: -0.28391008237610216 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 72.03214906017645 |
| - type: nauc_ndcg_at_1_max |
| value: 19.686864438615697 |
| - type: nauc_ndcg_at_1_std |
| value: -11.46718406152579 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 54.658404506399464 |
| - type: nauc_ndcg_at_20_max |
| value: 25.495663741198126 |
| - type: nauc_ndcg_at_20_std |
| value: -0.26800797289758815 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 55.5763557237763 |
| - type: nauc_ndcg_at_3_max |
| value: 26.4988763664988 |
| - type: nauc_ndcg_at_3_std |
| value: -2.3981007097238343 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 54.27240486490372 |
| - type: nauc_ndcg_at_5_max |
| value: 25.82259059583224 |
| - type: nauc_ndcg_at_5_std |
| value: -1.1890812042559784 |
| - type: nauc_precision_at_1000_diff1 |
| value: -7.60072118353888 |
| - type: nauc_precision_at_1000_max |
| value: 4.620559244156039 |
| - type: nauc_precision_at_1000_std |
| value: 3.5812750588401463 |
| - type: nauc_precision_at_100_diff1 |
| value: -9.45455804679522 |
| - type: nauc_precision_at_100_max |
| value: 6.631695936980273 |
| - type: nauc_precision_at_100_std |
| value: 6.451478574268801 |
| - type: nauc_precision_at_10_diff1 |
| value: -6.726955843425629 |
| - type: nauc_precision_at_10_max |
| value: 14.546007414428736 |
| - type: nauc_precision_at_10_std |
| value: 10.825230172002183 |
| - type: nauc_precision_at_1_diff1 |
| value: 72.03214906017645 |
| - type: nauc_precision_at_1_max |
| value: 19.686864438615697 |
| - type: nauc_precision_at_1_std |
| value: -11.46718406152579 |
| - type: nauc_precision_at_20_diff1 |
| value: -8.734846281871329 |
| - type: nauc_precision_at_20_max |
| value: 10.730418617259506 |
| - type: nauc_precision_at_20_std |
| value: 8.801245191066164 |
| - type: nauc_precision_at_3_diff1 |
| value: 17.69525577378896 |
| - type: nauc_precision_at_3_max |
| value: 29.697514372659484 |
| - type: nauc_precision_at_3_std |
| value: 6.020200289148097 |
| - type: nauc_precision_at_5_diff1 |
| value: 0.24533250177984312 |
| - type: nauc_precision_at_5_max |
| value: 20.531345723824952 |
| - type: nauc_precision_at_5_std |
| value: 9.158699123344162 |
| - type: nauc_recall_at_1000_diff1 |
| value: 10.77222496473275 |
| - type: nauc_recall_at_1000_max |
| value: 50.734546876280554 |
| - type: nauc_recall_at_1000_std |
| value: 55.206779538792986 |
| - type: nauc_recall_at_100_diff1 |
| value: 10.839466570515647 |
| - type: nauc_recall_at_100_max |
| value: 36.14527764646136 |
| - type: nauc_recall_at_100_std |
| value: 36.76142693997371 |
| - type: nauc_recall_at_10_diff1 |
| value: 20.82687898494084 |
| - type: nauc_recall_at_10_max |
| value: 32.686402220501726 |
| - type: nauc_recall_at_10_std |
| value: 21.652679688979624 |
| - type: nauc_recall_at_1_diff1 |
| value: 58.25983460638938 |
| - type: nauc_recall_at_1_max |
| value: 16.201489550987855 |
| - type: nauc_recall_at_1_std |
| value: -6.62306733738586 |
| - type: nauc_recall_at_20_diff1 |
| value: 18.19772056092292 |
| - type: nauc_recall_at_20_max |
| value: 34.47222550318253 |
| - type: nauc_recall_at_20_std |
| value: 27.38829232665364 |
| - type: nauc_recall_at_3_diff1 |
| value: 37.124181479070806 |
| - type: nauc_recall_at_3_max |
| value: 32.43172426208055 |
| - type: nauc_recall_at_3_std |
| value: 6.335659080755107 |
| - type: nauc_recall_at_5_diff1 |
| value: 28.13275823420512 |
| - type: nauc_recall_at_5_max |
| value: 32.34074997818357 |
| - type: nauc_recall_at_5_std |
| value: 12.824487132524897 |
| - type: ndcg_at_1 |
| value: 82.748 |
| - type: ndcg_at_10 |
| value: 88.48700000000001 |
| - type: ndcg_at_100 |
| value: 89.121 |
| - type: ndcg_at_1000 |
| value: 89.31700000000001 |
| - type: ndcg_at_20 |
| value: 88.809 |
| - type: ndcg_at_3 |
| value: 87.01299999999999 |
| - type: ndcg_at_5 |
| value: 87.96300000000001 |
| - type: precision_at_1 |
| value: 82.748 |
| - type: precision_at_10 |
| value: 10.546 |
| - type: precision_at_100 |
| value: 1.107 |
| - type: precision_at_1000 |
| value: 0.11399999999999999 |
| - type: precision_at_20 |
| value: 5.379 |
| - type: precision_at_3 |
| value: 33.173 |
| - type: precision_at_5 |
| value: 20.588 |
| - type: recall_at_1 |
| value: 76.75699999999999 |
| - type: recall_at_10 |
| value: 94.796 |
| - type: recall_at_100 |
| value: 97.174 |
| - type: recall_at_1000 |
| value: 98.349 |
| - type: recall_at_20 |
| value: 95.86 |
| - type: recall_at_3 |
| value: 90.814 |
| - type: recall_at_5 |
| value: 93.235 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB FiQA2018 (default) |
| revision: 27a168819829fe9bcd655c2df245fb19452e8e06 |
| split: test |
| type: mteb/fiqa |
| metrics: |
| - type: main_score |
| value: 47.035 |
| - type: map_at_1 |
| value: 24.295 |
| - type: map_at_10 |
| value: 39.029 |
| - type: map_at_100 |
| value: 41.016999999999996 |
| - type: map_at_1000 |
| value: 41.182 |
| - type: map_at_20 |
| value: 40.182 |
| - type: map_at_3 |
| value: 34.128 |
| - type: map_at_5 |
| value: 36.771 |
| - type: mrr_at_1 |
| value: 47.0679012345679 |
| - type: mrr_at_10 |
| value: 55.334852047815005 |
| - type: mrr_at_100 |
| value: 56.06915046612819 |
| - type: mrr_at_1000 |
| value: 56.10322909085006 |
| - type: mrr_at_20 |
| value: 55.82851614255695 |
| - type: mrr_at_3 |
| value: 52.98353909465019 |
| - type: mrr_at_5 |
| value: 54.21039094650203 |
| - type: nauc_map_at_1000_diff1 |
| value: 43.511658309461076 |
| - type: nauc_map_at_1000_max |
| value: 33.909990774712945 |
| - type: nauc_map_at_1000_std |
| value: -2.204768114675481 |
| - type: nauc_map_at_100_diff1 |
| value: 43.47352725497821 |
| - type: nauc_map_at_100_max |
| value: 33.831208204956994 |
| - type: nauc_map_at_100_std |
| value: -2.2426244577314565 |
| - type: nauc_map_at_10_diff1 |
| value: 43.44192893607367 |
| - type: nauc_map_at_10_max |
| value: 32.46143380861397 |
| - type: nauc_map_at_10_std |
| value: -3.424608611118555 |
| - type: nauc_map_at_1_diff1 |
| value: 48.36982230823535 |
| - type: nauc_map_at_1_max |
| value: 20.538023574672817 |
| - type: nauc_map_at_1_std |
| value: -6.313140012964799 |
| - type: nauc_map_at_20_diff1 |
| value: 43.3916601464767 |
| - type: nauc_map_at_20_max |
| value: 33.1621245847151 |
| - type: nauc_map_at_20_std |
| value: -2.4792616401386303 |
| - type: nauc_map_at_3_diff1 |
| value: 44.39544319440273 |
| - type: nauc_map_at_3_max |
| value: 28.173138602900078 |
| - type: nauc_map_at_3_std |
| value: -4.827558609939407 |
| - type: nauc_map_at_5_diff1 |
| value: 43.74632166484276 |
| - type: nauc_map_at_5_max |
| value: 30.962682241438205 |
| - type: nauc_map_at_5_std |
| value: -4.0383602707482975 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 49.72736651560055 |
| - type: nauc_mrr_at_1000_max |
| value: 43.775987123828216 |
| - type: nauc_mrr_at_1000_std |
| value: 0.2605801030626549 |
| - type: nauc_mrr_at_100_diff1 |
| value: 49.698356271296944 |
| - type: nauc_mrr_at_100_max |
| value: 43.79303269950675 |
| - type: nauc_mrr_at_100_std |
| value: 0.27383247751044537 |
| - type: nauc_mrr_at_10_diff1 |
| value: 49.74695781871661 |
| - type: nauc_mrr_at_10_max |
| value: 43.70095639468644 |
| - type: nauc_mrr_at_10_std |
| value: -0.0910007265618897 |
| - type: nauc_mrr_at_1_diff1 |
| value: 52.72283694395142 |
| - type: nauc_mrr_at_1_max |
| value: 42.44702827453944 |
| - type: nauc_mrr_at_1_std |
| value: -2.8273823855670255 |
| - type: nauc_mrr_at_20_diff1 |
| value: 49.66790615633498 |
| - type: nauc_mrr_at_20_max |
| value: 43.758962529366194 |
| - type: nauc_mrr_at_20_std |
| value: 0.3426322120672393 |
| - type: nauc_mrr_at_3_diff1 |
| value: 50.24600816852405 |
| - type: nauc_mrr_at_3_max |
| value: 44.05231137252421 |
| - type: nauc_mrr_at_3_std |
| value: 0.3241339755957089 |
| - type: nauc_mrr_at_5_diff1 |
| value: 49.5975151012115 |
| - type: nauc_mrr_at_5_max |
| value: 43.68322913701036 |
| - type: nauc_mrr_at_5_std |
| value: -0.006452533848350892 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 44.375754408381894 |
| - type: nauc_ndcg_at_1000_max |
| value: 39.06222884248439 |
| - type: nauc_ndcg_at_1000_std |
| value: 1.2790165406784537 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 43.55596660750431 |
| - type: nauc_ndcg_at_100_max |
| value: 38.58416939185971 |
| - type: nauc_ndcg_at_100_std |
| value: 1.3982431563388764 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 43.42342985549579 |
| - type: nauc_ndcg_at_10_max |
| value: 35.39654814350948 |
| - type: nauc_ndcg_at_10_std |
| value: -1.9691263385874018 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 52.72283694395142 |
| - type: nauc_ndcg_at_1_max |
| value: 42.44702827453944 |
| - type: nauc_ndcg_at_1_std |
| value: -2.8273823855670255 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 43.18638092853598 |
| - type: nauc_ndcg_at_20_max |
| value: 36.12317468609796 |
| - type: nauc_ndcg_at_20_std |
| value: 0.25078096107927306 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 44.586398632399366 |
| - type: nauc_ndcg_at_3_max |
| value: 37.89220961256707 |
| - type: nauc_ndcg_at_3_std |
| value: -2.448074667259283 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 43.64088923894009 |
| - type: nauc_ndcg_at_5_max |
| value: 35.94499252340929 |
| - type: nauc_ndcg_at_5_std |
| value: -2.4540364610254857 |
| - type: nauc_precision_at_1000_diff1 |
| value: -1.6609012856010976 |
| - type: nauc_precision_at_1000_max |
| value: 30.951360889282455 |
| - type: nauc_precision_at_1000_std |
| value: 10.832115521132394 |
| - type: nauc_precision_at_100_diff1 |
| value: 3.8635753172116454 |
| - type: nauc_precision_at_100_max |
| value: 37.50549346606815 |
| - type: nauc_precision_at_100_std |
| value: 12.984264349425006 |
| - type: nauc_precision_at_10_diff1 |
| value: 15.096155551489035 |
| - type: nauc_precision_at_10_max |
| value: 41.157377147091935 |
| - type: nauc_precision_at_10_std |
| value: 6.541970514670327 |
| - type: nauc_precision_at_1_diff1 |
| value: 52.72283694395142 |
| - type: nauc_precision_at_1_max |
| value: 42.44702827453944 |
| - type: nauc_precision_at_1_std |
| value: -2.8273823855670255 |
| - type: nauc_precision_at_20_diff1 |
| value: 10.77837369063063 |
| - type: nauc_precision_at_20_max |
| value: 39.02870175375101 |
| - type: nauc_precision_at_20_std |
| value: 11.493998523134003 |
| - type: nauc_precision_at_3_diff1 |
| value: 27.719913494785082 |
| - type: nauc_precision_at_3_max |
| value: 42.32147757624575 |
| - type: nauc_precision_at_3_std |
| value: 1.675159078162856 |
| - type: nauc_precision_at_5_diff1 |
| value: 21.13559680138858 |
| - type: nauc_precision_at_5_max |
| value: 42.94690948385399 |
| - type: nauc_precision_at_5_std |
| value: 4.269082271873189 |
| - type: nauc_recall_at_1000_diff1 |
| value: 29.211629532377664 |
| - type: nauc_recall_at_1000_max |
| value: 38.27913905193411 |
| - type: nauc_recall_at_1000_std |
| value: 33.777853794495186 |
| - type: nauc_recall_at_100_diff1 |
| value: 27.540258851819743 |
| - type: nauc_recall_at_100_max |
| value: 34.93970481970824 |
| - type: nauc_recall_at_100_std |
| value: 14.696131816776942 |
| - type: nauc_recall_at_10_diff1 |
| value: 33.429209402623314 |
| - type: nauc_recall_at_10_max |
| value: 26.83870557170468 |
| - type: nauc_recall_at_10_std |
| value: -1.7141062811893624 |
| - type: nauc_recall_at_1_diff1 |
| value: 48.36982230823535 |
| - type: nauc_recall_at_1_max |
| value: 20.538023574672817 |
| - type: nauc_recall_at_1_std |
| value: -6.313140012964799 |
| - type: nauc_recall_at_20_diff1 |
| value: 30.122935323793588 |
| - type: nauc_recall_at_20_max |
| value: 26.510122532461565 |
| - type: nauc_recall_at_20_std |
| value: 4.836919434308895 |
| - type: nauc_recall_at_3_diff1 |
| value: 38.95587878059384 |
| - type: nauc_recall_at_3_max |
| value: 25.25220801695804 |
| - type: nauc_recall_at_3_std |
| value: -3.7202422156547095 |
| - type: nauc_recall_at_5_diff1 |
| value: 35.913508616203146 |
| - type: nauc_recall_at_5_max |
| value: 26.70575052525446 |
| - type: nauc_recall_at_5_std |
| value: -3.047557854303276 |
| - type: ndcg_at_1 |
| value: 47.068 |
| - type: ndcg_at_10 |
| value: 47.035 |
| - type: ndcg_at_100 |
| value: 53.72 |
| - type: ndcg_at_1000 |
| value: 56.35 |
| - type: ndcg_at_20 |
| value: 49.830999999999996 |
| - type: ndcg_at_3 |
| value: 43.327 |
| - type: ndcg_at_5 |
| value: 44.18 |
| - type: precision_at_1 |
| value: 47.068 |
| - type: precision_at_10 |
| value: 12.948 |
| - type: precision_at_100 |
| value: 1.992 |
| - type: precision_at_1000 |
| value: 0.244 |
| - type: precision_at_20 |
| value: 7.670000000000001 |
| - type: precision_at_3 |
| value: 28.601 |
| - type: precision_at_5 |
| value: 20.772 |
| - type: recall_at_1 |
| value: 24.295 |
| - type: recall_at_10 |
| value: 53.681999999999995 |
| - type: recall_at_100 |
| value: 78.072 |
| - type: recall_at_1000 |
| value: 93.866 |
| - type: recall_at_20 |
| value: 62.18900000000001 |
| - type: recall_at_3 |
| value: 38.836 |
| - type: recall_at_5 |
| value: 44.779 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB HotpotQA (default) |
| revision: ab518f4d6fcca38d87c25209f94beba119d02014 |
| split: test |
| type: mteb/hotpotqa |
| metrics: |
| - type: main_score |
| value: 71.324 |
| - type: map_at_1 |
| value: 39.73 |
| - type: map_at_10 |
| value: 63.046 |
| - type: map_at_100 |
| value: 63.885999999999996 |
| - type: map_at_1000 |
| value: 63.94499999999999 |
| - type: map_at_20 |
| value: 63.548 |
| - type: map_at_3 |
| value: 59.655 |
| - type: map_at_5 |
| value: 61.795 |
| - type: mrr_at_1 |
| value: 79.45982444294395 |
| - type: mrr_at_10 |
| value: 85.08343247269632 |
| - type: mrr_at_100 |
| value: 85.24432818683675 |
| - type: mrr_at_1000 |
| value: 85.25040798349796 |
| - type: mrr_at_20 |
| value: 85.18723931309329 |
| - type: mrr_at_3 |
| value: 84.18861129867187 |
| - type: mrr_at_5 |
| value: 84.75647085302693 |
| - type: nauc_map_at_1000_diff1 |
| value: 18.712841235300363 |
| - type: nauc_map_at_1000_max |
| value: 17.872061255736362 |
| - type: nauc_map_at_1000_std |
| value: 5.493306103578567 |
| - type: nauc_map_at_100_diff1 |
| value: 18.67843452691231 |
| - type: nauc_map_at_100_max |
| value: 17.854433817054684 |
| - type: nauc_map_at_100_std |
| value: 5.505106020385131 |
| - type: nauc_map_at_10_diff1 |
| value: 18.431763547762873 |
| - type: nauc_map_at_10_max |
| value: 17.57826680597676 |
| - type: nauc_map_at_10_std |
| value: 4.849651427482733 |
| - type: nauc_map_at_1_diff1 |
| value: 71.41720657694039 |
| - type: nauc_map_at_1_max |
| value: 40.52361010802207 |
| - type: nauc_map_at_1_std |
| value: -3.4966985764484835 |
| - type: nauc_map_at_20_diff1 |
| value: 18.6008341594555 |
| - type: nauc_map_at_20_max |
| value: 17.7648824369578 |
| - type: nauc_map_at_20_std |
| value: 5.297255330344646 |
| - type: nauc_map_at_3_diff1 |
| value: 19.195615691261565 |
| - type: nauc_map_at_3_max |
| value: 17.466626379813636 |
| - type: nauc_map_at_3_std |
| value: 2.6243834953938427 |
| - type: nauc_map_at_5_diff1 |
| value: 18.55653332525488 |
| - type: nauc_map_at_5_max |
| value: 17.461088221922633 |
| - type: nauc_map_at_5_std |
| value: 3.853024189868032 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 71.1366541327153 |
| - type: nauc_mrr_at_1000_max |
| value: 43.32142719320606 |
| - type: nauc_mrr_at_1000_std |
| value: -1.5540399660833892 |
| - type: nauc_mrr_at_100_diff1 |
| value: 71.13849060952187 |
| - type: nauc_mrr_at_100_max |
| value: 43.32989371868815 |
| - type: nauc_mrr_at_100_std |
| value: -1.545670176348025 |
| - type: nauc_mrr_at_10_diff1 |
| value: 71.14457402293097 |
| - type: nauc_mrr_at_10_max |
| value: 43.372769272903284 |
| - type: nauc_mrr_at_10_std |
| value: -1.5393348801875264 |
| - type: nauc_mrr_at_1_diff1 |
| value: 71.41720657694039 |
| - type: nauc_mrr_at_1_max |
| value: 40.52361010802207 |
| - type: nauc_mrr_at_1_std |
| value: -3.4966985764484835 |
| - type: nauc_mrr_at_20_diff1 |
| value: 71.15310136746406 |
| - type: nauc_mrr_at_20_max |
| value: 43.368918600166595 |
| - type: nauc_mrr_at_20_std |
| value: -1.5098290368260359 |
| - type: nauc_mrr_at_3_diff1 |
| value: 70.8374180772855 |
| - type: nauc_mrr_at_3_max |
| value: 43.496540465503756 |
| - type: nauc_mrr_at_3_std |
| value: -1.8531058023308264 |
| - type: nauc_mrr_at_5_diff1 |
| value: 71.0445313174241 |
| - type: nauc_mrr_at_5_max |
| value: 43.48491122151075 |
| - type: nauc_mrr_at_5_std |
| value: -1.7318092266342708 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 25.295005668245402 |
| - type: nauc_ndcg_at_1000_max |
| value: 22.505081411406884 |
| - type: nauc_ndcg_at_1000_std |
| value: 8.4229330023338 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 24.384314427252505 |
| - type: nauc_ndcg_at_100_max |
| value: 22.07684629115631 |
| - type: nauc_ndcg_at_100_std |
| value: 8.829711507432597 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 23.485927495206663 |
| - type: nauc_ndcg_at_10_max |
| value: 20.98700547109215 |
| - type: nauc_ndcg_at_10_std |
| value: 6.32811700835107 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 71.41720657694039 |
| - type: nauc_ndcg_at_1_max |
| value: 40.52361010802207 |
| - type: nauc_ndcg_at_1_std |
| value: -3.4966985764484835 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 23.756288073742923 |
| - type: nauc_ndcg_at_20_max |
| value: 21.413410660850303 |
| - type: nauc_ndcg_at_20_std |
| value: 7.650727686015016 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 25.177960762972727 |
| - type: nauc_ndcg_at_3_max |
| value: 21.14305293776629 |
| - type: nauc_ndcg_at_3_std |
| value: 2.6923518302461122 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 23.990749939332094 |
| - type: nauc_ndcg_at_5_max |
| value: 20.956913803574537 |
| - type: nauc_ndcg_at_5_std |
| value: 4.404215868616478 |
| - type: nauc_precision_at_1000_diff1 |
| value: -2.8791075901918743 |
| - type: nauc_precision_at_1000_max |
| value: 19.15239727544697 |
| - type: nauc_precision_at_1000_std |
| value: 42.77060377512116 |
| - type: nauc_precision_at_100_diff1 |
| value: 1.3378406367413058 |
| - type: nauc_precision_at_100_max |
| value: 16.182541384634163 |
| - type: nauc_precision_at_100_std |
| value: 30.335004023396756 |
| - type: nauc_precision_at_10_diff1 |
| value: 5.54285946463118 |
| - type: nauc_precision_at_10_max |
| value: 13.895094021698995 |
| - type: nauc_precision_at_10_std |
| value: 13.626255441544547 |
| - type: nauc_precision_at_1_diff1 |
| value: 71.41720657694039 |
| - type: nauc_precision_at_1_max |
| value: 40.52361010802207 |
| - type: nauc_precision_at_1_std |
| value: -3.4966985764484835 |
| - type: nauc_precision_at_20_diff1 |
| value: 3.999435996481638 |
| - type: nauc_precision_at_20_max |
| value: 14.198331269969081 |
| - type: nauc_precision_at_20_std |
| value: 19.01442245585053 |
| - type: nauc_precision_at_3_diff1 |
| value: 12.534698984735535 |
| - type: nauc_precision_at_3_max |
| value: 15.937838211499326 |
| - type: nauc_precision_at_3_std |
| value: 4.941150608267901 |
| - type: nauc_precision_at_5_diff1 |
| value: 8.924874254304342 |
| - type: nauc_precision_at_5_max |
| value: 14.839503680284109 |
| - type: nauc_precision_at_5_std |
| value: 8.354174458200886 |
| - type: nauc_recall_at_1000_diff1 |
| value: -2.8791075901918277 |
| - type: nauc_recall_at_1000_max |
| value: 19.15239727544736 |
| - type: nauc_recall_at_1000_std |
| value: 42.77060377512101 |
| - type: nauc_recall_at_100_diff1 |
| value: 1.3378406367413018 |
| - type: nauc_recall_at_100_max |
| value: 16.18254138463397 |
| - type: nauc_recall_at_100_std |
| value: 30.335004023396756 |
| - type: nauc_recall_at_10_diff1 |
| value: 5.542859464631227 |
| - type: nauc_recall_at_10_max |
| value: 13.89509402169902 |
| - type: nauc_recall_at_10_std |
| value: 13.62625544154447 |
| - type: nauc_recall_at_1_diff1 |
| value: 71.41720657694039 |
| - type: nauc_recall_at_1_max |
| value: 40.52361010802207 |
| - type: nauc_recall_at_1_std |
| value: -3.4966985764484835 |
| - type: nauc_recall_at_20_diff1 |
| value: 3.999435996481615 |
| - type: nauc_recall_at_20_max |
| value: 14.198331269969106 |
| - type: nauc_recall_at_20_std |
| value: 19.014422455850642 |
| - type: nauc_recall_at_3_diff1 |
| value: 12.534698984735476 |
| - type: nauc_recall_at_3_max |
| value: 15.937838211499253 |
| - type: nauc_recall_at_3_std |
| value: 4.941150608267872 |
| - type: nauc_recall_at_5_diff1 |
| value: 8.924874254304386 |
| - type: nauc_recall_at_5_max |
| value: 14.839503680284134 |
| - type: nauc_recall_at_5_std |
| value: 8.35417445820087 |
| - type: ndcg_at_1 |
| value: 79.46 |
| - type: ndcg_at_10 |
| value: 71.324 |
| - type: ndcg_at_100 |
| value: 74.18 |
| - type: ndcg_at_1000 |
| value: 75.316 |
| - type: ndcg_at_20 |
| value: 72.551 |
| - type: ndcg_at_3 |
| value: 66.57300000000001 |
| - type: ndcg_at_5 |
| value: 69.241 |
| - type: precision_at_1 |
| value: 79.46 |
| - type: precision_at_10 |
| value: 14.915999999999999 |
| - type: precision_at_100 |
| value: 1.714 |
| - type: precision_at_1000 |
| value: 0.186 |
| - type: precision_at_20 |
| value: 7.852 |
| - type: precision_at_3 |
| value: 42.732 |
| - type: precision_at_5 |
| value: 27.743000000000002 |
| - type: recall_at_1 |
| value: 39.73 |
| - type: recall_at_10 |
| value: 74.578 |
| - type: recall_at_100 |
| value: 85.69200000000001 |
| - type: recall_at_1000 |
| value: 93.194 |
| - type: recall_at_20 |
| value: 78.521 |
| - type: recall_at_3 |
| value: 64.09899999999999 |
| - type: recall_at_5 |
| value: 69.35900000000001 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB IFlyTek (default) |
| revision: 421605374b29664c5fc098418fe20ada9bd55f8a |
| split: test |
| type: C-MTEB/IFlyTek-classification |
| metrics: |
| - type: accuracy |
| value: 0.0 |
| - type: f1 |
| value: 0.0 |
| - type: f1_weighted |
| value: 0.0 |
| - type: main_score |
| value: 0.0 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB IFlyTek (default) |
| revision: 421605374b29664c5fc098418fe20ada9bd55f8a |
| split: validation |
| type: C-MTEB/IFlyTek-classification |
| metrics: |
| - type: accuracy |
| value: 47.97999230473259 |
| - type: f1 |
| value: 35.99868153324778 |
| - type: f1_weighted |
| value: 45.93902403943046 |
| - type: main_score |
| value: 47.97999230473259 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB ImdbClassification (default) |
| revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
| split: test |
| type: mteb/imdb |
| metrics: |
| - type: accuracy |
| value: 87.066 |
| - type: ap |
| value: 81.39504087177659 |
| - type: ap_weighted |
| value: 81.39504087177659 |
| - type: f1 |
| value: 87.03207693979114 |
| - type: f1_weighted |
| value: 87.03207693979114 |
| - type: main_score |
| value: 87.066 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB JDReview (default) |
| revision: b7c64bd89eb87f8ded463478346f76731f07bf8b |
| split: test |
| type: C-MTEB/JDReview-classification |
| metrics: |
| - type: accuracy |
| value: 80.50656660412757 |
| - type: ap |
| value: 44.39524359482253 |
| - type: ap_weighted |
| value: 44.39524359482253 |
| - type: f1 |
| value: 74.47089881755461 |
| - type: f1_weighted |
| value: 82.26720272194022 |
| - type: main_score |
| value: 80.50656660412757 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB LCQMC (default) |
| revision: 17f9b096f80380fce5ed12a9be8be7784b337daf |
| split: test |
| type: C-MTEB/LCQMC |
| metrics: |
| - type: cosine_pearson |
| value: 67.51896934235906 |
| - type: cosine_spearman |
| value: 73.71926625669903 |
| - type: euclidean_pearson |
| value: 71.95413794810199 |
| - type: euclidean_spearman |
| value: 73.7192706889374 |
| - type: main_score |
| value: 73.71926625669903 |
| - type: manhattan_pearson |
| value: 71.99442345245122 |
| - type: manhattan_spearman |
| value: 73.70096693054006 |
| - type: pearson |
| value: 67.51896934235906 |
| - type: spearman |
| value: 73.71926625669903 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB MMarcoReranking (default) |
| revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6 |
| split: dev |
| type: C-MTEB/Mmarco-reranking |
| metrics: |
| - type: main_score |
| value: 32.47827175843312 |
| - type: map |
| value: 32.47827175843312 |
| - type: mrr |
| value: 31.68015873015873 |
| - type: nAUC_map_diff1 |
| value: 28.44752902999802 |
| - type: nAUC_map_max |
| value: -1.2720002819461194 |
| - type: nAUC_map_std |
| value: -17.183634811974066 |
| - type: nAUC_mrr_diff1 |
| value: 28.98249515778471 |
| - type: nAUC_mrr_max |
| value: -2.2626950880487264 |
| - type: nAUC_mrr_std |
| value: -18.15422633230884 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB MMarcoRetrieval (default) |
| revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 |
| split: test |
| type: C-MTEB/MMarcoRetrieval |
| metrics: |
| - type: main_score |
| value: 81.44200000000001 |
| - type: map_at_1 |
| value: 69.057 |
| - type: map_at_10 |
| value: 77.928 |
| - type: map_at_100 |
| value: 78.215 |
| - type: map_at_1000 |
| value: 78.223 |
| - type: map_at_20 |
| value: 78.125 |
| - type: map_at_3 |
| value: 76.149 |
| - type: map_at_5 |
| value: 77.279 |
| - type: mrr_at_1 |
| value: 71.30372492836676 |
| - type: mrr_at_10 |
| value: 78.42867944694576 |
| - type: mrr_at_100 |
| value: 78.68026411713835 |
| - type: mrr_at_1000 |
| value: 78.68715003278672 |
| - type: mrr_at_20 |
| value: 78.59844616726932 |
| - type: mrr_at_3 |
| value: 76.88634192932187 |
| - type: mrr_at_5 |
| value: 77.88061127029584 |
| - type: nauc_map_at_1000_diff1 |
| value: 79.16332823556652 |
| - type: nauc_map_at_1000_max |
| value: 42.136182569734615 |
| - type: nauc_map_at_1000_std |
| value: -14.021601911949988 |
| - type: nauc_map_at_100_diff1 |
| value: 79.1613187755636 |
| - type: nauc_map_at_100_max |
| value: 42.14855231595681 |
| - type: nauc_map_at_100_std |
| value: -14.000665861408082 |
| - type: nauc_map_at_10_diff1 |
| value: 79.06482011907491 |
| - type: nauc_map_at_10_max |
| value: 42.24609232573728 |
| - type: nauc_map_at_10_std |
| value: -14.077085057137879 |
| - type: nauc_map_at_1_diff1 |
| value: 81.21875382508544 |
| - type: nauc_map_at_1_max |
| value: 33.86760525968747 |
| - type: nauc_map_at_1_std |
| value: -20.298879297830517 |
| - type: nauc_map_at_20_diff1 |
| value: 79.11748052836347 |
| - type: nauc_map_at_20_max |
| value: 42.19786763146322 |
| - type: nauc_map_at_20_std |
| value: -13.95931426053525 |
| - type: nauc_map_at_3_diff1 |
| value: 79.04607858094613 |
| - type: nauc_map_at_3_max |
| value: 41.0549681711687 |
| - type: nauc_map_at_3_std |
| value: -15.79648875556265 |
| - type: nauc_map_at_5_diff1 |
| value: 78.98637879660431 |
| - type: nauc_map_at_5_max |
| value: 41.99578332353169 |
| - type: nauc_map_at_5_std |
| value: -14.858893068584978 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 79.65531133349177 |
| - type: nauc_mrr_at_1000_max |
| value: 43.06959597550469 |
| - type: nauc_mrr_at_1000_std |
| value: -12.827067228472465 |
| - type: nauc_mrr_at_100_diff1 |
| value: 79.65273047660082 |
| - type: nauc_mrr_at_100_max |
| value: 43.081843491541136 |
| - type: nauc_mrr_at_100_std |
| value: -12.805057022346073 |
| - type: nauc_mrr_at_10_diff1 |
| value: 79.55218639018759 |
| - type: nauc_mrr_at_10_max |
| value: 43.22658416132472 |
| - type: nauc_mrr_at_10_std |
| value: -12.733112379030551 |
| - type: nauc_mrr_at_1_diff1 |
| value: 81.93745763388229 |
| - type: nauc_mrr_at_1_max |
| value: 38.419758204798775 |
| - type: nauc_mrr_at_1_std |
| value: -18.398257441907354 |
| - type: nauc_mrr_at_20_diff1 |
| value: 79.60762520867635 |
| - type: nauc_mrr_at_20_max |
| value: 43.1410526852498 |
| - type: nauc_mrr_at_20_std |
| value: -12.72941439930238 |
| - type: nauc_mrr_at_3_diff1 |
| value: 79.62059579843309 |
| - type: nauc_mrr_at_3_max |
| value: 42.449859611207934 |
| - type: nauc_mrr_at_3_std |
| value: -14.078192368091425 |
| - type: nauc_mrr_at_5_diff1 |
| value: 79.47461249319058 |
| - type: nauc_mrr_at_5_max |
| value: 42.990249727432364 |
| - type: nauc_mrr_at_5_std |
| value: -13.4004037752339 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 78.85521456806612 |
| - type: nauc_ndcg_at_1000_max |
| value: 44.20182195097601 |
| - type: nauc_ndcg_at_1000_std |
| value: -10.78863328246398 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 78.78612088837218 |
| - type: nauc_ndcg_at_100_max |
| value: 44.67483956772654 |
| - type: nauc_ndcg_at_100_std |
| value: -9.952417799203559 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 78.27575531032375 |
| - type: nauc_ndcg_at_10_max |
| value: 45.3567853674585 |
| - type: nauc_ndcg_at_10_std |
| value: -9.843451827530492 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 81.93745763388229 |
| - type: nauc_ndcg_at_1_max |
| value: 38.419758204798775 |
| - type: nauc_ndcg_at_1_std |
| value: -18.398257441907354 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 78.46005207162577 |
| - type: nauc_ndcg_at_20_max |
| value: 45.11954152664807 |
| - type: nauc_ndcg_at_20_std |
| value: -9.544486301913391 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 78.36667983674094 |
| - type: nauc_ndcg_at_3_max |
| value: 42.9311716520143 |
| - type: nauc_ndcg_at_3_std |
| value: -13.742138987703386 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 78.11043344351806 |
| - type: nauc_ndcg_at_5_max |
| value: 44.569017736822126 |
| - type: nauc_ndcg_at_5_std |
| value: -12.018408823200332 |
| - type: nauc_precision_at_1000_diff1 |
| value: -20.925206348791804 |
| - type: nauc_precision_at_1000_max |
| value: 16.508882531465748 |
| - type: nauc_precision_at_1000_std |
| value: 28.256279950018325 |
| - type: nauc_precision_at_100_diff1 |
| value: -9.112393184096334 |
| - type: nauc_precision_at_100_max |
| value: 25.493617547909665 |
| - type: nauc_precision_at_100_std |
| value: 32.40172394495665 |
| - type: nauc_precision_at_10_diff1 |
| value: 17.208835586727638 |
| - type: nauc_precision_at_10_max |
| value: 37.7546142144074 |
| - type: nauc_precision_at_10_std |
| value: 21.54342493539188 |
| - type: nauc_precision_at_1_diff1 |
| value: 81.93745763388229 |
| - type: nauc_precision_at_1_max |
| value: 38.419758204798775 |
| - type: nauc_precision_at_1_std |
| value: -18.398257441907354 |
| - type: nauc_precision_at_20_diff1 |
| value: 5.981358362224364 |
| - type: nauc_precision_at_20_max |
| value: 32.9389498605972 |
| - type: nauc_precision_at_20_std |
| value: 27.379135010444607 |
| - type: nauc_precision_at_3_diff1 |
| value: 44.160977179222705 |
| - type: nauc_precision_at_3_max |
| value: 40.772105564552746 |
| - type: nauc_precision_at_3_std |
| value: 1.4884707594160764 |
| - type: nauc_precision_at_5_diff1 |
| value: 32.05559296191691 |
| - type: nauc_precision_at_5_max |
| value: 41.200449688782385 |
| - type: nauc_precision_at_5_std |
| value: 9.780866426114939 |
| - type: nauc_recall_at_1000_diff1 |
| value: 71.8206413337552 |
| - type: nauc_recall_at_1000_max |
| value: 90.63640978316079 |
| - type: nauc_recall_at_1000_std |
| value: 77.30548952215615 |
| - type: nauc_recall_at_100_diff1 |
| value: 72.73716194333042 |
| - type: nauc_recall_at_100_max |
| value: 84.00954968630633 |
| - type: nauc_recall_at_100_std |
| value: 62.53425171186474 |
| - type: nauc_recall_at_10_diff1 |
| value: 71.50695852530222 |
| - type: nauc_recall_at_10_max |
| value: 64.81575522599766 |
| - type: nauc_recall_at_10_std |
| value: 17.67037787116186 |
| - type: nauc_recall_at_1_diff1 |
| value: 81.21875382508544 |
| - type: nauc_recall_at_1_max |
| value: 33.86760525968747 |
| - type: nauc_recall_at_1_std |
| value: -20.298879297830517 |
| - type: nauc_recall_at_20_diff1 |
| value: 70.85924807023792 |
| - type: nauc_recall_at_20_max |
| value: 70.62057405576428 |
| - type: nauc_recall_at_20_std |
| value: 31.790496314992726 |
| - type: nauc_recall_at_3_diff1 |
| value: 74.86279950554011 |
| - type: nauc_recall_at_3_max |
| value: 47.821889540251064 |
| - type: nauc_recall_at_3_std |
| value: -8.318141348316889 |
| - type: nauc_recall_at_5_diff1 |
| value: 72.88547370934779 |
| - type: nauc_recall_at_5_max |
| value: 55.21595143637733 |
| - type: nauc_recall_at_5_std |
| value: -0.5790325911766804 |
| - type: ndcg_at_1 |
| value: 71.304 |
| - type: ndcg_at_10 |
| value: 81.44200000000001 |
| - type: ndcg_at_100 |
| value: 82.69 |
| - type: ndcg_at_1000 |
| value: 82.901 |
| - type: ndcg_at_20 |
| value: 82.114 |
| - type: ndcg_at_3 |
| value: 78.091 |
| - type: ndcg_at_5 |
| value: 80.00500000000001 |
| - type: precision_at_1 |
| value: 71.304 |
| - type: precision_at_10 |
| value: 9.764000000000001 |
| - type: precision_at_100 |
| value: 1.0370000000000001 |
| - type: precision_at_1000 |
| value: 0.105 |
| - type: precision_at_20 |
| value: 5.024 |
| - type: precision_at_3 |
| value: 29.244999999999997 |
| - type: precision_at_5 |
| value: 18.567 |
| - type: recall_at_1 |
| value: 69.057 |
| - type: recall_at_10 |
| value: 91.742 |
| - type: recall_at_100 |
| value: 97.295 |
| - type: recall_at_1000 |
| value: 98.97399999999999 |
| - type: recall_at_20 |
| value: 94.328 |
| - type: recall_at_3 |
| value: 82.918 |
| - type: recall_at_5 |
| value: 87.477 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB MSMARCO (default) |
| revision: c5a29a104738b98a9e76336939199e264163d4a0 |
| split: dev |
| type: mteb/msmarco |
| metrics: |
| - type: main_score |
| value: 41.827999999999996 |
| - type: map_at_1 |
| value: 22.245 |
| - type: map_at_10 |
| value: 34.741 |
| - type: map_at_100 |
| value: 35.958 |
| - type: map_at_1000 |
| value: 36.0 |
| - type: map_at_20 |
| value: 35.503 |
| - type: map_at_3 |
| value: 30.676 |
| - type: map_at_5 |
| value: 33.047 |
| - type: mrr_at_1 |
| value: 22.836676217765042 |
| - type: mrr_at_10 |
| value: 35.339734843316506 |
| - type: mrr_at_100 |
| value: 36.48494031661305 |
| - type: mrr_at_1000 |
| value: 36.5220826466321 |
| - type: mrr_at_20 |
| value: 36.061448870738566 |
| - type: mrr_at_3 |
| value: 31.363419293218588 |
| - type: mrr_at_5 |
| value: 33.6778892072588 |
| - type: nauc_map_at_1000_diff1 |
| value: 33.70269918652719 |
| - type: nauc_map_at_1000_max |
| value: -0.9666983449376146 |
| - type: nauc_map_at_1000_std |
| value: -24.106835117162635 |
| - type: nauc_map_at_100_diff1 |
| value: 33.69582567164444 |
| - type: nauc_map_at_100_max |
| value: -0.9713399181710164 |
| - type: nauc_map_at_100_std |
| value: -24.09732526417952 |
| - type: nauc_map_at_10_diff1 |
| value: 33.54760088792205 |
| - type: nauc_map_at_10_max |
| value: -1.108864145592058 |
| - type: nauc_map_at_10_std |
| value: -24.714000593926635 |
| - type: nauc_map_at_1_diff1 |
| value: 37.83176482911279 |
| - type: nauc_map_at_1_max |
| value: -1.2803428780118231 |
| - type: nauc_map_at_1_std |
| value: -21.43672521847787 |
| - type: nauc_map_at_20_diff1 |
| value: 33.56330277702434 |
| - type: nauc_map_at_20_max |
| value: -1.0453224620903316 |
| - type: nauc_map_at_20_std |
| value: -24.397377217635892 |
| - type: nauc_map_at_3_diff1 |
| value: 33.836449090455694 |
| - type: nauc_map_at_3_max |
| value: -1.4151762945866553 |
| - type: nauc_map_at_3_std |
| value: -24.53920025162081 |
| - type: nauc_map_at_5_diff1 |
| value: 33.540475708611254 |
| - type: nauc_map_at_5_max |
| value: -1.2270133827372984 |
| - type: nauc_map_at_5_std |
| value: -24.898031963382653 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 33.546755355679295 |
| - type: nauc_mrr_at_1000_max |
| value: -0.9999621583376623 |
| - type: nauc_mrr_at_1000_std |
| value: -23.885688782118415 |
| - type: nauc_mrr_at_100_diff1 |
| value: 33.540941435457846 |
| - type: nauc_mrr_at_100_max |
| value: -1.001220466565939 |
| - type: nauc_mrr_at_100_std |
| value: -23.875083905633048 |
| - type: nauc_mrr_at_10_diff1 |
| value: 33.37588602067944 |
| - type: nauc_mrr_at_10_max |
| value: -1.0813769231834895 |
| - type: nauc_mrr_at_10_std |
| value: -24.438406987527287 |
| - type: nauc_mrr_at_1_diff1 |
| value: 37.70984030766279 |
| - type: nauc_mrr_at_1_max |
| value: -1.3745841550868614 |
| - type: nauc_mrr_at_1_std |
| value: -21.46461137322961 |
| - type: nauc_mrr_at_20_diff1 |
| value: 33.40614386892839 |
| - type: nauc_mrr_at_20_max |
| value: -1.0449149378336973 |
| - type: nauc_mrr_at_20_std |
| value: -24.13679244294705 |
| - type: nauc_mrr_at_3_diff1 |
| value: 33.644563276200735 |
| - type: nauc_mrr_at_3_max |
| value: -1.4969606485922458 |
| - type: nauc_mrr_at_3_std |
| value: -24.348111206749714 |
| - type: nauc_mrr_at_5_diff1 |
| value: 33.36640792187642 |
| - type: nauc_mrr_at_5_max |
| value: -1.2313355299819755 |
| - type: nauc_mrr_at_5_std |
| value: -24.630079858307177 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 33.0122331192661 |
| - type: nauc_ndcg_at_1000_max |
| value: -0.24697428352372258 |
| - type: nauc_ndcg_at_1000_std |
| value: -23.013467138693887 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 32.86665293711552 |
| - type: nauc_ndcg_at_100_max |
| value: -0.276416624031757 |
| - type: nauc_ndcg_at_100_std |
| value: -22.45097004537971 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 32.06009904567439 |
| - type: nauc_ndcg_at_10_max |
| value: -0.9105345903791483 |
| - type: nauc_ndcg_at_10_std |
| value: -25.661880461901248 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 37.70984030766279 |
| - type: nauc_ndcg_at_1_max |
| value: -1.3745841550868614 |
| - type: nauc_ndcg_at_1_std |
| value: -21.46461137322961 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 32.067609578292775 |
| - type: nauc_ndcg_at_20_max |
| value: -0.732282304094851 |
| - type: nauc_ndcg_at_20_std |
| value: -24.550324249058423 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 32.60074846100642 |
| - type: nauc_ndcg_at_3_max |
| value: -1.5329621325967313 |
| - type: nauc_ndcg_at_3_std |
| value: -25.410306390920322 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 32.05683625760298 |
| - type: nauc_ndcg_at_5_max |
| value: -1.155409896292399 |
| - type: nauc_ndcg_at_5_std |
| value: -25.997867512038702 |
| - type: nauc_precision_at_1000_diff1 |
| value: -0.582363922011796 |
| - type: nauc_precision_at_1000_max |
| value: 15.367854085208096 |
| - type: nauc_precision_at_1000_std |
| value: 16.62922885462353 |
| - type: nauc_precision_at_100_diff1 |
| value: 13.413869212944443 |
| - type: nauc_precision_at_100_max |
| value: 9.540599900741062 |
| - type: nauc_precision_at_100_std |
| value: 8.598685767883458 |
| - type: nauc_precision_at_10_diff1 |
| value: 24.607692201117835 |
| - type: nauc_precision_at_10_max |
| value: 0.4073275292029154 |
| - type: nauc_precision_at_10_std |
| value: -26.55809497339693 |
| - type: nauc_precision_at_1_diff1 |
| value: 37.70984030766279 |
| - type: nauc_precision_at_1_max |
| value: -1.3745841550868614 |
| - type: nauc_precision_at_1_std |
| value: -21.46461137322961 |
| - type: nauc_precision_at_20_diff1 |
| value: 20.76545064732853 |
| - type: nauc_precision_at_20_max |
| value: 2.1323836200645387 |
| - type: nauc_precision_at_20_std |
| value: -19.423536825556933 |
| - type: nauc_precision_at_3_diff1 |
| value: 28.62040804487786 |
| - type: nauc_precision_at_3_max |
| value: -1.7875552566437067 |
| - type: nauc_precision_at_3_std |
| value: -27.4938024637869 |
| - type: nauc_precision_at_5_diff1 |
| value: 26.57961892416209 |
| - type: nauc_precision_at_5_max |
| value: -0.821025657887804 |
| - type: nauc_precision_at_5_std |
| value: -28.4053588476215 |
| - type: nauc_recall_at_1000_diff1 |
| value: 29.957070547516786 |
| - type: nauc_recall_at_1000_max |
| value: 37.51269513653321 |
| - type: nauc_recall_at_1000_std |
| value: 50.832935513386445 |
| - type: nauc_recall_at_100_diff1 |
| value: 29.124873637284093 |
| - type: nauc_recall_at_100_max |
| value: 7.456855039971972 |
| - type: nauc_recall_at_100_std |
| value: 5.513183800655616 |
| - type: nauc_recall_at_10_diff1 |
| value: 27.2239066879356 |
| - type: nauc_recall_at_10_max |
| value: -0.40501611552803435 |
| - type: nauc_recall_at_10_std |
| value: -28.66151209173145 |
| - type: nauc_recall_at_1_diff1 |
| value: 37.83176482911279 |
| - type: nauc_recall_at_1_max |
| value: -1.2803428780118231 |
| - type: nauc_recall_at_1_std |
| value: -21.43672521847787 |
| - type: nauc_recall_at_20_diff1 |
| value: 25.996575895775436 |
| - type: nauc_recall_at_20_max |
| value: 0.371917541705145 |
| - type: nauc_recall_at_20_std |
| value: -24.05013745494552 |
| - type: nauc_recall_at_3_diff1 |
| value: 29.23774283371172 |
| - type: nauc_recall_at_3_max |
| value: -1.792638771577912 |
| - type: nauc_recall_at_3_std |
| value: -27.680214935573588 |
| - type: nauc_recall_at_5_diff1 |
| value: 27.78539931643594 |
| - type: nauc_recall_at_5_max |
| value: -0.9461596361216702 |
| - type: nauc_recall_at_5_std |
| value: -29.02852975571309 |
| - type: ndcg_at_1 |
| value: 22.837 |
| - type: ndcg_at_10 |
| value: 41.827999999999996 |
| - type: ndcg_at_100 |
| value: 47.602 |
| - type: ndcg_at_1000 |
| value: 48.638999999999996 |
| - type: ndcg_at_20 |
| value: 44.506 |
| - type: ndcg_at_3 |
| value: 33.594 |
| - type: ndcg_at_5 |
| value: 37.81 |
| - type: precision_at_1 |
| value: 22.837 |
| - type: precision_at_10 |
| value: 6.65 |
| - type: precision_at_100 |
| value: 0.954 |
| - type: precision_at_1000 |
| value: 0.104 |
| - type: precision_at_20 |
| value: 3.8859999999999997 |
| - type: precision_at_3 |
| value: 14.302999999999999 |
| - type: precision_at_5 |
| value: 10.719 |
| - type: recall_at_1 |
| value: 22.245 |
| - type: recall_at_10 |
| value: 63.660000000000004 |
| - type: recall_at_100 |
| value: 90.187 |
| - type: recall_at_1000 |
| value: 98.095 |
| - type: recall_at_20 |
| value: 74.008 |
| - type: recall_at_3 |
| value: 41.349999999999994 |
| - type: recall_at_5 |
| value: 51.480000000000004 |
| task: |
| type: Retrieval |
| - dataset: |
| config: en |
| name: MTEB MTOPDomainClassification (en) |
| revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
| split: test |
| type: mteb/mtop_domain |
| metrics: |
| - type: accuracy |
| value: 93.86000911992704 |
| - type: f1 |
| value: 93.36462701030769 |
| - type: f1_weighted |
| value: 93.87166235541487 |
| - type: main_score |
| value: 93.86000911992704 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB MTOPIntentClassification (en) |
| revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
| split: test |
| type: mteb/mtop_intent |
| metrics: |
| - type: accuracy |
| value: 71.77610579115368 |
| - type: f1 |
| value: 52.094627301273746 |
| - type: f1_weighted |
| value: 74.31447677132623 |
| - type: main_score |
| value: 71.77610579115368 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB MassiveIntentClassification (en) |
| revision: 4672e20407010da34463acc759c162ca9734bca6 |
| split: test |
| type: mteb/amazon_massive_intent |
| metrics: |
| - type: accuracy |
| value: 72.14525891055817 |
| - type: f1 |
| value: 70.01668873115348 |
| - type: f1_weighted |
| value: 71.0196932891963 |
| - type: main_score |
| value: 72.14525891055817 |
| task: |
| type: Classification |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveIntentClassification (zh-CN) |
| revision: 4672e20407010da34463acc759c162ca9734bca6 |
| split: test |
| type: mteb/amazon_massive_intent |
| metrics: |
| - type: accuracy |
| value: 70.3866845998655 |
| - type: f1 |
| value: 68.0106461866208 |
| - type: f1_weighted |
| value: 69.47183715090725 |
| - type: main_score |
| value: 70.3866845998655 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB MassiveIntentClassification (en) |
| revision: 4672e20407010da34463acc759c162ca9734bca6 |
| split: validation |
| type: mteb/amazon_massive_intent |
| metrics: |
| - type: accuracy |
| value: 72.56763403836695 |
| - type: f1 |
| value: 68.74137086779079 |
| - type: f1_weighted |
| value: 71.17832082465809 |
| - type: main_score |
| value: 72.56763403836695 |
| task: |
| type: Classification |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveIntentClassification (zh-CN) |
| revision: 4672e20407010da34463acc759c162ca9734bca6 |
| split: validation |
| type: mteb/amazon_massive_intent |
| metrics: |
| - type: accuracy |
| value: 71.17560255779637 |
| - type: f1 |
| value: 67.53436094771642 |
| - type: f1_weighted |
| value: 69.85911870240461 |
| - type: main_score |
| value: 71.17560255779637 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB MassiveScenarioClassification (en) |
| revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
| split: test |
| type: mteb/amazon_massive_scenario |
| metrics: |
| - type: accuracy |
| value: 75.83725622057834 |
| - type: f1 |
| value: 74.81652741027294 |
| - type: f1_weighted |
| value: 75.64384667945804 |
| - type: main_score |
| value: 75.83725622057834 |
| task: |
| type: Classification |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveScenarioClassification (zh-CN) |
| revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
| split: test |
| type: mteb/amazon_massive_scenario |
| metrics: |
| - type: accuracy |
| value: 74.77471418964359 |
| - type: f1 |
| value: 74.50834305419674 |
| - type: f1_weighted |
| value: 74.51089478391411 |
| - type: main_score |
| value: 74.77471418964359 |
| task: |
| type: Classification |
| - dataset: |
| config: en |
| name: MTEB MassiveScenarioClassification (en) |
| revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
| split: validation |
| type: mteb/amazon_massive_scenario |
| metrics: |
| - type: accuracy |
| value: 75.2680767338908 |
| - type: f1 |
| value: 73.84891408751763 |
| - type: f1_weighted |
| value: 75.0958616975504 |
| - type: main_score |
| value: 75.2680767338908 |
| task: |
| type: Classification |
| - dataset: |
| config: zh-CN |
| name: MTEB MassiveScenarioClassification (zh-CN) |
| revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 |
| split: validation |
| type: mteb/amazon_massive_scenario |
| metrics: |
| - type: accuracy |
| value: 74.20068863748155 |
| - type: f1 |
| value: 73.56517145836091 |
| - type: f1_weighted |
| value: 74.02483580359413 |
| - type: main_score |
| value: 74.20068863748155 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB MedicalRetrieval (default) |
| revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 |
| split: test |
| type: C-MTEB/MedicalRetrieval |
| metrics: |
| - type: main_score |
| value: 59.93000000000001 |
| - type: map_at_1 |
| value: 52.5 |
| - type: map_at_10 |
| value: 57.489000000000004 |
| - type: map_at_100 |
| value: 58.006 |
| - type: map_at_1000 |
| value: 58.06 |
| - type: map_at_20 |
| value: 57.757999999999996 |
| - type: map_at_3 |
| value: 56.2 |
| - type: map_at_5 |
| value: 56.974999999999994 |
| - type: mrr_at_1 |
| value: 52.7 |
| - type: mrr_at_10 |
| value: 57.588531746031755 |
| - type: mrr_at_100 |
| value: 58.10652803307724 |
| - type: mrr_at_1000 |
| value: 58.160443460868684 |
| - type: mrr_at_20 |
| value: 57.858160152540975 |
| - type: mrr_at_3 |
| value: 56.3 |
| - type: mrr_at_5 |
| value: 57.074999999999996 |
| - type: nauc_map_at_1000_diff1 |
| value: 79.08319909758636 |
| - type: nauc_map_at_1000_max |
| value: 66.40358430901192 |
| - type: nauc_map_at_1000_std |
| value: 24.962984166768837 |
| - type: nauc_map_at_100_diff1 |
| value: 79.06038136198957 |
| - type: nauc_map_at_100_max |
| value: 66.39726845981066 |
| - type: nauc_map_at_100_std |
| value: 24.949716444423807 |
| - type: nauc_map_at_10_diff1 |
| value: 79.17675719820811 |
| - type: nauc_map_at_10_max |
| value: 66.51678197592413 |
| - type: nauc_map_at_10_std |
| value: 24.85733388904244 |
| - type: nauc_map_at_1_diff1 |
| value: 82.65982976105012 |
| - type: nauc_map_at_1_max |
| value: 66.0229036338153 |
| - type: nauc_map_at_1_std |
| value: 22.09004204696952 |
| - type: nauc_map_at_20_diff1 |
| value: 79.0706949814673 |
| - type: nauc_map_at_20_max |
| value: 66.41921898804029 |
| - type: nauc_map_at_20_std |
| value: 24.9286448686172 |
| - type: nauc_map_at_3_diff1 |
| value: 79.7198001441378 |
| - type: nauc_map_at_3_max |
| value: 67.00063808028989 |
| - type: nauc_map_at_3_std |
| value: 24.074213865142884 |
| - type: nauc_map_at_5_diff1 |
| value: 79.35098048907732 |
| - type: nauc_map_at_5_max |
| value: 66.80815275648563 |
| - type: nauc_map_at_5_std |
| value: 24.54538796165573 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 78.85646848963292 |
| - type: nauc_mrr_at_1000_max |
| value: 66.74961594120661 |
| - type: nauc_mrr_at_1000_std |
| value: 25.261834568256063 |
| - type: nauc_mrr_at_100_diff1 |
| value: 78.83397065160052 |
| - type: nauc_mrr_at_100_max |
| value: 66.74276613157386 |
| - type: nauc_mrr_at_100_std |
| value: 25.24809500309785 |
| - type: nauc_mrr_at_10_diff1 |
| value: 78.9530523067497 |
| - type: nauc_mrr_at_10_max |
| value: 66.85891502850205 |
| - type: nauc_mrr_at_10_std |
| value: 25.152891516847536 |
| - type: nauc_mrr_at_1_diff1 |
| value: 82.2465610017115 |
| - type: nauc_mrr_at_1_max |
| value: 66.66511371063773 |
| - type: nauc_mrr_at_1_std |
| value: 22.639906493776998 |
| - type: nauc_mrr_at_20_diff1 |
| value: 78.84558284790198 |
| - type: nauc_mrr_at_20_max |
| value: 66.76277766324108 |
| - type: nauc_mrr_at_20_std |
| value: 25.225303624662814 |
| - type: nauc_mrr_at_3_diff1 |
| value: 79.50156771547003 |
| - type: nauc_mrr_at_3_max |
| value: 67.33583901650987 |
| - type: nauc_mrr_at_3_std |
| value: 24.362469504761627 |
| - type: nauc_mrr_at_5_diff1 |
| value: 79.12943881636619 |
| - type: nauc_mrr_at_5_max |
| value: 67.14759555422152 |
| - type: nauc_mrr_at_5_std |
| value: 24.83799695076027 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 77.52019660815121 |
| - type: nauc_ndcg_at_1000_max |
| value: 65.83717552187926 |
| - type: nauc_ndcg_at_1000_std |
| value: 27.034582867678804 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 76.93747758970423 |
| - type: nauc_ndcg_at_100_max |
| value: 65.60810295420501 |
| - type: nauc_ndcg_at_100_std |
| value: 26.941487810863034 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 77.57948103065401 |
| - type: nauc_ndcg_at_10_max |
| value: 66.07222651913443 |
| - type: nauc_ndcg_at_10_std |
| value: 26.35911536261543 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 82.65982976105012 |
| - type: nauc_ndcg_at_1_max |
| value: 66.0229036338153 |
| - type: nauc_ndcg_at_1_std |
| value: 22.09004204696952 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 77.12409727019678 |
| - type: nauc_ndcg_at_20_max |
| value: 65.71984870176335 |
| - type: nauc_ndcg_at_20_std |
| value: 26.673365148606948 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 78.75978575557033 |
| - type: nauc_ndcg_at_3_max |
| value: 67.13135093269904 |
| - type: nauc_ndcg_at_3_std |
| value: 24.706967615687816 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 78.05104990867088 |
| - type: nauc_ndcg_at_5_max |
| value: 66.79111424562637 |
| - type: nauc_ndcg_at_5_std |
| value: 25.615575237732614 |
| - type: nauc_precision_at_1000_diff1 |
| value: 60.2983269810654 |
| - type: nauc_precision_at_1000_max |
| value: 58.90618542498941 |
| - type: nauc_precision_at_1000_std |
| value: 62.82775405244051 |
| - type: nauc_precision_at_100_diff1 |
| value: 64.11646723766715 |
| - type: nauc_precision_at_100_max |
| value: 60.282482009275654 |
| - type: nauc_precision_at_100_std |
| value: 39.473517969667135 |
| - type: nauc_precision_at_10_diff1 |
| value: 71.82001686121029 |
| - type: nauc_precision_at_10_max |
| value: 64.12657559785326 |
| - type: nauc_precision_at_10_std |
| value: 31.89716032543505 |
| - type: nauc_precision_at_1_diff1 |
| value: 82.65982976105012 |
| - type: nauc_precision_at_1_max |
| value: 66.0229036338153 |
| - type: nauc_precision_at_1_std |
| value: 22.09004204696952 |
| - type: nauc_precision_at_20_diff1 |
| value: 69.01813327818459 |
| - type: nauc_precision_at_20_max |
| value: 62.31511858543514 |
| - type: nauc_precision_at_20_std |
| value: 33.98133090177575 |
| - type: nauc_precision_at_3_diff1 |
| value: 75.85071053792088 |
| - type: nauc_precision_at_3_max |
| value: 67.4643531059972 |
| - type: nauc_precision_at_3_std |
| value: 26.61929747194295 |
| - type: nauc_precision_at_5_diff1 |
| value: 73.80236395769283 |
| - type: nauc_precision_at_5_max |
| value: 66.62363925820746 |
| - type: nauc_precision_at_5_std |
| value: 29.175770150771204 |
| - type: nauc_recall_at_1000_diff1 |
| value: 60.29832698106552 |
| - type: nauc_recall_at_1000_max |
| value: 58.9061854249895 |
| - type: nauc_recall_at_1000_std |
| value: 62.82775405244069 |
| - type: nauc_recall_at_100_diff1 |
| value: 64.11646723766702 |
| - type: nauc_recall_at_100_max |
| value: 60.282482009275654 |
| - type: nauc_recall_at_100_std |
| value: 39.47351796966711 |
| - type: nauc_recall_at_10_diff1 |
| value: 71.82001686121032 |
| - type: nauc_recall_at_10_max |
| value: 64.12657559785328 |
| - type: nauc_recall_at_10_std |
| value: 31.897160325435102 |
| - type: nauc_recall_at_1_diff1 |
| value: 82.65982976105012 |
| - type: nauc_recall_at_1_max |
| value: 66.0229036338153 |
| - type: nauc_recall_at_1_std |
| value: 22.09004204696952 |
| - type: nauc_recall_at_20_diff1 |
| value: 69.01813327818459 |
| - type: nauc_recall_at_20_max |
| value: 62.3151185854351 |
| - type: nauc_recall_at_20_std |
| value: 33.981330901775735 |
| - type: nauc_recall_at_3_diff1 |
| value: 75.85071053792085 |
| - type: nauc_recall_at_3_max |
| value: 67.4643531059972 |
| - type: nauc_recall_at_3_std |
| value: 26.619297471942975 |
| - type: nauc_recall_at_5_diff1 |
| value: 73.80236395769293 |
| - type: nauc_recall_at_5_max |
| value: 66.62363925820748 |
| - type: nauc_recall_at_5_std |
| value: 29.175770150771186 |
| - type: ndcg_at_1 |
| value: 52.5 |
| - type: ndcg_at_10 |
| value: 59.93000000000001 |
| - type: ndcg_at_100 |
| value: 62.697 |
| - type: ndcg_at_1000 |
| value: 64.28399999999999 |
| - type: ndcg_at_20 |
| value: 60.914 |
| - type: ndcg_at_3 |
| value: 57.336 |
| - type: ndcg_at_5 |
| value: 58.713 |
| - type: precision_at_1 |
| value: 52.5 |
| - type: precision_at_10 |
| value: 6.76 |
| - type: precision_at_100 |
| value: 0.8109999999999999 |
| - type: precision_at_1000 |
| value: 0.094 |
| - type: precision_at_20 |
| value: 3.5749999999999997 |
| - type: precision_at_3 |
| value: 20.200000000000003 |
| - type: precision_at_5 |
| value: 12.78 |
| - type: recall_at_1 |
| value: 52.5 |
| - type: recall_at_10 |
| value: 67.60000000000001 |
| - type: recall_at_100 |
| value: 81.10000000000001 |
| - type: recall_at_1000 |
| value: 93.89999999999999 |
| - type: recall_at_20 |
| value: 71.5 |
| - type: recall_at_3 |
| value: 60.6 |
| - type: recall_at_5 |
| value: 63.9 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB MedrxivClusteringP2P (default) |
| revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
| split: test |
| type: mteb/medrxiv-clustering-p2p |
| metrics: |
| - type: main_score |
| value: 34.731017731166105 |
| - type: v_measure |
| value: 34.731017731166105 |
| - type: v_measure_std |
| value: 1.5618103916501433 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB MedrxivClusteringS2S (default) |
| revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
| split: test |
| type: mteb/medrxiv-clustering-s2s |
| metrics: |
| - type: main_score |
| value: 31.545874099031675 |
| - type: v_measure |
| value: 31.545874099031675 |
| - type: v_measure_std |
| value: 1.4489482273302008 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB MindSmallReranking (default) |
| revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 |
| split: test |
| type: mteb/mind_small |
| metrics: |
| - type: main_score |
| value: 32.68299163296308 |
| - type: map |
| value: 32.68299163296308 |
| - type: mrr |
| value: 33.94301395316366 |
| - type: nAUC_map_diff1 |
| value: 12.502931744458973 |
| - type: nAUC_map_max |
| value: -21.63110475017275 |
| - type: nAUC_map_std |
| value: 0.6459544098312916 |
| - type: nAUC_mrr_diff1 |
| value: 11.816048638685693 |
| - type: nAUC_mrr_max |
| value: -15.973240530490395 |
| - type: nAUC_mrr_std |
| value: 1.9732078672552686 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB MultilingualSentiment (default) |
| revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a |
| split: test |
| type: C-MTEB/MultilingualSentiment-classification |
| metrics: |
| - type: accuracy |
| value: 72.95333333333333 |
| - type: f1 |
| value: 72.36132389042342 |
| - type: f1_weighted |
| value: 72.3613238904234 |
| - type: main_score |
| value: 72.95333333333333 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB MultilingualSentiment (default) |
| revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a |
| split: validation |
| type: C-MTEB/MultilingualSentiment-classification |
| metrics: |
| - type: accuracy |
| value: 71.95333333333333 |
| - type: f1 |
| value: 71.30311389484186 |
| - type: f1_weighted |
| value: 71.30311389484187 |
| - type: main_score |
| value: 71.95333333333333 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB NFCorpus (default) |
| revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 |
| split: test |
| type: mteb/nfcorpus |
| metrics: |
| - type: main_score |
| value: 37.749 |
| - type: map_at_1 |
| value: 6.4159999999999995 |
| - type: map_at_10 |
| value: 14.491000000000001 |
| - type: map_at_100 |
| value: 18.33 |
| - type: map_at_1000 |
| value: 19.953000000000003 |
| - type: map_at_20 |
| value: 15.973 |
| - type: map_at_3 |
| value: 10.417 |
| - type: map_at_5 |
| value: 12.303 |
| - type: mrr_at_1 |
| value: 48.91640866873065 |
| - type: mrr_at_10 |
| value: 57.561673792323944 |
| - type: mrr_at_100 |
| value: 58.053397955445895 |
| - type: mrr_at_1000 |
| value: 58.08848482119531 |
| - type: mrr_at_20 |
| value: 57.871279571741574 |
| - type: mrr_at_3 |
| value: 55.26315789473685 |
| - type: mrr_at_5 |
| value: 56.671826625387 |
| - type: nauc_map_at_1000_diff1 |
| value: 32.624665660376444 |
| - type: nauc_map_at_1000_max |
| value: 28.128385961803097 |
| - type: nauc_map_at_1000_std |
| value: 11.87883353166736 |
| - type: nauc_map_at_100_diff1 |
| value: 34.57600972779998 |
| - type: nauc_map_at_100_max |
| value: 27.32046439964767 |
| - type: nauc_map_at_100_std |
| value: 8.254050946905554 |
| - type: nauc_map_at_10_diff1 |
| value: 38.352776312240316 |
| - type: nauc_map_at_10_max |
| value: 20.71383460865022 |
| - type: nauc_map_at_10_std |
| value: -3.6182278698175008 |
| - type: nauc_map_at_1_diff1 |
| value: 51.364207086510284 |
| - type: nauc_map_at_1_max |
| value: 8.650050628738809 |
| - type: nauc_map_at_1_std |
| value: -17.30631242512481 |
| - type: nauc_map_at_20_diff1 |
| value: 36.6488106109285 |
| - type: nauc_map_at_20_max |
| value: 23.646170774047608 |
| - type: nauc_map_at_20_std |
| value: 0.9988156075379873 |
| - type: nauc_map_at_3_diff1 |
| value: 42.77938198571511 |
| - type: nauc_map_at_3_max |
| value: 13.04870823631715 |
| - type: nauc_map_at_3_std |
| value: -12.642790311189103 |
| - type: nauc_map_at_5_diff1 |
| value: 40.14997508237488 |
| - type: nauc_map_at_5_max |
| value: 16.841994515634102 |
| - type: nauc_map_at_5_std |
| value: -8.827177258110211 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 33.19158658938379 |
| - type: nauc_mrr_at_1000_max |
| value: 50.78735289388637 |
| - type: nauc_mrr_at_1000_std |
| value: 29.483390453949475 |
| - type: nauc_mrr_at_100_diff1 |
| value: 33.181857073965006 |
| - type: nauc_mrr_at_100_max |
| value: 50.82192854045011 |
| - type: nauc_mrr_at_100_std |
| value: 29.530433535813316 |
| - type: nauc_mrr_at_10_diff1 |
| value: 33.13575237370853 |
| - type: nauc_mrr_at_10_max |
| value: 50.840928702265245 |
| - type: nauc_mrr_at_10_std |
| value: 29.393982460321617 |
| - type: nauc_mrr_at_1_diff1 |
| value: 35.38635024440146 |
| - type: nauc_mrr_at_1_max |
| value: 45.58280413169544 |
| - type: nauc_mrr_at_1_std |
| value: 20.118650543521753 |
| - type: nauc_mrr_at_20_diff1 |
| value: 33.32009076370569 |
| - type: nauc_mrr_at_20_max |
| value: 50.67114851216221 |
| - type: nauc_mrr_at_20_std |
| value: 29.421770858743024 |
| - type: nauc_mrr_at_3_diff1 |
| value: 33.17533789218473 |
| - type: nauc_mrr_at_3_max |
| value: 50.00421069382273 |
| - type: nauc_mrr_at_3_std |
| value: 28.784501459911233 |
| - type: nauc_mrr_at_5_diff1 |
| value: 32.66135736896744 |
| - type: nauc_mrr_at_5_max |
| value: 50.401707427923505 |
| - type: nauc_mrr_at_5_std |
| value: 29.357909892487232 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 29.02160255181641 |
| - type: nauc_ndcg_at_1000_max |
| value: 44.98065565601714 |
| - type: nauc_ndcg_at_1000_std |
| value: 31.652110733336887 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 28.851190536083593 |
| - type: nauc_ndcg_at_100_max |
| value: 39.26997767014831 |
| - type: nauc_ndcg_at_100_std |
| value: 25.574099100530827 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 24.733756826812474 |
| - type: nauc_ndcg_at_10_max |
| value: 39.51573298713868 |
| - type: nauc_ndcg_at_10_std |
| value: 26.10826723752759 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 35.86747711483557 |
| - type: nauc_ndcg_at_1_max |
| value: 43.60593203885657 |
| - type: nauc_ndcg_at_1_std |
| value: 16.90139427357944 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 24.701717110335373 |
| - type: nauc_ndcg_at_20_max |
| value: 37.56137361178106 |
| - type: nauc_ndcg_at_20_std |
| value: 25.65000140011744 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 27.58703963162813 |
| - type: nauc_ndcg_at_3_max |
| value: 42.377949191047975 |
| - type: nauc_ndcg_at_3_std |
| value: 22.006636261926808 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 25.323540164365394 |
| - type: nauc_ndcg_at_5_max |
| value: 42.077483541800355 |
| - type: nauc_ndcg_at_5_std |
| value: 24.38614012402223 |
| - type: nauc_precision_at_1000_diff1 |
| value: -18.554231105026798 |
| - type: nauc_precision_at_1000_max |
| value: 8.600104573044353 |
| - type: nauc_precision_at_1000_std |
| value: 35.24043924606992 |
| - type: nauc_precision_at_100_diff1 |
| value: -12.366332039473939 |
| - type: nauc_precision_at_100_max |
| value: 21.684056644697822 |
| - type: nauc_precision_at_100_std |
| value: 44.19851905373012 |
| - type: nauc_precision_at_10_diff1 |
| value: 4.981890145850079 |
| - type: nauc_precision_at_10_max |
| value: 39.26695926876921 |
| - type: nauc_precision_at_10_std |
| value: 39.6193781427142 |
| - type: nauc_precision_at_1_diff1 |
| value: 36.226882155693254 |
| - type: nauc_precision_at_1_max |
| value: 45.64116702800358 |
| - type: nauc_precision_at_1_std |
| value: 18.56622209173858 |
| - type: nauc_precision_at_20_diff1 |
| value: -1.3537073154842467 |
| - type: nauc_precision_at_20_max |
| value: 34.211750289968315 |
| - type: nauc_precision_at_20_std |
| value: 42.88840705138113 |
| - type: nauc_precision_at_3_diff1 |
| value: 16.785704006680017 |
| - type: nauc_precision_at_3_max |
| value: 43.41048951027902 |
| - type: nauc_precision_at_3_std |
| value: 29.20950983049612 |
| - type: nauc_precision_at_5_diff1 |
| value: 10.141401144736987 |
| - type: nauc_precision_at_5_max |
| value: 42.61295259785708 |
| - type: nauc_precision_at_5_std |
| value: 34.36808976552582 |
| - type: nauc_recall_at_1000_diff1 |
| value: 13.070490227131154 |
| - type: nauc_recall_at_1000_max |
| value: 23.16211600933428 |
| - type: nauc_recall_at_1000_std |
| value: 20.18400228183049 |
| - type: nauc_recall_at_100_diff1 |
| value: 21.791873990847225 |
| - type: nauc_recall_at_100_max |
| value: 24.534035934410444 |
| - type: nauc_recall_at_100_std |
| value: 15.02352427792638 |
| - type: nauc_recall_at_10_diff1 |
| value: 31.695078384281018 |
| - type: nauc_recall_at_10_max |
| value: 17.87955239768676 |
| - type: nauc_recall_at_10_std |
| value: -1.8766363765059346 |
| - type: nauc_recall_at_1_diff1 |
| value: 51.364207086510284 |
| - type: nauc_recall_at_1_max |
| value: 8.650050628738809 |
| - type: nauc_recall_at_1_std |
| value: -17.30631242512481 |
| - type: nauc_recall_at_20_diff1 |
| value: 27.518789645287413 |
| - type: nauc_recall_at_20_max |
| value: 19.248306687993665 |
| - type: nauc_recall_at_20_std |
| value: 1.8973437807943836 |
| - type: nauc_recall_at_3_diff1 |
| value: 40.176896668779975 |
| - type: nauc_recall_at_3_max |
| value: 12.609773638086294 |
| - type: nauc_recall_at_3_std |
| value: -11.078650386618978 |
| - type: nauc_recall_at_5_diff1 |
| value: 34.52328172005921 |
| - type: nauc_recall_at_5_max |
| value: 15.927267077298449 |
| - type: nauc_recall_at_5_std |
| value: -6.882800988990083 |
| - type: ndcg_at_1 |
| value: 47.214 |
| - type: ndcg_at_10 |
| value: 37.749 |
| - type: ndcg_at_100 |
| value: 34.941 |
| - type: ndcg_at_1000 |
| value: 43.763000000000005 |
| - type: ndcg_at_20 |
| value: 35.096 |
| - type: ndcg_at_3 |
| value: 42.778 |
| - type: ndcg_at_5 |
| value: 40.916999999999994 |
| - type: precision_at_1 |
| value: 48.607 |
| - type: precision_at_10 |
| value: 27.771 |
| - type: precision_at_100 |
| value: 8.873000000000001 |
| - type: precision_at_1000 |
| value: 2.205 |
| - type: precision_at_20 |
| value: 20.294 |
| - type: precision_at_3 |
| value: 39.732 |
| - type: precision_at_5 |
| value: 35.294 |
| - type: recall_at_1 |
| value: 6.4159999999999995 |
| - type: recall_at_10 |
| value: 18.912000000000003 |
| - type: recall_at_100 |
| value: 35.716 |
| - type: recall_at_1000 |
| value: 67.38199999999999 |
| - type: recall_at_20 |
| value: 22.902 |
| - type: recall_at_3 |
| value: 11.331 |
| - type: recall_at_5 |
| value: 14.488000000000001 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB NQ (default) |
| revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 |
| split: test |
| type: mteb/nq |
| metrics: |
| - type: main_score |
| value: 64.57300000000001 |
| - type: map_at_1 |
| value: 40.624 |
| - type: map_at_10 |
| value: 57.27100000000001 |
| - type: map_at_100 |
| value: 58.025000000000006 |
| - type: map_at_1000 |
| value: 58.042 |
| - type: map_at_20 |
| value: 57.797 |
| - type: map_at_3 |
| value: 53.198 |
| - type: map_at_5 |
| value: 55.894999999999996 |
| - type: mrr_at_1 |
| value: 45.48088064889919 |
| - type: mrr_at_10 |
| value: 59.63950045062427 |
| - type: mrr_at_100 |
| value: 60.16941078517045 |
| - type: mrr_at_1000 |
| value: 60.180798143896816 |
| - type: mrr_at_20 |
| value: 60.028840407404324 |
| - type: mrr_at_3 |
| value: 56.542101197373306 |
| - type: mrr_at_5 |
| value: 58.66550791811501 |
| - type: nauc_map_at_1000_diff1 |
| value: 40.75719420870384 |
| - type: nauc_map_at_1000_max |
| value: 26.619453317408276 |
| - type: nauc_map_at_1000_std |
| value: -1.3661680695508878 |
| - type: nauc_map_at_100_diff1 |
| value: 40.74618842719167 |
| - type: nauc_map_at_100_max |
| value: 26.63090879971882 |
| - type: nauc_map_at_100_std |
| value: -1.3472110757216529 |
| - type: nauc_map_at_10_diff1 |
| value: 40.567377605059974 |
| - type: nauc_map_at_10_max |
| value: 26.643510588231212 |
| - type: nauc_map_at_10_std |
| value: -1.5369955980308176 |
| - type: nauc_map_at_1_diff1 |
| value: 44.266492092577074 |
| - type: nauc_map_at_1_max |
| value: 22.46429746791908 |
| - type: nauc_map_at_1_std |
| value: -4.60330121276408 |
| - type: nauc_map_at_20_diff1 |
| value: 40.723395327361416 |
| - type: nauc_map_at_20_max |
| value: 26.66921091235507 |
| - type: nauc_map_at_20_std |
| value: -1.3548217510569764 |
| - type: nauc_map_at_3_diff1 |
| value: 40.70669851540585 |
| - type: nauc_map_at_3_max |
| value: 24.92583855784867 |
| - type: nauc_map_at_3_std |
| value: -3.3252674375061337 |
| - type: nauc_map_at_5_diff1 |
| value: 40.71452737114635 |
| - type: nauc_map_at_5_max |
| value: 26.334549142409934 |
| - type: nauc_map_at_5_std |
| value: -2.2994574589513563 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 40.90181907233357 |
| - type: nauc_mrr_at_1000_max |
| value: 27.23972709490355 |
| - type: nauc_mrr_at_1000_std |
| value: 0.14293203689890255 |
| - type: nauc_mrr_at_100_diff1 |
| value: 40.894823740399 |
| - type: nauc_mrr_at_100_max |
| value: 27.2543152925248 |
| - type: nauc_mrr_at_100_std |
| value: 0.16145332462727016 |
| - type: nauc_mrr_at_10_diff1 |
| value: 40.70236566768764 |
| - type: nauc_mrr_at_10_max |
| value: 27.376281421880815 |
| - type: nauc_mrr_at_10_std |
| value: 0.2560670735926158 |
| - type: nauc_mrr_at_1_diff1 |
| value: 44.45758887813465 |
| - type: nauc_mrr_at_1_max |
| value: 24.93321743794642 |
| - type: nauc_mrr_at_1_std |
| value: -1.7773297601048152 |
| - type: nauc_mrr_at_20_diff1 |
| value: 40.880239024353024 |
| - type: nauc_mrr_at_20_max |
| value: 27.32513012463313 |
| - type: nauc_mrr_at_20_std |
| value: 0.22430042378671203 |
| - type: nauc_mrr_at_3_diff1 |
| value: 40.57781683564487 |
| - type: nauc_mrr_at_3_max |
| value: 26.35515394643849 |
| - type: nauc_mrr_at_3_std |
| value: -0.8399275552803087 |
| - type: nauc_mrr_at_5_diff1 |
| value: 40.58618030692413 |
| - type: nauc_mrr_at_5_max |
| value: 27.176058971300332 |
| - type: nauc_mrr_at_5_std |
| value: -0.290492953635725 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 40.19287643985469 |
| - type: nauc_ndcg_at_1000_max |
| value: 28.0748177881881 |
| - type: nauc_ndcg_at_1000_std |
| value: 0.5479989880947034 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 39.985152217871736 |
| - type: nauc_ndcg_at_100_max |
| value: 28.549199334840196 |
| - type: nauc_ndcg_at_100_std |
| value: 1.1976149852470748 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 39.258653400976094 |
| - type: nauc_ndcg_at_10_max |
| value: 28.78995376227268 |
| - type: nauc_ndcg_at_10_std |
| value: 0.7776396395610561 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 44.53452792113141 |
| - type: nauc_ndcg_at_1_max |
| value: 24.959865554376062 |
| - type: nauc_ndcg_at_1_std |
| value: -1.7064088232497185 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 39.86087719429661 |
| - type: nauc_ndcg_at_20_max |
| value: 28.83245623742156 |
| - type: nauc_ndcg_at_20_std |
| value: 1.214377441117911 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 39.559374584504305 |
| - type: nauc_ndcg_at_3_max |
| value: 25.79384722635462 |
| - type: nauc_ndcg_at_3_std |
| value: -2.6232036598581128 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 39.4735486654252 |
| - type: nauc_ndcg_at_5_max |
| value: 28.016157443317592 |
| - type: nauc_ndcg_at_5_std |
| value: -1.1037353009714006 |
| - type: nauc_precision_at_1000_diff1 |
| value: -11.339155238778668 |
| - type: nauc_precision_at_1000_max |
| value: 6.254052000341758 |
| - type: nauc_precision_at_1000_std |
| value: 15.950222098594327 |
| - type: nauc_precision_at_100_diff1 |
| value: -8.486482545268508 |
| - type: nauc_precision_at_100_max |
| value: 11.576264017770066 |
| - type: nauc_precision_at_100_std |
| value: 19.466754760840587 |
| - type: nauc_precision_at_10_diff1 |
| value: 5.8385831139291575 |
| - type: nauc_precision_at_10_max |
| value: 22.06247370356374 |
| - type: nauc_precision_at_10_std |
| value: 15.537018617227929 |
| - type: nauc_precision_at_1_diff1 |
| value: 44.53452792113141 |
| - type: nauc_precision_at_1_max |
| value: 24.959865554376062 |
| - type: nauc_precision_at_1_std |
| value: -1.7064088232497185 |
| - type: nauc_precision_at_20_diff1 |
| value: 0.46004341386248976 |
| - type: nauc_precision_at_20_max |
| value: 18.62298396479716 |
| - type: nauc_precision_at_20_std |
| value: 18.57322798830194 |
| - type: nauc_precision_at_3_diff1 |
| value: 23.61278777236713 |
| - type: nauc_precision_at_3_max |
| value: 24.430940659086023 |
| - type: nauc_precision_at_3_std |
| value: 3.654630451987505 |
| - type: nauc_precision_at_5_diff1 |
| value: 15.044899921606575 |
| - type: nauc_precision_at_5_max |
| value: 24.832870409550832 |
| - type: nauc_precision_at_5_std |
| value: 9.157167085230308 |
| - type: nauc_recall_at_1000_diff1 |
| value: 20.011280923509826 |
| - type: nauc_recall_at_1000_max |
| value: 77.0124542065231 |
| - type: nauc_recall_at_1000_std |
| value: 77.168446634178 |
| - type: nauc_recall_at_100_diff1 |
| value: 26.20020816542189 |
| - type: nauc_recall_at_100_max |
| value: 63.51438156454956 |
| - type: nauc_recall_at_100_std |
| value: 50.6515798452802 |
| - type: nauc_recall_at_10_diff1 |
| value: 30.672533975609245 |
| - type: nauc_recall_at_10_max |
| value: 38.04655658762951 |
| - type: nauc_recall_at_10_std |
| value: 10.401521020182201 |
| - type: nauc_recall_at_1_diff1 |
| value: 44.266492092577074 |
| - type: nauc_recall_at_1_max |
| value: 22.46429746791908 |
| - type: nauc_recall_at_1_std |
| value: -4.60330121276408 |
| - type: nauc_recall_at_20_diff1 |
| value: 32.410730671544556 |
| - type: nauc_recall_at_20_max |
| value: 43.56842328558742 |
| - type: nauc_recall_at_20_std |
| value: 18.786877985653163 |
| - type: nauc_recall_at_3_diff1 |
| value: 34.930964358124406 |
| - type: nauc_recall_at_3_max |
| value: 26.12903272130525 |
| - type: nauc_recall_at_3_std |
| value: -2.985516316701988 |
| - type: nauc_recall_at_5_diff1 |
| value: 33.47824880356667 |
| - type: nauc_recall_at_5_max |
| value: 32.449042774855855 |
| - type: nauc_recall_at_5_std |
| value: 0.6573399404508043 |
| - type: ndcg_at_1 |
| value: 45.452 |
| - type: ndcg_at_10 |
| value: 64.57300000000001 |
| - type: ndcg_at_100 |
| value: 67.56400000000001 |
| - type: ndcg_at_1000 |
| value: 67.927 |
| - type: ndcg_at_20 |
| value: 66.247 |
| - type: ndcg_at_3 |
| value: 57.32899999999999 |
| - type: ndcg_at_5 |
| value: 61.693 |
| - type: precision_at_1 |
| value: 45.452 |
| - type: precision_at_10 |
| value: 10.067 |
| - type: precision_at_100 |
| value: 1.176 |
| - type: precision_at_1000 |
| value: 0.121 |
| - type: precision_at_20 |
| value: 5.436 |
| - type: precision_at_3 |
| value: 25.628 |
| - type: precision_at_5 |
| value: 17.965999999999998 |
| - type: recall_at_1 |
| value: 40.624 |
| - type: recall_at_10 |
| value: 84.096 |
| - type: recall_at_100 |
| value: 96.734 |
| - type: recall_at_1000 |
| value: 99.401 |
| - type: recall_at_20 |
| value: 90.276 |
| - type: recall_at_3 |
| value: 65.892 |
| - type: recall_at_5 |
| value: 75.847 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB Ocnli (default) |
| revision: 66e76a618a34d6d565d5538088562851e6daa7ec |
| split: validation |
| type: C-MTEB/OCNLI |
| metrics: |
| - type: cosine_accuracy |
| value: 57.715213860314016 |
| - type: cosine_accuracy_threshold |
| value: 70.3215479850769 |
| - type: cosine_ap |
| value: 58.8326699963807 |
| - type: cosine_f1 |
| value: 67.816091954023 |
| - type: cosine_f1_threshold |
| value: 42.0912504196167 |
| - type: cosine_precision |
| value: 51.38813282525857 |
| - type: cosine_recall |
| value: 99.68321013727561 |
| - type: dot_accuracy |
| value: 57.715213860314016 |
| - type: dot_accuracy_threshold |
| value: 70.3215479850769 |
| - type: dot_ap |
| value: 58.8326699963807 |
| - type: dot_f1 |
| value: 67.816091954023 |
| - type: dot_f1_threshold |
| value: 42.0912504196167 |
| - type: dot_precision |
| value: 51.38813282525857 |
| - type: dot_recall |
| value: 99.68321013727561 |
| - type: euclidean_accuracy |
| value: 57.715213860314016 |
| - type: euclidean_accuracy_threshold |
| value: 77.04342603683472 |
| - type: euclidean_ap |
| value: 58.8326699963807 |
| - type: euclidean_f1 |
| value: 67.816091954023 |
| - type: euclidean_f1_threshold |
| value: 107.61815309524536 |
| - type: euclidean_precision |
| value: 51.38813282525857 |
| - type: euclidean_recall |
| value: 99.68321013727561 |
| - type: main_score |
| value: 57.877639415268 |
| - type: manhattan_accuracy |
| value: 57.877639415268 |
| - type: manhattan_accuracy_threshold |
| value: 1952.0273208618164 |
| - type: manhattan_ap |
| value: 58.70263102269272 |
| - type: manhattan_f1 |
| value: 67.84172661870504 |
| - type: manhattan_f1_threshold |
| value: 2661.929702758789 |
| - type: manhattan_precision |
| value: 51.44571740316422 |
| - type: manhattan_recall |
| value: 99.57761351636748 |
| - type: max_accuracy |
| value: 57.877639415268 |
| - type: max_ap |
| value: 58.8326699963807 |
| - type: max_f1 |
| value: 67.84172661870504 |
| - type: max_precision |
| value: 51.44571740316422 |
| - type: max_recall |
| value: 99.68321013727561 |
| - type: similarity_accuracy |
| value: 57.715213860314016 |
| - type: similarity_accuracy_threshold |
| value: 70.3215479850769 |
| - type: similarity_ap |
| value: 58.8326699963807 |
| - type: similarity_f1 |
| value: 67.816091954023 |
| - type: similarity_f1_threshold |
| value: 42.0912504196167 |
| - type: similarity_precision |
| value: 51.38813282525857 |
| - type: similarity_recall |
| value: 99.68321013727561 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB OnlineShopping (default) |
| revision: e610f2ebd179a8fda30ae534c3878750a96db120 |
| split: test |
| type: C-MTEB/OnlineShopping-classification |
| metrics: |
| - type: accuracy |
| value: 92.24 |
| - type: ap |
| value: 91.00427236651306 |
| - type: ap_weighted |
| value: 91.00427236651306 |
| - type: f1 |
| value: 92.23673939008314 |
| - type: f1_weighted |
| value: 92.24091264330853 |
| - type: main_score |
| value: 92.24 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB PAWSX (default) |
| revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 |
| split: test |
| type: C-MTEB/PAWSX |
| metrics: |
| - type: cosine_pearson |
| value: 13.62339858857132 |
| - type: cosine_spearman |
| value: 14.886062653753804 |
| - type: euclidean_pearson |
| value: 16.711204002219453 |
| - type: euclidean_spearman |
| value: 14.8864092068256 |
| - type: main_score |
| value: 14.886062653753804 |
| - type: manhattan_pearson |
| value: 16.658236019215405 |
| - type: manhattan_spearman |
| value: 14.868816375702131 |
| - type: pearson |
| value: 13.62339858857132 |
| - type: spearman |
| value: 14.886062653753804 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB QBQTC (default) |
| revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 |
| split: test |
| type: C-MTEB/QBQTC |
| metrics: |
| - type: cosine_pearson |
| value: 37.88649935702812 |
| - type: cosine_spearman |
| value: 38.59331260288544 |
| - type: euclidean_pearson |
| value: 37.492219553708004 |
| - type: euclidean_spearman |
| value: 38.59333707659388 |
| - type: main_score |
| value: 38.59331260288544 |
| - type: manhattan_pearson |
| value: 37.59659518440478 |
| - type: manhattan_spearman |
| value: 38.70529977801903 |
| - type: pearson |
| value: 37.88649935702812 |
| - type: spearman |
| value: 38.59331260288544 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB QuoraRetrieval (default) |
| revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 |
| split: test |
| type: mteb/quora |
| metrics: |
| - type: main_score |
| value: 88.828 |
| - type: map_at_1 |
| value: 70.86200000000001 |
| - type: map_at_10 |
| value: 85.146 |
| - type: map_at_100 |
| value: 85.76599999999999 |
| - type: map_at_1000 |
| value: 85.78 |
| - type: map_at_20 |
| value: 85.56200000000001 |
| - type: map_at_3 |
| value: 82.15299999999999 |
| - type: map_at_5 |
| value: 84.06400000000001 |
| - type: mrr_at_1 |
| value: 81.55 |
| - type: mrr_at_10 |
| value: 87.83961904761877 |
| - type: mrr_at_100 |
| value: 87.9277183426686 |
| - type: mrr_at_1000 |
| value: 87.92847303870963 |
| - type: mrr_at_20 |
| value: 87.90789174563797 |
| - type: mrr_at_3 |
| value: 86.88999999999969 |
| - type: mrr_at_5 |
| value: 87.54949999999965 |
| - type: nauc_map_at_1000_diff1 |
| value: 75.67356164383055 |
| - type: nauc_map_at_1000_max |
| value: 18.854168889852875 |
| - type: nauc_map_at_1000_std |
| value: -36.828725949066445 |
| - type: nauc_map_at_100_diff1 |
| value: 75.6814052182551 |
| - type: nauc_map_at_100_max |
| value: 18.80911976095899 |
| - type: nauc_map_at_100_std |
| value: -36.859464706791876 |
| - type: nauc_map_at_10_diff1 |
| value: 75.97345767589778 |
| - type: nauc_map_at_10_max |
| value: 18.011922065772556 |
| - type: nauc_map_at_10_std |
| value: -38.58646259978914 |
| - type: nauc_map_at_1_diff1 |
| value: 79.64088789104214 |
| - type: nauc_map_at_1_max |
| value: 13.492832118158285 |
| - type: nauc_map_at_1_std |
| value: -35.069831511434984 |
| - type: nauc_map_at_20_diff1 |
| value: 75.80068897926556 |
| - type: nauc_map_at_20_max |
| value: 18.45100586838551 |
| - type: nauc_map_at_20_std |
| value: -37.58585203895838 |
| - type: nauc_map_at_3_diff1 |
| value: 76.57306543713516 |
| - type: nauc_map_at_3_max |
| value: 16.00971160952194 |
| - type: nauc_map_at_3_std |
| value: -40.49530239296166 |
| - type: nauc_map_at_5_diff1 |
| value: 76.30622902785689 |
| - type: nauc_map_at_5_max |
| value: 16.939904072532965 |
| - type: nauc_map_at_5_std |
| value: -40.231106124451166 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 75.55280562909647 |
| - type: nauc_mrr_at_1000_max |
| value: 21.290797318017223 |
| - type: nauc_mrr_at_1000_std |
| value: -33.16189158862257 |
| - type: nauc_mrr_at_100_diff1 |
| value: 75.55304375022514 |
| - type: nauc_mrr_at_100_max |
| value: 21.29147005221132 |
| - type: nauc_mrr_at_100_std |
| value: -33.16308467728784 |
| - type: nauc_mrr_at_10_diff1 |
| value: 75.52812757986803 |
| - type: nauc_mrr_at_10_max |
| value: 21.324213275115707 |
| - type: nauc_mrr_at_10_std |
| value: -33.19426301054038 |
| - type: nauc_mrr_at_1_diff1 |
| value: 76.3028645714178 |
| - type: nauc_mrr_at_1_max |
| value: 21.43266251995086 |
| - type: nauc_mrr_at_1_std |
| value: -31.52048640923299 |
| - type: nauc_mrr_at_20_diff1 |
| value: 75.55919114256191 |
| - type: nauc_mrr_at_20_max |
| value: 21.295549776313212 |
| - type: nauc_mrr_at_20_std |
| value: -33.17019625161653 |
| - type: nauc_mrr_at_3_diff1 |
| value: 75.22527177974185 |
| - type: nauc_mrr_at_3_max |
| value: 20.97300902425478 |
| - type: nauc_mrr_at_3_std |
| value: -33.77499787473604 |
| - type: nauc_mrr_at_5_diff1 |
| value: 75.48235066264493 |
| - type: nauc_mrr_at_5_max |
| value: 21.267657410423936 |
| - type: nauc_mrr_at_5_std |
| value: -33.541730991271024 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 75.2765114539078 |
| - type: nauc_ndcg_at_1000_max |
| value: 20.20040796867573 |
| - type: nauc_ndcg_at_1000_std |
| value: -34.83796103543814 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 75.29839170813271 |
| - type: nauc_ndcg_at_100_max |
| value: 20.049897014114226 |
| - type: nauc_ndcg_at_100_std |
| value: -34.88744910962141 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 75.50031549242551 |
| - type: nauc_ndcg_at_10_max |
| value: 19.530680246918088 |
| - type: nauc_ndcg_at_10_std |
| value: -37.2132581016259 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 76.2640011251013 |
| - type: nauc_ndcg_at_1_max |
| value: 21.5044133820231 |
| - type: nauc_ndcg_at_1_std |
| value: -31.441810766378154 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 75.48825777295262 |
| - type: nauc_ndcg_at_20_max |
| value: 19.54316358183612 |
| - type: nauc_ndcg_at_20_std |
| value: -36.40213923640784 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 74.95053482032748 |
| - type: nauc_ndcg_at_3_max |
| value: 18.3619895361753 |
| - type: nauc_ndcg_at_3_std |
| value: -37.89625025926027 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 75.4723557202212 |
| - type: nauc_ndcg_at_5_max |
| value: 18.774895447817002 |
| - type: nauc_ndcg_at_5_std |
| value: -38.63359887929082 |
| - type: nauc_precision_at_1000_diff1 |
| value: -44.66021797376074 |
| - type: nauc_precision_at_1000_max |
| value: 4.986714020703288 |
| - type: nauc_precision_at_1000_std |
| value: 33.87174403678706 |
| - type: nauc_precision_at_100_diff1 |
| value: -44.37083805843052 |
| - type: nauc_precision_at_100_max |
| value: 4.205704372136468 |
| - type: nauc_precision_at_100_std |
| value: 33.751385069291466 |
| - type: nauc_precision_at_10_diff1 |
| value: -39.594958618164924 |
| - type: nauc_precision_at_10_max |
| value: 3.3213764067887017 |
| - type: nauc_precision_at_10_std |
| value: 24.514139443922584 |
| - type: nauc_precision_at_1_diff1 |
| value: 76.2640011251013 |
| - type: nauc_precision_at_1_max |
| value: 21.5044133820231 |
| - type: nauc_precision_at_1_std |
| value: -31.441810766378154 |
| - type: nauc_precision_at_20_diff1 |
| value: -42.515141197525665 |
| - type: nauc_precision_at_20_max |
| value: 3.270570048173852 |
| - type: nauc_precision_at_20_std |
| value: 29.464821564987304 |
| - type: nauc_precision_at_3_diff1 |
| value: -20.183805197256348 |
| - type: nauc_precision_at_3_max |
| value: 5.587068650916888 |
| - type: nauc_precision_at_3_std |
| value: 6.698987832594483 |
| - type: nauc_precision_at_5_diff1 |
| value: -32.29844211123163 |
| - type: nauc_precision_at_5_max |
| value: 3.678521295215363 |
| - type: nauc_precision_at_5_std |
| value: 15.83463108178084 |
| - type: nauc_recall_at_1000_diff1 |
| value: 57.09220215387219 |
| - type: nauc_recall_at_1000_max |
| value: 55.92562330520271 |
| - type: nauc_recall_at_1000_std |
| value: 55.55766930325453 |
| - type: nauc_recall_at_100_diff1 |
| value: 70.73870980404703 |
| - type: nauc_recall_at_100_max |
| value: 19.254114660154237 |
| - type: nauc_recall_at_100_std |
| value: -23.244583670383882 |
| - type: nauc_recall_at_10_diff1 |
| value: 72.37078223746651 |
| - type: nauc_recall_at_10_max |
| value: 14.81844588849609 |
| - type: nauc_recall_at_10_std |
| value: -49.52099798289318 |
| - type: nauc_recall_at_1_diff1 |
| value: 79.64088789104214 |
| - type: nauc_recall_at_1_max |
| value: 13.492832118158285 |
| - type: nauc_recall_at_1_std |
| value: -35.069831511434984 |
| - type: nauc_recall_at_20_diff1 |
| value: 73.92034634556907 |
| - type: nauc_recall_at_20_max |
| value: 12.42942772030806 |
| - type: nauc_recall_at_20_std |
| value: -49.565011337521874 |
| - type: nauc_recall_at_3_diff1 |
| value: 72.89507863914096 |
| - type: nauc_recall_at_3_max |
| value: 12.262751224508929 |
| - type: nauc_recall_at_3_std |
| value: -46.57376666697539 |
| - type: nauc_recall_at_5_diff1 |
| value: 72.22900202817202 |
| - type: nauc_recall_at_5_max |
| value: 12.163990495227434 |
| - type: nauc_recall_at_5_std |
| value: -51.071117062577656 |
| - type: ndcg_at_1 |
| value: 81.57 |
| - type: ndcg_at_10 |
| value: 88.828 |
| - type: ndcg_at_100 |
| value: 89.936 |
| - type: ndcg_at_1000 |
| value: 90.01599999999999 |
| - type: ndcg_at_20 |
| value: 89.459 |
| - type: ndcg_at_3 |
| value: 86.014 |
| - type: ndcg_at_5 |
| value: 87.619 |
| - type: precision_at_1 |
| value: 81.57 |
| - type: precision_at_10 |
| value: 13.535 |
| - type: precision_at_100 |
| value: 1.537 |
| - type: precision_at_1000 |
| value: 0.157 |
| - type: precision_at_20 |
| value: 7.1739999999999995 |
| - type: precision_at_3 |
| value: 37.763000000000005 |
| - type: precision_at_5 |
| value: 24.878 |
| - type: recall_at_1 |
| value: 70.86200000000001 |
| - type: recall_at_10 |
| value: 95.977 |
| - type: recall_at_100 |
| value: 99.644 |
| - type: recall_at_1000 |
| value: 99.984 |
| - type: recall_at_20 |
| value: 97.98 |
| - type: recall_at_3 |
| value: 87.878 |
| - type: recall_at_5 |
| value: 92.419 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB RedditClustering (default) |
| revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
| split: test |
| type: mteb/reddit-clustering |
| metrics: |
| - type: main_score |
| value: 53.22151362416875 |
| - type: v_measure |
| value: 53.22151362416875 |
| - type: v_measure_std |
| value: 4.187568171093669 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB RedditClusteringP2P (default) |
| revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 |
| split: test |
| type: mteb/reddit-clustering-p2p |
| metrics: |
| - type: main_score |
| value: 58.38112463653693 |
| - type: v_measure |
| value: 58.38112463653693 |
| - type: v_measure_std |
| value: 12.221880566455676 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB SCIDOCS (default) |
| revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 |
| split: test |
| type: mteb/scidocs |
| metrics: |
| - type: main_score |
| value: 21.714 |
| - type: map_at_1 |
| value: 5.208 |
| - type: map_at_10 |
| value: 13.075999999999999 |
| - type: map_at_100 |
| value: 15.334 |
| - type: map_at_1000 |
| value: 15.671 |
| - type: map_at_20 |
| value: 14.276 |
| - type: map_at_3 |
| value: 9.289 |
| - type: map_at_5 |
| value: 11.068 |
| - type: mrr_at_1 |
| value: 25.7 |
| - type: mrr_at_10 |
| value: 36.53083333333331 |
| - type: mrr_at_100 |
| value: 37.55081585500827 |
| - type: mrr_at_1000 |
| value: 37.59430456909269 |
| - type: mrr_at_20 |
| value: 37.13279102089857 |
| - type: mrr_at_3 |
| value: 33.08333333333333 |
| - type: mrr_at_5 |
| value: 35.09333333333329 |
| - type: nauc_map_at_1000_diff1 |
| value: 12.787713123133509 |
| - type: nauc_map_at_1000_max |
| value: 25.009736848507657 |
| - type: nauc_map_at_1000_std |
| value: 15.236663910283454 |
| - type: nauc_map_at_100_diff1 |
| value: 12.790709042725402 |
| - type: nauc_map_at_100_max |
| value: 24.925098118006847 |
| - type: nauc_map_at_100_std |
| value: 15.015816602174784 |
| - type: nauc_map_at_10_diff1 |
| value: 13.224119223857949 |
| - type: nauc_map_at_10_max |
| value: 23.147122480526107 |
| - type: nauc_map_at_10_std |
| value: 11.68007541947038 |
| - type: nauc_map_at_1_diff1 |
| value: 24.060065186932423 |
| - type: nauc_map_at_1_max |
| value: 15.813056289861308 |
| - type: nauc_map_at_1_std |
| value: 4.409040570335129 |
| - type: nauc_map_at_20_diff1 |
| value: 12.709789339255236 |
| - type: nauc_map_at_20_max |
| value: 24.131374778364897 |
| - type: nauc_map_at_20_std |
| value: 13.263463057764785 |
| - type: nauc_map_at_3_diff1 |
| value: 14.638035668154492 |
| - type: nauc_map_at_3_max |
| value: 20.171660202923068 |
| - type: nauc_map_at_3_std |
| value: 5.870864246185647 |
| - type: nauc_map_at_5_diff1 |
| value: 13.68973589831676 |
| - type: nauc_map_at_5_max |
| value: 20.87352332566476 |
| - type: nauc_map_at_5_std |
| value: 8.196922206894563 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 20.27923928575517 |
| - type: nauc_mrr_at_1000_max |
| value: 20.43476539310186 |
| - type: nauc_mrr_at_1000_std |
| value: 9.086257898498179 |
| - type: nauc_mrr_at_100_diff1 |
| value: 20.249544877396524 |
| - type: nauc_mrr_at_100_max |
| value: 20.4491314629493 |
| - type: nauc_mrr_at_100_std |
| value: 9.109279519370439 |
| - type: nauc_mrr_at_10_diff1 |
| value: 20.41175715492326 |
| - type: nauc_mrr_at_10_max |
| value: 20.355827731272182 |
| - type: nauc_mrr_at_10_std |
| value: 9.050079285224115 |
| - type: nauc_mrr_at_1_diff1 |
| value: 23.915902231767276 |
| - type: nauc_mrr_at_1_max |
| value: 15.52850265693499 |
| - type: nauc_mrr_at_1_std |
| value: 4.29701292794671 |
| - type: nauc_mrr_at_20_diff1 |
| value: 20.32272040046418 |
| - type: nauc_mrr_at_20_max |
| value: 20.531506937718476 |
| - type: nauc_mrr_at_20_std |
| value: 9.12290760511003 |
| - type: nauc_mrr_at_3_diff1 |
| value: 19.595989955386596 |
| - type: nauc_mrr_at_3_max |
| value: 20.380814283632976 |
| - type: nauc_mrr_at_3_std |
| value: 7.6948438186508845 |
| - type: nauc_mrr_at_5_diff1 |
| value: 20.549316997450543 |
| - type: nauc_mrr_at_5_max |
| value: 20.10743609189009 |
| - type: nauc_mrr_at_5_std |
| value: 8.321337612314704 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 13.562157258877319 |
| - type: nauc_ndcg_at_1000_max |
| value: 28.044569762293936 |
| - type: nauc_ndcg_at_1000_std |
| value: 21.629002656029655 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 13.565552060975996 |
| - type: nauc_ndcg_at_100_max |
| value: 28.243040397340337 |
| - type: nauc_ndcg_at_100_std |
| value: 21.195028071943252 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 14.626206452933701 |
| - type: nauc_ndcg_at_10_max |
| value: 24.62354235467142 |
| - type: nauc_ndcg_at_10_std |
| value: 13.630591725420302 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 23.915902231767276 |
| - type: nauc_ndcg_at_1_max |
| value: 15.52850265693499 |
| - type: nauc_ndcg_at_1_std |
| value: 4.29701292794671 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 13.709922324110547 |
| - type: nauc_ndcg_at_20_max |
| value: 26.279960777638273 |
| - type: nauc_ndcg_at_20_std |
| value: 16.28928883290933 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 15.040880440725592 |
| - type: nauc_ndcg_at_3_max |
| value: 21.22196654075134 |
| - type: nauc_ndcg_at_3_std |
| value: 7.300499957239256 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 14.9765729251872 |
| - type: nauc_ndcg_at_5_max |
| value: 22.033371543291263 |
| - type: nauc_ndcg_at_5_std |
| value: 9.965196118819666 |
| - type: nauc_precision_at_1000_diff1 |
| value: 2.8533310358503177 |
| - type: nauc_precision_at_1000_max |
| value: 25.73187660681489 |
| - type: nauc_precision_at_1000_std |
| value: 34.02524614249728 |
| - type: nauc_precision_at_100_diff1 |
| value: 6.776893109391378 |
| - type: nauc_precision_at_100_max |
| value: 29.618164171587452 |
| - type: nauc_precision_at_100_std |
| value: 32.075686109744275 |
| - type: nauc_precision_at_10_diff1 |
| value: 10.271876054657762 |
| - type: nauc_precision_at_10_max |
| value: 25.868155025861185 |
| - type: nauc_precision_at_10_std |
| value: 17.773239751669788 |
| - type: nauc_precision_at_1_diff1 |
| value: 23.915902231767276 |
| - type: nauc_precision_at_1_max |
| value: 15.52850265693499 |
| - type: nauc_precision_at_1_std |
| value: 4.29701292794671 |
| - type: nauc_precision_at_20_diff1 |
| value: 7.936691341508646 |
| - type: nauc_precision_at_20_max |
| value: 27.402771907150463 |
| - type: nauc_precision_at_20_std |
| value: 21.84488210613182 |
| - type: nauc_precision_at_3_diff1 |
| value: 11.385560757276366 |
| - type: nauc_precision_at_3_max |
| value: 23.303172357044453 |
| - type: nauc_precision_at_3_std |
| value: 8.52130696871279 |
| - type: nauc_precision_at_5_diff1 |
| value: 11.148869134691138 |
| - type: nauc_precision_at_5_max |
| value: 22.810159658927525 |
| - type: nauc_precision_at_5_std |
| value: 12.185276871335153 |
| - type: nauc_recall_at_1000_diff1 |
| value: 2.945669773991137 |
| - type: nauc_recall_at_1000_max |
| value: 26.026287068033522 |
| - type: nauc_recall_at_1000_std |
| value: 35.19319242132944 |
| - type: nauc_recall_at_100_diff1 |
| value: 6.656374129402131 |
| - type: nauc_recall_at_100_max |
| value: 29.382006365887197 |
| - type: nauc_recall_at_100_std |
| value: 32.0759161910764 |
| - type: nauc_recall_at_10_diff1 |
| value: 10.2223319715612 |
| - type: nauc_recall_at_10_max |
| value: 25.809267124090663 |
| - type: nauc_recall_at_10_std |
| value: 17.763663395998343 |
| - type: nauc_recall_at_1_diff1 |
| value: 24.060065186932423 |
| - type: nauc_recall_at_1_max |
| value: 15.813056289861308 |
| - type: nauc_recall_at_1_std |
| value: 4.409040570335129 |
| - type: nauc_recall_at_20_diff1 |
| value: 7.869155882388014 |
| - type: nauc_recall_at_20_max |
| value: 27.397709947819465 |
| - type: nauc_recall_at_20_std |
| value: 21.82500756071004 |
| - type: nauc_recall_at_3_diff1 |
| value: 11.524772275159814 |
| - type: nauc_recall_at_3_max |
| value: 23.47243651501392 |
| - type: nauc_recall_at_3_std |
| value: 8.536277281164557 |
| - type: nauc_recall_at_5_diff1 |
| value: 11.115700398648645 |
| - type: nauc_recall_at_5_max |
| value: 22.942164352144758 |
| - type: nauc_recall_at_5_std |
| value: 12.219902826983432 |
| - type: ndcg_at_1 |
| value: 25.7 |
| - type: ndcg_at_10 |
| value: 21.714 |
| - type: ndcg_at_100 |
| value: 30.103 |
| - type: ndcg_at_1000 |
| value: 35.658 |
| - type: ndcg_at_20 |
| value: 24.808 |
| - type: ndcg_at_3 |
| value: 20.575 |
| - type: ndcg_at_5 |
| value: 17.887 |
| - type: precision_at_1 |
| value: 25.7 |
| - type: precision_at_10 |
| value: 11.32 |
| - type: precision_at_100 |
| value: 2.338 |
| - type: precision_at_1000 |
| value: 0.366 |
| - type: precision_at_20 |
| value: 7.449999999999999 |
| - type: precision_at_3 |
| value: 19.267 |
| - type: precision_at_5 |
| value: 15.72 |
| - type: recall_at_1 |
| value: 5.208 |
| - type: recall_at_10 |
| value: 22.936999999999998 |
| - type: recall_at_100 |
| value: 47.503 |
| - type: recall_at_1000 |
| value: 74.413 |
| - type: recall_at_20 |
| value: 30.205 |
| - type: recall_at_3 |
| value: 11.693000000000001 |
| - type: recall_at_5 |
| value: 15.898000000000001 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB SICK-R (default) |
| revision: 20a6d6f312dd54037fe07a32d58e5e168867909d |
| split: test |
| type: mteb/sickr-sts |
| metrics: |
| - type: cosine_pearson |
| value: 80.05738628240671 |
| - type: cosine_spearman |
| value: 74.67900925815495 |
| - type: euclidean_pearson |
| value: 78.96070377803025 |
| - type: euclidean_spearman |
| value: 74.67900210067978 |
| - type: main_score |
| value: 74.67900925815495 |
| - type: manhattan_pearson |
| value: 78.48511276416917 |
| - type: manhattan_spearman |
| value: 74.39838905100096 |
| - type: pearson |
| value: 80.05738628240671 |
| - type: spearman |
| value: 74.67900925815495 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STS12 (default) |
| revision: a0d554a64d88156834ff5ae9920b964011b16384 |
| split: test |
| type: mteb/sts12-sts |
| metrics: |
| - type: cosine_pearson |
| value: 59.8887701896054 |
| - type: cosine_spearman |
| value: 61.76924747645064 |
| - type: euclidean_pearson |
| value: 59.33178599145238 |
| - type: euclidean_spearman |
| value: 61.76932878335521 |
| - type: main_score |
| value: 61.76924747645064 |
| - type: manhattan_pearson |
| value: 59.07929980423876 |
| - type: manhattan_spearman |
| value: 61.729703658805704 |
| - type: pearson |
| value: 59.8887701896054 |
| - type: spearman |
| value: 61.76924747645064 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STS13 (default) |
| revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
| split: test |
| type: mteb/sts13-sts |
| metrics: |
| - type: cosine_pearson |
| value: 73.46830128106028 |
| - type: cosine_spearman |
| value: 75.65482017135808 |
| - type: euclidean_pearson |
| value: 75.32146398293357 |
| - type: euclidean_spearman |
| value: 75.65482017135808 |
| - type: main_score |
| value: 75.65482017135808 |
| - type: manhattan_pearson |
| value: 75.11839624254772 |
| - type: manhattan_spearman |
| value: 75.52002809163668 |
| - type: pearson |
| value: 73.46830128106028 |
| - type: spearman |
| value: 75.65482017135808 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STS14 (default) |
| revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
| split: test |
| type: mteb/sts14-sts |
| metrics: |
| - type: cosine_pearson |
| value: 71.49707687668383 |
| - type: cosine_spearman |
| value: 68.16858223090989 |
| - type: euclidean_pearson |
| value: 71.22162204420484 |
| - type: euclidean_spearman |
| value: 68.16860152985392 |
| - type: main_score |
| value: 68.16858223090989 |
| - type: manhattan_pearson |
| value: 70.91495914707767 |
| - type: manhattan_spearman |
| value: 67.98861350196948 |
| - type: pearson |
| value: 71.49707687668383 |
| - type: spearman |
| value: 68.16858223090989 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STS15 (default) |
| revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
| split: test |
| type: mteb/sts15-sts |
| metrics: |
| - type: cosine_pearson |
| value: 76.4178409313179 |
| - type: cosine_spearman |
| value: 78.88504631843803 |
| - type: euclidean_pearson |
| value: 78.71618727706142 |
| - type: euclidean_spearman |
| value: 78.88501053846501 |
| - type: main_score |
| value: 78.88504631843803 |
| - type: manhattan_pearson |
| value: 78.89331900480339 |
| - type: manhattan_spearman |
| value: 79.04826734191282 |
| - type: pearson |
| value: 76.4178409313179 |
| - type: spearman |
| value: 78.88504631843803 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STS16 (default) |
| revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
| split: test |
| type: mteb/sts16-sts |
| metrics: |
| - type: cosine_pearson |
| value: 77.55495044464278 |
| - type: cosine_spearman |
| value: 80.3363642808641 |
| - type: euclidean_pearson |
| value: 79.96424489786347 |
| - type: euclidean_spearman |
| value: 80.3363642808641 |
| - type: main_score |
| value: 80.3363642808641 |
| - type: manhattan_pearson |
| value: 80.05244658476923 |
| - type: manhattan_spearman |
| value: 80.42606943747235 |
| - type: pearson |
| value: 77.55495044464278 |
| - type: spearman |
| value: 80.3363642808641 |
| task: |
| type: STS |
| - dataset: |
| config: en-en |
| name: MTEB STS17 (en-en) |
| revision: faeb762787bd10488a50c8b5be4a3b82e411949c |
| split: test |
| type: mteb/sts17-crosslingual-sts |
| metrics: |
| - type: cosine_pearson |
| value: 79.38935754316142 |
| - type: cosine_spearman |
| value: 85.46849738990943 |
| - type: euclidean_pearson |
| value: 83.64718180060812 |
| - type: euclidean_spearman |
| value: 85.46849738990943 |
| - type: main_score |
| value: 85.46849738990943 |
| - type: manhattan_pearson |
| value: 83.67702948761875 |
| - type: manhattan_spearman |
| value: 85.34710495908027 |
| - type: pearson |
| value: 79.38935754316142 |
| - type: spearman |
| value: 85.46849738990943 |
| task: |
| type: STS |
| - dataset: |
| config: zh-en |
| name: MTEB STS22 (zh-en) |
| revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 |
| split: test |
| type: mteb/sts22-crosslingual-sts |
| metrics: |
| - type: cosine_pearson |
| value: 73.89485634957391 |
| - type: cosine_spearman |
| value: 73.90825698961848 |
| - type: euclidean_pearson |
| value: 75.25124910546262 |
| - type: euclidean_spearman |
| value: 73.90825698961848 |
| - type: main_score |
| value: 73.90825698961848 |
| - type: manhattan_pearson |
| value: 75.11860084263171 |
| - type: manhattan_spearman |
| value: 73.69593141677598 |
| - type: pearson |
| value: 73.89485634957391 |
| - type: spearman |
| value: 73.90825698961848 |
| task: |
| type: STS |
| - dataset: |
| config: en |
| name: MTEB STS22 (en) |
| revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 |
| split: test |
| type: mteb/sts22-crosslingual-sts |
| metrics: |
| - type: cosine_pearson |
| value: 67.81059898950983 |
| - type: cosine_spearman |
| value: 68.30115315444206 |
| - type: euclidean_pearson |
| value: 69.27103790429173 |
| - type: euclidean_spearman |
| value: 68.30115315444206 |
| - type: main_score |
| value: 68.30115315444206 |
| - type: manhattan_pearson |
| value: 69.46849620900602 |
| - type: manhattan_spearman |
| value: 68.45651992521948 |
| - type: pearson |
| value: 67.81059898950983 |
| - type: spearman |
| value: 68.30115315444206 |
| task: |
| type: STS |
| - dataset: |
| config: zh |
| name: MTEB STS22 (zh) |
| revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 |
| split: test |
| type: mteb/sts22-crosslingual-sts |
| metrics: |
| - type: cosine_pearson |
| value: 74.44033998959976 |
| - type: cosine_spearman |
| value: 73.58772060484971 |
| - type: euclidean_pearson |
| value: 73.06488074477468 |
| - type: euclidean_spearman |
| value: 73.58772060484971 |
| - type: main_score |
| value: 73.58772060484971 |
| - type: manhattan_pearson |
| value: 73.00608049548906 |
| - type: manhattan_spearman |
| value: 73.55105762622729 |
| - type: pearson |
| value: 74.44033998959976 |
| - type: spearman |
| value: 73.58772060484971 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STSB (default) |
| revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 |
| split: test |
| type: C-MTEB/STSB |
| metrics: |
| - type: cosine_pearson |
| value: 68.09021885283748 |
| - type: cosine_spearman |
| value: 69.45189378034146 |
| - type: euclidean_pearson |
| value: 69.51961611366887 |
| - type: euclidean_spearman |
| value: 69.45189378034146 |
| - type: main_score |
| value: 69.45189378034146 |
| - type: manhattan_pearson |
| value: 69.30192429794056 |
| - type: manhattan_spearman |
| value: 69.22518486689475 |
| - type: pearson |
| value: 68.09021885283748 |
| - type: spearman |
| value: 69.45189378034146 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STSB (default) |
| revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 |
| split: validation |
| type: C-MTEB/STSB |
| metrics: |
| - type: cosine_pearson |
| value: 69.85149506856348 |
| - type: cosine_spearman |
| value: 72.05319943168429 |
| - type: euclidean_pearson |
| value: 72.41468551288946 |
| - type: euclidean_spearman |
| value: 72.05319943168429 |
| - type: main_score |
| value: 72.05319943168429 |
| - type: manhattan_pearson |
| value: 72.08871687183135 |
| - type: manhattan_spearman |
| value: 71.63960073768047 |
| - type: pearson |
| value: 69.85149506856348 |
| - type: spearman |
| value: 72.05319943168429 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB STSBenchmark (default) |
| revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
| split: test |
| type: mteb/stsbenchmark-sts |
| metrics: |
| - type: cosine_pearson |
| value: 71.64554885631986 |
| - type: cosine_spearman |
| value: 74.23136916844818 |
| - type: euclidean_pearson |
| value: 74.06782419242319 |
| - type: euclidean_spearman |
| value: 74.23136916844818 |
| - type: main_score |
| value: 74.23136916844818 |
| - type: manhattan_pearson |
| value: 74.0008422515175 |
| - type: manhattan_spearman |
| value: 74.10730250032161 |
| - type: pearson |
| value: 71.64554885631986 |
| - type: spearman |
| value: 74.23136916844818 |
| task: |
| type: STS |
| - dataset: |
| config: default |
| name: MTEB SciDocsRR (default) |
| revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
| split: test |
| type: mteb/scidocs-reranking |
| metrics: |
| - type: main_score |
| value: 84.06925933025143 |
| - type: map |
| value: 84.06925933025143 |
| - type: mrr |
| value: 95.48704382037715 |
| - type: nAUC_map_diff1 |
| value: -1.2531960147273353 |
| - type: nAUC_map_max |
| value: 53.63794852890932 |
| - type: nAUC_map_std |
| value: 66.72236818008908 |
| - type: nAUC_mrr_diff1 |
| value: 44.74963455842425 |
| - type: nAUC_mrr_max |
| value: 85.95161801239465 |
| - type: nAUC_mrr_std |
| value: 81.36111675398224 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB SciFact (default) |
| revision: 0228b52cf27578f30900b9e5271d331663a030d7 |
| split: test |
| type: mteb/scifact |
| metrics: |
| - type: main_score |
| value: 76.10900000000001 |
| - type: map_at_1 |
| value: 60.760999999999996 |
| - type: map_at_10 |
| value: 71.654 |
| - type: map_at_100 |
| value: 72.137 |
| - type: map_at_1000 |
| value: 72.149 |
| - type: map_at_20 |
| value: 72.021 |
| - type: map_at_3 |
| value: 68.772 |
| - type: map_at_5 |
| value: 70.36200000000001 |
| - type: mrr_at_1 |
| value: 63.66666666666667 |
| - type: mrr_at_10 |
| value: 72.70595238095237 |
| - type: mrr_at_100 |
| value: 73.04863520816613 |
| - type: mrr_at_1000 |
| value: 73.06075018033764 |
| - type: mrr_at_20 |
| value: 72.93638474099001 |
| - type: mrr_at_3 |
| value: 70.66666666666667 |
| - type: mrr_at_5 |
| value: 71.78333333333333 |
| - type: nauc_map_at_1000_diff1 |
| value: 71.35087668762627 |
| - type: nauc_map_at_1000_max |
| value: 54.011748108009186 |
| - type: nauc_map_at_1000_std |
| value: 7.678845781955053 |
| - type: nauc_map_at_100_diff1 |
| value: 71.34472898717338 |
| - type: nauc_map_at_100_max |
| value: 54.0145296319552 |
| - type: nauc_map_at_100_std |
| value: 7.676258294947637 |
| - type: nauc_map_at_10_diff1 |
| value: 71.37574967668972 |
| - type: nauc_map_at_10_max |
| value: 53.92702576795545 |
| - type: nauc_map_at_10_std |
| value: 7.758848033974705 |
| - type: nauc_map_at_1_diff1 |
| value: 73.89906137509777 |
| - type: nauc_map_at_1_max |
| value: 44.9910089561678 |
| - type: nauc_map_at_1_std |
| value: -3.560024114726528 |
| - type: nauc_map_at_20_diff1 |
| value: 71.3016202130441 |
| - type: nauc_map_at_20_max |
| value: 54.0858492905278 |
| - type: nauc_map_at_20_std |
| value: 7.758638712425257 |
| - type: nauc_map_at_3_diff1 |
| value: 71.0655101710049 |
| - type: nauc_map_at_3_max |
| value: 51.54952135715274 |
| - type: nauc_map_at_3_std |
| value: 3.191945160660174 |
| - type: nauc_map_at_5_diff1 |
| value: 71.71790408466262 |
| - type: nauc_map_at_5_max |
| value: 53.98746745769737 |
| - type: nauc_map_at_5_std |
| value: 6.607248876321941 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 71.35164281601696 |
| - type: nauc_mrr_at_1000_max |
| value: 55.476178340437 |
| - type: nauc_mrr_at_1000_std |
| value: 10.970185730462788 |
| - type: nauc_mrr_at_100_diff1 |
| value: 71.34535141418533 |
| - type: nauc_mrr_at_100_max |
| value: 55.47824801045241 |
| - type: nauc_mrr_at_100_std |
| value: 10.965792309322826 |
| - type: nauc_mrr_at_10_diff1 |
| value: 71.25095755664336 |
| - type: nauc_mrr_at_10_max |
| value: 55.697524040234235 |
| - type: nauc_mrr_at_10_std |
| value: 11.470516804375386 |
| - type: nauc_mrr_at_1_diff1 |
| value: 73.2821036264498 |
| - type: nauc_mrr_at_1_max |
| value: 50.30171935076129 |
| - type: nauc_mrr_at_1_std |
| value: 5.150119259795942 |
| - type: nauc_mrr_at_20_diff1 |
| value: 71.29800338727547 |
| - type: nauc_mrr_at_20_max |
| value: 55.54151075144868 |
| - type: nauc_mrr_at_20_std |
| value: 11.02928131756253 |
| - type: nauc_mrr_at_3_diff1 |
| value: 71.00310802253107 |
| - type: nauc_mrr_at_3_max |
| value: 55.197709395727045 |
| - type: nauc_mrr_at_3_std |
| value: 10.04566210045661 |
| - type: nauc_mrr_at_5_diff1 |
| value: 71.20451267727627 |
| - type: nauc_mrr_at_5_max |
| value: 56.42653941908357 |
| - type: nauc_mrr_at_5_std |
| value: 12.134736985103611 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 70.82229943036683 |
| - type: nauc_ndcg_at_1000_max |
| value: 55.754118035528776 |
| - type: nauc_ndcg_at_1000_std |
| value: 10.535060943270949 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 70.56950271178324 |
| - type: nauc_ndcg_at_100_max |
| value: 55.8594687697972 |
| - type: nauc_ndcg_at_100_std |
| value: 10.666914593212478 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 70.38141205486814 |
| - type: nauc_ndcg_at_10_max |
| value: 56.37560065613112 |
| - type: nauc_ndcg_at_10_std |
| value: 12.026555946404496 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 73.2821036264498 |
| - type: nauc_ndcg_at_1_max |
| value: 50.30171935076129 |
| - type: nauc_ndcg_at_1_std |
| value: 5.150119259795942 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 70.3390226823462 |
| - type: nauc_ndcg_at_20_max |
| value: 56.500553855618605 |
| - type: nauc_ndcg_at_20_std |
| value: 11.29004765829262 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 69.49806863319228 |
| - type: nauc_ndcg_at_3_max |
| value: 54.71563247265625 |
| - type: nauc_ndcg_at_3_std |
| value: 7.436156809946794 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 70.92542004817086 |
| - type: nauc_ndcg_at_5_max |
| value: 57.28530843114872 |
| - type: nauc_ndcg_at_5_std |
| value: 11.33887216009956 |
| - type: nauc_precision_at_1000_diff1 |
| value: -35.28994929585216 |
| - type: nauc_precision_at_1000_max |
| value: 14.893397453096902 |
| - type: nauc_precision_at_1000_std |
| value: 51.396256011227734 |
| - type: nauc_precision_at_100_diff1 |
| value: -23.15302836790124 |
| - type: nauc_precision_at_100_max |
| value: 20.885275194425965 |
| - type: nauc_precision_at_100_std |
| value: 47.38237004790941 |
| - type: nauc_precision_at_10_diff1 |
| value: 0.6508705327922056 |
| - type: nauc_precision_at_10_max |
| value: 35.86490378321761 |
| - type: nauc_precision_at_10_std |
| value: 47.767508836235066 |
| - type: nauc_precision_at_1_diff1 |
| value: 73.2821036264498 |
| - type: nauc_precision_at_1_max |
| value: 50.30171935076129 |
| - type: nauc_precision_at_1_std |
| value: 5.150119259795942 |
| - type: nauc_precision_at_20_diff1 |
| value: -10.7729801324503 |
| - type: nauc_precision_at_20_max |
| value: 29.079735099337757 |
| - type: nauc_precision_at_20_std |
| value: 45.845298013245014 |
| - type: nauc_precision_at_3_diff1 |
| value: 34.75492967022111 |
| - type: nauc_precision_at_3_max |
| value: 50.28675231734285 |
| - type: nauc_precision_at_3_std |
| value: 28.248258905786056 |
| - type: nauc_precision_at_5_diff1 |
| value: 20.985818128495684 |
| - type: nauc_precision_at_5_max |
| value: 49.058382386827766 |
| - type: nauc_precision_at_5_std |
| value: 42.45972725558781 |
| - type: nauc_recall_at_1000_diff1 |
| value: .nan |
| - type: nauc_recall_at_1000_max |
| value: .nan |
| - type: nauc_recall_at_1000_std |
| value: .nan |
| - type: nauc_recall_at_100_diff1 |
| value: 59.20634920634924 |
| - type: nauc_recall_at_100_max |
| value: 61.545284780578854 |
| - type: nauc_recall_at_100_std |
| value: 20.144724556489304 |
| - type: nauc_recall_at_10_diff1 |
| value: 64.66373744374164 |
| - type: nauc_recall_at_10_max |
| value: 63.97713233823764 |
| - type: nauc_recall_at_10_std |
| value: 26.664192167250576 |
| - type: nauc_recall_at_1_diff1 |
| value: 73.89906137509777 |
| - type: nauc_recall_at_1_max |
| value: 44.9910089561678 |
| - type: nauc_recall_at_1_std |
| value: -3.560024114726528 |
| - type: nauc_recall_at_20_diff1 |
| value: 62.63616557734212 |
| - type: nauc_recall_at_20_max |
| value: 67.73835460109973 |
| - type: nauc_recall_at_20_std |
| value: 23.744164332399645 |
| - type: nauc_recall_at_3_diff1 |
| value: 66.22059143579988 |
| - type: nauc_recall_at_3_max |
| value: 56.70083839895786 |
| - type: nauc_recall_at_3_std |
| value: 8.353413350691936 |
| - type: nauc_recall_at_5_diff1 |
| value: 67.74494960835959 |
| - type: nauc_recall_at_5_max |
| value: 65.60091243576522 |
| - type: nauc_recall_at_5_std |
| value: 21.75399137900112 |
| - type: ndcg_at_1 |
| value: 63.666999999999994 |
| - type: ndcg_at_10 |
| value: 76.10900000000001 |
| - type: ndcg_at_100 |
| value: 77.989 |
| - type: ndcg_at_1000 |
| value: 78.391 |
| - type: ndcg_at_20 |
| value: 77.199 |
| - type: ndcg_at_3 |
| value: 71.53699999999999 |
| - type: ndcg_at_5 |
| value: 73.662 |
| - type: precision_at_1 |
| value: 63.666999999999994 |
| - type: precision_at_10 |
| value: 10.033 |
| - type: precision_at_100 |
| value: 1.097 |
| - type: precision_at_1000 |
| value: 0.11299999999999999 |
| - type: precision_at_20 |
| value: 5.267 |
| - type: precision_at_3 |
| value: 27.889000000000003 |
| - type: precision_at_5 |
| value: 18.267 |
| - type: recall_at_1 |
| value: 60.760999999999996 |
| - type: recall_at_10 |
| value: 88.43299999999999 |
| - type: recall_at_100 |
| value: 96.667 |
| - type: recall_at_1000 |
| value: 100.0 |
| - type: recall_at_20 |
| value: 92.5 |
| - type: recall_at_3 |
| value: 76.461 |
| - type: recall_at_5 |
| value: 81.678 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB SprintDuplicateQuestions (default) |
| revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
| split: test |
| type: mteb/sprintduplicatequestions-pairclassification |
| metrics: |
| - type: cosine_accuracy |
| value: 99.76831683168317 |
| - type: cosine_accuracy_threshold |
| value: 75.98594427108765 |
| - type: cosine_ap |
| value: 94.10650199330435 |
| - type: cosine_f1 |
| value: 87.77429467084639 |
| - type: cosine_f1_threshold |
| value: 75.98594427108765 |
| - type: cosine_precision |
| value: 91.90371991247265 |
| - type: cosine_recall |
| value: 84.0 |
| - type: dot_accuracy |
| value: 99.76831683168317 |
| - type: dot_accuracy_threshold |
| value: 75.98594427108765 |
| - type: dot_ap |
| value: 94.10650199330435 |
| - type: dot_f1 |
| value: 87.77429467084639 |
| - type: dot_f1_threshold |
| value: 75.98594427108765 |
| - type: dot_precision |
| value: 91.90371991247265 |
| - type: dot_recall |
| value: 84.0 |
| - type: euclidean_accuracy |
| value: 99.76831683168317 |
| - type: euclidean_accuracy_threshold |
| value: 69.30232048034668 |
| - type: euclidean_ap |
| value: 94.10650199330436 |
| - type: euclidean_f1 |
| value: 87.77429467084639 |
| - type: euclidean_f1_threshold |
| value: 69.30232048034668 |
| - type: euclidean_precision |
| value: 91.90371991247265 |
| - type: euclidean_recall |
| value: 84.0 |
| - type: main_score |
| value: 94.31118902382526 |
| - type: manhattan_accuracy |
| value: 99.77227722772277 |
| - type: manhattan_accuracy_threshold |
| value: 1752.2960662841797 |
| - type: manhattan_ap |
| value: 94.31118902382526 |
| - type: manhattan_f1 |
| value: 87.98328108672936 |
| - type: manhattan_f1_threshold |
| value: 1752.2960662841797 |
| - type: manhattan_precision |
| value: 92.12253829321662 |
| - type: manhattan_recall |
| value: 84.2 |
| - type: max_accuracy |
| value: 99.77227722772277 |
| - type: max_ap |
| value: 94.31118902382526 |
| - type: max_f1 |
| value: 87.98328108672936 |
| - type: max_precision |
| value: 92.12253829321662 |
| - type: max_recall |
| value: 84.2 |
| - type: similarity_accuracy |
| value: 99.76831683168317 |
| - type: similarity_accuracy_threshold |
| value: 75.98594427108765 |
| - type: similarity_ap |
| value: 94.10650199330435 |
| - type: similarity_f1 |
| value: 87.77429467084639 |
| - type: similarity_f1_threshold |
| value: 75.98594427108765 |
| - type: similarity_precision |
| value: 91.90371991247265 |
| - type: similarity_recall |
| value: 84.0 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB StackExchangeClustering (default) |
| revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
| split: test |
| type: mteb/stackexchange-clustering |
| metrics: |
| - type: main_score |
| value: 65.28699161008417 |
| - type: v_measure |
| value: 65.28699161008417 |
| - type: v_measure_std |
| value: 5.01676559317753 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB StackExchangeClusteringP2P (default) |
| revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
| split: test |
| type: mteb/stackexchange-clustering-p2p |
| metrics: |
| - type: main_score |
| value: 32.18986400821423 |
| - type: v_measure |
| value: 32.18986400821423 |
| - type: v_measure_std |
| value: 1.7607695643068701 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB StackOverflowDupQuestions (default) |
| revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
| split: test |
| type: mteb/stackoverflowdupquestions-reranking |
| metrics: |
| - type: main_score |
| value: 51.373735094128726 |
| - type: map |
| value: 51.373735094128726 |
| - type: mrr |
| value: 52.03188661828367 |
| - type: nAUC_map_diff1 |
| value: 37.47429390492891 |
| - type: nAUC_map_max |
| value: 11.050764243820572 |
| - type: nAUC_map_std |
| value: 8.32183046644254 |
| - type: nAUC_mrr_diff1 |
| value: 38.215874831509836 |
| - type: nAUC_mrr_max |
| value: 12.326444149252634 |
| - type: nAUC_mrr_std |
| value: 9.234015034873362 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB SummEval (default) |
| revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
| split: test |
| type: mteb/summeval |
| metrics: |
| - type: cosine_pearson |
| value: 30.04547230414472 |
| - type: cosine_spearman |
| value: 30.62051882468504 |
| - type: dot_pearson |
| value: 30.04547442370404 |
| - type: dot_spearman |
| value: 30.62051882468504 |
| - type: main_score |
| value: 30.62051882468504 |
| - type: pearson |
| value: 30.04547230414472 |
| - type: spearman |
| value: 30.62051882468504 |
| task: |
| type: Summarization |
| - dataset: |
| config: default |
| name: MTEB T2Reranking (default) |
| revision: 76631901a18387f85eaa53e5450019b87ad58ef9 |
| split: dev |
| type: C-MTEB/T2Reranking |
| metrics: |
| - type: main_score |
| value: 67.491940360769 |
| - type: map |
| value: 67.491940360769 |
| - type: mrr |
| value: 77.88394939334343 |
| - type: nAUC_map_diff1 |
| value: -9.62446631785462 |
| - type: nAUC_map_max |
| value: 36.249702987605744 |
| - type: nAUC_map_std |
| value: -2.805167498766831 |
| - type: nAUC_mrr_diff1 |
| value: -6.20324917287488 |
| - type: nAUC_mrr_max |
| value: 31.812094369246875 |
| - type: nAUC_mrr_std |
| value: -4.075688771938606 |
| task: |
| type: Reranking |
| - dataset: |
| config: default |
| name: MTEB T2Retrieval (default) |
| revision: 8731a845f1bf500a4f111cf1070785c793d10e64 |
| split: test |
| type: C-MTEB/T2Retrieval |
| metrics: |
| - type: main_score |
| value: 84.787 |
| - type: map_at_1 |
| value: 27.672 |
| - type: map_at_10 |
| value: 77.376 |
| - type: map_at_100 |
| value: 80.938 |
| - type: map_at_1000 |
| value: 81.003 |
| - type: map_at_20 |
| value: 80.10300000000001 |
| - type: map_at_3 |
| value: 54.591 |
| - type: map_at_5 |
| value: 66.989 |
| - type: mrr_at_1 |
| value: 90.43485884622129 |
| - type: mrr_at_10 |
| value: 92.71924029124197 |
| - type: mrr_at_100 |
| value: 92.80671884902306 |
| - type: mrr_at_1000 |
| value: 92.80992943051154 |
| - type: mrr_at_20 |
| value: 92.7778595796147 |
| - type: mrr_at_3 |
| value: 92.34394178502532 |
| - type: mrr_at_5 |
| value: 92.58197439943898 |
| - type: nauc_map_at_1000_diff1 |
| value: 14.187138773935896 |
| - type: nauc_map_at_1000_max |
| value: 47.31682799510659 |
| - type: nauc_map_at_1000_std |
| value: 21.741024677228047 |
| - type: nauc_map_at_100_diff1 |
| value: 14.184796108320008 |
| - type: nauc_map_at_100_max |
| value: 47.20099045909537 |
| - type: nauc_map_at_100_std |
| value: 21.64510249356838 |
| - type: nauc_map_at_10_diff1 |
| value: 18.09013734024722 |
| - type: nauc_map_at_10_max |
| value: 33.957693353315754 |
| - type: nauc_map_at_10_std |
| value: 4.632860952606258 |
| - type: nauc_map_at_1_diff1 |
| value: 52.11313588305665 |
| - type: nauc_map_at_1_max |
| value: -26.469624300417475 |
| - type: nauc_map_at_1_std |
| value: -39.53640793241912 |
| - type: nauc_map_at_20_diff1 |
| value: 14.76789274084956 |
| - type: nauc_map_at_20_max |
| value: 44.60301273979322 |
| - type: nauc_map_at_20_std |
| value: 18.04392206281267 |
| - type: nauc_map_at_3_diff1 |
| value: 37.91974561569971 |
| - type: nauc_map_at_3_max |
| value: -13.879081458775136 |
| - type: nauc_map_at_3_std |
| value: -37.05923027567862 |
| - type: nauc_map_at_5_diff1 |
| value: 29.93643633903826 |
| - type: nauc_map_at_5_max |
| value: 3.3751192173497846 |
| - type: nauc_map_at_5_std |
| value: -25.135169956144026 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 50.63468735448507 |
| - type: nauc_mrr_at_1000_max |
| value: 80.9387975384782 |
| - type: nauc_mrr_at_1000_std |
| value: 47.45679490810955 |
| - type: nauc_mrr_at_100_diff1 |
| value: 50.63382425620111 |
| - type: nauc_mrr_at_100_max |
| value: 80.94557071660084 |
| - type: nauc_mrr_at_100_std |
| value: 47.4706521424307 |
| - type: nauc_mrr_at_10_diff1 |
| value: 50.65290897062318 |
| - type: nauc_mrr_at_10_max |
| value: 81.04383562705446 |
| - type: nauc_mrr_at_10_std |
| value: 47.512964315818635 |
| - type: nauc_mrr_at_1_diff1 |
| value: 50.784542961223664 |
| - type: nauc_mrr_at_1_max |
| value: 77.12020918334333 |
| - type: nauc_mrr_at_1_std |
| value: 41.62060992521132 |
| - type: nauc_mrr_at_20_diff1 |
| value: 50.642843237883554 |
| - type: nauc_mrr_at_20_max |
| value: 80.98753633072556 |
| - type: nauc_mrr_at_20_std |
| value: 47.51611228243667 |
| - type: nauc_mrr_at_3_diff1 |
| value: 50.55208949999196 |
| - type: nauc_mrr_at_3_max |
| value: 80.90088916202212 |
| - type: nauc_mrr_at_3_std |
| value: 47.108801496033706 |
| - type: nauc_mrr_at_5_diff1 |
| value: 50.575025944200426 |
| - type: nauc_mrr_at_5_max |
| value: 81.1682576960758 |
| - type: nauc_mrr_at_5_std |
| value: 47.66180863029945 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 19.248626358613777 |
| - type: nauc_ndcg_at_1000_max |
| value: 60.28098612146239 |
| - type: nauc_ndcg_at_1000_std |
| value: 35.566884443831256 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 18.724475810243103 |
| - type: nauc_ndcg_at_100_max |
| value: 59.05126018911714 |
| - type: nauc_ndcg_at_100_std |
| value: 34.989411651595724 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 18.365968693164454 |
| - type: nauc_ndcg_at_10_max |
| value: 49.3849363402422 |
| - type: nauc_ndcg_at_10_std |
| value: 22.725576324478418 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 50.784542961223664 |
| - type: nauc_ndcg_at_1_max |
| value: 77.12020918334333 |
| - type: nauc_ndcg_at_1_std |
| value: 41.62060992521132 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 18.609251604582088 |
| - type: nauc_ndcg_at_20_max |
| value: 53.4399532074586 |
| - type: nauc_ndcg_at_20_std |
| value: 27.89488334480925 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 14.778198776216014 |
| - type: nauc_ndcg_at_3_max |
| value: 66.3906486945579 |
| - type: nauc_ndcg_at_3_std |
| value: 37.77884123555143 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 14.738943082799125 |
| - type: nauc_ndcg_at_5_max |
| value: 58.68056564574427 |
| - type: nauc_ndcg_at_5_std |
| value: 31.79077827650123 |
| - type: nauc_precision_at_1000_diff1 |
| value: -32.96252557855019 |
| - type: nauc_precision_at_1000_max |
| value: 50.89186882409747 |
| - type: nauc_precision_at_1000_std |
| value: 65.12914817350996 |
| - type: nauc_precision_at_100_diff1 |
| value: -32.967363137612 |
| - type: nauc_precision_at_100_max |
| value: 52.371028986911206 |
| - type: nauc_precision_at_100_std |
| value: 66.22460187169995 |
| - type: nauc_precision_at_10_diff1 |
| value: -33.1994311022115 |
| - type: nauc_precision_at_10_max |
| value: 56.765578648101936 |
| - type: nauc_precision_at_10_std |
| value: 61.720110059524245 |
| - type: nauc_precision_at_1_diff1 |
| value: 50.784542961223664 |
| - type: nauc_precision_at_1_max |
| value: 77.12020918334333 |
| - type: nauc_precision_at_1_std |
| value: 41.62060992521132 |
| - type: nauc_precision_at_20_diff1 |
| value: -33.03547004009963 |
| - type: nauc_precision_at_20_max |
| value: 54.80165712635334 |
| - type: nauc_precision_at_20_std |
| value: 65.22155557747443 |
| - type: nauc_precision_at_3_diff1 |
| value: -29.686327824889048 |
| - type: nauc_precision_at_3_max |
| value: 67.05323085900143 |
| - type: nauc_precision_at_3_std |
| value: 54.17159434030293 |
| - type: nauc_precision_at_5_diff1 |
| value: -33.83722712457918 |
| - type: nauc_precision_at_5_max |
| value: 62.05827208111495 |
| - type: nauc_precision_at_5_std |
| value: 57.81072889200247 |
| - type: nauc_recall_at_1000_diff1 |
| value: 8.711894161388097 |
| - type: nauc_recall_at_1000_max |
| value: 65.25175068548329 |
| - type: nauc_recall_at_1000_std |
| value: 66.12749628458647 |
| - type: nauc_recall_at_100_diff1 |
| value: 8.66880971384389 |
| - type: nauc_recall_at_100_max |
| value: 52.0738003480146 |
| - type: nauc_recall_at_100_std |
| value: 46.06733146130884 |
| - type: nauc_recall_at_10_diff1 |
| value: 17.07505618926457 |
| - type: nauc_recall_at_10_max |
| value: 24.10644474612695 |
| - type: nauc_recall_at_10_std |
| value: -1.8168787700162745 |
| - type: nauc_recall_at_1_diff1 |
| value: 52.11313588305665 |
| - type: nauc_recall_at_1_max |
| value: -26.469624300417475 |
| - type: nauc_recall_at_1_std |
| value: -39.53640793241912 |
| - type: nauc_recall_at_20_diff1 |
| value: 11.613589798104606 |
| - type: nauc_recall_at_20_max |
| value: 38.97176582712362 |
| - type: nauc_recall_at_20_std |
| value: 20.84092179197353 |
| - type: nauc_recall_at_3_diff1 |
| value: 36.41927821015105 |
| - type: nauc_recall_at_3_max |
| value: -18.226003965381963 |
| - type: nauc_recall_at_3_std |
| value: -39.73849383747519 |
| - type: nauc_recall_at_5_diff1 |
| value: 29.151020828995026 |
| - type: nauc_recall_at_5_max |
| value: -4.815421889490071 |
| - type: nauc_recall_at_5_std |
| value: -30.957293364793664 |
| - type: ndcg_at_1 |
| value: 90.435 |
| - type: ndcg_at_10 |
| value: 84.787 |
| - type: ndcg_at_100 |
| value: 88.276 |
| - type: ndcg_at_1000 |
| value: 88.913 |
| - type: ndcg_at_20 |
| value: 86.502 |
| - type: ndcg_at_3 |
| value: 86.452 |
| - type: ndcg_at_5 |
| value: 84.951 |
| - type: precision_at_1 |
| value: 90.435 |
| - type: precision_at_10 |
| value: 42.034 |
| - type: precision_at_100 |
| value: 4.997 |
| - type: precision_at_1000 |
| value: 0.515 |
| - type: precision_at_20 |
| value: 23.299 |
| - type: precision_at_3 |
| value: 75.58399999999999 |
| - type: precision_at_5 |
| value: 63.243 |
| - type: recall_at_1 |
| value: 27.672 |
| - type: recall_at_10 |
| value: 83.404 |
| - type: recall_at_100 |
| value: 94.883 |
| - type: recall_at_1000 |
| value: 98.123 |
| - type: recall_at_20 |
| value: 89.252 |
| - type: recall_at_3 |
| value: 56.16 |
| - type: recall_at_5 |
| value: 70.13000000000001 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB TNews (default) |
| revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 |
| split: test |
| type: C-MTEB/TNews-classification |
| metrics: |
| - type: accuracy |
| value: 0.0 |
| - type: f1 |
| value: 0.0 |
| - type: f1_weighted |
| value: 0.0 |
| - type: main_score |
| value: 0.0 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB TNews (default) |
| revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 |
| split: validation |
| type: C-MTEB/TNews-classification |
| metrics: |
| - type: accuracy |
| value: 50.82700000000001 |
| - type: f1 |
| value: 48.96846676189542 |
| - type: f1_weighted |
| value: 50.893856756125246 |
| - type: main_score |
| value: 50.82700000000001 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB TRECCOVID (default) |
| revision: bb9466bac8153a0349341eb1b22e06409e78ef4e |
| split: test |
| type: mteb/trec-covid |
| metrics: |
| - type: main_score |
| value: 77.628 |
| - type: map_at_1 |
| value: 0.22399999999999998 |
| - type: map_at_10 |
| value: 1.9449999999999998 |
| - type: map_at_100 |
| value: 12.856000000000002 |
| - type: map_at_1000 |
| value: 30.894 |
| - type: map_at_20 |
| value: 3.688 |
| - type: map_at_3 |
| value: 0.658 |
| - type: map_at_5 |
| value: 1.018 |
| - type: mrr_at_1 |
| value: 86.0 |
| - type: mrr_at_10 |
| value: 92.16666666666667 |
| - type: mrr_at_100 |
| value: 92.16666666666667 |
| - type: mrr_at_1000 |
| value: 92.16666666666667 |
| - type: mrr_at_20 |
| value: 92.16666666666667 |
| - type: mrr_at_3 |
| value: 91.66666666666667 |
| - type: mrr_at_5 |
| value: 92.16666666666667 |
| - type: nauc_map_at_1000_diff1 |
| value: 3.874275710501894 |
| - type: nauc_map_at_1000_max |
| value: 42.30871182822161 |
| - type: nauc_map_at_1000_std |
| value: 73.1469042142082 |
| - type: nauc_map_at_100_diff1 |
| value: 10.741316614996673 |
| - type: nauc_map_at_100_max |
| value: 20.209806325112854 |
| - type: nauc_map_at_100_std |
| value: 54.852351862227444 |
| - type: nauc_map_at_10_diff1 |
| value: 5.126853472907795 |
| - type: nauc_map_at_10_max |
| value: -1.0189783202285525 |
| - type: nauc_map_at_10_std |
| value: 14.740708060291375 |
| - type: nauc_map_at_1_diff1 |
| value: 15.257728237315499 |
| - type: nauc_map_at_1_max |
| value: -16.76524076910754 |
| - type: nauc_map_at_1_std |
| value: 2.467734630732928 |
| - type: nauc_map_at_20_diff1 |
| value: 5.09579525723425 |
| - type: nauc_map_at_20_max |
| value: 0.49956021541095297 |
| - type: nauc_map_at_20_std |
| value: 21.83633208703783 |
| - type: nauc_map_at_3_diff1 |
| value: 17.747855564334554 |
| - type: nauc_map_at_3_max |
| value: -16.856658500402272 |
| - type: nauc_map_at_3_std |
| value: 0.7848912375580602 |
| - type: nauc_map_at_5_diff1 |
| value: 6.113580740023873 |
| - type: nauc_map_at_5_max |
| value: -10.302723521258908 |
| - type: nauc_map_at_5_std |
| value: 4.462343264994648 |
| - type: nauc_mrr_at_1000_diff1 |
| value: -0.30725281048945086 |
| - type: nauc_mrr_at_1000_max |
| value: -5.665166368661917 |
| - type: nauc_mrr_at_1000_std |
| value: 55.91692870501676 |
| - type: nauc_mrr_at_100_diff1 |
| value: -0.30725281048945086 |
| - type: nauc_mrr_at_100_max |
| value: -5.665166368661917 |
| - type: nauc_mrr_at_100_std |
| value: 55.91692870501676 |
| - type: nauc_mrr_at_10_diff1 |
| value: -0.30725281048945086 |
| - type: nauc_mrr_at_10_max |
| value: -5.665166368661917 |
| - type: nauc_mrr_at_10_std |
| value: 55.91692870501676 |
| - type: nauc_mrr_at_1_diff1 |
| value: -5.556325342940264 |
| - type: nauc_mrr_at_1_max |
| value: -2.792018844395178 |
| - type: nauc_mrr_at_1_std |
| value: 56.948870721906545 |
| - type: nauc_mrr_at_20_diff1 |
| value: -0.30725281048945086 |
| - type: nauc_mrr_at_20_max |
| value: -5.665166368661917 |
| - type: nauc_mrr_at_20_std |
| value: 55.91692870501676 |
| - type: nauc_mrr_at_3_diff1 |
| value: 0.47233468286134095 |
| - type: nauc_mrr_at_3_max |
| value: -8.804704067861577 |
| - type: nauc_mrr_at_3_std |
| value: 55.88586851744749 |
| - type: nauc_mrr_at_5_diff1 |
| value: -0.30725281048945086 |
| - type: nauc_mrr_at_5_max |
| value: -5.665166368661917 |
| - type: nauc_mrr_at_5_std |
| value: 55.91692870501676 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: -0.1345042550425933 |
| - type: nauc_ndcg_at_1000_max |
| value: 42.39024935806373 |
| - type: nauc_ndcg_at_1000_std |
| value: 72.7798720975461 |
| - type: nauc_ndcg_at_100_diff1 |
| value: -9.747555787088007 |
| - type: nauc_ndcg_at_100_max |
| value: 42.766831181803084 |
| - type: nauc_ndcg_at_100_std |
| value: 76.86015244416944 |
| - type: nauc_ndcg_at_10_diff1 |
| value: -26.27362680987509 |
| - type: nauc_ndcg_at_10_max |
| value: 30.431944046507574 |
| - type: nauc_ndcg_at_10_std |
| value: 66.53781705282887 |
| - type: nauc_ndcg_at_1_diff1 |
| value: -18.212689382945317 |
| - type: nauc_ndcg_at_1_max |
| value: -4.713531084924769 |
| - type: nauc_ndcg_at_1_std |
| value: 60.648981250362844 |
| - type: nauc_ndcg_at_20_diff1 |
| value: -20.086737440726655 |
| - type: nauc_ndcg_at_20_max |
| value: 34.37729157545477 |
| - type: nauc_ndcg_at_20_std |
| value: 72.61918470988022 |
| - type: nauc_ndcg_at_3_diff1 |
| value: -15.61833953537465 |
| - type: nauc_ndcg_at_3_max |
| value: -4.926385117627094 |
| - type: nauc_ndcg_at_3_std |
| value: 49.062914801546064 |
| - type: nauc_ndcg_at_5_diff1 |
| value: -26.24770979820179 |
| - type: nauc_ndcg_at_5_max |
| value: 10.694823304966544 |
| - type: nauc_ndcg_at_5_std |
| value: 55.162048508134575 |
| - type: nauc_precision_at_1000_diff1 |
| value: -7.8666941738399405 |
| - type: nauc_precision_at_1000_max |
| value: 47.7715994933915 |
| - type: nauc_precision_at_1000_std |
| value: 40.74870410349625 |
| - type: nauc_precision_at_100_diff1 |
| value: -8.388284738609048 |
| - type: nauc_precision_at_100_max |
| value: 48.91820270459412 |
| - type: nauc_precision_at_100_std |
| value: 79.23047106059042 |
| - type: nauc_precision_at_10_diff1 |
| value: -28.150110132914243 |
| - type: nauc_precision_at_10_max |
| value: 47.657016598848614 |
| - type: nauc_precision_at_10_std |
| value: 70.37939245057737 |
| - type: nauc_precision_at_1_diff1 |
| value: -5.556325342940264 |
| - type: nauc_precision_at_1_max |
| value: -2.792018844395178 |
| - type: nauc_precision_at_1_std |
| value: 56.948870721906545 |
| - type: nauc_precision_at_20_diff1 |
| value: -20.975405607247254 |
| - type: nauc_precision_at_20_max |
| value: 42.210933807639144 |
| - type: nauc_precision_at_20_std |
| value: 77.97688841426242 |
| - type: nauc_precision_at_3_diff1 |
| value: -8.477643393806025 |
| - type: nauc_precision_at_3_max |
| value: -1.736301997434395 |
| - type: nauc_precision_at_3_std |
| value: 48.117097306212315 |
| - type: nauc_precision_at_5_diff1 |
| value: -33.46599030296231 |
| - type: nauc_precision_at_5_max |
| value: 25.59065048625327 |
| - type: nauc_precision_at_5_std |
| value: 51.54564053698053 |
| - type: nauc_recall_at_1000_diff1 |
| value: 5.564800637005415 |
| - type: nauc_recall_at_1000_max |
| value: 38.79986663004598 |
| - type: nauc_recall_at_1000_std |
| value: 60.80970482914177 |
| - type: nauc_recall_at_100_diff1 |
| value: 12.714172839997923 |
| - type: nauc_recall_at_100_max |
| value: 11.122107061916715 |
| - type: nauc_recall_at_100_std |
| value: 40.48872128875498 |
| - type: nauc_recall_at_10_diff1 |
| value: 7.356834320854047 |
| - type: nauc_recall_at_10_max |
| value: -5.043558320134648 |
| - type: nauc_recall_at_10_std |
| value: 6.095900853363236 |
| - type: nauc_recall_at_1_diff1 |
| value: 15.257728237315499 |
| - type: nauc_recall_at_1_max |
| value: -16.76524076910754 |
| - type: nauc_recall_at_1_std |
| value: 2.467734630732928 |
| - type: nauc_recall_at_20_diff1 |
| value: 8.751020653150428 |
| - type: nauc_recall_at_20_max |
| value: -6.816867803112397 |
| - type: nauc_recall_at_20_std |
| value: 9.618881030590314 |
| - type: nauc_recall_at_3_diff1 |
| value: 16.423206556657334 |
| - type: nauc_recall_at_3_max |
| value: -19.342599897313622 |
| - type: nauc_recall_at_3_std |
| value: -4.494478463215018 |
| - type: nauc_recall_at_5_diff1 |
| value: 4.384028056599589 |
| - type: nauc_recall_at_5_max |
| value: -11.901838023947736 |
| - type: nauc_recall_at_5_std |
| value: -2.4395241014266964 |
| - type: ndcg_at_1 |
| value: 83.0 |
| - type: ndcg_at_10 |
| value: 77.628 |
| - type: ndcg_at_100 |
| value: 63.298 |
| - type: ndcg_at_1000 |
| value: 56.521 |
| - type: ndcg_at_20 |
| value: 76.32900000000001 |
| - type: ndcg_at_3 |
| value: 80.35799999999999 |
| - type: ndcg_at_5 |
| value: 79.266 |
| - type: precision_at_1 |
| value: 86.0 |
| - type: precision_at_10 |
| value: 82.6 |
| - type: precision_at_100 |
| value: 65.38000000000001 |
| - type: precision_at_1000 |
| value: 24.834 |
| - type: precision_at_20 |
| value: 81.10000000000001 |
| - type: precision_at_3 |
| value: 84.667 |
| - type: precision_at_5 |
| value: 83.6 |
| - type: recall_at_1 |
| value: 0.22399999999999998 |
| - type: recall_at_10 |
| value: 2.156 |
| - type: recall_at_100 |
| value: 15.928999999999998 |
| - type: recall_at_1000 |
| value: 53.191 |
| - type: recall_at_20 |
| value: 4.204 |
| - type: recall_at_3 |
| value: 0.6930000000000001 |
| - type: recall_at_5 |
| value: 1.097 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB ThuNewsClusteringP2P (default) |
| revision: 5798586b105c0434e4f0fe5e767abe619442cf93 |
| split: test |
| type: C-MTEB/ThuNewsClusteringP2P |
| metrics: |
| - type: main_score |
| value: 60.545793641097134 |
| - type: v_measure |
| value: 60.545793641097134 |
| - type: v_measure_std |
| value: 2.352957317776474 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB ThuNewsClusteringS2S (default) |
| revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d |
| split: test |
| type: C-MTEB/ThuNewsClusteringS2S |
| metrics: |
| - type: main_score |
| value: 57.01398421152894 |
| - type: v_measure |
| value: 57.01398421152894 |
| - type: v_measure_std |
| value: 1.2833070880511654 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB Touche2020 (default) |
| revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f |
| split: test |
| type: mteb/touche2020 |
| metrics: |
| - type: main_score |
| value: 29.907 |
| - type: map_at_1 |
| value: 2.8000000000000003 |
| - type: map_at_10 |
| value: 11.801 |
| - type: map_at_100 |
| value: 18.684 |
| - type: map_at_1000 |
| value: 20.324 |
| - type: map_at_20 |
| value: 14.863000000000001 |
| - type: map_at_3 |
| value: 5.976 |
| - type: map_at_5 |
| value: 8.017000000000001 |
| - type: mrr_at_1 |
| value: 42.857142857142854 |
| - type: mrr_at_10 |
| value: 58.21914480077746 |
| - type: mrr_at_100 |
| value: 58.86865834193187 |
| - type: mrr_at_1000 |
| value: 58.86865834193187 |
| - type: mrr_at_20 |
| value: 58.80220098976022 |
| - type: mrr_at_3 |
| value: 55.10204081632652 |
| - type: mrr_at_5 |
| value: 57.04081632653061 |
| - type: nauc_map_at_1000_diff1 |
| value: 6.6751169504360055 |
| - type: nauc_map_at_1000_max |
| value: -3.7567600769254965 |
| - type: nauc_map_at_1000_std |
| value: -1.1124604009386834 |
| - type: nauc_map_at_100_diff1 |
| value: 6.651926585741258 |
| - type: nauc_map_at_100_max |
| value: -5.864307686139932 |
| - type: nauc_map_at_100_std |
| value: -4.0284486690638674 |
| - type: nauc_map_at_10_diff1 |
| value: 9.029404549337796 |
| - type: nauc_map_at_10_max |
| value: -3.0948376595412075 |
| - type: nauc_map_at_10_std |
| value: -16.028908061051876 |
| - type: nauc_map_at_1_diff1 |
| value: 17.548361776107765 |
| - type: nauc_map_at_1_max |
| value: 0.45804783229218043 |
| - type: nauc_map_at_1_std |
| value: -5.479184148805003 |
| - type: nauc_map_at_20_diff1 |
| value: 4.328228113463324 |
| - type: nauc_map_at_20_max |
| value: -9.231540897130337 |
| - type: nauc_map_at_20_std |
| value: -13.57340782183861 |
| - type: nauc_map_at_3_diff1 |
| value: 14.867523300835748 |
| - type: nauc_map_at_3_max |
| value: 0.07164381913812982 |
| - type: nauc_map_at_3_std |
| value: -14.682908765951844 |
| - type: nauc_map_at_5_diff1 |
| value: 10.234632572678201 |
| - type: nauc_map_at_5_max |
| value: -3.275900458361846 |
| - type: nauc_map_at_5_std |
| value: -16.435789249899617 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 20.695606512580003 |
| - type: nauc_mrr_at_1000_max |
| value: 4.156101549595945 |
| - type: nauc_mrr_at_1000_std |
| value: 8.649527259525529 |
| - type: nauc_mrr_at_100_diff1 |
| value: 20.695606512580003 |
| - type: nauc_mrr_at_100_max |
| value: 4.156101549595945 |
| - type: nauc_mrr_at_100_std |
| value: 8.649527259525529 |
| - type: nauc_mrr_at_10_diff1 |
| value: 20.698427366086086 |
| - type: nauc_mrr_at_10_max |
| value: 3.954042031248478 |
| - type: nauc_mrr_at_10_std |
| value: 9.39591288641001 |
| - type: nauc_mrr_at_1_diff1 |
| value: 16.54599380455322 |
| - type: nauc_mrr_at_1_max |
| value: 0.24848203752485892 |
| - type: nauc_mrr_at_1_std |
| value: 1.2530112824133868 |
| - type: nauc_mrr_at_20_diff1 |
| value: 20.85307910624566 |
| - type: nauc_mrr_at_20_max |
| value: 4.245739796463719 |
| - type: nauc_mrr_at_20_std |
| value: 8.861423973155963 |
| - type: nauc_mrr_at_3_diff1 |
| value: 16.903075214698525 |
| - type: nauc_mrr_at_3_max |
| value: 2.093666156251405 |
| - type: nauc_mrr_at_3_std |
| value: 4.126025928366229 |
| - type: nauc_mrr_at_5_diff1 |
| value: 20.24944648545998 |
| - type: nauc_mrr_at_5_max |
| value: 6.445821430696172 |
| - type: nauc_mrr_at_5_std |
| value: 9.596892048537528 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 20.415433650593336 |
| - type: nauc_ndcg_at_1000_max |
| value: 12.366922931280424 |
| - type: nauc_ndcg_at_1000_std |
| value: 27.300631515965605 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 20.856515124531192 |
| - type: nauc_ndcg_at_100_max |
| value: 0.4578830622365307 |
| - type: nauc_ndcg_at_100_std |
| value: 19.22066872166263 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 18.56708265272367 |
| - type: nauc_ndcg_at_10_max |
| value: 3.4531275220348503 |
| - type: nauc_ndcg_at_10_std |
| value: 2.264480721925588 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 13.704789776500043 |
| - type: nauc_ndcg_at_1_max |
| value: -3.7049166878413837 |
| - type: nauc_ndcg_at_1_std |
| value: 4.422031642782982 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 14.90643434193072 |
| - type: nauc_ndcg_at_20_max |
| value: -8.621048644057323 |
| - type: nauc_ndcg_at_20_std |
| value: -0.4555067324883121 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 19.361812396724805 |
| - type: nauc_ndcg_at_3_max |
| value: 5.817962867013526 |
| - type: nauc_ndcg_at_3_std |
| value: -0.8753252050514689 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 19.512803989022927 |
| - type: nauc_ndcg_at_5_max |
| value: 4.4466800234390655 |
| - type: nauc_ndcg_at_5_std |
| value: 3.1909856882261396 |
| - type: nauc_precision_at_1000_diff1 |
| value: -13.554283710669521 |
| - type: nauc_precision_at_1000_max |
| value: 47.14610434487349 |
| - type: nauc_precision_at_1000_std |
| value: 36.39173956289614 |
| - type: nauc_precision_at_100_diff1 |
| value: 10.856084336362487 |
| - type: nauc_precision_at_100_max |
| value: 25.678326608203704 |
| - type: nauc_precision_at_100_std |
| value: 59.36676183382602 |
| - type: nauc_precision_at_10_diff1 |
| value: 14.37198412094934 |
| - type: nauc_precision_at_10_max |
| value: 9.104503069700481 |
| - type: nauc_precision_at_10_std |
| value: 10.71451519713279 |
| - type: nauc_precision_at_1_diff1 |
| value: 16.54599380455322 |
| - type: nauc_precision_at_1_max |
| value: 0.24848203752485892 |
| - type: nauc_precision_at_1_std |
| value: 1.2530112824133868 |
| - type: nauc_precision_at_20_diff1 |
| value: 3.2143260768611457 |
| - type: nauc_precision_at_20_max |
| value: -7.636401715682449 |
| - type: nauc_precision_at_20_std |
| value: 14.782100594161287 |
| - type: nauc_precision_at_3_diff1 |
| value: 21.50971938731398 |
| - type: nauc_precision_at_3_max |
| value: 9.186546888214185 |
| - type: nauc_precision_at_3_std |
| value: -6.357345547475153 |
| - type: nauc_precision_at_5_diff1 |
| value: 20.90086453668611 |
| - type: nauc_precision_at_5_max |
| value: 7.176558075115805 |
| - type: nauc_precision_at_5_std |
| value: 1.3728705241884456 |
| - type: nauc_recall_at_1000_diff1 |
| value: 28.074510232752107 |
| - type: nauc_recall_at_1000_max |
| value: 38.34389209274438 |
| - type: nauc_recall_at_1000_std |
| value: 63.33063604431367 |
| - type: nauc_recall_at_100_diff1 |
| value: 19.850551726108478 |
| - type: nauc_recall_at_100_max |
| value: -5.164677831273209 |
| - type: nauc_recall_at_100_std |
| value: 23.866106801134855 |
| - type: nauc_recall_at_10_diff1 |
| value: 11.964369616221267 |
| - type: nauc_recall_at_10_max |
| value: -7.987814146906532 |
| - type: nauc_recall_at_10_std |
| value: -12.980265883915317 |
| - type: nauc_recall_at_1_diff1 |
| value: 17.548361776107765 |
| - type: nauc_recall_at_1_max |
| value: 0.45804783229218043 |
| - type: nauc_recall_at_1_std |
| value: -5.479184148805003 |
| - type: nauc_recall_at_20_diff1 |
| value: 5.681314516223625 |
| - type: nauc_recall_at_20_max |
| value: -18.663653531510523 |
| - type: nauc_recall_at_20_std |
| value: -8.289499326600785 |
| - type: nauc_recall_at_3_diff1 |
| value: 14.990419827341613 |
| - type: nauc_recall_at_3_max |
| value: -3.66710023682901 |
| - type: nauc_recall_at_3_std |
| value: -15.496276012407181 |
| - type: nauc_recall_at_5_diff1 |
| value: 11.548731015024577 |
| - type: nauc_recall_at_5_max |
| value: -4.628411094603573 |
| - type: nauc_recall_at_5_std |
| value: -13.165660161388459 |
| - type: ndcg_at_1 |
| value: 39.796 |
| - type: ndcg_at_10 |
| value: 29.907 |
| - type: ndcg_at_100 |
| value: 41.347 |
| - type: ndcg_at_1000 |
| value: 52.688 |
| - type: ndcg_at_20 |
| value: 30.651 |
| - type: ndcg_at_3 |
| value: 35.419 |
| - type: ndcg_at_5 |
| value: 31.715 |
| - type: precision_at_1 |
| value: 42.857 |
| - type: precision_at_10 |
| value: 25.918000000000003 |
| - type: precision_at_100 |
| value: 8.469 |
| - type: precision_at_1000 |
| value: 1.614 |
| - type: precision_at_20 |
| value: 20.305999999999997 |
| - type: precision_at_3 |
| value: 36.735 |
| - type: precision_at_5 |
| value: 30.612000000000002 |
| - type: recall_at_1 |
| value: 2.8000000000000003 |
| - type: recall_at_10 |
| value: 18.722 |
| - type: recall_at_100 |
| value: 52.001 |
| - type: recall_at_1000 |
| value: 86.88 |
| - type: recall_at_20 |
| value: 27.805000000000003 |
| - type: recall_at_3 |
| value: 7.420999999999999 |
| - type: recall_at_5 |
| value: 10.663 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB ToxicConversationsClassification (default) |
| revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de |
| split: test |
| type: mteb/toxic_conversations_50k |
| metrics: |
| - type: accuracy |
| value: 64.833984375 |
| - type: ap |
| value: 11.670668141786788 |
| - type: ap_weighted |
| value: 11.670668141786788 |
| - type: f1 |
| value: 49.77377634658719 |
| - type: f1_weighted |
| value: 72.52437665595998 |
| - type: main_score |
| value: 64.833984375 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB TweetSentimentExtractionClassification (default) |
| revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
| split: test |
| type: mteb/tweet_sentiment_extraction |
| metrics: |
| - type: accuracy |
| value: 64.88398415393323 |
| - type: f1 |
| value: 64.84361328633659 |
| - type: f1_weighted |
| value: 63.59840236775296 |
| - type: main_score |
| value: 64.88398415393323 |
| task: |
| type: Classification |
| - dataset: |
| config: default |
| name: MTEB TwentyNewsgroupsClustering (default) |
| revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
| split: test |
| type: mteb/twentynewsgroups-clustering |
| metrics: |
| - type: main_score |
| value: 46.12488681435932 |
| - type: v_measure |
| value: 46.12488681435932 |
| - type: v_measure_std |
| value: 1.5095413626412524 |
| task: |
| type: Clustering |
| - dataset: |
| config: default |
| name: MTEB TwitterSemEval2015 (default) |
| revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
| split: test |
| type: mteb/twittersemeval2015-pairclassification |
| metrics: |
| - type: cosine_accuracy |
| value: 85.17017345174942 |
| - type: cosine_accuracy_threshold |
| value: 73.24392199516296 |
| - type: cosine_ap |
| value: 71.5689998301689 |
| - type: cosine_f1 |
| value: 66.13515565679575 |
| - type: cosine_f1_threshold |
| value: 69.43247318267822 |
| - type: cosine_precision |
| value: 63.545719844357976 |
| - type: cosine_recall |
| value: 68.94459102902375 |
| - type: dot_accuracy |
| value: 85.17017345174942 |
| - type: dot_accuracy_threshold |
| value: 73.24392199516296 |
| - type: dot_ap |
| value: 71.56899897384942 |
| - type: dot_f1 |
| value: 66.13515565679575 |
| - type: dot_f1_threshold |
| value: 69.43247318267822 |
| - type: dot_precision |
| value: 63.545719844357976 |
| - type: dot_recall |
| value: 68.94459102902375 |
| - type: euclidean_accuracy |
| value: 85.17017345174942 |
| - type: euclidean_accuracy_threshold |
| value: 73.15199971199036 |
| - type: euclidean_ap |
| value: 71.56900157150807 |
| - type: euclidean_f1 |
| value: 66.13515565679575 |
| - type: euclidean_f1_threshold |
| value: 78.18889617919922 |
| - type: euclidean_precision |
| value: 63.545719844357976 |
| - type: euclidean_recall |
| value: 68.94459102902375 |
| - type: main_score |
| value: 71.56900157150807 |
| - type: manhattan_accuracy |
| value: 85.03904154497228 |
| - type: manhattan_accuracy_threshold |
| value: 1829.3399810791016 |
| - type: manhattan_ap |
| value: 71.17145701434644 |
| - type: manhattan_f1 |
| value: 65.9017661467062 |
| - type: manhattan_f1_threshold |
| value: 1959.2126846313477 |
| - type: manhattan_precision |
| value: 64.43156037307789 |
| - type: manhattan_recall |
| value: 67.44063324538259 |
| - type: max_accuracy |
| value: 85.17017345174942 |
| - type: max_ap |
| value: 71.56900157150807 |
| - type: max_f1 |
| value: 66.13515565679575 |
| - type: max_precision |
| value: 64.43156037307789 |
| - type: max_recall |
| value: 68.94459102902375 |
| - type: similarity_accuracy |
| value: 85.17017345174942 |
| - type: similarity_accuracy_threshold |
| value: 73.24392199516296 |
| - type: similarity_ap |
| value: 71.5689998301689 |
| - type: similarity_f1 |
| value: 66.13515565679575 |
| - type: similarity_f1_threshold |
| value: 69.43247318267822 |
| - type: similarity_precision |
| value: 63.545719844357976 |
| - type: similarity_recall |
| value: 68.94459102902375 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB TwitterURLCorpus (default) |
| revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
| split: test |
| type: mteb/twitterurlcorpus-pairclassification |
| metrics: |
| - type: cosine_accuracy |
| value: 89.17995886211045 |
| - type: cosine_accuracy_threshold |
| value: 70.09445428848267 |
| - type: cosine_ap |
| value: 86.05864466686238 |
| - type: cosine_f1 |
| value: 78.36959806588094 |
| - type: cosine_f1_threshold |
| value: 67.36087203025818 |
| - type: cosine_precision |
| value: 76.92821121328983 |
| - type: cosine_recall |
| value: 79.8660301817062 |
| - type: dot_accuracy |
| value: 89.17995886211045 |
| - type: dot_accuracy_threshold |
| value: 70.09446620941162 |
| - type: dot_ap |
| value: 86.05863925325839 |
| - type: dot_f1 |
| value: 78.36959806588094 |
| - type: dot_f1_threshold |
| value: 67.36087799072266 |
| - type: dot_precision |
| value: 76.92821121328983 |
| - type: dot_recall |
| value: 79.8660301817062 |
| - type: euclidean_accuracy |
| value: 89.17995886211045 |
| - type: euclidean_accuracy_threshold |
| value: 77.33762264251709 |
| - type: euclidean_ap |
| value: 86.05864473129277 |
| - type: euclidean_f1 |
| value: 78.36959806588094 |
| - type: euclidean_f1_threshold |
| value: 80.79496026039124 |
| - type: euclidean_precision |
| value: 76.92821121328983 |
| - type: euclidean_recall |
| value: 79.8660301817062 |
| - type: main_score |
| value: 86.05864473129277 |
| - type: manhattan_accuracy |
| value: 89.15279233127644 |
| - type: manhattan_accuracy_threshold |
| value: 1960.135269165039 |
| - type: manhattan_ap |
| value: 86.00803071652211 |
| - type: manhattan_f1 |
| value: 78.28386279840602 |
| - type: manhattan_f1_threshold |
| value: 2062.195587158203 |
| - type: manhattan_precision |
| value: 75.81331602106326 |
| - type: manhattan_recall |
| value: 80.92085001539883 |
| - type: max_accuracy |
| value: 89.17995886211045 |
| - type: max_ap |
| value: 86.05864473129277 |
| - type: max_f1 |
| value: 78.36959806588094 |
| - type: max_precision |
| value: 76.92821121328983 |
| - type: max_recall |
| value: 80.92085001539883 |
| - type: similarity_accuracy |
| value: 89.17995886211045 |
| - type: similarity_accuracy_threshold |
| value: 70.09445428848267 |
| - type: similarity_ap |
| value: 86.05864466686238 |
| - type: similarity_f1 |
| value: 78.36959806588094 |
| - type: similarity_f1_threshold |
| value: 67.36087203025818 |
| - type: similarity_precision |
| value: 76.92821121328983 |
| - type: similarity_recall |
| value: 79.8660301817062 |
| task: |
| type: PairClassification |
| - dataset: |
| config: default |
| name: MTEB VideoRetrieval (default) |
| revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 |
| split: test |
| type: C-MTEB/VideoRetrieval |
| metrics: |
| - type: main_score |
| value: 76.259 |
| - type: map_at_1 |
| value: 63.1 |
| - type: map_at_10 |
| value: 72.214 |
| - type: map_at_100 |
| value: 72.595 |
| - type: map_at_1000 |
| value: 72.604 |
| - type: map_at_20 |
| value: 72.482 |
| - type: map_at_3 |
| value: 70.45 |
| - type: map_at_5 |
| value: 71.565 |
| - type: mrr_at_1 |
| value: 63.1 |
| - type: mrr_at_10 |
| value: 72.2140873015873 |
| - type: mrr_at_100 |
| value: 72.59450024817937 |
| - type: mrr_at_1000 |
| value: 72.60392898169485 |
| - type: mrr_at_20 |
| value: 72.48216831972637 |
| - type: mrr_at_3 |
| value: 70.45000000000005 |
| - type: mrr_at_5 |
| value: 71.56500000000004 |
| - type: nauc_map_at_1000_diff1 |
| value: 74.28035434647866 |
| - type: nauc_map_at_1000_max |
| value: 24.9986630946608 |
| - type: nauc_map_at_1000_std |
| value: -39.39693868573278 |
| - type: nauc_map_at_100_diff1 |
| value: 74.26844900002635 |
| - type: nauc_map_at_100_max |
| value: 25.018503837474537 |
| - type: nauc_map_at_100_std |
| value: -39.37583943184342 |
| - type: nauc_map_at_10_diff1 |
| value: 74.26680601603067 |
| - type: nauc_map_at_10_max |
| value: 25.09926459399667 |
| - type: nauc_map_at_10_std |
| value: -39.89610049620442 |
| - type: nauc_map_at_1_diff1 |
| value: 77.10780012025063 |
| - type: nauc_map_at_1_max |
| value: 19.806657111504965 |
| - type: nauc_map_at_1_std |
| value: -37.2015516410441 |
| - type: nauc_map_at_20_diff1 |
| value: 74.21838781014907 |
| - type: nauc_map_at_20_max |
| value: 25.063007078847527 |
| - type: nauc_map_at_20_std |
| value: -39.43384966895805 |
| - type: nauc_map_at_3_diff1 |
| value: 74.18933637722046 |
| - type: nauc_map_at_3_max |
| value: 24.353710639698495 |
| - type: nauc_map_at_3_std |
| value: -40.981597611432065 |
| - type: nauc_map_at_5_diff1 |
| value: 74.10850958842639 |
| - type: nauc_map_at_5_max |
| value: 24.97711616082726 |
| - type: nauc_map_at_5_std |
| value: -40.679823827809464 |
| - type: nauc_mrr_at_1000_diff1 |
| value: 74.28035434647866 |
| - type: nauc_mrr_at_1000_max |
| value: 24.9986630946608 |
| - type: nauc_mrr_at_1000_std |
| value: -39.39693868573278 |
| - type: nauc_mrr_at_100_diff1 |
| value: 74.26844900002635 |
| - type: nauc_mrr_at_100_max |
| value: 25.018503837474537 |
| - type: nauc_mrr_at_100_std |
| value: -39.37583943184342 |
| - type: nauc_mrr_at_10_diff1 |
| value: 74.26680601603067 |
| - type: nauc_mrr_at_10_max |
| value: 25.09926459399667 |
| - type: nauc_mrr_at_10_std |
| value: -39.89610049620442 |
| - type: nauc_mrr_at_1_diff1 |
| value: 77.10780012025063 |
| - type: nauc_mrr_at_1_max |
| value: 19.806657111504965 |
| - type: nauc_mrr_at_1_std |
| value: -37.2015516410441 |
| - type: nauc_mrr_at_20_diff1 |
| value: 74.21838781014907 |
| - type: nauc_mrr_at_20_max |
| value: 25.063007078847527 |
| - type: nauc_mrr_at_20_std |
| value: -39.43384966895805 |
| - type: nauc_mrr_at_3_diff1 |
| value: 74.18933637722046 |
| - type: nauc_mrr_at_3_max |
| value: 24.353710639698495 |
| - type: nauc_mrr_at_3_std |
| value: -40.981597611432065 |
| - type: nauc_mrr_at_5_diff1 |
| value: 74.10850958842639 |
| - type: nauc_mrr_at_5_max |
| value: 24.97711616082726 |
| - type: nauc_mrr_at_5_std |
| value: -40.679823827809464 |
| - type: nauc_ndcg_at_1000_diff1 |
| value: 73.61936333916759 |
| - type: nauc_ndcg_at_1000_max |
| value: 27.116854943289013 |
| - type: nauc_ndcg_at_1000_std |
| value: -37.54463044764084 |
| - type: nauc_ndcg_at_100_diff1 |
| value: 73.34360970001946 |
| - type: nauc_ndcg_at_100_max |
| value: 27.730908397954856 |
| - type: nauc_ndcg_at_100_std |
| value: -36.847816086959746 |
| - type: nauc_ndcg_at_10_diff1 |
| value: 73.26740567929617 |
| - type: nauc_ndcg_at_10_max |
| value: 28.07969523618901 |
| - type: nauc_ndcg_at_10_std |
| value: -39.5613267347007 |
| - type: nauc_ndcg_at_1_diff1 |
| value: 77.10780012025063 |
| - type: nauc_ndcg_at_1_max |
| value: 19.806657111504965 |
| - type: nauc_ndcg_at_1_std |
| value: -37.2015516410441 |
| - type: nauc_ndcg_at_20_diff1 |
| value: 73.03206507305698 |
| - type: nauc_ndcg_at_20_max |
| value: 27.998040243323953 |
| - type: nauc_ndcg_at_20_std |
| value: -37.708810040181056 |
| - type: nauc_ndcg_at_3_diff1 |
| value: 73.14698395698991 |
| - type: nauc_ndcg_at_3_max |
| value: 26.27353390787337 |
| - type: nauc_ndcg_at_3_std |
| value: -42.26161377498877 |
| - type: nauc_ndcg_at_5_diff1 |
| value: 72.95385082418426 |
| - type: nauc_ndcg_at_5_max |
| value: 27.616141938964926 |
| - type: nauc_ndcg_at_5_std |
| value: -41.73620432852272 |
| - type: nauc_precision_at_1000_diff1 |
| value: 62.222222222221426 |
| - type: nauc_precision_at_1000_max |
| value: 84.86150015561716 |
| - type: nauc_precision_at_1000_std |
| value: 70.37659508247653 |
| - type: nauc_precision_at_100_diff1 |
| value: 57.83489866534799 |
| - type: nauc_precision_at_100_max |
| value: 79.34448289119527 |
| - type: nauc_precision_at_100_std |
| value: 41.03229527104971 |
| - type: nauc_precision_at_10_diff1 |
| value: 67.38577178030296 |
| - type: nauc_precision_at_10_max |
| value: 47.06185741341976 |
| - type: nauc_precision_at_10_std |
| value: -35.28392180735959 |
| - type: nauc_precision_at_1_diff1 |
| value: 77.10780012025063 |
| - type: nauc_precision_at_1_max |
| value: 19.806657111504965 |
| - type: nauc_precision_at_1_std |
| value: -37.2015516410441 |
| - type: nauc_precision_at_20_diff1 |
| value: 62.29131652661053 |
| - type: nauc_precision_at_20_max |
| value: 54.85838779956423 |
| - type: nauc_precision_at_20_std |
| value: -14.987239340181135 |
| - type: nauc_precision_at_3_diff1 |
| value: 69.30752094380478 |
| - type: nauc_precision_at_3_max |
| value: 33.47195304146769 |
| - type: nauc_precision_at_3_std |
| value: -46.94646447336974 |
| - type: nauc_precision_at_5_diff1 |
| value: 67.75036818851238 |
| - type: nauc_precision_at_5_max |
| value: 39.96410162002936 |
| - type: nauc_precision_at_5_std |
| value: -46.30185321551299 |
| - type: nauc_recall_at_1000_diff1 |
| value: 62.22222222222214 |
| - type: nauc_recall_at_1000_max |
| value: 84.86150015561832 |
| - type: nauc_recall_at_1000_std |
| value: 70.37659508247839 |
| - type: nauc_recall_at_100_diff1 |
| value: 57.83489866534869 |
| - type: nauc_recall_at_100_max |
| value: 79.34448289119563 |
| - type: nauc_recall_at_100_std |
| value: 41.03229527105021 |
| - type: nauc_recall_at_10_diff1 |
| value: 67.38577178030309 |
| - type: nauc_recall_at_10_max |
| value: 47.061857413419816 |
| - type: nauc_recall_at_10_std |
| value: -35.283921807359164 |
| - type: nauc_recall_at_1_diff1 |
| value: 77.10780012025063 |
| - type: nauc_recall_at_1_max |
| value: 19.806657111504965 |
| - type: nauc_recall_at_1_std |
| value: -37.2015516410441 |
| - type: nauc_recall_at_20_diff1 |
| value: 62.29131652661064 |
| - type: nauc_recall_at_20_max |
| value: 54.858387799564234 |
| - type: nauc_recall_at_20_std |
| value: -14.987239340180611 |
| - type: nauc_recall_at_3_diff1 |
| value: 69.30752094380468 |
| - type: nauc_recall_at_3_max |
| value: 33.47195304146769 |
| - type: nauc_recall_at_3_std |
| value: -46.94646447336982 |
| - type: nauc_recall_at_5_diff1 |
| value: 67.75036818851241 |
| - type: nauc_recall_at_5_max |
| value: 39.96410162002941 |
| - type: nauc_recall_at_5_std |
| value: -46.30185321551307 |
| - type: ndcg_at_1 |
| value: 63.1 |
| - type: ndcg_at_10 |
| value: 76.259 |
| - type: ndcg_at_100 |
| value: 77.985 |
| - type: ndcg_at_1000 |
| value: 78.227 |
| - type: ndcg_at_20 |
| value: 77.208 |
| - type: ndcg_at_3 |
| value: 72.684 |
| - type: ndcg_at_5 |
| value: 74.698 |
| - type: precision_at_1 |
| value: 63.1 |
| - type: precision_at_10 |
| value: 8.88 |
| - type: precision_at_100 |
| value: 0.966 |
| - type: precision_at_1000 |
| value: 0.099 |
| - type: precision_at_20 |
| value: 4.625 |
| - type: precision_at_3 |
| value: 26.367 |
| - type: precision_at_5 |
| value: 16.8 |
| - type: recall_at_1 |
| value: 63.1 |
| - type: recall_at_10 |
| value: 88.8 |
| - type: recall_at_100 |
| value: 96.6 |
| - type: recall_at_1000 |
| value: 98.5 |
| - type: recall_at_20 |
| value: 92.5 |
| - type: recall_at_3 |
| value: 79.10000000000001 |
| - type: recall_at_5 |
| value: 84.0 |
| task: |
| type: Retrieval |
| - dataset: |
| config: default |
| name: MTEB Waimai (default) |
| revision: 339287def212450dcaa9df8c22bf93e9980c7023 |
| split: test |
| type: C-MTEB/waimai-classification |
| metrics: |
| - type: accuracy |
| value: 85.77999999999999 |
| - type: ap |
| value: 68.2216414015039 |
| - type: ap_weighted |
| value: 68.2216414015039 |
| - type: f1 |
| value: 83.82399258910729 |
| - type: f1_weighted |
| value: 85.78923735869199 |
| - type: main_score |
| value: 85.77999999999999 |
| task: |
| type: Classification |
| pipeline_tag: feature-extraction |
| tags: |
| - mteb |
| - sentence-transformers |
| library_name: transformers |
| --- |
| ## MiniCPM-Embedding-Light |
|
|
| **MiniCPM-Embedding-Light** 是面壁智能与清华大学自然语言处理实验室（THUNLP）、东北大学信息检索小组（NEUIR）共同开发的中英双语言文本嵌入模型，有如下特点： |
| - 出色的中文、英文检索能力。 |
| - 出色的中英跨语言检索能力。 |
| - 支持长文本（最长8192token）。 |
| - 提供稠密向量与token级别的稀疏向量。 |
| - 可变的稠密向量维度（套娃表征）。 |
|
|
| MiniCPM-Embedding-Light结构上采取双向注意力和 Weighted Mean Pooling [1]。采取多阶段训练方式，共使用包括开源数据、机造数据、闭源数据在内的约 260M 条训练数据。 |
|
|
| 欢迎关注 UltraRAG 系列： |
|
|
| - 检索模型：[MiniCPM-Embedding-Light](https://huggingface.co/openbmb/MiniCPM-Embedding-Light) |
| - 重排模型：[MiniCPM-Reranker-Light](https://huggingface.co/openbmb/MiniCPM-Reranker-Light) |
| - 领域自适应RAG框架：[UltraRAG](https://github.com/openbmb/UltraRAG) |
|
|
| **MiniCPM-Embedding-Light** is a bilingual & cross-lingual text embedding model developed by ModelBest Inc. , THUNLP and NEUIR , featuring: |
|
|
| - Exceptional Chinese and English retrieval capabilities. |
| - Outstanding cross-lingual retrieval capabilities between Chinese and English. |
| - Long-text support (up to 8192 tokens). |
| - Dense vectors and token-level sparse vectors. |
| - Variable dense vector dimensions (Matryoshka representation [2]). |
|
|
| MiniCPM-Embedding-Light incorporates bidirectional attention and Weighted Mean Pooling [1] in its architecture. The model underwent multi-stage training using approximately 260 million training examples, including open-source, synthetic, and proprietary data. |
|
|
| We also invite you to explore the UltraRAG series: |
|
|
| - Retrieval Model: [MiniCPM-Embedding-Light](https://huggingface.co/openbmb/MiniCPM-Embedding-Light) |
| - Re-ranking Model: [MiniCPM-Reranker-Light](https://huggingface.co/openbmb/MiniCPM-Reranker-Light) |
| - Domain Adaptive RAG Framework: [UltraRAG](https://github.com/openbmb/UltraRAG) |
|
|
| [1] Muennighoff, N. (2022). Sgpt: Gpt sentence embeddings for semantic search. arXiv preprint arXiv:2202.08904. |
| [2] Kusupati, Aditya, et al. "Matryoshka representation learning." Advances in Neural Information Processing Systems 35 (2022): 30233-30249. |
|
|
| ## 模型信息 Model Information |
|
|
| - 模型大小：440M |
| - 嵌入维度：1024 |
| - 最大输入token数：8192 |
|
|
| - Model Size: 440M |
| - Embedding Dimension: 1024 |
| - Max Input Tokens: 8192 |
|
|
| ## 使用方法 Usage |
|
|
| ### 输入格式 Input Format |
|
|
| 本模型支持 query 侧指令，格式如下： |
|
|
| MiniCPM-Embedding-Light supports query-side instructions in the following format: |
|
|
| ``` |
| Instruction: {{ instruction }} Query: {{ query }} |
| ``` |
|
|
| 例如： |
|
|
| For example: |
|
|
| ``` |
| Instruction: 为这个医学问题检索相关回答。Query: 咽喉癌的成因是什么？ |
| ``` |
|
|
| ``` |
| Instruction: Given a claim about climate change, retrieve documents that support or refute the claim. Query: However the warming trend is slower than most climate models have forecast. |
| ``` |
|
|
| 也可以不提供指令，即采取如下格式： |
|
|
| MiniCPM-Embedding-Light also works in instruction-free mode in the following format: |
|
|
| ``` |
| Query: {{ query }} |
| ``` |
|
|
| ### 环境要求 Requirements |
|
|
| ``` |
| transformers==4.37.2 |
| ``` |
|
|
| ### 示例脚本 Demo |
|
|
| #### Huggingface Transformers |
| ```python |
| from transformers import AutoModel |
| import torch |
| |
| model_name = "openbmb/MiniCPM-Embedding-Light" |
| model = AutoModel.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.float16).to("cuda") |
| |
| # you can use flash_attention_2 for faster inference |
| # model = AutoModel.from_pretrained(model_name, trust_remote_code=True, attn_implementation="flash_attention_2", torch_dtype=torch.float16).to("cuda") |
| |
| model.eval() |
| |
| queries = ["MiniCPM-o 2.6 A GPT-4o Level MLLM for Vision, Speech and Multimodal Live Streaming on Your Phone"] |
| passages = ["MiniCPM-o 2.6 is the latest and most capable model in the MiniCPM-o series. The model is built in an end-to-end fashion based on SigLip-400M, Whisper-medium-300M, ChatTTS-200M, and Qwen2.5-7B with a total of 8B parameters. It exhibits a significant performance improvement over MiniCPM-V 2.6, and introduces new features for real-time speech conversation and multimodal live streaming."] |
| |
| embeddings_query_dense, embeddings_query_sparse = model.encode_query(queries, return_sparse_vectors=True) |
| embeddings_doc_dense, embeddings_doc_sparse = model.encode_corpus(passages, return_sparse_vectors=True) |
| |
| dense_scores = (embeddings_query_dense @ embeddings_doc_dense.T) |
| print(dense_scores.tolist()) # [[0.6512398719787598]] |
| print(model.compute_sparse_score_dicts(embeddings_query_sparse, embeddings_doc_sparse)) # [[0.27202296]] |
| |
| dense_scores, sparse_scores, mixed_scores = model.compute_score(queries, passages) |
| print(dense_scores) # [[0.65123993]] |
| print(sparse_scores) # [[0.27202296]] |
| print(mixed_scores) # [[0.73284686]] |
| ``` |
|
|
| #### Sentence Transformers |
|
|
| ```python |
| import torch |
| from sentence_transformers import SentenceTransformer |
| |
| |
| model_name = "openbmb/MiniCPM-Embedding-Light" |
| model = SentenceTransformer(model_name, trust_remote_code=True, model_kwargs={"torch_dtype": torch.float16}) |
| |
| # you can use flash_attention_2 for faster inference |
| # model = SentenceTransformer(model_name, trust_remote_code=True, model_kwargs={"attn_implementation": "flash_attention_2", "torch_dtype": torch.float16}) |
| |
| queries = ["中国的首都是哪里？"] # "What is the capital of China?" |
| passages = ["beijing", "shanghai"] # "北京", "上海" |
| |
| INSTRUCTION = "Query: " |
| |
| embeddings_query = model.encode(queries, prompt=INSTRUCTION) |
| embeddings_doc = model.encode(passages) |
| |
| scores = (embeddings_query @ embeddings_doc.T) |
| print(scores.tolist()) # [[0.40356746315956116, 0.36183440685272217]] |
| ``` |
|
|
| #### Infinity |
|
|
| ```python |
| import asyncio |
| from infinity_emb import AsyncEngineArray, EngineArgs, AsyncEmbeddingEngine |
| import numpy as np |
| |
| array = AsyncEngineArray.from_args([ |
| EngineArgs(model_name_or_path = "openbmb/MiniCPM-Embedding-Light", engine="torch", dtype="float16", bettertransformer=False, pooling_method="mean", trust_remote_code=True), |
| ]) |
| queries = ["中国的首都是哪里？"] # "What is the capital of China?" |
| passages = ["beijing", "shanghai"] # "北京", "上海" |
| |
| INSTRUCTION = "Query:" |
| queries = [f"{INSTRUCTION} {query}" for query in queries] |
| |
| |
| async def embed_text(engine: AsyncEmbeddingEngine,sentences): |
| async with engine: |
| embeddings, usage = await engine.embed(sentences=sentences) |
| return embeddings |
| |
| queries_embedding = asyncio.run(embed_text(array[0],queries)) |
| passages_embedding = asyncio.run(embed_text(array[0],passages)) |
| |
| scores = (np.array(queries_embedding) @ np.array(passages_embedding).T) |
| print(scores.tolist()) # [[0.40356746315956116, 0.36183443665504456]] |
| ``` |
|
|
| #### FlagEmbedding |
| ```python |
| from FlagEmbedding import FlagModel |
| |
| |
| model = FlagModel("openbmb/MiniCPM-Embedding-Light", |
| query_instruction_for_retrieval="Query: ", |
| pooling_method="mean", |
| trust_remote_code=True, |
| normalize_embeddings=True, |
| use_fp16=True) |
| # You can hack the __init__() method of the FlagEmbedding BaseEmbedder class to use flash_attention_2 for faster inference |
| # self.model = AutoModel.from_pretrained( |
| # model_name_or_path, |
| # trust_remote_code=trust_remote_code, |
| # cache_dir=cache_dir, |
| # # torch_dtype=torch.float16, # we need to add this line to use fp16 |
| # # attn_implementation="flash_attention_2", # we need to add this line to use flash_attention_2 |
| # ) |
| |
| queries = ["中国的首都是哪里？"] # "What is the capital of China?" |
| passages = ["beijing", "shanghai"] # "北京", "上海" |
| |
| |
| embeddings_query = model.encode_queries(queries) |
| embeddings_doc = model.encode_corpus(passages) |
| |
| scores = (embeddings_query @ embeddings_doc.T) |
| print(scores.tolist()) # [[0.40356746315956116, 0.36183440685272217]] |
| ``` |
|
|
|
|
| ## 实验结果 Evaluation Results |
|
|
| ### 中文与英文检索结果 CN/EN Retrieval Results |
|
|
| | 模型 Model | C-MTEB/Retrieval(NDCG@10) | BEIR(NDCG@10) | |
| |----------------------------------------------------|-------------------|---------------| |
| | bge-large-zh-v1.5 | 70.46 | - | |
| | gte-large-zh | 72.49 | - | |
| | Conan-embedding-v1 | 76.67 | | |
| | bge-large-en-v1.5 | - | 54.29 | |
| | modernbert-embed-large | - | 54.36 | |
| | snowflake-arctic-embed-l | - | 55.98 | |
| | gte-en-large-v1.5 | - | 57.91 | |
| | me5-large | 63.66 | 51.43 | |
| | bge-m3(Dense) | 65.43 | 48.82 | |
| | gte-multilingual-base(Dense) | 71.95 | 51.08 | |
| | jina-embeddings-v3 | 68.60 | 53.88 | |
| | gte-Qwen2-1.5B-instruct | 71.86 | 58.29 | |
| | MiniCPM-Embedding | 76.76 | 58.56 | |
| | MiniCPM-Embedding-Light(Dense) | 72.71 | 55.27 | |
| | MiniCPM-Embedding-Light(Dense+Sparse) | 73.13 | 56.31 | |
| | MiniCPM-Embedding-Light(Dense+Sparse)+MiniCPM-Reranker-Light | 76.34 | 61.49 | |
|
|
|
|
| ### 中英跨语言检索结果 CN-EN Cross-lingual Retrieval Results |
|
|
| | 模型 Model | MKQA En-Zh_CN (Recall@20) | NeuCLIR22 (NDCG@10) | NeuCLIR23 (NDCG@10) | |
| |------------------------------|--------------------|--------------------|--------------------| |
| | me5-large | 44.3 | 9.01 | 25.33 | |
| | bge-m3(Dense) | 66.4 | 30.49 | 41.09 | |
| | gte-multilingual-base(Dense) | 68.2 | 39.46 | 45.86 | |
| | MiniCPM-Embedding | 72.95 | 52.65 | 49.95 | |
| | MiniCPM-Embedding-Light(Dense) | 68.29 | 41.17 | 45.83 | |
| | MiniCPM-Embedding-Light(Dense)+MiniCPM-Reranker-Light | 71.86 | 54.32 | 56.50 | |
| |
| ## 许可证 License |
| |
| - 本仓库中代码依照 [Apache-2.0 协议](https://github.com/openbmb/MiniCPM/blob/main/LICENSE)开源。 |
| - MiniCPM-Embedding-Light 模型权重的使用则需要遵循 [MiniCPM 模型协议](https://github.com/openbmb/MiniCPM/blob/main/MiniCPM%20Model%20License.md)。 |
| - MiniCPM-Embedding-Light 模型权重对学术研究完全开放。如需将模型用于商业用途，请填写[此问卷](https://modelbest.feishu.cn/share/base/form/shrcnpV5ZT9EJ6xYjh3Kx0J6v8g)。 |
| |
| * The code in this repo is released under the [Apache-2.0](https://github.com/openbmb/MiniCPM/blob/main/LICENSE) License. |
| * The usage of MiniCPM-Embedding-Light model weights must strictly follow [MiniCPM Model License.md](https://github.com/openbmb/MiniCPM/blob/main/MiniCPM%20Model%20License.md). |
| * The models and weights of MiniCPM-Embedding-Light are completely free for academic research. After filling out a ["questionnaire"](https://modelbest.feishu.cn/share/base/form/shrcnpV5ZT9EJ6xYjh3Kx0J6v8g) for registration, MiniCPM-Embedding-Light weights are also available for free commercial use. |
| |
