| ---
|
| language:
|
| - en
|
| license: mit
|
| tags:
|
| - sentence-transformers
|
| - multi-vector
|
| - colbert
|
| - late-interaction
|
| - generated_from_trainer
|
| - dataset_size:501907
|
| - loss:MultiVectorMultipleNegativesRankingLoss
|
| base_model: prajjwal1/bert-tiny
|
| widget:
|
| - text: 'Kroger Pharmacy - Keller 976 Keller Pkwy, Keller TX 76248 Phone Number: (817)
|
| 431-5178'
|
| - text: Moyie Springs, Idaho. Moyie Springs is a city in Boundary County, Idaho, United
|
| States. The population was 718 at the 2010 census.
|
| - text: cunningham funeral home in colbert ok
|
| - text: A.O. Smith stock price target raised to $60 from $58 at Boenning & Scattergood.
|
| 8:26 a.m. July 26, 2017 - Tomi Kilgore
|
| - text: 'There are 140 calories in a 4 pieces serving of Kirkland Signature Four Cheese
|
| Ravioli. Calorie breakdown: 45% fat, 31% carbs, 24% protein.'
|
| datasets:
|
| - sentence-transformers/msmarco-bm25
|
| pipeline_tag: feature-extraction
|
| library_name: sentence-transformers
|
| metrics:
|
| - maxsim_accuracy@1
|
| - maxsim_accuracy@3
|
| - maxsim_accuracy@5
|
| - maxsim_accuracy@10
|
| - maxsim_precision@1
|
| - maxsim_precision@3
|
| - maxsim_precision@5
|
| - maxsim_precision@10
|
| - maxsim_recall@1
|
| - maxsim_recall@3
|
| - maxsim_recall@5
|
| - maxsim_recall@10
|
| - maxsim_ndcg@10
|
| - maxsim_mrr@10
|
| - maxsim_map@100
|
| model-index:
|
| - name: BERT tiny multi-vector encoder trained on MS MARCO
|
| results:
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoMSMARCO
|
| type: NanoMSMARCO
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.16
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.32
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.42
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.7
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.16
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.10666666666666666
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.084
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.07
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.16
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.32
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.42
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.7
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.3858968432351718
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.292015873015873
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3048587220806822
|
| name: Maxsim Map@100
|
| - type: maxsim_accuracy@1
|
| value: 0.16
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.32
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.42
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.7
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.16
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.10666666666666666
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.084
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.07
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.16
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.32
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.42
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.7
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.3858968432351718
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.292015873015873
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3048587220806822
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoNQ
|
| type: NanoNQ
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.28
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.4
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.48
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.66
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.28
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.13333333333333333
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.09600000000000002
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.066
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.27
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.39
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.46
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.61
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.42995107279160477
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.3832698412698412
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3841358170885305
|
| name: Maxsim Map@100
|
| - type: maxsim_accuracy@1
|
| value: 0.28
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.4
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.48
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.66
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.28
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.13333333333333333
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.09600000000000002
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.066
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.27
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.39
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.46
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.61
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.42995107279160477
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.3832698412698412
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3841358170885305
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoFiQA2018
|
| type: NanoFiQA2018
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.26
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.4
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.48
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.58
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.26
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.18
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.132
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.08
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.12285714285714285
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.251047619047619
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.3117142857142857
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.38704761904761903
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.3027040168258662
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.36041269841269835
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.24465363135397997
|
| name: Maxsim Map@100
|
| - type: maxsim_accuracy@1
|
| value: 0.26
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.4
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.48
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.58
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.26
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.18
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.132
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.08
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.12285714285714285
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.251047619047619
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.3117142857142857
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.38704761904761903
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.3027040168258662
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.36041269841269835
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.24465363135397997
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-nano-beir
|
| name: Multi Vector Nano BEIR
|
| dataset:
|
| name: NanoBEIR mean
|
| type: NanoBEIR_mean
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.23333333333333336
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.37333333333333335
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.45999999999999996
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.6466666666666666
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.23333333333333336
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.13999999999999999
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.10400000000000002
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.07200000000000001
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.1842857142857143
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.32034920634920633
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.3972380952380952
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.5656825396825397
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.37285064428421427
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.34523280423280417
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.31121605684106424
|
| name: Maxsim Map@100
|
| - type: maxsim_accuracy@1
|
| value: 0.40367346938775506
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.5673469387755101
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.6259654631083202
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.7445839874411302
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.40367346938775506
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.2545368916797488
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.1988320251177394
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.14285400313971744
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.2296541932962195
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.3527886883553923
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.4044498317393773
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.5012192071371501
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.44684493129737013
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.506674777603349
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.37732753591032614
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoClimateFEVER
|
| type: NanoClimateFEVER
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.18
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.3
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.34
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.5
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.18
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.1
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.07600000000000001
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.05800000000000001
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.09166666666666667
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.12999999999999998
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.16
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.2383333333333333
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.19191034232336726
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.2662698412698412
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.14946591363353165
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoDBPedia
|
| type: NanoDBPedia
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.62
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.78
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.84
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.94
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.62
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.4733333333333333
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.44800000000000006
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.39199999999999996
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.052136771709552124
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.11703776658357738
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.16684861747261473
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.2876252581653851
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.4820307033364284
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.7217380952380953
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3543191098253603
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoFEVER
|
| type: NanoFEVER
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.56
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.72
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.82
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.86
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.56
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.24666666666666665
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.172
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.092
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.5266666666666667
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.6766666666666667
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.7833333333333333
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.8233333333333333
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.6792624024637341
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.6485555555555556
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.6348848591340011
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoHotpotQA
|
| type: NanoHotpotQA
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.72
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.9
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.92
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.96
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.72
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.41333333333333333
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.268
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.148
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.36
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.62
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.67
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.74
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.6839586445702376
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.8070238095238095
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.6020088740548019
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoNFCorpus
|
| type: NanoNFCorpus
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.42
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.52
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.52
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.6
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.42
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.32666666666666666
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.26799999999999996
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.22399999999999998
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.044696247294946014
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.07181046561572595
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.0834824676415163
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.10608315478868971
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.28518605272369923
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.4765238095238095
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.12589300813746968
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoQuoraRetrieval
|
| type: NanoQuoraRetrieval
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.74
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.88
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.9
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.92
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.74
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.35999999999999993
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.22799999999999998
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.12
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.654
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.8586666666666667
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.8859999999999999
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.9126666666666666
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.8354929187617376
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.8162222222222222
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.8099312372179969
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoSCIDOCS
|
| type: NanoSCIDOCS
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.26
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.42
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.52
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.74
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.26
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.18
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.15200000000000002
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.11800000000000001
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.054000000000000006
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.11000000000000001
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.15400000000000003
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.23999999999999996
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.22168572688545177
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.38710317460317456
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.15605007993528686
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoArguAna
|
| type: NanoArguAna
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.18
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.38
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.42
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.56
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.18
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.12666666666666665
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.084
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.05600000000000001
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.18
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.38
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.42
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.56
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.3539147678833996
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.29019047619047617
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.3044243083606632
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoSciFact
|
| type: NanoSciFact
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.48
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.58
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.6
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.68
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.48
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.21333333333333332
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.136
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.078
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.445
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.565
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.59
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.67
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.564998292690912
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.5397142857142857
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.5383929411495326
|
| name: Maxsim Map@100
|
| - task:
|
| type: multi-vector-information-retrieval
|
| name: Multi Vector Information Retrieval
|
| dataset:
|
| name: NanoTouche2020
|
| type: NanoTouche2020
|
| metrics:
|
| - type: maxsim_accuracy@1
|
| value: 0.3877551020408163
|
| name: Maxsim Accuracy@1
|
| - type: maxsim_accuracy@3
|
| value: 0.7755102040816326
|
| name: Maxsim Accuracy@3
|
| - type: maxsim_accuracy@5
|
| value: 0.8775510204081632
|
| name: Maxsim Accuracy@5
|
| - type: maxsim_accuracy@10
|
| value: 0.9795918367346939
|
| name: Maxsim Accuracy@10
|
| - type: maxsim_precision@1
|
| value: 0.3877551020408163
|
| name: Maxsim Precision@1
|
| - type: maxsim_precision@3
|
| value: 0.44897959183673464
|
| name: Maxsim Precision@3
|
| - type: maxsim_precision@5
|
| value: 0.4408163265306122
|
| name: Maxsim Precision@5
|
| - type: maxsim_precision@10
|
| value: 0.35510204081632657
|
| name: Maxsim Precision@10
|
| - type: maxsim_recall@1
|
| value: 0.024481017655879133
|
| name: Maxsim Recall@1
|
| - type: maxsim_recall@3
|
| value: 0.09602376403984346
|
| name: Maxsim Recall@3
|
| - type: maxsim_recall@5
|
| value: 0.15246910845015463
|
| name: Maxsim Recall@5
|
| - type: maxsim_recall@10
|
| value: 0.24076032744792322
|
| name: Maxsim Recall@10
|
| - type: maxsim_ndcg@10
|
| value: 0.39199232237420195
|
| name: Maxsim Ndcg@10
|
| - type: maxsim_mrr@10
|
| value: 0.5977324263038547
|
| name: Maxsim Mrr@10
|
| - type: maxsim_map@100
|
| value: 0.2962394648624034
|
| name: Maxsim Map@100
|
| ---
|
|
|
|
|
| # BERT tiny multi-vector encoder trained on MS MARCO
|
|
|
| This is a [Multi-Vector Encoder](https://www.sbert.net/docs/multi_vector_encoder/usage/usage.html) model finetuned in two stages from [prajjwal1/bert-tiny](https://huggingface.co/prajjwal1/bert-tiny) on the [msmarco-bm25](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25) dataset using the [sentence-transformers](https://www.SBERT.net) library. It maps inputs to sequences of 128-dimensional token-level vectors and scores them with late interaction (MaxSim), useful for semantic search with late interaction.
|
| ## Model Details
|
|
|
| ### Model Description
|
| - **Model Type:** Multi-Vector Encoder
|
| - **Base model:** [prajjwal1/bert-tiny](https://huggingface.co/prajjwal1/bert-tiny)
|
| - **Maximum Sequence Length:** 512 tokens
|
| - **Maximum Query Length:** 32 tokens
|
| - **Maximum Document Length:** 256 tokens
|
| - **Output Dimensionality:** 128 dimensions
|
| - **Similarity Function:** MaxSim
|
| - **Supported Modality:** Text
|
| - **Training Dataset:**
|
| - [msmarco-bm25](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25)
|
| - **Language:** en
|
| - **License:** mit
|
|
|
| ### Model Sources
|
|
|
| - **Documentation:** [Sentence Transformers Documentation](https://sbert.net)
|
| - **Documentation:** [Multi-Vector Encoder Documentation](https://www.sbert.net/docs/multi_vector_encoder/usage/usage.html)
|
| - **Repository:** [Sentence Transformers on GitHub](https://github.com/huggingface/sentence-transformers)
|
| - **Hugging Face:** [Multi-Vector Encoders on Hugging Face](https://huggingface.co/models?library=sentence-transformers&other=multi-vector)
|
|
|
| ### Full Model Architecture
|
|
|
| ```
|
| MultiVectorEncoder(
|
| (0): Transformer({'transformer_task': 'feature-extraction', 'modality_config': {'text': {'method': 'forward', 'method_output_name': 'last_hidden_state'}}, 'module_output_name': 'token_embeddings', 'query_length': 32, 'document_length': 256, 'query_expansion': {'strategy': 'min', 'attend': False, 'token': None, 'length': 32}, 'architecture': 'BertModel'})
|
| (1): Dense({'in_features': 128, 'out_features': 128, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity', 'module_input_name': 'token_embeddings', 'module_output_name': 'token_embeddings'})
|
| (2): MultiVectorMask({'skiplist_words': [], 'skiplist_tasks': ['document'], 'keep_only_token_ids': None})
|
| (3): Normalize({'module_input_name': 'token_embeddings', 'module_output_name': 'token_embeddings'})
|
| )
|
| ```
|
|
|
| ## Usage
|
|
|
| ### Direct Usage (Sentence Transformers)
|
|
|
| First install the Sentence Transformers library:
|
|
|
| ```bash
|
| pip install -U sentence-transformers
|
| ```
|
| Then you can load this model and run inference.
|
| ```python
|
| from sentence_transformers import MultiVectorEncoder
|
|
|
| # Download from the 🤗 Hub
|
| model = MultiVectorEncoder("multi-vector-encoder-testing/bert-tiny-multi-vector")
|
| # Run inference: each input becomes a sequence of per-token vectors (variable length).
|
| queries = [
|
| 'calories in kirkland ravioli',
|
| ]
|
| documents = [
|
| 'There are 140 calories in a 4 pieces serving of Kirkland Signature Four Cheese Ravioli. Calorie breakdown: 45% fat, 31% carbs, 24% protein.',
|
| 'There are 140 calories in a 4 pieces serving of Kirkland Signature Four Cheese Ravioli. Calorie breakdown: 45% fat, 31% carbs, 24% protein.',
|
| 'Current Local Time: Cleveland, Ohio is in the Eastern Time Zone: The Current Time in Cleveland, Ohio is: Thursday 1/18/2018 10:41 PM EST Cleveland, Ohio is in the Eastern Time Zone',
|
| ]
|
| query_embeddings = model.encode_query(queries)
|
| document_embeddings = model.encode_document(documents)
|
| print(query_embeddings[0].shape, document_embeddings[0].shape)
|
| # (32, 128) (39, 128)
|
|
|
| # Get the MaxSim similarity scores
|
| similarities = model.similarity(query_embeddings, document_embeddings)
|
| print(similarities)
|
| # tensor([[25.4110, 25.4110, 8.3390]])
|
| ```
|
| <!--
|
| ### Direct Usage (Transformers)
|
|
|
| <details><summary>Click to see the direct usage in Transformers</summary>
|
|
|
| </details>
|
| -->
|
|
|
| <!--
|
| ### Downstream Usage (Sentence Transformers)
|
|
|
| You can finetune this model on your own dataset.
|
|
|
| <details><summary>Click to expand</summary>
|
|
|
| </details>
|
| -->
|
|
|
| <!--
|
| ### Out-of-Scope Use
|
|
|
| *List how the model may foreseeably be misused and address what users ought not to do with the model.*
|
| -->
|
|
|
| ## Evaluation
|
|
|
| ### Metrics
|
|
|
| #### Multi Vector Information Retrieval
|
|
|
| * Datasets: `NanoMSMARCO`, `NanoNQ`, `NanoFiQA2018`, `NanoClimateFEVER`, `NanoDBPedia`, `NanoFEVER`, `NanoFiQA2018`, `NanoHotpotQA`, `NanoMSMARCO`, `NanoNFCorpus`, `NanoNQ`, `NanoQuoraRetrieval`, `NanoSCIDOCS`, `NanoArguAna`, `NanoSciFact` and `NanoTouche2020`
|
| * Evaluated with [<code>MultiVectorInformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/multi_vector_encoder/evaluation.html#sentence_transformers.multi_vector_encoder.evaluation.MultiVectorInformationRetrievalEvaluator)
|
|
|
| | Metric | NanoMSMARCO | NanoNQ | NanoFiQA2018 | NanoClimateFEVER | NanoDBPedia | NanoFEVER | NanoHotpotQA | NanoNFCorpus | NanoQuoraRetrieval | NanoSCIDOCS | NanoArguAna | NanoSciFact | NanoTouche2020 |
|
| |:--------------------|:------------|:---------|:-------------|:-----------------|:------------|:-----------|:-------------|:-------------|:-------------------|:------------|:------------|:------------|:---------------|
|
| | maxsim_accuracy@1 | 0.16 | 0.28 | 0.26 | 0.18 | 0.62 | 0.56 | 0.72 | 0.42 | 0.74 | 0.26 | 0.18 | 0.48 | 0.3878 |
|
| | maxsim_accuracy@3 | 0.32 | 0.4 | 0.4 | 0.3 | 0.78 | 0.72 | 0.9 | 0.52 | 0.88 | 0.42 | 0.38 | 0.58 | 0.7755 |
|
| | maxsim_accuracy@5 | 0.42 | 0.48 | 0.48 | 0.34 | 0.84 | 0.82 | 0.92 | 0.52 | 0.9 | 0.52 | 0.42 | 0.6 | 0.8776 |
|
| | maxsim_accuracy@10 | 0.7 | 0.66 | 0.58 | 0.5 | 0.94 | 0.86 | 0.96 | 0.6 | 0.92 | 0.74 | 0.56 | 0.68 | 0.9796 |
|
| | maxsim_precision@1 | 0.16 | 0.28 | 0.26 | 0.18 | 0.62 | 0.56 | 0.72 | 0.42 | 0.74 | 0.26 | 0.18 | 0.48 | 0.3878 |
|
| | maxsim_precision@3 | 0.1067 | 0.1333 | 0.18 | 0.1 | 0.4733 | 0.2467 | 0.4133 | 0.3267 | 0.36 | 0.18 | 0.1267 | 0.2133 | 0.449 |
|
| | maxsim_precision@5 | 0.084 | 0.096 | 0.132 | 0.076 | 0.448 | 0.172 | 0.268 | 0.268 | 0.228 | 0.152 | 0.084 | 0.136 | 0.4408 |
|
| | maxsim_precision@10 | 0.07 | 0.066 | 0.08 | 0.058 | 0.392 | 0.092 | 0.148 | 0.224 | 0.12 | 0.118 | 0.056 | 0.078 | 0.3551 |
|
| | maxsim_recall@1 | 0.16 | 0.27 | 0.1229 | 0.0917 | 0.0521 | 0.5267 | 0.36 | 0.0447 | 0.654 | 0.054 | 0.18 | 0.445 | 0.0245 |
|
| | maxsim_recall@3 | 0.32 | 0.39 | 0.251 | 0.13 | 0.117 | 0.6767 | 0.62 | 0.0718 | 0.8587 | 0.11 | 0.38 | 0.565 | 0.096 |
|
| | maxsim_recall@5 | 0.42 | 0.46 | 0.3117 | 0.16 | 0.1668 | 0.7833 | 0.67 | 0.0835 | 0.886 | 0.154 | 0.42 | 0.59 | 0.1525 |
|
| | maxsim_recall@10 | 0.7 | 0.61 | 0.387 | 0.2383 | 0.2876 | 0.8233 | 0.74 | 0.1061 | 0.9127 | 0.24 | 0.56 | 0.67 | 0.2408 |
|
| | **maxsim_ndcg@10** | **0.3859** | **0.43** | **0.3027** | **0.1919** | **0.482** | **0.6793** | **0.684** | **0.2852** | **0.8355** | **0.2217** | **0.3539** | **0.565** | **0.392** |
|
| | maxsim_mrr@10 | 0.292 | 0.3833 | 0.3604 | 0.2663 | 0.7217 | 0.6486 | 0.807 | 0.4765 | 0.8162 | 0.3871 | 0.2902 | 0.5397 | 0.5977 |
|
| | maxsim_map@100 | 0.3049 | 0.3841 | 0.2447 | 0.1495 | 0.3543 | 0.6349 | 0.602 | 0.1259 | 0.8099 | 0.1561 | 0.3044 | 0.5384 | 0.2962 |
|
|
|
| #### Multi Vector Nano BEIR
|
|
|
| * Dataset: `NanoBEIR_mean`
|
| * Evaluated with [<code>MultiVectorNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/multi_vector_encoder/evaluation.html#sentence_transformers.multi_vector_encoder.evaluation.MultiVectorNanoBEIREvaluator) with these parameters:
|
| ```json
|
| {
|
| "dataset_names": [
|
| "msmarco",
|
| "nq",
|
| "fiqa2018"
|
| ],
|
| "dataset_id": "sentence-transformers/NanoBEIR-en"
|
| }
|
| ```
|
|
|
| | Metric | Value |
|
| |:--------------------|:-----------|
|
| | maxsim_accuracy@1 | 0.2333 |
|
| | maxsim_accuracy@3 | 0.3733 |
|
| | maxsim_accuracy@5 | 0.46 |
|
| | maxsim_accuracy@10 | 0.6467 |
|
| | maxsim_precision@1 | 0.2333 |
|
| | maxsim_precision@3 | 0.14 |
|
| | maxsim_precision@5 | 0.104 |
|
| | maxsim_precision@10 | 0.072 |
|
| | maxsim_recall@1 | 0.1843 |
|
| | maxsim_recall@3 | 0.3203 |
|
| | maxsim_recall@5 | 0.3972 |
|
| | maxsim_recall@10 | 0.5657 |
|
| | **maxsim_ndcg@10** | **0.3729** |
|
| | maxsim_mrr@10 | 0.3452 |
|
| | maxsim_map@100 | 0.3112 |
|
|
|
| #### Multi Vector Nano BEIR
|
|
|
| * Dataset: `NanoBEIR_mean`
|
| * Evaluated with [<code>MultiVectorNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/multi_vector_encoder/evaluation.html#sentence_transformers.multi_vector_encoder.evaluation.MultiVectorNanoBEIREvaluator) with these parameters:
|
| ```json
|
| {
|
| "dataset_names": [
|
| "climatefever",
|
| "dbpedia",
|
| "fever",
|
| "fiqa2018",
|
| "hotpotqa",
|
| "msmarco",
|
| "nfcorpus",
|
| "nq",
|
| "quoraretrieval",
|
| "scidocs",
|
| "arguana",
|
| "scifact",
|
| "touche2020"
|
| ],
|
| "dataset_id": "sentence-transformers/NanoBEIR-en"
|
| }
|
| ```
|
|
|
| | Metric | Value |
|
| |:--------------------|:-----------|
|
| | maxsim_accuracy@1 | 0.4037 |
|
| | maxsim_accuracy@3 | 0.5673 |
|
| | maxsim_accuracy@5 | 0.626 |
|
| | maxsim_accuracy@10 | 0.7446 |
|
| | maxsim_precision@1 | 0.4037 |
|
| | maxsim_precision@3 | 0.2545 |
|
| | maxsim_precision@5 | 0.1988 |
|
| | maxsim_precision@10 | 0.1429 |
|
| | maxsim_recall@1 | 0.2297 |
|
| | maxsim_recall@3 | 0.3528 |
|
| | maxsim_recall@5 | 0.4044 |
|
| | maxsim_recall@10 | 0.5012 |
|
| | **maxsim_ndcg@10** | **0.4468** |
|
| | maxsim_mrr@10 | 0.5067 |
|
| | maxsim_map@100 | 0.3773 |
|
|
|
| <!--
|
| ## Bias, Risks and Limitations
|
|
|
| *What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.*
|
| -->
|
|
|
| <!--
|
| ### Recommendations
|
|
|
| *What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.*
|
| -->
|
|
|
| ## Training Details
|
|
|
| ### Training Dataset
|
|
|
| #### msmarco-bm25
|
|
|
| * Dataset: [msmarco-bm25](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25) at [ce8a493](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25/tree/ce8a493a65af5e872c3c92f72a89e2e99e175f02)
|
| * Size: 501,907 training samples
|
| * Columns: <code>query</code>, <code>positive</code>, and <code>negative</code>
|
| * Approximate statistics based on the first 100 samples:
|
| | | query | positive | negative |
|
| |:---------|:---------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
|
| | type | string | string | string |
|
| | modality | text | text | text |
|
| | details | <ul><li>min: 4 tokens</li><li>mean: 8.81 tokens</li><li>max: 16 tokens</li></ul> | <ul><li>min: 25 tokens</li><li>mean: 78.08 tokens</li><li>max: 174 tokens</li></ul> | <ul><li>min: 22 tokens</li><li>mean: 76.59 tokens</li><li>max: 180 tokens</li></ul> |
|
| * Samples:
|
| | query | positive | negative |
|
| |:-------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
| | <code>sociopath define</code> | <code>Updated September 08, 2016. Both psychopaths and sociopaths are defined as someone who is suffering from Antisocial Personality Disorder. Both groups show a pervasive pattern of disregard for the rights and feelings of others. There are, however, subtle differences between the two groups.</code> | <code>Define sociopath. sociopath synonyms, sociopath pronunciation, sociopath translation, English dictionary definition of sociopath. n. A psychopath or a person with antisocial personality disorder. soâ²ci·o·pathâ²ic adj. soâ²ci·opâ²a·thy n. n psychiatry another name for psychopath...</code> |
|
| | <code>what county is tarrytown ny in</code> | <code>ABOUT US. The Music Hall, an 1885 landmark in Tarrytown, NY is Westchester County's oldest theater and one of the region's busiest music venues, welcoming 85,000 visitors every year, including tens of thousands of children. Please complete all required fields!</code> | <code>Tarrytown, NY Other Information. 1 Located in WESTCHESTER County, New York. 2 Tarrytown, NY is also known as: 3 N TARRYTOWN, NY. NORTH TARRYTOWN, 1 NY. PHILIPSE MANOR, 2 NY. POCANTICO HILLS, 3 NY. SLEEPY HOLLOW, NY. SLEEPY HOLLOW MANOR, NY.</code> |
|
| | <code>what temperature do you grill a t-bone at</code> | <code>Step 2. Move your T-bones to the medium heat side of your grill and continue grilling. If you like your T-bone medium rare, grill for four to five minutes on each side or until a meat thermometer reads 130 to 140 degrees Fahrenheit.For a medium steak, grill six to seven minutes per side or until a meat thermometer reads 140 to 150 degrees.iming and technique are the keys to grilling a 1-inch-thick T-bone steak to perfection. This thicker cut requires different treatment from a T-bone, for example, 1/2- to 3/4-inch thick.</code> | <code>Preheat your grill using two temperature settings. If you are using a gas grill, set one side to high and the other to a medium setting, then close the lid for 10 to 15 minutes.iming and technique are the keys to grilling a 1-inch-thick T-bone steak to perfection. This thicker cut requires different treatment from a T-bone, for example, 1/2- to 3/4-inch thick.</code> |
|
| * Loss: [<code>MultiVectorMultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/multi_vector_encoder/losses.html#multivectormultiplenegativesrankingloss) with these parameters:
|
| ```json
|
| {
|
| "scale": 1.0,
|
| "similarity_fct": "colbert_scores",
|
| "mini_batch_size": null,
|
| "score_mini_batch_size": null,
|
| "gather_across_devices": false
|
| }
|
| ```
|
|
|
| ### Evaluation Dataset
|
|
|
| #### msmarco-bm25
|
|
|
| * Dataset: [msmarco-bm25](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25) at [ce8a493](https://huggingface.co/datasets/sentence-transformers/msmarco-bm25/tree/ce8a493a65af5e872c3c92f72a89e2e99e175f02)
|
| * Size: 1,024 evaluation samples
|
| * Columns: <code>query</code>, <code>positive</code>, and <code>negative</code>
|
| * Approximate statistics based on the first 100 samples:
|
| | | query | positive | negative |
|
| |:---------|:---------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
|
| | type | string | string | string |
|
| | modality | text | text | text |
|
| | details | <ul><li>min: 4 tokens</li><li>mean: 8.65 tokens</li><li>max: 15 tokens</li></ul> | <ul><li>min: 21 tokens</li><li>mean: 80.76 tokens</li><li>max: 174 tokens</li></ul> | <ul><li>min: 16 tokens</li><li>mean: 79.88 tokens</li><li>max: 224 tokens</li></ul> |
|
| * Samples:
|
| | query | positive | negative |
|
| |:----------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
| | <code>what is the bottom lip piercing called</code> | <code>1 Standard Lip Piercing â This is a single piercing done off centered on the lower lip. 2 A captive bead ring (CBR) or labret stud can be worn. 3 Monroe Piercing â This is a single piercing done on the left side of the upper lip and is named for the mole on Marilyn Monroeâs lip. 4 Usually a labret stud is worn in this piercing.</code> | <code>Also known as lower-lip piercing or bottom lip piercing .The labret piercing is placed at the labrum (below bottom lip, above chin). Popular among men and women, this style looks super cool and trendy.</code> |
|
| | <code>what are the mind and body</code> | <code>This is known as dualism. Dualism is the view that the mind and body both exist. There are two basic types of dualism: o Descartes dualism: The view that the mind and body function separately, without interchange. o Cartesian dualism argues that there is a two-way interaction between mental and physical substances. Dualism is in contrast to monism that states the mind and body are the same thing.</code> | <code>Quotes About What Matters In Life. âWhat is in your mind position or disposition your mind, body and spirit in the best or worst way. What you are yet to accept into your mind exposes your mind to and keep your mind on what you are yet to accept and what has not yet come into your mind least controls your mind, body and spirit. Browse By Tag.</code> |
|
| | <code>what breed of dogs have green eyes</code> | <code>Best Answer: There are many dog breeds that CAN have green eyes, such as Australian Shepherds, Border Collies, Siberian Huskies, and others, but it is an uncommon occurrence. However it won't be a bright green like a cat's eye. It'll be a somewhat subdued shade of blueish-grey with green overtones.</code> | <code>What are some dog breeds that have or can have green eyes? What breed is this dog? What is the breed of a dog, which looks like a fox and has light blue eyes, called? Rohit Akut, love and respect animals be friendly have had lots of different pets..</code> |
|
| * Loss: [<code>MultiVectorMultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/multi_vector_encoder/losses.html#multivectormultiplenegativesrankingloss) with these parameters:
|
| ```json
|
| {
|
| "scale": 1.0,
|
| "similarity_fct": "colbert_scores",
|
| "mini_batch_size": null,
|
| "score_mini_batch_size": null,
|
| "gather_across_devices": false
|
| }
|
| ```
|
|
|
| ### Training Hyperparameters
|
| #### Non-Default Hyperparameters
|
|
|
| - `per_device_train_batch_size`: 128
|
| - `max_steps`: 10000
|
| - `learning_rate`: 1e-05
|
| - `warmup_steps`: 0.05
|
| - `weight_decay`: 0.01
|
| - `bf16`: True
|
| - `disable_tqdm`: True
|
| - `per_device_eval_batch_size`: 32
|
| - `load_best_model_at_end`: True
|
| - `seed`: 12
|
| - `batch_sampler`: no_duplicates
|
|
|
| #### All Hyperparameters
|
| <details><summary>Click to expand</summary>
|
|
|
| - `per_device_train_batch_size`: 128
|
| - `num_train_epochs`: 3.0
|
| - `max_steps`: 10000
|
| - `learning_rate`: 1e-05
|
| - `lr_scheduler_type`: linear
|
| - `lr_scheduler_kwargs`: None
|
| - `warmup_steps`: 0.05
|
| - `optim`: adamw_torch_fused
|
| - `optim_args`: None
|
| - `weight_decay`: 0.01
|
| - `adam_beta1`: 0.9
|
| - `adam_beta2`: 0.999
|
| - `adam_epsilon`: 1e-08
|
| - `optim_target_modules`: None
|
| - `gradient_accumulation_steps`: 1
|
| - `average_tokens_across_devices`: True
|
| - `max_grad_norm`: 1.0
|
| - `label_smoothing_factor`: 0.0
|
| - `bf16`: True
|
| - `fp16`: False
|
| - `bf16_full_eval`: False
|
| - `fp16_full_eval`: False
|
| - `tf32`: None
|
| - `gradient_checkpointing`: False
|
| - `gradient_checkpointing_kwargs`: None
|
| - `torch_compile`: False
|
| - `torch_compile_backend`: None
|
| - `torch_compile_mode`: None
|
| - `use_liger_kernel`: False
|
| - `liger_kernel_config`: None
|
| - `use_cache`: False
|
| - `neftune_noise_alpha`: None
|
| - `torch_empty_cache_steps`: None
|
| - `auto_find_batch_size`: False
|
| - `log_on_each_node`: True
|
| - `logging_nan_inf_filter`: True
|
| - `include_num_input_tokens_seen`: no
|
| - `log_level`: passive
|
| - `log_level_replica`: warning
|
| - `disable_tqdm`: True
|
| - `project`: huggingface
|
| - `trackio_space_id`: None
|
| - `trackio_bucket_id`: None
|
| - `trackio_static_space_id`: None
|
| - `per_device_eval_batch_size`: 32
|
| - `prediction_loss_only`: True
|
| - `eval_on_start`: False
|
| - `eval_do_concat_batches`: True
|
| - `eval_use_gather_object`: False
|
| - `eval_accumulation_steps`: None
|
| - `include_for_metrics`: []
|
| - `batch_eval_metrics`: False
|
| - `save_only_model`: False
|
| - `save_on_each_node`: False
|
| - `enable_jit_checkpoint`: False
|
| - `push_to_hub`: False
|
| - `hub_private_repo`: None
|
| - `hub_model_id`: None
|
| - `hub_strategy`: every_save
|
| - `hub_always_push`: False
|
| - `hub_revision`: None
|
| - `load_best_model_at_end`: True
|
| - `ignore_data_skip`: False
|
| - `restore_callback_states_from_checkpoint`: False
|
| - `full_determinism`: False
|
| - `seed`: 12
|
| - `data_seed`: None
|
| - `use_cpu`: False
|
| - `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
|
| - `parallelism_config`: None
|
| - `dataloader_drop_last`: False
|
| - `dataloader_num_workers`: 0
|
| - `dataloader_pin_memory`: True
|
| - `dataloader_persistent_workers`: False
|
| - `dataloader_prefetch_factor`: None
|
| - `dataloader_multiprocessing_context`: None
|
| - `dataloader_in_order`: True
|
| - `remove_unused_columns`: True
|
| - `label_names`: None
|
| - `train_sampling_strategy`: random
|
| - `length_column_name`: length
|
| - `ddp_find_unused_parameters`: None
|
| - `ddp_bucket_cap_mb`: None
|
| - `ddp_broadcast_buffers`: False
|
| - `ddp_static_graph`: None
|
| - `ddp_backend`: None
|
| - `ddp_timeout`: 1800
|
| - `fsdp`: None
|
| - `fsdp_config`: None
|
| - `deepspeed`: None
|
| - `debug`: []
|
| - `skip_memory_metrics`: True
|
| - `do_predict`: False
|
| - `resume_from_checkpoint`: None
|
| - `local_rank`: -1
|
| - `prompts`: None
|
| - `batch_sampler`: no_duplicates
|
| - `multi_dataset_batch_sampler`: proportional
|
| - `router_mapping`: {}
|
| - `learning_rate_mapping`: {}
|
| - `warmup_ratio`: None
|
| - `max_length`: None
|
|
|
| </details>
|
|
|
| ### Training Logs
|
| <details><summary>Click to expand</summary>
|
|
|
| | Epoch | Step | Training Loss | Validation Loss | NanoMSMARCO_maxsim_ndcg@10 | NanoNQ_maxsim_ndcg@10 | NanoFiQA2018_maxsim_ndcg@10 | NanoBEIR_mean_maxsim_ndcg@10 | NanoClimateFEVER_maxsim_ndcg@10 | NanoDBPedia_maxsim_ndcg@10 | NanoFEVER_maxsim_ndcg@10 | NanoHotpotQA_maxsim_ndcg@10 | NanoNFCorpus_maxsim_ndcg@10 | NanoQuoraRetrieval_maxsim_ndcg@10 | NanoSCIDOCS_maxsim_ndcg@10 | NanoArguAna_maxsim_ndcg@10 | NanoSciFact_maxsim_ndcg@10 | NanoTouche2020_maxsim_ndcg@10 |
|
| |:----------:|:--------:|:-------------:|:---------------:|:--------------------------:|:---------------------:|:---------------------------:|:----------------------------:|:-------------------------------:|:--------------------------:|:------------------------:|:---------------------------:|:---------------------------:|:---------------------------------:|:--------------------------:|:--------------------------:|:--------------------------:|:-----------------------------:|
|
| | -1 | 0 | - | - | 0.4328 | 0.2992 | 0.2658 | 0.3999 | 0.1431 | 0.3914 | 0.6310 | 0.5766 | 0.2569 | 0.7976 | 0.2029 | 0.2863 | 0.5060 | 0.4096 |
|
| | 0.0003 | 1 | 1.7377 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0127 | 50 | 1.7342 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0255 | 100 | 1.7067 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0382 | 150 | 1.6564 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0510 | 200 | 1.6343 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0637 | 250 | 1.6082 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0765 | 300 | 1.5952 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.0892 | 350 | 1.5606 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1020 | 400 | 1.5329 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1147 | 450 | 1.4977 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1275 | 500 | 1.5205 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1402 | 550 | 1.4602 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1530 | 600 | 1.4465 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1657 | 650 | 1.4251 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1785 | 700 | 1.3911 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.1912 | 750 | 1.3776 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2040 | 800 | 1.3530 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2167 | 850 | 1.3297 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2295 | 900 | 1.3348 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2422 | 950 | 1.3423 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2550 | 1000 | 1.2872 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2677 | 1050 | 1.2880 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2805 | 1100 | 1.2762 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.2932 | 1150 | 1.2545 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3060 | 1200 | 1.2463 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3187 | 1250 | 1.2492 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3315 | 1300 | 1.2145 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3442 | 1350 | 1.2369 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3570 | 1400 | 1.1995 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3697 | 1450 | 1.1950 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3825 | 1500 | 1.2016 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.3952 | 1550 | 1.1622 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4080 | 1600 | 1.1886 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4207 | 1650 | 1.1866 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4335 | 1700 | 1.1774 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4462 | 1750 | 1.1462 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4589 | 1800 | 1.1655 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4717 | 1850 | 1.1309 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4844 | 1900 | 1.1613 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.4972 | 1950 | 1.1465 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5099 | 2000 | 1.1741 | 0.8207 | 0.3460 | 0.4138 | 0.2792 | 0.3463 | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5227 | 2050 | 1.1287 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5354 | 2100 | 1.1325 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5482 | 2150 | 1.1121 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5609 | 2200 | 1.1046 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5737 | 2250 | 1.1026 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5864 | 2300 | 1.1245 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.5992 | 2350 | 1.1253 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6119 | 2400 | 1.1051 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6247 | 2450 | 1.0870 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6374 | 2500 | 1.0775 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6502 | 2550 | 1.0713 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6629 | 2600 | 1.0946 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6757 | 2650 | 1.0682 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.6884 | 2700 | 1.0537 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7012 | 2750 | 1.0500 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7139 | 2800 | 1.0920 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7267 | 2850 | 1.0873 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7394 | 2900 | 1.0598 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7522 | 2950 | 1.0745 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7649 | 3000 | 1.0587 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7777 | 3050 | 1.0532 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.7904 | 3100 | 1.0609 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8032 | 3150 | 1.0611 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8159 | 3200 | 1.0592 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8287 | 3250 | 1.0413 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8414 | 3300 | 1.0224 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8542 | 3350 | 1.0372 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8669 | 3400 | 1.0544 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8797 | 3450 | 0.9922 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.8924 | 3500 | 1.0215 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9052 | 3550 | 1.0052 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9179 | 3600 | 1.0061 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9306 | 3650 | 1.0136 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9434 | 3700 | 1.0236 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9561 | 3750 | 0.9937 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9689 | 3800 | 1.0128 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9816 | 3850 | 1.0399 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 0.9944 | 3900 | 0.9840 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0071 | 3950 | 0.9999 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0199 | 4000 | 1.0204 | 0.7238 | 0.3782 | 0.4405 | 0.2881 | 0.3690 | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0326 | 4050 | 1.0075 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0454 | 4100 | 1.0111 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0581 | 4150 | 0.9972 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0709 | 4200 | 1.0144 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0836 | 4250 | 1.0054 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.0964 | 4300 | 1.0123 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1091 | 4350 | 0.9976 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1219 | 4400 | 0.9798 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1346 | 4450 | 1.0313 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1474 | 4500 | 0.9913 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1601 | 4550 | 0.9911 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1729 | 4600 | 0.9880 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1856 | 4650 | 0.9834 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.1984 | 4700 | 0.9489 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2111 | 4750 | 0.9462 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2239 | 4800 | 0.9670 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2366 | 4850 | 0.9766 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2494 | 4900 | 0.9655 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2621 | 4950 | 0.9525 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2749 | 5000 | 0.9818 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.2876 | 5050 | 0.9699 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3004 | 5100 | 0.9679 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3131 | 5150 | 0.9810 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3259 | 5200 | 0.9723 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3386 | 5250 | 0.9660 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3514 | 5300 | 0.9688 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3641 | 5350 | 0.9701 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3768 | 5400 | 0.9532 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.3896 | 5450 | 0.9817 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4023 | 5500 | 0.9401 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4151 | 5550 | 0.9399 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4278 | 5600 | 0.9505 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4406 | 5650 | 0.9515 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4533 | 5700 | 0.9576 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4661 | 5750 | 0.9672 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4788 | 5800 | 0.9414 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.4916 | 5850 | 0.9355 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5043 | 5900 | 0.9615 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5171 | 5950 | 0.9436 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | **1.5298** | **6000** | **0.9647** | **0.6846** | **0.3859** | **0.4304** | **0.3025** | **0.373** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** |
|
| | 1.5426 | 6050 | 0.9528 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5553 | 6100 | 0.9359 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5681 | 6150 | 0.9409 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5808 | 6200 | 0.9534 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.5936 | 6250 | 0.9594 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6063 | 6300 | 0.9496 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6191 | 6350 | 0.9398 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6318 | 6400 | 0.9368 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6446 | 6450 | 0.9482 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6573 | 6500 | 0.9393 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6701 | 6550 | 0.9601 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6828 | 6600 | 0.9297 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.6956 | 6650 | 0.9263 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7083 | 6700 | 0.9391 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7211 | 6750 | 0.9424 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7338 | 6800 | 0.9357 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7466 | 6850 | 0.9470 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7593 | 6900 | 0.9315 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7721 | 6950 | 0.9289 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7848 | 7000 | 0.9206 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.7976 | 7050 | 0.9493 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8103 | 7100 | 0.9530 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8230 | 7150 | 0.9379 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8358 | 7200 | 0.9349 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8485 | 7250 | 0.9017 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8613 | 7300 | 0.9407 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8740 | 7350 | 0.9158 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8868 | 7400 | 0.9412 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.8995 | 7450 | 0.9240 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9123 | 7500 | 0.9258 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9250 | 7550 | 0.9456 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9378 | 7600 | 0.9428 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9505 | 7650 | 0.9630 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9633 | 7700 | 0.9452 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9760 | 7750 | 0.9222 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 1.9888 | 7800 | 0.9288 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0015 | 7850 | 0.9349 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0143 | 7900 | 0.9355 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0270 | 7950 | 0.9041 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0398 | 8000 | 0.9093 | 0.6678 | 0.3805 | 0.4371 | 0.2995 | 0.3724 | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0525 | 8050 | 0.9139 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0653 | 8100 | 0.9533 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0780 | 8150 | 0.9314 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.0908 | 8200 | 0.9206 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1035 | 8250 | 0.9299 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1163 | 8300 | 0.9174 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1290 | 8350 | 0.9039 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1418 | 8400 | 0.9092 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1545 | 8450 | 0.9086 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1673 | 8500 | 0.9605 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1800 | 8550 | 0.9296 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.1928 | 8600 | 0.9196 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2055 | 8650 | 0.9146 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2183 | 8700 | 0.9130 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2310 | 8750 | 0.9037 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2438 | 8800 | 0.9357 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2565 | 8850 | 0.9313 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2693 | 8900 | 0.9179 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2820 | 8950 | 0.9597 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.2947 | 9000 | 0.9274 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3075 | 9050 | 0.9092 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3202 | 9100 | 0.9072 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3330 | 9150 | 0.9166 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3457 | 9200 | 0.8974 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3585 | 9250 | 0.9166 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3712 | 9300 | 0.9281 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3840 | 9350 | 0.9292 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.3967 | 9400 | 0.9166 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4095 | 9450 | 0.9265 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4222 | 9500 | 0.9142 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4350 | 9550 | 0.9131 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4477 | 9600 | 0.8902 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4605 | 9650 | 0.9464 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4732 | 9700 | 0.9167 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4860 | 9750 | 0.9122 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.4987 | 9800 | 0.9217 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.5115 | 9850 | 0.9245 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.5242 | 9900 | 0.8963 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.5370 | 9950 | 0.9256 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
|
| | 2.5497 | 10000 | 0.9121 | 0.6630 | 0.3796 | 0.4370 | 0.3010 | 0.3725 | - | - | - | - | - | - | - | - | - | - |
|
| | -1 | -1 | - | - | 0.3859 | 0.4300 | 0.3027 | 0.4468 | 0.1919 | 0.4820 | 0.6793 | 0.6840 | 0.2852 | 0.8355 | 0.2217 | 0.3539 | 0.5650 | 0.3920 |
|
|
|
| * The bold row denotes the saved checkpoint.
|
| </details>
|
|
|
| ### Training Time
|
| - **Training**: 15.5 minutes
|
| - **Evaluation**: 19.4 seconds
|
| - **Total**: 15.9 minutes
|
|
|
| ### Framework Versions
|
| - Python: 3.11.6
|
| - Sentence Transformers: 6.1.0.dev0
|
| - Transformers: 5.16.1
|
| - PyTorch: 2.10.0+cu128
|
| - Accelerate: 1.14.0
|
| - Datasets: 4.8.4
|
| - Tokenizers: 0.23.2
|
|
|
| ## Citation
|
|
|
| ### BibTeX
|
|
|
| #### Sentence Transformers
|
| ```bibtex
|
| @inproceedings{reimers-2019-sentence-bert,
|
| title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
|
| author = "Reimers, Nils and Gurevych, Iryna",
|
| booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
|
| month = "11",
|
| year = "2019",
|
| publisher = "Association for Computational Linguistics",
|
| url = "https://arxiv.org/abs/1908.10084",
|
| }
|
| ```
|
|
|
| #### MultiVectorMultipleNegativesRankingLoss
|
| ```bibtex
|
| @misc{henderson2017efficient,
|
| title={Efficient Natural Language Response Suggestion for Smart Reply},
|
| author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil},
|
| year={2017},
|
| eprint={1705.00652},
|
| archivePrefix={arXiv},
|
| primaryClass={cs.CL}
|
| }
|
| ```
|
|
|
| <!--
|
| ## Glossary
|
|
|
| *Clearly define terms in order to be accessible across audiences.*
|
| -->
|
|
|
| <!--
|
| ## Model Card Authors
|
|
|
| *Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.*
|
| -->
|
|
|
| <!--
|
| ## Model Card Contact
|
|
|
| *Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.*
|
| --> |
