README.md

metadata

tags:
  - ColBERT
  - PyLate
  - sentence-transformers
  - sentence-similarity
  - feature-extraction
  - generated_from_trainer
  - dataset_size:10000000
  - loss:Contrastive
base_model: answerdotai/ModernBERT-base
datasets:
  - bclavie/msmarco-10m-triplets
pipeline_tag: sentence-similarity
library_name: PyLate
metrics:
  - MaxSim_accuracy@1
  - MaxSim_accuracy@3
  - MaxSim_accuracy@5
  - MaxSim_accuracy@10
  - MaxSim_precision@1
  - MaxSim_precision@3
  - MaxSim_precision@5
  - MaxSim_precision@10
  - MaxSim_recall@1
  - MaxSim_recall@3
  - MaxSim_recall@5
  - MaxSim_recall@10
  - MaxSim_ndcg@10
  - MaxSim_mrr@10
  - MaxSim_map@100
model-index:
  - name: PyLate model based on answerdotai/ModernBERT-base
    results:
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoClimateFEVER
          type: NanoClimateFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.24
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.5
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.56
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.72
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.24
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.18
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.136
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.09
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.12
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.23166666666666663
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.26733333333333337
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3456666666666666
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.28659074634439036
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.38899206349206344
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.22237332831909778
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoDBPedia
          type: NanoDBPedia
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.72
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.88
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.92
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.92
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.72
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.6333333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.608
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.514
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.08698943062762281
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.15785394286086044
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.25560775585402196
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.35140662490283503
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6310474103595507
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8023333333333333
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4766459270446468
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFEVER
          type: NanoFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.84
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.98
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 1
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.84
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.33333333333333326
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.20799999999999996
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.10599999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.7866666666666667
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.9266666666666667
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.9566666666666667
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.9633333333333333
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.8973997659405684
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.905
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.8618368368368369
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFiQA2018
          type: NanoFiQA2018
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.46
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.66
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.74
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.78
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.46
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.32666666666666666
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.244
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.14
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.2685793650793651
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.47251587301587306
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.5722857142857143
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.6081190476190477
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5290995192750131
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.579
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4613817582717077
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoHotpotQA
          type: NanoHotpotQA
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.88
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.96
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.96
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.98
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.88
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.49999999999999983
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.32799999999999996
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.17199999999999996
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.44
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.75
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.82
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.86
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.8190215640428958
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.9195238095238095
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.7542864635927516
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoMSMARCO
          type: NanoMSMARCO
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.46
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.74
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.76
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.86
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.46
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.24666666666666665
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.15200000000000002
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.08599999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.46
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.74
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.76
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.86
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6683749348060735
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.6068333333333333
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.6153476502857308
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNFCorpus
          type: NanoNFCorpus
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.46
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.52
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.54
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.56
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.46
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.3533333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.29600000000000004
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.23600000000000002
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.043124364328590487
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.07617110540472594
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.0888523411805491
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.10700228981811344
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.30567453687850876
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.494
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.13042780558280634
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNQ
          type: NanoNQ
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.38
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.66
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.78
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.9
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.38
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.22
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.15600000000000003
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.092
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.37
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.62
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.73
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.84
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6117132890493197
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.5528888888888889
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.5368699134199134
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoQuoraRetrieval
          type: NanoQuoraRetrieval
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.9
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.98
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 1
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.9
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.3933333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.24799999999999997
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.13599999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.7973333333333333
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.9386666666666668
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.9593333333333334
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.9933333333333334
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.9500644303763519
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.945
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.9287460317460318
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSCIDOCS
          type: NanoSCIDOCS
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.42
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.62
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.7
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.82
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.42
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.30666666666666664
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.24
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.172
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.08866666666666667
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.19066666666666662
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.24666666666666665
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3526666666666666
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.34178587115776565
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.5458571428571429
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.254228684034207
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoArguAna
          type: NanoArguAna
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.14
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.48
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.66
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.82
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.14
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.15999999999999998
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.132
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.08199999999999999
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.14
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.48
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.66
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.82
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.46517990763149614
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.3529603174603174
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.3567092878593359
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSciFact
          type: NanoSciFact
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.66
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.76
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.8
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.84
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.66
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.2733333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.176
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.09399999999999999
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.625
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.745
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.79
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.83
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.7435557624884191
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.720888888888889
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.7171289121218996
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoTouche2020
          type: NanoTouche2020
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.7346938775510204
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.9591836734693877
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.9591836734693877
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.9591836734693877
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.7346938775510204
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.7006802721088434
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.6285714285714286
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.5122448979591837
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.05035830341896132
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.14458823716252048
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.20896761189253432
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3315966017926261
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5870329612095947
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8401360544217688
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4348641866623347
            name: Maxsim Map@100
      - task:
          type: nano-beir
          name: Nano BEIR
        dataset:
          name: NanoBEIR mean
          type: NanoBEIR_mean
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.5611302982731554
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.7460910518053375
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.7983987441130298
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.8583987441130299
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.5611302982731554
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.3559497645211931
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.2732747252747253
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.18709576138147568
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.3289783177016312
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.4979842942392806
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.5627471864009862
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.6356249664717402
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6028108230430728
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.6656472178615036
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.519295906598254
            name: Maxsim Map@100

PyLate model based on answerdotai/ModernBERT-base

This is a PyLate model finetuned from answerdotai/ModernBERT-base on the msmarco-10m-triplets dataset. It maps sentences & paragraphs to sequences of 128-dimensional dense vectors and can be used for semantic textual similarity using the MaxSim operator.

Model Details

Model Description

• Model Type: PyLate model
• Base model: answerdotai/ModernBERT-base
• Document Length: 512 tokens
• Query Length: 32 tokens
• Output Dimensionality: 128 tokens
• Similarity Function: MaxSim
• Training Dataset:
  • msmarco-10m-triplets

Model Sources

• Documentation: PyLate Documentation
• Repository: PyLate on GitHub
• Hugging Face: PyLate models on Hugging Face

Full Model Architecture

ColBERT(
  (0): Transformer({'max_seq_length': 512, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
  (1): Dense({'in_features': 768, 'out_features': 128, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity', 'use_residual': False})
)

Usage

First install the PyLate library:

pip install -U pylate

Retrieval

Use this model with PyLate to index and retrieve documents. The index uses FastPLAID for efficient similarity search.

Indexing documents

Load the ColBERT model and initialize the PLAID index, then encode and index your documents:

from pylate import indexes, models, retrieve

# Step 1: Load the ColBERT model
model = models.ColBERT(
    model_name_or_path="pylate_model_id",
)

# Step 2: Initialize the PLAID index
index = indexes.PLAID(
    index_folder="pylate-index",
    index_name="index",
    override=True,  # This overwrites the existing index if any
)

# Step 3: Encode the documents
documents_ids = ["1", "2", "3"]
documents = ["document 1 text", "document 2 text", "document 3 text"]

documents_embeddings = model.encode(
    documents,
    batch_size=32,
    is_query=False,  # Ensure that it is set to False to indicate that these are documents, not queries
    show_progress_bar=True,
)

# Step 4: Add document embeddings to the index by providing embeddings and corresponding ids
index.add_documents(
    documents_ids=documents_ids,
    documents_embeddings=documents_embeddings,
)

Note that you do not have to recreate the index and encode the documents every time. Once you have created an index and added the documents, you can re-use the index later by loading it:

# To load an index, simply instantiate it with the correct folder/name and without overriding it
index = indexes.PLAID(
    index_folder="pylate-index",
    index_name="index",
)

Retrieving top-k documents for queries

Once the documents are indexed, you can retrieve the top-k most relevant documents for a given set of queries. To do so, initialize the ColBERT retriever with the index you want to search in, encode the queries and then retrieve the top-k documents to get the top matches ids and relevance scores:

# Step 1: Initialize the ColBERT retriever
retriever = retrieve.ColBERT(index=index)

# Step 2: Encode the queries
queries_embeddings = model.encode(
    ["query for document 3", "query for document 1"],
    batch_size=32,
    is_query=True,  #  # Ensure that it is set to False to indicate that these are queries
    show_progress_bar=True,
)

# Step 3: Retrieve top-k documents
scores = retriever.retrieve(
    queries_embeddings=queries_embeddings,
    k=10,  # Retrieve the top 10 matches for each query
)

Reranking

If you only want to use the ColBERT model to perform reranking on top of your first-stage retrieval pipeline without building an index, you can simply use rank function and pass the queries and documents to rerank:

from pylate import rank, models

queries = [
    "query A",
    "query B",
]

documents = [
    ["document A", "document B"],
    ["document 1", "document C", "document B"],
]

documents_ids = [
    [1, 2],
    [1, 3, 2],
]

model = models.ColBERT(
    model_name_or_path="pylate_model_id",
)

queries_embeddings = model.encode(
    queries,
    is_query=True,
)

documents_embeddings = model.encode(
    documents,
    is_query=False,
)

reranked_documents = rank.rerank(
    documents_ids=documents_ids,
    queries_embeddings=queries_embeddings,
    documents_embeddings=documents_embeddings,
)

Evaluation

Metrics

Py Late Information Retrieval

• Dataset: ['NanoClimateFEVER', 'NanoDBPedia', 'NanoFEVER', 'NanoFiQA2018', 'NanoHotpotQA', 'NanoMSMARCO', 'NanoNFCorpus', 'NanoNQ', 'NanoQuoraRetrieval', 'NanoSCIDOCS', 'NanoArguAna', 'NanoSciFact', 'NanoTouche2020']
• Evaluated with pylate.evaluation.pylate_information_retrieval_evaluator.PyLateInformationRetrievalEvaluator

Metric	NanoClimateFEVER	NanoDBPedia	NanoFEVER	NanoFiQA2018	NanoHotpotQA	NanoMSMARCO	NanoNFCorpus	NanoNQ	NanoQuoraRetrieval	NanoSCIDOCS	NanoArguAna	NanoSciFact	NanoTouche2020
MaxSim_accuracy@1	0.24	0.72	0.84	0.46	0.88	0.46	0.46	0.38	0.9	0.42	0.14	0.66	0.7347
MaxSim_accuracy@3	0.5	0.88	0.98	0.66	0.96	0.74	0.52	0.66	0.98	0.62	0.48	0.76	0.9592
MaxSim_accuracy@5	0.56	0.92	1.0	0.74	0.96	0.76	0.54	0.78	1.0	0.7	0.66	0.8	0.9592
MaxSim_accuracy@10	0.72	0.92	1.0	0.78	0.98	0.86	0.56	0.9	1.0	0.82	0.82	0.84	0.9592
MaxSim_precision@1	0.24	0.72	0.84	0.46	0.88	0.46	0.46	0.38	0.9	0.42	0.14	0.66	0.7347
MaxSim_precision@3	0.18	0.6333	0.3333	0.3267	0.5	0.2467	0.3533	0.22	0.3933	0.3067	0.16	0.2733	0.7007
MaxSim_precision@5	0.136	0.608	0.208	0.244	0.328	0.152	0.296	0.156	0.248	0.24	0.132	0.176	0.6286
MaxSim_precision@10	0.09	0.514	0.106	0.14	0.172	0.086	0.236	0.092	0.136	0.172	0.082	0.094	0.5122
MaxSim_recall@1	0.12	0.087	0.7867	0.2686	0.44	0.46	0.0431	0.37	0.7973	0.0887	0.14	0.625	0.0504
MaxSim_recall@3	0.2317	0.1579	0.9267	0.4725	0.75	0.74	0.0762	0.62	0.9387	0.1907	0.48	0.745	0.1446
MaxSim_recall@5	0.2673	0.2556	0.9567	0.5723	0.82	0.76	0.0889	0.73	0.9593	0.2467	0.66	0.79	0.209
MaxSim_recall@10	0.3457	0.3514	0.9633	0.6081	0.86	0.86	0.107	0.84	0.9933	0.3527	0.82	0.83	0.3316
MaxSim_ndcg@10	0.2866	0.631	0.8974	0.5291	0.819	0.6684	0.3057	0.6117	0.9501	0.3418	0.4652	0.7436	0.587
MaxSim_mrr@10	0.389	0.8023	0.905	0.579	0.9195	0.6068	0.494	0.5529	0.945	0.5459	0.353	0.7209	0.8401
MaxSim_map@100	0.2224	0.4766	0.8618	0.4614	0.7543	0.6153	0.1304	0.5369	0.9287	0.2542	0.3567	0.7171	0.4349

Nano BEIR

• Dataset: NanoBEIR_mean
• Evaluated with pylate.evaluation.nano_beir_evaluator.NanoBEIREvaluator

Metric	Value
MaxSim_accuracy@1	0.5611
MaxSim_accuracy@3	0.7461
MaxSim_accuracy@5	0.7984
MaxSim_accuracy@10	0.8584
MaxSim_precision@1	0.5611
MaxSim_precision@3	0.3559
MaxSim_precision@5	0.2733
MaxSim_precision@10	0.1871
MaxSim_recall@1	0.329
MaxSim_recall@3	0.498
MaxSim_recall@5	0.5627
MaxSim_recall@10	0.6356
MaxSim_ndcg@10	0.6028
MaxSim_mrr@10	0.6656
MaxSim_map@100	0.5193

Training Details

Training Dataset

msmarco-10m-triplets

• Dataset: msmarco-10m-triplets at 8c5139a
• Size: 10,000,000 training samples
• Columns: query, positive, and negative
• Approximate statistics based on the first 1000 samples:

  	query	positive	negative
  type	string	string	string
  details	min: 4 tokens mean: 9.31 tokens max: 31 tokens	min: 20 tokens mean: 31.95 tokens max: 32 tokens	min: 18 tokens mean: 31.91 tokens max: 32 tokens

• Samples:

  query	positive	negative
  the most important factor that influences k+ secretion is __________.	The regulation of K+ distribution between the intracellular and extracellular space is referred to as internal K+ balance. The most important factors regulating this movement under normal conditions are insulin and catecholamines (1).	They are both also important for secretion and flow of bile: 1 Cholecystokinin: The name of this hormone describes its effect on the biliary system-cholecysto = gallbladder and kinin = movement. 2 Secretin: This hormone is secreted in response to acid in the duodenum.
  how much did the mackinac bridge cost to build	The cost to design the project was $3,500,000 (Steinman Company). The cost to construct the bridge was $70, 268,500. Two primary contractors were hired to build the bridge: American Bridge for superstructure - $44,532,900; and Merritt-Chapman and Scott of New York for the foundations - $25,735,600.	When your child needs a dental tooth bridge, you need to know the average cost so you can factor the price into your budget. Several factors affect the price of a bridge, which can run between $700 to $1,500 per tooth. If you have insurance or your child is covered by Medicaid, part of the cost may be covered.
  when do concussion symptoms appear	Then you can get advice on what to do next. For milder symptoms, the doctor may recommend rest and ask you to watch your child closely for changes, such as a headache that gets worse. Symptoms of a concussion don't always show up right away, and can develop within 24 to 72 hours after an injury.	Concussion: A traumatic injury to soft tissue, usually the brain, as a result of a violent blow, shaking, or spinning. A brain concussion can cause immediate but temporary impairment of brain functions, such as thinking, vision, equilibrium, and consciousness. After a person has had a concussion, he or she is at increased risk for recurrence. Moreover, after a person has several concussions, less of a blow can cause injury, and the person can require more time to recover.

• Loss: pylate.losses.contrastive.Contrastive

Training Hyperparameters

Non-Default Hyperparameters

• eval_strategy: steps
• per_device_train_batch_size: 64
• learning_rate: 3e-05
• max_steps: 50000
• fp16: True
• dataloader_drop_last: True
• dataloader_num_workers: 8
• ddp_find_unused_parameters: False
• torch_compile: True
• torch_compile_backend: inductor
• eval_on_start: True

All Hyperparameters

Click to expand

• overwrite_output_dir: False
• do_predict: False
• eval_strategy: steps
• prediction_loss_only: True
• per_device_train_batch_size: 64
• per_device_eval_batch_size: 8
• per_gpu_train_batch_size: None
• per_gpu_eval_batch_size: None
• gradient_accumulation_steps: 1
• eval_accumulation_steps: None
• torch_empty_cache_steps: None
• learning_rate: 3e-05
• weight_decay: 0.0
• adam_beta1: 0.9
• adam_beta2: 0.999
• adam_epsilon: 1e-08
• max_grad_norm: 1.0
• num_train_epochs: 3.0
• max_steps: 50000
• lr_scheduler_type: linear
• lr_scheduler_kwargs: {}
• warmup_ratio: 0.0
• warmup_steps: 0
• log_level: passive
• log_level_replica: warning
• log_on_each_node: True
• logging_nan_inf_filter: True
• save_safetensors: True
• save_on_each_node: False
• save_only_model: False
• restore_callback_states_from_checkpoint: False
• no_cuda: False
• use_cpu: False
• use_mps_device: False
• seed: 42
• data_seed: None
• jit_mode_eval: False
• use_ipex: False
• bf16: False
• fp16: True
• fp16_opt_level: O1
• half_precision_backend: auto
• bf16_full_eval: False
• fp16_full_eval: False
• tf32: None
• local_rank: 0
• ddp_backend: None
• tpu_num_cores: None
• tpu_metrics_debug: False
• debug: []
• dataloader_drop_last: True
• dataloader_num_workers: 8
• dataloader_prefetch_factor: None
• past_index: -1
• disable_tqdm: False
• remove_unused_columns: True
• label_names: None
• load_best_model_at_end: False
• ignore_data_skip: False
• fsdp: []
• fsdp_min_num_params: 0
• fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
• fsdp_transformer_layer_cls_to_wrap: None
• accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
• parallelism_config: None
• deepspeed: None
• label_smoothing_factor: 0.0
• optim: adamw_torch
• optim_args: None
• adafactor: False
• group_by_length: False
• length_column_name: length
• ddp_find_unused_parameters: False
• ddp_bucket_cap_mb: None
• ddp_broadcast_buffers: False
• dataloader_pin_memory: True
• dataloader_persistent_workers: False
• skip_memory_metrics: True
• use_legacy_prediction_loop: False
• push_to_hub: False
• resume_from_checkpoint: None
• hub_model_id: None
• hub_strategy: every_save
• hub_private_repo: None
• hub_always_push: False
• hub_revision: None
• gradient_checkpointing: False
• gradient_checkpointing_kwargs: None
• include_inputs_for_metrics: False
• include_for_metrics: []
• eval_do_concat_batches: True
• fp16_backend: auto
• push_to_hub_model_id: None
• push_to_hub_organization: None
• mp_parameters:
• auto_find_batch_size: False
• full_determinism: False
• torchdynamo: None
• ray_scope: last
• ddp_timeout: 1800
• torch_compile: True
• torch_compile_backend: inductor
• torch_compile_mode: None
• include_tokens_per_second: False
• include_num_input_tokens_seen: False
• neftune_noise_alpha: None
• optim_target_modules: None
• batch_eval_metrics: False
• eval_on_start: True
• use_liger_kernel: False
• liger_kernel_config: None
• eval_use_gather_object: False
• average_tokens_across_devices: False
• prompts: None
• batch_sampler: batch_sampler
• multi_dataset_batch_sampler: proportional
• router_mapping: {}
• learning_rate_mapping: {}