Release MA-Retriever

.gitattributes

@@ -1,35 +1 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 1024,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 MetaSyn Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,108 @@
 ---
+library_name: sentence-transformers
+pipeline_tag: sentence-similarity
 license: mit
+base_model: BAAI/bge-large-en-v1.5
+tags:
+- sentence-transformers
+- information-retrieval
+- biomedical
+- metasyn
+datasets:
+- THUIR/MetaSyn
 ---
+
+# MA-Retriever
+
+MA-Retriever is the dense retriever released with MetaSyn. It fine-tunes
+`BAAI/bge-large-en-v1.5` for retrieving articles linked to the published
+included-study lists of systematic reviews, scoping reviews, and
+meta-analyses.
+
+## Resources
+
+- Paper: [arXiv:2606.17041](https://arxiv.org/abs/2606.17041)
+- Dataset: [THUIR/MetaSyn](https://huggingface.co/datasets/THUIR/MetaSyn)
+- Code and evaluator: [THUIR/MetaSyn](https://github.com/THUIR/MetaSyn)
+
+## Use
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("BFTree/MA-Retriever")
+
+def encode_protocol(research_question, population, intervention_or_exposure,
+                    comparison, outcome):
+    fields = [
+        ("Research Question", research_question),
+        ("Population", population),
+        ("Intervention or Exposure", intervention_or_exposure),
+        ("Comparison", comparison),
+        ("Outcome", outcome),
+    ]
+    protocol = ". ".join(f"{name}: {value}" for name, value in fields if value)
+    query = "Represent this sentence for searching relevant passages: " + protocol
+    return model.encode(query, normalize_embeddings=True)
+
+def encode_article(title, abstract):
+    document = f"Title: {title}. Abstract: {abstract}"
+    return model.encode(document, normalize_embeddings=True)
+```
+
+The protocol query concatenates the research question, Population,
+Intervention or Exposure, Comparison, and Outcome. It does not use the
+source-review title. Documents use title and abstract. Retrieval uses cosine
+similarity over normalized embeddings.
+
+## Training
+
+The base revision is `d4aa6901d3a41ba39fb536a557fa166f842b0e09`. We used
+40 reviews sampled from the training split to select the epoch count. Recall@20
+is the primary metric, with Recall@100 and Recall@200 used as tie-breakers. The
+released checkpoint was retrained for one epoch using the full training split
+with Multiple Negatives Ranking Loss, batch size 64, learning rate 2e-5,
+maximum sequence length 512, and seed 718. After excluding articles linked to
+a test review, 334 of the 336 training reviews contribute positive pairs.
+
+There are 5,585 constructed training pairs. The sentence-transformers training
+loader used 5,568 examples in complete batches during the epoch; this accounts
+for the smaller sample count shown by automatically generated trainer metadata.
+
+## Test retrieval
+
+All metrics are macro-averaged over the 86 held-out reviews after removing the
+source review itself from each ranking.
+
+| K | Recall@K | Precision@K |
+|---:|---:|---:|
+| 5 | 24.0% | 47.9% |
+| 10 | 38.8% | 42.0% |
+| 20 | 53.5% | 34.0% |
+| 50 | 75.3% | 22.6% |
+| 100 | 84.2% | 14.0% |
+| 200 | 91.7% | 8.4% |
+
+The source-review split is disjoint, and every article linked to a test review
+is excluded as a positive retriever-training example.
+
+## Limitations
+
+The model is trained on reviews from Nature Portfolio and a PubMed-centered
+corpus. Performance may not transfer unchanged to other databases, languages,
+or domains. A high-recall pool still requires protocol-based screening; model
+scores should not be treated as inclusion decisions.
+
+## Citation
+
+```bibtex
+@misc{metasyn2026,
+  title         = {Benchmarking {LLM} Agents on Meta-Analysis Articles from {Nature} Portfolio},
+  author        = {Anzhe Xie and Weihang Su and Yujia Zhou and Yiqun Liu and Min Zhang and Qingyao Ai},
+  year          = {2026},
+  eprint        = {2606.17041},
+  archivePrefix = {arXiv},
+  primaryClass  = {cs.CL},
+  url           = {https://arxiv.org/abs/2606.17041}
+}
+```

checksums.json

@@ -0,0 +1,67 @@
+[
+  {
+    "path": ".gitattributes",
+    "bytes": 50,
+    "sha256": "ee05350c7fda0af4da116020dd7b976d731de3beed4f5e717d5bcc95ead24550"
+  },
+  {
+    "path": "1_Pooling/config.json",
+    "bytes": 297,
+    "sha256": "13e69897522ee8255104483ed9f219465d1be3936654a54a318758738052789e"
+  },
+  {
+    "path": "LICENSE",
+    "bytes": 1072,
+    "sha256": "8b8b52057027c23382997be7fd01368a0bbd0ea3596e20973bffa7053b4e8fe3"
+  },
+  {
+    "path": "config.json",
+    "bytes": 742,
+    "sha256": "3d0c874396f8eebd33c1554428f72475494ec68e5bc3abb3e740cfe1f73ccb03"
+  },
+  {
+    "path": "config_sentence_transformers.json",
+    "bytes": 283,
+    "sha256": "ab257f3f160a1f6d256c13c29740444e974992a89716e6f64ab122f1f45d7bfd"
+  },
+  {
+    "path": "model.safetensors",
+    "bytes": 1340612432,
+    "sha256": "608a6b98c77ce79d9d14d8d6480c0f7586741083cb5be761cfe448efd4a84322"
+  },
+  {
+    "path": "modules.json",
+    "bytes": 349,
+    "sha256": "84e40c8e006c9b1d6c122e02cba9b02458120b5fb0c87b746c41e0207cf642cf"
+  },
+  {
+    "path": "sentence_bert_config.json",
+    "bytes": 52,
+    "sha256": "84e39fda68ccbff05bfa723ae9c0e70e23e2ec373b76e0f8c6e71af72a693cbf"
+  },
+  {
+    "path": "special_tokens_map.json",
+    "bytes": 695,
+    "sha256": "5d5b662e421ea9fac075174bb0688ee0d9431699900b90662acd44b2a350503a"
+  },
+  {
+    "path": "tokenizer.json",
+    "bytes": 711649,
+    "sha256": "91f1def9b9391fdabe028cd3f3fcc4efd34e5d1f08c3bf2de513ebb5911a1854"
+  },
+  {
+    "path": "tokenizer_config.json",
+    "bytes": 1272,
+    "sha256": "7b082b48a08a0b5c5939d99472e33addce3d96202155e33f7928cecd171186a0"
+  },
+  {
+    "path": "training_metadata.json",
+    "bytes": 1317,
+    "sha256": "3037d33b70f00f62a16a2182f79051bd48026f8caeccfbd93f0809b2ce5e6319"
+  },
+  {
+    "path": "vocab.txt",
+    "bytes": 231508,
+    "sha256": "07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3"
+  }
+]

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "BAAI/bge-large-en-v1.5",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

config_sentence_transformers.json

@@ -0,0 +1,14 @@
+{
+  "__version__": {
+    "sentence_transformers": "3.2.1",
+    "transformers": "4.43.1",
+    "pytorch": "2.3.1+cu121"
+  },
+  "model_type": "SentenceTransformer",
+  "prompts": {
+    "query": "",
+    "document": ""
+  },
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608a6b98c77ce79d9d14d8d6480c0f7586741083cb5be761cfe448efd4a84322
+size 1340612432

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": true
+}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,58 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

training_metadata.json

@@ -0,0 +1,37 @@
+{
+  "base_model": "BAAI/bge-large-en-v1.5",
+  "base_model_revision": "d4aa6901d3a41ba39fb536a557fa166f842b0e09",
+  "seed": 718,
+  "query_template": "Research_Question + PI/ECO with BGE retrieval instruction",
+  "document_template": "Title + Abstract",
+  "train_reviews": 336,
+  "test_reviews": 86,
+  "model_selection_reviews": 40,
+  "model_selection_rule": "Recall@20, then Recall@100, then Recall@200",
+  "selected_epochs": 1,
+  "final_retraining_reviews": 336,
+  "training_reviews_with_positive_pairs": 334,
+  "test_linked_articles_excluded_from_positive_training": 1649,
+  "constructed_training_pairs": 5585,
+  "complete_batch_examples_used": 5568,
+  "batch_size": 64,
+  "learning_rate": 2e-05,
+  "max_seq_length": 512,
+  "loss": "MultipleNegativesRankingLoss",
+  "batch_sampler": "NoDuplicatesDataLoader",
+  "epochs": 1,
+  "test_metrics": {
+    "recall@5": 0.24032354463558983,
+    "recall@10": 0.38838341990475195,
+    "recall@20": 0.534827514606953,
+    "recall@50": 0.7526966587841311,
+    "recall@100": 0.8420887384792756,
+    "recall@200": 0.917380895710928,
+    "precision@5": 0.4790697674418605,
+    "precision@10": 0.4197674418604652,
+    "precision@20": 0.3401162790697674,
+    "precision@50": 0.22604651162790698,
+    "precision@100": 0.14046511627906977,
+    "precision@200": 0.08406976744186047
+  }
+}