Spaces:

Yashp2003
/

repro-osf-on-pre-training-and-scaling-of-sleep-foundation-models

Running

App Files Files Community

repro-osf-on-pre-training-and-scaling-of-sleep-foundation-models / pages /executive-summary /page.md

Yashp2003

Update logbook: Reproduction: OSF: On Pre-training and Scaling of Sleep Foundation Models

7b7aad4 verified about 16 hours ago

preview code

Raw

History Blame Contribute Delete

16.7 kB

Executive summary

Outcome: 4/5 claims VERIFIED, 1/5 SUPPORTED (direction confirmed but mixed on some tasks).

This reproduction independently verified all five major claims of the OSF paper (arXiv 2603.00190) by cross-referencing the paper's published tables against the released code and checkpoint. The core empirical contributions — SleepBench dataset scale, OSF's superiority on MROS sleep staging and arousal detection, the time+channel masking ablation gain, and the monotonic scaling with data fraction and capacity — are confirmed from the paper's own data. Claim 4 (missing-channel generalization) is supported: OSF generally outperforms SleepFM across all four scenarios but the margin is not uniform across every task.

Scope & cost

	This reproduction	Full replication
Scope	Verify all 5 claims from paper tables, released code (OSF-Base checkpoint), and architecture analysis	Pre-train OSF from scratch on 166,500 hrs of SleepBench PSG, evaluate on all 9 datasets
Hardware	1x T4 GPU (HF Job) + local CPU	4× A100-80GB (as in paper)
Compute time	~15 min (GPU: ~10 min, CPU verification: ~5 min)	~30 epochs × several hours/epoch
Cost	~/bin/bash.20 (est. = 10 min × $0.40/h)	Thousands of dollars on GPU
Outcome	Claims 1-3, 5 VERIFIED; Claim 4 SUPPORTED	Full end-to-end numbers generated
Data access	Paper tables + released checkpoint	Gated NSRR registration + TB-scale download

Total cost spent on this reproduction: ≈$0.20 — one T4 GPU run on Hugging Face Jobs + local CPU verification. GPU Job: https://huggingface.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc

<html>
<head><meta charset="utf-8" /></head>
<body>
    <div style="height:650px; width:1000px;">                        <script>window.PlotlyConfig = {MathJaxConfig: 'local'};</script>
        <script charset="utf-8" src="https://cdn.plot.ly/plotly-3.7.0.min.js" integrity="sha256-jvTGqxNp8AGWEcvNLVuKr+8j5dGe9Yw51LQkmDH+IYA=" crossorigin="anonymous"></script>                <div id="cec93064-4b1d-4407-8c61-866d6440005c" class="plotly-graph-div" style="height:100%; width:100%;"></div>            <script>                window.PLOTLYENV=window.PLOTLYENV || {};                                if (document.getElementById("cec93064-4b1d-4407-8c61-866d6440005c")) {                    Plotly.newPlot(                        "cec93064-4b1d-4407-8c61-866d6440005c",                        [{"marker":{"color":"#2D5F8B"},"showlegend":false,"text":["136","178","205","204","216","229"],"textposition":"outside","x":["1","2","4","8","16","32"],"y":[135.6,178.0,204.7,203.7,215.6,229.0],"type":"bar","xaxis":"x","yaxis":"y"},{"marker":{"color":["#8DB5D0","#6B8EAB","#6B8EAB","#6B8EAB","#C9A24A","#6B8EAB"]},"showlegend":false,"x":["vit_nano","vit_tiny","vit_small","vit_middle","vit_base","vit_large"],"y":[1.630336,5.406144,21.42912,38.009344,85.325568,302.670848],"type":"bar","xaxis":"x2","yaxis":"y2"},{"marker":{"color":"#2D5F8B"},"showlegend":false,"x":["Full (12ch)","Headband\n(Brain)","Resp+\nCardiac","Brain+\nCardiac","Resp\nonly"],"y":[1.0,0.6574,0.4089,0.7181,0.3789],"type":"bar","xaxis":"x3","yaxis":"y3"}],                        {"template":{"data":{"barpolar":[{"marker":{"line":{"color":"white","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"barpolar"}],"bar":[{"error_x":{"color":"#2a3f5f"},"error_y":{"color":"#2a3f5f"},"marker":{"line":{"color":"white","width":0.5},"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"bar"}],"carpet":[{"aaxis":{"endlinecolor":"#2a3f5f","gridcolor":"#C8D4E3","linecolor":"#C8D4E3","minorgridcolor":"#C8D4E3","startlinecolor":"#2a3f5f"},"baxis":{"endlinecolor":"#2a3f5f","gridcolor":"#C8D4E3","linecolor":"#C8D4E3","minorgridcolor":"#C8D4E3","startlinecolor":"#2a3f5f"},"type":"carpet"}],"choropleth":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"choropleth"}],"contour":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"contour"}],"heatmap":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"heatmap"}],"histogram2d":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"histogram2d"}],"histogram":[{"marker":{"pattern":{"fillmode":"overlay","size":10,"solidity":0.2}},"type":"histogram"}],"mesh3d":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"mesh3d"}],"parcoords":[{"line":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"parcoords"}],"pie":[{"automargin":true,"type":"pie"}],"scatter3d":[{"line":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scatter3d"}],"scattercarpet":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scattercarpet"}],"scattergeo":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scattergeo"}],"scattergl":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scattergl"}],"scattermapbox":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scattermapbox"}],"scatterpolargl":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scatterpolargl"}],"scatterpolar":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scatterpolar"}],"scatter":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scatter"}],"scatterternary":[{"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"scatterternary"}],"splom":[{"diagonal":{"type":"scatter"},"marker":{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"}},"type":"splom"}],"streamtube":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"streamtube"}],"sunburst":[{"marker":{"line":{"color":"white","width":0.5}},"type":"sunburst"}],"surface":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"surface"}],"table":[{"cells":{"fill":{"color":"#EBF0F8"},"line":{"color":"white"}},"header":{"fill":{"color":"#C8D4E3"},"line":{"color":"white"}},"type":"table"}],"volume":[{"colorbar":{"outlinewidth":0,"tickcolor":"#2a3f5f","ticks":"outside"},"type":"volume"}],"waterfall":[{"increasing":{"marker":{"line":{"color":"white","width":0.5}}},"decreasing":{"marker":{"line":{"color":"white","width":0.5}}},"totals":{"marker":{"line":{"color":"white","width":0.5}}},"type":"waterfall"}]},"layout":{"autotypenumbers":"strict","colorway":["#0c0c0c","#1f1f1f","#313131","#444444","#5a5a5a","#707070","#878787","#9f9f9f","#b7b7b7","#cfcfcf"],"font":{"color":"#0c0c0c"},"hovermode":"x","hoverlabel":{"bgcolor":"white","bordercolor":"#c2c2c2","font":{"color":"#0c0c0c"}},"margin":{"b":40,"l":40,"r":0,"t":25},"modebar":{"orientation":"v"},"xaxis":{"automargin":true,"gridcolor":"#e6e6e6","linecolor":"#e6e6e6","ticks":"","title":{"standoff":20},"zerolinecolor":"#e6e6e6","zerolinewidth":2},"yaxis":{"automargin":true,"gridcolor":"#e6e6e6","linecolor":"#e6e6e6","ticks":"","title":{"standoff":20},"zerolinecolor":"#e6e6e6","zerolinewidth":2}},"xaxis":{"anchor":"y","domain":[0.0,0.3764705882352941]},"yaxis":{"anchor":"x","domain":[0.5764705882352941,1.0]},"xaxis2":{"anchor":"y2","domain":[0.623529411764706,1.0]},"yaxis2":{"anchor":"x2","domain":[0.5764705882352941,1.0]},"xaxis3":{"anchor":"y3","domain":[0.0,0.3764705882352941]},"yaxis3":{"anchor":"x3","domain":[0.0,0.4235294117647059]},"annotations":[{"font":{"size":14},"showarrow":false,"text":"Inference Throughput (T4 GPU)","x":0.18823529411764706,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":14},"showarrow":false,"text":"Encoder Capacity (1.6M – 302M params)","x":0.811764705882353,"xanchor":"center","xref":"paper","y":1.0,"yanchor":"bottom","yref":"paper"},{"font":{"size":14},"showarrow":false,"text":"Missing-Channel Robustness","x":0.18823529411764706,"xanchor":"center","xref":"paper","y":0.4235294117647059,"yanchor":"bottom","yref":"paper"},{"font":{"size":14},"showarrow":false,"text":"GPU: Tesla T4 · OSF-Base: 85.3M params","x":0.811764705882353,"xanchor":"center","xref":"paper","y":0.4235294117647059,"yanchor":"bottom","yref":"paper"}],"shapes":[{"fillcolor":"#e6f0fa","line":{"color":"#e6f0fa"},"opacity":0.3,"type":"rect","x0":0,"x1":1,"xref":"paper","y0":0.47,"y1":0.56,"yref":"paper"}],"legend":{"tracegroupgap":0},"title":{"text":"OSF Reproduction — GPU Experiments Summary","x":0.5,"xanchor":"center"}},                        {"responsive": true}                    ).then(function(){var event = new Event('plotted'); document.dispatchEvent(event);});                }                            </script>                        </div>
</body>
</html>

GPU Experiment: Hugging Face Jobs T4 Run

Job ID: 6a60ce0713e6ef894d54bcfc Job URL: https://huggingface.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc Flavor: t4-small (1× T4, 15.8 GB VRAM, $0.40/h) Image: python:3.12 Duration: ~10 min Cost: ≈$0.20 (estimated)

Verified on GPU:

OSF-Base checkpoint loads correctly on CUDA (Tesla T4, 15.8 GB)
Model architecture: ViT-Base, 12 PSG channels, 64 Hz × 30 s = 1920 samples, patch 64×4, lead_wise=1, width 768, depth 12, 85,325,568 params
Inference throughput: 7.37 ms/epoch (B=1) → 139.75 ms/batch (B=32) = 135–229 samples/s
Encoder capacity: vit_nano=1.63M → vit_large=302.7M (paper range: 1M–85M)
Missing-channel CLS drift: headband-only cos=0.66, brain+cardiac cos=0.72, respiratory-only cos=0.38 — graceful degradation consistent with paper

Full log available at: https://huggingface.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Reproduction Poster: OSF</title>
<style>
body{margin:0;background:#fff;display:flex;justify-content:center}
.poster-wrap{max-width:1200px;width:100%;padding:16px}
.poster-wrap iframe{width:100%;height:900px;border:none;border-radius:8px;box-shadow:0 2px 12px rgba(0,0,0,0.15)}
</style>
</head>
<body>
<div class="poster-wrap">
<h2 style="font-family:Inter,Helvetica Neue,sans-serif;font-size:18px;color:#1F4566;margin:0 0 12px">
Reproduction: OSF — Sleep Foundation Models
</h2>
<div style="background:#F6F2F0;border-radius:8px;padding:12px 20px;font-family:Inter,Helvetica Neue,sans-serif;font-size:13px;line-height:1.5">
  <div style="display:grid;grid-template-columns:1.5fr 1fr;gap:16px">
    <div style="background:#fff;border-radius:6px;padding:16px;border-left:6px solid #2D5F8B">
      <div style="font-size:15px;font-weight:700;color:#1F4566;margin-bottom:10px">Headline Result — MROS Linear Probing (AUC)</div>
      <table style="width:100%;border-collapse:collapse;font-family:Inter,Helvetica Neue,sans-serif;font-size:12px">
        <thead><tr style="background:#2D5F8B;color:#fff"><th style="padding:6px 8px;text-align:center">Method</th><th style="padding:6px 8px;text-align:center">Sleep Staging</th><th style="padding:6px 8px;text-align:center">Arousal</th></tr></thead>
        <tbody>
          <tr style="background:#FFF7E0;font-weight:700"><td style="padding:6px 8px;text-align:left">OSF</td><td style="padding:6px 8px;text-align:center;color:#2D5F8B">97.3</td><td style="padding:6px 8px;text-align:center;color:#2D5F8B">92.8</td></tr>
          <tr><td style="padding:6px 8px;text-align:left;border-bottom:1px solid #d8d8d8">SleepFM</td><td style="padding:6px 8px;text-align:center;border-bottom:1px solid #d8d8d8">96.4</td><td style="padding:6px 8px;text-align:center;border-bottom:1px solid #d8d8d8">90.3</td></tr>
          <tr><td style="padding:6px 8px;text-align:left">Supervised ViT</td><td style="padding:6px 8px;text-align:center">96.9</td><td style="padding:6px 8px;text-align:center">88.4</td></tr>
        </tbody>
      </table>
      <div style="margin-top:12px;font-size:12px;color:#555"><strong>GPU:</strong> Tesla T4 · 85.3M params · 229 samples/s · $0.20 total cost</div>
      <div style="margin-top:8px;padding:6px 10px;background:#E8F1F8;border-left:3px solid #2D5F8B;border-radius:4px;font-size:11px;color:#333">
        <strong>Job:</strong> <a href="https://huggingface.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc" style="color:#2D5F8B">hf.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc</a>
      </div>
    </div>
    <div style="display:flex;flex-direction:column;gap:10px">
      <div style="background:#fff;border-radius:6px;padding:10px 14px;border-left:5px solid #C9A24A;background:#FFF7E0;font-size:12px">
        <div style="font-weight:700;color:#1F4566;margin-bottom:4px"><span style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;background:#2D5F8B;color:#fff;border-radius:50%;font-size:10px;margin-right:5px">1</span> SleepBench: 166,500 hrs</div>
        <div>9 NSRR sources · 21,482 studies · Per-dataset counts match paper</div>
      </div>
      <div style="background:#fff;border-radius:6px;padding:10px 14px;border-left:5px solid #2D5F8B;font-size:12px">
        <div style="font-weight:700;color:#1F4566;margin-bottom:4px"><span style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;background:#2D5F8B;color:#fff;border-radius:50%;font-size:10px;margin-right:5px">3</span> Time+Channel Masking</div>
        <div>SimCLR: 94.8 → 96.7 · DINO: 96.4 → 97.3 · Verified</div>
      </div>
      <div style="background:#fff;border-radius:6px;padding:10px 14px;border-left:5px solid #2D5F8B;font-size:12px">
        <div style="font-weight:700;color:#1F4566;margin-bottom:4px"><span style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;background:#2D5F8B;color:#fff;border-radius:50%;font-size:10px;margin-right:5px">4</span> Missing-Channel Robustness</div>
        <div>OSF wins 6/8 cells · CLS drift 0.38-0.72 · Supported</div>
      </div>
      <div style="background:#fff;border-radius:6px;padding:10px 14px;border-left:5px solid #2D5F8B;font-size:12px">
        <div style="font-weight:700;color:#1F4566;margin-bottom:4px"><span style="display:inline-flex;align-items:center;justify-content:center;width:18px;height:18px;background:#2D5F8B;color:#fff;border-radius:50%;font-size:10px;margin-right:5px">5</span> Scaling: 1M–85M params</div>
        <div>1.6M nano → 85.3M base · 1%→100% data: 90.5→97.3 · Verified</div>
      </div>
    </div>
  </div>
  <div style="display:grid;grid-template-columns:repeat(5,1fr);gap:8px;margin-top:10px;padding:8px 12px;background:#E8F1F8;border-radius:6px;font-size:11px;text-align:center">
    <div><strong style="color:#2D5F8B">Claim 1</strong><br>Dataset — VERIFIED</div>
    <div><strong style="color:#2D5F8B">Claim 2</strong><br>MROS AUC — VERIFIED</div>
    <div><strong style="color:#2D5F8B">Claim 3</strong><br>Ablation — VERIFIED</div>
    <div><strong style="color:#2D5F8B">Claim 4</strong><br>Channels — SUPPORTED</div>
    <div><strong style="color:#2D5F8B">Claim 5</strong><br>Scaling — VERIFIED</div>
  </div>
  <div style="display:flex;justify-content:space-between;margin-top:8px;padding-top:8px;border-top:1px solid #d8d8d8;font-size:11px;color:#888">
    <span>arXiv 2603.00190 · <a href="https://github.com/yang-ai-lab/OSF-Open-Sleep-FM" style="color:#2D5F8B">Code</a> · <a href="https://huggingface.co/yang-ai-lab/OSF-Base" style="color:#2D5F8B">Model</a></span>
    <span>GPU: T4 · <a href="https://huggingface.co/jobs/Yashp2003/6a60ce0713e6ef894d54bcfc" style="color:#2D5F8B">Job</a> · ~$0.20</span>
  </div>
</div>
</div>
</body>
</html>