Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>SupraLabs | Is One Epoch Really All You Need for SLMs?</title> | |
| <script src="https://cdn.jsdelivr.net/npm/chart.js"></script> | |
| <style> | |
| :root { | |
| --bg: #0f0f0f; | |
| --surface: #1a1a1a; | |
| --border: #333; | |
| --text: #e0e0e0; | |
| --accent: #536bfe; /* Supra Blue */ | |
| --muted: #888; | |
| --success: #00e676; | |
| --warning: #ffb300; | |
| --font-mono: 'JetBrains Mono', 'Fira Code', monospace; | |
| } | |
| * { | |
| margin: 0; | |
| padding: 0; | |
| box-sizing: border-box; | |
| } | |
| body { | |
| background-color: var(--bg); | |
| color: var(--text); | |
| font-family: 'Inter', -apple-system, sans-serif; | |
| line-height: 1.6; | |
| padding: 2rem; | |
| } | |
| code, pre, .mono { | |
| font-family: var(--font-mono); | |
| } | |
| .container { | |
| max-width: 1000px; | |
| margin: 0 auto; | |
| } | |
| /* --- Header --- */ | |
| header { | |
| border-bottom: 2px solid var(--border); | |
| padding-bottom: 2rem; | |
| margin-bottom: 3rem; | |
| display: flex; | |
| justify-content: space-between; | |
| align-items: flex-end; | |
| } | |
| .logo-area a { | |
| text-decoration: none; | |
| color: inherit; | |
| } | |
| .logo-area h1 { | |
| font-size: 1.2rem; | |
| text-transform: uppercase; | |
| letter-spacing: 2px; | |
| color: var(--accent); | |
| line-height: 1; | |
| display: flex; | |
| align-items: center; | |
| gap: 10px; | |
| } | |
| nav a { | |
| color: var(--text); | |
| text-decoration: none; | |
| margin-left: 1.5rem; | |
| font-size: 0.9rem; | |
| border-bottom: 1px solid transparent; | |
| } | |
| nav a:hover { | |
| border-bottom: 1px solid var(--accent); | |
| } | |
| /* --- Hero Section --- */ | |
| .hero { | |
| margin-bottom: 4rem; | |
| } | |
| .hero h2 { | |
| font-size: 3rem; | |
| line-height: 1.1; | |
| margin-bottom: 1.5rem; | |
| font-weight: 800; | |
| } | |
| .hero p { | |
| font-size: 1.2rem; | |
| color: var(--muted); | |
| max-width: 750px; | |
| } | |
| .section-label { | |
| display: block; | |
| font-family: var(--font-mono); | |
| color: var(--accent); | |
| font-size: 0.8rem; | |
| margin-top: 3rem; | |
| margin-bottom: 1rem; | |
| text-transform: uppercase; | |
| } | |
| /* --- General Content Styles --- */ | |
| .card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| padding: 2.5rem; | |
| margin-bottom: 2rem; | |
| } | |
| h3 { | |
| font-size: 1.6rem; | |
| margin-bottom: 1rem; | |
| font-weight: 700; | |
| } | |
| p { | |
| margin-bottom: 1rem; | |
| color: #ccc; | |
| } | |
| ul { | |
| margin-left: 1.5rem; | |
| margin-bottom: 1.5rem; | |
| } | |
| li { | |
| margin-bottom: 0.5rem; | |
| } | |
| .winner-badge { | |
| color: var(--success); | |
| font-weight: bold; | |
| font-family: var(--font-mono); | |
| } | |
| /* --- Responsive Interactive Table --- */ | |
| .table-container { | |
| overflow-x: auto; | |
| margin: 2rem 0; | |
| border: 1px solid var(--border); | |
| } | |
| table { | |
| width: 100%; | |
| border-collapse: collapse; | |
| text-align: left; | |
| font-size: 0.95rem; | |
| } | |
| th, td { | |
| padding: 1rem; | |
| border-bottom: 1px solid var(--border); | |
| } | |
| th { | |
| background-color: rgba(26, 26, 26, 0.8); | |
| font-family: var(--font-mono); | |
| color: var(--accent); | |
| font-size: 0.85rem; | |
| text-transform: uppercase; | |
| } | |
| tr:hover { | |
| background-color: rgba(83, 107, 254, 0.05); | |
| } | |
| .highlight-row { | |
| border-left: 4px solid var(--success); | |
| background-color: rgba(0, 230, 118, 0.02); | |
| } | |
| /* --- Chart Section --- */ | |
| .chart-box { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| padding: 2rem; | |
| margin-bottom: 2rem; | |
| } | |
| /* --- Grid for Downloads / Source Links --- */ | |
| .links-grid { | |
| display: grid; | |
| grid-template-columns: 1fr 1fr; | |
| gap: 1.5rem; | |
| margin-bottom: 2rem; | |
| } | |
| .link-card { | |
| background: var(--surface); | |
| border: 1px solid var(--border); | |
| padding: 1.5rem; | |
| display: flex; | |
| flex-direction: column; | |
| justify-content: space-between; | |
| } | |
| .link-card h4 { | |
| font-family: var(--font-mono); | |
| font-size: 0.95rem; | |
| margin-bottom: 0.5rem; | |
| color: var(--accent); | |
| } | |
| .btn { | |
| display: inline-block; | |
| background: transparent; | |
| color: var(--text); | |
| border: 1px solid var(--accent); | |
| padding: 0.6rem 1.2rem; | |
| font-family: var(--font-mono); | |
| font-size: 0.8rem; | |
| text-decoration: none; | |
| text-align: center; | |
| cursor: pointer; | |
| transition: all 0.2s ease; | |
| margin-top: 1rem; | |
| } | |
| .btn:hover { | |
| background: var(--accent); | |
| color: #000; | |
| font-weight: bold; | |
| transform: translateY(-1px); | |
| } | |
| .btn-success { | |
| border-color: var(--success); | |
| } | |
| .btn-success:hover { | |
| background: var(--success); | |
| } | |
| /* --- Hardware Stats --- */ | |
| .stats-grid { | |
| display: grid; | |
| grid-template-columns: 1fr 1fr 1fr; | |
| gap: 1rem; | |
| margin-top: 4rem; | |
| border-top: 1px solid var(--border); | |
| padding-top: 2rem; | |
| } | |
| .stat-box { | |
| padding: 1rem; | |
| border-left: 2px solid var(--accent); | |
| } | |
| .stat-box small { | |
| display: block; | |
| color: var(--muted); | |
| font-family: var(--font-mono); | |
| } | |
| footer { | |
| margin-top: 6rem; | |
| padding-bottom: 2rem; | |
| font-size: 0.8rem; | |
| color: var(--muted); | |
| text-align: center; | |
| } | |
| @media (max-width: 768px) { | |
| .hero h2 { font-size: 2rem; } | |
| header { flex-direction: column; align-items: flex-start; gap: 1rem; } | |
| nav a { margin-left: 0; margin-right: 1rem; } | |
| .links-grid, .stats-grid { grid-template-columns: 1fr; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="container"> | |
| <header> | |
| <div class="logo-area" style="font-size: 1.5em;"> | |
| <a href="index.html"><h1><img src="./image.png" style="height: 2em" alt="Logo"> SupraLabs_</h1></a> | |
| </div> | |
| <nav> | |
| <a href="#summary">Core Learnings</a> | |
| <a href="#benchmarks">Epoch Matrix</a> | |
| <a href="#charts">Visualizations</a> | |
| <a href="#downloads">Artifact Exports</a> | |
| <a href="https://huggingface.co/SupraLabs" target="_blank">HuggingFace</a> | |
| </nav> | |
| </header> | |
| <section class="hero"> | |
| <h2>Experiment #3:<br>Is One Epoch Really All You Need for SLMs?</h2> | |
| <p>A rigorous, controlled empirical scan to stress-test multi-epoch training boundaries on consumer hardware. We ran a <strong>5,114,304 parameter architecture</strong> through a sweeping 1-to-8 epoch matrix using an isolated, stationary pool of <strong>20,000,000 high-quality tokens</strong> from FineWeb-Edu.</p> | |
| </section> | |
| <span class="section-label" id="summary">// Empirical_Discoveries_&_Tipping_Points</span> | |
| <div class="card"> | |
| <h3>Breaking the Chinchilla Dogma for SLMs</h3> | |
| <p>Modern LLM design dictates that models should ideally train on unique tokens for exactly one epoch to guarantee generalized logic. Our sweep completely dismantles this assumption for extreme Small Language Models (SLMs) in the megabyte range:</p> | |
| <ul> | |
| <li><strong>The Single-Epoch Problem:</strong> At 1 Epoch, the model suffers from massive syntactic fragmentation. Its limited internal parameter real-estate isn't given enough optimization cycles to cement basic structural patterns.</li> | |
| <li><strong>Deep Memorization vs. Collapse:</strong> Repeating data up to Epoch 5-6 acts as an intense structural reinforcement phase. Grammatical robustness jumps massively without deteriorating logic. However, pushing past Epoch 6 triggers a fascinating mechanical divergence.</li> | |
| <li><strong>The Epoch 6 Tipping Point:</strong> At Epoch 6, factual reasoning reaches its perfect zenith. By Epoch 7 and 8, the model begins severe parameter oscillation. It starts shifts from generalized grammar mastery to raw, star-memorized token parroting, locking the latent space.</li> | |
| </ul> | |
| </div> | |
| <span class="section-label" id="benchmarks">// The_8-Epoch_Data_Matrix</span> | |
| <div class="card" style="padding: 1.5rem;"> | |
| <h3>Controlled Sweeping Metrics</h3> | |
| <p>All benchmarks processed zero-shot using uniform learning schedules. Lower Perplexity (PPL) and Lower Training Loss denote superior data mapping.</p> | |
| <div class="table-container"> | |
| <table> | |
| <thead> | |
| <tr> | |
| <th>Benchmark/Metric</th> | |
| <th>Epoch 1</th> | |
| <th>Epoch 2</th> | |
| <th>Epoch 3</th> | |
| <th>Epoch 4</th> | |
| <th>Epoch 5</th> | |
| <th style="color: var(--success)">Epoch 6 (🏆 Win)</th> | |
| <th>Epoch 7</th> | |
| <th>Epoch 8</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| <tr> | |
| <td class="mono">Final Pretrain Loss (↓)</td> | |
| <td>6.574</td> | |
| <td>5.871</td> | |
| <td>5.459</td> | |
| <td>5.173</td> | |
| <td>4.974</td> | |
| <td>4.858</td> | |
| <td>4.734</td> | |
| <td class="mono" style="color: var(--warning)">4.632</td> | |
| </tr> | |
| <tr> | |
| <td class="mono">BLiMP (Grammar ↑)</td> | |
| <td>54.01%</td> | |
| <td>56.31%</td> | |
| <td>57.85%</td> | |
| <td>61.07%</td> | |
| <td>61.53%</td> | |
| <td>59.11%</td> | |
| <td class="winner-badge">62.17%</td> | |
| <td>60.67%</td> | |
| </tr> | |
| <tr> | |
| <td class="mono">ARC-Easy (Facts ↑)</td> | |
| <td>26.56%</td> | |
| <td>26.77%</td> | |
| <td>27.78%</td> | |
| <td>29.17%</td> | |
| <td>29.80%</td> | |
| <td class="winner-badge">30.30%</td> | |
| <td>30.26%</td> | |
| <td>29.63%</td> | |
| </tr> | |
| <tr> | |
| <td class="mono">BoolQ (Reading Logic ↑)</td> | |
| <td>37.83%</td> | |
| <td>37.83%</td> | |
| <td>37.83%</td> | |
| <td>37.83%</td> | |
| <td>37.83%</td> | |
| <td>37.86%</td> | |
| <td class="winner-badge">37.95%</td> | |
| <td>37.92%</td> | |
| </tr> | |
| <tr> | |
| <td class="mono">Wikitext (Byte PPL ↓)</td> | |
| <td>5.1477</td> | |
| <td>4.2237</td> | |
| <td>3.8080</td> | |
| <td>3.5645</td> | |
| <td>3.4472</td> | |
| <td>3.3916</td> | |
| <td>3.1886</td> | |
| <td class="winner-badge">3.1550</td> | |
| </tr> | |
| <tr class="highlight-row"> | |
| <td style="font-weight: bold;">EVALUATION STATUS</td> | |
| <td>Underfit</td> | |
| <td>Adapting</td> | |
| <td>Glitch Crisis</td> | |
| <td>Stable</td> | |
| <td>High Quality</td> | |
| <td style="color: var(--success); font-weight: bold;">OPTIMAL PEAK</td> | |
| <td style="color: var(--warning)">Oscillating</td> | |
| <td style="color: var(--warning)">Overfit Parrot</td> | |
| </tr> | |
| </tbody> | |
| </table> | |
| </div> | |
| </div> | |
| <span class="section-label" id="charts">// Visualizing_The_Tipping_Points</span> | |
| <div class="chart-box"> | |
| <h3>Downstream Metric Development Across Epoch Space</h3> | |
| <div style="position: relative; height:380px; width:100%"> | |
| <canvas id="lineMetricChart"></canvas> | |
| </div> | |
| </div> | |
| <div class="chart-box"> | |
| <h3>The Divergence: Training Loss vs. True Grammatical Accuracy (BLiMP)</h3> | |
| <p style="font-size: 0.85rem; color: var(--muted); margin-bottom: 1.5rem;">Crucial observation: While the training loss curves downward indefinitely, the true linguistic capability spikes and drops, confirming structural fragmentation past Epoch 7.</p> | |
| <div style="position: relative; height:350px; width:100%"> | |
| <canvas id="lossVsBlimpChart"></canvas> | |
| </div> | |
| </div> | |
| <span class="section-label">// Generative_Evolution_Analysis</span> | |
| <div class="card"> | |
| <h3>Qualitative Output Transformations</h3> | |
| <p>Tracking raw generation weights given the prompt <code>"Artificial intelligence is "</code>:</p> | |
| <div style="margin-top: 1.5rem;"> | |
| <p><strong style="color: var(--accent);">Epoch 1:</strong> <i>"...by the year-ponbmi, which they may be well of a single health;, so was that has much to help with a tergs..."</i><br> | |
| <small style="color: var(--muted)">Analysis: Highly fragmented syntax. Inventing structural gibberish due to under-tokenization.</small></p> | |
| <hr style="border: 0; border-top: 1px solid #222; margin: 1rem 0;"> | |
| <p><strong style="color: var(--accent);">Epoch 3 (The Glitch Phase):</strong> <i>"...Artificial intelligence is בोΩϤЧЉᵸ²хайϻجΠ״..."</i><br> | |
| <small style="color: var(--muted)">Analysis: Transition volatility. Weights start intense formatting compression, creating dense corrupted tokens.</small></p> | |
| <hr style="border: 0; border-top: 1px solid #222; margin: 1rem 0;"> | |
| <p><strong style="color: var(--success);">Epoch 5/6 (Structural Mastery):</strong> <i>"...The earliest examples are known for a large number... | Despite its knowledge, the same person's most complexity was only as being..."</i><br> | |
| <small style="color: var(--success)">Analysis: Complete repair of character flaws. The model spontaneously masters Markdown tags, layout partitions, and tabular boundaries.</small></p> | |
| </div> | |
| </div> | |
| <section class="stats-grid" id="hardware"> | |
| <div class="stat-box"> | |
| <small>COMPUTE ENVIRONMENT</small> | |
| <strong>Isolated Tensor Engine (T4 GPU)</strong> | |
| </div> | |
| <div class="stat-box"> | |
| <small>STATIONARY TOKEN POOL</small> | |
| <strong>20,000,000 FW-Edu Base</strong> | |
| </div> | |
| <div class="stat-box"> | |
| <small>PEAK EVAL PROFILE</small> | |
| <strong>Epoch 6 Optimization</strong> | |
| </div> | |
| </section> | |
| <footer> | |
| <p>© 2026 SupraLabs. High performance. Small footprints. Proudly open-source.</p> | |
| </footer> | |
| </div> | |
| <script> | |
| // Line Chart for Tracking downstream Metrics across Epochs | |
| const ctxLine = document.getElementById('lineMetricChart').getContext('2d'); | |
| new Chart(ctxLine, { | |
| type: 'line', | |
| data: { | |
| labels: ['Epoch 1', 'Epoch 2', 'Epoch 3', 'Epoch 4', 'Epoch 5', 'Epoch 6', 'Epoch 7', 'Epoch 8'], | |
| datasets: [ | |
| { | |
| label: 'ARC-Easy: Factual Knowledge Accuracy (%)', | |
| data: [26.56, 26.77, 27.78, 29.17, 29.80, 30.30, 30.26, 29.63], | |
| borderColor: '#00e676', | |
| backgroundColor: 'transparent', | |
| tension: 0.2, | |
| borderWidth: 3 | |
| }, | |
| { | |
| label: 'Wikitext: Byte Perplexity (Lower = Better)', | |
| data: [5.15, 4.22, 3.81, 3.56, 3.45, 3.39, 3.19, 3.16], | |
| borderColor: '#2196f3', | |
| backgroundColor: 'transparent', | |
| tension: 0.2, | |
| borderWidth: 2, | |
| borderDash: [5, 5] | |
| }, | |
| { | |
| label: 'BoolQ: Reading Logic Accuracy (%)', | |
| data: [37.83, 37.83, 37.83, 37.83, 37.83, 37.86, 37.95, 37.92], | |
| borderColor: '#ffeb3b', | |
| backgroundColor: 'transparent', | |
| tension: 0.1, | |
| borderWidth: 1.5 | |
| } | |
| ] | |
| }, | |
| options: { | |
| responsive: true, | |
| maintainAspectRatio: false, | |
| plugins: { | |
| legend: { labels: { color: '#bbb', font: { size: 11 } } } | |
| }, | |
| scales: { | |
| y: { grid: { color: '#222' }, ticks: { color: '#888' } }, | |
| x: { grid: { color: '#222' }, ticks: { color: '#aaa' } } | |
| } | |
| } | |
| }); | |
| // Scatter / Combo Dual-Axis View charting Loss vs BLiMP capabilities | |
| const ctxLossBlimp = document.getElementById('lossVsBlimpChart').getContext('2d'); | |
| new Chart(ctxLossBlimp, { | |
| type: 'line', | |
| data: { | |
| labels: ['Epoch 1', 'Epoch 2', 'Epoch 3', 'Epoch 4', 'Epoch 5', 'Epoch 6', 'Epoch 7', 'Epoch 8'], | |
| datasets: [ | |
| { | |
| label: 'Training Loss (Linear Collapse ↓)', | |
| data: [6.574, 5.871, 5.459, 5.173, 4.974, 4.858, 4.734, 4.632], | |
| borderColor: 'rgba(229, 57, 53, 0.5)', | |
| backgroundColor: 'transparent', | |
| yAxisID: 'yLoss', | |
| tension: 0.1, | |
| borderWidth: 2 | |
| }, | |
| { | |
| label: 'BLiMP Grammatical Accuracy (Peak & Fracture ↕)', | |
| data: [61.95, 62.00, 60.34, 61.07, 61.53, 59.11, 62.17, 60.67], // Mapped normalized relative scale | |
| borderColor: '#536bfe', | |
| backgroundColor: 'rgba(83, 107, 254, 0.2)', | |
| fill: true, | |
| yAxisID: 'yBlimp', | |
| tension: 0.3, | |
| borderWidth: 4 | |
| } | |
| ] | |
| }, | |
| options: { | |
| responsive: true, | |
| maintainAspectRatio: false, | |
| plugins: { | |
| legend: { labels: { color: '#bbb' } } | |
| }, | |
| scales: { | |
| x: { grid: { display: false }, ticks: { color: '#aaa' } }, | |
| yLoss: { | |
| type: 'linear', | |
| position: 'left', | |
| title: { display: true, text: 'Pretrain Training Loss', color: 'rgba(229, 57, 53, 0.8)' }, | |
| grid: { color: '#222' }, | |
| ticks: { color: '#888' } | |
| }, | |
| yBlimp: { | |
| type: 'linear', | |
| position: 'right', | |
| title: { display: true, text: 'BLiMP Accuracy Score (%)', color: '#536bfe' }, | |
| grid: { display: false }, | |
| ticks: { color: '#888' } | |
| } | |
| } | |
| } | |
| }); | |
| </script> | |
| </body> | |
| </html> |