Hugging Face's logo

Spaces:
SupraLabs
/
Research-SupraLabs
Running

App Files Community
1
LH-Tech-AI commited on
Commit
2b06042
·
verified ·
1 Parent(s): 39282ec

Create epochs-vs-data.html

Browse files
Files changed (1) hide show
  1. epochs-vs-data.html +332 -0
epochs-vs-data.html ADDED
@@ -0,0 +1,332 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
+ <title>SupraLabs | Satiating the Latent Space: Unique Tokens vs. Cycles</title>
7
+ <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
8
+ <style>
9
+ :root {
10
+ --bg: #0f0f0f;
11
+ --surface: #1a1a1a;
12
+ --border: #333;
13
+ --text: #e0e0e0;
14
+ --accent: #536bfe; /* Supra Blue */
15
+ --muted: #888;
16
+ --success: #00e676;
17
+ --warning: #ffb300;
18
+ --font-mono: 'JetBrains Mono', 'Fira Code', monospace;
19
+ }
20
+ * { margin: 0; padding: 0; box-sizing: border-box; }
21
+ body {
22
+ background-color: var(--bg);
23
+ color: var(--text);
24
+ font-family: 'Inter', -apple-system, sans-serif;
25
+ line-height: 1.6;
26
+ padding: 2rem;
27
+ }
28
+ code, pre, .mono { font-family: var(--font-mono); }
29
+ .container { max-width: 1000px; margin: 0 auto; }
30
+ header {
31
+ border-bottom: 2px solid var(--border);
32
+ padding-bottom: 2rem;
33
+ margin-bottom: 3rem;
34
+ display: flex;
35
+ justify-content: space-between;
36
+ align-items: flex-end;
37
+ }
38
+ .logo-area a { text-decoration: none; color: inherit; }
39
+ .logo-area h1 {
40
+ font-size: 1.2rem;
41
+ text-transform: uppercase;
42
+ letter-spacing: 2px;
43
+ color: var(--accent);
44
+ line-height: 1;
45
+ display: flex;
46
+ align-items: center;
47
+ gap: 10px;
48
+ }
49
+ nav a {
50
+ color: var(--text);
51
+ text-decoration: none;
52
+ margin-left: 1.5rem;
53
+ font-size: 0.9rem;
54
+ border-bottom: 1px solid transparent;
55
+ }
56
+ nav a:hover { border-bottom: 1px solid var(--accent); }
57
+ .hero { margin-bottom: 4rem; }
58
+ .hero h2 { font-size: 3rem; line-height: 1.1; margin-bottom: 1.5rem; font-weight: 800; }
59
+ .hero p { font-size: 1.2rem; color: var(--muted); max-width: 750px; }
60
+ .section-label {
61
+ display: block;
62
+ font-family: var(--font-mono);
63
+ color: var(--accent);
64
+ font-size: 0.8rem;
65
+ margin-top: 3rem;
66
+ margin-bottom: 1rem;
67
+ text-transform: uppercase;
68
+ }
69
+ .card { background: var(--surface); border: 1px solid var(--border); padding: 2.5rem; margin-bottom: 2rem; }
70
+ h3 { font-size: 1.6rem; margin-bottom: 1rem; font-weight: 700; }
71
+ p { margin-bottom: 1rem; color: #ccc; }
72
+ ul { margin-left: 1.5rem; margin-bottom: 1.5rem; }
73
+ li { margin-bottom: 0.5rem; }
74
+ .winner-badge { color: var(--success); font-weight: bold; font-family: var(--font-mono); }
75
+ .table-container { overflow-x: auto; margin: 2rem 0; border: 1px solid var(--border); }
76
+ table { width: 100%; border-collapse: collapse; text-align: left; font-size: 0.95rem; }
77
+ th, td { padding: 1rem; border-bottom: 1px solid var(--border); }
78
+ th { background-color: rgba(26, 26, 26, 0.8); font-family: var(--font-mono); color: var(--accent); font-size: 0.85rem; text-transform: uppercase; }
79
+ tr:hover { background-color: rgba(83, 107, 254, 0.05); }
80
+ .highlight-row { border-left: 4px solid var(--success); background-color: rgba(0, 230, 118, 0.02); }
81
+ .chart-box { background: var(--surface); border: 1px solid var(--border); padding: 2rem; margin-bottom: 2rem; }
82
+ .stats-grid {
83
+ display: grid;
84
+ grid-template-columns: 1fr 1fr 1fr;
85
+ gap: 1rem;
86
+ margin-top: 4rem;
87
+ border-top: 1px solid var(--border);
88
+ padding-top: 2rem;
89
+ }
90
+ .stat-box { padding: 1rem; border-left: 2px solid var(--accent); }
91
+ .stat-box small { display: block; color: var(--muted); font-family: var(--font-mono); }
92
+ footer { margin-top: 6rem; padding-bottom: 2rem; font-size: 0.8rem; color: var(--muted); text-align: center; }
93
+ @media (max-width: 768px) {
94
+ .hero h2 { font-size: 2rem; }
95
+ header { flex-direction: column; align-items: flex-start; gap: 1rem; }
96
+ nav a { margin-left: 0; margin-right: 1rem; }
97
+ .stats-grid { grid-template-columns: 1fr; }
98
+ }
99
+ </style>
100
+ </head>
101
+ <body>
102
+ <div class="container">
103
+ <header>
104
+ <div class="logo-area" style="font-size: 1.5em;">
105
+ <a href="index.html"><h1><img src="./image.png" style="height: 2em" alt="Logo"> SupraLabs_</h1></a>
106
+ </div>
107
+ <nav>
108
+ <a href="#summary">Core Learnings</a>
109
+ <a href="#benchmarks">Pretrain Matrix</a>
110
+ <a href="#charts">Visualizations</a>
111
+ <a href="https://huggingface.co/SupraLabs" target="_blank">HuggingFace</a>
112
+ </nav>
113
+ </header>
114
+
115
+ <section class="hero">
116
+ <h2>Experiment #6:<br>More Epochs vs. More Data for SLMs</h2>
117
+ <p>A rigorous, mathematical verification of information limits under a strict compute constraint. We held total token exposure starr at <strong>200,000,000 processed steps</strong>, testing total unique data volume directly against looping recurrent cycles.</p>
118
+ </section>
119
+
120
+ <span class="section-label" id="summary">// Data_Entropy_&_Reasoning_Loss</span>
121
+ <div class="card">
122
+ <h3>Isolating Token Freshness in the Static Latent Block</h3>
123
+ <p>Chinchilla compute curves dictate linear resource scaling. Our targeted isolation runs reveal an asymmetric divergence between static loss optimization and objective downstream capability inside sub-10M environments:</p>
124
+ <ul>
125
+ <li><strong>The Logic Divergence Cliff (Run 1):</strong> Maximizing unique data exposure (200M unique steps × 1 Epoch) delivers the highest reasoning performance, claiming 33.42% accuracy on factual deduction (ARC-Easy). Fresh token entropy is essential for non-repetitive learning.</li>
126
+ <li><strong>The Perplexity Sweetspot (Run 2):</strong> Running a micro-cycle (100M unique steps × 2 Epochs) yields a slight boost in base linguistic perplexity (236.80). The immediate repetition helps tiny architectures reinforce core syntactic boundaries.</li>
127
+ <li><strong>The Overfitting Illusion (Run 5):</strong> Compressing unique data down to 25M while repeating for 8 full epochs drops training loss to its absolute minimum (4.196). However, this triggers semantic memorization, ruining factual reasoning properties.</li>
128
+ </ul>
129
+ </div>
130
+
131
+ <span class="section-label" id="benchmarks">// Information_Density_Matrix</span>
132
+ <div class="card" style="padding: 1.5rem;">
133
+ <h3>Symmetric Token Matrix Results</h3>
134
+ <p>Every configuration is locked to exactly 200M total token exposure steps. Validation loss tracking alone is deceptive due to language overfitting parameters.</p>
135
+
136
+ <div class="table-container">
137
+ <table>
138
+ <thead>
139
+ <tr>
140
+ <th>Benchmark / Metric</th>
141
+ <th style="color: var(--success)">Run 1: 200M × 1 (🏆 Facts Win)</th>
142
+ <th style="color: var(--success)">Run 2: 100M × 2 (🏆 PPL Win)</th>
143
+ <th>Run 3: 50M × 4</th>
144
+ <th>Run 4: 40M × 5</th>
145
+ <th>Run 5: 25M × 8</th>
146
+ </tr>
147
+ </thead>
148
+ <tbody>
149
+ <tr>
150
+ <td class="mono">Unique Tokens Pool</td>
151
+ <td>200,000,000</td>
152
+ <td>100,000,000</td>
153
+ <td>50,000,000</td>
154
+ <td>40,000,000</td>
155
+ <td>25,000,000</td>
156
+ </tr>
157
+ <tr>
158
+ <td class="mono">Training Epochs Block</td>
159
+ <td>1 Epoch</td>
160
+ <td>2 Epochs</td>
161
+ <td>4 Epochs</td>
162
+ <td>5 Epochs</td>
163
+ <td>8 Epochs</td>
164
+ </tr>
165
+ <tr>
166
+ <td class="mono">Final Pretrain Loss (↓)</td>
167
+ <td>3.789</td>
168
+ <td>3.771</td>
169
+ <td>3.785</td>
170
+ <td>3.771</td>
171
+ <td style="color: var(--warning)">3.719</td>
172
+ </tr>
173
+ <tr>
174
+ <td class="mono">Final Pretrain Train Loss (↓)</td>
175
+ <td>4.240</td>
176
+ <td>4.229</td>
177
+ <td>4.235</td>
178
+ <td>4.225</td>
179
+ <td style="color: var(--warning)">4.196</td>
180
+ </tr>
181
+ <tr>
182
+ <td class="mono">ARC-Easy Zero-Shot (↑)</td>
183
+ <td class="winner-badge">33.42%</td>
184
+ <td>31.57%</td>
185
+ <td>31.82%</td>
186
+ <td>31.69%</td>
187
+ <td>30.93%</td>
188
+ </tr>
189
+ <tr>
190
+ <td class="mono">Wikitext Byte PPL (↓)</td>
191
+ <td>1.4824</td>
192
+ <td class="winner-badge">1.4750</td>
193
+ <td>1.4851</td>
194
+ <td>1.4918</td>
195
+ <td>1.5017</td>
196
+ </tr>
197
+ <tr>
198
+ <td class="mono">Wikitext Word PPL (↓)</td>
199
+ <td>243.3377</td>
200
+ <td class="winner-badge">236.8014</td>
201
+ <td>245.8708</td>
202
+ <td>252.0078</td>
203
+ <td>261.4054</td>
204
+ </tr>
205
+ <tr class="highlight-row">
206
+ <td style="font-weight: bold;">PRETRAIN ASSESSMENT</td>
207
+ <td style="color: var(--success); font-weight: bold;">MAXIMUM KNOWLEDGE</td>
208
+ <td style="color: var(--success); font-weight: bold;">SYNTAX ENHANCED</td>
209
+ <td>Recycling Decay</td>
210
+ <td>Memorization Masking</td>
211
+ <td style="color: var(--warning)">Severe Overfitting</td>
212
+ </tr>
213
+ </tbody>
214
+ </table>
215
+ </div>
216
+ </div>
217
+
218
+ <span class="section-label" id="charts">// Plotting_The_Entropy_Divergence</span>
219
+ <div class="chart-box">
220
+ <h3>Factual Logic Degradation vs. Looping Cycles</h3>
221
+ <div style="position: relative; height:350px; width:100%">
222
+ <canvas id="epochsArcChart"></canvas>
223
+ </div>
224
+ </div>
225
+
226
+ <div class="chart-box">
227
+ <h3>The Overfitting Paradox: True Language PPL vs. Apparent Training Loss</h3>
228
+ <p style="font-size: 0.85rem; color: var(--muted); margin-bottom: 1.5rem;">Crucial observation: While recycling data (increasing epochs) forces the loss curve downward artificially, true out-of-distribution Perplexity steadily degrades.</p>
229
+ <div style="position: relative; height:350px; width:100%">
230
+ <canvas id="epochsDivergenceChart"></canvas>
231
+ </div>
232
+ </div>
233
+
234
+ <section class="stats-grid" id="hardware">
235
+ <div class="stat-box">
236
+ <small>CONSTANT MATRIX SIZE</small>
237
+ <strong>200M Processing Steps</strong>
238
+ </div>
239
+ <div class="stat-box">
240
+ <small>COMPUTE TOPOLOGY</small>
241
+ <strong>Shallow & Wide SOTA Layout</strong>
242
+ </div>
243
+ <div class="stat-box">
244
+ <small>DATASET ROUTING ENGINE</small>
245
+ <strong>FineWeb-Edu Target Stream</strong>
246
+ </div>
247
+ </section>
248
+
249
+ <footer>
250
+ <p>&copy; 2026 SupraLabs. High performance. Small footprints. Proudly open-source.</p>
251
+ </footer>
252
+ </div>
253
+
254
+ <script>
255
+ // ARC Accuracy across runs
256
+ const ctxArc = document.getElementById('epochsArcChart').getContext('2d');
257
+ new Chart(ctxArc, {
258
+ type: 'bar',
259
+ data: {
260
+ labels: ['Run 1: 200M×1', 'Run 2: 100M×2', 'Run 3: 50M×4', 'Run 4: 40M×5', 'Run 5: 25M×8'],
261
+ datasets: [{
262
+ label: 'ARC-Easy: Factual Knowledge Accuracy (%)',
263
+ data: [33.42, 31.57, 31.82, 31.69, 30.93],
264
+ backgroundColor: ['rgba(0, 230, 118, 0.3)', 'rgba(83, 107, 254, 0.2)', 'rgba(83, 107, 254, 0.2)', 'rgba(83, 107, 254, 0.2)', 'rgba(229, 57, 53, 0.2)'],
265
+ borderColor: ['#00e676', '#536bfe', '#536bfe', '#536bfe', '#e53935'],
266
+ borderWidth: 2
267
+ }]
268
+ },
269
+ options: {
270
+ responsive: true,
271
+ maintainAspectRatio: false,
272
+ plugins: { legend: { labels: { color: '#bbb' } } },
273
+ scales: {
274
+ y: { grid: { color: '#222' }, ticks: { color: '#888' }, min: 28 },
275
+ x: { grid: { display: false }, ticks: { color: '#aaa' } }
276
+ }
277
+ }
278
+ });
279
+
280
+ // Paradox Divergence Line
281
+ const ctxDiv = document.getElementById('epochsDivergenceChart').getContext('2d');
282
+ new Chart(ctxDiv, {
283
+ type: 'line',
284
+ data: {
285
+ labels: ['Run 1 (1 Ep)', 'Run 2 (2 Ep)', 'Run 3 (4 Ep)', 'Run 4 (5 Ep)', 'Run 5 (8 Ep)'],
286
+ datasets: [
287
+ {
288
+ label: 'Apparent Pretrain Loss (Deceptive Collapse ↓)',
289
+ data: [4.240, 4.229, 4.235, 4.225, 4.196],
290
+ borderColor: 'rgba(229, 57, 53, 0.6)',
291
+ backgroundColor: 'transparent',
292
+ yAxisID: 'yLoss',
293
+ tension: 0.1,
294
+ borderWidth: 2
295
+ },
296
+ {
297
+ label: 'Wikitext Word Perplexity (True Evaluation Stagnation ↑)',
298
+ data: [243.33, 236.80, 245.87, 252.00, 261.40],
299
+ borderColor: '#536bfe',
300
+ backgroundColor: 'transparent',
301
+ yAxisID: 'yPpl',
302
+ tension: 0.2,
303
+ borderWidth: 3
304
+ }
305
+ ]
306
+ },
307
+ options: {
308
+ responsive: true,
309
+ maintainAspectRatio: false,
310
+ plugins: { legend: { labels: { color: '#bbb' } } },
311
+ scales: {
312
+ x: { grid: { display: false }, ticks: { color: '#aaa' } },
313
+ yLoss: {
314
+ type: 'linear',
315
+ position: 'left',
316
+ title: { display: true, text: 'Pretrain Loss Metric', color: 'rgba(229, 57, 53, 0.8)' },
317
+ grid: { color: '#222' },
318
+ ticks: { color: '#888' }
319
+ },
320
+ yPpl: {
321
+ type: 'linear',
322
+ position: 'right',
323
+ title: { display: true, text: 'Word Perplexity Scale', color: '#536bfe' },
324
+ grid: { display: false },
325
+ ticks: { color: '#888' }
326
+ }
327
+ }
328
+ }
329
+ });
330
+ </script>
331
+ </body>
332
+ </html>