File size: 6,119 Bytes
b5ee41d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
\relax 
\providecommand\hyper@newdestlabel[2]{}
\providecommand\HyField@AuxAddToFields[1]{}
\providecommand\HyField@AuxAddToCoFields[2]{}
\citation{hu2021lora}
\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{1}{section.1}\protected@file@percent }
\newlabel{eq:base}{{1}{1}{Introduction}{equation.1.1}{}}
\newlabel{eq:delta}{{2}{1}{Introduction}{equation.1.2}{}}
\citation{hu2021lora}
\@writefile{toc}{\contentsline {section}{\numberline {2}Method}{2}{section.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.1}Weight-Diff Extraction}{2}{subsection.2.1}\protected@file@percent }
\newlabel{eq:weight_diff}{{3}{2}{Weight-Diff Extraction}{equation.2.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.2}Truncated SVD Compression}{2}{subsection.2.2}\protected@file@percent }
\newlabel{eq:svd}{{4}{2}{Truncated SVD Compression}{equation.2.4}{}}
\newlabel{eq:lora_a}{{5}{2}{Truncated SVD Compression}{equation.2.5}{}}
\newlabel{eq:lora_b}{{6}{2}{Truncated SVD Compression}{equation.2.6}{}}
\newlabel{eq:lora_forward}{{7}{2}{Truncated SVD Compression}{equation.2.7}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.3}Tensor-by-Tensor Processing}{2}{subsection.2.3}\protected@file@percent }
\citation{shen2024moe}
\@writefile{toc}{\contentsline {subsection}{\numberline {2.4}Target Module Selection}{3}{subsection.2.4}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {2.5}Computational Complexity}{3}{subsection.2.5}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {3}Experiments}{3}{section.3}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {3.1}Setup}{3}{subsection.3.1}\protected@file@percent }
\@writefile{lot}{\contentsline {table}{\numberline {1}{\ignorespaces Experimental configuration.}}{3}{table.caption.1}\protected@file@percent }
\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}}
\newlabel{tab:setup}{{1}{3}{Experimental configuration}{table.caption.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.2}Delta Magnitude Analysis}{3}{subsection.3.2}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {1}{\ignorespaces Weight delta magnitudes ($\|\Delta \|_F$) per layer and attention projection module.}}{4}{figure.caption.2}\protected@file@percent }
\newlabel{fig:delta_magnitudes}{{1}{4}{Weight delta magnitudes ($\|\Delta \|_F$) per layer and attention projection module}{figure.caption.2}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.3}Rank Selection}{4}{subsection.3.3}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {2}{\ignorespaces Reconstruction quality vs. LoRA rank. At $r{=}16$, we retain 91.8\% cumulative spectral energy with a 7.2~MB adapter.}}{5}{figure.caption.3}\protected@file@percent }
\newlabel{fig:rank_vs_error}{{2}{5}{Reconstruction quality vs. LoRA rank. At $r{=}16$, we retain 91.8\% cumulative spectral energy with a 7.2~MB adapter}{figure.caption.3}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.4}Pipeline Architecture}{5}{subsection.3.4}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {3}{\ignorespaces Extraction pipeline: (1) download both source models, (2) compute element-wise deltas, (3) apply truncated SVD per-tensor, (4) construct LoRA A/B matrices with symmetric singular value distribution.}}{5}{figure.caption.4}\protected@file@percent }
\newlabel{fig:pipeline}{{3}{5}{Extraction pipeline: (1) download both source models, (2) compute element-wise deltas, (3) apply truncated SVD per-tensor, (4) construct LoRA A/B matrices with symmetric singular value distribution}{figure.caption.4}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {3.5}Layer-wise Analysis}{5}{subsection.3.5}\protected@file@percent }
\@writefile{lof}{\contentsline {figure}{\numberline {4}{\ignorespaces Layer-wise analysis of delta magnitudes: per-layer total $\|\Delta \|$, module breakdown, and layer group aggregation.}}{6}{figure.caption.5}\protected@file@percent }
\newlabel{fig:layer_analysis}{{4}{6}{Layer-wise analysis of delta magnitudes: per-layer total $\|\Delta \|$, module breakdown, and layer group aggregation}{figure.caption.5}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4}Results}{6}{section.4}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {4.1}Quantitative Results}{6}{subsection.4.1}\protected@file@percent }
\@writefile{lot}{\contentsline {table}{\numberline {2}{\ignorespaces Quantitative results comparing the base Opus model with the Opus+Kimi-LoRA adapter.}}{6}{table.caption.6}\protected@file@percent }
\newlabel{tab:results}{{2}{6}{Quantitative results comparing the base Opus model with the Opus+Kimi-LoRA adapter}{table.caption.6}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {4.2}Compression Efficiency}{6}{subsection.4.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {4.3}Qualitative Results}{6}{subsection.4.3}\protected@file@percent }
\citation{rombach2022ldm}
\@writefile{toc}{\contentsline {section}{\numberline {5}Discussion}{7}{section.5}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {5.1}Why This Works}{7}{subsection.5.1}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {5.2}Limitations}{7}{subsection.5.2}\protected@file@percent }
\@writefile{toc}{\contentsline {subsection}{\numberline {5.3}Future Work}{7}{subsection.5.3}\protected@file@percent }
\bibstyle{unsrt}
\bibcite{hu2021lora}{1}
\bibcite{dettmers2023qlora}{2}
\bibcite{rombach2022ldm}{3}
\bibcite{qwen2025}{4}
\bibcite{shen2024moe}{5}
\bibcite{dettmers2022optimizer}{6}
\bibcite{mangrulkar2022peft}{7}
\bibcite{unsloth2024}{8}
\bibcite{moonshot2025}{9}
\bibcite{anthropic2026}{10}
\bibcite{nous2026hermes}{11}
\bibcite{lordx64_2026}{12}
\bibcite{bas95_2026}{13}
\@writefile{toc}{\contentsline {section}{\numberline {6}Conclusion}{8}{section.6}\protected@file@percent }
\gdef \@abspage@last{8}