update

app.py

@@ -60,13 +60,17 @@ class MultiHeadSelfAttention(nn.Module):
 class SwiGLUFFN(nn.Module):
     def __init__(self, d_model, ffn_hidden_dim, dropout):
         super().__init__()
-        self.lineargate    = nn.Linear(d_model, ffn_hidden_dim, bias=False)
-        self.linearvalue   = nn.Linear(d_model, ffn_hidden_dim, bias=False)
-        self.linearout     = nn.Linear(ffn_hidden_dim, d_model, bias=False)
-        self.dropout = nn.Dropout(dropout)
+        self.linear_gate  = nn.Linear(d_model, ffn_hidden_dim, bias=False)
+        self.linear_value = nn.Linear(d_model, ffn_hidden_dim, bias=False)
+        self.linear_out   = nn.Linear(ffn_hidden_dim, d_model, bias=False)
+        self.dropout      = nn.Dropout(dropout)
 
     def forward(self, x):
-        return self.dropout(self.out(F.silu(self.gate(x)) * self.value(x)))
+        return self.dropout(
+            self.linear_out(
+                F.silu(self.linear_gate(x)) * self.linear_value(x)
+            )
+        )
 
 
 class TransformerBlock(nn.Module):